From bfcdf927f812a8b2cbfb01183df57cbc632db92d Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Mon, 6 Jul 2026 10:29:06 +0100 Subject: [PATCH 01/10] Fix SignalLock.incrementAsyncWork (wrong guard, can lead to ISE) --- .../service/accord/AccordExecutor.java | 18 +++---- .../AccordExecutorAbstractLockLoop.java | 4 +- .../accord/AccordExecutorSignalLoop.java | 4 +- .../service/accord/AccordExecutorSimple.java | 2 +- .../cassandra/service/accord/AccordTask.java | 2 +- .../utils/concurrent/SignalLock.java | 8 +++- .../simulator/test/AccordExecutorTest.java | 48 +++++++++++++++---- 7 files changed, 60 insertions(+), 26 deletions(-) diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java index b1aff08f13..cff519396a 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java @@ -1176,7 +1176,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor) task).preLoadContext())); else - task.runInternal(); + task.run(); // NOTE: cannot safely release owner here, in case an immediate-execution runs before we can release our references and store their changes to the cache } @@ -1631,7 +1631,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor name, Agent agent) { super(lock, executorId, agent); - Invariants.require(threads < SignalLock.MAX_THREADS); + Invariants.require(threads <= SignalLock.MAX_THREADS); this.lock = lock; this.loops = new AccordExecutorLoops(mode, threads, name, this::task); this.readyToRunLimit = Math.min(threads * 4, SignalLock.MAX_SIGNAL_COUNT); @@ -397,7 +397,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop try { setRunning(task); - task.runInternal(); + task.run(); } catch (Throwable t) { diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java index 4e89f068ab..ab6c8c25e7 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java @@ -95,7 +95,7 @@ class AccordExecutorSimple extends AccordExecutor return; } - try { task.preRunExclusive(); task.runInternal(); } + try { task.preRunExclusive(); task.run(); } catch (Throwable t) { task.fail(t); } finally { diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/AccordTask.java index e3fe9656e0..a766cc2097 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTask.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTask.java @@ -672,7 +672,7 @@ public abstract class AccordTask extends Task implements Function= 1 && increment <= MAX_SIGNAL_COUNT); while (true) { long cur = state; int count = asyncSignalCount(cur); - if (count == MAX_THREADS) + if (count == MAX_SIGNAL_COUNT) return false; if (signalIfWaiting) diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index 44888830cc..103f45b2db 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -18,7 +18,9 @@ package org.apache.cassandra.simulator.test; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -36,6 +38,7 @@ import accord.api.AsyncExecutor; import accord.local.SequentialAsyncExecutor; import org.apache.cassandra.concurrent.ExecutorFactory; +import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; import org.apache.cassandra.service.accord.AccordExecutor; @@ -43,47 +46,71 @@ import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; import org.apache.cassandra.service.accord.api.AccordAgent; +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; import static org.apache.cassandra.service.accord.AccordService.toFuture; // TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit public class AccordExecutorTest extends SimulationTestBase { - static final int THREAD_COUNT = 8; + static final int EXECUTOR_THREAD_COUNT = 44; @Test public void signalLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, THREAD_COUNT, -1, -1, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent())); + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, -1, -1, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), + 16); } @Test public void signalSpinLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, THREAD_COUNT, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent())); + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), + 16); } @Test public void ayncSubmitTest() { - executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, THREAD_COUNT, i -> "Loop" + i, new AccordAgent())); + executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, i -> "Loop" + i, new AccordAgent()), + 16); } - public void executorTest(SerializableSupplier supplier) + public void executorTest(SerializableSupplier supplier, int submissionThreads) { simulate(arr(() -> { try { DatabaseDescriptor.daemonInitialization(); + ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); AccordExecutor executor = supplier.get(); Lock lock = executor.unsafeLock(); SequentialAsyncExecutor sequentialExecutor = executor.newSequentialExecutor(); - Executor lockExecutor = ExecutorFactory.Global.executorFactory().sequential("lock"); - ConcurrentLinkedQueue> await = new ConcurrentLinkedQueue<>(); + Executor lockExecutor = executorFactory().sequential("lock"); for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) + { for (float lockChance : new float[] { 0f, 0.01f, 0.1f }) - submitLoop(lock, executor, sequentialExecutor, lockExecutor, 200, 10, await, sleepChance, lockChance); + { + List> done = new ArrayList<>(); + for (int i = 0 ; i < submissionThreads ; ++i) + { + int id = i; + done.add(submit.submit(() -> { + try + { + submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance); + } + catch (ExecutionException | InterruptedException e) + { + throw new RuntimeException(e); + } + })); + } + for (Future f : done) + f.get(); + } + } } catch (Throwable t) { @@ -93,8 +120,9 @@ public class AccordExecutorTest extends SimulationTestBase () -> {}, 1L); } - private static void submitLoop(Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, ConcurrentLinkedQueue> await, float sleepChance, float lockChance) throws ExecutionException, InterruptedException + private static void submitLoop(int id, Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance) throws ExecutionException, InterruptedException { + ConcurrentLinkedQueue> await = new ConcurrentLinkedQueue<>(); while (outerLoop-- > 0) { for (int i = 0; i < innerLoop; ++i) @@ -105,7 +133,7 @@ public class AccordExecutorTest extends SimulationTestBase while (!await.isEmpty()) await.poll().get(); done.set(true); - System.out.println("Loop " + (1 + outerLoop)); + System.out.println("Loop " + id + '.' + (1 + outerLoop)); } } From 13956ddf214599f5d47fa55e60f5252c395c1d60 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Thu, 9 Jul 2026 11:25:51 +0100 Subject: [PATCH 02/10] rename PreLoadContext to ExecutionContext, and SequentialAsyncExecutor -> ExclusiveAsyncExecutor --- .gitmodules | 4 +- .../architecture/accord-architecture.adoc | 2 +- .../db/virtual/AccordDebugKeyspace.java | 18 ++--- .../service/accord/AccordCommandStore.java | 22 +++--- .../service/accord/AccordCommandStores.java | 4 +- .../service/accord/AccordExecutor.java | 61 ++++++++-------- .../cassandra/service/accord/AccordTask.java | 70 +++++++++---------- .../accord/debug/DebugBlockedTxns.java | 8 +-- .../service/accord/debug/DebugTxnGraph.java | 8 +-- .../accord/interop/AccordInteropAdapter.java | 10 +-- .../interop/AccordInteropExecution.java | 6 +- .../accord/interop/AccordInteropPersist.java | 4 +- .../test/accord/AccordDropTableBase.java | 4 +- .../test/accord/AccordLoadTest.java | 6 +- .../test/accord/AccordRegainRangesTest.java | 4 +- .../simulator/test/AccordExecutorTest.java | 9 ++- .../CompactionAccordIteratorsTest.java | 2 +- .../accord/AccordCommandStoreTest.java | 6 +- .../service/accord/AccordCommandTest.java | 8 +-- .../service/accord/AccordTaskTest.java | 14 ++-- .../service/accord/AccordTestUtils.java | 10 +-- ...SimpleSimulatedAccordCommandStoreTest.java | 4 +- .../accord/SimulatedAccordCommandStore.java | 10 +-- .../accord/SimulatedAccordTaskTest.java | 10 +-- .../CommandsForKeySerializerTest.java | 16 ++--- 25 files changed, 159 insertions(+), 161 deletions(-) diff --git a/.gitmodules b/.gitmodules index 616dacf610..4b1eea6020 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "modules/accord"] path = modules/accord - url = https://github.com/apache/cassandra-accord.git - branch = trunk + url = https://github.com/belliottsmith/cassandra-accord.git + branch = executor-fair diff --git a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc index 4acb2395d6..ac7c6b7180 100644 --- a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc +++ b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc @@ -87,7 +87,7 @@ finally { } .... -In other words, `CommandStore` collects the `PreLoadContext`, state +In other words, `CommandStore` collects the `ExecutionContext`, state required to be in memory for command execution (possible dependencies, such as `TxnId`s, and `Key`s of commands, but also `CommandsForKeys` that will be needed during execution). Once the context is collected and diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 8c63b5767e..35b11418a6 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -71,9 +71,9 @@ import accord.local.CommandStores.LatentStoreSelector; import accord.local.Commands; import accord.local.Commands.NotifyWaitingOnPlus; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.MaxConflicts; import accord.local.Node; -import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; @@ -337,16 +337,16 @@ public class AccordDebugKeyspace extends VirtualKeyspace if (prev != null && info.status() == prev.status() && info.position() == prev.position()) ++uniquePos; else uniquePos = 0; prev = info; - PreLoadContext preLoadContext = info.preLoadContext(); + ExecutionContext executionContext = info.preLoadContext(); rows.add(info.status().name(), info.position(), uniquePos) .lazyCollect(columns -> { columns.add("description", info.describe()) .add("command_store_id", info.commandStoreId()) - .add("txn_id", preLoadContext, PreLoadContext::primaryTxnId, TO_STRING) - .add("txn_id_additional", preLoadContext, PreLoadContext::additionalTxnId, TO_STRING) - .add("keys", preLoadContext, PreLoadContext::keys, TO_STRING) - .add("keys_loading", preLoadContext, PreLoadContext::loadKeys, TO_STRING) - .add("keys_loading_for", preLoadContext, PreLoadContext::loadKeysFor, TO_STRING); + .add("txn_id", executionContext, ExecutionContext::primaryTxnId, TO_STRING) + .add("txn_id_additional", executionContext, ExecutionContext::additionalTxnId, TO_STRING) + .add("keys", executionContext, ExecutionContext::keys, TO_STRING) + .add("keys_loading", executionContext, ExecutionContext::loadKeys, TO_STRING) + .add("keys_loading_for", executionContext, ExecutionContext::loadKeysFor, TO_STRING); }); } }); @@ -740,7 +740,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace collector.partition(commandStore.id()) .collect(rows -> { // TODO (desired): support maybe execute immediately with safeStore - Future future = toFuture(commandStore.chain((PreLoadContext.Empty) metadata::toString, safeStore -> { addRows(safeStore, rows); })); + Future future = toFuture(commandStore.chain((ExecutionContext.Empty) metadata::toString, safeStore -> { addRows(safeStore, rows); })); if (!future.awaitUntilThrowUncheckedOnInterrupt(collector.deadlineNanos())) throw new InternalTimeoutException(); }); @@ -1895,7 +1895,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace AccordService.getBlocking(accord.node() .commandStores() .forId(commandStoreId) - .chain(PreLoadContext.contextFor(txnId, TXN_OPS), apply) + .chain(ExecutionContext.contextFor(txnId, TXN_OPS), apply) .flatMap(i -> i)); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index f4dd59a9c7..7bdb69efc0 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -58,13 +58,13 @@ import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.CommandSummaries; +import accord.local.ExecutionContext; import accord.local.MaxConflicts; import accord.local.MaxDecidedRX; import accord.local.MinimalCommand; import accord.local.MinimalCommand.MinimalWithDeps; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; -import accord.local.PreLoadContext.Empty; +import accord.local.ExecutionContext.Empty; import accord.local.RedundantBefore; import accord.local.RedundantBefore.Bounds; import accord.local.RedundantStatus.Property; @@ -216,7 +216,7 @@ public class AccordCommandStore extends CommandStore public final String loggingId; public final Journal journal; private final AccordExecutor sharedExecutor; - final AccordExecutor.SequentialExecutor exclusiveExecutor; + final AccordExecutor.ExclusiveExecutor exclusiveExecutor; private final ExclusiveCaches caches; private final RangeIndex rangeIndex; private final TableId tableId; @@ -403,27 +403,27 @@ public class AccordCommandStore extends CommandStore return lastSystemTimestampMicros.accumulateAndGet(node.now(), (a, b) -> Math.max(a + 1, b)); } @Override - public AsyncChain chain(PreLoadContext loadCtx, Function function) + public AsyncChain chain(ExecutionContext loadCtx, Function function) { return AccordTask.create(this, loadCtx, function).chain(); } @Override - public AsyncChain chain(PreLoadContext preLoadContext, Consumer consumer) + public AsyncChain chain(ExecutionContext executionContext, Consumer consumer) { - return AccordTask.create(this, preLoadContext, consumer).chain(); + return AccordTask.create(this, executionContext, consumer).chain(); } @Override - public AsyncChain priorityChain(PreLoadContext preLoadContext, Consumer consumer) + public AsyncChain priorityChain(ExecutionContext executionContext, Consumer consumer) { - return AccordTask.create(this, preLoadContext, consumer).priorityChain(); + return AccordTask.create(this, executionContext, consumer).priorityChain(); } @Override - public AsyncChain priorityChain(PreLoadContext preLoadContext, Function function) + public AsyncChain priorityChain(ExecutionContext executionContext, Function function) { - return AccordTask.create(this, preLoadContext, function).priorityChain(); + return AccordTask.create(this, executionContext, function).priorityChain(); } @Override @@ -729,7 +729,7 @@ public class AccordCommandStore extends CommandStore if (!maybeShouldReplay(txnId)) return AsyncChains.success(null); - return commandStore.chain(PreLoadContext.contextFor(txnId, "Replay"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Replay"), safeStore -> { Replay replay = shouldReplay(txnId, safeStore.unsafeGet(txnId).current().participants()); if (replay == Replay.NONE) return null; diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index f362c960c4..929d3d58cb 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -36,7 +36,7 @@ import accord.api.LocalListeners; import accord.api.ProgressLog; import accord.local.CommandStores; import accord.local.NodeCommandStoreService; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.local.ShardDistributor; import accord.utils.RandomSource; import accord.utils.Reduce; @@ -148,7 +148,7 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu } @Override - public SequentialAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someSequentialExecutor() { int idx = ((int) Thread.currentThread().getId()) & mask; return executors[idx].newSequentialExecutor(); diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java index cff519396a..4277ec55c8 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java @@ -44,8 +44,7 @@ import accord.api.Agent; import accord.api.RoutingKey; import accord.impl.AbstractAsyncExecutor; import accord.local.Command; -import accord.local.PreLoadContext; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExecutionContext; import accord.local.cfk.CommandsForKey; import accord.messages.Accept; import accord.messages.Commit; @@ -611,25 +610,25 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor void cancel(AccordTask task) @@ -751,7 +750,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(SequentialExecutor.class, Thread.class, "owner"); - public class SequentialExecutor extends TaskQueue implements SequentialAsyncExecutor + private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(ExclusiveExecutor.class, Thread.class, "owner"); + public class ExclusiveExecutor extends TaskQueue implements accord.local.ExclusiveAsyncExecutor { final int commandStoreId; final SequentialQueueTask selfTask; @@ -1278,12 +1277,12 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor)) return true; - PreLoadContext context = ((AccordTask) task).preLoadContext(); + ExecutionContext context = ((AccordTask) task).preLoadContext(); return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); } @@ -1522,7 +1521,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor result; final Runnable run; - final @Nullable SequentialExecutor executor; + final @Nullable ExclusiveExecutor executor; PlainRunnable(Runnable run) { @@ -1697,7 +1696,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor result, Runnable run, @Nullable SequentialExecutor executor, long queuePosition) + PlainRunnable(AsyncPromise result, Runnable run, @Nullable ExclusiveExecutor executor, long queuePosition) { this.result = result; this.run = run; @@ -1727,7 +1726,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor) task).preLoadContext(); @@ -2119,7 +2118,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor extends Task implements Function function; - public ForFunction(AccordCommandStore commandStore, PreLoadContext loadCtx, Function function) + public ForFunction(AccordCommandStore commandStore, ExecutionContext loadCtx, Function function) { super(commandStore, loadCtx); this.function = function; @@ -129,7 +129,7 @@ public abstract class AccordTask extends Task implements Function consumer; - private ForConsumer(AccordCommandStore commandStore, PreLoadContext loadCtx, Consumer consumer) + private ForConsumer(AccordCommandStore commandStore, ExecutionContext loadCtx, Consumer consumer) { super(commandStore, loadCtx); this.consumer = consumer; @@ -143,12 +143,12 @@ public abstract class AccordTask extends Task implements Function AccordTask create(CommandStore commandStore, PreLoadContext ctx, Function function) + public static AccordTask create(CommandStore commandStore, ExecutionContext ctx, Function function) { return new ForFunction<>((AccordCommandStore) commandStore, ctx, function); } - public static AccordTask create(CommandStore commandStore, PreLoadContext ctx, Consumer consumer) + public static AccordTask create(CommandStore commandStore, ExecutionContext ctx, Consumer consumer) { return new ForConsumer((AccordCommandStore) commandStore, ctx, consumer); } @@ -201,7 +201,7 @@ public abstract class AccordTask extends Task implements Function loggingIdUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordTask.class, String.class, "loggingId"); @@ -220,10 +220,10 @@ public abstract class AccordTask extends Task implements Function callback; - public AccordTask(@Nonnull AccordCommandStore commandStore, PreLoadContext preLoadContext) + public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext) { this.commandStore = commandStore; - this.preLoadContext = preLoadContext; + this.executionContext = executionContext; this.loggingId = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); if (logger.isTraceEnabled()) @@ -245,7 +245,7 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function extends Task implements Function keys() { - return preLoadContext.keys(); + return executionContext.keys(); } // TODO (expected): try to execute immediately BUT consider ordering requirements @@ -363,22 +363,22 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function extends Task implements Function keys, boolean isToCompleteRangeScan) { - if (preLoadContext.loadKeys() == LoadKeys.NONE) + if (executionContext.loadKeys() == LoadKeys.NONE) return; - if (!isToCompleteRangeScan && preLoadContext.loadKeysFor() == RECOVERY) + if (!isToCompleteRangeScan && executionContext.loadKeysFor() == RECOVERY) { Invariants.require(rangeScanner == null); rangeScanner = new RangeTxnScanner(); @@ -441,12 +441,12 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function commands() @@ -680,7 +680,7 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function extends Task implements Function intersectingKeys = new ObjectHashSet<>(); final KeyWatcher keyWatcher = new KeyWatcher(); - final Ranges ranges = ((AbstractRanges) preLoadContext.keys()).toRanges(); + final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); final AccordCache.Type.Instance commandsForKeyCache; public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) @@ -1082,9 +1082,9 @@ public abstract class AccordTask extends Task implements Function cancelled); } - PreLoadContext preLoadContext() + ExecutionContext preLoadContext() { - return preLoadContext; + return executionContext; } @Override @@ -1109,7 +1109,7 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function extends Task implements Function visitRootTxnAsync(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -188,7 +188,7 @@ public class DebugBlockedTxns private AsyncChain visitTxnAsync(CommandStore commandStore, TxnId txnId, Timestamp rootExecuteAt, @Nullable TokenKey byKey, int depth, boolean recurse) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -231,7 +231,7 @@ public class DebugBlockedTxns private AsyncChain visitKeysAsync(CommandStore commandStore, TokenKey key, Timestamp rootExecuteAt, int depth) { - return commandStore.chain(PreLoadContext.contextFor(RoutingKeys.of(key.toUnseekable()), SYNC, READ_WRITE, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(RoutingKeys.of(key.toUnseekable()), SYNC, READ_WRITE, "Populate txn_blocked_by"), safeStore -> { visitKeysSync(safeStore, key, rootExecuteAt, depth); }); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java index 8bc6a6a53a..a821db6340 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java @@ -40,7 +40,7 @@ import javax.annotation.Nullable; import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommandStore; import accord.primitives.PartialDeps; import accord.primitives.Participants; @@ -221,7 +221,7 @@ public abstract class DebugTxnGraph private AsyncChain> submitRoot(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -232,7 +232,7 @@ public abstract class DebugTxnGraph private AsyncChain> submitParent(CommandStore commandStore, TxnId txnId, P param, Map infos, Set visitedParent, int depth) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -344,7 +344,7 @@ public abstract class DebugTxnGraph private AsyncChain populateTxnAsync(CommandStore commandStore, TxnId txnId, Map visited) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); visited.putIfAbsent(txnId, command == null || command.saveStatus() == SaveStatus.Uninitialised ? SaveInfo.NONE : new SaveInfo(command.saveStatus(), command.executeAtIfKnown())); }); diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java index 61de28c26c..b79fc5f841 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java @@ -30,7 +30,7 @@ import accord.coordinate.CoordinationAdapter.Adapters.TxnAdapter; import accord.coordinate.ExecuteFlag.CoordinationFlags; import accord.coordinate.ExecutePath; import accord.local.Node; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.messages.Apply; import accord.primitives.Ballot; import accord.primitives.Deps; @@ -83,14 +83,14 @@ public class AccordInteropAdapter extends TxnAdapter } @Override - public void execute(Node node, SequentialAsyncExecutor executor, Topologies any, FullRoute route, Ballot ballot, ExecutePath path, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer callback) + public void execute(Node node, ExclusiveAsyncExecutor executor, Topologies any, FullRoute route, Ballot ballot, ExecutePath path, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer callback) { if (!doInteropExecute(node, executor, route, ballot, txnId, txn, executeAt, stableDeps, callback)) super.execute(node, executor, any, route, ballot, path, flags, txnId, txn, executeAt, stableDeps, sendDeps, callback); } @Override - public void persist(Node node, SequentialAsyncExecutor executor, Topologies any, Route require, Route sendTo, FullRoute route, Ballot ballot, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, boolean informDurableOnDone, BiConsumer callback) + public void persist(Node node, ExclusiveAsyncExecutor executor, Topologies any, Route require, Route sendTo, FullRoute route, Ballot ballot, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, boolean informDurableOnDone, BiConsumer callback) { if (applyKind == Minimal && doInteropPersist(node, executor, any, require, sendTo, ballot, txnId, txn, executeAt, deps, writes, result, route, informDurableOnDone, callback)) return; @@ -98,7 +98,7 @@ public class AccordInteropAdapter extends TxnAdapter super.persist(node, executor, any, require, sendTo, route, ballot, flags, txnId, txn, executeAt, deps, writes, result, informDurableOnDone, callback); } - private boolean doInteropExecute(Node node, SequentialAsyncExecutor executor, FullRoute route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer callback) + private boolean doInteropExecute(Node node, ExclusiveAsyncExecutor executor, FullRoute route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer callback) { // Unrecoverable repair always needs to be run by AccordInteropExecution AccordUpdate.Kind updateKind = AccordUpdate.kind(txn.update()); @@ -121,7 +121,7 @@ public class AccordInteropAdapter extends TxnAdapter return true; } - private boolean doInteropPersist(Node node, SequentialAsyncExecutor executor, Topologies any, Route require, Route sendTo, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, boolean informDurableOnDone, BiConsumer callback) + private boolean doInteropPersist(Node node, ExclusiveAsyncExecutor executor, Topologies any, Route require, Route sendTo, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, boolean informDurableOnDone, BiConsumer callback) { Update update = txn.update(); ConsistencyLevel consistencyLevel = update instanceof AccordUpdate ? ((AccordUpdate) update).cassandraCommitCL() : null; diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java index 49b467cc7f..8b215a903f 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java @@ -33,7 +33,7 @@ import accord.coordinate.CoordinationAdapter; import accord.coordinate.ExecuteFlag.CoordinationFlags; import accord.local.Node; import accord.local.Node.Id; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.messages.Commit; import accord.messages.Commit.Kind; import accord.primitives.AbstractRanges; @@ -125,7 +125,7 @@ public class AccordInteropExecution implements ReadCoordinator private final Timestamp executeAt; private final Deps deps; private final BiConsumer callback; - private final SequentialAsyncExecutor executor; + private final ExclusiveAsyncExecutor executor; private final ConsistencyLevel consistencyLevel; private final AccordEndpointMapper endpointMapper; @@ -141,7 +141,7 @@ public class AccordInteropExecution implements ReadCoordinator private volatile long uniqueHlc; public AccordInteropExecution(Node node, TxnId txnId, Txn txn, AccordUpdate.Kind updateKind, FullRoute route, Ballot ballot, Timestamp executeAt, Deps deps, BiConsumer callback, - SequentialAsyncExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper) throws TopologyException + ExclusiveAsyncExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper) throws TopologyException { requireArgument(!txn.read().keys().isEmpty() || updateKind == AccordUpdate.Kind.UNRECOVERABLE_REPAIR); this.node = node; diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java index 7cdb7032a8..95551f7092 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java @@ -26,7 +26,7 @@ import accord.coordinate.Persist; import accord.coordinate.tracking.AllTracker; import accord.coordinate.tracking.QuorumTracker; import accord.local.Node; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.messages.Apply; import accord.primitives.Ballot; import accord.primitives.Deps; @@ -53,7 +53,7 @@ import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; */ public class AccordInteropPersist extends Persist { - public AccordInteropPersist(Node node, SequentialAsyncExecutor executor, Topologies topologies, TxnId txnId, Route sendTo, Ballot ballot, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, ConsistencyLevel consistencyLevel, ExecuteFlag.CoordinationFlags flags, boolean informDurableOnDone, Apply.Kind applyKind, BiConsumer callback) + public AccordInteropPersist(Node node, ExclusiveAsyncExecutor executor, Topologies topologies, TxnId txnId, Route sendTo, Ballot ballot, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, ConsistencyLevel consistencyLevel, ExecuteFlag.CoordinationFlags flags, boolean informDurableOnDone, Apply.Kind applyKind, BiConsumer callback) { super(node, executor, topologies, txnId, ballot, sendTo, txn, executeAt, deps, writes, result.toPersistable(), fullRoute, flags, informDurableOnDone, AccordInteropApply.FACTORY, applyKind, consistencyLevel == ALL ? AllTracker::new : QuorumTracker::new, (ignore, fail) -> { if (fail != null) callback.accept(null, fail); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java index 8b96f0d1da..04db462483 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java @@ -24,9 +24,9 @@ import org.assertj.core.api.Assertions; import accord.api.RoutingKey; import accord.local.CommandStores; +import accord.local.ExecutionContext; import accord.local.LoadKeys; import accord.local.Node; -import accord.local.PreLoadContext; import accord.local.cfk.CommandsForKey; import accord.primitives.Ranges; import accord.primitives.Routable; @@ -135,7 +135,7 @@ public class AccordDropTableBase extends TestBaseImpl TableId tableId = TableId.fromString(s); AccordService accord = (AccordService) AccordService.instance(); TxnId syntheticTxnId = new TxnId(TxnId.MAX_EPOCH, 0, Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range, new Node.Id(1)); - PreLoadContext ctx = PreLoadContext.contextFor(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), LoadKeys.SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = ExecutionContext.contextFor(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), LoadKeys.SYNC, READ_WRITE, "Test"); CommandStores stores = accord.node().commandStores(); for (int storeId : stores.ids()) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java index cee395977d..8270da184e 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java @@ -56,7 +56,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.local.CommandStore; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.primitives.PartialDeps; import accord.primitives.TxnId; @@ -832,7 +832,7 @@ public class AccordLoadTest extends AccordTestBase if (storeId.get() >= 0) { CommandStore commandStore = service.node().commandStores().forId(storeId.get()); - List> result = AccordService.getBlocking(commandStore.submit(PreLoadContext.contextFor(candidate, "LoadTest"), safeStore -> { + List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.contextFor(candidate, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(candidate); PartialDeps deps = safeCommand.current().partialDeps(); if (deps == null) @@ -855,7 +855,7 @@ public class AccordLoadTest extends AccordTestBase for (List info : result) { TxnId txnId = TxnId.parse(info.get(0)); - AccordService.getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, "LoadTest"), safeStore -> { + AccordService.getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(txnId); if (safeCommand.current().executeAt != null) info.add(safeCommand.current().executeAt.toString()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java index 990c875024..eeb57dd204 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java @@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory; import accord.local.CommandStore; import accord.local.CommandStores.PreviouslyOwned; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.primitives.AbstractRanges; import accord.primitives.Ranges; import accord.topology.ActiveEpochs; @@ -121,7 +121,7 @@ public class AccordRegainRangesTest extends AccordTestBase Ranges range = Ranges.EMPTY; for (CommandStore commandStore : AccordService.instance().node().commandStores().all()) { - Ranges safeToReadRanges = getBlocking(commandStore.submit((PreLoadContext.Empty) () -> "No overlapping safeToReadRanges", safeCommandStore -> { + Ranges safeToReadRanges = getBlocking(commandStore.submit((ExecutionContext.Empty) () -> "No overlapping safeToReadRanges", safeCommandStore -> { Ranges mergedRanges = Ranges.EMPTY; for (Ranges r : safeCommandStore.safeToReadAt().values()) mergedRanges = mergedRanges.union(AbstractRanges.UnionMode.MERGE_ADJACENT, r); diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index 103f45b2db..eadb7a4f55 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -35,9 +35,8 @@ import java.util.function.BooleanSupplier; import org.junit.Test; import accord.api.AsyncExecutor; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; -import org.apache.cassandra.concurrent.ExecutorFactory; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; @@ -85,7 +84,7 @@ public class AccordExecutorTest extends SimulationTestBase ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); AccordExecutor executor = supplier.get(); Lock lock = executor.unsafeLock(); - SequentialAsyncExecutor sequentialExecutor = executor.newSequentialExecutor(); + ExclusiveAsyncExecutor sequentialExecutor = executor.newSequentialExecutor(); Executor lockExecutor = executorFactory().sequential("lock"); for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) @@ -120,7 +119,7 @@ public class AccordExecutorTest extends SimulationTestBase () -> {}, 1L); } - private static void submitLoop(int id, Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance) throws ExecutionException, InterruptedException + private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance) throws ExecutionException, InterruptedException { ConcurrentLinkedQueue> await = new ConcurrentLinkedQueue<>(); while (outerLoop-- > 0) @@ -137,7 +136,7 @@ public class AccordExecutorTest extends SimulationTestBase } } - private static void submitRecursive(Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, int count, Collection> await, float sleepChance, float lockChance) + private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> await, float sleepChance, float lockChance) { AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; await.add(toFuture(submitTo.chain(() -> { diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index 4901566aae..a9a5c6db58 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -92,7 +92,7 @@ import org.apache.cassandra.utils.FBUtilities; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.PreLoadContext.contextFor; +import static accord.local.ExecutionContext.contextFor; import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE; import static accord.primitives.Routable.Domain.Range; import static accord.primitives.Timestamp.Flag.HLC_BOUND; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java index f3d94e7827..fc691f60fc 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java @@ -32,7 +32,7 @@ import org.slf4j.LoggerFactory; import accord.api.Key; import accord.api.Result.PersistableResult; import accord.local.Command; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.StoreParticipants; import accord.local.cfk.CommandsForKey; import accord.primitives.Ballot; @@ -170,8 +170,8 @@ public class AccordCommandStoreTest AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null)); cfk.initialize(); - cfk.set(cfk.current().update(new TestSafeCommandStore(PreLoadContext.contextFor(command1.txnId(), "Test")), command1).cfk()); - cfk.set(cfk.current().update(new TestSafeCommandStore(PreLoadContext.contextFor(command1.txnId(), "Test")), command2).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.contextFor(command1.txnId(), "Test")), command1).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.contextFor(command1.txnId(), "Test")), command2).cfk()); CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run(); logger.info("E: {}", cfk); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index 214e17e6d1..dcd3486852 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -29,7 +29,7 @@ import accord.api.RoutingKey; import accord.local.Command; import accord.local.LoadKeys; import accord.local.Node; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; @@ -98,7 +98,7 @@ public class AccordCommandTest public void basicCycleTest() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((PreLoadContext.Empty)() -> "Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(1); @@ -174,7 +174,7 @@ public class AccordCommandTest Commit commit = Commit.SerializerSupport.create(txnId, route, 1, 1, Commit.Kind.StableWithTxnAndDeps, Ballot.ZERO, executeAt, partialTxn, deps, fullRoute); getBlocking(commandStore.execute(commit, commit::apply)); - getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, Keys.of(key).toParticipants(), LoadKeys.SYNC, READ_WRITE, "Test"), safeStore -> { + getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, Keys.of(key).toParticipants(), LoadKeys.SYNC, READ_WRITE, "Test"), safeStore -> { Command before = safeStore.ifInitialised(txnId).current(); Assert.assertEquals(commit.executeAt, before.executeAt()); Assert.assertTrue(before.hasBeen(Status.Committed)); @@ -191,7 +191,7 @@ public class AccordCommandTest public void computeDeps() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((PreLoadContext.Empty)()->"Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)()->"Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); TxnId txnId1 = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(2); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java index c5fafc9103..ede89a4d77 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java @@ -46,7 +46,7 @@ import org.slf4j.LoggerFactory; import accord.api.RoutingKey; import accord.local.CheckedCommands; import accord.local.Command; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; @@ -87,7 +87,7 @@ import org.apache.cassandra.utils.concurrent.Condition; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.PreLoadContext.contextFor; +import static accord.local.ExecutionContext.contextFor; import static accord.utils.Property.qt; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; import static org.apache.cassandra.service.accord.AccordService.getBlocking; @@ -127,7 +127,7 @@ public class AccordTaskTest AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, "Test"), instance -> { + getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "Test"), instance -> { // TODO review: This change to `ifInitialized` was done in a lot of places and it doesn't preserve this property // I fixed this reference to point to `ifLoadedAndInitialised` and but didn't update other places Assert.assertNull(instance.ifInitialised(txnId)); @@ -141,7 +141,7 @@ public class AccordTaskTest AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, "Test"), safe -> { + getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "Test"), safe -> { StoreParticipants participants = StoreParticipants.empty(txnId); SafeCommand command = safe.get(txnId, participants); Assert.assertNotNull(command); @@ -155,7 +155,7 @@ public class AccordTaskTest Txn txn = AccordTestUtils.createWriteTxn((int)clock.incrementAndGet()); TokenKey key = ((PartitionKey) Iterables.getOnlyElement(txn.keys())).toUnseekable(); - getBlocking(commandStore.execute((PreLoadContext.Empty)() -> "Test", instance -> { + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", instance -> { SafeCommandsForKey cfk = instance.ifLoadedAndInitialised(key); Assert.assertNull(cfk); })); @@ -298,7 +298,7 @@ public class AccordTaskTest awaitDone(commandStore, ids, participants); assertNoReferences(commandStore, ids, participants); - PreLoadContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); Consumer consumer = Mockito.mock(Consumer.class); Map failed = selectFailedTxn(rs, ids); @@ -363,7 +363,7 @@ public class AccordTaskTest assertNoReferences(commandStore, ids, participants); createCommand(commandStore, rs, ids); - PreLoadContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); Consumer consumer = Mockito.mock(Consumer.class); String errorMsg = "txn_ids " + ids; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index fd2301046a..fc4fdd4367 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -51,12 +51,12 @@ import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.Node; import accord.local.Node.Id; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; import accord.local.SafeCommandStore; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.durability.DurabilityService; @@ -244,7 +244,7 @@ public class AccordTestUtils public static AsyncChain> processTxnResult(AccordCommandStore commandStore, TxnId txnId, PartialTxn txn, Timestamp executeAt) throws Throwable { AtomicReference>> result = new AtomicReference<>(); - getBlocking(commandStore.execute((PreLoadContext.Empty)() -> "Test", + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", safeStore -> result.set(processTxnResultDirect(safeStore, txnId, txn, executeAt)))); return result.get(); } @@ -371,7 +371,7 @@ public class AccordTestUtils } @Override - public SequentialAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someSequentialExecutor() { return null; } @@ -419,7 +419,7 @@ public class AccordTestUtils Node.Id node = new Id(1); Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node))); AccordCommandStore store = createAccordCommandStore(node, now, topology); - store.execute((PreLoadContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20)); + store.execute((ExecutionContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20)); return store; } diff --git a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java index 0d2b8ccb93..fae6c912fa 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java @@ -21,7 +21,7 @@ package org.apache.cassandra.service.accord; import org.assertj.core.api.Assertions; import org.junit.Test; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.StoreParticipants; import accord.primitives.SaveStatus; import accord.primitives.TxnId; @@ -41,7 +41,7 @@ public class SimpleSimulatedAccordCommandStoreTest extends SimulatedAccordComman for (int i = 0, examples = 100; i < examples; i++) { TxnId id = AccordGens.txnIds().next(rs); - instance.process(PreLoadContext.contextFor(id, "Test"), (safe) -> { + instance.process(ExecutionContext.contextFor(id, "Test"), (safe) -> { var safeCommand = safe.get(id, StoreParticipants.empty(id)); var command = safeCommand.current(); Assertions.assertThat(command.saveStatus()).isEqualTo(SaveStatus.Uninitialised); diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 0fabb67352..228e2e564a 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -54,12 +54,12 @@ import accord.local.CommandStore; import accord.local.CommandStores; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.Node; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.local.TimeService; import accord.local.durability.DurabilityService; import accord.messages.BeginRecovery; @@ -182,7 +182,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable } @Override - public SequentialAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someSequentialExecutor() { return null; } @@ -448,7 +448,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable return process(request, request); } - public T process(PreLoadContext loadCtx, Function function) throws ExecutionException, InterruptedException + public T process(ExecutionContext loadCtx, Function function) throws ExecutionException, InterruptedException { var result = processAsync(loadCtx, function); processAll(); @@ -460,7 +460,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable return processAsync(request, request); } - public AsyncResult processAsync(PreLoadContext loadCtx, Function function) + public AsyncResult processAsync(ExecutionContext loadCtx, Function function) { return commandStore.submit(loadCtx, function); } diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java index 805ba040b8..ce061f40bf 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java @@ -32,7 +32,7 @@ import org.junit.Test; import accord.api.RoutingKey; import accord.impl.basic.SimulatedFault; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommandStore; import accord.messages.PreAccept; import accord.primitives.FullRoute; @@ -107,7 +107,7 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase { case Task: { - PreLoadContext ctx = (PreLoadContext.Empty)()->"Test"; + ExecutionContext ctx = (ExecutionContext.Empty)()->"Test"; instance.maybeCacheEvict(ctx.keys()); operation(instance, ctx, actionGen.next(rs), rs::nextBoolean).chain().begin(counter); } @@ -178,7 +178,7 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase private enum Action { SUCCESS, FAILURE, LOAD_FAILURE } - private static AccordTask operation(SimulatedAccordCommandStore instance, PreLoadContext ctx, Action action, BooleanSupplier delay) + private static AccordTask operation(SimulatedAccordCommandStore instance, ExecutionContext ctx, Action action, BooleanSupplier delay) { return new SimulatedOperation(instance.commandStore, ctx, action == Action.FAILURE ? SimulatedOperation.Action.FAILURE : SimulatedOperation.Action.SUCCESS); } @@ -201,9 +201,9 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase enum Action { SUCCESS, FAILURE} private final Action action; - public SimulatedOperation(AccordCommandStore commandStore, PreLoadContext preLoadContext, Action action) + public SimulatedOperation(AccordCommandStore commandStore, ExecutionContext executionContext, Action action) { - super(commandStore, preLoadContext); + super(commandStore, executionContext); this.action = action; } diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 138dbb4f12..5130dce553 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -66,13 +66,13 @@ import accord.local.CommandBuilder; import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.Node; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; import accord.local.RedundantBefore; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.cfk.CommandsForKey; @@ -505,8 +505,8 @@ public class CommandsForKeySerializerTest { int next = source.nextInt(commands.size()); Command command = commands.get(next); - if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(PreLoadContext.contextFor(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); - else cfk = cfk.update(new TestSafeCommandStore(PreLoadContext.contextFor(command.txnId(), "Test")), command).cfk(); + if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(ExecutionContext.contextFor(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); + else cfk = cfk.update(new TestSafeCommandStore(ExecutionContext.contextFor(command.txnId(), "Test")), command).cfk(); commands.set(next, commands.get(commands.size() - 1)); commands.remove(commands.size() - 1); } @@ -667,8 +667,8 @@ public class CommandsForKeySerializerTest } @Override public boolean inStore() { return true; } - @Override public AsyncChain chain(PreLoadContext context, Consumer consumer) { throw new UnsupportedOperationException();} - @Override public AsyncChain chain(PreLoadContext context, Function apply) { throw new UnsupportedOperationException(); } + @Override public AsyncChain chain(ExecutionContext context, Consumer consumer) { throw new UnsupportedOperationException();} + @Override public AsyncChain chain(ExecutionContext context, Function apply) { throw new UnsupportedOperationException(); } @Override public Journal.Replayer replayer(AbstractReplayer.Mode mode) { throw new UnsupportedOperationException(); } @@ -702,7 +702,7 @@ public class CommandsForKeySerializerTest public static class TestSafeCommandStore extends AbstractSafeCommandStore { - public TestSafeCommandStore(PreLoadContext context) + public TestSafeCommandStore(ExecutionContext context) { super(context, TestCommandStore.INSTANCE); } @@ -718,7 +718,7 @@ public class CommandsForKeySerializerTest @Override public NodeCommandStoreService node() { return new NodeCommandStoreService() { @Override public AsyncExecutor someExecutor() { throw new UnsupportedOperationException(); } - @Override public SequentialAsyncExecutor someSequentialExecutor() { throw new UnsupportedOperationException(); } + @Override public ExclusiveAsyncExecutor someSequentialExecutor() { throw new UnsupportedOperationException(); } @Override public long epoch() { return 0;} @Override public Node.Id id() { return Node.Id.NONE; } @Override public Timeouts timeouts() { return null; } From 1cffe9a8a032ce37889e821fb46b165bcd942323 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Thu, 9 Jul 2026 11:42:47 +0100 Subject: [PATCH 03/10] Executor Fairness --- .../cassandra/concurrent/CassandraThread.java | 57 +- .../apache/cassandra/config/AccordConfig.java | 80 +- .../db/virtual/AccordDebugKeyspace.java | 2 +- .../cassandra/service/accord/AccordCache.java | 1 - .../service/accord/AccordCommandStore.java | 40 +- .../service/accord/AccordCommandStores.java | 6 +- .../service/accord/AccordExecutor.java | 2629 +++++++++++++---- .../AccordExecutorAbstractLockLoop.java | 110 +- .../accord/AccordExecutorAbstractLoop.java | 22 +- .../accord/AccordExecutorSemiSyncSubmit.java | 12 - .../accord/AccordExecutorSignalLoop.java | 446 +-- .../service/accord/AccordExecutorSimple.java | 12 + .../accord/AccordExecutorSyncSubmit.java | 5 +- .../accord/AccordFetchCoordinator.java | 2 +- .../accord/AccordSafeCommandStore.java | 2 +- .../service/accord/AccordService.java | 2 +- .../cassandra/service/accord/AccordTask.java | 221 +- .../service/accord/debug/DebugExecution.java | 8 +- .../accord/interop/AccordInteropAdapter.java | 2 +- .../interop/AccordInteropExecution.java | 2 +- .../accord/interop/AccordInteropPersist.java | 2 +- .../utils/concurrent/SignalLock.java | 6 - .../cassandra/simulator/ActionSchedule.java | 5 +- .../simulator/test/AccordExecutorTest.java | 217 +- .../CompactionAccordIteratorsTest.java | 2 +- .../service/accord/AccordCommandTest.java | 2 +- .../service/accord/AccordTaskTest.java | 2 +- .../service/accord/AccordTestUtils.java | 6 +- .../accord/SimulatedAccordCommandStore.java | 4 +- .../CommandsForKeySerializerTest.java | 4 +- .../cassandra/utils/AccordGenerators.java | 2 +- 31 files changed, 2782 insertions(+), 1131 deletions(-) diff --git a/src/java/org/apache/cassandra/concurrent/CassandraThread.java b/src/java/org/apache/cassandra/concurrent/CassandraThread.java index 62cbcdb6a4..d66adc2442 100644 --- a/src/java/org/apache/cassandra/concurrent/CassandraThread.java +++ b/src/java/org/apache/cassandra/concurrent/CassandraThread.java @@ -18,14 +18,21 @@ package org.apache.cassandra.concurrent; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + import org.apache.cassandra.metrics.ThreadLocalMetrics; +import org.apache.cassandra.service.accord.AccordExecutor; import io.netty.util.concurrent.FastThreadLocalThread; -public class CassandraThread extends FastThreadLocalThread +public class CassandraThread extends FastThreadLocalThread implements AccordExecutor.AccordTaskRunner { private ThreadLocalMetrics threadLocalMetrics; private ExecutorLocals executorLocals; + private AccordExecutor accordActiveExecutor; + private AccordExecutor accordLockedExecutor; + private volatile AccordExecutor.Task accordActiveTask; + private static final AtomicReferenceFieldUpdater accordActiveTaskUpdater = AtomicReferenceFieldUpdater.newUpdater(CassandraThread.class, AccordExecutor.Task.class, "accordActiveTask"); private final ImmediateTaskHolder immediateTaskHolder; @@ -89,6 +96,54 @@ public class CassandraThread extends FastThreadLocalThread return current != null ? current : ExecutorLocals.none(); } + public final AccordExecutor accordActiveExecutor() + { + return accordActiveExecutor; + } + + public final void setAccordActiveExecutor(AccordExecutor newExecutor) + { + accordActiveExecutor = newExecutor; + } + + @Override + public final AccordExecutor accordLockedExecutor() + { + return accordLockedExecutor; + } + + @Override + public final boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor) + { + if (accordLockedExecutor != null) + return false; + accordLockedExecutor = newLockedExecutor; + return true; + } + + @Override + public final void clearAccordLockedExecutor() + { + accordLockedExecutor = null; + } + + public final AccordExecutor.Task accordActiveTask() + { + return accordActiveTask; + } + + // to be called only by the thread itself, so can (eventually) avoid any memory barriers + public final AccordExecutor.Task accordActiveSelfTask() + { + // TODO (expected): with newer JDK use accordActiveTaskUpdater.getPlain + return accordActiveTask; + } + + public final void setAccordActiveTask(AccordExecutor.Task newActiveTask) + { + accordActiveTaskUpdater.lazySet(this, newActiveTask); + } + // final to avoid skipping of the cleanup logic in child classes final public void run() { diff --git a/src/java/org/apache/cassandra/config/AccordConfig.java b/src/java/org/apache/cassandra/config/AccordConfig.java index b9fe93b9a4..fe226fe92f 100644 --- a/src/java/org/apache/cassandra/config/AccordConfig.java +++ b/src/java/org/apache/cassandra/config/AccordConfig.java @@ -35,7 +35,6 @@ import org.apache.cassandra.service.accord.serializers.Version; import org.apache.cassandra.service.consensus.TransactionalMode; import static org.apache.cassandra.config.AccordConfig.CatchupMode.NORMAL; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.HLC_FIFO; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_POOL_PER_SHARD; import static org.apache.cassandra.config.AccordConfig.QueueSubmissionModel.SYNC; import static org.apache.cassandra.config.AccordConfig.RangeIndexMode.in_memory; @@ -134,25 +133,65 @@ public class AccordConfig FIFO, /** - * If the work has an associated TxnId, prioritise by its HLC (and FIFO otherwise) + * If the work has an associated TxnId of Ballot, prioritise by the newest HLC (and FIFO otherwise) */ HLC_FIFO, /** - * Prioritise Apply, Stable, Commit, Accept, and PreAccept messages in that order. - * Within a given message type, prioritise by HLC. - * Other messages will be mixed with PreAccept messages, but using the next counter rather than the HLC of the TxnId. - * Note: this can have some performance edge cases for contended keys, as we may process Stable messages for later commands before - * we process earlier Accept/Commit, which may delay execution + * If the work has an associated TxnId, prioritise by its HLC (and FIFO otherwise) */ - PHASE_HLC_FIFO, + ORIG_HLC_FIFO + } + + public enum QueueBalancingModel + { + /** + * Always pick the task by priority + */ + PRIORITY_ONLY, /** - * Prioritise Apply, Stable, Commit, Accept, and PreAccept messages from the original coordinator only, in that order. - * Within a given message type, prioritise by HLC. - * Other messages will be mixed with PreAccept messages, but using the next counter rather than the HLC of the TxnId. + * Pick by phase first, so the highest phase with work ALWAYS runs. + * Within a phase, pick by priority. */ - ORIG_PHASE_HLC_FIFO + PHASE_ONLY, + + /** + * Always pick the task by priority. + */ + PRIORITY_BUDGET, + + /** + * Pick by phase first, so the highest phase with budget ALWAYS runs. + * Within a phase, pick by priority. + */ + PHASE_BUDGET, + + /** + * Pick by phase first, selecting the phase that has processed the least work recently relative to arrivals. + * Within a phase, pick by priority. + */ + PHASE_FAIR, + + /** + * Pick by phase first, selecting the phase that has processed the least work recently relative to arrivals. + * However, each phase has a budget that is consumed on dispatch, and we pick only from those phases with budget. + * If there is no phase with budget, the budget resets. + * Within a phase, pick by priority. + */ + PHASE_BUDGET_FAIR, + + /** + * While phases are within a threshold of imbalance, pick tasks by priority. + * Once the threshold is crossed, over-processed phases have a small penalty applied + * and work is prioritised by phase with the phase with the least recent work processed picked first, + * until the imbalance is resolved. + * + * Within a phase, pick by priority. + */ + BLENDED_PRIORITY_PHASE_FAIR, + + BLENDED_PRIORITY_PHASE_BUDGET_FAIR, } public QueueShardModel queue_shard_model = THREAD_POOL_PER_SHARD; @@ -168,7 +207,13 @@ public class AccordConfig */ public volatile OptionaldPositiveInt queue_thread_count = OptionaldPositiveInt.UNDEFINED; - public QueuePriorityModel queue_priority_model = HLC_FIFO; + public QueuePriorityModel queue_priority_model; + public QueueBalancingModel queue_balancing_model; + + public Integer queue_flow_imbalance_onset = null; + public Integer queue_flow_imbalance_width_shift = null; + public String queue_active_limits; + public String queue_budgets; /** * If set, the signal loop does not match park/unpark pairs, but instead consumers perform timed-park spin waits @@ -181,15 +226,6 @@ public class AccordConfig */ public DurationSpec.LongMicrosecondsBound queue_stop_check_interval; - /** - * If set, the signal loop reduces the number of threads it is using when the time spent parked exceeds real-time - * by this interval. - */ - public DurationSpec.LongMicrosecondsBound queue_signal_stop_check_interval_credit; - - // yield to other executor threads after executing this many tasks in a row, if there are waiting threads and tasks - public int queue_yield_interval = 100; - /** * If the HLC is older than this, queue by FIFO instead */ diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 35b11418a6..86238688f5 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -1875,7 +1875,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace Node node = AccordService.unsafeInstance().node(); if (Route.isFullRoute(route)) { - PrepareRecovery.recover(node, node.someSequentialExecutor(), txnId, NotKnownToBeInvalid, (FullRoute) route, null, LatentStoreSelector.standard(), (success, fail) -> { + PrepareRecovery.recover(node, node.someExclusiveExecutor(), txnId, NotKnownToBeInvalid, (FullRoute) route, null, LatentStoreSelector.standard(), (success, fail) -> { if (fail != null) result.setFailure(fail); else result.setSuccess(null); }); diff --git a/src/java/org/apache/cassandra/service/accord/AccordCache.java b/src/java/org/apache/cassandra/service/accord/AccordCache.java index 6b6c48b2fc..0d679b2f51 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCache.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCache.java @@ -345,7 +345,6 @@ public class AccordCache implements CacheSize Collection> load(LoadExecutor loadExecutor, P1 p1, P2 p2, AccordCacheEntry node) { - Type parent = node.owner.parent(); return node.load(loadExecutor, p1, p2).waiters(); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index 7bdb69efc0..60205bf890 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -27,7 +27,6 @@ import java.util.Objects; import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.concurrent.locks.Lock; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.IntFunction; @@ -59,12 +58,12 @@ import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.CommandSummaries; import accord.local.ExecutionContext; +import accord.local.ExecutionContext.Empty; import accord.local.MaxConflicts; import accord.local.MaxDecidedRX; import accord.local.MinimalCommand; import accord.local.MinimalCommand.MinimalWithDeps; import accord.local.NodeCommandStoreService; -import accord.local.ExecutionContext.Empty; import accord.local.RedundantBefore; import accord.local.RedundantBefore.Bounds; import accord.local.RedundantStatus.Property; @@ -100,6 +99,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; +import org.apache.cassandra.service.accord.AccordExecutor.AccordTaskRunner; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfo; import org.apache.cassandra.service.accord.api.TokenKey; @@ -167,12 +167,12 @@ public class AccordCommandStore extends CommandStore public static final class ExclusiveCaches extends Caches implements CommandStoreCaches { - private final Lock lock; + private final AccordExecutor owner; - public ExclusiveCaches(Lock lock, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) + public ExclusiveCaches(AccordExecutor owner, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) { super(global, commands, commandsForKeys); - this.lock = lock; + this.owner = owner; } @Override @@ -190,8 +190,8 @@ public class AccordCommandStore extends CommandStore @Override public void close() { - global().tryShrinkOrEvict(lock); - lock.unlock(); + global().tryShrinkOrEvict(owner.unsafeLock()); + owner.unlock(AccordTaskRunner.get()); } } @@ -261,7 +261,7 @@ public class AccordCommandStore extends CommandStore { commands = exclusive.commands.newInstance(this); commandsForKey = exclusive.commandsForKey.newInstance(this); - this.caches = new ExclusiveCaches(sharedExecutor.unsafeLock(), exclusive.global, commands, commandsForKey); + this.caches = new ExclusiveCaches(sharedExecutor, exclusive.global, commands, commandsForKey); } this.exclusiveExecutor = sharedExecutor.executor(id); @@ -323,13 +323,13 @@ public class AccordCommandStore extends CommandStore public ExclusiveCaches lockCaches() { //noinspection LockAcquiredButNotSafelyReleased - caches.lock.lock(); + caches.owner.lock(AccordTaskRunner.get()); return caches; } public ExclusiveCaches tryLockCaches() { - if (caches.lock.tryLock()) + if (caches.owner.tryLock(AccordTaskRunner.get())) return caches; return null; } @@ -403,27 +403,15 @@ public class AccordCommandStore extends CommandStore return lastSystemTimestampMicros.accumulateAndGet(node.now(), (a, b) -> Math.max(a + 1, b)); } @Override - public AsyncChain chain(ExecutionContext loadCtx, Function function) + public AsyncChain chain(ExecutionContext context, Function function) { - return AccordTask.create(this, loadCtx, function).chain(); + return AccordTask.create(this, context, function).chain(); } @Override - public AsyncChain chain(ExecutionContext executionContext, Consumer consumer) + public AsyncChain chain(ExecutionContext context, Consumer consumer) { - return AccordTask.create(this, executionContext, consumer).chain(); - } - - @Override - public AsyncChain priorityChain(ExecutionContext executionContext, Consumer consumer) - { - return AccordTask.create(this, executionContext, consumer).priorityChain(); - } - - @Override - public AsyncChain priorityChain(ExecutionContext executionContext, Function function) - { - return AccordTask.create(this, executionContext, function).priorityChain(); + return AccordTask.create(this, context, consumer).chain(); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index 929d3d58cb..f82215919e 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -31,12 +31,12 @@ import org.slf4j.LoggerFactory; import accord.api.Agent; import accord.api.DataStore; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.LocalListeners; import accord.api.ProgressLog; import accord.local.CommandStores; import accord.local.NodeCommandStoreService; -import accord.local.ExclusiveAsyncExecutor; import accord.local.ShardDistributor; import accord.utils.RandomSource; import accord.utils.Reduce; @@ -148,10 +148,10 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu } @Override - public ExclusiveAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someExclusiveExecutor() { int idx = ((int) Thread.currentThread().getId()) & mask; - return executors[idx].newSequentialExecutor(); + return executors[idx].newExclusiveExecutor(); } public synchronized void setCapacity(long bytes) diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java index 4277ec55c8..c600178513 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java @@ -21,7 +21,7 @@ package org.apache.cassandra.service.accord; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; -import java.util.Queue; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; import java.util.concurrent.RejectedExecutionException; @@ -41,6 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.Agent; +import accord.api.ExclusiveAsyncExecutor; import accord.api.RoutingKey; import accord.impl.AbstractAsyncExecutor; import accord.local.Command; @@ -54,28 +55,31 @@ import accord.messages.Request; import accord.primitives.Ballot; import accord.primitives.SaveStatus; import accord.primitives.TxnId; -import accord.utils.ArrayBuffers.BufferList; +import accord.utils.ArrayBuffers; +import accord.utils.IntrusiveHeapNode; import accord.utils.IntrusivePriorityHeap; import accord.utils.Invariants; import accord.utils.QuadConsumer; import accord.utils.QuadFunction; import accord.utils.QuintConsumer; +import accord.utils.TinyEnumSet; import accord.utils.TriConsumer; import accord.utils.TriFunction; import accord.utils.UnhandledEnum; import accord.utils.async.AsyncCallbacks.CallAndCallback; -import accord.utils.async.AsyncCallbacks.FlatCallAndCallback; -import accord.utils.async.AsyncCallbacks.RunAndCallback; import accord.utils.async.AsyncCallbacks.RunOrFail; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; import accord.utils.async.Cancellable; import org.apache.cassandra.cache.CacheSize; +import org.apache.cassandra.concurrent.CassandraThread; import org.apache.cassandra.concurrent.DebuggableTask; import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; import org.apache.cassandra.concurrent.ExecutorLocals; import org.apache.cassandra.concurrent.Shutdownable; +import org.apache.cassandra.config.AccordConfig; +import org.apache.cassandra.config.AccordConfig.QueueBalancingModel; import org.apache.cassandra.config.AccordConfig.QueuePriorityModel; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.metrics.AccordCacheMetrics; @@ -89,29 +93,52 @@ import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistograms import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; import org.apache.cassandra.service.accord.AccordCacheEntry.UniqueSave; +import org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup; +import org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup; +import org.apache.cassandra.service.accord.AccordExecutor.Task.GroupKind; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExclusiveExecutor; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugSequentialExecutor; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; import org.apache.cassandra.utils.Closeable; -import org.apache.cassandra.utils.MonotonicClock; import org.apache.cassandra.utils.WithResources; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.concurrent.Future; -import io.netty.util.concurrent.FastThreadLocal; - +import static accord.primitives.Routable.Domain.Range; import static accord.utils.Invariants.createIllegalState; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.PHASE_HLC_FIFO; +import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.ORIG_HLC_FIFO; import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER; import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; import static org.apache.cassandra.service.accord.AccordCache.registerJfrListener; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordTask.State.LOADING; -import static org.apache.cassandra.service.accord.AccordTask.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordTask.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.COUNTER_LOWBITS; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.COUNTER_MASKS; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.minCounterValue; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.selectByOverflowBits; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.setOverflowWhenLessEqual; +import static org.apache.cassandra.service.accord.AccordExecutor.RunnableTaskQueue.RUNNABLE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.ACCEPT; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.APPLY; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.COMMIT; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.PREACCEPT; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.RANGE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.STABLE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.COMMAND_STORE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.OTHER; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_SCAN; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.SAVE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.MAX_TRANCHE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.ASSIGNED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.INITIALIZED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_SCAN_RANGES; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -123,11 +150,82 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutorflow. + private static final int BLEND_SHIFT = 6, BLEND_TOTAL = 1 << BLEND_SHIFT; + private static final int FLOW_ONSET, FLOW_WIDTH_SHIFT; + private static final boolean BALANCE_BY_POSITION; + private static final long GLOBAL_QUEUE_LIMITS, GLOBAL_QUEUE_BUDGETS; + private static final long EXCLUSIVE_QUEUE_LIMITS, EXCLUSIVE_QUEUE_BUDGETS; + + static + { + AccordConfig config = DatabaseDescriptor.getAccord(); + AGE_TO_FIFO = config.queue_priority_age_to_fifo.to(TimeUnit.MICROSECONDS); + PRIORITY_MODEL = config.queue_priority_model != null ? config.queue_priority_model : QueuePriorityModel.HLC_FIFO; + BALANCING_MODEL = config.queue_balancing_model != null ? config.queue_balancing_model : QueueBalancingModel.BLENDED_PRIORITY_PHASE_BUDGET_FAIR; + FLOW_ONSET = config.queue_flow_imbalance_onset == null ? 4 : config.queue_flow_imbalance_onset; + FLOW_WIDTH_SHIFT = config.queue_flow_imbalance_width_shift == null ? 5 : config.queue_flow_imbalance_width_shift; + Invariants.require(FLOW_ONSET >= 0 && FLOW_WIDTH_SHIFT >= 0); + switch (BALANCING_MODEL) + { + default: throw new UnhandledEnum(BALANCING_MODEL); + case PRIORITY_ONLY: + case BLENDED_PRIORITY_PHASE_FAIR: + case BLENDED_PRIORITY_PHASE_BUDGET_FAIR: + case PRIORITY_BUDGET: + BALANCE_BY_POSITION = true; + break; + case PHASE_ONLY: + case PHASE_FAIR: + case PHASE_BUDGET: + case PHASE_BUDGET_FAIR: + BALANCE_BY_POSITION = false; + } + + { + long global = COUNTER_MASKS, exclusive = COUNTER_MASKS; + global ^= (0x7f ^ 1) << RANGE_SCAN.ordinal(); + if (config.queue_active_limits != null) + { + long[] limits = parseEnumParams(config.queue_active_limits, "queue_active_limits"); + global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), global, limits[0]); + exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), global, limits[1]); + } + GLOBAL_QUEUE_LIMITS = global; + EXCLUSIVE_QUEUE_LIMITS = exclusive; + } + + { + long global = encodeCounters(Map.of(COMMAND_STORE, 16L, LOAD, 16L, SAVE, 16L, OTHER, 16L, RANGE_LOAD, 2L, RANGE_SCAN, 1L)); + long exclusive = encodeCounters(Map.of(APPLY, 8L, STABLE, 7L, COMMIT, 6L, ACCEPT, 5L, OTHER, 4L, PREACCEPT, 4L, RANGE, 1L)); + if (config.queue_budgets != null) + { + long[] limits = parseEnumParams(config.queue_budgets, "queue_budgets"); + // any that are unset, set to the lowest value of any other specified + if (limits[0] != 0) + { + long min = minCounterValue(limits[0], 0); + long mins = min * COUNTER_LOWBITS; + if (min > 1) + { + mins ^= (min ^ 1) << (RANGE_SCAN.ordinal() * 8); // set the default RANGE_SCAN budget to 1 + mins ^= (min ^ 2) << (RANGE_LOAD.ordinal() * 8); // set the default RANGE_LOAD budget to 2 + } + global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), mins, limits[0]); + } + if (limits[1] != 0) + exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), minCounterValue(limits[1], 0) * COUNTER_LOWBITS, limits[1]); + } + GLOBAL_QUEUE_BUDGETS = global; + EXCLUSIVE_QUEUE_BUDGETS = exclusive; + } + } + public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); - private static final FastThreadLocal paranoidPriorityInversionCheck = new FastThreadLocal<>(); public interface AccordExecutorFactory { @@ -136,6 +234,92 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor threadLocal = ThreadLocal.withInitial(ThreadLocalAccordTaskRunner::new); + + @Override + public AccordExecutor accordActiveExecutor() + { + return activeExecutor; + } + + @Override + public void setAccordActiveExecutor(AccordExecutor newExecutor) + { + activeExecutor = newExecutor; + } + + @Override + public AccordExecutor accordLockedExecutor() + { + return lockedExecutor; + } + + @Override + public boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor) + { + if (lockedExecutor != null) + return false; + lockedExecutor = newLockedExecutor; + return true; + } + + @Override + public void clearAccordLockedExecutor() + { + lockedExecutor = null; + } + + @Override + public void setAccordActiveTask(Task newActiveTask) + { + activeTask = newActiveTask; + } + + @Override + public Task accordActiveTask() + { + return activeTask; + } + + @Override + public Task accordActiveSelfTask() + { + return activeTask; + } + } + // WARNING: this is a shared object, so close is NOT idempotent public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable { @@ -152,7 +336,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor deferred; + + WithDeferred(Runnable run, Runnable deferred) + { + this.run = run; + this.deferred = new ArrayDeque<>(1); + this.deferred.add(deferred); + } + + WithDeferred(Runnable run, ArrayDeque deferred) + { + this.run = run; + this.deferred = deferred; + } + + @Override + public void run() + { + Runnable register = deferred.poll(); + if (!deferred.isEmpty()) + { + Invariants.require(runs[firstIndex] != null); + Invariants.require(!(runs[firstIndex] instanceof WithDeferred)); + runs[firstIndex] = new WithDeferred(runs[firstIndex], deferred); + } + registerWait(register); + try { run.run(); } + catch (Throwable t) { owner.agent.onException(t); } + } } - public String toString() + final AccordExecutor owner; + + int firstTranche; + int firstIndex; + long[] mins = new long[8]; + int[] counts = new int[8]; + Runnable[] runs = new Runnable[8]; + + // the next tranche, that all new work is being collected against + // (and will move to the array accounting once a new wait is registered) + long nextMin; + int nextTranche; + int nextCount; + + private Tranches(AccordExecutor owner) { - return run.toString() + " @" + position; + this.owner = owner; + } + + private int trancheToIndex(int tranche) + { + return firstIndex + trancheToIndexOffset(tranche); + } + + private int trancheToIndexOffset(int tranche) + { + int offset = tranche - firstTranche; + if (offset < 0) + offset += MAX_TRANCHE + 1; + return offset; + } + + int size() + { + return trancheToIndexOffset(nextTranche); + } + + int capacity() + { + return counts.length; + } + + int addNew(long position) + { + Invariants.require(position >= nextMin); + ++nextCount; + return nextTranche; + } + + void addInherited(int tranche, long position) + { + if (tranche == nextTranche) + { + Invariants.require(position >= nextMin); + ++nextCount; + } + else + { + int index = trancheToIndex(tranche); + Invariants.require(counts[index] > 0); + Invariants.require(mins[index] <= position); + ++counts[index]; + } + } + + void complete(int tranche) + { + if (tranche == nextTranche) + { + --nextCount; + } + else + { + + if (counts[trancheToIndex(tranche)] == 1) + owner.drainUnqueuedNewWorkExclusive(); // make sure we don't have any pending + + if (--counts[trancheToIndex(tranche)] == 0 && tranche == firstTranche) + { + do advance(); + while (firstTranche != nextTranche && counts[firstIndex] == 0); + } + + if (firstIndex >= counts.length / 2) + compact(); + } + } + + public void finishAll(long nextPosition) + { + while (firstTranche != nextTranche) + { + logger.warn("{} processed all pending tasks (<{}) but found {} waiting for {}", this, + nextPosition, counts[firstIndex], size() == 1 ? nextMin : mins[firstIndex + 1]); + advance(); + } + } + + private void advance() + { + Runnable run = runs[firstIndex]; + runs[firstIndex] = null; + ++firstIndex; + if (firstTranche == MAX_TRANCHE) firstTranche = 0; + else ++firstTranche; + try { run.run(); } + catch (Throwable t) { owner.agent.onException(t); } + } + + public void registerWait(Runnable run) + { + int newNextTranche = (nextTranche + 1) % (MAX_TRANCHE + 1); + if (newNextTranche == firstTranche) + { + Runnable cur = runs[firstIndex]; + if (cur instanceof WithDeferred) ((WithDeferred) cur).deferred.add(run); + else runs[firstIndex] = new WithDeferred(runs[firstIndex], run); + return; + } + + if ((firstIndex + size()) == capacity()) + growOrCompact(); + + int index = firstIndex + size(); + mins[index] = nextMin; + counts[index] = nextCount; + runs[index] = run; + nextMin = owner.minPosition = owner.nextPosition; + nextCount = 0; + nextTranche = newNextTranche; + } + + private void compact() + { + if (size() <= capacity()/4 && capacity() > 8) resize(capacity()/2); + else compact(mins, counts, runs); + } + + private void growOrCompact() + { + if (size() > capacity()/2) resize(capacity() * 2); + else compact(); + } + + private void resize(int newSize) + { + Invariants.require(newSize > 0); + long[] newMins = new long[newSize]; + int[] newCounts = new int[newSize]; + Runnable[] newRuns = new Runnable[newSize]; + compact(newMins, newCounts, newRuns); + mins = newMins; + counts = newCounts; + runs = newRuns; + } + + private void compact(long[] newMins, int[] newCounts, Runnable[] newRuns) + { + int size = size(); + System.arraycopy(mins, firstIndex, newMins, 0, size); + System.arraycopy(counts, firstIndex, newCounts, 0, size); + System.arraycopy(runs, firstIndex, newRuns, 0, size); + firstIndex = 0; } } - private final Lock lock; - final Agent agent; - final int executorId; - private final AccordCache cache; - - private final TaskQueue> scanningRanges = new TaskQueue<>(SCANNING_RANGES); // never queried, just parked here while scanning - private final TaskQueue> loading = new TaskQueue<>(LOADING); // never queried, just parked here while loading - private final TaskQueue running = new TaskQueue<>(RUNNING); - - private final TaskQueue> waitingToLoadRangeTxns = new TaskQueue<>(WAITING_TO_LOAD); - private final TaskQueue> waitingToLoad = new TaskQueue<>(WAITING_TO_LOAD); - private final TaskQueue waitingToRun = new TaskQueue<>(WAITING_TO_RUN); - - private final ExclusiveGlobalCaches caches; - - private List waitingForQuiescence; - private Queue waitingForCompletion; - final LogLinearDecayingHistograms histograms; final LogLinearDecayingHistogram elapsedPreparingToRun; final LogLinearDecayingHistogram elapsedWaitingToRun; @@ -215,11 +571,20 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor> scanningRanges = new StandaloneTaskQueue<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning + private final StandaloneTaskQueue> waitingToLoadRangeTxns = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_TO_LOAD)); + private final StandaloneTaskQueue> waitingToLoad = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD)); + private final StandaloneTaskQueue> loading = new StandaloneTaskQueue<>(LOADING); + private final RunnableTaskQueue runnable = new RunnableTaskQueue<>(); + + private final Tranches tranches = new Tranches(this); + + /** + * Newly submitted work must take a position >= minPosition, but this condition does not apply to consequences of + * previously submitted work; this inherits the originating operation's position and tranche. + * This is to ensure afterSubmittedAndConsequences functions correctly. + */ + private long minPosition = 1; + private long nextPosition = 1; + int tasks; + private int activeLoads, activeRangeLoads; private boolean hasPausedLoading; - int tasks; - final DebugExecutor debug = DebugExecutor.maybeDebug(); + + private List waitingForQuiescence; AccordExecutor(Lock lock, int executorId, Agent agent) { @@ -267,7 +649,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor w.run.run()); - waitingForCompletion = null; - } + tranches.finishAll(nextPosition); } public void afterSubmittedAndConsequences(Runnable run) { - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + + lock(self); try { + drainUnqueuedNewWorkExclusive(); if (tasks == 0) { run.run(); return; } - if (waitingForCompletion != null) // escape hatch for some bug that means we lose a notification for a given task's queue position - maybeNotifyWaitingForCompletion(); - if (waitingForCompletion == null) - waitingForCompletion = new ArrayDeque<>(); - - long position = nextPosition; - minPosition = position; - waitingForCompletion.add(new WaitForCompletion(position, run)); + tranches.registerWait(run); } finally { - unlock(); + unlock(self); } } - void maybeUnpauseLoading() + final void maybeUnpauseLoading() { if (!hasPausedLoading) return; - if (cache.weightedSize() < maxWorkingCapacityInBytes || (loading.isEmpty() && waitingToRun.isEmpty())) + if (cache.weightedSize() < maxWorkingCapacityInBytes || (loading.isEmpty() || runnable.hasWaiting())) { hasPausedLoading = false; enqueueLoadsExclusive(); @@ -466,7 +812,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor> queue = waitingToLoadRangeTxns.isEmpty() || activeRangeLoads >= maxQueuedRangeLoads ? waitingToLoad : waitingToLoadRangeTxns; + StandaloneTaskQueue> queue = waitingToLoadRangeTxns.isEmpty() || activeRangeLoads >= maxQueuedRangeLoads ? waitingToLoad : waitingToLoadRangeTxns; AccordTask next = queue.peek(); if (next == null) return; @@ -474,7 +820,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor= maxWorkingCapacityInBytes) { // we have too much in memory already, and we have work waiting to run, so let that complete before queueing more - if (!loading.isEmpty() || !waitingToRun.isEmpty()) + if (!loading.isEmpty() || runnable.hasWaiting()) { AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); hasPausedLoading = true; @@ -542,43 +888,24 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor task, AccordCacheEntry load) { - boolean isForRangeTxn = task.hasRanges(); + boolean isForRangeTxn = task.isRange(); if (!isForRangeTxn) return false; for (AccordTask t : load.loadingOrWaiting().waiters()) { - if (!t.hasRanges()) + if (!t.isRange()) return false; } return true; } - @Override - public Cancellable execute(RunOrFail runOrFail) - { - PlainChain submit = new PlainChain(runOrFail); - return submit(submit); - } - - public AsyncChain buildDebuggable(Callable task, Object describe) - { - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return submit(new DebuggableChain(new CallAndCallback<>(task, callback), null, 0, describe)); - } - }; - } - private void parkRangeLoad(AccordTask task) { if (task.queued() != waitingToLoadRangeTxns) { - task.unqueueIfQueued(); - task.addToQueue(waitingToLoadRangeTxns); + task.unqueue(); + waitingToLoadRangeTxns.enqueue(task); } } @@ -590,13 +917,13 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor task) { task.onWaitingToRun(); - task.addToQueue(task.commandStore.exclusiveExecutor); + task.commandStore.exclusiveExecutor.enqueue(task); } private void waitingToRun(Task task, @Nullable ExclusiveExecutor queue) { task.onWaitingToRun(); - task.addToQueue(queue == null ? waitingToRun : queue); + if (queue == null) runnable.enqueue(task); + else queue.enqueue(task); } public ExclusiveExecutor executor() @@ -626,7 +954,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor i, operation); } - void submitPriority(AccordTask operation) - { - submit(AccordExecutor::submitPriorityExclusive, i -> i, operation); - } - void submitExclusive(AccordTask task) { - assignQueuePosition(task); - submitInternalExclusive(task); - } - - void submitPriorityExclusive(AccordTask task) - { - assignMinQueuePosition(task); - submitInternalExclusive(task); - } - - private void submitInternalExclusive(AccordTask task) - { + registerExclusive(task); task.setupExclusive(); - ++tasks; updateQueue(task); enqueueLoadsExclusive(); } public void submitExclusive(Runnable runnable) { - submitPlainExclusive(new PlainRunnable(null, runnable)); + submitPlainExclusive(new PlainRunnable(null, runnable, OTHER)); + } + + private Cancellable submitPlain(Plain task) + { + submit(AccordExecutor::submitPlainExclusive, i -> i, task); + return task; } private void submitPlainExclusive(Plain task) { - ++tasks; - assignQueuePosition(task); + registerExclusive(task); waitingToRun(task, task.executor()); } - Cancellable submitPlainExclusive(Task parent, AbstractIOTask task) + Cancellable submitPlainExclusive(Task parent, GlobalGroup group, AbstractIOTask task) { - return submitPlainExclusive(parent, new WrappedIOTask(task)); + return submitPlainExclusive(parent, new WrappedIOTask(task, group, parent.position, parent.tranche())); } - T submitPlainExclusive(Task parent, T task) + final T submitPlainExclusive(Task parent, T task) { Invariants.require(isOwningThread()); - ++tasks; - if (parent != null) inheritQueuePosition(parent, task); - else assignFifoQueuePosition(task); + if (parent == null) registerExclusive(task); + else registerConsequenceExclusive(parent, task); task.onWaitingToRun(); - waitingToRun.append(task); + runnable.enqueue(task); return task; } - private void assignQueuePosition(Task task) + private void registerConsequenceExclusive(Task parent, Task task) { - if (task.queuePosition != 0) updateNextPosition(task); - else assignFifoQueuePosition(task); + ++tasks; + int tranche = parent.tranche(); + tranches.addInherited(tranche, parent.position); + task.position = parent.position; + task.setInheritedTranche(tranche); } - private void assignQueuePosition(AccordTask task) + private void registerExclusive(Task task) { - if (task.queuePosition != 0) updateNextPosition(task); + ++tasks; + if (task.hasInherited()) + { + tranches.addInherited(task.tranche(), task.position); + } else { - long priority_bits = PRIORITY_BITS; - TxnId txnId = null; - switch (PRIORITY_MODEL) + long position = task.position; + if (position == 0) { - case ORIG_PHASE_HLC_FIFO: - case PHASE_HLC_FIFO: - { - // TODO (expected): we should process messages for a TxnId together, to avoid processing delayed messages out of order - ExecutionContext context = task.preLoadContext(); - if (context instanceof Request) - { - MessageType type = ((Request) context).type(); - if (type instanceof StandardMessage) - { - TxnId txnId0 = context.primaryTxnId(); - switch ((StandardMessage)type) - { - case APPLY_REQ: - { - priority_bits = 0L; - txnId = txnId0; - break; - } - case READ_EPHEMERAL_REQ: - case READ_REQ: - case STABLE_THEN_READ_REQ: - { - priority_bits = 1000000000000000L; - txnId = txnId0; - break; - } - case COMMIT_REQ: - { - Commit commit = (Commit) context; - if (PRIORITY_MODEL == PHASE_HLC_FIFO || commit.ballot.equals(Ballot.ZERO)) - txnId = commit.txnId; - if (commit.kind.saveStatus == SaveStatus.Stable) priority_bits = 1000000000000L; - else priority_bits = 2000000000000L; - break; - } - case ACCEPT_REQ: - { - Accept accept = (Accept) context; - if (PRIORITY_MODEL == PHASE_HLC_FIFO || accept.ballot.equals(Ballot.ZERO)) - txnId = accept.txnId; - priority_bits = 3000000000000L; - break; - } - case GET_EPHEMERAL_READ_DEPS_REQ: - case PRE_ACCEPT_REQ: - { - txnId = txnId0; - break; - } - } - } - } - break; - } - case HLC_FIFO: - { - txnId = task.preLoadContext().primaryTxnId(); - break; - } - case FIFO: - { - break; - } + task.position = position = nextPosition++; } - - if (txnId != null) + else { - long hlc = txnId.hlc(); - long delta = nextPosition - hlc; - if (delta < AGE_TO_FIFO) - { - long position = hlc; - if (delta <= 0) nextPosition = position + 1; - else if (position < minPosition) position = minPosition; - position |= priority_bits; - task.queuePosition = position; - return; - } + long delta = nextPosition - position; + if (delta >= AGE_TO_FIFO) + task.position = position = nextPosition++; + else if (delta <= 0) + nextPosition = position + 1; + else if (position < minPosition) + task.position = position = minPosition; } - - assignFifoQueuePosition(task); - } - - } - - private void assignMinQueuePosition(Task task) - { - task.queuePosition = minPosition | PRIORITY_BITS; - } - - private void assignFifoQueuePosition(Task task) - { - task.queuePosition = nextPosition++ | PRIORITY_BITS; - } - - private void updateNextPosition(Task task) - { - nextPosition = Math.max(nextPosition, (task.queuePosition & ~PRIORITY_BITS) + 1); - } - - private void inheritQueuePosition(Task parent, Task task) - { - task.queuePosition = parent.queuePosition; - } - - void completeTaskExclusive(Task task) - { - // for integration with SequentialExecutor, we must : - // - first take the position so that represents the just-executed task - // - call cleanup to submit any following task on the relevant sub-queue - // - remove the previous task from the running collection only if still present (SequentialExecutor will have removed it) - long position = task.queuePosition; - try - { - task.cleanupExclusive(this); - } - finally - { - --tasks; - if (running.contains(task)) - running.remove(task); - - if (waitingForCompletion != null && waitingForCompletion.peek().maybeNotify <= position) - maybeNotifyWaitingForCompletion(); - - cache.tryShrinkOrEvict(lock); + task.setTranche(tranches.addNew(position)); } } - private void maybeNotifyWaitingForCompletion() + final void completeTaskExclusive(Task task) { - long min = minPosition(waitingToRun.peek(), - minPosition(waitingToLoad.peek(), - minPosition(waitingToLoadRangeTxns.peek(), - minPosition(running.peek(), - minPosition(loading.peek(), - minPosition(scanningRanges.peek(), Long.MAX_VALUE)))))); - - while (!waitingForCompletion.isEmpty() && waitingForCompletion.peek().position - min <= 0) - waitingForCompletion.poll().run.run(); - if (waitingForCompletion.isEmpty()) - waitingForCompletion = null; - else - waitingForCompletion.peek().maybeNotify = min; + runnable.complete(task); + try { task.cleanupExclusive(this); } + finally { cache.tryShrinkOrEvict(lock); } } - private static long minPosition(@Nullable Task task, long min) + final void unregisterExclusive(Task task) { - return task == null ? min : Long.min(task.queuePosition, min); + int tranch = task.tranche(); + --tasks; + tranches.complete(tranch); } - void cancelExclusive(AccordTask task) + final void cancelExclusive(AccordTask task) { AccordTask.State state = task.state(); switch (state) @@ -972,7 +1171,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor> tasks = loaded.loading().copyWaiters()) + try (ArrayBuffers.BufferList> tasks = loaded.loading().copyWaiters()) { if (fail != null) { @@ -999,35 +1198,74 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor submit(Runnable run) { - PlainRunnable task = new PlainRunnable(new AsyncPromise<>(), run); - submit(task); + Task inherit = inherit(); + PlainRunnable task = inherit == null ? new PlainRunnable(new AsyncPromise<>(), run, OTHER) + : new PlainRunnable(new AsyncPromise<>(), run, OTHER, inherit.position, inherit.tranche()); + submitPlain(task); return task.result; } - public void execute(Runnable command) + public void execute(Runnable run) { - submit(new PlainRunnable(null, command)); + Task inherit = inherit(); + PlainRunnable task = inherit == null ? new PlainRunnable(null, run, OTHER) + : new PlainRunnable(null, run, OTHER, inherit.position, inherit.tranche()); + submitPlain(task); } - private Cancellable submit(Plain task) + @Override + public Cancellable execute(RunOrFail runOrFail) { - submit(AccordExecutor::submitPlainExclusive, i -> i, task); - return task; + Task inherit = inherit(); + PlainChain submit = inherit == null ? new PlainChain(runOrFail, null, ExclusiveGroup.OTHER) + : new PlainChain(runOrFail, null, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + return submitPlain(submit); + } + + public AsyncChain buildDebuggable(Callable call, Object describe) + { + return new AsyncChains.Head<>() + { + @Override + protected Cancellable start(BiConsumer callback) + { + Task inherit = inherit(); + return submitPlain(inherit == null ? new DebuggableChain(new CallAndCallback<>(call, callback), null, describe) + : new DebuggableChain(new CallAndCallback<>(call, callback), null, inherit.position, inherit.tranche(), describe)); + } + }; } public void executeDirectlyWithLock(Runnable command) { - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + lock(self); try { + self.setAccordActiveExecutor(this); command.run(); } finally { beforeUnlockExternal(); - unlock(); + self.clearAccordLockedExecutor(); + unlock(self); } } @@ -1077,60 +1315,287 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor runningUpdater = AtomicReferenceFieldUpdater.newUpdater(TaskRunner.class, Task.class, "running"); + private volatile AccordTaskRunner wrapped; + + protected void setWrapped(AccordTaskRunner wrapped) + { + this.wrapped = wrapped; + } @Override public DebuggableTask running() { - Task running = this.running; + Task running = wrapped.accordActiveTask(); return running == null ? null : running.debuggable(); } - - Task runningTask() - { - return running; - } - - void setRunning(Task debuggable) - { - runningUpdater.lazySet(this, debuggable); - } - - void clearRunning() - { - runningUpdater.lazySet(this, null); - } } - public static abstract class Task extends IntrusivePriorityHeap.Node + public static abstract class Task extends IntrusiveHeapNode { + enum State + { + INITIALIZED(), + WAITING_TO_SCAN_RANGES(INITIALIZED), + SCANNING_RANGES(WAITING_TO_SCAN_RANGES), + WAITING_TO_LOAD(INITIALIZED, SCANNING_RANGES), + LOADING(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD), + WAITING_TO_RUN(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD, LOADING), + ASSIGNED(WAITING_TO_RUN), + RUNNING(ASSIGNED), + PERSISTING(RUNNING), + FINISHED(RUNNING, PERSISTING), + CANCELLED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED), + FAILED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED, RUNNING, PERSISTING); + + private final int permittedFrom; + static final State[] VALUES = values(); + + State() + { + this.permittedFrom = 0; + } + + State(State ... permittedFroms) + { + int permittedFrom = 0; + for (State state : permittedFroms) + permittedFrom |= 1 << state.ordinal(); + this.permittedFrom = permittedFrom; + } + + boolean isPermittedFrom(int prevOrdinal) + { + return (permittedFrom & (1 << prevOrdinal)) != 0; + } + + boolean isExecuted() + { + return this.compareTo(PERSISTING) >= 0; + } + + boolean isComplete() + { + return this.compareTo(FINISHED) >= 0; + } + + static State forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + + enum GlobalGroup + { + COMMAND_STORE, + LOAD, + SAVE, + OTHER, + RANGE_LOAD(true), + RANGE_SCAN(true), + ; + + final int bits; + + GlobalGroup() + { + this(false); + } + + GlobalGroup(boolean isRange) + { + this.bits = (ordinal() << GLOBAL_GROUP_SHIFT) | (isRange ? RANGE_BIT : 0); + } + } + + enum ExclusiveGroup + { + APPLY, + STABLE, + COMMIT, + ACCEPT, + OTHER, + PREACCEPT, + RANGE(true), + ; + + final int bits; + + ExclusiveGroup() + { + this(false); + } + + ExclusiveGroup(boolean isRange) + { + this.bits = (ordinal() << EXCLUSIVE_GROUP_SHIFT) | (isRange ? RANGE_BIT : 0); + } + } + + public enum GroupKind + { + EXCLUSIVE(ExclusiveGroup.values().length, EXCLUSIVE_GROUP_SHIFT), + GLOBAL(GlobalGroup.values().length, GLOBAL_GROUP_SHIFT), + NONE(0, 0); + + final int count; + final byte shift; + GroupKind(int count, int shift) + { + this.count = count; + this.shift = (byte) shift; + } + } + + private static final int STATE_MASK = 0xf; + private static final int GROUP_MASK = 0x7; + private static final int GLOBAL_GROUP_SHIFT = 7; + private static final int EXCLUSIVE_GROUP_SHIFT = 4; + private static final int RANGE_BIT = 1 << 10; + private static final int CLEANUP_BIT = 1 << 11; + private static final int HAS_TRANCHE_BIT = 1 << 12; + private static final int HAS_INHERITED_BIT = 1 << 13; + private static final int TRANCHE_SHIFT = 14; + static final int MAX_TRANCHE = 0x3ff; + public final WithResources resources; Task next; - long queuePosition; + + long position; + private int info; + + // TODO (expected): do we need this? we should be able to determine the queue from state() if needed for e.g. cancellation + private TaskQueue queued; + + // TODO (expected): expose via executors vtable + // TODO (expected): use just one long and some flag bits to indicate which point it represents, and report incrementally public long createdAt = nanoTime(), waitingToRunAt, runningAt, cleanupAt; - protected Task() + protected Task(GlobalGroup group, State state) { resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(group, ExclusiveGroup.OTHER, state); + } + + protected Task(ExclusiveGroup group, State state) + { + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(GlobalGroup.OTHER, group, state); + } + + protected Task(GlobalGroup group, State state, long position, int tranche) + { + this(group, state); + this.position = position; + setInheritedTranche(tranche); + } + + protected Task(ExclusiveGroup group, State state, long position, int tranche) + { + this(group, state); + this.position = position; + setInheritedTranche(tranche); + } + + protected Task(ExecutionContext context) + { + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + ExclusiveGroup group = ExclusiveGroup.OTHER; + TxnId txnId = context.primaryTxnId(); + if (txnId != null) + { + if (txnId.is(Range)) group = ExclusiveGroup.RANGE; + else + { + switch (PRIORITY_MODEL) + { + case HLC_FIFO: + case ORIG_HLC_FIFO: + { + // TODO (expected): we should process messages for a TxnId together, to avoid processing delayed messages out of order + if (context instanceof Request) + { + MessageType type = ((Request) context).type(); + if (type instanceof StandardMessage) + { + switch ((StandardMessage)type) + { + case APPLY_REQ: + { + group = APPLY; + break; + } + case READ_EPHEMERAL_REQ: + case READ_REQ: + case STABLE_THEN_READ_REQ: + { + group = STABLE; + break; + } + case COMMIT_REQ: + { + Commit commit = (Commit) context; + if (PRIORITY_MODEL == ORIG_HLC_FIFO && !commit.ballot.equals(Ballot.ZERO)) + txnId = null; + if (commit.kind.saveStatus == SaveStatus.Stable) group = STABLE; + else group = COMMIT; + break; + } + case ACCEPT_REQ: + { + Accept accept = (Accept) context; + if (PRIORITY_MODEL == ORIG_HLC_FIFO && !accept.ballot.equals(Ballot.ZERO)) + txnId = null; + group = ExclusiveGroup.ACCEPT; + break; + } + case GET_EPHEMERAL_READ_DEPS_REQ: + case PRE_ACCEPT_REQ: + { + group = ExclusiveGroup.PREACCEPT; + break; + } + default: + { + txnId = null; + } + } + } + } + else + { + txnId = null; + } + break; + } + case FIFO: + { + txnId = null; + break; + } + } + } + } + + this.info = init(GlobalGroup.OTHER, group, INITIALIZED); + if (txnId != null) + this.position = txnId.hlc(); } public final Task unwrap() { - if (this instanceof SequentialQueueTask) - return ((SequentialQueueTask) this).queue.task; + if (this instanceof ExclusiveExecutorTask) + return ((ExclusiveExecutorTask) this).queue.task; return this; } final void setReadyToCleanup() { - queuePosition |= Long.MIN_VALUE; + info |= CLEANUP_BIT; } final boolean isReadyToCleanup() { - return 0 != (queuePosition & Long.MIN_VALUE); + return 0 != (info & CLEANUP_BIT); } final void onRunning() @@ -1165,6 +1630,8 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor>> GLOBAL_GROUP_SHIFT) & GROUP_MASK; + } + + final int exclusiveGroupOrdinal() + { + return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; + } + + @Nullable + final TaskQueue queued() + { + return queued; + } + + final void unqueueIfQueued() + { + if (queued != null) + unqueue(); + } + + final void unqueue() + { + Invariants.require(queued != null); + queued.unqueue(this); + queued = null; + } + + final void unsetQueue(TaskQueue queue) + { + Invariants.require(queued == queue); + queued = null; + } + + final void setQueue(TaskQueue queue) + { + Invariants.require(queued == null); + Invariants.require(isCompatible(queue)); + queued = queue; + } + + private boolean isCompatible(TaskQueue queue) + { + int self = stateOrdinal(); + return TinyEnumSet.contains(queue.states, self); + } + + private static int init(GlobalGroup global, ExclusiveGroup exclusive, State state) + { + return global.bits | exclusive.bits | state.ordinal(); + } + + final void setTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; + } + + final void setInheritedTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; + } + + final int tranche() + { + Invariants.require((info & HAS_TRANCHE_BIT) != 0); + return info >>> TRANCHE_SHIFT; + } + + final boolean hasInherited() + { + return (info & HAS_INHERITED_BIT) != 0; + } } // run the task even on a stopped commandStore @@ -1215,16 +1805,22 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(ExclusiveExecutor.class, Thread.class, "owner"); - public class ExclusiveExecutor extends TaskQueue implements accord.local.ExclusiveAsyncExecutor + public final class ExclusiveExecutor extends MultiTaskQueue implements ExclusiveAsyncExecutor { final int commandStoreId; - final SequentialQueueTask selfTask; + final ExclusiveExecutorTask selfTask; private Task task; - private volatile Thread owner, waiting; + volatile Thread owner, waiting; private boolean stopped; private volatile boolean visibleStopped; private boolean terminated; - final DebugSequentialExecutor debug; + final DebugExclusiveExecutor debug; ExclusiveExecutor(AccordExecutor executor) { @@ -1284,15 +1879,14 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor) task).preLoadContext())); + task.fail(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).executionContext())); else task.run(); // NOTE: cannot safely release owner here, in case an immediate-execution runs before we can release our references and store their changes to the cache @@ -1330,8 +1924,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor)) return true; - ExecutionContext context = ((AccordTask) task).preLoadContext(); - + ExecutionContext context = ((AccordTask) task).executionContext(); return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); } @@ -1345,71 +1938,78 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor chain(Runnable run) { - long position = inheritQueuePosition(); - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return execute(new RunAndCallback(run, callback), position); - } - }; + return AsyncChains.chain(this, run); } @Override - public AsyncChain chain(Callable call) + public AsyncChain chain(Callable call) { - long position = inheritQueuePosition(); - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return execute(new CallAndCallback<>(call, callback), position); - } - }; + return AsyncChains.chain(this, call); } @Override - public AsyncChain flatChain(Callable> call) + public AsyncChain flatChain(Callable> call) { - long position = inheritQueuePosition(); - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return execute(new FlatCallAndCallback<>(call, callback), position); - } - }; + return AsyncChains.flatChain(this, call); } - @Override - public Cancellable execute(RunOrFail runOrFail) + Task inherit() { - return execute(runOrFail, inheritQueuePosition()); - } - - private long inheritQueuePosition() - { - return inExecutor() && task != null ? task.queuePosition : 0; - } - - private Cancellable execute(RunOrFail runOrFail, long queuePosition) - { - PlainChain submit = new PlainChain(runOrFail, ExclusiveExecutor.this, queuePosition); - return AccordExecutor.this.submit(submit); + Thread thread = Thread.currentThread(); + if (thread == owner) + return task; + return AccordTaskRunner.get(thread).accordActiveSelfTask(); } @Override public void execute(Runnable run) { - PlainRunnable submit = new PlainRunnable(null, run, this, inheritQueuePosition()); - AccordExecutor.this.submit(submit); + Task inherit = inherit(); + PlainRunnable submit = inherit == null ? new PlainRunnable(null, run, this, ExclusiveGroup.OTHER) + : new PlainRunnable(null, run, this, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + submitPlain(submit); + } + + @Override + public Cancellable execute(RunOrFail runOrFail) + { + Task inherit = inherit(); + PlainChain submit = inherit == null ? new PlainChain(runOrFail, ExclusiveExecutor.this, ExclusiveGroup.OTHER) + : new PlainChain(runOrFail, ExclusiveExecutor.this, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + return submitPlain(submit); } @Override public boolean tryExecuteImmediately(Runnable run) { - Thread self = Thread.currentThread(); + Thread thread = Thread.currentThread(); Thread owner = this.owner; - if (owner != null ? owner != self : !ownerUpdater.compareAndSet(this, null, self)) + if (owner != null && owner != thread) return false; + AccordTaskRunner self = AccordTaskRunner.get(thread); + AccordExecutor active = self.accordActiveExecutor(); + if (active != null && active != AccordExecutor.this) + return false; // prevent cross-executor locking/execution + + if (!ownerUpdater.compareAndSet(this, null, thread)) + return false; + + if (active == null) + self.setAccordActiveExecutor(AccordExecutor.this); + try { run.run(); } catch (Throwable t) { agent.onException(t); } finally { + if (active == null) + self.setAccordActiveExecutor(null); + if (owner == null) { Thread waiting = this.waiting; @@ -1559,80 +2133,834 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor extends IntrusivePriorityHeap { - final AccordTask.State kind; + final int states; + public TaskQueue(int states) {this.states = states; } - TaskQueue(AccordTask.State kind) + void unqueue(T task) { - this.kind = kind; - } - - TaskQueue(AccordTask.State kind, boolean tiny) - { - super(tiny); - this.kind = kind; + throw new UnsupportedOperationException(); } @Override - public int compare(T o1, T o2) + public final int compare(T o1, T o2) { - return Long.compare(o1.queuePosition, o2.queuePosition); + return Long.compare(o1.position, o2.position); } - protected void append(T task) + @Override + protected final void ensureHeapified() { - super.append(task); + super.ensureHeapified(); } - protected void update(T task) + protected final boolean requeueSingle(T requeue) { - super.update(task); + int oldIndex = updateNode(requeue); + if (oldIndex < 0) + return false; + + int newIndex = heapIndex(requeue); + return Math.min(oldIndex, newIndex) == 0 && heapifiedSize() > 0; } - protected T poll() - { - ensureHeapified(); - return pollNode(); - } - - protected T peek() + final T peekSingle() { ensureHeapified(); return peekNode(); } - protected T get(int index) + final T pollSingle() { - return super.get(index); + ensureHeapified(); + return pollNode(); } - protected void remove(T remove) + final boolean isEmptySingle() { - super.remove(remove); + return isEmptyInternal(); } - @Override - protected boolean removeIfContains(T node) + final int enqueueSingle(T enqueue) { - return super.removeIfContains(node); + return insertNode(enqueue); } - protected boolean contains(T contains) + final boolean unqueueSingle(T unqueue) { - return super.contains(contains); + int heapIndex = heapIndex(unqueue); + removeNode(unqueue); + return heapIndex == 0; + } + + final boolean tryUnqueueSingle(T remove) + { + return removeNodeIfContains(remove); + } + + final T getSingle(int index) + { + return super.getNode(index); + } + + final boolean isQueuedSingle(T test) + { + return containsNode(test); + } + } + + static final class StandaloneTaskQueue extends TaskQueue + { + StandaloneTaskQueue(int states) + { + super(states); + } + + StandaloneTaskQueue(Task.State state) + { + super(TinyEnumSet.encode(state)); + } + + void enqueue(T enqueue) + { + enqueue.setQueue(this); + enqueueSingle(enqueue); + } + + void unqueue(T unqueue) + { + unqueue.unsetQueue(this); + removeNode(unqueue); + } + + T peek() + { + return peekSingle(); + } + + T poll() + { + return pollSingle(); + } + + boolean isEmpty() + { + return isEmptySingle(); + } + } + + /** + * A {@link TaskQueue} sub-divided into up to eight per-group sub-queues (groups are phases for the exclusive + * executor, or work classes globally) plus a policy for choosing which sub-queue to serve next. The policy balances + * three competing concerns: ordering/priority (run the oldest / highest-priority work), fairness (do not let one + * group monopolise the executor), and throughput (do not let an even split leave a busy group's backlog growing + * without bound). + * + *

Packed counters

+ * Per-group state is held in {@code long}s, one 7-bit lane per group; bit 7 of each byte is a spare "guard" bit used + * to run branch-free min/compare across all eight lanes at once (see {@code minCounters}). The lanes are: + *
    + *
  • {@code positions[]} - the head task's position (HLC, or submission order) per group; drives FIFO/priority.
    • + *
  • {@code recent} - a windowed count of recently served tasks per group (service received).
    • + *
  • {@code arrivals} - a windowed, size-biased count of recent arrivals per group (demand offered).
    • + *
  • {@code current} - tasks currently in flight per group, used to enforce {@code queue_active_limits}.
    • + *
  • {@code hasWork}/{@code dirty} - guard-bit masks: which groups have queued work / need a position refresh.
    • + *
+ * + *

Windowing: a bounded memory of the past

+ * {@code recent} and {@code arrivals} are not running totals. Whenever incrementing {@code recent} tips any lane to + * its maximum (0x80), both {@code recent} and {@code arrivals} are halved (see {@code incrementRecents}), + * and each lane also saturates at 0x7f. They are therefore an exponentially-decaying window keyed on service/time. + * This is what stops a one-off burst of arrivals granting a group a permanent boost: the burst saturates the lane + * briefly and then decays away as other work is served. + * + *

The fair selection counter

+ * The fair models pick the group with the smallest {@code effective} counter, where per lane + * 
effective = min(0x7f, max(0, recent - arrivals) + bias)
+ *
    + *
  • {@code max(0, recent - arrivals)} - a group whose arrivals have outpaced its recent service clamps to 0 and + * is therefore preferred; in steady state each group's service tracks its arrival rate, which bounds backlog. + * The clamp is symmetric, so a group cannot bank "credit" during a lull and later use it to monopolise service.
    • + *
+ * + *

Choosing a group ({@code minGroup} / {@code pollByBlend})

+ *
    + *
  • {@code PRIORITY_ONLY} - always the lowest {@code positions} (oldest / highest priority), ignoring fairness.
    • + *
  • {@code PHASE_OVERRIDE} - strict phase order: the lowest-index group with work, always.
    • + *
  • {@code PHASE_FAIR} - pure fairness: the smallest {@code effective}; ties broken by index, within a group by + * position. It has no priority fallback, so it is deliberately completing-first under contention.
    • + *
  • {@code PRIORITY_FAIR} (default) - a deficit round-robin blend of two strategies, chosen per poll: flow + * (smallest {@code effective}, i.e. least fairly serviced) and age (oldest {@code positions}, i.e. FIFO). + * The flow weight ramps up with the flow imbalance ({@code max-min} of {@code effective}) over + * {@code queue_flow_imbalance_onset..+width}, trading against age zero-sum; when balanced it is pure age. The + * ramp is smooth (not a hard mode switch), so there is no boundary to oscillate around and no hysteresis is + * needed. Age also drains stale/standing backlogs for free: a backlog's items are the oldest work, so age + * clears them without a separate backlog-aware strategy (the {@code effective} flow counter tracks arrival + * rate and is deliberately blind to standing stock).
    • + *
+ * + *

Concurrency limits and eligibility

+ * A group whose in-flight {@code current} count has reached its {@code queue_active_limits} limit is + * {@code saturated} and excluded from selection ({@code disabled}), as is any group with no queued work. + * + *

Note on {@code positions} freshness

+ * {@code positions} is refreshed lazily from the sub-queue heads via the {@code dirty} mask; {@code dirty} must use + * the same guard-bit layout as {@code hasWork} so the refresh loop in {@code minGroupByPriority} actually runs - + * otherwise it silently degenerates to lowest-index-with-work and starves high-index / new-work groups. + * + *

NB it extends {@link TaskQueue} to keep the type hierarchy simple for method dispatch, and for efficiency for + * anonymous ExclusiveExecutors which do not use multiple queues, while letting ExclusiveExecutor share a parent + * class for both use cases. + */ + static abstract class MultiTaskQueue extends TaskQueue + { + private static final TaskQueue[] NO_QUEUES = new TaskQueue[0]; + private static final long[] NO_POSITIONS = new long[0]; + static final long COUNTER_OVERFLOWS = 0x8080808080808080L; + static final long COUNTER_MASKS = 0x7f7f7f7f7f7f7f7fL; + static final long COUNTER_LOWBITS = 0x0101010101010101L; + + final TaskQueue[] queues; + final long[] positions; + final byte groupShift; + final long baseBudget, limits; + + /** sets overflow bits for each counter when it needs its position updated */ + long dirty; + /** sets overflow bits for each counter when there's associated work */ + long hasWork; + /** Stores recent dequeue counts for up to 8 sub queues. */ + long dispatches; + /** Stores recent enqueue counts for up to 8 sub queues. */ + long arrivals; + /** Stores currently-active counts for up to 8 sub queues. We can use this to impose limits on specific queues. */ + long active; + /** Stores a biased budget for each task type, so that we may prefer to serve one type of task over another */ + long budget; + /** deficit-round-robin credits for the two PRIORITY_FAIR strategies (flow/age). */ + int creditFlow, creditAge; + + int waitingCount; + + MultiTaskQueue(int waitingStates, GroupKind groups, long budget, long limits) + { + super(waitingStates); + this.baseBudget = budget; + this.limits = limits; + int queueCount = groups.count; + Invariants.require(queueCount <= 8); + queues = queueCount == 0 ? NO_QUEUES : new TaskQueue[queueCount]; + positions = queueCount > 0 && BALANCE_BY_POSITION ? new long[queueCount] : NO_POSITIONS; + groupShift = groups.shift; + } + + final int group(Task task) + { + if (groupShift == 0) + return -1; + + return (task.info >>> groupShift) & Task.GROUP_MASK; + } + + final TaskQueue queue(Task task) + { + int group = group(task); + if (group < 0) + return this; + + return queue(group); + } + + final TaskQueue queue(int group) + { + TaskQueue queue = queues[group]; + if (queue == null) + queues[group] = queue = new TaskQueue<>(0); + + return queue; + } + + private int pollGroup() + { + if (hasWork == 0) + return -1; + + switch (BALANCING_MODEL) + { + default: throw new UnhandledEnum(PRIORITY_MODEL); + case PRIORITY_ONLY: return pollGroupByPriority(); + case PHASE_ONLY: return pollGroupByIndex(); + case PRIORITY_BUDGET: return pollGroupByPriorityWithBudget(); + case PHASE_BUDGET: return pollGroupByIndexWithBudget(); + case PHASE_FAIR: return pollGroupByPhaseFair(); + case PHASE_BUDGET_FAIR: return pollGroupByPhaseBudgetFair(); + case BLENDED_PRIORITY_PHASE_FAIR: return pollGroupByBlended(); + case BLENDED_PRIORITY_PHASE_BUDGET_FAIR: return pollGroupByBlendedWithBudget(); + } + } + + private int pollGroupByPriority() + { + return pollGroupByPriority(unsaturatedWithWork()); + } + + private int pollGroupByPriority(long enabled) + { + long refresh = dirty & hasWork; + while (refresh != 0) + { + int bitIndex = Long.numberOfTrailingZeros(refresh); + int group = bitIndex / 8; + positions[group] = queues[group].peekSingle().position; + refresh ^= 1L << bitIndex; + } + dirty = 0; + + long minPosition = Long.MAX_VALUE; + int minGroup = -1; + long visit = enabled >>> 7; + while (visit != 0) + { + int bitIndex = Long.numberOfTrailingZeros(visit); + int group = bitIndex / 8; + long position = positions[group]; + if (position < minPosition) + { + minGroup = group; + minPosition = position; + } + visit ^= 1L << bitIndex; + } + + return minGroup; + } + + private int pollGroupByIndex() + { + long visit = (hasWork & unsaturated()) >>> 7; + if (visit == 0) + return -1; + + int bitIndex = Long.numberOfTrailingZeros(visit); + return bitIndex / 8; + } + + private int pollGroupByPriorityWithBudget() + { + return pollGroupByPriority(unsaturatedWithWorkAndBudget()); + } + + private int pollGroupByIndexWithBudget() + { + long visit = (hasWork & unsaturated()); + if (visit == 0) + return -1; + + visit = withBudgetOrReset(visit); + visit >>>= 7; + int bitIndex = Long.numberOfTrailingZeros(visit); + return bitIndex / 8; + } + + private int pollGroupByPhaseFair() + { + return minCounterIndex(recentFlowImbalances()); + } + + private int pollGroupByPhaseBudgetFair() + { + return minCounterIndex(recentFlowImbalances(), saturatedOrWithoutWorkOrWithoutBudget()); + } + + // PRIORITY_FAIR selection: a deficit round-robin blend of two strategies, chosen per poll: + // flow -> minCounterIndex(recent - arrivals + bias) (least fairly serviced) + // age -> minGroupByPriority() (earliest-queued work) + // wFlow = ramp(flow imbalance F = max-min of the selection counters); wAge = BLEND_TOTAL - wFlow. As flow gets + // uneven, polls trade from age to flow zero-sum; when balanced it is pure age (FIFO). A ramp (not a hard + // threshold) means there is no mode cliff to oscillate around, so no anti-oscillation penalty is needed. Age is + // also what drains a stale/standing backlog -- its items are the oldest -- so no separate stock strategy is needed. + private int pollGroupByBlended() + { + return pollGroupByBlended(saturatedOrWithoutWork()); + } + + private int pollGroupByBlended(long disabled) + { + long withoutWork = hasWork ^ COUNTER_OVERFLOWS; + long counters = recentFlowImbalances(); + long minMax = minMaxCounterValue(counters, withoutWork); + long min = minMax & 0x7f; + long max = minMax >>> 8; + int flowImbalance = (int) (max - min); + + int flowWeight = flowWeight(flowImbalance); + int priorityWeight = BLEND_TOTAL - flowWeight; + + creditFlow += flowWeight; + creditAge += priorityWeight; + + if (creditFlow >= creditAge) + { + creditFlow -= BLEND_TOTAL; + if (disabled != withoutWork) + min = minCounterValue(counters, disabled); + + return minCounterIndex(counters, min, disabled); + } + else + { + creditAge -= BLEND_TOTAL; + return pollGroupByPriority(disabled ^ COUNTER_OVERFLOWS); + } + } + + private int pollGroupByBlendedWithBudget() + { + return pollGroupByBlended(saturatedOrWithoutWorkOrWithoutBudget()); + } + + private long saturated() + { + return ((active | COUNTER_OVERFLOWS) - limits) & COUNTER_OVERFLOWS; + } + + private long unsaturated() + { + return saturated() ^ COUNTER_OVERFLOWS; + } + + private long unsaturatedWithWork() + { + return hasWork & unsaturated(); + } + + private long unsaturatedWithWorkAndBudget() + { + return withBudgetOrReset(hasWork & unsaturated()); + } + + private long saturatedOrWithoutWorkOrWithoutBudget() + { + return withoutBudgetOrReset(saturatedOrWithoutWork()); + } + + private long withoutBudgetOrReset(long disabled) + { + return COUNTER_OVERFLOWS ^ withBudgetOrReset(COUNTER_OVERFLOWS ^ disabled); + } + + private long withBudgetOrReset(long enabled) + { + long hasBudget = hasBudget(); + if ((hasBudget & enabled) != 0) + return hasBudget & enabled; + + budget = baseBudget; + return enabled; + } + + private long hasBudget() + { + return setOverflowWhenLessEqual(budget, 0) ^ COUNTER_OVERFLOWS; + } + + private long saturatedOrWithoutWork() + { + return (hasWork ^ COUNTER_OVERFLOWS) | saturated(); + } + + // arrivals is a windowed measure of ARRIVAL (incremented on enqueue, size-biased; decays on recent's overflow). + // Combined with recent (service) as effective = max(0, recent - arrivals): a queue whose arrivals outpace its + // service clamps to 0 and is preferred, so service converges to arrival rate (bounding the busy queue's backlog). + private long recentFlowImbalances() + { + return clampedSubtract(dispatches, arrivals); + } + + static long minCounterValue(long counters, long disabled) + { + long mins = counters; + mins |= overflowsToLowMasks(disabled); + mins = minCounters(mins, mins >>> 8); // each slot is min of slots [i..i+1] + mins = minCounters(mins, mins >>> 16); // each slot is min of slots [i..i+3] + mins = minCounters(mins, mins >>> 32); // each slot is min of slots [i..i+7] + return mins & 0x7f; + } + + static long minMaxCounterValue(long counters, long disabled) + { + long mins = counters; + long maxs = counters ^ COUNTER_MASKS; + long overflowMasks = overflowsToLowMasks(disabled); + mins |= overflowMasks; + maxs |= overflowMasks; + mins = minCounters(mins, mins >>> 8) & 0x007f007f007f007fL; // each slot is min of slots [i..i+1] + maxs = (minCounters(maxs, maxs << 8) & 0x7f007f007f007f00L); // each slot is min of slots ~[i..i+1] + long minmaxs = mins | maxs; + minmaxs = minCounters(minmaxs, minmaxs >>> 16); // each slot is min of slots [i..i+3] + minmaxs = minCounters(minmaxs, minmaxs >>> 32); // each slot is min of slots [i..i+7] + return (minmaxs ^ 0x7f00) & 0x7f7f; + } + + /** + * If provided two counters (containing 8 7 bit counters each), + * returns the minimum of each matching counter + */ + private static long minCounters(long a, long b) + { + // set overflow bits where a <= b + long selecta = setOverflowWhenLessEqual (a, b); + return selectByOverflowBits(selecta, a, b); + } + + static long setOverflowWhenLessEqual(long a, long b) + { + return ((b | COUNTER_OVERFLOWS) - a) & COUNTER_OVERFLOWS; + } + + // select a if overflow bit is set; b if it is unset + static long selectByOverflowBits(long selecta, long a, long b) + { + selecta = overflowsToLowMasks(selecta); + a &= selecta; + b &= ~selecta; + return a | b; + } + + static long overflowsToLowMasks(long v) + { + return v - (v >>> 7); + } + + private static int flowWeight(int flowImbalance) + { + if (flowImbalance <= FLOW_ONSET) return 0; + return Math.min(BLEND_TOTAL, ((flowImbalance - FLOW_ONSET) << BLEND_SHIFT) >>> FLOW_WIDTH_SHIFT); + } + + // per-lane max(0, a - b), carry-free: zero both a and b in lanes where a <= b, then subtract + private static long clampedSubtract(long a, long b) + { + long keep = ~overflowsToLowMasks(setOverflowWhenLessEqual(a, b)); + return (a & keep) - (b & keep); + } + + private int minCounterIndex(long counters) + { + return minCounterIndex(counters, saturatedOrWithoutWork()); + } + + private int minCounterIndex(long counters, long disabled) + { + return minCounterIndex(counters, minCounterValue(counters, disabled), disabled); + } + + private int minCounterIndex(long counters, long minCounterValue, long disabled) + { + long mins = minCounterValue * COUNTER_LOWBITS; + long select = ((mins | COUNTER_OVERFLOWS) - counters) & COUNTER_OVERFLOWS; + // now unset those overflow bits associated with disabled queues + select &= ~disabled; + if (select == 0) + return -1; + return (Long.numberOfTrailingZeros(select) - 7) / 8; + } + + final T pollMulti() + { + int group = pollGroup(); + if (group < 0) + { + // group < 0 can mean EITHER we don't have any nested queues OR those queues are either empty or DISABLED + T result = pollSingle(); + if (result != null) + --waitingCount; + return result; + } + + --waitingCount; + incrementActive(group); + incrementDispatches(group); + decrementBudget(group); + + TaskQueue queue = queues[group]; + T head = queue.pollSingle(); + // NOTE: must clear dirty when emptied, symmetrically with unqueue(): the fair selection paths + // never consume the dirty bit, so a group drained during a fairness episode would otherwise + // retain a stale dirty bit and NPE in minGroupByPriority.peekSingle() when balance is restored. + if (queue.isEmptySingle()) { unsetHasWork(group); unsetDirty(group); } + else setDirty(group); + return head; + } + + final void enqueueMulti(T task) + { + task.setQueue(this); + int group = group(task); + if (group < 0) + { + enqueueSingle(task); + } + else + { + TaskQueue queue = queue(group); + int result = queue.enqueueSingle(task); + incrementArrivals(group); + if (result < 0) setHasWork(group); + if (result != 0) setDirty(group); + } + ++waitingCount; + } + + final void requeue(T task) + { + int group = group(task); + if (group < 0) requeueSingle(task); + else + { + TaskQueue queue = queue(group); + Invariants.require(queue != null && queue.isQueuedSingle(task)); + if (queue.requeueSingle(task)) + setDirty(group); + } + } + + final void unqueueMulti(T task) + { + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + Invariants.require(queue.isQueuedSingle(task)); + unqueue(task, group, queue); + } + + // if there is an active collection, we return false and do not remove ourselves from it + boolean tryUnqueueWaiting(T task) + { + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + if (!queue.isQueuedSingle(task)) + return false; + + unqueue(task, group, queue); + return true; + } + + private void unqueue(T task, int group, TaskQueue queue) + { + task.unsetQueue(this); + boolean dirty = queue.unqueueSingle(task); + --waitingCount; + if (group >= 0) + { + if (queue.isEmptySingle()) + { + unsetHasWork(group); + unsetDirty(group); + } + else if (dirty) setDirty(group); + } + } + + final void incrementActive(int group) + { + active += lowBit(group); + } + + final void decrementActive(int group) + { + active -= lowBit(group); + } + + final void incrementDispatches(Task task) + { + int group = group(task); + if (group >= 0) + incrementDispatches(group); + } + + final void incrementDispatches(int group) + { + dispatches += lowBit(group); + if ((dispatches & COUNTER_OVERFLOWS) != 0) + { + dispatches = (dispatches >>> 1) & COUNTER_MASKS; + arrivals = (arrivals >>> 1) & COUNTER_MASKS; // arrivals (arrival) decays on the service/time clock + } + } + + final void decrementBudget(int group) + { + // no need to manage underflow; if the budget is being used, + // a queue should only dispatch work when there's non-zero budget + budget -= lowBit(group); + } + + final void decrementDispatches(Task task) + { + int group = group(task); + if (group >= 0) + decrementDispatches(group); + } + + final void decrementDispatches(int group) + { + long lowBit = lowBit(group); + dispatches -= lowBit; + dispatches += (dispatches >>> 7) & lowBit; + } + + final void incrementArrivals(int group) + { + int shift = group * 8; + long overflowBit = 0x80L << shift; + arrivals += 1L << shift; + // if we overflow, unset the overflow bit and set all other bits for the counter + long overflow = arrivals & overflowBit; + arrivals ^= overflow; + arrivals |= overflow - (overflow >>> 7); + } + + final void setHasWork(int group) + { + hasWork |= overflowBit(group); + } + + final void unsetHasWork(int group) + { + hasWork &= ~overflowBit(group); + } + + final void setDirty(int group) + { + dirty |= overflowBit(group); + } + + final void unsetDirty(int group) + { + dirty &= ~overflowBit(group); + } + + final boolean hasWaiting() + { + return waitingCount > 0; + } + + final boolean isWaiting(T task) + { + return queue(task).isQueuedSingle(task); + } + + final int waitingCount() + { + return waitingCount; + } + + final long lowBit(int group) + { + return 1L << (group * 8); + } + + final long overflowBit(int group) + { + return 0x80L << (group * 8); + } + } + + static final class RunnableTaskQueue extends MultiTaskQueue + { + static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, ASSIGNED, RUNNING); + + final TaskQueue assigned; + + RunnableTaskQueue() + { + super(RUNNABLE, GroupKind.GLOBAL, GLOBAL_QUEUE_BUDGETS, GLOBAL_QUEUE_LIMITS); + this.assigned = new TaskQueue<>(0); + } + + T poll() + { + T next = pollMulti(); + if (next == null) + return null; + + next.setState(ASSIGNED); + assigned.enqueueSingle(next); + + return next; + } + + void enqueue(T enqueue) + { + enqueueMulti(enqueue); + } + + void unqueue(T unqueue) + { + if (assigned.isQueuedSingle(unqueue)) + { + int group = group(unqueue); + if (group >= 0) + decrementActive(group); + + unqueue.unsetQueue(this); + assigned.unqueueSingle(unqueue); + } + else + { + super.unqueueMulti(unqueue); + } + } + + int waitingOrAssignedCount() + { + return waitingCount + assigned.size(); + } + + boolean hasAssignedOrWaiting() + { + return waitingCount > 0 || !assigned.isEmptySingle(); + } + + boolean hasAssigned() + { + return !assigned.isEmptySingle(); + } + + boolean isAssigned(T task) + { + return assigned.isQueuedSingle(task); + } + + void complete(T task) + { + if (assigned.tryUnqueueSingle(task)) + { + int group = group(task); + if (group >= 0) + decrementActive(group); + task.unsetQueue(this); + } } } static class CancelTask extends Task { final Task cancel; - private CancelTask(Task cancel) { this.cancel = cancel; } + private CancelTask(Task cancel) + { + super(GlobalGroup.OTHER, WAITING_TO_RUN); + this.cancel = cancel; + } + @Override void submitExclusive(AccordExecutor owner) { cancel.cancelExclusive(owner); } @Override protected void preRunExclusive() { throw new UnsupportedOperationException(); } @Override protected void run() { throw new UnsupportedOperationException(); } @Override protected void fail(Throwable fail) { throw new UnsupportedOperationException(); } - @Override protected void addToQueue(TaskQueue queue) { throw new UnsupportedOperationException(); } + @Override protected boolean isNewWork() { return false; } + @Override String toDescription() { return "Cancel " + cancel.toDescription(); } } static IntFunction constant(O out) @@ -1642,18 +2970,31 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor result, Runnable run, GlobalGroup group, long position, int tranche) { - this(null, run); + super(group, position, tranche); + this.result = result; + this.run = run; + this.executor = null; } - PlainRunnable(AsyncPromise result, Runnable run) + PlainRunnable(AsyncPromise result, Runnable run, GlobalGroup group) { - this(result, run, null, 0); + super(group); + this.result = result; + this.run = run; + this.executor = null; } - PlainRunnable(AsyncPromise result, Runnable run, @Nullable ExclusiveExecutor executor, long queuePosition) + PlainRunnable(AsyncPromise result, Runnable run, ExclusiveExecutor executor, ExclusiveGroup group, long position, int tranche) { + super(group, position, tranche); this.result = result; this.run = run; this.executor = executor; - this.queuePosition = queuePosition; + } + + PlainRunnable(AsyncPromise result, Runnable run, ExclusiveExecutor executor, ExclusiveGroup group) + { + super(group); + this.result = result; + this.run = run; + this.executor = executor; + } + + @Override + String toDescription() + { + // TODO (expected): ensure this is usefully descriptive, or accept a separate description + return run.toString(); } @Override @@ -1735,17 +3101,18 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor LoadRunnable newLoad(AccordCacheEntry entry, boolean isForRange) { - return isForRange ? new LoadRangeRunnable<>(entry) : new LoadRunnable<>(entry); + return new LoadRunnable<>(entry, isForRange ? RANGE_LOAD : LOAD); } class LoadRunnable extends IOTask @@ -1794,12 +3161,14 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor entry; Object result = FailureHolder.NOT_STARTED; - LoadRunnable(AccordCacheEntry entry) + LoadRunnable(AccordCacheEntry entry, GlobalGroup group) { + super(group); + Invariants.require(group == LOAD || group == RANGE_LOAD); this.entry = entry; } - boolean isForRange() { return false; } + boolean isForRange() { return is(RANGE_LOAD); } void postRunExclusive() { @@ -1807,6 +3176,12 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor extends LoadRunnable - { - LoadRangeRunnable(AccordCacheEntry entry) { super(entry); } - @Override boolean isForRange() { return true; } - } - static abstract class AbstractIOTask { abstract protected void runInternal(); @@ -1849,11 +3218,18 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor entry, UniqueSave identity, Runnable run) { + super(SAVE); this.entry = entry; this.identity = identity; this.run = run; @@ -1905,6 +3282,12 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor) task).preLoadContext().reason(); + return ((AccordTask) task).executionContext().reason(); if (task instanceof DebuggableTask) return ((DebuggableTask) task).description(); @@ -2075,7 +3463,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor) task).preLoadContext(); + return ((AccordTask) task).executionContext(); if (task instanceof WrappedIOTask && ((WrappedIOTask) task).wrapped instanceof AccordTask.RangeTxnScanner) return ((AccordTask.RangeTxnScanner) ((WrappedIOTask) task).wrapped).preLoadContext(); return null; @@ -2093,19 +3481,24 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor taskSnapshot() { List result = new ArrayList<>(); - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + lock(self); try { - addToSnapshot(result, waitingToLoad, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); - addToSnapshot(result, waitingToLoadRangeTxns, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); addToSnapshot(result, scanningRanges, TaskInfo.Status.SCANNING_RANGES, TaskInfo.Status.SCANNING_RANGES); - addToSnapshot(result, loading, TaskInfo.Status.LOADING, TaskInfo.Status.LOADING); - addToSnapshot(result, waitingToRun, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); - addToSnapshot(result, running, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); + addToSnapshot(result, waitingToLoadRangeTxns, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); + addToSnapshot(result, waitingToLoad, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); + addToSnapshot(result, loading, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); + for (TaskQueue queue : runnable.queues) + { + if (queue != null) + addToSnapshot(result, queue, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); + } + addToSnapshot(result, runnable.assigned, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); } finally { - unlock(); + unlock(self); } result.sort(TaskInfo::compareTo); return result; @@ -2115,13 +3508,19 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor q0 : q.queues) + { + if (q0 != null) + { + for (int j = 0 ; j < q0.size() ; ++j) + snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q0.getSingle(j))); + } + } } else { @@ -2133,17 +3532,65 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor> get, String input, String describe) + { + long result = 0; + for (String spec : input.split(",")) + { + if (spec.trim().isEmpty()) continue; + + String[] split = spec.split(":"); + if (split.length != 2) + throw new IllegalArgumentException("Invalid specifier " + spec + " in " + describe + ": " + input); + + try + { + Enum queue = get.apply(split[0]); + long value = Long.parseLong(split[1]); + if (value <= 0 || value >= 128) + throw new IllegalArgumentException("Invalid limit " + value + " in queue_active_limits: " + input); + + result |= value << (queue.ordinal() * 8); + } + catch (Throwable t) + { + throw new IllegalArgumentException("Invalid queue identifier " + split[0] + " in " + describe + ": " + input); + } + } + + return result; + } + + private static long encodeCounters(Map, Long> counters) + { + long result = 0; + for (Map.Entry, Long> e : counters.entrySet()) + result |= e.getValue() << (e.getKey().ordinal() * 8); + return result; } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java index d8051f40dc..8e3e41f7c3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java @@ -24,7 +24,7 @@ import accord.api.Agent; import accord.utils.QuadFunction; import accord.utils.QuintConsumer; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.concurrent.CassandraThread; import org.apache.cassandra.service.accord.AccordExecutorLoops.LoopTask; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutorLoop; @@ -33,7 +33,6 @@ import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXE abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop { - private static final int YIELD_INTERVAL = DatabaseDescriptor.getAccord().queue_yield_interval; int runningThreads; boolean shutdown; @@ -44,11 +43,10 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop abstract void notifyWork(); abstract void notifyWorkExclusive(); - void loopYieldExclusive() throws InterruptedException {} abstract void awaitExclusive() throws InterruptedException; abstract void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4); - void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + final void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) { // if we're a loop thread, we will poll the waitingToRun queue when we come around // NOTE: this assumes no synchronous blocking tasks are submitted to this executor @@ -56,7 +54,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop else submitExternal(sync, async, p1s, p1a, p2, p3, p4); } - void submitExternalExclusive(QuintConsumer sync, P1s p1s, P2 p2, P3 p3, P4 p4) + final void submitExternalExclusive(QuintConsumer sync, P1s p1s, P2 p2, P3 p3, P4 p4) { try { @@ -84,6 +82,73 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop notifyWorkExclusive(); } + final boolean hasWaitingToRun() + { + updateWaitingToRunExclusive(); + return hasAlreadyWaitingToRun(); + } + + final Task pollWaitingToRunExclusive() + { + updateWaitingToRunExclusive(); + return pollAlreadyWaitingToRunExclusive(); + } + + final void updateWaitingToRunExclusive() + { + drainUnqueuedExclusive(); + super.updateWaitingToRunExclusive(); + } + + final void drainUnqueuedExclusive() + { + Task cur = Task.reverse(acquireUnqueuedExclusive()); + while (cur != null) + cur = enqueueOneExclusive(cur); + } + + final void drainUnqueuedNewWorkExclusive() + { + Task cur = acquireUnqueuedExclusive(); + Task prev = null, requeue = null, requeueLast = null; + while (cur != null) + { + Task next = cur.next; + if (cur.isNewWork()) + { + cur.next = prev; + prev = cur; + } + else + { + if (requeue == null) requeue = cur; + else requeueLast.next = cur; + requeueLast = cur; + } + cur = next; + } + + if (requeue != null) + { + while (true) + { + Task next = unqueued; + requeueLast.next = next; + if (unqueuedUpdater.compareAndSet(this, next, requeue)) + break; + } + } + + cur = prev; + while (cur != null) + { + Task next = cur.next; + cur.submitExclusive(this); + cur.next = null; + cur = next; + } + } + @Override final void beforeUnlockExternal() { @@ -130,11 +195,15 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop @Override public void run() { - Thread self = Thread.currentThread(); + Thread thread = Thread.currentThread(); + AccordTaskRunner self = AccordTaskRunner.get(thread); + self.setAccordActiveExecutor(AccordExecutorAbstractLockLoop.this); + setWrapped(self); + Task task; while (true) { - lock(); + lock(self); try { enterLockLoop(); @@ -144,7 +213,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop if (task != null) { - setRunning(task); + self.setAccordActiveTask(task); try { task.preRunExclusive(); @@ -157,7 +226,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop finally { completeTaskExclusive(task); - clearRunning(); + self.setAccordActiveTask(null); } } else @@ -185,7 +254,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop } finally { - unlock(); + unlock(self); } } } @@ -200,12 +269,15 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop @Override public void run() { - int count = 0; + CassandraThread self = (CassandraThread) Thread.currentThread(); + self.setAccordActiveExecutor(AccordExecutorAbstractLockLoop.this); + setWrapped(self); + Task task = null; while (true) { if (DEBUG_EXECUTION) debug.onLock(); - lock(); + lock(self); try { if (DEBUG_EXECUTION) debug.onEnterLock(); @@ -215,13 +287,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop Task tmp = task; task = null; completeTaskExclusive(tmp); - clearRunning(); - } - - if (count >= YIELD_INTERVAL) - { - loopYieldExclusive(); - count = 0; + self.setAccordActiveTask(null); } while (true) @@ -230,7 +296,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop if (task != null) { - setRunning(task); + self.setAccordActiveTask(task); task.preRunExclusive(); if (DEBUG_EXECUTION) debug.onExitLock(); exitLockLoop(); @@ -250,7 +316,6 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop awaitExclusive(); if (DEBUG_EXECUTION) debug.onEnterLock(); resumeLoop(); - count = 0; } } catch (Throwable t) @@ -276,12 +341,11 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop finally { if (DEBUG_EXECUTION) debug.onExitLock(); - unlock(); + unlock(self); } try { - ++count; task.run(); } catch (Throwable t) diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java index a76d1bac18..3cbd3db144 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java @@ -29,8 +29,8 @@ import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; abstract class AccordExecutorAbstractLoop extends AccordExecutor { - private volatile Task unqueued; - private static final AtomicReferenceFieldUpdater unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordExecutorAbstractLoop.class, Task.class, "unqueued"); + volatile Task unqueued; + static final AtomicReferenceFieldUpdater unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordExecutorAbstractLoop.class, Task.class, "unqueued"); AccordExecutorAbstractLoop(Lock lock, int executorId, Agent agent) { @@ -67,30 +67,18 @@ abstract class AccordExecutorAbstractLoop extends AccordExecutor if (hasUnqueued() || tasks > 0) return true; - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + lock(self); try { return hasUnqueued() || tasks > 0; } finally { - unlock(); + unlock(self); } } - final void updateWaitingToRunExclusive() - { - drainUnqueuedExclusive(); - super.updateWaitingToRunExclusive(); - } - - final void drainUnqueuedExclusive() - { - Task cur = Task.reverse(acquireUnqueuedExclusive()); - while (cur != null) - cur = enqueueOneExclusive(cur); - } - final Task acquireUnqueuedExclusive() { return unqueuedUpdater.getAndSet(this, null); diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java index f97e206ef5..67e897d479 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java @@ -46,18 +46,6 @@ public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSync this.loops = new AccordExecutorLoops(mode, threads, name, this::task); } - @Override - void loopYieldExclusive() throws InterruptedException - { - if (waiting > 0 && hasWaitingToRun()) - { - pauseLoop(); - hasWork.signal(); - awaitWork(); - resumeLoop(); - } - } - @Override void awaitExclusive() throws InterruptedException { diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java index 9ec7fa95d2..ce7de7fefc 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java @@ -100,6 +100,217 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop return new LoopTask(index, id); } + @Override + final void drainUnqueuedNewWorkExclusive() + { + updatePendingUnqueued(); + Task requeue = null, requeueTail = null; + Task cur = pendingNewHead; + while (cur != null) + { + Task next = cur.next; + if (cur.isNewWork()) + { + cur.next = null; + cur.submitExclusive(this); + --pendingCount; + } + else + { + if (requeue == null) requeue = cur; + else requeueTail.next = cur; + requeueTail = cur; + } + cur = next; + } + + pendingNewHead = requeue; + pendingNewTail = requeueTail; + } + + private boolean enqueueOnePending() + { + if (pendingCount == 0) + return false; + + if (pendingSequentialHead != null) + { + pendingSequentialHead = enqueueOneCleanup(pendingSequentialHead); + if (pendingSequentialHead == null) + pendingSequentialTail = null; + } + else if (pendingCleanupHead != null) + { + pendingCleanupHead = enqueueOneCleanup(pendingCleanupHead); + if (pendingCleanupHead == null) + pendingCleanupTail = null; + } + else + { + pendingNewHead = enqueueOneSubmit(pendingNewHead); + if (pendingNewHead == null) + pendingNewTail = null; + } + --pendingCount; + return true; + } + + private void fetchWorkExclusive() + { + boolean hasReadyToRun = hasReadyToRun(); + boolean hadWaitingToRun = hasAlreadyWaitingToRun(); + { + int prevPendingUnqueued = pendingCount; + updateAndEnqueuePendingUntilHasWaitingToRun(prevPendingUnqueued / 2); + } + + if (hadWaitingToRun && hasReadyToRun) + { + lock.addAndGetEnabledThreadCount(1); + } + else if (hadWaitingToRun) + { + readyToRunTarget = Math.min(readyToRunLimit, readyToRunTarget + (1+readyToRunTarget)/2); + } + else if (hasReadyToRun && readyToRunTarget > 1) + { + --readyToRunTarget; + } + + boolean hasDrainedSignal = false; + while (true) + { + long state = lock.state(); + int signals = SignalLock.asyncSignalCount(state); + int waiters = SignalLock.waitingEnabledThreadCount(state); + if (signals >= readyToRunTarget) + { + if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; + else if (hasDrainedSignal) + lock.signalLockWorkExclusive(); + return; + } + else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) + { + // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary + lock.propagateAsyncWorkSignals(1); + } + + Task task = pollAlreadyWaitingToRunExclusive(); + if (task == null) + { + if (updateAndEnqueuePendingUntilHasWaitingToRun(0)) + continue; + + lock.clearLockWork(); + hasDrainedSignal = true; + if (!updateAndEnqueuePendingUntilHasWaitingToRun(0)) + { + if (tasks == 0) + notifyQuiescentExclusive(); + return; + } + } + else + { + try { task.preRunExclusive(); } + catch (Throwable t) + { + try { task.fail(t); } + catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } + try { completeTaskExclusive(task); } + catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } + continue; + } + if (DEBUG_EXECUTION) DebugTask.get(task).onPreRun(); + + addReadyToRun(task); + boolean incremented = lock.incrementAsyncWork(false); + Invariants.require(incremented); + } + } + } + + private boolean updatePendingUnqueued() + { + if (!hasUnqueued()) + return false; + + int count = 0; + Task addSequentialHead = null, addSequentialTail = null; + Task addCleanupHead = null, addCleanupTail = null; + Task addNewHead = null, addNewTail = null; + { + Task cur = Task.reverse(acquireUnqueuedExclusive()); + while (cur != null) + { + Task next = cur.next; + if (!cur.isReadyToCleanup()) + { + if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); + else addNewHead = reverseOne(addNewHead, cur); + } + else if (cur instanceof ExclusiveExecutorTask) + { + if (addSequentialHead == null) addSequentialHead = addSequentialTail = setNextNull(cur); + else addSequentialHead = reverseOne(addSequentialHead, cur); + } + else + { + if (addCleanupHead == null) addCleanupHead = addCleanupTail = setNextNull(cur); + else addCleanupHead = reverseOne(addCleanupHead, cur); + } + ++count; + cur = next; + } + } + + pendingCount += count; + if (addSequentialHead != null) + { + if (pendingSequentialHead == null) pendingSequentialHead = addSequentialHead; + else pendingSequentialTail.next = addSequentialHead; + pendingSequentialTail = addSequentialTail; + } + if (addCleanupHead != null) + { + if (pendingCleanupHead == null) pendingCleanupHead = addCleanupHead; + else pendingCleanupTail.next = addCleanupHead; + pendingCleanupTail = addCleanupTail; + } + if (addNewHead != null) + { + if (pendingNewHead == null) pendingNewHead = addNewHead; + else pendingNewTail.next = addNewHead; + pendingNewTail = addNewTail; + } + return true; + } + + private Task reverseOne(Task prev, Task cur) + { + cur.next = prev; + return cur; + } + + private Task setNextNull(Task cur) + { + cur.next = null; + return cur; + } + + private boolean updateAndEnqueuePendingUntilHasWaitingToRun(int processAtLeast) + { + updatePendingUnqueued(); + int count = 0; + while (enqueueOnePending()) + { + if (++count >= processAtLeast && hasAlreadyWaitingToRun()) + return true; + } + return hasAlreadyWaitingToRun(); + } + class LoopTask extends AccordExecutorLoops.LoopTask { final int index; @@ -110,7 +321,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop this.index = index; } - private Task awaitWork() + private Task awaitWork(AccordTaskRunner self) { while (true) { @@ -123,11 +334,11 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop } catch (Throwable t) { - unlock(); + unlock(self); throw t; } - if (!unlockAndAcquire()) + if (!unlockAndAcquire(self)) continue; } @@ -138,11 +349,8 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop } } - private Task cleanupAndMaybeGetWork(@Nullable Task cleanup) + private Task cleanupAndMaybeGetWork(AccordTaskRunner self, @Nullable Task cleanup) { - if (cleanup == null) - return null; - if (lock.tryAcquireAsyncWork()) { if (shutdown) @@ -151,22 +359,21 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop return pushCleanupAndReturn(cleanup, nonNull(pollReadyToRun())); } - if (!tryLock()) + if (!tryLock(self)) return pushCleanupAndReturn(cleanup, null); try { completeTaskExclusive(cleanup); - clearRunning(); fetchWorkExclusive(); } catch (Throwable t) { - unlock(); + unlock(self); throw t; } - if (unlockAndAcquire()) + if (unlockAndAcquire(self)) { if (shutdown) throw new ShutdownException(); @@ -178,225 +385,52 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop private Task pushCleanupAndReturn(Task cleanup, Task result) { - clearRunning(); cleanup.setReadyToCleanup(); if (push(cleanup) == null) lock.signalLockWork(); return result; } - final boolean tryLock() + final boolean tryLock(AccordTaskRunner self) { - return onTryLock(lock.tryLock(index)); + if (self.accordLockedExecutor() != null) + return false; + + return onTryLock(self, lock.tryLock(index)); } - final boolean unlockAndAcquire() + final boolean unlockAndAcquire(AccordTaskRunner self) { - if (Invariants.isParanoid()) paranoidUnlockExclusive(); + self.clearAccordLockedExecutor(); if (DEBUG_EXECUTION) debug.onExitLock(); return lock.unlockAndAcquireAsyncWork(); } - private boolean enqueueOnePending() - { - if (pendingCount == 0) - return false; - - if (pendingSequentialHead != null) - { - pendingSequentialHead = enqueueOneCleanup(pendingSequentialHead); - if (pendingSequentialHead == null) - pendingSequentialTail = null; - } - else if (pendingCleanupHead != null) - { - pendingCleanupHead = enqueueOneCleanup(pendingCleanupHead); - if (pendingCleanupHead == null) - pendingCleanupTail = null; - } - else - { - pendingNewHead = enqueueOneSubmit(pendingNewHead); - if (pendingNewHead == null) - pendingNewTail = null; - } - --pendingCount; - return true; - } - - private void fetchWorkExclusive() - { - boolean hasReadyToRun = hasReadyToRun(); - boolean hadWaitingToRun = hasAlreadyWaitingToRun(); - { - int prevPendingUnqueued = pendingCount; - updateAndEnqueuePendingUntilHasWaitingToRun(prevPendingUnqueued / 2); - } - - if (hadWaitingToRun && hasReadyToRun) - { - lock.addAndGetEnabledThreadCount(1); - } - else if (hadWaitingToRun) - { - readyToRunTarget = Math.min(readyToRunLimit, readyToRunTarget + (1+readyToRunTarget)/2); - } - else if (hasReadyToRun && readyToRunTarget > 1) - { - --readyToRunTarget; - } - - boolean hasDrainedSignal = false; - while (true) - { - long state = lock.state(); - int signals = SignalLock.asyncSignalCount(state); - int waiters = SignalLock.waitingEnabledThreadCount(state); - if (signals >= readyToRunTarget) - { - if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; - else if (hasDrainedSignal) - lock.signalLockWorkExclusive(); - return; - } - else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) - { - // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary - lock.propagateAsyncWorkSignals(1); - } - - Task task = pollAlreadyWaitingToRunExclusive(); - if (task == null) - { - if (updateAndEnqueuePendingUntilHasWaitingToRun(0)) - continue; - - lock.clearLockWork(); - hasDrainedSignal = true; - if (!updateAndEnqueuePendingUntilHasWaitingToRun(0)) - { - if (tasks == 0) - notifyQuiescentExclusive(); - return; - } - } - else - { - try { task.preRunExclusive(); } - catch (Throwable t) - { - try { task.fail(t); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - try { completeTaskExclusive(task); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - continue; - } - if (DEBUG_EXECUTION) DebugTask.get(task).onPreRun(); - - addReadyToRun(task); - boolean incremented = lock.incrementAsyncWork(false); - Invariants.require(incremented); - } - } - } - - private boolean updatePendingUnqueued() - { - if (!hasUnqueued()) - return false; - - int count = 0; - Task addSequentialHead = null, addSequentialTail = null; - Task addCleanupHead = null, addCleanupTail = null; - Task addNewHead = null, addNewTail = null; - { - Task cur = Task.reverse(acquireUnqueuedExclusive()); - while (cur != null) - { - Task next = cur.next; - if (!cur.isReadyToCleanup()) - { - if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); - else addNewHead = reverseOne(addNewHead, cur); - } - else if (cur instanceof SequentialQueueTask) - { - if (addSequentialHead == null) addSequentialHead = addSequentialTail = setNextNull(cur); - else addSequentialHead = reverseOne(addSequentialHead, cur); - } - else - { - if (addCleanupHead == null) addCleanupHead = addCleanupTail = setNextNull(cur); - else addCleanupHead = reverseOne(addCleanupHead, cur); - } - ++count; - cur = next; - } - } - - pendingCount += count; - if (addSequentialHead != null) - { - if (pendingSequentialHead == null) pendingSequentialHead = addSequentialHead; - else pendingSequentialTail.next = addSequentialHead; - pendingSequentialTail = addSequentialTail; - } - if (addCleanupHead != null) - { - if (pendingCleanupHead == null) pendingCleanupHead = addCleanupHead; - else pendingCleanupTail.next = addCleanupHead; - pendingCleanupTail = addCleanupTail; - } - if (addNewHead != null) - { - if (pendingNewHead == null) pendingNewHead = addNewHead; - else pendingNewTail.next = addNewHead; - pendingNewTail = addNewTail; - } - return true; - } - - private Task reverseOne(Task prev, Task cur) - { - cur.next = prev; - return cur; - } - - private Task setNextNull(Task cur) - { - cur.next = null; - return cur; - } - - private boolean updateAndEnqueuePendingUntilHasWaitingToRun(int processAtLeast) - { - updatePendingUnqueued(); - int count = 0; - while (enqueueOnePending()) - { - if (++count >= processAtLeast && hasAlreadyWaitingToRun()) - return true; - } - return hasAlreadyWaitingToRun(); - } - @Override public void run() { - lock.register(index, Thread.currentThread()); + Thread thread = Thread.currentThread(); + AccordTaskRunner self = AccordTaskRunner.get(thread); + self.setAccordActiveExecutor(AccordExecutorSignalLoop.this); + setWrapped(self); + + lock.register(index, thread); Task task = null; while (true) { try { - try { task = cleanupAndMaybeGetWork(task); } - catch (Throwable t) { task = null; throw t; } + if (task != null) + { + try { task = cleanupAndMaybeGetWork(self, task); } + catch (Throwable t) { task = null; throw t; } + } if (task == null) - task = awaitWork(); + task = awaitWork(self); try { - setRunning(task); + self.setAccordActiveTask(task); task.run(); } catch (Throwable t) @@ -421,6 +455,10 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop { agent.onException(t); } + finally + { + self.setAccordActiveTask(null); + } } } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java index ab6c8c25e7..ee37744d8d 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java @@ -128,6 +128,18 @@ class AccordExecutorSimple extends AccordExecutor } } + final boolean hasWaitingToRun() + { + updateWaitingToRunExclusive(); + return hasAlreadyWaitingToRun(); + } + + final Task pollWaitingToRunExclusive() + { + updateWaitingToRunExclusive(); + return pollAlreadyWaitingToRunExclusive(); + } + @Override boolean isOwningThread() { diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java index 0d852ccde3..e4efac3ad4 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java @@ -91,14 +91,15 @@ public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) { - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + lock(self); try { submitExternalExclusive(sync, p1s, p2, p3, p4); } finally { - unlock(); + unlock(self); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index 45aeaa2f18..e268113b86 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -397,7 +397,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements public AccordFetchCoordinator(Node node, Ranges ranges, SyncPoint syncPoint, DataStore.FetchRanges fetchRanges, CommandStore commandStore) throws TopologyException { - super(node, node.someSequentialExecutor(), ranges, syncPoint, fetchRanges, commandStore); + super(node, node.someExclusiveExecutor(), ranges, syncPoint, fetchRanges, commandStore); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java index 6a9817bc97..d0399e1b92 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java @@ -64,7 +64,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore KeyBarriers.await(node, node.someSequentialExecutor(), found, syncLocal, syncRemote)) + .flatMap(found -> KeyBarriers.await(node, node.someExclusiveExecutor(), found, syncLocal, syncRemote)) .flatMap(success -> { if (success) return null; diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/AccordTask.java index fe2889e2a6..90781c740f 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTask.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTask.java @@ -71,7 +71,6 @@ import org.apache.cassandra.metrics.LogLinearDecayingHistograms; import org.apache.cassandra.service.accord.AccordCacheEntry.Status; import org.apache.cassandra.service.accord.AccordCommandStore.Caches; import org.apache.cassandra.service.accord.AccordExecutor.Task; -import org.apache.cassandra.service.accord.AccordExecutor.TaskQueue; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; @@ -87,18 +86,17 @@ import static accord.local.LoadKeysFor.WRITE; import static accord.primitives.Routable.Domain.Key; import static accord.primitives.Txn.Kind.EphemeralRead; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; -import static org.apache.cassandra.service.accord.AccordTask.State.ASSIGNED; -import static org.apache.cassandra.service.accord.AccordTask.State.CANCELLED; -import static org.apache.cassandra.service.accord.AccordTask.State.FAILED; -import static org.apache.cassandra.service.accord.AccordTask.State.FINISHED; -import static org.apache.cassandra.service.accord.AccordTask.State.INITIALIZED; -import static org.apache.cassandra.service.accord.AccordTask.State.LOADING; -import static org.apache.cassandra.service.accord.AccordTask.State.PERSISTING; -import static org.apache.cassandra.service.accord.AccordTask.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordTask.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_SCAN_RANGES; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.ASSIGNED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FAILED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FINISHED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.PERSISTING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_SCAN_RANGES; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask.SANITY_CHECK; @@ -111,9 +109,9 @@ public abstract class AccordTask extends Task implements Function function; - public ForFunction(AccordCommandStore commandStore, ExecutionContext loadCtx, Function function) + public ForFunction(AccordCommandStore commandStore, ExecutionContext context, Function function) { - super(commandStore, loadCtx); + super(commandStore, context); this.function = function; } @@ -129,9 +127,9 @@ public abstract class AccordTask extends Task implements Function consumer; - private ForConsumer(AccordCommandStore commandStore, ExecutionContext loadCtx, Consumer consumer) + private ForConsumer(AccordCommandStore commandStore, ExecutionContext context, Consumer consumer) { - super(commandStore, loadCtx); + super(commandStore, context); this.consumer = consumer; } @@ -143,63 +141,16 @@ public abstract class AccordTask extends Task implements Function AccordTask create(CommandStore commandStore, ExecutionContext ctx, Function function) + public static AccordTask create(CommandStore commandStore, ExecutionContext context, Function function) { - return new ForFunction<>((AccordCommandStore) commandStore, ctx, function); + return new ForFunction<>((AccordCommandStore) commandStore, context, function); } - public static AccordTask create(CommandStore commandStore, ExecutionContext ctx, Consumer consumer) + public static AccordTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) { - return new ForConsumer((AccordCommandStore) commandStore, ctx, consumer); + return new ForConsumer((AccordCommandStore) commandStore, context, consumer); } - public enum State - { - INITIALIZED(), - WAITING_TO_SCAN_RANGES(INITIALIZED), - SCANNING_RANGES(WAITING_TO_SCAN_RANGES), - WAITING_TO_LOAD(INITIALIZED, SCANNING_RANGES), - LOADING(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD), - WAITING_TO_RUN(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD, LOADING), - ASSIGNED(WAITING_TO_RUN), - RUNNING(ASSIGNED), - PERSISTING(RUNNING), - FINISHED(RUNNING, PERSISTING), - CANCELLED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED), - FAILED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED, RUNNING, PERSISTING); - - private final int permittedFrom; - - State() - { - this.permittedFrom = 0; - } - - State(State ... permittedFroms) - { - int permittedFrom = 0; - for (State state : permittedFroms) - permittedFrom |= 1 << state.ordinal(); - this.permittedFrom = permittedFrom; - } - - boolean isPermittedFrom(State prev) - { - return (permittedFrom & (1 << prev.ordinal())) != 0; - } - - boolean isExecuted() - { - return this.compareTo(PERSISTING) >= 0; - } - - boolean isComplete() - { - return this.compareTo(FINISHED) >= 0; - } - } - - private State state = INITIALIZED; final AccordCommandStore commandStore; private final ExecutionContext executionContext; private volatile String loggingId; @@ -214,14 +165,13 @@ public abstract class AccordTask extends Task implements Function> waitingToLoad; @Nullable RangeTxnScanner rangeScanner; - boolean hasRanges; @Nullable CommandSummaries commandsForRanges; - @Nullable private TaskQueue queued; private BiConsumer callback; public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext) { + super(executionContext); this.commandStore = commandStore; this.executionContext = executionContext; this.loggingId = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); @@ -250,13 +200,13 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function no loading or waiting; found %s", this, AccordTask::toDescription); - } - } - Unseekables keys() { return executionContext.keys(); @@ -334,20 +273,6 @@ public abstract class AccordTask extends Task implements Function priorityChain() - { - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - preSetup(callback); - commandStore.executor().submitPriority(AccordTask.this); - return AccordTask.this; - } - }; - } - private void preSetup(BiConsumer callback) { Invariants.require(this.callback == null); @@ -358,7 +283,7 @@ public abstract class AccordTask extends Task implements Function parent) { - this.queuePosition = parent.queuePosition; + this.position = parent.position; // note we use the caches "unsafely" here deliberately, as we only reference commands we already have references to // so we do not mutate anything, except the atomic counter of references if (parent.commands != null) @@ -393,8 +318,8 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function extends Task implements Function extends Task implements Function extends Task implements Function> ensureWaitingToLoad() { - Invariants.require(state.compareTo(WAITING_TO_LOAD) <= 0, "Expected status to be on or before WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); + Invariants.require(compareTo(WAITING_TO_LOAD) <= 0, "Expected status to be on or before WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); if (waitingToLoad == null) waitingToLoad = new ArrayDeque<>(); return waitingToLoad; @@ -610,7 +534,7 @@ public abstract class AccordTask extends Task implements Function pollWaitingToLoad() { - Invariants.require(state == State.WAITING_TO_LOAD, "Expected status to be WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); + Invariants.require(is(WAITING_TO_LOAD), "Expected status to be WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); if (waitingToLoad == null) return null; @@ -658,8 +582,7 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function extends Task implements Function finish(result, null); safeStore.persistFieldUpdatesInternal(changes == null ? onFlush : null); if (changes != null) save(changes, onFlush); @@ -735,12 +657,12 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function extends Task implements Function extends Task implements Function extends Task implements Function extends Task implements Function v.revert()); } - protected void addToQueue(TaskQueue queue) - { - if (state == CANCELLED) - return; - - Invariants.require(queue.kind == state || (queue.kind == State.WAITING_TO_LOAD && state == WAITING_TO_SCAN_RANGES), "Invalid queue type: %s vs %s", queue.kind, this, AccordTask::toDescription); - Invariants.require(this.queued == null, "Already queued with state: %s", this, AccordTask::toDescription); - queued = queue; - queue.append(this); - } - - @Nullable - TaskQueue queued() - { - return queued; - } - - TaskQueue unqueue() - { - TaskQueue wasQueued = queued; - queued.remove(this); - queued = null; - return wasQueued; - } - - TaskQueue unqueueIfQueued() - { - if (queued == null) - return null; - return unqueue(); - } - public class RangeTxnAndKeyScanner extends RangeTxnScanner { class KeyWatcher implements AccordCache.Listener @@ -965,7 +851,6 @@ public abstract class AccordTask extends Task implements Function intersectingKeys = new ObjectHashSet<>(); final KeyWatcher keyWatcher = new KeyWatcher(); final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); @@ -1102,9 +987,9 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function= 1 && increment <= MAX_SIGNAL_COUNT); while (true) { long cur = state; diff --git a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java index de181ef688..e1a50740ef 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java +++ b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java @@ -401,10 +401,13 @@ public class ActionSchedule implements CloseableIterator, LongConsumer advance(scheduled.poll()); } - Action perform = runnable.poll(); + Action perform = runnableByDeadline.peek(); if (perform == null) throw new NoSuchElementException(); + if (perform.deadline() < now - currentJitter) runnable.remove(perform); + else perform = runnable.poll(); + if (!runnableByDeadline.remove(perform) && perform.deadline() > 0) throw new IllegalStateException(); time.tick(perform.deadline()); diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index eadb7a4f55..52a2b85409 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -21,6 +21,8 @@ package org.apache.cassandra.simulator.test; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -28,14 +30,19 @@ import java.util.concurrent.Future; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.LockSupport; import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +import javax.annotation.Nullable; import org.junit.Test; import accord.api.AsyncExecutor; -import accord.local.ExclusiveAsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; +import accord.utils.async.Cancellable; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; @@ -44,37 +51,144 @@ import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.SignalLock; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; -import static org.apache.cassandra.service.accord.AccordService.toFuture; // TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit public class AccordExecutorTest extends SimulationTestBase { - static final int EXECUTOR_THREAD_COUNT = 44; - @Test public void signalLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, -1, -1, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new AccordAgent()), 16); } @Test public void signalSpinLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), 16); } @Test public void ayncSubmitTest() { - executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, i -> "Loop" + i, new AccordAgent()), + int threads = 32; + executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, threads, i -> "Loop" + i, new AccordAgent()), 16); } + static class Submitted + { + final AtomicInteger nextId = new AtomicInteger(); + final AtomicInteger doneBefore = new AtomicInteger(); + final ConcurrentHashMap>> consequences = new ConcurrentHashMap<>(); + + boolean isDone() + { + return isDoneBetween(0, nextId.get()); + } + + boolean isDoneBefore(int before) + { + if (!isDoneBetween(doneBefore.get(), before)) + return false; + doneBefore.accumulateAndGet(before, Integer::max); + return true; + } + + boolean isDoneBetween(int from, int before) + { + for (int id = from ; id < before ; ++id) + { + for (Future future : consequences.get(id)) + { + if (!future.isDone()) + return false; + } + } + return true; + } + + Collection> start() + { + ConcurrentLinkedQueue> result = new ConcurrentLinkedQueue<>(); + + while (true) + { + int id = nextId.get(); + Object prev = consequences.putIfAbsent(id, result); + nextId.compareAndSet(id, id + 1); + if (prev == null) + return result; + } + } + } + + static class Control extends ConcurrentLinkedQueue + { + final Submitted submitted; + final AtomicInteger count = new AtomicInteger(); + final float cancelChance; + float processChance; + + Control(float cancelChance, Submitted submitted) + { + this(submitted, cancelChance, ThreadLocalRandom.current().nextFloat() * 0.5f); + } + + Control(Submitted submitted, float cancelChance, float processChance) + { + this.submitted = submitted; + this.cancelChance = cancelChance; + this.processChance = processChance; + } + + void submit(AsyncExecutor executor, Collection> consequences, Consumer>> run) + { + AsyncPromise future = new AsyncPromise<>(); + Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { + if (fail == null) future.trySuccess(null); + else + { + future.tryFailure(fail); + if (fail instanceof CancellationException) + run.accept(submitted.start()); + } + }); + consequences.add(future); + + if (cancel != null && ThreadLocalRandom.current().nextFloat() <= cancelChance) + { + add(cancel); + count.incrementAndGet(); + } + + if (count.get() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance) + { + int cancelCount = 0; + do + { + ++cancelCount; + + float delta = ThreadLocalRandom.current().nextFloat() - 0.5f; + if (delta < 0) processChance /= delta; + else processChance *= -delta; + if (processChance < 0.001f || processChance > 0.999f) + processChance = cancelChance; + } while (count.decrementAndGet() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance); + + // do outside of loop to avoid reentry + while (cancelCount-- > 0) + remove().cancel(); + } + } + } + public void executorTest(SerializableSupplier supplier, int submissionThreads) { simulate(arr(() -> { @@ -84,30 +198,38 @@ public class AccordExecutorTest extends SimulationTestBase ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); AccordExecutor executor = supplier.get(); Lock lock = executor.unsafeLock(); - ExclusiveAsyncExecutor sequentialExecutor = executor.newSequentialExecutor(); + ExclusiveAsyncExecutor sequentialExecutor = executor.newExclusiveExecutor(); Executor lockExecutor = executorFactory().sequential("lock"); for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) { for (float lockChance : new float[] { 0f, 0.01f, 0.1f }) { - List> done = new ArrayList<>(); - for (int i = 0 ; i < submissionThreads ; ++i) + for (float cancelChance : new float[] { 0f, 0.01f, 0.1f }) { - int id = i; - done.add(submit.submit(() -> { - try - { - submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance); - } - catch (ExecutionException | InterruptedException e) - { - throw new RuntimeException(e); - } - })); + System.out.println(String.format("sleepChance %.2f, lockChance %.2f, cancelChance %.2f", sleepChance, lockChance, cancelChance)); + List> done = new ArrayList<>(); + Submitted submitted = new Submitted(); + for (int i = 0; i < submissionThreads; ++i) + { + int id = i; + done.add(submit.submit(() -> { + try + { + submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance, new Control(cancelChance, submitted), submitted); + } + catch (ExecutionException | InterruptedException e) + { + throw new RuntimeException(e); + } + })); + } + for (Future f : done) + f.get(); + + if (!submitted.isDone()) + throw new AssertionError(); } - for (Future f : done) - f.get(); } } } @@ -119,27 +241,34 @@ public class AccordExecutorTest extends SimulationTestBase () -> {}, 1L); } - private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance) throws ExecutionException, InterruptedException + private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance, Control control, Submitted submitted) throws ExecutionException, InterruptedException { - ConcurrentLinkedQueue> await = new ConcurrentLinkedQueue<>(); + ConcurrentLinkedQueue> awaitConsequences = new ConcurrentLinkedQueue<>(); while (outerLoop-- > 0) { + List>> allAwaitSubmitted = new ArrayList<>(); for (int i = 0; i < innerLoop; ++i) - submitRecursive(lock, executor, sequentialExecutor, 1 + i, await, sleepChance, lockChance); + { + Collection> awaitSubmitted = submitted.start(); + allAwaitSubmitted.add(awaitSubmitted); + submitRecursive(lock, executor, sequentialExecutor, 1 + i, awaitSubmitted, awaitConsequences, submitted, sleepChance, lockChance, control); + } AtomicBoolean done = new AtomicBoolean(); submitUntil(lock, lockExecutor, sleepChance, done::get); - while (!await.isEmpty()) - await.poll().get(); + for (Collection> awaitSubmitted : allAwaitSubmitted) + await(awaitSubmitted, CancellationException.class); + await(awaitConsequences, null); done.set(true); System.out.println("Loop " + id + '.' + (1 + outerLoop)); } } - private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> await, float sleepChance, float lockChance) + private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) { AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; - await.add(toFuture(submitTo.chain(() -> { + + control.submit(submitTo, consequences, nextConsequences -> { ThreadLocalRandom rnd = ThreadLocalRandom.current(); boolean locked = false; if (rnd.nextFloat() < lockChance) @@ -147,10 +276,21 @@ public class AccordExecutorTest extends SimulationTestBase if (rnd.nextBoolean()) locked = lock.tryLock(); else { locked = true; lock.lock(); } } + if (ThreadLocalRandom.current().nextFloat() < 0.01f) + { + int expectDoneBefore = submitted.nextId.get(); + AsyncPromise afterConsequences = new AsyncPromise<>(); + executor.afterSubmittedAndConsequences(() -> { + if (!submitted.isDoneBefore(expectDoneBefore)) + throw new AssertionError(); + afterConsequences.setSuccess(null); + }); + awaitConsequences.add(afterConsequences); + } try { if (count > 1) - submitRecursive(lock, executor, sequentialExecutor, count -1, await, sleepChance, lockChance); + submitRecursive(lock, executor, sequentialExecutor, count -1, nextConsequences, awaitConsequences, submitted, sleepChance, lockChance, control); if (rnd.nextFloat() < sleepChance) LockSupport.parkNanos(rnd.nextInt(10000, 100000)); } @@ -159,7 +299,7 @@ public class AccordExecutorTest extends SimulationTestBase if (locked) lock.unlock(); } - }).beginAsResult())); + }); } private static void submitUntil(Lock lock, Executor executor, float sleepChance, BooleanSupplier done) @@ -188,4 +328,17 @@ public class AccordExecutorTest extends SimulationTestBase } }); } + + private static void await(Collection> await, @Nullable Class ignore) throws InterruptedException, ExecutionException + { + for (Future future : await) + { + try { future.get(); } + catch (ExecutionException e) + { + if (ignore == null || !(ignore.isInstance(e.getCause()))) + throw e; + } + } + } } diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index a9a5c6db58..5d6408f676 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -90,9 +90,9 @@ import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.utils.FBUtilities; +import static accord.local.ExecutionContext.contextFor; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.ExecutionContext.contextFor; import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE; import static accord.primitives.Routable.Domain.Range; import static accord.primitives.Timestamp.Flag.HLC_BOUND; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index dcd3486852..3509c2e506 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -27,9 +27,9 @@ import org.junit.Test; import accord.api.Key; import accord.api.RoutingKey; import accord.local.Command; +import accord.local.ExecutionContext; import accord.local.LoadKeys; import accord.local.Node; -import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java index ede89a4d77..54dee77c04 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java @@ -85,9 +85,9 @@ import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Condition; +import static accord.local.ExecutionContext.contextFor; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.ExecutionContext.contextFor; import static accord.utils.Property.qt; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; import static org.apache.cassandra.service.accord.AccordService.getBlocking; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index fc4fdd4367..ae43676851 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -37,6 +37,7 @@ import org.junit.Assert; import accord.api.AsyncExecutor; import accord.api.Data; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.ProgressLog.NoOpProgressLog; import accord.api.RemoteListeners.NoOpRemoteListeners; @@ -56,7 +57,6 @@ import accord.local.Node; import accord.local.Node.Id; import accord.local.NodeCommandStoreService; import accord.local.SafeCommandStore; -import accord.local.ExclusiveAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.durability.DurabilityService; @@ -132,7 +132,7 @@ public class AccordTestUtils public static class Commands { - public static Command notDefined(TxnId txnId, PartialTxn txn) + public static Command notDefined(TxnId txnId) { return Command.NotDefined.notDefined(txnId, NotDefined, NotDurable, StoreParticipants.empty(txnId), Ballot.ZERO); } @@ -371,7 +371,7 @@ public class AccordTestUtils } @Override - public ExclusiveAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 228e2e564a..00705915dd 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -35,6 +35,7 @@ import javax.annotation.Nullable; import org.assertj.core.api.Assertions; import accord.api.AsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.LocalListeners; import accord.api.ProgressLog; @@ -59,7 +60,6 @@ import accord.local.Node; import accord.local.NodeCommandStoreService; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.ExclusiveAsyncExecutor; import accord.local.TimeService; import accord.local.durability.DurabilityService; import accord.messages.BeginRecovery; @@ -182,7 +182,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable } @Override - public ExclusiveAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 5130dce553..9975fd6b2b 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -49,6 +49,7 @@ import org.junit.Test; import accord.api.Agent; import accord.api.AsyncExecutor; import accord.api.DataStore; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.Key; import accord.api.OwnershipEventListener; @@ -72,7 +73,6 @@ import accord.local.NodeCommandStoreService; import accord.local.RedundantBefore; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.ExclusiveAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.cfk.CommandsForKey; @@ -718,7 +718,7 @@ public class CommandsForKeySerializerTest @Override public NodeCommandStoreService node() { return new NodeCommandStoreService() { @Override public AsyncExecutor someExecutor() { throw new UnsupportedOperationException(); } - @Override public ExclusiveAsyncExecutor someSequentialExecutor() { throw new UnsupportedOperationException(); } + @Override public ExclusiveAsyncExecutor someExclusiveExecutor() { throw new UnsupportedOperationException(); } @Override public long epoch() { return 0;} @Override public Node.Id id() { return Node.Id.NONE; } @Override public Timeouts timeouts() { return null; } diff --git a/test/unit/org/apache/cassandra/utils/AccordGenerators.java b/test/unit/org/apache/cassandra/utils/AccordGenerators.java index 0e7d7a3685..a05b644b53 100644 --- a/test/unit/org/apache/cassandra/utils/AccordGenerators.java +++ b/test/unit/org/apache/cassandra/utils/AccordGenerators.java @@ -175,7 +175,7 @@ public class AccordGenerators switch (targetType) { case notDefined: - return AccordTestUtils.Commands.notDefined(id, txn); + return AccordTestUtils.Commands.notDefined(id); case preaccepted: return AccordTestUtils.Commands.preaccepted(id, txn, executeAt); case committed: From 37b4fbf08fc21ff849774b89dd5bc978fec1bd81 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Wed, 15 Jul 2026 22:50:50 +0100 Subject: [PATCH 04/10] Accord Executor QoS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fairness: tasks are grouped by kind of work, and for each group we track work arrival and service via decaying counters. If work for some group(s) begin to be served at a much lower rate than some other group (e.g. because that group has a large backlog, that by simple priority comes first), then we begin processing some fraction of the work by service-ratio rather than priority. This means that we cannot starve request processing because, e.g., a sync point has produced thousands of state save tasks. Incremental processing: tasks that process many keys may be declared INCR or ASYNC. ASYNC indicates the keys are needed (so should be loaded), but may be used in follow-up work so the task may be executed if the data is not yet loaded. An ASYNC task may be followed by an INCR task that processes keys as they are loaded, with limits to the batch sizes. This limits the latency impact of processing larger transactions such as sync points. INCR tasks may be declared to be processed in either an arbitrary order with no isolation, or “atomically” with its parent task. In the latter case, the complete unit of work appears to execute as a single execution to external observers (blocking progress on unprocessed keys). Key-level ordering: AccordCacheEntry can inflate a special Queue object that can be used to impose ordering constraints: FIFO for atomic work (once it has been part processed, or submitted as part of a parent task); prioritised for tasks that have a key-level priority to impose (i.e. PreAccept/Accept/Commit/Stable/Apply want to be ordered by TxnId); and unsequenced tasks for those that have no sequencing restrictions. Additional improvements: - AccordCacheEntry are now explicitly LOCKED for the duration of their usage, which may span multiple executions for long running INCR tasks. - ExclusiveExecutor releases its ownership lock immediately, since AccordCacheEntry locks guarantee safety (as they will not be released until the task is cleaned up) - CassandraThread tracks active and locked AccordExecutor, so that we may safely and more aggressively use executeMaybeImmediately, safe in the knowledge it will never permit cross-executor executions or multiple executor locks to be held. Also: - Rename PreLoadContext -> ExecutionContext - Break out AccordExecutor/AccordTask into separate package --- modules/accord | 2 +- .../cassandra/concurrent/CassandraThread.java | 14 +- .../apache/cassandra/config/AccordConfig.java | 47 +- .../db/virtual/AccordDebugKeyspace.java | 4 +- .../cassandra/service/accord/AccordCache.java | 240 +-- .../service/accord/AccordCacheEntry.java | 1149 ++++++++++++-- .../service/accord/AccordCommandStore.java | 39 +- .../service/accord/AccordCommandStores.java | 34 +- .../service/accord/AccordExecutor.java | 1207 +++++++-------- .../AccordExecutorAbstractLockLoop.java | 19 +- .../accord/AccordExecutorAbstractLoop.java | 21 +- .../accord/AccordExecutorSignalLoop.java | 119 +- .../service/accord/AccordExecutorSimple.java | 22 +- .../service/accord/AccordSafeCommand.java | 91 +- .../accord/AccordSafeCommandStore.java | 140 +- .../accord/AccordSafeCommandsForKey.java | 86 +- .../service/accord/AccordSafeState.java | 42 +- .../service/accord/AccordService.java | 1 + .../cassandra/service/accord/AccordTask.java | 1348 +++++++++++------ .../service/accord/InMemoryRangeIndex.java | 3 +- .../cassandra/service/accord/RangeIndex.java | 2 +- .../accord/debug/DebugBlockedTxns.java | 6 +- .../service/accord/debug/DebugExecution.java | 54 +- .../service/accord/debug/DebugTxnGraph.java | 6 +- .../accord/journal/JournalRangeIndex.java | 8 +- .../test/accord/AccordDropTableBase.java | 12 +- .../test/accord/AccordLoadTest.java | 16 +- .../AccordJournalConsistentExpungeTest.java | 3 +- .../test/AccordExecutorAndCacheTest.java | 329 ++++ .../CompactionAccordIteratorsTest.java | 13 +- .../service/accord/AccordCacheEntryTest.java | 14 +- .../service/accord/AccordCacheTest.java | 86 +- .../accord/AccordCommandStoreTest.java | 17 +- .../service/accord/AccordCommandTest.java | 2 +- .../service/accord/AccordTaskTest.java | 13 +- .../service/accord/AccordTestUtils.java | 21 +- ...SimpleSimulatedAccordCommandStoreTest.java | 2 +- .../accord/SimulatedAccordTaskTest.java | 26 +- .../CommandsForKeySerializerTest.java | 32 +- 39 files changed, 3317 insertions(+), 1973 deletions(-) create mode 100644 test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java diff --git a/modules/accord b/modules/accord index fff32de2e9..84ac4db032 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit fff32de2e915772fbc70d16b1c32346313877838 +Subproject commit 84ac4db03251c3b842c98a29868e072792662f51 diff --git a/src/java/org/apache/cassandra/concurrent/CassandraThread.java b/src/java/org/apache/cassandra/concurrent/CassandraThread.java index d66adc2442..b013a5f0ad 100644 --- a/src/java/org/apache/cassandra/concurrent/CassandraThread.java +++ b/src/java/org/apache/cassandra/concurrent/CassandraThread.java @@ -31,6 +31,7 @@ public class CassandraThread extends FastThreadLocalThread implements AccordExec private ExecutorLocals executorLocals; private AccordExecutor accordActiveExecutor; private AccordExecutor accordLockedExecutor; + private int accordLockedExecutorDepth; private volatile AccordExecutor.Task accordActiveTask; private static final AtomicReferenceFieldUpdater accordActiveTaskUpdater = AtomicReferenceFieldUpdater.newUpdater(CassandraThread.class, AccordExecutor.Task.class, "accordActiveTask"); @@ -113,18 +114,19 @@ public class CassandraThread extends FastThreadLocalThread implements AccordExec } @Override - public final boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor) + public final boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) { - if (accordLockedExecutor != null) - return false; - accordLockedExecutor = newLockedExecutor; + if (accordLockedExecutor == null) accordLockedExecutor = newLockedExecutor; + else if (accordLockedExecutor != newLockedExecutor) return false; + ++accordLockedExecutorDepth; return true; } @Override - public final void clearAccordLockedExecutor() + public final void exitAccordLockedExecutor() { - accordLockedExecutor = null; + if (--accordLockedExecutorDepth == 0) + accordLockedExecutor = null; } public final AccordExecutor.Task accordActiveTask() diff --git a/src/java/org/apache/cassandra/config/AccordConfig.java b/src/java/org/apache/cassandra/config/AccordConfig.java index fe226fe92f..9813f3ffa0 100644 --- a/src/java/org/apache/cassandra/config/AccordConfig.java +++ b/src/java/org/apache/cassandra/config/AccordConfig.java @@ -156,31 +156,12 @@ public class AccordConfig */ PHASE_ONLY, - /** - * Always pick the task by priority. - */ - PRIORITY_BUDGET, - - /** - * Pick by phase first, so the highest phase with budget ALWAYS runs. - * Within a phase, pick by priority. - */ - PHASE_BUDGET, - /** * Pick by phase first, selecting the phase that has processed the least work recently relative to arrivals. * Within a phase, pick by priority. */ PHASE_FAIR, - /** - * Pick by phase first, selecting the phase that has processed the least work recently relative to arrivals. - * However, each phase has a budget that is consumed on dispatch, and we pick only from those phases with budget. - * If there is no phase with budget, the budget resets. - * Within a phase, pick by priority. - */ - PHASE_BUDGET_FAIR, - /** * While phases are within a threshold of imbalance, pick tasks by priority. * Once the threshold is crossed, over-processed phases have a small penalty applied @@ -190,8 +171,6 @@ public class AccordConfig * Within a phase, pick by priority. */ BLENDED_PRIORITY_PHASE_FAIR, - - BLENDED_PRIORITY_PHASE_BUDGET_FAIR, } public QueueShardModel queue_shard_model = THREAD_POOL_PER_SHARD; @@ -212,8 +191,29 @@ public class AccordConfig public Integer queue_flow_imbalance_onset = null; public Integer queue_flow_imbalance_width_shift = null; + public String queue_active_limits; - public String queue_budgets; + + public Boolean queue_nonsync_enabled; + + /** + * Size at which we will begin processing a task that is ASYNC, INCR OR INCR_ATOMIC. + * Note that a size of zero will effectively give implicit priority to INCR_ATOMIC tasks, as they may immediately + * take a FIFO queue slot (which is processed preferentially). + */ + public Integer queue_nonsync_min_batch_size; + + /** + * If there are more than min_batch_size keys ready for an ASYNC, INCR or INCR_ATOMIC task, + * process up to this many keys at once. + */ + public Integer queue_nonsync_max_batch_size; + + /** + * An ASYNC, INCR or INCR_ATOMIC task that is ready to run but waiting for batch_size work will proceed + * once this number of tasks are blocked behind it, regardless of batch_size. + */ + public Integer queue_nonsync_blocked_limit; /** * If set, the signal loop does not match park/unpark pairs, but instead consumers perform timed-park spin waits @@ -240,9 +240,6 @@ public class AccordConfig */ public volatile OptionaldPositiveInt command_store_shard_count = OptionaldPositiveInt.UNDEFINED; - public volatile OptionaldPositiveInt max_queued_loads = OptionaldPositiveInt.UNDEFINED; - public volatile OptionaldPositiveInt max_queued_range_loads = OptionaldPositiveInt.UNDEFINED; - public volatile OptionaldPositiveInt progress_log_concurrency = OptionaldPositiveInt.UNDEFINED; public DurationSpec.IntMillisecondsBound progress_log_query_fallback_timeout = new DurationSpec.IntMillisecondsBound("1m"); diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 86238688f5..3a8c6e32bd 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -1650,7 +1650,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace { try (AccordCommandStore.ExclusiveCaches caches = commandStore.lockCaches()) { - AccordCacheEntry entry = caches.commands().getUnsafe(txnId); + AccordCacheEntry entry = caches.commands().getUnsafe(txnId); return entry == null ? null : entry.getExclusive(); } } @@ -1895,7 +1895,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace AccordService.getBlocking(accord.node() .commandStores() .forId(commandStoreId) - .chain(ExecutionContext.contextFor(txnId, TXN_OPS), apply) + .chain(ExecutionContext.unsequenced(txnId, TXN_OPS), apply) .flatMap(i -> i)); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCache.java b/src/java/org/apache/cassandra/service/accord/AccordCache.java index 0d679b2f51..a75e6a6e27 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCache.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCache.java @@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory; import accord.api.RoutingKey; import accord.local.Command; +import accord.local.SafeState; import accord.local.cfk.CommandsForKey; import accord.local.cfk.Serialize; import accord.primitives.Routable; @@ -66,6 +67,7 @@ import org.apache.cassandra.metrics.LogLinearHistogram; import org.apache.cassandra.metrics.ShardedHitRate; import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.AccordCacheEntry.Loading; import org.apache.cassandra.service.accord.AccordCacheEntry.Status; import org.apache.cassandra.service.accord.AccordSafeCommandsForKey.CommandsForKeyCacheEntry; import org.apache.cassandra.service.accord.events.CacheEvents; @@ -79,6 +81,8 @@ import org.apache.cassandra.utils.ObjectSizes; import static accord.utils.Invariants.illegalState; import static accord.utils.Invariants.require; +import static org.apache.cassandra.service.accord.AccordCacheEntry.AGE_MASK; +import static org.apache.cassandra.service.accord.AccordCacheEntry.GENERATION_MASK; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED; @@ -106,7 +110,7 @@ public class AccordCache implements CacheSize VALIDATE_LOAD_ON_EVICT = value; } - public interface Adapter + public interface Adapter & AccordSafeState> { enum Shrink { EVICT, DONE, PERFORM_WITHOUT_LOCK } @@ -122,10 +126,10 @@ public class AccordCache implements CacheSize long estimateHeapSize(V value); long estimateShrunkHeapSize(Object shrunk); boolean validate(AccordCommandStore commandStore, K key, V value); - S safeRef(AccordCacheEntry node); + S safeRef(AccordCacheEntry node); default Comparator keyComparator() { return null; } - default AccordCacheEntry newEntry(K key, AccordCache.Type.Instance owner) + default AccordCacheEntry newEntry(K key, AccordCache.Type.Instance owner) { return AccordCacheEntry.createReadyToLoad(key, owner); } @@ -151,8 +155,8 @@ public class AccordCache implements CacheSize private final List> types = new CopyOnWriteArrayList<>(); final AccordCacheEntry.SaveExecutor saveExecutor; - private final IntrusiveLinkedList> evictQueue = new IntrusiveLinkedList<>(); - private final IntrusiveLinkedList> noEvictQueue = new IntrusiveLinkedList<>(); + private final IntrusiveLinkedList> evictQueue = new IntrusiveLinkedList<>(); + private final IntrusiveLinkedList> noEvictQueue = new IntrusiveLinkedList<>(); private long unreferencedBytes; private int unreferenced; @@ -192,16 +196,16 @@ public class AccordCache implements CacheSize */ void processNoEvictQueue() { - noEvictGeneration = (noEvictGeneration + 1) & 0xffff; + noEvictGeneration = (noEvictGeneration + 1) & GENERATION_MASK; if (noEvictQueue.isEmpty()) return; - Iterator> iter = noEvictQueue.iterator(); + Iterator> iter = noEvictQueue.iterator(); int skipCount = 3; while (skipCount > 0 && iter.hasNext()) { - AccordCacheEntry entry = iter.next(); - int age = (noEvictGeneration - entry.noEvictGeneration()) & 0xffff; + AccordCacheEntry entry = iter.next(); + int age = (noEvictGeneration - entry.noEvictGeneration()) & GENERATION_MASK; if (age >= entry.noEvictMaxAge()) { evictNoEvict.warn(entry, age, entry.noEvictMaxAge()); @@ -226,14 +230,14 @@ public class AccordCache implements CacheSize while (bytesCached > maxSizeInBytes && !evictQueue.isEmpty()) { - AccordCacheEntry node = evictQueue.peek(); + AccordCacheEntry node = evictQueue.peek(); shrinkOrEvict(lock, node); } tryShrinkOrEvict = false; } @VisibleForTesting - private void shrinkOrEvict(Lock lock, AccordCacheEntry node) + private void shrinkOrEvict(Lock lock, AccordCacheEntry node) { require(node.references() == 0); @@ -244,7 +248,7 @@ public class AccordCache implements CacheSize } else { - IntrusiveLinkedList> queue; + IntrusiveLinkedList> queue; queue = node.isNoEvict() ? noEvictQueue : evictQueue; node.unlink(); if (shrink == Shrink.DONE) @@ -272,7 +276,7 @@ public class AccordCache implements CacheSize } @VisibleForTesting - public void tryEvict(AccordCacheEntry node) + public void tryEvict(AccordCacheEntry node) { require(node.references() == 0); @@ -290,7 +294,6 @@ public class AccordCache implements CacheSize case LOADING: node.loading().loading.cancel(); case WAITING_TO_LOAD: - Invariants.paranoid(node.loadingOrWaiting().waiters == null); case LOADED: node.unlink(); evict(node, true); @@ -305,7 +308,7 @@ public class AccordCache implements CacheSize } } - public void saveWhenReadyExclusive(AccordCacheEntry entry, Runnable onSuccess) + public void saveWhenReadyExclusive(AccordCacheEntry entry, Runnable onSuccess) { if (!entry.isSavingOrWaiting() && !entry.saveWhenReady()) onSuccess.run(); @@ -313,7 +316,7 @@ public class AccordCache implements CacheSize entry.savingOrWaitingToSave().identity.onSuccess(onSuccess); } - private void evict(AccordCacheEntry node, boolean updateUnreferenced) + private void evict(AccordCacheEntry node, boolean updateUnreferenced) { if (logger.isTraceEnabled()) logger.trace("Evicting {}", node); @@ -336,25 +339,25 @@ public class AccordCache implements CacheSize if (node.status() == LOADED && VALIDATE_LOAD_ON_EVICT) owner.validateLoadEvicted(node); - AccordCacheEntry self = node.owner.remove(node.key()); + AccordCacheEntry self = node.owner.remove(node.key()); Invariants.require(self.references() == 0); require(self == node, "Leaked node detected; was attempting to remove %s but cache had %s", node, self); node.notifyListeners(Listener::onEvict); node.evicted(); } - Collection> load(LoadExecutor loadExecutor, P1 p1, P2 p2, AccordCacheEntry node) + Loading load(LoadExecutor loadExecutor, P1 p1, P2 p2, AccordCacheEntry node) { - return node.load(loadExecutor, p1, p2).waiters(); + return node.load(loadExecutor, p1, p2); } - void loaded(AccordCacheEntry node, V value) + void loaded(AccordCacheEntry node, V value) { node.loaded(value); node.notifyListeners(Listener::onUpdate); } - void failedToLoad(AccordCacheEntry node) + void failedToLoad(AccordCacheEntry node) { Invariants.require(node.references() == 0); if (node.isUnqueued()) @@ -367,38 +370,38 @@ public class AccordCache implements CacheSize evict(node, true); } - void saved(AccordCacheEntry node, Object identity, Throwable fail) + void saved(AccordCacheEntry node, Object identity, Throwable fail) { if (node.saved(identity, fail) && node.references() == 0 && node.isUnqueued()) evictQueue.addFirst(node); // add to front since we have just saved, so we were eligible for eviction } - public > void release(S safeRef, AccordTask owner) + public & AccordSafeState> void release(S safeRef, AccordTask owner) { safeRef.global().owner.release(safeRef, owner); } - public > Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) + public & AccordSafeState> Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) { Type instance = new Type<>(keyClass, adapter, metrics); types.add(instance); return instance; } - public > Type newType( + public & AccordSafeState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, Function quickShrink, TriFunction validateFunction, ToLongFunction heapEstimator, - Function, S> safeRefFactory, + Function, S> safeRefFactory, AccordCacheMetrics.Shard metrics) { return newType(keyClass, loadFunction, saveFunction, quickShrink, (i, j) -> j, (c, i, j) -> (V)j, validateFunction, heapEstimator, i -> 0, safeRefFactory, metrics); } - public > Type newType( + public & AccordSafeState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, @@ -408,7 +411,7 @@ public class AccordCache implements CacheSize TriFunction validateFunction, ToLongFunction heapEstimator, ToLongFunction shrunkHeapEstimator, - Function, S> safeRefFactory, + Function, S> safeRefFactory, AccordCacheMetrics.Shard metrics) { return newType(keyClass, new FunctionalAdapter<>(loadFunction, saveFunction, quickShrink, @@ -425,18 +428,18 @@ public class AccordCache implements CacheSize public interface Listener { - default void onAdd(AccordCacheEntry state) {} - default void onUpdate(AccordCacheEntry state) {} - default void onEvict(AccordCacheEntry state) {} + default void onAdd(AccordCacheEntry state) {} + default void onUpdate(AccordCacheEntry state) {} + default void onEvict(AccordCacheEntry state) {} } - public class Type> implements CacheSize + public class Type & AccordSafeState> implements CacheSize { - public class Instance implements Iterable> + public class Instance implements Iterable> { final AccordCommandStore commandStore; // TODO (desired): don't need to store key separately as stored in node; ideally use a hash set that allows us to get the current entry - private final Map> cache = new Object2ObjectHashMap<>(); + private final Map> cache = new Object2ObjectHashMap<>(); private List> listeners = null; // TODO (expected): update this after releasing the lock private OrderedKeys orderedKeys; @@ -446,50 +449,51 @@ public class AccordCache implements CacheSize this.commandStore = commandStore; } - public S acquire(K key) + public final S acquire(K key) { - AccordCacheEntry node = acquire(key, false); + AccordCacheEntry node = acquire(key, false); return adapter.safeRef(node); } - public S acquireIfLoaded(K key) + public final S acquireIfLoadedAndPermitted(K key) { - AccordCacheEntry node = acquire(key, true); + AccordCacheEntry node = acquire(key, true); if (node == null) return null; return adapter.safeRef(node); } - public S acquire(AccordCacheEntry node) + public final S acquire(AccordCacheEntry node) { Invariants.require(node.owner == this); acquireExisting(node, false); return adapter.safeRef(node); } - public void recordPreAcquired(AccordSafeState ref) + public final void recordPreAcquired(AccordCacheEntry entry) { - Invariants.require(ref.global().owner == this); - incrementCacheHits(); + Invariants.require(entry.owner == this); + if (entry.isLoaded()) incrementCacheHits(); + else incrementCacheMisses(); } - private AccordCacheEntry acquire(K key, boolean onlyIfLoaded) + private AccordCacheEntry acquire(K key, boolean onlyIfLoadedAndPermitted) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node == null - ? acquireAbsent(key, onlyIfLoaded) - : acquireExisting(node, onlyIfLoaded); + ? acquireAbsent(key, onlyIfLoadedAndPermitted) + : acquireExisting(node, onlyIfLoadedAndPermitted); } /* * Can only return a LOADING Node (or null) */ - private AccordCacheEntry acquireAbsent(K key, boolean onlyIfLoaded) + private AccordCacheEntry acquireAbsent(K key, boolean onlyIfLoaded) { incrementCacheMisses(); if (onlyIfLoaded) return null; - AccordCacheEntry node = adapter.newEntry(key, this); + AccordCacheEntry node = adapter.newEntry(key, this); node.increment(); Object prev = cache.put(key, node); @@ -506,7 +510,7 @@ public class AccordCache implements CacheSize /* * Can't return EVICTED or INITIALIZED */ - private AccordCacheEntry acquireExisting(AccordCacheEntry node, boolean onlyIfLoaded) + private AccordCacheEntry acquireExisting(AccordCacheEntry node, boolean onlyIfLoadedAndPermitted) { boolean isLoaded = node.isLoaded(); if (isLoaded) @@ -514,8 +518,11 @@ public class AccordCache implements CacheSize else incrementCacheMisses(); - if (onlyIfLoaded && !isLoaded) - return null; + if (onlyIfLoadedAndPermitted) + { + if (!isLoaded || node.hasFifoOrLocked()) + return null; + } if (node.increment() == 1) { @@ -527,21 +534,21 @@ public class AccordCache implements CacheSize return node; } - public void release(AccordSafeState safeRef, AccordTask owner) + public final void release(S safeRef, AccordTask owner) { K key = safeRef.global().key(); logger.trace("Releasing resources for {}: {}", key, safeRef); - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); - require(!safeRef.isUnsafe()); + require(!safeRef.isReleased()); require(safeRef.global() != null, "safeRef node is null for %s", key); require(safeRef.global() == node, "safeRef node not in map: %s != %s", safeRef.global(), node); require(node.references() > 0, "references (%d) are zero for %s (%s)", node.references(), key, node); require(node.isUnqueued()); boolean evict = false; - if (safeRef.hasUpdate()) + if (safeRef.isModified()) { V update = safeRef.current(); if (update != null) @@ -555,15 +562,12 @@ public class AccordCache implements CacheSize } node.notifyListeners(Listener::onUpdate); } - else if (node.isLoadingOrWaiting()) - { - node.loadingOrWaiting().remove(owner); - } else { evict = node.is(LOADED) && node.isNull(); } - safeRef.markUnsafe(); + node.remove(owner, safeRef.isSafe()); + safeRef.setReleased(); if (node.decrement() == 0) { @@ -596,9 +600,9 @@ public class AccordCache implements CacheSize tryShrinkOrEvict = true; } - AccordCacheEntry remove(K key) + final AccordCacheEntry remove(K key) { - AccordCacheEntry result = cache.remove(key); + AccordCacheEntry result = cache.remove(key); if (orderedKeys != null && result != null) orderedKeys.remove(key); return result; @@ -609,7 +613,7 @@ public class AccordCache implements CacheSize return Type.this; } - public Iterable keysBetween(K start, boolean startInclusive, K end, boolean endInclusive) + public final Iterable keysBetween(K start, boolean startInclusive, K end, boolean endInclusive) { if (orderedKeys == null) orderedKeys = new OrderedKeys<>(adapter.keyComparator(), cache.keySet()); @@ -618,15 +622,15 @@ public class AccordCache implements CacheSize } @Override - public Iterator> iterator() + public final Iterator> iterator() { return cache.values().iterator(); } - void validateLoadEvicted(AccordCacheEntry node) + final void validateLoadEvicted(AccordCacheEntry node) { @SuppressWarnings("unchecked") - AccordCacheEntry state = (AccordCacheEntry) node; + AccordCacheEntry state = (AccordCacheEntry) node; K key = state.key(); V evicted = state.tryGetFull(); if (evicted == null) @@ -649,46 +653,46 @@ public class AccordCache implements CacheSize } @VisibleForTesting - public AccordCacheEntry getUnsafe(K key) + public final AccordCacheEntry getUnsafe(K key) { return cache.get(key); } @VisibleForTesting - public boolean isReferenced(K key) + public final boolean isReferenced(K key) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null && node.references() > 0; } @VisibleForTesting - boolean keyIsReferenced(Object key, Class> valClass) + final boolean keyIsReferenced(Object key, Class> valClass) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null && node.references() > 0; } @VisibleForTesting - boolean keyIsCached(Object key, Class> valClass) + final boolean keyIsCached(Object key, Class> valClass) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null; } @VisibleForTesting - int references(Object key, Class> valClass) + final int references(Object key, Class> valClass) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null ? node.references() : 0; } - void notifyListeners(BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) + final void notifyListeners(BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) { notifyListeners(listeners, notify, node); notifyListeners(typeListeners, notify, node); } - void notifyListeners(List> listeners, BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) + final void notifyListeners(List> listeners, BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) { if (listeners != null) { @@ -698,20 +702,20 @@ public class AccordCache implements CacheSize } } - public void register(Listener l) + public final void register(Listener l) { if (listeners == null) listeners = new ArrayList<>(); listeners.add(l); } - public void unregister(Listener l) + public final void unregister(Listener l) { if (!tryUnregister(l)) throw illegalState("Listener was not registered"); } - public boolean tryUnregister(Listener l) + public final boolean tryUnregister(Listener l) { if (listeners == null || !listeners.remove(l)) return false; @@ -719,9 +723,23 @@ public class AccordCache implements CacheSize listeners = null; return true; } + + final boolean isCommandsForKey() + { + return getClass() == KeyInstance.class; + } } - private final Class keyClass; + // KeyInstance exists to provide us slightly easier discrimination about the Type an AccordCacheEntry is associated with + public final class KeyInstance extends Instance + { + public KeyInstance(AccordCommandStore commandStore) + { + super(commandStore); + } + } + + private final Class keyClass; // type of key, useful primarily for toString(), but also piggyback for deciding Instance type private Adapter adapter; private long bytesCached; private int size; @@ -734,6 +752,8 @@ public class AccordCache implements CacheSize public Type(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) { + // Integer and String permitted for testing, but the Invariant exists only to enforce that we construct the right kind of Instance + Invariants.require(keyClass == RoutingKey.class || keyClass == TxnId.class || keyClass == String.class || keyClass == Integer.class); this.keyClass = keyClass; this.adapter = adapter; this.objectSize = metrics.objectSize; @@ -751,7 +771,7 @@ public class AccordCache implements CacheSize // can be safely garbage collected if empty Instance newInstance(AccordCommandStore commandStore) { - return new Instance(commandStore); + return keyClass == RoutingKey.class ? new KeyInstance(commandStore) : new Instance(commandStore); } private void incrementCacheHits() @@ -867,17 +887,17 @@ public class AccordCache implements CacheSize } @VisibleForTesting - AccordCacheEntry head() + AccordCacheEntry head() { - Iterator> iter = evictQueue.iterator(); + Iterator> iter = evictQueue.iterator(); return iter.hasNext() ? iter.next() : null; } @VisibleForTesting - AccordCacheEntry tail() + AccordCacheEntry tail() { - AccordCacheEntry last = null; - Iterator> iter = evictQueue.iterator(); + AccordCacheEntry last = null; + Iterator> iter = evictQueue.iterator(); while (iter.hasNext()) last = iter.next(); return last; @@ -888,7 +908,7 @@ public class AccordCache implements CacheSize return size() == 0; } - Iterable> evictionQueue() + Iterable> evictionQueue() { return evictQueue::iterator; } @@ -937,10 +957,10 @@ public class AccordCache implements CacheSize return; type.register(new AccordCache.Listener<>() { - private final IdentityHashMap, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>(); + private final IdentityHashMap, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>(); @Override - public void onAdd(AccordCacheEntry state) + public void onAdd(AccordCacheEntry state) { CacheEvents.Add add = new CacheEvents.Add(); CacheEvents.Evict evict = new CacheEvents.Evict(); @@ -957,7 +977,7 @@ public class AccordCache implements CacheSize } @Override - public void onEvict(AccordCacheEntry state) + public void onEvict(AccordCacheEntry state) { CacheEvents.Evict event = pendingEvicts.remove(state); if (event == null) return; @@ -967,7 +987,7 @@ public class AccordCache implements CacheSize }); } - private static void updateMutable(AccordCache.Type type, AccordCacheEntry state, CacheEvents event) + private static void updateMutable(AccordCache.Type type, AccordCacheEntry state, CacheEvents event) { event.status = state.status().name(); @@ -987,7 +1007,7 @@ public class AccordCache implements CacheSize event.update(); } - static class FunctionalAdapter implements Adapter + static class FunctionalAdapter & AccordSafeState> implements Adapter { final BiFunction load; final QuadFunction save; @@ -997,8 +1017,8 @@ public class AccordCache implements CacheSize final TriFunction validate; final ToLongFunction estimateHeapSize; final ToLongFunction estimateShrunkHeapSize; - final Function, S> newSafeRef; - final BiFunction.Instance, AccordCacheEntry> newNode; + final Function, S> newSafeRef; + final BiFunction.Instance, AccordCacheEntry> newNode; FunctionalAdapter(BiFunction load, QuadFunction save, @@ -1007,8 +1027,8 @@ public class AccordCache implements CacheSize TriFunction validate, ToLongFunction estimateHeapSize, ToLongFunction estimateShrunkHeapSize, - Function, S> newSafeRef, - BiFunction.Instance, AccordCacheEntry> newNode) + Function, S> newSafeRef, + BiFunction.Instance, AccordCacheEntry> newNode) { this.load = load; this.save = save; @@ -1082,13 +1102,13 @@ public class AccordCache implements CacheSize } @Override - public S safeRef(AccordCacheEntry node) + public S safeRef(AccordCacheEntry node) { return newSafeRef.apply(node); } @Override - public AccordCacheEntry newEntry(K key, Type.Instance owner) + public AccordCacheEntry newEntry(K key, Type.Instance owner) { return newNode.apply(key, owner); } @@ -1100,7 +1120,7 @@ public class AccordCache implements CacheSize } } - static class SettableWrapper extends FunctionalAdapter + static class SettableWrapper & AccordSafeState> extends FunctionalAdapter { volatile BiFunction load; @@ -1110,9 +1130,9 @@ public class AccordCache implements CacheSize this.load = super.load; } - public static Adapter loadOnly(BiFunction load) + public static & AccordSafeState> Adapter loadOnly(BiFunction load) { - SettableWrapper result = new SettableWrapper<>(new NoOpAdapter<>()); + SettableWrapper result = new SettableWrapper<>(new NoOpAdapter()); result.load = load; return result; } @@ -1124,7 +1144,7 @@ public class AccordCache implements CacheSize } } - static class NoOpAdapter implements Adapter + static class NoOpAdapter & AccordSafeState> implements Adapter { @Override public V load(AccordCommandStore commandStore, K key) { return null; } @Override public Runnable save(AccordCommandStore commandStore, K key, @Nullable V value, @Nullable Object shrunk) { return null; } @@ -1135,7 +1155,7 @@ public class AccordCache implements CacheSize @Override public long estimateHeapSize(V value) { return 0; } @Override public long estimateShrunkHeapSize(Object shrunk) { return 0; } @Override public boolean validate(AccordCommandStore commandStore, K key, V value) { return false; } - @Override public S safeRef(AccordCacheEntry node) { return null; } + @Override public S safeRef(AccordCacheEntry node) { return null; } } public static class CommandsForKeyAdapter implements Adapter @@ -1239,7 +1259,7 @@ public class AccordCache implements CacheSize } @Override - public AccordSafeCommandsForKey safeRef(AccordCacheEntry node) + public AccordSafeCommandsForKey safeRef(AccordCacheEntry node) { return new AccordSafeCommandsForKey(node); } @@ -1251,7 +1271,7 @@ public class AccordCache implements CacheSize } @Override - public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) + public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) { CommandsForKeyCacheEntry entry = new CommandsForKeyCacheEntry(key, owner); entry.readyToLoad(); @@ -1376,19 +1396,19 @@ public class AccordCache implements CacheSize } @Override - public AccordSafeCommand safeRef(AccordCacheEntry node) + public AccordSafeCommand safeRef(AccordCacheEntry node) { return new AccordSafeCommand(node); } @Override - public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) + public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) { - AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); + AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); if (txnId.is(Txn.Kind.EphemeralRead)) { node.initialize(null); - int maxAge = (int)Math.min(0xff, 1 + DatabaseDescriptor.getReadRpcTimeout(TimeUnit.SECONDS)); + int maxAge = (int)Math.min(AGE_MASK, 1 + DatabaseDescriptor.getReadRpcTimeout(TimeUnit.SECONDS)); node.markNoEvict(owner.parent().parent().noEvictGeneration, maxAge); } else diff --git a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java b/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java index 77b56f3203..11d8a2b105 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java @@ -18,27 +18,43 @@ package org.apache.cassandra.service.accord; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.function.BiConsumer; +import java.util.function.BiPredicate; import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Ints; +import accord.local.SafeState; import accord.utils.ArrayBuffers.BufferList; import accord.utils.IntrusiveLinkedList; import accord.utils.IntrusiveLinkedListNode; import accord.utils.Invariants; +import accord.utils.SortedArrays; +import accord.utils.TriConsumer; +import accord.utils.UnhandledEnum; import accord.utils.async.Cancellable; import org.apache.cassandra.service.accord.AccordCache.Adapter; import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; +import org.apache.cassandra.service.accord.AccordExecutor.IOTask; import org.apache.cassandra.utils.ObjectSizes; +import static accord.utils.Invariants.nonNull; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.HOLD_QUEUE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.UNLOCKED; +import static org.apache.cassandra.service.accord.AccordCacheEntry.Queue.compare; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_LOAD; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_SAVE; @@ -52,7 +68,7 @@ import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.WAITIN /** * Global (per CommandStore) state of a cached entity (Command or CommandsForKey). */ -public class AccordCacheEntry extends IntrusiveLinkedListNode +public class AccordCacheEntry & AccordSafeState> extends IntrusiveLinkedListNode { public enum Status { @@ -128,9 +144,12 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode } } - static final int STATUS_MASK = 0x0000001F; - static final int SHRUNK = 0x00000040; - static final int NO_EVICT = 0x00000020; + static final int STATUS_MASK = 0x0000001F; + static final int NO_EVICT = 0x00000020; + static final int SHRUNK = 0x00000040; + static final int LOCKED_MASK = 0x00000180; + static final int LOCKED_SHIFT = Integer.numberOfTrailingZeros(LOCKED_MASK); + static final int LOCKED_HOLDING_QUEUE = HOLD_QUEUE.ordinal() << LOCKED_SHIFT; static final int IS_NOT_EVICTED = 0xF; static final int IS_LOADED = 0x8; static final int IS_NESTED = 0x4; @@ -138,117 +157,1045 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode static final int IS_LOADING_OR_WAITING = 0x2; static final int IS_SAVING_OR_WAITING_MASK = 0xE; static final int IS_SAVING_OR_WAITING = 0xC; + static final int GENERATION_SHIFT = 9; + static final int GENERATION_MASK = 0x7fff; + static final int AGE_SHIFT = 24; + static final int AGE_MASK = 0xff; + + static class Queue + { + private static final int DEFAULT_CAPACITY = 4; + private static final int LOCKED_INDEX = 0; + private static final int PRIORITY_START_INDEX = LOCKED_INDEX + 1; + /** + * [priorityHead..priorityTail) stores a priority-sorted list of tasks + * (fifoTail...fifoHead] stores a fifo queue that runs ahead of any priority tasks + * (fifoTail-unsequencedCount...fifoTail] stores unsequenced tasks that are waiting for a queued incremental task. + * This only happens for TxnId cache entries, since they may lockAndHoldQueue. Once the lock is released, + * any pending unsequenced tasks are notified and immediately made (irrevocably) runnable for this entry. + */ + private AccordTask[] tasks; + // TODO (expected): use bytes/shorts for indexes to keep size down, and have an expanded version of the Queue + // with better algorithmic complexity (e.g. Hash -> IntrusivePriorityHeap) + private int priorityHead, priorityTail, fifoHead, fifoTail; + private int unsequencedSize; + + public Queue() + { + tasks = new AccordTask[DEFAULT_CAPACITY]; + priorityHead = priorityTail = PRIORITY_START_INDEX; + fifoHead = fifoTail = DEFAULT_CAPACITY - 1; + } + + private Queue(Queue copy) + { + tasks = copy.tasks.clone(); + priorityHead = copy.priorityHead; + priorityTail = copy.priorityTail; + fifoHead = copy.fifoHead; + fifoTail = copy.fifoTail; + } + + // returns true if no fifo tasks already queue (i.e. so we become head) + boolean addFifo(AccordTask task) + { + ensureCapacity(); + boolean isHead = fifoHead == fifoTail; + if (unsequencedSize > 0) // simply displace the unsequence task, as they're an unordered list + tasks[fifoTail - unsequencedSize] = tasks[fifoTail]; + tasks[fifoTail--] = task; + validate(); + return isHead; + } + + boolean addUnsequenced(AccordTask task) + { + Invariants.require(task.isUnsequenced()); + ensureCapacity(); + tasks[fifoTail - unsequencedSize++] = task; + return unsequencedSize == 1 && sequencedSize() == 1; + } + + boolean isLocked(AccordTask task) + { + return tasks[LOCKED_INDEX] == task; + } + + AccordTask lockedBy() + { + return tasks[LOCKED_INDEX]; + } + + boolean removeIfHead(AccordTask task) + { + return removeIfFifoHead(task) || removeIfPriorityHead(task); + } + + boolean removeIfFifoHead(AccordTask task) + { + if (!hasFifo()) + return false; + + if (task != tasks[fifoHead]) + return false; + tasks[fifoHead--] = null; + return true; + } + + boolean removeIfPriorityHead(AccordTask task) + { + if (!hasPriority()) + return false; + + if (task != tasks[priorityHead]) + return false; + tasks[priorityHead++] = null; + return true; + } + + void lock(AccordTask task) + { + tasks[LOCKED_INDEX] = task; + } + + void unlock(AccordTask task) + { + Invariants.require(tasks[LOCKED_INDEX] == task); + tasks[LOCKED_INDEX] = null; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + boolean addPrioritised(AccordTask task) + { + ensureCapacity(); + int insertPos = Arrays.binarySearch(tasks, priorityHead, priorityTail, task, Queue::compare); + if (insertPos < 0) + insertPos = -1 - insertPos; + + if (priorityHead == PRIORITY_START_INDEX || insertPos > (priorityTail + priorityHead)/2) + { + System.arraycopy(tasks, insertPos, tasks, insertPos + 1, priorityTail - insertPos); + tasks[insertPos] = task; + priorityTail++; + } + else + { + System.arraycopy(tasks, priorityHead, tasks, priorityHead - 1, insertPos - priorityHead); + tasks[insertPos - 1] = task; + priorityHead--; + } + + validate(); + return fifoHead == fifoTail && tasks[priorityHead] == task; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + void addWaitingToLoad(AccordTask task) + { + ensureCapacity(); + tasks[priorityTail++] = task; + } + + private boolean hasTailRoom() + { + if (priorityTail + unsequencedSize <= fifoTail) + return true; + Invariants.require(priorityTail + unsequencedSize == 1 + fifoTail); + return false; + } + + private void ensureCapacity() + { + if (!hasTailRoom()) + { + if (fifoHead == fifoTail && unsequencedSize == 0 && fifoTail < tasks.length - 1) fifoHead = fifoTail = tasks.length - 1; + else if (priorityHead == priorityTail && priorityHead > PRIORITY_START_INDEX) priorityHead = priorityTail = PRIORITY_START_INDEX; + else if (totalSize() >= (tasks.length - 1) / 2) compact(new AccordTask[tasks.length * 2]); + else compact(tasks); + Invariants.require(hasTailRoom()); + } + } + + private void compact(AccordTask[] into) + { + if (priorityHead == priorityTail) priorityHead = priorityTail = PRIORITY_START_INDEX; + else + { + int priorityLength = priorityTail - priorityHead; + System.arraycopy(tasks, priorityHead, into, PRIORITY_START_INDEX, priorityLength); + int newTail = PRIORITY_START_INDEX + priorityLength; + Invariants.require(newTail <= priorityTail); + if (into == tasks) + Arrays.fill(into, newTail, priorityTail, null); + priorityHead = PRIORITY_START_INDEX; + priorityTail = newTail; + } + + if (fifoHead == fifoTail && unsequencedSize == 0) fifoHead = fifoTail = into.length - 1; + else + { + int fifoLength = fifoHead - fifoTail; + int copyLength = fifoLength + unsequencedSize; + int copyFrom = (fifoTail - unsequencedSize) + 1; + int copyTo = into.length - copyLength; + Invariants.require(copyTo >= copyFrom); + System.arraycopy(tasks, copyFrom, into, copyTo, copyLength); + if (into == tasks) + Arrays.fill(into, copyFrom, copyTo, null); + fifoHead = into.length - 1; + fifoTail = fifoHead - fifoLength; + } + + if (tasks != into) + { + into[LOCKED_INDEX] = tasks[LOCKED_INDEX]; + tasks = into; + } + validate(); + } + + private void validate() + { + for (int i = PRIORITY_START_INDEX ; i < priorityHead ; ++i) + Invariants.require(tasks[i] == null); + for (int i = priorityHead ; i < priorityTail ; ++i) + Invariants.require(tasks[i] != null); + for (int i = priorityTail; i <= fifoTail - unsequencedSize; ++i) + Invariants.require(tasks[i] == null); + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoHead ; ++i) + Invariants.require(tasks[i] != null); + for (int i = fifoHead + 1 ; i < tasks.length ; ++i) + Invariants.require(tasks[i] == null); + } + + AccordTask peek() + { + if (hasFifo()) return tasks[fifoHead]; + if (hasPriority()) return tasks[priorityHead]; + return null; + } + + AccordTask peekFifo() + { + return hasFifo() ? tasks[fifoHead] : null; + } + + // second task + AccordTask peekBehind() + { + int fifoSize = fifoSize(); + if (fifoSize > 1) + return tasks[fifoHead - 1]; + int priorityIndex = priorityHead + (1 - fifoSize); + if (priorityIndex < priorityTail) + return tasks[priorityIndex]; + return null; + } + + boolean hasFifo() + { + return fifoHead != fifoTail; + } + + boolean hasPriority() + { + return priorityHead != priorityTail; + } + + boolean hasUnsequenced() + { + return unsequencedSize > 0; + } + + int sequencedSize() + { + return prioritySize() + fifoSize(); + } + + int unsequencedSize() + { + return unsequencedSize; + } + + int totalSize() + { + return sequencedSize() + unsequencedSize; + } + + int prioritySize() + { + return priorityTail - priorityHead; + } + + int fifoSize() + { + return fifoHead - fifoTail; + } + + // true iff was head + boolean removeFifoOrPriority(AccordTask task, boolean permitMissing) + { + int fifoIndex = fifoIndexOf(task); + if (fifoIndex >= 0) + { + if (fifoIndex == fifoHead) + { + tasks[fifoHead--] = null; + validate(); + return true; + } + else + { + if (remove(fifoIndex, fifoTail + 1, fifoHead + 1)) ++fifoTail; + else --fifoHead; + validate(); + return false; + } + } + + int priorityIndex = priorityIndexOf(task); + Invariants.require(priorityIndex >= 0 || permitMissing); + if (priorityIndex >= 0) + { + if (priorityIndex == priorityHead) + { + tasks[priorityHead++] = null; + return !hasFifo(); + } + + if (remove(priorityIndex, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return false; + } + + return false; + } + + boolean removeUnsequenced(AccordTask task) + { + int unsequencedIndex = unsequencedIndexOf(task); + if (unsequencedIndex < 0) + return false; + + --unsequencedSize; + tasks[unsequencedIndex] = tasks[fifoTail - unsequencedSize]; + tasks[fifoTail - unsequencedSize] = null; + return true; + } + + // return true IFF was head + private boolean removePriority(AccordTask task, boolean permitAbsent) + { + int i = priorityIndexOf(task); + if (i < 0) + { + Invariants.require(permitAbsent); + return false; + } + + boolean wasHead = i == priorityHead; + if (remove(i, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return wasHead; + } + + // return true if we move the start forwards, false if we moved the end back + private boolean remove(int i, int start, int end) + { + if (i < (start + end)/2) + { + System.arraycopy(tasks, start, tasks, start + 1, i - start); + tasks[start] = null; + return true; + } + else + { + System.arraycopy(tasks, i + 1, tasks, i, end - (i + 1)); + tasks[end - 1] = null; + return false; + } + } + + boolean contains(AccordTask task) + { + return indexOf(task) >= 0; + } + + private int indexOf(AccordTask task) + { + if (tasks[priorityHead] == task) + return priorityHead; + + if (tasks[fifoHead] == task) + return fifoHead; + + int i = priorityIndexOf(task); + if (i >= 0) + return i; + + return fifoIndexOf(task); + } + + private int priorityIndexOf(AccordTask task) + { + if (priorityTail - priorityHead > 16) + { + if (tasks[priorityHead] == task) + return priorityHead; + + int i = SortedArrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, Queue::compare, SortedArrays.Search.CEIL); + if (i < 0) + return -1; + + while (i < priorityTail) + { + if (tasks[i] == task) + return i; + if (compare(task, tasks[i]) != 0) + break; + ++i; + } + } + + for (int i = priorityHead ; i < priorityTail ; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int fifoIndexOf(AccordTask task) + { + for (int i = fifoHead ; i > fifoTail ; --i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int unsequencedIndexOf(AccordTask task) + { + for (int i = (fifoTail - unsequencedSize) + 1 ; i <= fifoTail ; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + int drainUnsequenced(TriConsumer, P1, P2> forEach, P1 p1, P2 p2) + { + for (int i = (fifoTail - unsequencedSize) + 1 ; i <= fifoTail ; ++i) + { + AccordTask task = tasks[i]; + tasks[i] = null; + // should not be reentrant + forEach.accept(task, p1, p2); + } + int count = unsequencedSize; + unsequencedSize = 0; + return count; + } + + RunnableStatus ensureHeadFifo(AccordTask task) + { + if (hasFifo()) + { + Invariants.require(tasks[fifoHead] == task); + return NOT_RUNNABLE; + } + + if (tasks[priorityHead] == task) + { + tasks[priorityHead++] = null; + addFifo(task); + return STILL_RUNNABLE; + } + else + { + boolean wasPriorityHead = removePriority(task, false); + boolean isFifoHead = addFifo(task); + if (!isFifoHead) + return NOT_RUNNABLE; + if (wasPriorityHead) + return STILL_RUNNABLE; + if (hasPriority() || hasUnsequenced()) + return NEWLY_BLOCKING_RUNNABLE; + return NEWLY_RUNNABLE; + } + } + + static int compare(AccordTask a, AccordTask b) + { + Invariants.require(a != null && b != null); + int c = Long.compare(a.position, b.position); + if (c == 0) + c = a.executionContext().executionKind().compareTo(b.executionContext().executionKind()); + if (c == 0) + c = Long.compare(a.createdAt, b.createdAt); + if (c == 0) + c = Long.compare(a.loadedAt, b.loadedAt); + if (c == 0) + c = a.loggingId().compareTo(b.loggingId()); + return c; + } + + public boolean hasQueued() + { + return hasFifo() || hasPriority(); + } + } + static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCacheEntry<>(null, null)); private final K key; - final AccordCache.Type.Instance owner; + final AccordCache.Type.Instance owner; private Object state; + /** + * Either a single AccordTask or a Queue object. The meaning of a single task is defined by various flags. + * If locked, then the task is not logically part of the queue unless LOCKED_HOLDING_QUEUE. + * If unlocked, or LOCKED_HOLDING_QUEUE, the task represents a single-item queue. + * If the task forms a single-item queue, whether it is FIFO or prioritised is determined by the task's isCacheQueuedFifo flag. + */ + private Object queue; // private int status; + private int unsequenced; int sizeOnHeap; private volatile int references; private static final AtomicIntegerFieldUpdater referencesUpdater = AtomicIntegerFieldUpdater.newUpdater(AccordCacheEntry.class, "references"); - AccordCacheEntry(K key, AccordCache.Type.Instance owner) + AccordCacheEntry(K key, AccordCache.Type.Instance owner) { this.key = key; this.owner = owner; } - void unlink() + private RunnableStatus validate(RunnableStatus status) + { + Invariants.require(queue != null); + AccordTask head = queue instanceof Queue ? ((Queue) queue).peek() : (AccordTask) queue; + Invariants.require(isRunnable(head) || status == NOT_RUNNABLE); + return status; + } + + // TODO (expected): don't unwrap when only one entry, since this may cause us to flap when locking unsequenced tasks + private void maybeUnwrap(Queue q) + { + int size = q.sequencedSize(); + switch (size) + { + case 0: + Invariants.require(q.unsequencedSize() == 0); + queue = isLocked() ? nonNull(q.lockedBy()) : null; + break; + + case 1: + if (isLocked() || q.unsequencedSize() > 0) + break; + queue = q.peek(); + } + } + + private boolean maybeUnwrap(Queue q, AccordTask lockedBy) + { + if (q.sequencedSize() == 0) + { + Invariants.require(q.unsequencedSize() == 0); + queue = lockedBy; + return true; + } + return false; + } + + // assumes already queued with priority + final RunnableStatus moveToFifo(AccordTask task) + { + if (queue != task) + { + Queue q = (Queue) queue; + RunnableStatus status = q.ensureHeadFifo(task); + if (status == NEWLY_BLOCKING_RUNNABLE && isLoaded()) + onChangedHead(q, null, q.peekBehind()); + return validate(isRunnable(task) ? status : NOT_RUNNABLE); + } + return validate(isRunnable(task) ? STILL_RUNNABLE : NOT_RUNNABLE); + } + + // drains ONLY those queued with addWaitingToLoad; addFifo are included in the result but are not removed from the collection + public final BufferList> drainWaitingToLoad() + { + Invariants.require(isLoading()); + Invariants.require(!isLocked()); + BufferList> list = new BufferList<>(); + if (queue != null) + { + if (queue instanceof Queue) + { + Queue q = (Queue) queue; + for (int i = q.priorityHead ; i < q.priorityTail ; ++i) + { + list.add(q.tasks[i]); + q.tasks[i] = null; + } + q.priorityHead = q.priorityTail = Queue.PRIORITY_START_INDEX; + for (int i = q.fifoHead ; i > q.fifoTail ; --i) + list.add(q.tasks[i]); + + maybeUnwrap(q); + } + else + { + AccordTask task = (AccordTask) queue; + list.add(task); + Invariants.require(!isLocked()); + if (!task.isCacheQueuedFifo()) + queue = null; + } + } + return list; + } + + final void remove(AccordTask task, boolean ownsLock) + { + if (queue instanceof Queue) + { + Queue q = (Queue) queue; + boolean remove; + boolean isLocked = isLocked() && q.isLocked(task); + Invariants.require(isLocked == ownsLock); + if (isLocked) + { + // if locked, we've already released unsequenced/pririty/fifo positions unless isLockedHoldingQueue + remove = isLockedHoldingQueue(); + status &= ~LOCKED_MASK; + q.unlock(task); + } + else if (task.isUnsequenced(this)) + { + if (task.isCacheQueued()) + { + if (!q.removeUnsequenced(task)) + releaseUnsequenced(q, task); + } + remove = false; + } + else remove = task.isCacheQueued(); + + if (remove) + { + boolean wasHead = remove && q.removeFifoOrPriority(task, false); + if (isLoaded() && wasHead) + { + unsequenced += q.drainUnsequenced(AccordTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); + onChangedHead(q, q.peek(), null); + } + } + + if (remove || isLocked) + maybeUnwrap(q); + } + else if (queue == task) + { + boolean isLocked = isLocked(); + Invariants.require(isLocked == ownsLock); + if (isLocked) + { + status &= ~LOCKED_MASK; + } + else if (task.isUnsequenced(this)) + { + if (task.isCacheQueued()) --unsequenced; // nothing to release if we hit zero + else Invariants.require(isLoading()); + } + queue = null; + } + else + { + Invariants.require(!ownsLock); + if (task.isUnsequenced(this) && task.isCacheQueued()) + --unsequenced; // nothing to release if we hit zero + } + } + + final boolean isCommandsForKey() + { + return getClass() == AccordSafeCommandsForKey.CommandsForKeyCacheEntry.class; + } + + final RunnableStatus headStatus(AccordTask task) + { + if (queue == task) + return validate(isRunnable(task) ? NEWLY_RUNNABLE : NOT_RUNNABLE); + + Queue q = (Queue) queue; + if (q.peek() != task || !isRunnable(task)) + return NOT_RUNNABLE; + + return validate(q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); + } + + private Queue ensureQueue() + { + if (queue instanceof Queue) + return (Queue) queue; + + AccordTask head = (AccordTask) this.queue; + Queue q = new Queue(); + if (isLocked()) + { + if (isLockedHoldingQueue()) + q.addFifo(head); + q.lock(head); + } + else if (head.isCacheQueuedFifo()) q.addFifo(head); + else q.addPrioritised(head); + this.queue = q; + return q; + } + + final void addWaitingToLoad(AccordTask task) + { + Invariants.require(isLoading()); + if (queue == null) queue = task; + else ensureQueue().addWaitingToLoad(task); + } + + final AccordTask head() + { + if (queue == null) + return null; + + if (queue instanceof Queue) + return ((Queue) queue).peek(); + + if (isLocked() && !isLockedHoldingQueue()) + return null; + + return (AccordTask) queue; + } + + final RunnableStatus addUnsequenced(AccordTask task) + { + Invariants.require(isLoaded()); + + AccordTask head = head(); + if (head != null && head.holdsLocksBetweenRuns()) + { + boolean wait = compare(task, head) > 0 || (unsequenced == 0 && head.hasIncrementalStarted()); + if (wait) + { + if (ensureQueue().addUnsequenced(task) && isLoaded() && unsequenced == 0) + head.onChangeHeadStatus(this, STILL_RUNNABLE_NEWLY_BLOCKING); + return NOT_RUNNABLE; + } + else + { + ++unsequenced; + if (unsequenced == 1 && isLoaded()) + head.onChangeHeadStatus(this, NOT_RUNNABLE); + return NEWLY_RUNNABLE; + } + } + + ++unsequenced; + return NEWLY_RUNNABLE; + } + + final int waitingCount() + { + Invariants.require(isLoading()); + return queue == null ? 0 : queue instanceof Queue + ? ((Queue)queue).sequencedSize() + : isLocked() == isLockedHoldingQueue() ? 1 : 0; + } + + public enum RunnableStatus + { + NOT_RUNNABLE, STILL_RUNNABLE, NEWLY_RUNNABLE, NEWLY_BLOCKING_RUNNABLE, STILL_RUNNABLE_NEWLY_BLOCKING + } + + private boolean isRunnable(AccordTask head) + { + return !head.holdsLocksBetweenRuns() || unsequenced == 0; + } + + private RunnableStatus add(AccordTask task, BiPredicate> add) + { + Object prev = this.queue; + if (prev == null) + { + queue = task; + return validate(isRunnable(task) ? NEWLY_RUNNABLE : NOT_RUNNABLE); + } + + Queue q = ensureQueue(); + if (!add.test(q, task)) + { + if (isLoaded() && q.totalSize() == 2) + { + AccordTask head = q.peek(); + if (isRunnable(head)) + head.onChangeHeadStatus(this, STILL_RUNNABLE_NEWLY_BLOCKING); + } + return NOT_RUNNABLE; + } + + boolean isRunnable = isRunnable(task); + int sequencedSize = q.sequencedSize(); + int unsequencedSize = q.unsequencedSize(); + if (sequencedSize + unsequencedSize == 1) // could have one locked and one waiting + return validate(isRunnable ? NEWLY_RUNNABLE : NOT_RUNNABLE); + + if (isLoaded() && sequencedSize > 1) + onChangedHead(q, null, q.peekBehind()); + + return validate(isRunnable ? NEWLY_BLOCKING_RUNNABLE : NOT_RUNNABLE); + } + + final RunnableStatus addPrioritised(AccordTask task) + { + Invariants.require(!isLoading()); + return add(task, Queue::addPrioritised); + } + + final RunnableStatus addFifo(AccordTask task) + { + return add(task, Queue::addFifo); + } + + private void onChangedHead(Queue q, @Nullable AccordTask notifyNewHead, @Nullable AccordTask notifyPrevHead) + { + if (notifyNewHead != null && isRunnable(notifyNewHead)) + notifyNewHead.onChangeHeadStatus(this, q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); + if (notifyPrevHead != null && isRunnable(notifyPrevHead)) + notifyPrevHead.onChangeHeadStatus(this, NOT_RUNNABLE); + } + + public enum LockMode + { + /** + * Invalid as a parameter to lock methods, but represents the unlocked state + */ + UNLOCKED, + + /** + * If we're sequenced, remove ourselves from the relevant queue so that the next task can queue itself up. + */ + RELEASE_QUEUE, + + /** + * Hold onto our queue position (which we expect to be the head position, as we're queued to execute). + * This is used exclusively for INCR tasks that may hold onto their TxnId for multiple rounds of execution, + * and prevents later tasks from being scheduled when they will be unable to obtain the lock. + */ + HOLD_QUEUE, + + /** + * Skip all queue accounting (sequenced or unsequenced). This mode is used by optimistic referencing via + * tryLockCaches. + */ + UNQUEUED + } + + /** + * On lock we remove ourselves from the priority/fifo queues and notify the new head + */ + public final V lockExclusive(AccordTask owner, LockMode lockMode) + { + Invariants.require(!isLocked()); + Invariants.require(isRunnable(owner) || owner.isUnsequenced(this)); + + if (queue == owner) + { + Invariants.require(lockMode != UNLOCKED); + } + else if (queue == null) + { + queue = owner; + switch (lockMode) + { + default: throw UnhandledEnum.unknown(lockMode); + case UNLOCKED: throw UnhandledEnum.invalid(UNLOCKED); + case HOLD_QUEUE: throw UnhandledEnum.invalid(HOLD_QUEUE, "Must already be head of the queue"); + case RELEASE_QUEUE: + Invariants.require(owner.isUnsequenced(this) && owner.isCacheQueued(), "Must already be head of the queue"); + --unsequenced; + case UNQUEUED: + } + } + else + { + Queue q = ensureQueue(); + switch (lockMode) + { + default: throw UnhandledEnum.unknown(lockMode); + case UNLOCKED: throw UnhandledEnum.invalid(UNLOCKED); + case HOLD_QUEUE: + Invariants.require(!owner.isUnsequenced(this)); + if (q.hasFifo()) Invariants.require(q.peekFifo() == owner); + else + { + boolean wasHead = q.removeIfPriorityHead(owner); + Invariants.require(wasHead); + q.addFifo(owner); + } + q.lock(owner); + break; + case RELEASE_QUEUE: + if (owner.isUnsequenced(this)) releaseUnsequenced(q, owner); + else + { + boolean wasHead = q.removeIfHead(owner); + Invariants.require(wasHead); + if (isLoaded()) + { + unsequenced += q.drainUnsequenced(AccordTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); + onChangedHead(q, q.peek(), null); + } + if (maybeUnwrap(q, owner)) + break; + } + case UNQUEUED: + q.lock(owner); + } + } + + status |= lockMode.ordinal() << LOCKED_SHIFT; + return getExclusive(); + } + + private void releaseUnsequenced(Queue q, AccordTask release) + { + Invariants.require(release.isCacheQueued()); + if (--unsequenced == 0) + { + AccordTask head = q.peek(); + if (head != null && head.holdsLocksBetweenRuns()) + onChangedHead(q, head, null); + } + } + + final boolean hasFifoOrLocked() + { + if (isLocked()) + return true; + + if (queue == null) + return false; + + if (queue instanceof AccordTask) + return ((AccordTask) queue).isCacheQueuedFifo(); + + return ((Queue)queue).hasFifo(); + } + + final void unlink() { remove(); } - boolean isUnqueued() + final boolean isUnqueued() { return isFree(); } - public K key() + public final K key() { return key; } - public int references() + public final int references() { return references; } - public int increment() + public final int increment() { return referencesUpdater.incrementAndGet(this); } - public int decrement() + public final int decrement() { return referencesUpdater.decrementAndGet(this); } - boolean isLoaded() + final boolean isLocked() + { + return (status & LOCKED_MASK) != 0; + } + + final boolean isLockedHoldingQueue() + { + return (status & LOCKED_MASK) == LOCKED_HOLDING_QUEUE; + } + + final boolean isLoaded() { return (status & IS_LOADED) != 0; } - boolean isModified() + final boolean isModified() { return (status & IS_NOT_EVICTED) >= MODIFIED.ordinal(); } - boolean isNested() + final boolean isNested() { Invariants.require(isLoaded()); return (status & IS_NESTED) != 0; } - boolean isShrunk() + final boolean isShrunk() { return (status & SHRUNK) != 0; } - public boolean is(Status status) + public final boolean is(Status status) { return (this.status & STATUS_MASK) == status.ordinal(); } - boolean isLoadingOrWaiting() + final boolean isLoading() { return (status & IS_LOADING_OR_WAITING_MASK) == IS_LOADING_OR_WAITING; } - boolean isSavingOrWaiting() + final boolean isSavingOrWaiting() { return (status & IS_SAVING_OR_WAITING_MASK) == IS_SAVING_OR_WAITING; } - public boolean isComplete() + public final boolean isComplete() { return !is(LOADING) && !is(SAVING); } - int noEvictGeneration() + final int noEvictGeneration() { Invariants.require(isNoEvict()); - return (status >>> 8) & 0xffff; + return (status >>> GENERATION_SHIFT) & GENERATION_MASK; } - int noEvictMaxAge() + final int noEvictMaxAge() { Invariants.require(isNoEvict()); - return status >>> 24; + return status >>> AGE_SHIFT; } - boolean isNoEvict() + final boolean isNoEvict() { return (status & NO_EVICT) != 0; } - int sizeOnHeap() + final int sizeOnHeap() { return sizeOnHeap; } - void updateSize(AccordCache.Type parent) + final void updateSize(AccordCache.Type parent) { // TODO (expected): we aren't weighing the keys int newSizeOnHeap = Ints.saturatedCast(EMPTY_SIZE + estimateOnHeapSize(parent.adapter())); @@ -256,7 +1203,7 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode sizeOnHeap = newSizeOnHeap; } - void initSize(AccordCache.Type parent) + final void initSize(AccordCache.Type parent) { // TODO (expected): we aren't weighing the keys sizeOnHeap = Ints.saturatedCast(EMPTY_SIZE); @@ -265,7 +1212,7 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode } @Override - public String toString() + public final String toString() { return "Node{" + status() + ", key=" + key() + @@ -273,7 +1220,7 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode "}@" + Integer.toHexString(System.identityHashCode(this)); } - public Status status() + public final Status status() { return Status.VALUES[(status & STATUS_MASK)]; } @@ -290,42 +1237,36 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode status |= newStatus.ordinal(); } - public void initialize(V value) + public final void initialize(V value) { Invariants.require(state == null); setStatus(LOADED); state = value; } - public void readyToLoad() + public final void readyToLoad() { Invariants.require(state == null); setStatus(WAITING_TO_LOAD); - state = new WaitingToLoad(); } - public void markNoEvict(int generation, int maxAge) + public final void markNoEvict(int generation, int maxAge) { - Invariants.require((maxAge & ~0xff) == 0); - Invariants.require((generation & ~0xffff) == 0); + Invariants.require((maxAge & ~AGE_MASK) == 0); + Invariants.require((generation & ~GENERATION_MASK) == 0); status |= NO_EVICT; - status |= generation << 8; - status |= maxAge << 24; + status |= generation << GENERATION_SHIFT; + status |= maxAge << AGE_SHIFT; } - public LoadingOrWaiting loadingOrWaiting() - { - return (LoadingOrWaiting)state; - } - - void notifyListeners(BiConsumer, AccordCacheEntry> notify) + final void notifyListeners(BiConsumer, AccordCacheEntry> notify) { owner.notifyListeners(notify, this); } - public interface LoadExecutor + interface LoadExecutor { - Cancellable load(P1 p1, P2 p2, AccordCacheEntry entry); + IOTask load(P1 p1, P2 p2, AccordCacheEntry entry); } // functions as both an identity object, and a register of listeners @@ -355,32 +1296,31 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode } } - public interface SaveExecutor + interface SaveExecutor { - Cancellable save(AccordCacheEntry saving, UniqueSave identity, Runnable save); + Cancellable save(AccordCacheEntry saving, UniqueSave identity, Runnable save); } - public Loading load(LoadExecutor loadExecutor, P1 p1, P2 p2) + final Loading load(LoadExecutor loadExecutor, P1 p1, P2 p2) { Invariants.require(is(WAITING_TO_LOAD), "%s", this); - WaitingToLoad cur = (WaitingToLoad)state; - Loading loading = cur.load(loadExecutor.load(p1, p2, this)); + Loading loading = new Loading(loadExecutor.load(p1, p2, this)); setStatus(LOADING); state = loading; return loading; } - public Loading testLoad() + public final Loading testLoad() { Invariants.require(is(WAITING_TO_LOAD)); - Loading loading = ((WaitingToLoad)state).load(() -> {}); + Loading loading = new Loading(null); setStatus(LOADING); state = loading; return loading; } - public Loading loading() + public final Loading loading() { Invariants.require(is(LOADING), "%s", this); return (Loading) state; @@ -388,7 +1328,7 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode // must own the cache's lock when invoked. this is true of most methods in the class, // but this one is less obvious so named as to draw attention - public V getExclusive() + public final V getExclusive() { Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); Invariants.require(isLoaded(), "%s", this); @@ -399,14 +1339,19 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode updateSize(parent); } - return (V)unwrap(); + return (V) maybeUnwrap(); } - public Object getOrShrunkExclusive() + public final void releaseExclusive(S safeState, AccordTask task) + { + owner.release(safeState, task); + } + + public final Object getOrShrunkExclusive() { Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); Invariants.require(isLoaded(), "%s", this); - return unwrap(); + return maybeUnwrap(); } public V tryGetExclusive() @@ -414,10 +1359,10 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); if (!isLoaded() || isShrunk()) return null; - return (V)unwrap(); + return (V) maybeUnwrap(); } - private Object unwrap() + private Object maybeUnwrap() { return isNested() ? ((Nested)state).state : state; } @@ -475,7 +1420,7 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode if (isShrunk() || state == null) return Shrink.EVICT; - V cur = (V)unwrap(); + V cur = (V) maybeUnwrap(); Shrink shrink = adapter.decideFullShrink(key, cur); if (shrink == Shrink.PERFORM_WITHOUT_LOCK) return Shrink.PERFORM_WITHOUT_LOCK; @@ -489,12 +1434,12 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode V tryGetFull() { - return isShrunk() ? null : (V)unwrap(); + return isShrunk() ? null : (V) maybeUnwrap(); } Object tryGetShrunk() { - return isShrunk() ? unwrap() : null; + return isShrunk() ? maybeUnwrap() : null; } boolean isNull() @@ -583,7 +1528,7 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode return (SavingOrWaitingToSave) state; } - public AccordCacheEntry evicted() + public AccordCacheEntry evicted() { if (isNoEvict()) setStatusUnsafe(EVICTED); @@ -597,12 +1542,12 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode return ((FailedToSave)state).cause; } - void tryApplyShrink(Object cur, Object upd, IntrusiveLinkedList> queue) + void tryApplyShrink(Object cur, Object upd, IntrusiveLinkedList> queue) { if (references() > 0 || !isUnqueued()) return; - if (isLoaded() && unwrap() == cur && upd != cur && upd != null) + if (isLoaded() && maybeUnwrap() == cur && upd != cur && upd != null) applyShrink(owner.parent(), cur, upd); queue.addLast(this); } @@ -632,74 +1577,18 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode private long estimateOnHeapSize(Adapter adapter) { - Object current = unwrap(); + Object current = maybeUnwrap(); if (current == null) return 0; else if (isShrunk()) return adapter.estimateShrunkHeapSize(current); return adapter.estimateHeapSize((V)current); } - public static abstract class LoadingOrWaiting + public static class Loading { - Collection> waiters; + final IOTask loading; - public LoadingOrWaiting() + Loading(IOTask loading) { - } - - public LoadingOrWaiting(Collection> waiters) - { - this.waiters = waiters; - } - - public Collection> waiters() - { - return waiters != null ? waiters : Collections.emptyList(); - } - - public BufferList> copyWaiters() - { - BufferList> list = new BufferList<>(); - if (waiters != null) - list.addAll(waiters); - return list; - } - - public void add(AccordTask waiter) - { - if (waiters == null) - waiters = new ArrayList<>(); - waiters.add(waiter); - } - - public void remove(AccordTask waiter) - { - if (waiters != null) - { - waiters.remove(waiter); - if (waiters.isEmpty()) - waiters = null; - } - } - } - - static class WaitingToLoad extends LoadingOrWaiting - { - public Loading load(Cancellable loading) - { - Invariants.paranoid(waiters == null || !waiters.isEmpty()); - Loading result = new Loading(waiters, loading); - waiters = Collections.emptyList(); - return result; - } - } - - static class Loading extends LoadingOrWaiting - { - public final Cancellable loading; - - public Loading(Collection> waiters, Cancellable loading) - { - super(waiters); this.loading = loading; } } @@ -755,9 +1644,9 @@ public class AccordCacheEntry extends IntrusiveLinkedListNode } } - public static AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) + public static & AccordSafeState> AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) { - AccordCacheEntry node = new AccordCacheEntry<>(key, owner); + AccordCacheEntry node = new AccordCacheEntry<>(key, owner); node.readyToLoad(); return node; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index 60205bf890..aeae9844e5 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -56,7 +56,6 @@ import accord.impl.progresslog.TxnState; import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; -import accord.local.CommandSummaries; import accord.local.ExecutionContext; import accord.local.ExecutionContext.Empty; import accord.local.MaxConflicts; @@ -178,13 +177,16 @@ public class AccordCommandStore extends CommandStore @Override public AccordSafeCommand acquireIfLoaded(TxnId txnId) { - return commands().acquireIfLoaded(txnId); + // note: we must return false if the entry is locked to enforce ordering. + // note importantly that this is also coupled to the safety of synchronously releasing ExclusiveExecutor.owner, + // rather than waiting until the (potentially asynchronous) cleanup of the task completes + return commands().acquireIfLoadedAndPermitted(txnId); } @Override public AccordSafeCommandsForKey acquireIfLoaded(RoutingKey key) { - return commandsForKeys().acquireIfLoaded(key); + return commandsForKeys().acquireIfLoadedAndPermitted(key); } @Override @@ -299,7 +301,7 @@ public class AccordCommandStore extends CommandStore void tryPreSetup(AccordTask task) { if (inStore() && current != null) - task.presetup(current.task); + task.preSetup(current.task); } public final TableId tableId() @@ -432,13 +434,19 @@ public class AccordCommandStore extends CommandStore return taskExecutor().tryExecuteImmediately(run); } - public AccordSafeCommandStore begin(AccordTask operation, @Nullable CommandSummaries commandsForRanges) + public AccordSafeCommandStore begin(AccordSafeCommandStore safeStore) { require(current == null); - current = AccordSafeCommandStore.create(operation, commandsForRanges, this); + current = safeStore; return current; } + public void complete(AccordSafeCommandStore store) + { + require(current == store); + current = null; + } + public boolean hasSafeStore() { return current != null; @@ -454,19 +462,6 @@ public class AccordCommandStore extends CommandStore return progressLog; } - public void complete(AccordSafeCommandStore store) - { - require(current == store); - current.postExecute(); - current = null; - } - - public void abort(AccordSafeCommandStore store) - { - Invariants.require(store == current); - current = null; - } - @Override public void shutdown() { @@ -650,7 +645,7 @@ public class AccordCommandStore extends CommandStore public Ready() { super(1); } @Override public void run() { decrement(); } - void maybeFlush(ExclusiveCaches caches, AccordCacheEntry e) + void maybeFlush(ExclusiveCaches caches, AccordCacheEntry e) { if (e.isModified()) { @@ -665,7 +660,7 @@ public class AccordCommandStore extends CommandStore { if (ranges == null) { - for (AccordCacheEntry e : caches.commandsForKeys()) + for (AccordCacheEntry e : caches.commandsForKeys()) ready.maybeFlush(caches, e); } else @@ -717,7 +712,7 @@ public class AccordCommandStore extends CommandStore if (!maybeShouldReplay(txnId)) return AsyncChains.success(null); - return commandStore.chain(ExecutionContext.contextFor(txnId, "Replay"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Replay"), safeStore -> { Replay replay = shouldReplay(txnId, safeStore.unsafeGet(txnId).current().participants()); if (replay == Replay.NONE) return null; diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index f82215919e..14accb0fb8 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -55,7 +55,6 @@ import org.apache.cassandra.journal.Descriptor; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore.DurablyAppliedTo; import org.apache.cassandra.service.accord.AccordExecutor.AccordExecutorFactory; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; @@ -73,7 +72,6 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu private final int mask; private long cacheSize, workingSetSize; - private int maxQueuedLoads, maxQueuedRangeLoads; private boolean shrinkingOn; AccordCommandStores(NodeCommandStoreService node, Agent agent, DataStore store, RandomSource random, @@ -87,12 +85,9 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu cacheSize = DatabaseDescriptor.getAccordCacheSizeInMiB() << 20; workingSetSize = DatabaseDescriptor.getAccordWorkingSetSizeInMiB() << 20; - AccordConfig config = DatabaseDescriptor.getAccord(); - maxQueuedLoads = maxQueuedLoads(config); - maxQueuedRangeLoads = maxQueuedRangeLoads(config); shrinkingOn = DatabaseDescriptor.getAccordCacheShrinkingOn(); refreshCapacities(); - ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(() -> { + ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(() -> { for (AccordExecutor executor : executors) { executor.executeDirectlyWithLock(() -> { @@ -173,13 +168,6 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu refreshCapacities(); } - public synchronized void setMaxQueuedLoads(int total, int range) - { - maxQueuedLoads = total; - maxQueuedRangeLoads = range; - refreshCapacities(); - } - public synchronized void setShrinking(boolean on) { shrinkingOn = on; @@ -213,14 +201,11 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu { long capacityPerExecutor = cacheSize / executors.length; long workingSetPerExecutor = workingSetSize < 0 ? Long.MAX_VALUE : workingSetSize / executors.length; - int maxLoadsPerExecutor = Math.max(1, (maxQueuedLoads + executors.length - 1) / executors.length); - int maxRangeLoadsPerExecutor = Math.max(1, (maxQueuedRangeLoads + executors.length - 1) / executors.length); for (AccordExecutor executor : executors) { executor.executeDirectlyWithLock(() -> { executor.setCapacity(capacityPerExecutor); executor.setWorkingSetSize(workingSetPerExecutor); - executor.setMaxQueuedLoads(maxLoadsPerExecutor, maxRangeLoadsPerExecutor); executor.cacheExclusive().setShrinkingOn(shrinkingOn); }); } @@ -341,21 +326,4 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu return Math.max(1, config.queue_shard_count.or(DatabaseDescriptor.getAvailableProcessors() / 8)); } } - - private static int threads(AccordConfig config) - { - return config.queue_thread_count.or(2 * FBUtilities.getAvailableProcessors()); - } - - public static int maxQueuedLoads(AccordConfig config) - { - return config.max_queued_loads.or(FBUtilities.getAvailableProcessors()); - } - - public static int maxQueuedRangeLoads(AccordConfig config) - { - return config.max_queued_range_loads.or(maxQueuedLoads(config) / 4); - } - - } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java index c600178513..67a4d815c9 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java @@ -26,6 +26,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.LockSupport; @@ -46,6 +47,7 @@ import accord.api.RoutingKey; import accord.impl.AbstractAsyncExecutor; import accord.local.Command; import accord.local.ExecutionContext; +import accord.local.ExecutionContext.ExecutionSequence; import accord.local.cfk.CommandsForKey; import accord.messages.Accept; import accord.messages.Commit; @@ -96,17 +98,20 @@ import org.apache.cassandra.service.accord.AccordCacheEntry.UniqueSave; import org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup; import org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup; import org.apache.cassandra.service.accord.AccordExecutor.Task.GroupKind; +import org.apache.cassandra.service.accord.AccordExecutor.Task.State; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExclusiveExecutor; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.WithResources; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.concurrent.Future; +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY; +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY_ATOMIC; import static accord.primitives.Routable.Domain.Range; -import static accord.utils.Invariants.createIllegalState; import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.ORIG_HLC_FIFO; import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER; import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; @@ -131,14 +136,17 @@ import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGrou import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_SCAN; import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.SAVE; import static org.apache.cassandra.service.accord.AccordExecutor.Task.MAX_TRANCHE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.ASSIGNED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.INITIALIZED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.EXECUTED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FAILED_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_REQUIRED; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING_OR_EXECUTED; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_REQUIRED; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_SCAN_RANGES; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -158,37 +166,39 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor= 0 && FLOW_WIDTH_SHIFT >= 0); switch (BALANCING_MODEL) { default: throw new UnhandledEnum(BALANCING_MODEL); case PRIORITY_ONLY: case BLENDED_PRIORITY_PHASE_FAIR: - case BLENDED_PRIORITY_PHASE_BUDGET_FAIR: - case PRIORITY_BUDGET: BALANCE_BY_POSITION = true; break; case PHASE_ONLY: case PHASE_FAIR: - case PHASE_BUDGET: - case PHASE_BUDGET_FAIR: BALANCE_BY_POSITION = false; } { + // TODO (required): pick default max loads/saves/range loads based on number of threads long global = COUNTER_MASKS, exclusive = COUNTER_MASKS; - global ^= (0x7f ^ 1) << RANGE_SCAN.ordinal(); + global ^= (0x7fL ^ 1) << (RANGE_SCAN.ordinal() * 8); if (config.queue_active_limits != null) { long[] limits = parseEnumParams(config.queue_active_limits, "queue_active_limits"); @@ -199,30 +209,6 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor 1) - { - mins ^= (min ^ 1) << (RANGE_SCAN.ordinal() * 8); // set the default RANGE_SCAN budget to 1 - mins ^= (min ^ 2) << (RANGE_LOAD.ordinal() * 8); // set the default RANGE_LOAD budget to 2 - } - global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), mins, limits[0]); - } - if (limits[1] != 0) - exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), minCounterValue(limits[1], 0) * COUNTER_LOWBITS, limits[1]); - } - GLOBAL_QUEUE_BUDGETS = global; - EXCLUSIVE_QUEUE_BUDGETS = exclusive; - } } public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); @@ -240,15 +226,14 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor> scanningRanges = new StandaloneTaskQueue<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning - private final StandaloneTaskQueue> waitingToLoadRangeTxns = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_TO_LOAD)); - private final StandaloneTaskQueue> waitingToLoad = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD)); - private final StandaloneTaskQueue> loading = new StandaloneTaskQueue<>(LOADING); - private final RunnableTaskQueue runnable = new RunnableTaskQueue<>(); + final StandaloneTaskQueue> scanningRanges = new StandaloneTaskQueue<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning + final StandaloneTaskQueue> loading = new StandaloneTaskQueue<>(TinyEnumSet.encode(LOADING_REQUIRED, LOADING_OPTIONAL)); + final StandaloneTaskQueue> waitingOnCacheQueues = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL)); + final RunnableTaskQueue runnable = new RunnableTaskQueue<>(); private final Tranches tranches = new Tranches(this); @@ -610,7 +583,6 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor waitingForQuiescence; @@ -662,16 +634,17 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor= maxWorkingCapacityInBytes || !runnable.hasWaitingToRun()) + { + AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); + hasPausedLoading = true; + runnable.stop(RANGE_SCAN.ordinal()); + runnable.stop(LOAD.ordinal()); + runnable.stop(RANGE_LOAD.ordinal()); } } @@ -808,142 +798,6 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor> queue = waitingToLoadRangeTxns.isEmpty() || activeRangeLoads >= maxQueuedRangeLoads ? waitingToLoad : waitingToLoadRangeTxns; - AccordTask next = queue.peek(); - if (next == null) - return; - - if (hasPausedLoading || cache.weightedSize() >= maxWorkingCapacityInBytes) - { - // we have too much in memory already, and we have work waiting to run, so let that complete before queueing more - if (!loading.isEmpty() || runnable.hasWaiting()) - { - AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); - hasPausedLoading = true; - return; - } - } - - switch (next.state()) - { - default: - { - failExclusive(next, createIllegalState("Unexpected state: " + next.toDescription())); - break; - } - case WAITING_TO_SCAN_RANGES: - if (activeRangeLoads >= maxQueuedRangeLoads) - { - parkRangeLoad(next); - } - else - { - ++activeRangeLoads; - ++activeLoads; - next.rangeScanner().start(this); - updateQueue(next); - } - break; - - case WAITING_TO_LOAD: - while (true) - { - AccordCacheEntry load = next.peekWaitingToLoad(); - boolean isForRange = isForRange(next, load); - if (isForRange && activeRangeLoads >= maxQueuedRangeLoads) - { - parkRangeLoad(next); - continue outer; - } - - Invariants.require(load != null); - ++activeLoads; - if (isForRange) - ++activeRangeLoads; - - for (AccordTask task : cache.load(this, next, isForRange, load)) - { - if (task == next) continue; - if (task.onLoading(load)) - updateQueue(task); - } - Object prev = next.pollWaitingToLoad(); - Invariants.require(prev == load); - if (next.peekWaitingToLoad() == null) - break; - - Invariants.require(next.state() == WAITING_TO_LOAD, "Invalid state: %s", next); - if (activeLoads >= maxQueuedLoads) - return; - } - Invariants.require(next.state().compareTo(LOADING) >= 0, "Invalid state: %s", next); - updateQueue(next); - } - } - } - - private boolean isForRange(AccordTask task, AccordCacheEntry load) - { - boolean isForRangeTxn = task.isRange(); - if (!isForRangeTxn) - return false; - - for (AccordTask t : load.loadingOrWaiting().waiters()) - { - if (!t.isRange()) - return false; - } - return true; - } - - private void parkRangeLoad(AccordTask task) - { - if (task.queued() != waitingToLoadRangeTxns) - { - task.unqueue(); - waitingToLoadRangeTxns.enqueue(task); - } - } - - private void updateQueue(AccordTask task) - { - task.unqueueIfQueued(); - switch (task.state()) - { - default: throw new AssertionError("Unexpected state: " + task.toDescription()); - case WAITING_TO_SCAN_RANGES: - case WAITING_TO_LOAD: - waitingToLoad.enqueue(task); - break; - case SCANNING_RANGES: - scanningRanges.enqueue(task); - break; - case LOADING: - loading.enqueue(task); - break; - case WAITING_TO_RUN: - waitingToRun(task); - break; - } - } - - private void waitingToRun(AccordTask task) - { - task.onWaitingToRun(); - task.commandStore.exclusiveExecutor.enqueue(task); - } - - private void waitingToRun(Task task, @Nullable ExclusiveExecutor queue) - { - task.onWaitingToRun(); - if (queue == null) runnable.enqueue(task); - else queue.enqueue(task); - } - public ExclusiveExecutor executor() { return new ExclusiveExecutor(this); @@ -968,13 +822,14 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor Cancellable load(AccordTask parent, Boolean isForRange, AccordCacheEntry entry) + public LoadRunnable load(AccordTask parent, Boolean isForRange, AccordCacheEntry entry) { - return submitPlainExclusive(parent, newLoad(entry, isForRange)); + LoadRunnable load = newLoad(entry, isForRange); + return submitPlainExclusive(parent, load); } @Override - public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) + public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) { return submitPlainExclusive(null, new SaveRunnable(entry, identity, save)); } @@ -1003,15 +858,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor void submit(AccordTask operation) { - submit(AccordExecutor::submitExclusive, i -> i, operation); - } - - void submitExclusive(AccordTask task) - { - registerExclusive(task); - task.setupExclusive(); - updateQueue(task); - enqueueLoadsExclusive(); + submit((self, task) -> task.submitExclusive(self), i -> i, operation); } public void submitExclusive(Runnable runnable) @@ -1028,7 +875,10 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor T submitPlainExclusive(Task parent, T task) { Invariants.require(isOwningThread()); + task.setStateExclusive(WAITING_TO_RUN); if (parent == null) registerExclusive(task); else registerConsequenceExclusive(parent, task); - task.onWaitingToRun(); - runnable.enqueue(task); + task.onLoaded(); + runnable.enqueue(task, true); return task; } @@ -1052,10 +903,10 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor task) { - AccordTask.State state = task.state(); + State state = task.state(); switch (state) { default: throw new UnhandledEnum(state); case SCANNING_RANGES: - case LOADING: - case WAITING_TO_LOAD: - case WAITING_TO_SCAN_RANGES: + case LOADING_REQUIRED: + case LOADING_OPTIONAL: + case WAITING_ON_REQUIRED: + case WAITING_ON_OPTIONAL: case WAITING_TO_RUN: - task.unqueueIfQueued(); - try { task.cancelExclusive(); } - finally { completeTaskExclusive(task); } - break; - - case INITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase - case ASSIGNED: + if (!task.hasIncrementalStarted()) + { + task.unqueueIfQueued(); + try { task.cancelExclusive(); } + finally { cleanupTaskExclusive(task, false); } + break; + } + case UNINITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase case RUNNING: - case PERSISTING: - case FINISHED: + case INCOMPLETE: + case EXECUTED: case CANCELLED: - case FAILED: + case FAILED_TO_LOAD: // cannot safely cancel } } void onScannedRangesExclusive(AccordTask task, Throwable fail) { - --activeLoads; - --activeRangeLoads; // the task may have already been cancelled, in which case we don't need to fail it - if (!task.state().isExecuted()) - { - if (fail != null) - { - failExclusive(task, fail); - } - else - { - task.rangeScanner().scannedExclusive(); - updateQueue(task); - } - } - enqueueLoadsExclusive(); + if (task.state().isExecuted()) + return; + + if (fail != null) failExclusive(task, fail, FAILED_TO_LOAD); + else task.rangeScanner().scannedExclusive(); } - private void failExclusive(AccordTask task, Throwable fail) + private void failExclusive(AccordTask task, Throwable fail, State newState) { if (task.state().isExecuted()) return; - try { task.failExclusive(fail); } + try { task.failExclusive(fail, newState); } catch (Throwable t) { agent.onException(t); } finally { task.unqueueIfQueued(); - completeTaskExclusive(task); + cleanupTaskExclusive(task, false); } } - private void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) + private void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) { cache.saved(state, identity, fail); } - private void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail, boolean isForRange) + private void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail) { - --activeLoads; - if (isForRange) - --activeRangeLoads; + if (loaded.status() == EVICTED) + return; - if (loaded.status() != EVICTED) + try (ArrayBuffers.BufferList> tasks = loaded.drainWaitingToLoad()) { - try (ArrayBuffers.BufferList> tasks = loaded.loading().copyWaiters()) + if (fail != null) { - if (fail != null) - { - for (AccordTask task : tasks) - failExclusive(task, fail); - cache.failedToLoad(loaded); - } - else - { - cache.loaded(loaded, value); - for (AccordTask task : tasks) - { - if (task.onLoad(loaded)) - { - Invariants.require(task.queued() == loading); - task.unqueue(); - waitingToRun(task); - } - } - } + for (AccordTask task : tasks) + failExclusive(task, fail, FAILED_TO_LOAD); + cache.failedToLoad(loaded); + } + else + { + cache.loaded(loaded, value); + for (AccordTask task : tasks) + task.onLoadOneExclusive(loaded); } } - enqueueLoadsExclusive(); + maybePauseLoading(); } private Task inherit() @@ -1264,7 +1100,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor= 1, "Must permit at least one load"); - Invariants.requireArgument(range >= 1, "Must permit at least one range load"); - maxQueuedLoads = total; - maxQueuedRangeLoads = range; - } - @Override public long capacity() { @@ -1332,24 +1159,39 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor= 0; + return this.compareTo(EXECUTED) >= 0; } - boolean isComplete() + boolean hasStarted() { - return this.compareTo(FINISHED) >= 0; + return this.compareTo(RUNNING) >= 0; } static State forOrdinal(int ordinal) @@ -1384,27 +1231,25 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor next < prev ? prev + 1 : next); ExclusiveGroup group = ExclusiveGroup.OTHER; TxnId txnId = context.primaryTxnId(); if (txnId != null) @@ -1511,7 +1373,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor queued() + { + return queued; + } + + final void unqueueIfQueued() + { + if (queued != null) + { + queued.unqueue(this); + queued = null; + } + } + + final void unqueue(TaskQueue expected) + { + Invariants.require(queued == expected, "%s != %s", queued, expected); + queued.unqueue(this); + queued = null; + } + + final void unsetQueue(TaskQueue expected) + { + Invariants.require(queued == expected, "%s != %s", queued, expected); + queued = null; + } + + final void setQueue(TaskQueue queue) + { + Invariants.require(queued == null); + Invariants.require(isCompatible(queue)); + queued = queue; } final void onRunning() @@ -1609,71 +1562,31 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor describeState() + { + State state = state(); + if (state == RUNNING || state == EXECUTED) { - Task next = cur.next; - cur.next = prev; - prev = cur; - cur = next; + RunState runState = runState(); + if (runState == RunState.NONE) + return state; + return runState; } - return prev; - } - - public DebuggableTask debuggable() { return null; } - - abstract String toDescription(); - - abstract void submitExclusive(AccordExecutor owner); - - /** - * Prepare to run while holding the state cache lock - */ - abstract protected void preRunExclusive(); - - /** - * Run the command; the state cache lock may or may not be held depending on the executor implementation - */ - protected abstract void run(); - - /** - * Fail the command; the state cache lock may or may not be held depending on the executor implementation - */ - abstract protected void fail(Throwable fail); - - abstract protected boolean isNewWork(); - - /** - * Cleanup the command while holding the state cache lock - */ - protected void cleanupExclusive(AccordExecutor executor) - { - executor.unregisterExclusive(this); - cleanupAt = nanoTime(); - if (runningAt != 0) - { - if (waitingToRunAt == 0) - waitingToRunAt = runningAt; - executor.elapsedWaitingToRun.increment(runningAt - waitingToRunAt, runningAt); - executor.elapsedPreparingToRun.increment(waitingToRunAt - createdAt, runningAt); - executor.elapsedRunning.increment(cleanupAt - runningAt, cleanupAt); - executor.elapsed.increment(cleanupAt - createdAt, cleanupAt); - } - if (DEBUG_EXECUTION) DebugTask.get(this).onCompleted(executor.debug); - } - - void cancelExclusive(AccordExecutor owner) {} - - public final State state() - { return State.forOrdinal(stateOrdinal()); } @@ -1687,14 +1600,24 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; } - @Nullable - final TaskQueue queued() - { - return queued; - } - - final void unqueueIfQueued() - { - if (queued != null) - unqueue(); - } - - final void unqueue() - { - Invariants.require(queued != null); - queued.unqueue(this); - queued = null; - } - - final void unsetQueue(TaskQueue queue) - { - Invariants.require(queued == queue); - queued = null; - } - - final void setQueue(TaskQueue queue) - { - Invariants.require(queued == null); - Invariants.require(isCompatible(queue)); - queued = queue; - } - private boolean isCompatible(TaskQueue queue) { int self = stateOrdinal(); return TinyEnumSet.contains(queue.states, self); } - private static int init(GlobalGroup global, ExclusiveGroup exclusive, State state) + final boolean isSync() { - return global.bits | exclusive.bits | state.ordinal(); + return 0 == (info & NONSYNC_BIT); } - final void setTranche(int tranche) + final boolean isNonSync() { - Invariants.require(tranche <= MAX_TRANCHE); - info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; + return !isSync(); } - final void setInheritedTranche(int tranche) + final void setNonSyncExclusive() { - Invariants.require(tranche <= MAX_TRANCHE); - info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; + info |= NONSYNC_BIT; + } + + final boolean isIncremental() + { + return 0 != (info & INCREMENTAL_MASK); + } + + final void setIncrementalExclusive() + { + info |= INCREMENTAL | NONSYNC_BIT; + } + + final boolean hasIncrementalStarted() + { + return (info & INCREMENTAL_MASK) >= INCREMENTAL_STARTED; + } + + final void setIncrementalStartedExclusive() + { + Invariants.require(isIncremental()); + if (!isIncrementalFinishing()) + info = (info & ~INCREMENTAL_MASK) | INCREMENTAL_STARTED; + } + + final boolean isIncrementalFinishing() + { + return (info & INCREMENTAL_MASK) >= INCREMENTAL_FINISHING; + } + + final void setIncrementalFinishingExclusive() + { + Invariants.require(isIncremental()); + info |= INCREMENTAL_FINISHING; + } + + final void setSequencedExclusive(ExecutionSequence sequence) + { + Invariants.require(isUnsequenced()); + info |= sequence.ordinal() << SEQUENCED_SHIFT; + } + + final boolean isUnsequenced() + { + return (info & SEQUENCED_MASK) == 0; + } + + final boolean isSequencedByPriority() + { + return (info & SEQUENCED_MASK) == SEQUENCED_PRIORITY; + } + + final boolean isSequencedByPriorityAtomic() + { + return (info & SEQUENCED_MASK) >= SEQUENCED_ATOMIC; + } + + final boolean isCacheQueuedFifo() + { + return (info & SEQUENCED_MASK) == SEQUENCED_ATOMIC_AND_QUEUED; + } + + final boolean isCacheQueued() + { + return 0 != (info & CACHE_QUEUED_BIT); + } + + // supersedes priority, in whichever order they're called + final void setCacheQueuedFifoExclusive() + { + Invariants.require(isSequencedByPriorityAtomic()); + info |= SEQUENCED_ATOMIC_AND_QUEUED | CACHE_QUEUED_BIT; + } + + final void setCacheQueuedExclusive() + { + info |= CACHE_QUEUED_BIT; } final int tranche() @@ -1789,10 +1760,51 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor>> TRANCHE_SHIFT; } + final void setTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; + } + + final void setInheritedWithTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; + } + final boolean hasInherited() { return (info & HAS_INHERITED_BIT) != 0; } + + final void setInheritedRangeScan() + { + info = info | HAS_INHERITED_RANGE_SCAN_BIT; + } + + final boolean hasInheritedRangeScan() + { + return (info & HAS_INHERITED_RANGE_SCAN_BIT) != 0; + } + + static int init(GlobalGroup global, ExclusiveGroup exclusive) + { + return (global.ordinal() << GLOBAL_GROUP_SHIFT) | (exclusive.ordinal() << EXCLUSIVE_GROUP_SHIFT); + } + + static Task reverse(Task unqueued) + { + Task prev = null; + Task cur = unqueued; + while (cur != null) + { + Task next = cur.next; + cur.next = prev; + prev = cur; + cur = next; + } + return prev; + } } // run the task even on a stopped commandStore @@ -1811,7 +1823,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor) task).executionContext())); + task.reportFailure(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).executionContext())); else task.run(); - // NOTE: cannot safely release owner here, in case an immediate-execution runs before we can release our references and store their changes to the cache + + // NOTE: we can ONLY safely release owner here due to AccordCacheEntry locking, which remains in place until AccordTask.releaseResourcesExclusive + // this also relies on AccordSafeCommandStore$ExclusiveCaches.acquireIfLoaded returning false when the entry is locked + owner = null; } private boolean reject(Task task) @@ -1928,45 +1943,49 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor extends TaskQueue @@ -2218,7 +2243,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor[] queues; final long[] positions; final byte groupShift; - final long baseBudget, limits; + final long limits; + /** sets overflow bits for a queue that has been stopped */ + long stopped; /** sets overflow bits for each counter when it needs its position updated */ long dirty; /** sets overflow bits for each counter when there's associated work */ long hasWork; /** Stores recent dequeue counts for up to 8 sub queues. */ long dispatches; + // TODO (required): increment arrivals based on internal queue for ExclusiveExecutors + // also: experiment with decaying on arrival schedule rather than poll schedule, since this should respond to work growth more accurately /** Stores recent enqueue counts for up to 8 sub queues. */ long arrivals; /** Stores currently-active counts for up to 8 sub queues. We can use this to impose limits on specific queues. */ long active; - /** Stores a biased budget for each task type, so that we may prefer to serve one type of task over another */ - long budget; /** deficit-round-robin credits for the two PRIORITY_FAIR strategies (flow/age). */ int creditFlow, creditAge; int waitingCount; - MultiTaskQueue(int waitingStates, GroupKind groups, long budget, long limits) + MultiTaskQueue(int waitingStates, GroupKind groups, long limits) { super(waitingStates); - this.baseBudget = budget; this.limits = limits; int queueCount = groups.count; Invariants.require(queueCount <= 8); @@ -2364,6 +2390,16 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor>> groupShift) & Task.GROUP_MASK; } + void stop(int group) + { + stopped |= overflowBit(group); + } + + void restart(int group) + { + stopped &= ~overflowBit(group); + } + final TaskQueue queue(Task task) { int group = group(task); @@ -2392,12 +2428,8 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor>>= 7; - int bitIndex = Long.numberOfTrailingZeros(visit); - return bitIndex / 8; - } - private int pollGroupByPhaseFair() { return minCounterIndex(recentFlowImbalances()); } - private int pollGroupByPhaseBudgetFair() - { - return minCounterIndex(recentFlowImbalances(), saturatedOrWithoutWorkOrWithoutBudget()); - } - // PRIORITY_FAIR selection: a deficit round-robin blend of two strategies, chosen per poll: // flow -> minCounterIndex(recent - arrivals + bias) (least fairly serviced) // age -> minGroupByPriority() (earliest-queued work) @@ -2516,11 +2526,6 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor queue = queues[group]; T head = queue.pollSingle(); @@ -2695,7 +2669,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor queue = queue(group); int result = queue.enqueueSingle(task); - incrementArrivals(group); + if (incrementArrivals) + incrementArrivals(group); if (result < 0) setHasWork(group); if (result != 0) setDirty(group); } @@ -2790,13 +2765,6 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor>> 7) & lowBit; } + final void incrementArrivals(Task task) + { + int group = group(task); + if (group >= 0) + incrementArrivals(group); + } + final void incrementArrivals(int group) { int shift = group * 8; @@ -2842,9 +2817,9 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor 0; + return unsaturatedWithWork() != 0; } final boolean isWaiting(T task) @@ -2870,13 +2845,13 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor extends MultiTaskQueue { - static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, ASSIGNED, RUNNING); + static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, RUNNING); final TaskQueue assigned; RunnableTaskQueue() { - super(RUNNABLE, GroupKind.GLOBAL, GLOBAL_QUEUE_BUDGETS, GLOBAL_QUEUE_LIMITS); + super(RUNNABLE, GroupKind.GLOBAL, GLOBAL_QUEUE_LIMITS); this.assigned = new TaskQueue<>(0); } @@ -2886,15 +2861,13 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor LoadRunnable newLoad(AccordCacheEntry entry, boolean isForRange) + LoadRunnable newLoad(AccordCacheEntry entry, boolean isForRange) { return new LoadRunnable<>(entry, isForRange ? RANGE_LOAD : LOAD); } - class LoadRunnable extends IOTask + public class LoadRunnable extends IOTask { - final AccordCacheEntry entry; + final AccordCacheEntry entry; Object result = FailureHolder.NOT_STARTED; - LoadRunnable(AccordCacheEntry entry, GlobalGroup group) + LoadRunnable(AccordCacheEntry entry, GlobalGroup group) { super(group); Invariants.require(group == LOAD || group == RANGE_LOAD); this.entry = entry; } - boolean isForRange() { return is(RANGE_LOAD); } - void postRunExclusive() { - if (!(result instanceof FailureHolder)) onLoadedExclusive(entry, (V)result, null, isForRange()); - else onLoadedExclusive(entry, null, ((FailureHolder)result).fail, isForRange()); + if (!(result instanceof FailureHolder)) onLoadedExclusive(entry, (V)result, null); + else onLoadedExclusive(entry, null, ((FailureHolder)result).fail); } @Override @@ -3194,7 +3163,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor entry; + final AccordCacheEntry entry; final UniqueSave identity; final Runnable run; Throwable failure = NOT_STARTED; - SaveRunnable(AccordCacheEntry entry, UniqueSave identity, Runnable run) + SaveRunnable(AccordCacheEntry entry, UniqueSave identity, Runnable run) { super(SAVE); this.entry = entry; @@ -3301,7 +3270,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor readyToRun = new ConcurrentLinkedQueue<>(); - private Task pendingSequentialHead, pendingSequentialTail; - private Task pendingCleanupHead, pendingCleanupTail; + private Task pendingExecutedHead, pendingExecutedTail; private Task pendingNewHead, pendingNewTail; private int pendingCount; @@ -105,14 +106,17 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop { updatePendingUnqueued(); Task requeue = null, requeueTail = null; + Task submit = null, submitTail = null; Task cur = pendingNewHead; while (cur != null) { Task next = cur.next; + cur.next = null; if (cur.isNewWork()) { - cur.next = null; - cur.submitExclusive(this); + if (submit == null) submit = cur; + else submitTail.next = cur; + submitTail = cur; --pendingCount; } else @@ -126,6 +130,13 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop pendingNewHead = requeue; pendingNewTail = requeueTail; + while (submit != null) + { + Task next = submit.next; + submit.next = null; + submit.submitExclusive(this); + submit = next; + } } private boolean enqueueOnePending() @@ -133,25 +144,23 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop if (pendingCount == 0) return false; - if (pendingSequentialHead != null) + --pendingCount; + if (pendingExecutedHead != null) { - pendingSequentialHead = enqueueOneCleanup(pendingSequentialHead); - if (pendingSequentialHead == null) - pendingSequentialTail = null; - } - else if (pendingCleanupHead != null) - { - pendingCleanupHead = enqueueOneCleanup(pendingCleanupHead); - if (pendingCleanupHead == null) - pendingCleanupTail = null; + Task executed = pendingExecutedHead; + pendingExecutedHead = destructiveNext(executed); + if (pendingExecutedHead == null) + pendingExecutedTail = null; + cleanupTaskExclusive(executed, true); } else { - pendingNewHead = enqueueOneSubmit(pendingNewHead); + Task submit = pendingNewHead; + pendingNewHead = destructiveNext(submit); if (pendingNewHead == null) pendingNewTail = null; + submit.submitExclusive(this); } - --pendingCount; return true; } @@ -178,22 +187,29 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop } boolean hasDrainedSignal = false; + int loops = 0; while (true) { long state = lock.state(); int signals = SignalLock.asyncSignalCount(state); int waiters = SignalLock.waitingEnabledThreadCount(state); - if (signals >= readyToRunTarget) + if (signals > 0) { - if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; - else if (hasDrainedSignal) - lock.signalLockWorkExclusive(); - return; - } - else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) - { - // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary - lock.propagateAsyncWorkSignals(1); + if (++loops > MAX_LOOPS) + return; + + if (signals >= readyToRunTarget) + { + if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; + else if (hasDrainedSignal) + lock.signalLockWorkExclusive(); + return; + } + else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) + { + // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary + lock.propagateAsyncWorkSignals(1); + } } Task task = pollAlreadyWaitingToRunExclusive(); @@ -216,9 +232,9 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop try { task.preRunExclusive(); } catch (Throwable t) { - try { task.fail(t); } + try { task.failExclusive(t, Task.State.FAILED_OTHER); } catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - try { completeTaskExclusive(task); } + try { cleanupTaskExclusive(task, false); } catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } continue; } @@ -237,28 +253,22 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop return false; int count = 0; - Task addSequentialHead = null, addSequentialTail = null; - Task addCleanupHead = null, addCleanupTail = null; + Task addExecutedHead = null, addExecutedTail = null; Task addNewHead = null, addNewTail = null; { Task cur = Task.reverse(acquireUnqueuedExclusive()); while (cur != null) { Task next = cur.next; - if (!cur.isReadyToCleanup()) + if (cur.is(UNINITIALIZED)) { if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); else addNewHead = reverseOne(addNewHead, cur); } - else if (cur instanceof ExclusiveExecutorTask) - { - if (addSequentialHead == null) addSequentialHead = addSequentialTail = setNextNull(cur); - else addSequentialHead = reverseOne(addSequentialHead, cur); - } else { - if (addCleanupHead == null) addCleanupHead = addCleanupTail = setNextNull(cur); - else addCleanupHead = reverseOne(addCleanupHead, cur); + if (addExecutedHead == null) addExecutedHead = addExecutedTail = setNextNull(cur); + else addExecutedHead = reverseOne(addExecutedHead, cur); } ++count; cur = next; @@ -266,17 +276,11 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop } pendingCount += count; - if (addSequentialHead != null) + if (addExecutedHead != null) { - if (pendingSequentialHead == null) pendingSequentialHead = addSequentialHead; - else pendingSequentialTail.next = addSequentialHead; - pendingSequentialTail = addSequentialTail; - } - if (addCleanupHead != null) - { - if (pendingCleanupHead == null) pendingCleanupHead = addCleanupHead; - else pendingCleanupTail.next = addCleanupHead; - pendingCleanupTail = addCleanupTail; + if (pendingExecutedHead == null) pendingExecutedHead = addExecutedHead; + else pendingExecutedTail.next = addExecutedHead; + pendingExecutedTail = addExecutedTail; } if (addNewHead != null) { @@ -349,22 +353,22 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop } } - private Task cleanupAndMaybeGetWork(AccordTaskRunner self, @Nullable Task cleanup) + private Task executedAndMaybeGetWork(AccordTaskRunner self, @Nullable Task executed) { if (lock.tryAcquireAsyncWork()) { if (shutdown) throw new ShutdownException(); - return pushCleanupAndReturn(cleanup, nonNull(pollReadyToRun())); + return pushExecutedAndReturn(executed, nonNull(pollReadyToRun())); } if (!tryLock(self)) - return pushCleanupAndReturn(cleanup, null); + return pushExecutedAndReturn(executed, null); try { - completeTaskExclusive(cleanup); + cleanupTaskExclusive(executed, true); fetchWorkExclusive(); } catch (Throwable t) @@ -383,10 +387,9 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop return null; } - private Task pushCleanupAndReturn(Task cleanup, Task result) + private Task pushExecutedAndReturn(Task complete, Task result) { - cleanup.setReadyToCleanup(); - if (push(cleanup) == null) + if (push(complete) == null) lock.signalLockWork(); return result; } @@ -401,7 +404,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop final boolean unlockAndAcquire(AccordTaskRunner self) { - self.clearAccordLockedExecutor(); + self.exitAccordLockedExecutor(); if (DEBUG_EXECUTION) debug.onExitLock(); return lock.unlockAndAcquireAsyncWork(); } @@ -422,7 +425,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop { if (task != null) { - try { task = cleanupAndMaybeGetWork(self, task); } + try { task = executedAndMaybeGetWork(self, task); } catch (Throwable t) { task = null; throw t; } } if (task == null) @@ -435,7 +438,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop } catch (Throwable t) { - try { task.fail(t); } + try { task.failExecution(t); } catch (Throwable t2) { try diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java index ee37744d8d..cadd7a766e 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java @@ -82,7 +82,8 @@ class AccordExecutorSimple extends AccordExecutor protected void run() { - Thread self = Thread.currentThread(); + AccordTaskRunner self = AccordTaskRunner.get(); + self.setAccordActiveExecutor(AccordExecutorSimple.this); lock.lock(); try { @@ -95,11 +96,24 @@ class AccordExecutorSimple extends AccordExecutor return; } - try { task.preRunExclusive(); task.run(); } - catch (Throwable t) { task.fail(t); } + // TODO (expected): dedup with AbstractLockLoop, and cleanup (executed flag is a bit ugly) + self.setAccordActiveTask(task); + boolean executed = false; + try + { + task.preRunExclusive(); + executed = true; + task.run(); + } + catch (Throwable t) + { + executed = false; + task.failExecution(t); + } finally { - completeTaskExclusive(task); + cleanupTaskExclusive(task, executed); + self.setAccordActiveTask(null); } } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java index 488ec61cb0..1263a3e1ec 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java @@ -20,51 +20,22 @@ package org.apache.cassandra.service.accord; import java.util.Objects; -import com.google.common.annotations.VisibleForTesting; - import accord.api.Journal; import accord.local.Command; import accord.local.SafeCommand; import accord.primitives.TxnId; -import org.apache.cassandra.utils.concurrent.Ref; +import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; -public class AccordSafeCommand extends SafeCommand implements AccordSafeState +public class AccordSafeCommand extends SafeCommand implements AccordSafeState { - public static class DebugAccordSafeCommand extends AccordSafeCommand - { - final Ref selfRef; - public DebugAccordSafeCommand(AccordCacheEntry global) - { - super(global); - selfRef = new Ref<>(this, null); - selfRef.debug(global.key().toString()); - } - - @Override - public void markUnsafe() - { - super.markUnsafe(); - selfRef.release(); - } - - public static void trace(AccordSafeCommand safeCommand, String message) - { - ((DebugAccordSafeCommand)safeCommand).selfRef.debug(message); - } - } - - private boolean unsafe; - private final AccordCacheEntry global; + private final AccordCacheEntry global; private Command original; - private Command current; - public AccordSafeCommand(AccordCacheEntry global) + public AccordSafeCommand(AccordCacheEntry global) { super(global.key()); this.global = global; - this.original = null; - this.current = null; } @Override @@ -73,7 +44,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState extends Task implements Function, Cancellable, DebuggableTask +public final class AccordTask extends Task implements Cancellable, DebuggableTask { private static final Logger logger = LoggerFactory.getLogger(AccordTask.class); private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES); - static class ForFunction extends AccordTask - { - private final Function function; - - public ForFunction(AccordCommandStore commandStore, ExecutionContext context, Function function) - { - super(commandStore, context); - this.function = function; - } - - @Override - public R apply(SafeCommandStore commandStore) - { - return function.apply(commandStore); - } - } - - // TODO (desired): these anonymous ops are somewhat tricky to debug. We may want to at least give them names. - static class ForConsumer extends AccordTask - { - private final Consumer consumer; - - private ForConsumer(AccordCommandStore commandStore, ExecutionContext context, Consumer consumer) - { - super(commandStore, context); - this.consumer = consumer; - } - - @Override - public Void apply(SafeCommandStore commandStore) - { - consumer.accept(commandStore); - return null; - } - } - - public static AccordTask create(CommandStore commandStore, ExecutionContext context, Function function) - { - return new ForFunction<>((AccordCommandStore) commandStore, context, function); - } - public static AccordTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) { - return new ForConsumer((AccordCommandStore) commandStore, context, consumer); + return new AccordTask<>((AccordCommandStore) commandStore, context, safeStore -> { + consumer.accept(safeStore); + return null; + }); + } + + public static AccordTask create(CommandStore commandStore, ExecutionContext context, Function function) + { + return new AccordTask<>((AccordCommandStore) commandStore, context, function); + } + + final class NonSyncState extends ExecutionContext.Wrapped implements ExecutionContext + { + RoutingKeys active; + ObjectHashSet notBlocking; + ObjectHashSet blocking; // cache entries on which we're blocking work + int loaded, processed; + int ready; + + public NonSyncState() + { + super(executionContext); + } + + @Override + public Unseekables keys() + { + return active; + } + + void addLoaded() + { + ++loaded; + } + + void onNotHead(AccordCacheEntry entry) + { + if ((notBlocking == null || !notBlocking.remove((RoutingKey) entry.key())) && blocking != null) + blocking.remove((RoutingKey) entry.key()); + } + + void onNewHead(AccordCacheEntry entry) + { + ensureNotBlocking().add((RoutingKey) entry.key()); + } + + void onNewBlockingHead(AccordCacheEntry entry) + { + ensureBlocking().add((RoutingKey) entry.key()); + } + + void onStillHeadNewBlocking(AccordCacheEntry entry) + { + notBlocking.remove((RoutingKey) entry.key()); + ensureBlocking().add((RoutingKey) entry.key()); + } + + private ObjectHashSet ensureBlocking() + { + if (blocking == null) + blocking = new ObjectHashSet<>(); + return blocking; + } + + private ObjectHashSet ensureNotBlocking() + { + if (notBlocking == null) + notBlocking = new ObjectHashSet<>(); + return notBlocking; + } + + private int readyCount() + { + return (blocking == null ? 0 : blocking.size()) + (notBlocking == null ? 0 : notBlocking.size()); + } + + boolean isLoaded() + { + return loaded >= Math.min(keys, NONSYNC_MIN_BATCH_SIZE); + } + + boolean isWaitReady() + { + if (readyCount() >= Math.min(keys - processed, NONSYNC_MIN_BATCH_SIZE)) + return true; + + return blocking != null && blocking.size() >= NONSYNC_BLOCKED_LIMIT; + } + + void preRunExclusive() + { + try (BufferList keys = new BufferList<>()) + { + if ((blocking == null || !populate(keys, blocking)) && notBlocking != null) + populate(keys, notBlocking); + + keys.forEach(key -> preExecute(refs.get(key), AccordTask.this, RELEASE_QUEUE)); + keys.sort(RoutingKey::compareTo); + active = RoutingKeys.of(keys); + } + processed += active.size(); + if (processed == keys && isIncremental()) + setIncrementalFinishingExclusive(); + } + + private boolean populate(List keys, ObjectHashSet from) + { + if (keys.size() + from.size() <= AccordExecutor.NONSYNC_MAX_BATCH_SIZE) + { + keys.addAll(from); + from.clear(); + return keys.size() == AccordExecutor.NONSYNC_MAX_BATCH_SIZE; + } + + Iterator iterator = from.iterator(); + while (iterator.hasNext()) + { + if (keys.size() == AccordExecutor.NONSYNC_MAX_BATCH_SIZE) + return true; + + RoutingKey key = iterator.next(); + keys.add(key); + iterator.remove(); + } + + return false; + } + + void postRunExclusive() + { + if (active != null) + { + for (RoutingKey key : active) + postExecute(refs.remove(key), AccordTask.this); + active = null; + } + ready = readyCount(); + } } final AccordCommandStore commandStore; private final ExecutionContext executionContext; - private volatile String loggingId; - private static final AtomicLong nextLoggingId = new AtomicLong(Clock.Global.currentTimeMillis()); - private static final AtomicReferenceFieldUpdater loggingIdUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordTask.class, String.class, "loggingId"); + private final Function function; - // TODO (desired): merge all of these maps into one - @Nullable Object2ObjectHashMap commands; - @Nullable Object2ObjectHashMap commandsForKey; - @Nullable Object2ObjectHashMap> loading; + // TODO (expected): simple custom map that allows (at least): + // - efficient putIfAbsent + // - efficient small collections (2-4 entries) + // - forEach with parameters to avoid boxing lambdas + // - destructive forEach + // - forEach over specific SafeState types + Object2ObjectHashMap> refs = new Object2ObjectHashMap<>(); + + /** + * if is(LOADING), this is the number of cache entries we're waiting to complete loading before we can transition to WAITING_ON_CACHE_QUEUES; + * if is(WAITING_ON_CACHE_QUEUES), it's the number we're waiting to be head of before we can run + *

+ * if isNonSync(), this counts only txnId; otherwise it counts keys and txnId + */ + int waitingForState; + + /** + * Only set when isNonSync() + *

+ * if is(LOADING), this is the cache entries that have finished loading + * otherwise it's the cache entries for which we're at the head of the queue and are ready to run with + */ + @Nullable NonSyncState nonSync; + + int keys; // TODO (expected): not counting keys we add during execution LogLinearDecayingHistograms.Buffer histogramBuffer; - // TODO (desired): collection supporting faster deletes but still fast poll (e.g. some ordered collection) - @Nullable ArrayDeque> waitingToLoad; @Nullable RangeTxnScanner rangeScanner; @Nullable CommandSummaries commandsForRanges; + byte runState; private BiConsumer callback; - public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext) + public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext, Function function) { - super(executionContext); + super(executionContext, commandStore.executor().uniqueCreatedAt); this.commandStore = commandStore; this.executionContext = executionContext; - this.loggingId = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); - + this.function = function; if (logger.isTraceEnabled()) logger.trace("Created {} on {}", this, commandStore); } - private String loggingId() + String loggingId() { - String id = loggingId; - if (id == null) - { - id = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); - if (!loggingIdUpdater.compareAndSet(this, null, id)) - id = loggingId; - } - return id; + return executor().executorId + "/" + Long.toHexString(createdAt); } @Override public String toString() { - return executionContext.describe() + ' ' + toBriefString(); + return "@[" + commandStore.id() + ',' + commandStore.node().id() + "] " + executionContext.describe() + ' ' + toBriefString(); } public String toBriefString() { - return '{' + loggingId() + ',' + state() + '}'; + return '{' + loggingId() + ',' + describeState() + '}'; } public String toDescription() { return toBriefString() + ": " - + (queued() == null ? "unqueued" : state()) + + (queued() == null ? "unqueued" : describeState()) + ", primaryTxnId: " + executionContext.primaryTxnId() - + ", waitingToLoad: " + summarise(waitingToLoad) - + ", loading:" + summarise(loading, AccordSafeState::global) - + ", cfks:" + summarise(commandsForKey, AccordSafeState::global) - + ", txns:" + summarise(commands, AccordSafeState::global); + + ", state: " + summarise(refs, AccordSafeState::global); } @@ -252,11 +373,6 @@ public abstract class AccordTask extends Task implements Function keys() - { - return executionContext.keys(); - } - // TODO (expected): try to execute immediately BUT consider ordering requirements // esp. with deferred actions on e.g. CommandsForKey (not yet supported but also important for performance) public AsyncChain chain() @@ -267,7 +383,7 @@ public abstract class AccordTask extends Task implements Function callback) { preSetup(callback); - commandStore.executor().submit(AccordTask.this); + executor().submit(AccordTask.this); return AccordTask.this; } }; @@ -281,86 +397,147 @@ public abstract class AccordTask extends Task implements Function parent) + public void preSetup(AccordTask parent) { this.position = parent.position; + setInheritedWithTranche(parent.tranche()); + // note we use the caches "unsafely" here deliberately, as we only reference commands we already have references to // so we do not mutate anything, except the atomic counter of references - if (parent.commands != null) + + LoadKeys loadKeys = loadKeys(executionContext); + if (loadKeys != NONE) { - for (TxnId txnId : executionContext.txnIds()) - presetupExclusive(txnId, AccordTask::ensureCommands, parent.commands, commandStore.cachesUnsafe().commands()); + Unseekables parentKeysOrRanges = parent.executionContext.keys(); + Unseekables keysOrRanges = executionContext.keys(); + + boolean isKeySubset = parent.isIncremental() ? parent.nonSync.active.containsAll(keysOrRanges) : parentKeysOrRanges.containsAll(keysOrRanges); + if (isKeySubset) + setInheritedRangeScan(); + + setSequencedExclusive(executionContext.executionSequence()); + if (isSequencedByPriorityAtomic()) + { + boolean isTxnIdSubset = executionContext.isTxnIdSubsetOf(parent.executionContext); + Invariants.require(isKeySubset, "Must start ATOMIC tasks from a task declaring a superset of the required keys (for ASYNC/INCR tasks this means the keys active for the batch in question)"); + Invariants.require(isTxnIdSubset, "Must start ATOMIC tasks from a task declaring a superset of the required TxnIds"); + // TODO (required): we're appending to the fifo queue - does this maintain correct order? + setCacheQueuedFifoExclusive(); + } + + if (loadKeys != SYNC) + { + setNonSyncExclusive(); + nonSync = new NonSyncState(); + if (loadKeys == INCR) + { + setIncrementalExclusive(); + // forbid BY_PRIORITY sequencing to avoid priority inversion deadlocks on INCR tasks that lock a TxnId but await some key that has a higher priority task (that is waiting on our locked TxnId) - solvable in future if necessary + Invariants.require(isSequencedByPriorityAtomic() || isUnsequenced(), "INCR tasks may currently only be ATOMIC or UNSEQUENCED"); + } + } + + if (keysOrRanges.equals(parentKeysOrRanges)) + { + // TODO (desired): custom map we can more cheaply fork/copy + parent.refs.forEach((key, val) -> { + if (val instanceof AccordSafeCommandsForKey) + preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); + }); + } + else + { + switch (keysOrRanges.domain()) + { + case Key: + for (RoutingKey key : (AbstractUnseekableKeys) keysOrRanges) + preSetup(key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); + break; + + case Range: + AbstractRanges ranges = (AbstractRanges) keysOrRanges; + parent.refs.forEach((key, val) -> { + if (val instanceof AccordSafeCommandsForKey && ranges.contains((RoutingKey) key)) + preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); + }); + break; + } + } } - if (parent.commandsForKey == null) return; - if (executionContext.keys().domain() != Key) return; - switch (executionContext.loadKeys()) - { - default: throw new UnhandledEnum(executionContext.loadKeys()); - case NONE: - break; - - case ASYNC: - case INCR: - case SYNC: - for (RoutingKey key : (AbstractUnseekableKeys) executionContext.keys()) - presetupExclusive(key, AccordTask::ensureCommandsForKey, parent.commandsForKey, commandStore.cachesUnsafe().commandsForKeys()); - break; - } + for (TxnId txnId : executionContext.txnIds()) + preSetup(txnId, parent.refs, commandStore.cachesUnsafe().commands()); } @Override void submitExclusive(AccordExecutor owner) { - owner.submitExclusive(this); - } - - public void setupExclusive() - { + owner.registerExclusive(this); setupInternal(commandStore.cachesExclusive()); - setState(rangeScanner != null ? WAITING_TO_SCAN_RANGES - : waitingToLoad != null ? WAITING_TO_LOAD - : loading != null ? LOADING : WAITING_TO_RUN); } private void setupInternal(Caches caches) { + boolean hasPreSetup = hasInherited(); + LoadKeys loadKeys = loadKeys(executionContext); + if (loadKeys != NONE) { - boolean hasPreSetup = commands != null; - for (TxnId txnId : executionContext.txnIds()) + if (loadKeys != SYNC && !hasPreSetup) { - if (hasPreSetup && completePresetupExclusive(txnId, commands, caches.commands())) - continue; - setupExclusive(txnId, AccordTask::ensureCommands, caches.commands()); + setNonSyncExclusive(); + nonSync = new NonSyncState(); + if (loadKeys == INCR) + setIncrementalExclusive(); + } + + Unseekables keysOrRanges = executionContext.keys(); + switch (keysOrRanges.domain()) + { + case Range: + if (!hasInheritedRangeScan()) setupRangeLoadsExclusive(caches); + else refs.forEach((k, v) -> { + if (v instanceof AccordSafeCommandsForKey) + completePresetupExclusive((AccordSafeCommandsForKey)v); + }); + break; + case Key: + setupKeyLoadsExclusive(hasPreSetup, caches, (AbstractUnseekableKeys) keysOrRanges, hasInheritedRangeScan()); + break; } } - if (executionContext.keys().isEmpty()) + for (TxnId txnId : executionContext.txnIds()) + { + if (hasPreSetup && completePresetupExclusive(txnId)) + continue; + setupExclusive(txnId, caches.commands(), 1); + } + + if (is(SCANNING_RANGES)) return; - switch (executionContext.keys().domain()) - { - case Key: setupKeyLoadsExclusive(caches, (AbstractUnseekableKeys) executionContext.keys(), false); break; - case Range: setupRangeLoadsExclusive(caches); - } + onSetupOrScannedExclusive(); } - private void setupKeyLoadsExclusive(Caches caches, Iterable keys, boolean isToCompleteRangeScan) + private void setupKeyLoadsExclusive(boolean hasPreSetup, Caches caches, Iterable setupKeys, boolean doNotScanRanges) { - if (executionContext.loadKeys() == LoadKeys.NONE) + if (executionContext.loadKeys() == NONE) return; - if (!isToCompleteRangeScan && executionContext.loadKeysFor() == RECOVERY) + if (!doNotScanRanges && executionContext.loadKeysFor() == RECOVERY) { Invariants.require(rangeScanner == null); rangeScanner = new RangeTxnScanner(); + rangeScanner.start(); } - boolean hasPreSetup = commandsForKey != null; - for (RoutingKey key : keys) + int waitsForIncrement = isSync() ? 1 : 0; + for (RoutingKey setupKey : setupKeys) { - if (hasPreSetup && completePresetupExclusive(key, commandsForKey, caches.commandsForKeys())) continue; - setupExclusive(key, AccordTask::ensureCommandsForKey, caches.commandsForKeys()); + if (hasPreSetup && completePresetupExclusive(setupKey)) + continue; + + setupExclusive(setupKey, caches.commandsForKeys(), waitsForIncrement); } } @@ -369,123 +546,365 @@ public abstract class AccordTask extends Task implements Function> void presetupExclusive(K k, Function, Map> loaded, Map parentMap, AccordCache.Type.Instance cache) + private & AccordSafeState> void preSetup(K k, Map> parentMap, AccordCache.Type.Instance cache) { - AccordSafeState ref = parentMap.get(k); + S ref = (S) parentMap.get(k); if (ref == null) return; - AccordCacheEntry node = ref.global(); + AccordCacheEntry node = ref.global(); int refs = node.increment(); Invariants.require(refs > 1); - loaded.apply(this).put(k, cache.parent().adapter().safeRef(node)); + S safeState = cache.parent().adapter().safeRef(node); + this.refs.put(k, safeState); + if (cache.isCommandsForKey()) + keys++; } - // expects to hold lock - private > boolean completePresetupExclusive(K k, Map map, AccordCache.Type.Instance cache) + private & AccordSafeState> boolean completePresetupExclusive(K k) { - AccordSafeState preacquired = map.get(k); + S preacquired = (S) refs.get(k); if (preacquired != null) { - cache.recordPreAcquired(preacquired); + completePresetupExclusive(preacquired); return true; } return false; } + private & AccordSafeState> void completePresetupExclusive(S preacquired) + { + AccordCacheEntry entry = preacquired.global(); + if (entry.isLoaded()) completeSetupOfLoaded(entry); + else completeSetupOfLoading(entry, true); + } + // expects to hold lock - private > void setupExclusive(K k, Function, Map> loaded, AccordCache.Type.Instance cache) + private & AccordSafeState> void setupExclusive(K k, AccordCache.Type.Instance cache, int waitForIncrement) { S safeRef = cache.acquire(k); - Status entryStatus = safeRef.global().status(); - Map map; + AccordCacheEntry entry = safeRef.global(); + Status entryStatus = entry.status(); + boolean submitLoad = false; + boolean isLoaded; switch (entryStatus) { default: throw new UnhandledEnum(entryStatus); case WAITING_TO_LOAD: + submitLoad = true; case LOADING: - map = ensureLoading(); + isLoaded = false; + waitingForState += waitForIncrement; break; case WAITING_TO_SAVE: case SAVING: case LOADED: case MODIFIED: case FAILED_TO_SAVE: - map = loaded.apply(this); + isLoaded = true; } - Object prev = map.putIfAbsent(k, safeRef); + Object prev = refs.putIfAbsent(k, safeRef); if (prev != null) { - noSpamLogger.warn("PreLoadContext {} contained key {} more than once", map, k); + noSpamLogger.warn("ExecutionContext {} contained key {} more than once", refs, k); cache.release(safeRef, this); + waitingForState -= waitForIncrement; } - else if (map == loading) - { - if (entryStatus == Status.WAITING_TO_LOAD) - ensureWaitingToLoad().add(safeRef.global()); - safeRef.global().loadingOrWaiting().add(this); - Invariants.paranoid(safeRef.global().loadingOrWaiting().waiters().size() == safeRef.global().references()); - } - } - - // expects to hold lock - public boolean onLoad(AccordCacheEntry state) - { - AccordSafeState safeRef = loading == null ? null : loading.remove(state.key()); - Invariants.require(safeRef != null && safeRef.global() == state, "Expected to find %s loading; found %s", state, this, AccordTask::toDescription); - if (safeRef.getClass() == AccordSafeCommand.class) - ensureCommands().put((TxnId)state.key(), (AccordSafeCommand) safeRef); else - ensureCommandsForKey().put((RoutingKey) state.key(), (AccordSafeCommandsForKey) safeRef); + { + if (entry.isCommandsForKey()) + keys++; - if (!loading.isEmpty()) - return false; - - loading = null; - if (compareTo(WAITING_TO_LOAD) < 0) - return false; - - Invariants.require(waitingToLoad == null, "Invalid state: %s", this, AccordTask::toDescription); - setState(WAITING_TO_RUN); - return true; + if (isLoaded) completeSetupOfLoaded(entry); + else + { + if (submitLoad) executor().cacheUnsafe().load(executor(), this, is(ExclusiveGroup.RANGE), entry); + completeSetupOfLoading(entry, !submitLoad); + } + } } - // expects to hold lock - public boolean onLoading(AccordCacheEntry state) + private void completeSetupOfLoaded(AccordCacheEntry entry) { - boolean removed = waitingToLoad != null && waitingToLoad.remove(state); - Invariants.require(removed, "%s not found in waitingToLoad %s", state, this, AccordTask::toDescription); - if (!waitingToLoad.isEmpty()) - return false; - - return onEmptyWaitingToLoad(); + if (isOptional(entry)) + { + nonSync.addLoaded(); + if (isCacheQueuedFifo()) + addQueuedOptionalKey(entry, entry.addFifo(this)); + } + else if (isCacheQueuedFifo()) + { + entry.addFifo(this); + } } - private boolean onEmptyWaitingToLoad() + private void completeSetupOfLoading(AccordCacheEntry entry, boolean alreadyLoading) { - waitingToLoad = null; - if (compareTo(WAITING_TO_LOAD) < 0) - return false; + if (alreadyLoading) + { + Loading loading = entry.loading(); + if (loading.loading != null && loading.loading.is(RANGE_LOAD) && loading.loading.is(WAITING_TO_RUN) && !is(ExclusiveGroup.RANGE)) + { + // requeue anything setup as a range load that's now needed for a key-based operation, so it can use the correct the queue limits + loading.loading.unqueue(executor().runnable); + loading.loading.override(LOAD); + executor().runnable.enqueue(loading.loading, false); + } + } - setState(loading == null ? WAITING_TO_RUN : LOADING); - return true; + if (isCacheQueuedFifo()) entry.addFifo(this); + else entry.addWaitingToLoad(this); + Invariants.paranoid(entry.waitingCount() == entry.references()); + } + + private void onSetupOrScannedExclusive() + { + if (waitingForState > 0) + { + setStateExclusive(LOADING_REQUIRED); + executor().loading.enqueue(this); + } + else onLoadedRequiredExclusive(); + } + + private void onLoadedRequiredExclusive() + { + if (isSync() || nonSync.isLoaded()) + { + waitOnCacheQueuesExclusive(); + } + else + { + setStateExclusive(LOADING_OPTIONAL); + executor().loading.enqueue(this); + } + } + + boolean isUnsequenced(AccordCacheEntry entry) + { + return isUnsequenced() && (entry.isCommandsForKey() || !isIncremental()); + } + + boolean isOptional(AccordCacheEntry entry) + { + return isNonSync() && entry.isCommandsForKey(); + } + + boolean holdsLocksBetweenRuns() + { // TODO (desired): encode as a state bit + return isIncremental() && executionContext.primaryTxnId() != null; + } + + private void waitOnCacheQueuesExclusive() + { + Invariants.require(waitingForState == 0); + onLoaded(); + executor().runnable.incrementArrivals(this); + commandStore.exclusiveExecutor.incrementArrivals(this); + + this.refs.forEach((key, safeState) -> { + AccordCacheEntry entry = global(safeState); + boolean optional = isOptional(entry); + if (entry.isLoaded()) + { + RunnableStatus status = addToCacheQueue(entry, false); + if (optional) addQueuedOptionalKey(entry, status); + else if (status == NOT_RUNNABLE) + ++waitingForState; + } + else Invariants.require(optional); + }); + + // TODO (desired): exception-safe rollback for addUnsequenced + setCacheQueuedExclusive(); + if (waitingForState == 0) waitOnOptionalCacheQueuesExclusive(); + else + { + setStateExclusive(WAITING_ON_REQUIRED); + executor().waitingOnCacheQueues.enqueue(this); + } + } + + private void waitOnOptionalCacheQueuesExclusive() + { + if (isSync() || nonSync.isWaitReady()) waitToRunExclusive(); + else + { + setStateExclusive(WAITING_ON_OPTIONAL); + executor().waitingOnCacheQueues.enqueue(this); + } + } + + RunnableStatus addToCacheQueue(AccordCacheEntry loaded, boolean addIfFifo) + { + if (isCacheQueuedFifo()) return addIfFifo ? loaded.addFifo(this) : loaded.headStatus(this); + else if (isUnsequenced(loaded)) return loaded.addUnsequenced(this); + else return loaded.addPrioritised(this); + } + + void onLoadOneExclusive(AccordCacheEntry loaded) + { + if (isOptional(loaded)) + { + // if we're incremental/async we don't block on keys loading, so we don't need to decrement anything + // however, if we're in fifo mode this loaded key might be ready for us to run with + State state = state(); + switch (state) + { + default: throw new UnhandledEnum(state); + case WAITING_ON_REQUIRED: + case WAITING_ON_OPTIONAL: + case WAITING_TO_RUN: + case RUNNING: + RunnableStatus status = addToCacheQueue(loaded, false); + if (status != NOT_RUNNABLE) + addQueuedOptionalKey(loaded, status); + // fall-through + case LOADING_REQUIRED: + nonSync.addLoaded(); + break; + + case LOADING_OPTIONAL: + addLoadedOptionalKey(); + break; + } + } + else + { + if (--waitingForState == 0) + { + if (is(LOADING_REQUIRED)) + { + unqueue(executor().loading); + onLoadedRequiredExclusive(); + } + else Invariants.require(is(SCANNING_RANGES)); + } + } + } + + void addLoadedOptionalKey() + { + nonSync.addLoaded(); + if (is(LOADING_OPTIONAL) && nonSync.isLoaded()) + { + unqueue(executor().loading); + waitOnCacheQueuesExclusive(); + } + } + + // TODO (expected): add vs setup vs onChange; some callers don't need to try + void addQueuedOptionalKey(AccordCacheEntry loaded, RunnableStatus status) + { + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case NOT_RUNNABLE: break; + case STILL_RUNNABLE: + case NEWLY_RUNNABLE: + nonSync.onNewHead(loaded); + break; + case STILL_RUNNABLE_NEWLY_BLOCKING: + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(loaded); + break; + } + + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady()) + { + unqueue(executor().waitingOnCacheQueues); + waitToRunExclusive(); + } + } + + void onChangeHeadStatus(AccordCacheEntry entry, RunnableStatus status) + { + if (isSync() || !entry.isCommandsForKey()) onChangeRequiredHeadStatus(entry, status); + if (isNonSync() && entry.isCommandsForKey()) onChangeOptionalHeadStatus(entry, status); + } + + private void incrementWaitingWhileAlreadyWaiting() + { + Invariants.require(isState(WAITING)); + if (waitingForState == 0) + { + if (is(WAITING_ON_OPTIONAL)) setStateExclusive(WAITING_ON_REQUIRED); + else + { + // TODO (expected): this is potentially costly; maybe we don't want to swap these in and out (but harder to maintain invariants) + unqueue(commandStore.exclusiveExecutor); + setStateExclusive(WAITING_ON_REQUIRED); + executor().waitingOnCacheQueues.enqueue(this); + } + } + Invariants.require(waitingForState < refs.size()); + ++waitingForState; + } + + void onChangeRequiredHeadStatus(AccordCacheEntry entry, RunnableStatus newStatus) + { + if (newStatus == NOT_RUNNABLE) + { + incrementWaitingWhileAlreadyWaiting(); + } + else if (newStatus != STILL_RUNNABLE_NEWLY_BLOCKING) + { + Invariants.require(is(WAITING_ON_REQUIRED)); + if (--waitingForState == 0) + { + unqueue(executor().waitingOnCacheQueues); + waitOnOptionalCacheQueuesExclusive(); + } + } + } + + void onChangeOptionalHeadStatus(AccordCacheEntry entry, RunnableStatus status) + { + Invariants.require(isState(WAITING_OR_RUNNING)); + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case STILL_RUNNABLE: throw UnhandledEnum.invalid(STILL_RUNNABLE); // onChange -> changed (but this means no change) + case NOT_RUNNABLE: + nonSync.onNotHead(entry); + if (is(WAITING_TO_RUN) && !nonSync.isWaitReady()) + { + unqueue(commandStore.exclusiveExecutor); + setStateExclusive(WAITING_ON_OPTIONAL); + executor().waitingOnCacheQueues.enqueue(this); + } + return; + + case STILL_RUNNABLE_NEWLY_BLOCKING: + nonSync.onStillHeadNewBlocking(entry); + break; + + case NEWLY_RUNNABLE: + nonSync.onNewHead(entry); + break; + + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(entry); + break; + } + + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady()) + { + unqueue(executor().waitingOnCacheQueues); + waitToRunExclusive(); + } + } + + void waitToRunExclusive() + { + setStateExclusive(WAITING_TO_RUN); + commandStore.exclusiveExecutor.enqueue(this, false); } public ExecutionContext executionContext() @@ -493,70 +912,132 @@ public abstract class AccordTask extends Task implements Function commands() + @Override + protected void preRunExclusive() { - return commands; - } - - public Map ensureCommands() - { - if (commands == null) - commands = new Object2ObjectHashMap<>(); - return commands; - } - - public Map commandsForKey() - { - return commandsForKey; - } - - public Map ensureCommandsForKey() - { - if (commandsForKey == null) - commandsForKey = new Object2ObjectHashMap<>(); - return commandsForKey; - } - - private Map> ensureLoading() - { - if (loading == null) - loading = new Object2ObjectHashMap<>(); - return loading; - } - - private ArrayDeque> ensureWaitingToLoad() - { - Invariants.require(compareTo(WAITING_TO_LOAD) <= 0, "Expected status to be on or before WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); - if (waitingToLoad == null) - waitingToLoad = new ArrayDeque<>(); - return waitingToLoad; - } - - public AccordCacheEntry pollWaitingToLoad() - { - Invariants.require(is(WAITING_TO_LOAD), "Expected status to be WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); - if (waitingToLoad == null) - return null; - - AccordCacheEntry next = waitingToLoad.poll(); - if (waitingToLoad.isEmpty()) - onEmptyWaitingToLoad(); - return next; - } - - public AccordCacheEntry peekWaitingToLoad() - { - return waitingToLoad == null ? null : waitingToLoad.peek(); - } - - private void maybeSanityCheck(AccordSafeCommand safeCommand) - { - if (SANITY_CHECK) + super.preRunExclusive(); + if (rangeScanner != null) { - DebugTask debug = DebugTask.get(this); - if (debug.sanityCheck == null) - debug.sanityCheck = new ArrayList<>(commands.size()); - debug.sanityCheck.add(safeCommand.current()); + commandsForRanges = rangeScanner.finish(commandStore.cachesExclusive()); + rangeScanner = null; + } + + if (isSync()) + { + refs.forEach((k, v) -> { + preExecute(v, this, RELEASE_QUEUE); + }); + } + else + { + if (!hasIncrementalStarted()) + { + TxnId primaryTxnId = executionContext.primaryTxnId(); + if (primaryTxnId != null) + { + LockMode lockMode = holdsLocksBetweenRuns() ? HOLD_QUEUE : RELEASE_QUEUE; + preExecute(refs.get(primaryTxnId), this, lockMode); + TxnId additionalTxnId = executionContext.additionalTxnId(); + if (additionalTxnId != null) + preExecute(refs.get(additionalTxnId), this, lockMode); + } + + if (isIncremental() && isSequencedByPriorityAtomic() && !isCacheQueuedFifo()) + { + setCacheQueuedFifoExclusive(); + refs.forEach((key, safeState) -> { + AccordCacheEntry entry = global(safeState); + RunnableStatus status = entry.moveToFifo(this); + if (entry.isLoaded()) + { + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case NOT_RUNNABLE: + case STILL_RUNNABLE: + case STILL_RUNNABLE_NEWLY_BLOCKING: + break; + case NEWLY_RUNNABLE: + nonSync.onNewHead(entry); + break; + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(entry); + break; + } + } + }); + } + + if (isIncremental()) + setIncrementalStartedExclusive(); + } + nonSync.preRunExclusive(); + } + } + + @Override + public void run() + { + onRunning(); + AccordSafeCommandStore safeStore = null; + try (Closeable close = resources.get()) + { + if (Tracing.isTracing()) + Tracing.trace(executionContext.describe()); + + commandStore.begin(safeStore = new AccordSafeCommandStore(this, isSync() ? executionContext : nonSync)); + + R result = function.apply(safeStore); + + boolean finished = !isIncremental() || isIncrementalFinishing(); + if (finished) + { + List changes = new ArrayList<>(); + // TODO (expected): save any TxnId we add so that we don't need to iterate all of refs + refs.forEach((key, value) -> { + if (value instanceof AccordSafeCommand) + { + AccordSafeCommand safeCommand = (AccordSafeCommand) value; + Journal.CommandUpdate diff = safeCommand.update(); + if (diff != null) + { + changes.add(diff); + maybeSanityCheck(safeCommand); + } + } + }); + + boolean flush = !changes.isEmpty() || safeStore.fieldUpdates() != null; + if (flush) + { + setRunState(PERSISTING); + Runnable onFlush = () -> finish(result, null); + safeStore.persistFieldUpdatesInternal(changes.isEmpty() ? onFlush : null); + if (!changes.isEmpty()) + save(changes, onFlush); + finished = false; + } + } + + // TODO (required): exception handling here needs improving + safeStore.postExecute(); + commandStore.complete(safeStore); + safeStore = null; + + if (finished) + finish(result, null); + } + catch (Throwable t) + { + if (safeStore != null) + refs.forEach((k, v) -> v.setAbandoned()); + throw t; + } + finally + { + if (safeStore != null) + commandStore.complete(safeStore); + onRunComplete(); } } @@ -579,147 +1060,71 @@ public abstract class AccordTask extends Task implements Function v.preExecute()); - if (commandsForKey != null) - commandsForKey.forEach((k, v) -> v.preExecute()); - } - - @Override - public void run() - { - onRunning(); - AccordSafeCommandStore safeStore = null; - try (Closeable close = resources.get()) - { - if (Tracing.isTracing()) - Tracing.trace(executionContext.describe()); - - setState(RUNNING); - - safeStore = commandStore.begin(this, commandsForRanges); - R result = apply(safeStore); - - List changes = null; - if (commands != null) - { - for (AccordSafeCommand safeCommand : commands.values()) - { - if (safeCommand.txnId().is(EphemeralRead)) - continue; - - Journal.CommandUpdate diff = safeCommand.update(); - if (diff == null) - continue; - - if (changes == null) - changes = new ArrayList<>(commands.size()); - changes.add(diff); - - maybeSanityCheck(safeCommand); - } - } - - boolean flush = changes != null || safeStore.fieldUpdates() != null; - if (flush) - { - setState(PERSISTING); - Runnable onFlush = () -> finish(result, null); - safeStore.persistFieldUpdatesInternal(changes == null ? onFlush : null); - if (changes != null) save(changes, onFlush); - } - - commandStore.complete(safeStore); - safeStore = null; - onRunComplete(); - if (!flush) - finish(result, null); - } - catch (Throwable t) - { - if (safeStore != null) - { - revert(); - commandStore.abort(safeStore); - } - throw t; + DebugTask debug = DebugTask.get(this); + if (debug.sanityCheck == null) + debug.sanityCheck = new ArrayList<>(2); + debug.sanityCheck.add(safeCommand.current()); } } - public void fail(Throwable throwable) + public void reportFailure(Throwable throwable) { - if (state().isComplete()) - return; - try - { - setState(FAILED); - commandStore.agent().onException(throwable); - } - finally { if (callback != null) callback.accept(null, throwable); } - } - - @Override - protected boolean isNewWork() - { - return true; - } - - public void failExclusive(Throwable throwable) - { - fail(throwable); - } - - @Override - protected void cleanupExclusive(AccordExecutor executor) - { - Invariants.expect(state().isExecuted()); - releaseResources(commandStore.cachesExclusive()); - super.cleanupExclusive(executor); - executor.keys.increment(commandsForKey == null ? 0 : commandsForKey.size(), runningAt); - if (histogramBuffer != null) + finally { - histogramBuffer.flush(cleanupAt); - histogramBuffer = null; + commandStore.agent().onException(throwable); } } - @Nullable - public RangeTxnScanner rangeScanner() + @Override + protected void cleanupExclusive(AccordExecutor executor, boolean executed) { - return rangeScanner; + if (is(RUNNING) && isNonSync()) + { + nonSync.postRunExclusive(); + if (isIncremental() && !isIncrementalFinishing()) + { + setStateExclusive(INCOMPLETE); + waitOnOptionalCacheQueuesExclusive(); + return; + } + } + + executor.keys.increment(keys, runningAt); + releaseResourcesExclusive(commandStore.cachesExclusive()); + super.cleanupExclusive(executor, executed); + if (histogramBuffer != null) + { + histogramBuffer.flush(completeAt); + histogramBuffer = null; + } } @Override public void cancel() { - if (!state().isComplete()) - commandStore.executor().cancel(this); + if (!state().hasStarted()) + executor().cancel(this); } void cancelExclusive() { logger.info("Cancelling {}", executionContext); - setState(CANCELLED); + setStateExclusive(CANCELLED); if (rangeScanner != null) rangeScanner.cancelled = true; if (callback != null) { - if (commandStore.executor().isInLoop()) callback.accept(null, new CancellationException()); - else commandStore.executor().submit(() -> callback.accept(null, new CancellationException())); + if (executor().isInLoop()) callback.accept(null, new CancellationException()); + else executor().submit(() -> callback.accept(null, new CancellationException())); } } @@ -730,113 +1135,54 @@ public abstract class AccordTask extends Task implements Function caches.commands().release(v, this)); - commands.clear(); - commands = null; - if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedCommands(); - } - if (commandsForKey != null) - { - commandsForKey.forEach((k, v) -> caches.commandsForKeys().release(v, this)); - commandsForKey.clear(); - commandsForKey = null; - if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedCommandsForKeys(); - } - if (waitingToLoad != null) - { - while (!waitingToLoad.isEmpty()) - waitingToLoad.poll().loadingOrWaiting().remove(this); - waitingToLoad = null; - } - if (loading != null) - { - loading.forEach((k, v) -> caches.global().release(v, this)); - loading.clear(); - loading = null; - } + + refs.forEach((key, safeState) -> { + AccordSafeState.postExecute(safeState, this); + }); + if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedState(); } catch (Throwable t) { - releaseResourcesSlow(caches, t); + releaseResourcesSlowExclusive(t); commandStore.agent().onException(t); } + finally + { + refs = null; + } } - private void releaseResourcesSlow(Caches caches, Throwable suppressedBy) + private void releaseResourcesSlowExclusive(Throwable suppressedBy) { - if (commands != null) - { - safeRelease(commands, caches.commands(), suppressedBy); - commands.clear(); - commands = null; - } - if (commandsForKey != null) - { - safeRelease(commandsForKey, caches.commandsForKeys(), suppressedBy); - commandsForKey.clear(); - commandsForKey = null; - } - if (waitingToLoad != null) - { - while (!waitingToLoad.isEmpty()) + if (refs == null) + return; + + refs.forEach((k, safeState) -> { + if (!safeState.isReleased()) { - try { waitingToLoad.poll().loadingOrWaiting().remove(this); } + try { AccordSafeState.postExecute(safeState, this); } catch (Throwable t) { suppressedBy.addSuppressed(t); } } - waitingToLoad = null; - } - if (loading != null) - { - safeRelease(loading, caches.global(), suppressedBy); - loading.clear(); - loading = null; - } - } - - private void safeRelease(Map> map, AccordCache.Type.Instance cache, Throwable suppressedBy) - { - for (AccordSafeState safeState : map.values()) - { - if (safeState.isUnsafe()) continue; - try { cache.release(safeState, this); } - catch (Throwable t) { suppressedBy.addSuppressed(t); } - } - } - - private void safeRelease(Map> map, AccordCache cache, Throwable suppressedBy) - { - for (AccordSafeState safeState : map.values()) - { - if (safeState.isUnsafe()) continue; - try { cache.release(safeState, this); } - catch (Throwable t) { suppressedBy.addSuppressed(t); } - } - } - - void revert() - { - if (commands != null) - commands.forEach((k, v) -> v.revert()); - if (commandsForKey != null) - commandsForKey.forEach((k, v) -> v.revert()); + }); + refs = null; } public class RangeTxnAndKeyScanner extends RangeTxnScanner @@ -844,17 +1190,17 @@ public abstract class AccordTask extends Task implements Function { @Override - public void onUpdate(AccordCacheEntry state) + public void onUpdate(AccordCacheEntry state) { if (ranges.contains(state.key())) - reference(state); + reference((AccordCacheEntry) state); } } final Set intersectingKeys = new ObjectHashSet<>(); - final KeyWatcher keyWatcher = new KeyWatcher(); final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); final AccordCache.Type.Instance commandsForKeyCache; + KeyWatcher keyWatcher = new KeyWatcher(); public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) { @@ -877,11 +1223,8 @@ public abstract class AccordTask extends Task implements Function entry) + private void reference(AccordCacheEntry entry) { - if (loading != null && loading.containsKey(entry.key())) - return; - switch (entry.status()) { default: throw new AssertionError("Unhandled Status: " + entry.status()); @@ -894,7 +1237,7 @@ public abstract class AccordTask extends Task implements Function extends Task implements Function extends Task implements Function extends Task implements Function extends Task implements Function extends Task implements Function extends Task implements Function byId = new TreeMap<>(summaries); return (CommandSummaries.ByTxnIdSnapshot) () -> byId; } @@ -1061,4 +1418,29 @@ public abstract class AccordTask extends Task implements Function entry = caches.commands().getUnsafe(txnId); + AccordCacheEntry entry = caches.commands().getUnsafe(txnId); if (entry == null) { loadFromDisk.add(txnId); @@ -115,7 +115,6 @@ public class InMemoryRangeIndex extends InMemoryRangeSummaryIndex implements Ran public void finish(Map into) { - cleanupExclusive(null); owner.search(this, into::put, null); } diff --git a/src/java/org/apache/cassandra/service/accord/RangeIndex.java b/src/java/org/apache/cassandra/service/accord/RangeIndex.java index a31179a4eb..228aad3781 100644 --- a/src/java/org/apache/cassandra/service/accord/RangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/RangeIndex.java @@ -82,7 +82,7 @@ public interface RangeIndex return null; } - public CommandSummaries.Summary ifRelevant(AccordCacheEntry state) + public CommandSummaries.Summary ifRelevant(AccordCacheEntry state) { if (state.key().domain() != Routable.Domain.Range) return null; diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java index 15cbd81ff2..e5ff023c80 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java @@ -178,7 +178,7 @@ public class DebugBlockedTxns private AsyncChain visitRootTxnAsync(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -188,7 +188,7 @@ public class DebugBlockedTxns private AsyncChain visitTxnAsync(CommandStore commandStore, TxnId txnId, Timestamp rootExecuteAt, @Nullable TokenKey byKey, int depth, boolean recurse) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -231,7 +231,7 @@ public class DebugBlockedTxns private AsyncChain visitKeysAsync(CommandStore commandStore, TokenKey key, Timestamp rootExecuteAt, int depth) { - return commandStore.chain(ExecutionContext.contextFor(RoutingKeys.of(key.toUnseekable()), SYNC, READ_WRITE, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequencedReadWrite(RoutingKeys.of(key.toUnseekable()), "Populate txn_blocked_by"), safeStore -> { visitKeysSync(safeStore, key, rootExecuteAt, depth); }); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java index e01f877d14..037a09f1c4 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java @@ -40,10 +40,10 @@ public class DebugExecution { private static final Logger logger = LoggerFactory.getLogger(DebugExecution.class); public static final boolean DEBUG_EXECUTION = CassandraRelevantProperties.ACCORD_DEBUG_EXECUTION.getBoolean(false); - private static final long REPORT_MIN_LATENCY_MICROS = 20_000; + private static final long REPORT_MIN_LATENCY_MICROS = 50_000; private static final long REPORT_CPU_RATIO = 2; - private static final long REPORT_MAX_LATENCY_MICROS = 50_000; - private static final long REPORT_CPU_MICROS = 10_000; + private static final long REPORT_MAX_LATENCY_MICROS = 100_000; + private static final long REPORT_CPU_MICROS = 50_000; // TODO (expected): use sharded histogram so we can report global stats public static class DebugExecutor @@ -94,43 +94,16 @@ public class DebugExecution long lockedForCpuMicros = (unlockedAtCpu - lockedAtCpu)/1000; if (lockedForMicros >= REPORT_MAX_LATENCY_MICROS) { - report("Held lock for {}us (cpu:{}us)\n", lockedForMicros, lockedForCpuMicros); + report("Held lock for {}us (cpu:{}us)", lockedForMicros, lockedForCpuMicros); } else if (lockedForMicros >= REPORT_MIN_LATENCY_MICROS && (lockedForMicros / lockedForCpuMicros) >= REPORT_CPU_RATIO) { - report("Held lock for {}us with cpu time only {}us\n", lockedForMicros, lockedForCpuMicros); + report("Held lock for {}us with cpu time only {}us", lockedForMicros, lockedForCpuMicros); } locked.increment(lockedForMicros); } } - public static class DebugExecutorLoop - { - final DebugExecutor owner; - long lockAt; - - public DebugExecutorLoop(DebugExecutor owner) - { - this.owner = owner; - } - - public void onLock() - { - lockAt = nanoTime(); - } - - public void onEnterLock() - { - owner.onEnterLock(lockAt); - lockAt = 0; - } - - public void onExitLock() - { - owner.onExitLock(); - } - } - public static class DebugExclusiveExecutor { public static DebugExclusiveExecutor maybeDebug(DebugExecutor owner, int commandStoreId) @@ -200,7 +173,7 @@ public class DebugExecution public List sanityCheck; // for AccordTask only long polledAt, preRunAt, runCompleteAt, completedAt; - long releasedRangeScannerAt, releasedCommandsAt, releasedCommandsForKeyAt; + long releasedRangeScannerAt, releasedStateAt; long runningAtCpu, runCompleteAtCpu; Thread thread; @@ -231,14 +204,9 @@ public class DebugExecution releasedRangeScannerAt = nanoTime(); } - public void onReleasedCommands() + public void onReleasedState() { - releasedCommandsAt = nanoTime(); - } - - public void onReleasedCommandsForKeys() - { - releasedCommandsForKeyAt = nanoTime(); + releasedStateAt = nanoTime(); } public void onCompleted(DebugExecutor owner) @@ -254,9 +222,9 @@ public class DebugExecution runningMicros = (runCompleteAt - task.runningAt) / 1000; owner.running.increment(runningMicros); } - long runToCleanMicros = (task.cleanupAt - runCompleteAt)/1000; + long runToCleanMicros = (task.completeAt - runCompleteAt) / 1000; owner.runToCleanup.increment(runToCleanMicros); - long cleanupMicros = (completedAt - task.cleanupAt)/1000; + long cleanupMicros = (completedAt - task.completeAt) / 1000; owner.cleanup.increment(cleanupMicros); long totalMicros = (completedAt - polledAt)/1000; owner.taskTotal.increment(totalMicros); @@ -266,7 +234,7 @@ public class DebugExecution String reason = ""; if (totalMicros > REPORT_MAX_LATENCY_MICROS) reason += "LONG TIME "; if (totalCpu > REPORT_CPU_MICROS) reason += "HIGH CPU "; - if ((totalMicros > REPORT_MIN_LATENCY_MICROS && (totalMicros/totalCpu) >= REPORT_CPU_RATIO)) reason += "LOW RATIO "; + if ((totalMicros > REPORT_MIN_LATENCY_MICROS && (totalCpu == 0 || (totalMicros/totalCpu) >= REPORT_CPU_RATIO))) reason += "LOW RATIO "; report("{}{}: total {}us cpu:{}us ({}), pollToRun {}us, running {}us, runToClean {}us, cleanup {}us", reason, task, totalMicros, totalCpu, thread, pollToRunMicros, runningMicros, runToCleanMicros, cleanupMicros); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java index a821db6340..7807eff9bc 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java @@ -221,7 +221,7 @@ public abstract class DebugTxnGraph private AsyncChain> submitRoot(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -232,7 +232,7 @@ public abstract class DebugTxnGraph private AsyncChain> submitParent(CommandStore commandStore, TxnId txnId, P param, Map infos, Set visitedParent, int depth) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -344,7 +344,7 @@ public abstract class DebugTxnGraph private AsyncChain populateTxnAsync(CommandStore commandStore, TxnId txnId, Map visited) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); visited.putIfAbsent(txnId, command == null || command.saveStatus() == SaveStatus.Uninitialised ? SaveInfo.NONE : new SaveInfo(command.saveStatus(), command.executeAtIfKnown())); }); diff --git a/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java b/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java index eba2e534b4..c3be7c87a0 100644 --- a/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java @@ -107,7 +107,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements } @Override - public void onUpdate(AccordCacheEntry state) + public void onUpdate(AccordCacheEntry state) { Summary summary = loader.ifRelevant(state); if (summary != null) @@ -162,7 +162,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements if (isMaybeRelevant(i)) { TxnId txnId = i.txnId; - AccordCacheEntry entry = c.getUnsafe(txnId); + AccordCacheEntry entry = c.getUnsafe(txnId); Invariants.expect(entry != null, "%s found interval %s but no matching transaction in cache", owner.commandStore, i); if (entry != null) { @@ -267,7 +267,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements } @Override - public void onUpdate(AccordCacheEntry state) + public void onUpdate(AccordCacheEntry state) { TxnId txnId = state.key(); if (txnId.is(Routable.Domain.Range)) @@ -334,7 +334,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements } @Override - public void onEvict(AccordCacheEntry state) + public void onEvict(AccordCacheEntry state) { TxnId txnId = state.key(); if (txnId.is(Routable.Domain.Range)) diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java index 04db462483..edff67507a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java @@ -22,12 +22,12 @@ import com.google.common.base.Throwables; import org.assertj.core.api.Assertions; -import accord.api.RoutingKey; import accord.local.CommandStores; import accord.local.ExecutionContext; import accord.local.LoadKeys; import accord.local.Node; import accord.local.cfk.CommandsForKey; +import accord.local.cfk.SafeCommandsForKey; import accord.primitives.Ranges; import accord.primitives.Routable; import accord.primitives.Txn; @@ -43,7 +43,6 @@ import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordSafeCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandsForKey; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.TokenRange; @@ -135,19 +134,16 @@ public class AccordDropTableBase extends TestBaseImpl TableId tableId = TableId.fromString(s); AccordService accord = (AccordService) AccordService.instance(); TxnId syntheticTxnId = new TxnId(TxnId.MAX_EPOCH, 0, Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range, new Node.Id(1)); - ExecutionContext ctx = ExecutionContext.contextFor(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), LoadKeys.SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = ExecutionContext.unsequencedReadWrite(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), "Test"); CommandStores stores = accord.node().commandStores(); for (int storeId : stores.ids()) { AccordCommandStore store = (AccordCommandStore) stores.forId(storeId); getBlocking(store.chain(ctx, input -> { AccordSafeCommandStore safe = (AccordSafeCommandStore) input; - for (RoutingKey key : safe.commandsForKeysKeys()) + for (SafeCommandsForKey safeCfk : safe.safeCommandsForKeys()) { - AccordSafeCommandsForKey safeCFK = (AccordSafeCommandsForKey) safe.ifLoadedAndInitialised(key); - if (safeCFK == null) // we read and found a key, but its null at load time... so ignore it - continue; - CommandsForKey cfk = safeCFK.current(); + CommandsForKey cfk = safeCfk.current(); CommandsForKey.TxnInfo minUndecided = cfk.minUndecidedManaged(); if (minUndecided != null) throw new AssertionError("Undecided txn: " + minUndecided); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java index 8270da184e..f7e52f108a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java @@ -355,6 +355,12 @@ public class AccordLoadTest extends AccordTestBase return builder.setArtificialLatencies(LATENCIES); } + private static SettingsBuilder populate(SettingsBuilder builder, int keyCount) + { + return builder.setKeySelector(roundrobin(keyCount)) + .setReadRatio(0f); + } + private static SettingsBuilder ycsbA(SettingsBuilder builder, int keyCount) { return builder.setKeySelector(ycsbZipfian(keyCount)) @@ -832,7 +838,7 @@ public class AccordLoadTest extends AccordTestBase if (storeId.get() >= 0) { CommandStore commandStore = service.node().commandStores().forId(storeId.get()); - List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.contextFor(candidate, "LoadTest"), safeStore -> { + List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.unsequenced(candidate, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(candidate); PartialDeps deps = safeCommand.current().partialDeps(); if (deps == null) @@ -855,7 +861,7 @@ public class AccordLoadTest extends AccordTestBase for (List info : result) { TxnId txnId = TxnId.parse(info.get(0)); - AccordService.getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "LoadTest"), safeStore -> { + AccordService.getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(txnId); if (safeCommand.current().executeAt != null) info.add(safeCommand.current().executeAt.toString()); @@ -915,7 +921,7 @@ public class AccordLoadTest extends AccordTestBase cluster.forEach(() -> { refresh(AccordExecutorMetrics.INSTANCE.elapsedRunning); refresh(AccordExecutorMetrics.INSTANCE.elapsed); - System.out.printf("%tT.%tL (%d %d %d %d %d %d)ms (%d %d %d %d %d %d)ms (%d %d %d %d %.0f, %d %d %d)us %d %d %d\n", nowMillis, nowMillis, + System.out.printf("%tT.%tL (%d %d %d %d %d %d)ms (%d %d %d %d %d %d)ms (%d %d %d)us %d %.0f (%d %d %d)us %d %d %d\n", nowMillis, nowMillis, getLatency(AccordCoordinatorMetrics.readMetrics.preacceptLatency, 0.5), getLatency(AccordCoordinatorMetrics.readMetrics.executeLatency, 0.5), getLatency(AccordCoordinatorMetrics.readMetrics.applyLatency, 0.5), @@ -1092,13 +1098,13 @@ public class AccordLoadTest extends AccordTestBase try { test.setup(); - test.testLoad(withArtificialLatencies(ycsbA(new SettingsBuilder(), 100_000) + test.testLoad(populate(new SettingsBuilder(), 1_000_000) // .setRatePerSecond(400).setMinRatePerSecond(200) // .setRatePerSecond(800).setMinRatePerSecond(200) .setRatePerSecond(1600).setMinRatePerSecond(200) .setIncreaseRatePerSecondInterval(5000) // .setTraceLast(5000) - ).build()); + .build()); } finally { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java index 8770df01c9..1b864aea1b 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java @@ -38,6 +38,7 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordSafeCommand; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.utils.ByteBufferUtil; @@ -86,7 +87,7 @@ public class AccordJournalConsistentExpungeTest extends TestBaseImpl Node node = service.node(); AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable()); - Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); + Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); TxnId txnId = TxnId.NONE; diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java new file mode 100644 index 0000000000..6bf1f46dfd --- /dev/null +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.simulator.test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.LockSupport; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +import javax.annotation.Nullable; + +import org.junit.Test; + +import accord.api.AsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; +import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; +import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.SignalLock; + +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; + +// TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit +public class AccordExecutorAndCacheTest extends SimulationTestBase +{ + @Test + public void signalLoopTest() + { + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new AccordAgent()), + 16); + } + + static class Submitted + { + final AtomicInteger nextId = new AtomicInteger(); + final AtomicInteger doneBefore = new AtomicInteger(); + final ConcurrentHashMap>> consequences = new ConcurrentHashMap<>(); + + boolean isDone() + { + return isDoneBetween(0, nextId.get()); + } + + boolean isDoneBefore(int before) + { + if (!isDoneBetween(doneBefore.get(), before)) + return false; + doneBefore.accumulateAndGet(before, Integer::max); + return true; + } + + boolean isDoneBetween(int from, int before) + { + for (int id = from ; id < before ; ++id) + { + for (Future future : consequences.get(id)) + { + if (!future.isDone()) + return false; + } + } + return true; + } + + Collection> start() + { + ConcurrentLinkedQueue> result = new ConcurrentLinkedQueue<>(); + + while (true) + { + int id = nextId.get(); + Object prev = consequences.putIfAbsent(id, result); + nextId.compareAndSet(id, id + 1); + if (prev == null) + return result; + } + } + } + + static class Control extends ConcurrentLinkedQueue + { + final Submitted submitted; + final AtomicInteger count = new AtomicInteger(); + final float cancelChance; + float processChance; + + Control(float cancelChance, Submitted submitted) + { + this(submitted, cancelChance, ThreadLocalRandom.current().nextFloat() * 0.5f); + } + + Control(Submitted submitted, float cancelChance, float processChance) + { + this.submitted = submitted; + this.cancelChance = cancelChance; + this.processChance = processChance; + } + + void submit(AsyncExecutor executor, Collection> consequences, Consumer>> run) + { + AsyncPromise future = new AsyncPromise<>(); + Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { + if (fail == null) future.trySuccess(null); + else + { + future.tryFailure(fail); + if (fail instanceof CancellationException) + run.accept(submitted.start()); + } + }); + consequences.add(future); + + if (cancel != null && ThreadLocalRandom.current().nextFloat() <= cancelChance) + { + add(cancel); + count.incrementAndGet(); + } + + if (count.get() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance) + { + int cancelCount = 0; + do + { + ++cancelCount; + + float delta = ThreadLocalRandom.current().nextFloat() - 0.5f; + if (delta < 0) processChance /= delta; + else processChance *= -delta; + if (processChance < 0.001f || processChance > 0.999f) + processChance = cancelChance; + } while (count.decrementAndGet() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance); + + // do outside of loop to avoid reentry + while (cancelCount-- > 0) + remove().cancel(); + } + } + } + + public void executorTest(SerializableSupplier supplier, int submissionThreads) + { + simulate(arr(() -> { + try + { + DatabaseDescriptor.daemonInitialization(); + ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); + AccordExecutor executor = supplier.get(); + Lock lock = executor.unsafeLock(); + ExclusiveAsyncExecutor sequentialExecutor = executor.newExclusiveExecutor(); + Executor lockExecutor = executorFactory().sequential("lock"); + + for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) + { + for (float lockChance : new float[] { 0f, 0.01f, 0.1f }) + { + for (float cancelChance : new float[] { 0f, 0.01f, 0.1f }) + { + System.out.println(String.format("sleepChance %.2f, lockChance %.2f, cancelChance %.2f", sleepChance, lockChance, cancelChance)); + List> done = new ArrayList<>(); + Submitted submitted = new Submitted(); + for (int i = 0; i < submissionThreads; ++i) + { + int id = i; + done.add(submit.submit(() -> { + try + { + submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance, new Control(cancelChance, submitted), submitted); + } + catch (ExecutionException | InterruptedException e) + { + throw new RuntimeException(e); + } + })); + } + for (Future f : done) + f.get(); + + if (!submitted.isDone()) + throw new AssertionError(); + } + } + } + } + catch (Throwable t) + { + throw new RuntimeException(t); + } + }), + () -> {}, 1L); + } + + private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance, Control control, Submitted submitted) throws ExecutionException, InterruptedException + { + ConcurrentLinkedQueue> awaitConsequences = new ConcurrentLinkedQueue<>(); + while (outerLoop-- > 0) + { + List>> allAwaitSubmitted = new ArrayList<>(); + for (int i = 0; i < innerLoop; ++i) + { + Collection> awaitSubmitted = submitted.start(); + allAwaitSubmitted.add(awaitSubmitted); + submitRecursive(lock, executor, sequentialExecutor, 1 + i, awaitSubmitted, awaitConsequences, submitted, sleepChance, lockChance, control); + } + + AtomicBoolean done = new AtomicBoolean(); + submitUntil(lock, lockExecutor, sleepChance, done::get); + for (Collection> awaitSubmitted : allAwaitSubmitted) + await(awaitSubmitted, CancellationException.class); + await(awaitConsequences, null); + done.set(true); + System.out.println("Loop " + id + '.' + (1 + outerLoop)); + } + } + + private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) + { + AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; + + control.submit(submitTo, consequences, nextConsequences -> { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + boolean locked = false; + if (rnd.nextFloat() < lockChance) + { + if (rnd.nextBoolean()) locked = lock.tryLock(); + else { locked = true; lock.lock(); } + } + if (ThreadLocalRandom.current().nextFloat() < 0.01f) + { + int expectDoneBefore = submitted.nextId.get(); + AsyncPromise afterConsequences = new AsyncPromise<>(); + executor.afterSubmittedAndConsequences(() -> { + if (!submitted.isDoneBefore(expectDoneBefore)) + throw new AssertionError(); + afterConsequences.setSuccess(null); + }); + awaitConsequences.add(afterConsequences); + } + try + { + if (count > 1) + submitRecursive(lock, executor, sequentialExecutor, count -1, nextConsequences, awaitConsequences, submitted, sleepChance, lockChance, control); + if (rnd.nextFloat() < sleepChance) + LockSupport.parkNanos(rnd.nextInt(10000, 100000)); + } + finally + { + if (locked) + lock.unlock(); + } + }); + } + + private static void submitUntil(Lock lock, Executor executor, float sleepChance, BooleanSupplier done) + { + if (done.getAsBoolean()) + return; + + executor.execute(() -> { + + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + boolean tryLock = rnd.nextBoolean(); + boolean locked = !tryLock; + if (tryLock) locked = lock.tryLock(); + else lock.lock(); + try + { + if (rnd.nextFloat() < sleepChance) + LockSupport.parkNanos(rnd.nextInt(10000, 100000)); + + submitUntil(lock, executor, sleepChance, done); + } + finally + { + if (locked) + lock.unlock(); + } + }); + } + + private static void await(Collection> await, @Nullable Class ignore) throws InterruptedException, ExecutionException + { + for (Future future : await) + { + try { future.get(); } + catch (ExecutionException e) + { + if (ignore == null || !(ignore.isInstance(e.getCause()))) + throw e; + } + } + } +} diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index 5d6408f676..69f3722bd0 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -91,6 +91,7 @@ import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.utils.FBUtilities; import static accord.local.ExecutionContext.contextFor; +import static accord.local.ExecutionContext.unsequencedReadWrite; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE; @@ -322,28 +323,28 @@ public class CompactionAccordIteratorsTest PartialDeps partialDeps = Deps.NONE.intersecting(AccordTestUtils.fullRange(txn)); PartialTxn partialTxn = txn.slice(commandStore.unsafeGetRangesForEpoch().currentRanges(), true); Route partialRoute = route.overlapping(commandStore.unsafeGetRangesForEpoch().currentRanges()); - getBlocking(commandStore.execute(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.execute(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.preaccept(safe, txnId, partialTxn, route, (a, b) -> {}); })); flush(commandStore); - getBlocking(commandStore.execute(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.execute(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, txnId, partialDeps, (a, b) -> {}); })); flush(commandStore); - getBlocking(commandStore.execute(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.execute(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, partialTxn, txnId, partialDeps, (a, b) -> {}); })); flush(commandStore); - getBlocking(commandStore.chain(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.chain(unsequencedReadWrite(txnId, route, "Test"), safe -> { return AccordTestUtils.processTxnResultDirect(safe, txnId, partialTxn, txnId); - }).flatMap(i -> i).flatMap(result -> commandStore.chain(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + }).flatMap(i -> i).flatMap(result -> commandStore.chain(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.apply(safe, txnId, route, txnId, partialDeps, partialTxn, result.left, result.right, (a, b) -> {}); }))); flush(commandStore); // The apply chain is asychronous, so it is easiest to just spin until it is applied // in order to have the updated state in the system table spinAssertEquals(true, 5, () -> { - return getBlocking(commandStore.submit(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + return getBlocking(commandStore.submit(unsequencedReadWrite(txnId, route, "Test"), safe -> { StoreParticipants participants = StoreParticipants.all(route); Command command = safe.get(txnId, participants).current(); return command.hasBeen(Status.Applied); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java index a91c082039..4c174a37b7 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java @@ -20,14 +20,24 @@ package org.apache.cassandra.service.accord; import org.junit.Assert; import org.junit.Test; +import accord.local.SafeState; + import org.apache.cassandra.service.accord.AccordCache.Type; +import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; import org.apache.cassandra.service.accord.AccordCacheEntry.Status; public class AccordCacheEntryTest { - static class CacheEntry extends AccordCacheEntry + static class TestSafeState extends SafeState implements AccordSafeState { - public CacheEntry(String key, Type.Instance instance) + @Override public AccordCacheEntry global() { return null; } + @Override public void preExecute(AccordTask owner, LockMode lockMode) {} + @Override public void postExecute(AccordTask owner) {} + } + + static class CacheEntry extends AccordCacheEntry + { + public CacheEntry(String key, Type.Instance instance) { super(key, instance); } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java index 5cb919a66e..aedf100df0 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java @@ -23,13 +23,18 @@ import org.agrona.concurrent.NoOpLock; import org.junit.Assert; import org.junit.Test; +import accord.local.ExecutionContext; +import accord.local.SafeState; + import org.apache.cassandra.cache.CacheSize; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.ManualExecutor; import org.apache.cassandra.metrics.AccordCacheMetrics; +import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; import static org.apache.cassandra.service.accord.AccordTestUtils.testLoad; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; @@ -40,85 +45,56 @@ public class AccordCacheTest { private static final long DEFAULT_NODE_SIZE = nodeSize(0); - private static abstract class TestSafeState implements AccordSafeState + private static abstract class TestSafeState & AccordSafeState> extends SafeState implements AccordSafeState { - protected boolean invalidated = false; - protected final AccordCacheEntry global; - private T original = null; + protected final AccordCacheEntry global; - public TestSafeState(AccordCacheEntry global) + public TestSafeState(AccordCacheEntry global) { this.global = global; } - public AccordCacheEntry global() + public AccordCacheEntry global() { return global; } - @Override - public T key() - { - return global.key(); - } + public final T key() { return global.key(); } - @Override - public T current() + public void preExecute(AccordTask owner, LockMode lockMode) { - return global.getExclusive(); - } - - @Override - public void set(T update) - { - global.setExclusive(update); - } - - @Override - public T original() - { - return original; - } - - @Override - public void preExecute() - { - original = global.getExclusive(); - } - - @Override - public Throwable failure() - { - return global.failure(); - } - - @Override - public void markUnsafe() - { - invalidated = true; - } - - @Override - public boolean isUnsafe() - { - return invalidated; + requireUninitialised(); + current = global.lockExclusive(owner, lockMode); + setSafe(); } } - private static class SafeString extends TestSafeState + private static class SafeString extends TestSafeState { - public SafeString(AccordCacheEntry global) + public SafeString(AccordCacheEntry global) { super(global); } + + @Override + public void postExecute(AccordTask owner) + { + global.releaseExclusive(this, owner); + } } - private static class SafeInt extends TestSafeState + private static class SafeInt extends TestSafeState { - public SafeInt(AccordCacheEntry global) + public SafeInt(AccordCacheEntry global) { super(global); } + + @Override + public void postExecute(AccordTask owner) + { + global.releaseExclusive(this, owner); + } } private static long emptyNodeSize() @@ -264,6 +240,7 @@ public class AccordCacheTest assertCacheMetrics(cacheMetrics, 0, 3, 3, 3); SafeString safeString = instance.acquire("1"); + safeString.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals(Status.LOADED, safeString.global.status()); assertCacheState(cache, 1, 3, nodeSize(1) * 3); @@ -392,6 +369,7 @@ public class AccordCacheTest assertCacheState(cache, 1, 1, nodeSize(1)); SafeString safeString2 = instance.acquire("0"); + safeString2.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals("0", safeString2.current()); Assert.assertEquals(Status.LOADED, safeString1.global.status()); Assert.assertEquals(2, instance.references("0", SafeString.class)); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java index fc691f60fc..a8bbec8fdf 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java @@ -18,8 +18,6 @@ package org.apache.cassandra.service.accord; -import java.util.NavigableMap; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; import org.junit.Assert; @@ -73,6 +71,7 @@ import org.apache.cassandra.utils.Pair; import static accord.primitives.Status.Durability.AllQuorums; import static com.google.common.collect.Iterables.getOnlyElement; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; import static org.apache.cassandra.service.accord.AccordService.getBlocking; import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.preaccepted; import static org.apache.cassandra.service.accord.AccordTestUtils.ballot; @@ -140,6 +139,7 @@ public class AccordCommandStoreTest promised, executeAt, txn, dependencies, accepted, waitingOn, result.left, TxnDataResult.PERSISTABLE); AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null)); + safeCommand.preExecute(new AccordTask<>(null, ExecutionContext.unsequenced(txnId, "Test"), null), RELEASE_QUEUE); safeCommand.set(expected); // In practice we should never need to save it with the condition boolean set // Not sure why this test does that @@ -168,10 +168,10 @@ public class AccordCommandStoreTest Command command2 = preaccepted(txnId2, txn, timestamp(1, clock.incrementAndGet(), 1)); AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null)); - cfk.initialize(); + cfk.preExecute(new AccordTask<>(null, ExecutionContext.unsequenced(txnId1, "Test"), null), RELEASE_QUEUE); - cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.contextFor(command1.txnId(), "Test")), command1).cfk()); - cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.contextFor(command1.txnId(), "Test")), command2).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command1).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command2).cfk()); CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run(); logger.info("E: {}", cfk); @@ -180,11 +180,4 @@ public class AccordCommandStoreTest Assert.assertEquals(cfk.current(), actual); } - - private static > NavigableMap toNavigableMap(V safeState) - { - TreeMap map = new TreeMap<>(); - map.put(safeState.key(), safeState); - return map; - } } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index 3509c2e506..189b08ed0d 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -174,7 +174,7 @@ public class AccordCommandTest Commit commit = Commit.SerializerSupport.create(txnId, route, 1, 1, Commit.Kind.StableWithTxnAndDeps, Ballot.ZERO, executeAt, partialTxn, deps, fullRoute); getBlocking(commandStore.execute(commit, commit::apply)); - getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, Keys.of(key).toParticipants(), LoadKeys.SYNC, READ_WRITE, "Test"), safeStore -> { + getBlocking(commandStore.execute(ExecutionContext.unsequencedReadWrite(txnId, Keys.of(key).toParticipants(), "Test"), safeStore -> { Command before = safeStore.ifInitialised(txnId).current(); Assert.assertEquals(commit.executeAt, before.executeAt()); Assert.assertTrue(before.hasBeen(Status.Committed)); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java index 54dee77c04..9ca2b11678 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java @@ -86,6 +86,7 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Condition; import static accord.local.ExecutionContext.contextFor; +import static accord.local.ExecutionContext.unsequencedReadWrite; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; import static accord.utils.Property.qt; @@ -127,7 +128,7 @@ public class AccordTaskTest AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "Test"), instance -> { + getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "Test"), instance -> { // TODO review: This change to `ifInitialized` was done in a lot of places and it doesn't preserve this property // I fixed this reference to point to `ifLoadedAndInitialised` and but didn't update other places Assert.assertNull(instance.ifInitialised(txnId)); @@ -141,7 +142,7 @@ public class AccordTaskTest AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "Test"), safe -> { + getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "Test"), safe -> { StoreParticipants participants = StoreParticipants.empty(txnId); SafeCommand command = safe.get(txnId, participants); Assert.assertNotNull(command); @@ -198,7 +199,7 @@ public class AccordTaskTest route.overlapping(ranges); PartialDeps deps = PartialDeps.builder(ranges, true).build(); - Command command = getBlocking(commandStore.submit(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + Command command = getBlocking(commandStore.submit(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.preaccept(safe, txnId, partialTxn, route, appendDiffToLog(commandStore)); CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, partialTxn, executeAt, deps, appendDiffToLog(commandStore)); return safe.ifInitialised(txnId).current(); @@ -246,7 +247,7 @@ public class AccordTaskTest Route partialRoute = route.overlapping(ranges); PartialDeps deps = PartialDeps.builder(ranges, true).build(); - Command command = getBlocking(commandStore.submit(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + Command command = getBlocking(commandStore.submit(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.preaccept(safe, txnId, partialTxn, route, appendDiffToLog(commandStore)); CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, executeAt, deps, appendDiffToLog(commandStore)); CheckedCommands.commit(safe, SaveStatus.Committed, Ballot.ZERO, txnId, route, partialTxn, executeAt, deps, appendDiffToLog(commandStore)); @@ -441,7 +442,7 @@ public class AccordTaskTest AssertionError error = null; for (T key : keys) { - AccordCacheEntry node = cache.getUnsafe(key); + AccordCacheEntry node = cache.getUnsafe(key); if (node == null) continue; try { @@ -476,7 +477,7 @@ public class AccordTaskTest { for (T key : keys) { - AccordCacheEntry node = cache.getUnsafe(key); + AccordCacheEntry node = cache.getUnsafe(key); if (node == null) continue; Awaitility.await("For node " + node.key() + " to complete") .atMost(Duration.ofMinutes(1)) diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index ae43676851..297d13a79d 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -57,6 +57,7 @@ import accord.local.Node; import accord.local.Node.Id; import accord.local.NodeCommandStoreService; import accord.local.SafeCommandStore; +import accord.local.SafeState; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.durability.DurabilityService; @@ -82,7 +83,6 @@ import accord.topology.TopologyManager; import accord.utils.SortedArrays.SortedArrayList; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; -import accord.utils.async.Cancellable; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.concurrent.ExecutorPlus; @@ -104,6 +104,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.AccordExecutor.IOTask; import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.journal.AccordJournal; @@ -114,7 +115,6 @@ import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Condition; -import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static accord.primitives.Routable.Domain.Key; @@ -123,6 +123,7 @@ import static accord.primitives.SaveStatus.PreAccepted; import static accord.primitives.Status.Durability.NotDurable; import static accord.primitives.Txn.Kind.Write; import static java.lang.String.format; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.service.accord.AccordService.getBlocking; @@ -162,16 +163,16 @@ public class AccordTestUtils } } - public static AccordCacheEntry loaded(K key, V value) + public static & AccordSafeState> AccordCacheEntry loaded(K key, V value) { - AccordCacheEntry global = new AccordCacheEntry<>(key, null); + AccordCacheEntry global = new AccordCacheEntry<>(key, null); global.initialize(value); return global; } public static AccordSafeCommand safeCommand(Command command) { - AccordCacheEntry global = loaded(command.txnId(), command); + AccordCacheEntry global = loaded(command.txnId(), command); return new AccordSafeCommand(global); } @@ -188,9 +189,9 @@ public class AccordTestUtils return new LoadExecutor<>() { @Override - public Cancellable load(P1 p1, P2 p2, AccordCacheEntry entry) + public IOTask load(P1 p1, P2 p2, AccordCacheEntry entry) { - Future future = executor.submit(() -> { + executor.submit(() -> { V v; try { v = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); } catch (Throwable t) @@ -200,19 +201,19 @@ public class AccordTestUtils } entry.loaded(v); }); - return () -> future.cancel(true); + return null; } }; } - public static void testLoad(ManualExecutor executor, AccordSafeState safeState, V val) + public static & AccordSafeState> void testLoad(ManualExecutor executor, S safeState, V val) { Assert.assertEquals(AccordCacheEntry.Status.WAITING_TO_LOAD, safeState.global().status()); safeState.global().load(loadExecutor(executor), null, null); Assert.assertEquals(AccordCacheEntry.Status.LOADING, safeState.global().status()); executor.runOne(); Assert.assertEquals(AccordCacheEntry.Status.LOADED, safeState.global().status()); - safeState.preExecute(); + safeState.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals(val, safeState.current()); } diff --git a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java index fae6c912fa..f62a784f1e 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java @@ -41,7 +41,7 @@ public class SimpleSimulatedAccordCommandStoreTest extends SimulatedAccordComman for (int i = 0, examples = 100; i < examples; i++) { TxnId id = AccordGens.txnIds().next(rs); - instance.process(ExecutionContext.contextFor(id, "Test"), (safe) -> { + instance.process(ExecutionContext.unsequenced(id, "Test"), (safe) -> { var safeCommand = safe.get(id, StoreParticipants.empty(id)); var command = safeCommand.current(); Assertions.assertThat(command.saveStatus()).isEqualTo(SaveStatus.Uninitialised); diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java index ce061f40bf..fcd7ccdde6 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java @@ -23,6 +23,7 @@ import java.util.concurrent.locks.LockSupport; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.BooleanSupplier; +import java.util.function.Function; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -180,7 +181,10 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase private static AccordTask operation(SimulatedAccordCommandStore instance, ExecutionContext ctx, Action action, BooleanSupplier delay) { - return new SimulatedOperation(instance.commandStore, ctx, action == Action.FAILURE ? SimulatedOperation.Action.FAILURE : SimulatedOperation.Action.SUCCESS); + + Function function = action == Action.FAILURE ? safeStore -> { throw new SimulatedFault("Operation failed for keys " + ctx.keys()); } + : safeStore -> null; + return AccordTask.create(instance.commandStore, ctx, function); } private static class Counter implements BiConsumer @@ -196,26 +200,6 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase } } - private static class SimulatedOperation extends AccordTask - { - enum Action { SUCCESS, FAILURE} - private final Action action; - - public SimulatedOperation(AccordCommandStore commandStore, ExecutionContext executionContext, Action action) - { - super(commandStore, executionContext); - this.action = action; - } - - @Override - public Void apply(SafeCommandStore safe) - { - if (action == Action.FAILURE) - throw new SimulatedFault("Operation failed for keys " + keys()); - return null; - } - } - private static class SimulatedLoadFunctionWrapper implements FunctionWrapper { final Supplier actions; diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 9975fd6b2b..448583ac00 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -505,8 +505,8 @@ public class CommandsForKeySerializerTest { int next = source.nextInt(commands.size()); Command command = commands.get(next); - if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(ExecutionContext.contextFor(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); - else cfk = cfk.update(new TestSafeCommandStore(ExecutionContext.contextFor(command.txnId(), "Test")), command).cfk(); + if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(ExecutionContext.unsequenced(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); + else cfk = cfk.update(new TestSafeCommandStore(ExecutionContext.unsequenced(command.txnId(), "Test")), command).cfk(); commands.set(next, commands.get(commands.size() - 1)); commands.remove(commands.size() - 1); } @@ -547,33 +547,10 @@ public class CommandsForKeySerializerTest static class TestSafeCommand extends SafeCommand { - final Command command; TestSafeCommand(Command command) { super(command.txnId); - this.command = command; - } - - @Override - public Command current() - { - return command; - } - - @Override - public void markUnsafe() - { - } - - @Override - public boolean isUnsafe() - { - return false; - } - - @Override - protected void set(Command command) - { + current = command; } } @@ -704,7 +681,7 @@ public class CommandsForKeySerializerTest { public TestSafeCommandStore(ExecutionContext context) { - super(context, TestCommandStore.INSTANCE); + super(context); } @Override protected CommandStoreCaches tryGetCaches() { return null; } @@ -712,6 +689,7 @@ public class CommandsForKeySerializerTest @Override protected SafeCommandsForKey add(SafeCommandsForKey safeCfk, CommandStoreCaches caches) { return null; } @Override protected SafeCommand getInternal(TxnId txnId) { return null; } @Override protected SafeCommandsForKey getInternal(RoutingKey key) { return null; } + @Override public CommandStore commandStore() { return TestCommandStore.INSTANCE; } @Override public DataStore dataStore() { return null; } @Override public Agent agent() { return null; } @Override public ProgressLog progressLog() { return null; } From 15bede148a02dffe16b43a8c05e0157156e19582 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Mon, 27 Jul 2026 17:27:22 +0100 Subject: [PATCH 05/10] split AccordExecutor et al into own package --- .../architecture/accord-architecture.adoc | 4 +- modules/accord | 2 +- .../cassandra/concurrent/CassandraThread.java | 16 +- .../db/virtual/AccordDebugKeyspace.java | 11 +- .../cassandra/db/virtual/QueriesTable.java | 2 +- .../apache/cassandra/io/util/PathUtils.java | 1 - .../cassandra/metrics/AccordCacheMetrics.java | 2 +- .../metrics/AccordExecutorMetrics.java | 4 +- .../metrics/AccordReplicaMetrics.java | 6 +- .../service/accord/AccordCommandStore.java | 131 +- .../service/accord/AccordCommandStores.java | 14 +- .../service/accord/AccordDurableOnFlush.java | 10 +- .../service/accord/AccordExecutor.java | 3563 ----------------- .../accord/AccordFetchCoordinator.java | 2 +- .../service/accord/AccordKeyspace.java | 12 +- .../service/accord/AccordSafeState.java | 47 - .../service/accord/AccordService.java | 1 + .../service/accord/IAccordService.java | 1 + .../service/accord/InMemoryRangeIndex.java | 1 + .../cassandra/service/accord/RangeIndex.java | 9 +- .../accord/debug/DebugBlockedTxns.java | 2 - .../service/accord/debug/DebugExecution.java | 2 +- .../AbstractLockLoop.java} | 36 +- .../AbstractLoop.java} | 14 +- .../AbstractSemiSyncSubmit.java} | 14 +- .../accord/{ => execution}/AccordCache.java | 114 +- .../{ => execution}/AccordCacheEntry.java | 644 +-- .../execution/AccordCacheEntryQueue.java | 518 +++ .../accord/execution/AccordExecutor.java | 823 ++++ .../AccordExecutorAsyncSubmit.java | 12 +- .../AccordExecutorSemiSyncSubmit.java | 12 +- .../AccordExecutorSignalLoop.java | 38 +- .../{ => execution}/AccordExecutorSimple.java | 16 +- .../AccordExecutorSyncSubmit.java | 22 +- .../service/accord/execution/CancelTask.java | 53 + .../accord/execution/ExclusiveExecutor.java | 390 ++ .../service/accord/execution/IOTask.java | 64 + .../service/accord/execution/IOTaskLoad.java | 89 + .../service/accord/execution/IOTaskSave.java | 77 + .../accord/execution/IOTaskWrapper.java | 75 + .../Loops.java} | 14 +- .../service/accord/execution/Plain.java | 110 + .../service/accord/execution/PlainChain.java | 87 + .../execution/PlainChainDebuggable.java | 67 + .../accord/execution/PlainRunnable.java | 99 + .../SafeTask.java} | 215 +- .../SaferCommand.java} | 18 +- .../SaferCommandStore.java} | 33 +- .../SaferCommandsForKey.java} | 23 +- .../service/accord/execution/SaferState.java | 47 + .../service/accord/execution/Task.java | 724 ++++ .../service/accord/execution/TaskInfo.java | 89 + .../service/accord/execution/TaskQueue.java | 110 + .../accord/execution/TaskQueueMulti.java | 612 +++ .../accord/execution/TaskQueueRunnable.java | 100 + .../accord/execution/TaskQueueStandalone.java | 61 + .../service/accord/execution/TaskRunner.java | 124 + .../service/accord/execution/Tranches.java | 254 ++ .../service/accord/execution/Unstoppable.java | 26 + .../accord/execution/Unterminatable.java | 24 + .../accord/journal/JournalRangeIndex.java | 8 +- .../service/accord/txn/TxnNamedRead.java | 2 +- .../cassandra/service/accord/txn/TxnRead.java | 2 +- .../service/accord/txn/TxnWrite.java | 2 +- .../accord/AccordExecutorBurnTest.java | 1 + .../distributed/test/TestBaseImpl.java | 2 +- .../test/accord/AccordBootstrapTest.java | 4 +- .../test/accord/AccordDropTableBase.java | 6 +- .../accord/AccordIncrementalRepairTest.java | 4 +- .../test/accord/AccordMetricsTest.java | 2 +- .../test/accord/AccordMoveTest.java | 4 +- ...AccordRecoverFromAvailabilityLossTest.java | 4 +- .../AccordJournalConsistentExpungeTest.java | 6 +- .../fuzz/topology/AccordRebootstrapTest.java | 2 +- ...actPairOfSequencesPaxosSimulationTest.java | 8 +- .../test/AccordExecutorAndCacheTest.java | 329 -- .../simulator/test/AccordExecutorTest.java | 55 +- .../org/apache/cassandra/cql3/CQLTester.java | 2 +- .../CompactionAccordIteratorsTest.java | 5 +- .../accord/AccordCommandStoreTest.java | 17 +- .../service/accord/AccordExpungeTest.java | 3 +- .../service/accord/AccordKeyspaceTest.java | 4 +- .../service/accord/AccordTestUtils.java | 59 +- ...{AccordTaskTest.java => SafeTaskTest.java} | 21 +- .../accord/SimulatedAccordCommandStore.java | 4 + .../accord/SimulatedAccordTaskTest.java | 5 +- .../{ => execution}/AccordCacheEntryTest.java | 14 +- .../{ => execution}/AccordCacheTest.java | 24 +- .../execution/AccordExecutionTestUtils.java | 72 + 89 files changed, 5328 insertions(+), 5034 deletions(-) delete mode 100644 src/java/org/apache/cassandra/service/accord/AccordExecutor.java delete mode 100644 src/java/org/apache/cassandra/service/accord/AccordSafeState.java rename src/java/org/apache/cassandra/service/accord/{AccordExecutorAbstractLockLoop.java => execution/AbstractLockLoop.java} (88%) rename src/java/org/apache/cassandra/service/accord/{AccordExecutorAbstractLoop.java => execution/AbstractLoop.java} (84%) rename src/java/org/apache/cassandra/service/accord/{AccordExecutorAbstractSemiSyncSubmit.java => execution/AbstractSemiSyncSubmit.java} (65%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordCache.java (90%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordCacheEntry.java (60%) create mode 100644 src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorAsyncSubmit.java (88%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorSemiSyncSubmit.java (89%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorSignalLoop.java (92%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorSimple.java (90%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorSyncSubmit.java (79%) create mode 100644 src/java/org/apache/cassandra/service/accord/execution/CancelTask.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/IOTask.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java rename src/java/org/apache/cassandra/service/accord/{AccordExecutorLoops.java => execution/Loops.java} (87%) create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Plain.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/PlainChain.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java rename src/java/org/apache/cassandra/service/accord/{AccordTask.java => execution/SafeTask.java} (84%) rename src/java/org/apache/cassandra/service/accord/{AccordSafeCommand.java => execution/SaferCommand.java} (77%) rename src/java/org/apache/cassandra/service/accord/{AccordSafeCommandStore.java => execution/SaferCommandStore.java} (85%) rename src/java/org/apache/cassandra/service/accord/{AccordSafeCommandsForKey.java => execution/SaferCommandsForKey.java} (72%) create mode 100644 src/java/org/apache/cassandra/service/accord/execution/SaferState.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Task.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Tranches.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java delete mode 100644 test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java rename test/unit/org/apache/cassandra/service/accord/{AccordTaskTest.java => SafeTaskTest.java} (95%) rename test/unit/org/apache/cassandra/service/accord/{ => execution}/AccordCacheEntryTest.java (88%) rename test/unit/org/apache/cassandra/service/accord/{ => execution}/AccordCacheTest.java (95%) create mode 100644 test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java diff --git a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc index ac7c6b7180..ebe8631b77 100644 --- a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc +++ b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc @@ -103,11 +103,11 @@ to ensure transactional integrity, changes to commands are tracked and are recorded into `Journal` for crash-recovery. `ProgressLog` and `CommandsForKey` are up -On Cassandra side, concurrent execution is controlled by `AccordTask`, +On Cassandra side, concurrent execution is controlled by `SafeTask`, which contains cache loading logic and persistence callbacks. Since Accord may potentially hold a large number of command states in memory, their states may be _shrunk_ to their binary representation to save some -memory, or they can get fully evicted. This also means that `AccordTask` +memory, or they can get fully evicted. This also means that `SafeTask` will have to reload relevant dependencies from preload context before command execution can begin. diff --git a/modules/accord b/modules/accord index 84ac4db032..8d2cdb35ae 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 84ac4db03251c3b842c98a29868e072792662f51 +Subproject commit 8d2cdb35ae96205b44da16582797bf42aa6fddcc diff --git a/src/java/org/apache/cassandra/concurrent/CassandraThread.java b/src/java/org/apache/cassandra/concurrent/CassandraThread.java index b013a5f0ad..f2702ed7c0 100644 --- a/src/java/org/apache/cassandra/concurrent/CassandraThread.java +++ b/src/java/org/apache/cassandra/concurrent/CassandraThread.java @@ -21,19 +21,21 @@ package org.apache.cassandra.concurrent; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.cassandra.metrics.ThreadLocalMetrics; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.Task; +import org.apache.cassandra.service.accord.execution.TaskRunner; import io.netty.util.concurrent.FastThreadLocalThread; -public class CassandraThread extends FastThreadLocalThread implements AccordExecutor.AccordTaskRunner +public class CassandraThread extends FastThreadLocalThread implements TaskRunner { private ThreadLocalMetrics threadLocalMetrics; private ExecutorLocals executorLocals; private AccordExecutor accordActiveExecutor; private AccordExecutor accordLockedExecutor; private int accordLockedExecutorDepth; - private volatile AccordExecutor.Task accordActiveTask; - private static final AtomicReferenceFieldUpdater accordActiveTaskUpdater = AtomicReferenceFieldUpdater.newUpdater(CassandraThread.class, AccordExecutor.Task.class, "accordActiveTask"); + private volatile Task accordActiveTask; + private static final AtomicReferenceFieldUpdater accordActiveTaskUpdater = AtomicReferenceFieldUpdater.newUpdater(CassandraThread.class, Task.class, "accordActiveTask"); private final ImmediateTaskHolder immediateTaskHolder; @@ -129,19 +131,19 @@ public class CassandraThread extends FastThreadLocalThread implements AccordExec accordLockedExecutor = null; } - public final AccordExecutor.Task accordActiveTask() + public final Task accordActiveTask() { return accordActiveTask; } // to be called only by the thread itself, so can (eventually) avoid any memory barriers - public final AccordExecutor.Task accordActiveSelfTask() + public final Task accordActiveSelfTask() { // TODO (expected): with newer JDK use accordActiveTaskUpdater.getPlain return accordActiveTask; } - public final void setAccordActiveTask(AccordExecutor.Task newActiveTask) + public final void setAccordActiveTask(Task newActiveTask) { accordActiveTaskUpdater.lazySet(this, newActiveTask); } diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 3a8c6e32bd..6e2fad78b4 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -132,11 +132,8 @@ import org.apache.cassandra.schema.Indexes; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.accord.AccordCache; -import org.apache.cassandra.service.accord.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStores; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.AccordKeyspace; import org.apache.cassandra.service.accord.AccordOperations; import org.apache.cassandra.service.accord.AccordService; @@ -154,6 +151,10 @@ import org.apache.cassandra.service.accord.debug.DebugTxnDepsAll; import org.apache.cassandra.service.accord.debug.DebugTxnDepsOrdered; import org.apache.cassandra.service.accord.debug.DebugTxnGraph; import org.apache.cassandra.service.accord.debug.TxnKindsAndDomains; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.TaskInfo; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.service.consensus.migration.TableMigrationState; @@ -331,8 +332,8 @@ public class AccordDebugKeyspace extends VirtualKeyspace int executorId = executor.executorId(); collector.partition(executorId).collect(rows -> { int uniquePos = 0; - AccordExecutor.TaskInfo prev = null; - for (AccordExecutor.TaskInfo info : executor.taskSnapshot()) + TaskInfo prev = null; + for (TaskInfo info : executor.taskSnapshot()) { if (prev != null && info.status() == prev.status() && info.position() == prev.position()) ++uniquePos; else uniquePos = 0; diff --git a/src/java/org/apache/cassandra/db/virtual/QueriesTable.java b/src/java/org/apache/cassandra/db/virtual/QueriesTable.java index e58e874c14..673d7335ad 100644 --- a/src/java/org/apache/cassandra/db/virtual/QueriesTable.java +++ b/src/java/org/apache/cassandra/db/virtual/QueriesTable.java @@ -25,8 +25,8 @@ import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import static java.lang.Long.max; import static java.util.concurrent.TimeUnit.NANOSECONDS; diff --git a/src/java/org/apache/cassandra/io/util/PathUtils.java b/src/java/org/apache/cassandra/io/util/PathUtils.java index 334763f271..780be3faaf 100644 --- a/src/java/org/apache/cassandra/io/util/PathUtils.java +++ b/src/java/org/apache/cassandra/io/util/PathUtils.java @@ -662,7 +662,6 @@ public final class PathUtils DeleteOnExit.clearOnExitThreads(); } - private static final class DeleteOnExit implements Runnable { private boolean isRegistered; diff --git a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java index 59efb88044..00627e019a 100644 --- a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java @@ -24,8 +24,8 @@ import java.util.function.ToLongFunction; import com.codahale.metrics.Gauge; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import static org.apache.cassandra.metrics.AccordMetricUtils.fromAccordService; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; diff --git a/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java b/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java index 3d5132131a..2c48fe7d3d 100644 --- a/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java @@ -23,10 +23,10 @@ import java.util.concurrent.TimeUnit; import com.codahale.metrics.Gauge; import org.apache.cassandra.metrics.ShardedDecayingHistograms.ShardedDecayingHistogram; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; -import static org.apache.cassandra.service.accord.AccordExecutor.HISTOGRAMS; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.HISTOGRAMS; public class AccordExecutorMetrics { diff --git a/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java b/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java index 6bc46f89c7..556ffb574d 100644 --- a/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java @@ -33,11 +33,11 @@ import org.apache.cassandra.metrics.LogLinearDecayingHistograms.LogLinearDecayin import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistogramsShard; import org.apache.cassandra.metrics.ShardedDecayingHistograms.ShardedDecayingHistogram; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.tracing.Tracing; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; -import static org.apache.cassandra.service.accord.AccordExecutor.HISTOGRAMS; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.HISTOGRAMS; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; public class AccordReplicaMetrics @@ -168,7 +168,7 @@ public class AccordReplicaMetrics private static LogLinearDecayingHistograms.Buffer buffer(SafeCommandStore safeStore) { - return ((AccordSafeCommandStore) safeStore).histogramBuffer(); + return ((SaferCommandStore) safeStore).histogramBuffer(); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index aeae9844e5..2dbbc07e00 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -40,7 +40,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.Agent; -import accord.api.AsyncExecutor; import accord.api.DataStore; import accord.api.Journal; import accord.api.LocalListeners; @@ -72,7 +71,6 @@ import accord.local.cfk.CommandsForKey; import accord.primitives.PartialTxn; import accord.primitives.Range; import accord.primitives.Ranges; -import accord.primitives.RoutableKey; import accord.primitives.Route; import accord.primitives.SaveStatus; import accord.primitives.Status; @@ -98,10 +96,17 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; -import org.apache.cassandra.service.accord.AccordExecutor.AccordTaskRunner; -import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfo; -import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.ExclusiveExecutor; +import org.apache.cassandra.service.accord.execution.SafeTask; +import org.apache.cassandra.service.accord.execution.SaferCommand; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandsForKey; +import org.apache.cassandra.service.accord.execution.TaskRunner; +import org.apache.cassandra.service.accord.execution.Unterminatable; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.journal.JournalRangeIndex; import org.apache.cassandra.service.accord.txn.TxnRead; @@ -138,10 +143,10 @@ public class AccordCommandStore extends CommandStore public static class Caches { private final AccordCache global; - private final AccordCache.Type.Instance commands; - private final AccordCache.Type.Instance commandsForKeys; + private final AccordCache.Type.Instance commands; + private final AccordCache.Type.Instance commandsForKeys; - Caches(AccordCache global, AccordCache.Type.Instance commandCache, AccordCache.Type.Instance commandsForKeyCache) + Caches(AccordCache global, AccordCache.Type.Instance commandCache, AccordCache.Type.Instance commandsForKeyCache) { this.global = global; this.commands = commandCache; @@ -153,29 +158,29 @@ public class AccordCommandStore extends CommandStore return global; } - public final AccordCache.Type.Instance commands() + public final AccordCache.Type.Instance commands() { return commands; } - public final AccordCache.Type.Instance commandsForKeys() + public final AccordCache.Type.Instance commandsForKeys() { return commandsForKeys; } } - public static final class ExclusiveCaches extends Caches implements CommandStoreCaches + public static final class ExclusiveCaches extends Caches implements CommandStoreCaches { private final AccordExecutor owner; - public ExclusiveCaches(AccordExecutor owner, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) + public ExclusiveCaches(AccordExecutor owner, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) { super(global, commands, commandsForKeys); this.owner = owner; } @Override - public AccordSafeCommand acquireIfLoaded(TxnId txnId) + public SaferCommand acquireIfLoaded(TxnId txnId) { // note: we must return false if the entry is locked to enforce ordering. // note importantly that this is also coupled to the safety of synchronously releasing ExclusiveExecutor.owner, @@ -184,7 +189,7 @@ public class AccordCommandStore extends CommandStore } @Override - public AccordSafeCommandsForKey acquireIfLoaded(RoutingKey key) + public SaferCommandsForKey acquireIfLoaded(RoutingKey key) { return commandsForKeys().acquireIfLoadedAndPermitted(key); } @@ -193,7 +198,7 @@ public class AccordCommandStore extends CommandStore public void close() { global().tryShrinkOrEvict(owner.unsafeLock()); - owner.unlock(AccordTaskRunner.get()); + owner.unlock(TaskRunner.get()); } } @@ -213,12 +218,10 @@ public class AccordCommandStore extends CommandStore = AtomicReferenceFieldUpdater.newUpdater(AccordCommandStore.class, Termination.class, "terminated"); static final AtomicLong nextSafeRedundantBeforeTicket = new AtomicLong(); - private static final AtomicLong lastSystemTimestampMicros = new AtomicLong(); - public final String loggingId; public final Journal journal; private final AccordExecutor sharedExecutor; - final AccordExecutor.ExclusiveExecutor exclusiveExecutor; + private final ExclusiveExecutor exclusiveExecutor; private final ExclusiveCaches caches; private final RangeIndex rangeIndex; private final TableId tableId; @@ -227,8 +230,8 @@ public class AccordCommandStore extends CommandStore volatile SafeRedundantBefore safeRedundantBefore; volatile Termination terminated; - private AccordSafeCommandStore current; - LogLinearDecayingHistograms.Buffer metricsBuffer; + private SaferCommandStore current; + public LogLinearDecayingHistograms.Buffer metricsBuffer; public AccordCommandStore(int id, NodeCommandStoreService node, @@ -257,15 +260,15 @@ public class AccordCommandStore extends CommandStore return a; }).orElseThrow(() -> Invariants.illegalState("CommandStore %d created with no ranges", id)); - final AccordCache.Type.Instance commands; - final AccordCache.Type.Instance commandsForKey; + final AccordCache.Type.Instance commands; + final AccordCache.Type.Instance commandsForKey; try (AccordExecutor.ExclusiveGlobalCaches exclusive = sharedExecutor.lockCaches()) { commands = exclusive.commands.newInstance(this); commandsForKey = exclusive.commandsForKey.newInstance(this); this.caches = new ExclusiveCaches(sharedExecutor, exclusive.global, commands, commandsForKey); } - this.exclusiveExecutor = sharedExecutor.executor(id); + this.exclusiveExecutor = sharedExecutor.newExclusiveExecutor(id); { AccordConfig.RangeIndexMode mode = getAccord().range_index_mode; @@ -298,12 +301,6 @@ public class AccordCommandStore extends CommandStore return exclusiveExecutor.inExecutor(); } - void tryPreSetup(AccordTask task) - { - if (inStore() && current != null) - task.preSetup(current.task); - } - public final TableId tableId() { return tableId; @@ -317,7 +314,7 @@ public class AccordCommandStore extends CommandStore // TODO (desired): we use this for executing callbacks with mutual exclusivity, // but we don't need to block the actual CommandStore - could quite easily // inflate a separate queue dynamically in AccordExecutor - public AsyncExecutor taskExecutor() + public ExclusiveExecutor exclusiveExecutor() { return exclusiveExecutor; } @@ -325,13 +322,13 @@ public class AccordCommandStore extends CommandStore public ExclusiveCaches lockCaches() { //noinspection LockAcquiredButNotSafelyReleased - caches.owner.lock(AccordTaskRunner.get()); + caches.owner.lock(TaskRunner.get()); return caches; } public ExclusiveCaches tryLockCaches() { - if (caches.owner.tryLock(AccordTaskRunner.get())) + if (caches.owner.tryLock(TaskRunner.get())) return caches; return null; } @@ -359,89 +356,50 @@ public class AccordCommandStore extends CommandStore journal.saveCommand(id, new CommandUpdate(before, after), onFlush); } - boolean validateCommand(TxnId txnId, Command evicting) - { - if (!Invariants.isParanoid()) - return true; - - Command reloaded = loadCommand(txnId); - return Objects.equals(evicting, reloaded); - } - @VisibleForTesting public void sanityCheckCommand(RedundantBefore redundantBefore, Command command) { ((AccordJournal) journal).sanityCheck(id, redundantBefore, command); } - CommandsForKey loadCommandsForKey(RoutableKey key) - { - CommandsForKey cfk = CommandsForKeyAccessor.load(id, (TokenKey) key); - if (cfk == null) - return null; - RedundantBefore.QuickBounds bounds = safeGetRedundantBefore().get(key); - if (!Invariants.expect(bounds != null, "No RedundantBefore information found when loading key %s", key)) - return cfk; - return cfk.withCleanCfkBeforeAtLeast(bounds.cleanCfkBefore(), false); - } - - boolean validateCommandsForKey(RoutableKey key, CommandsForKey evicting) - { - if (!Invariants.isParanoid()) - return true; - - CommandsForKey reloaded = CommandsForKeyAccessor.load(id, (TokenKey) key); - return Objects.equals(evicting, reloaded); - } - - @Nullable - Runnable saveCommandsForKey(RoutingKey key, CommandsForKey after, Object serialized) - { - return CommandsForKeyAccessor.systemTableUpdater(id, (TokenKey) key, after, serialized, nextSystemTimestampMicros()); - } - - public long nextSystemTimestampMicros() - { - return lastSystemTimestampMicros.accumulateAndGet(node.now(), (a, b) -> Math.max(a + 1, b)); - } @Override public AsyncChain chain(ExecutionContext context, Function function) { - return AccordTask.create(this, context, function).chain(); + return SafeTask.create(this, context, function).chain(); } @Override public AsyncChain chain(ExecutionContext context, Consumer consumer) { - return AccordTask.create(this, context, consumer).chain(); + return SafeTask.create(this, context, consumer).chain(); } @Override public AsyncChain chain(Callable call) { - return taskExecutor().chain(call); + return exclusiveExecutor().chain(call); } @Override public void execute(Runnable run) { - taskExecutor().execute(run); + exclusiveExecutor().execute(run); } @Override public boolean tryExecuteImmediately(Runnable run) { - return taskExecutor().tryExecuteImmediately(run); + return exclusiveExecutor().tryExecuteImmediately(run); } - public AccordSafeCommandStore begin(AccordSafeCommandStore safeStore) + public SaferCommandStore begin(SaferCommandStore safeStore) { require(current == null); current = safeStore; return current; } - public void complete(AccordSafeCommandStore store) + public void complete(SaferCommandStore store) { require(current == store); current = null; @@ -452,12 +410,12 @@ public class AccordCommandStore extends CommandStore return current != null; } - DataStore dataStore() + public DataStore dataStore() { return dataStore; } - ProgressLog progressLog() + public ProgressLog progressLog() { return progressLog; } @@ -759,7 +717,7 @@ public class AccordCommandStore extends CommandStore AsyncChain saveState(Descriptor descriptor) { - return chain((AccordExecutor.Unterminatable)() -> "Save State", safeStore -> { + return chain((Unterminatable)() -> "Save State", safeStore -> { File storeDir = storeSaveDir(); { File[] tmpDirs = listTmpSaveDirs(storeDir); @@ -897,7 +855,7 @@ public class AccordCommandStore extends CommandStore // TODO (expected): handle journal failures, and consider how we handle partial failures. // Very likely we will not be able to safely or cleanly handle partial failures of this logic, but decide and document. // TODO (desired): consider merging with PersistentField? This version is cheaper to manage which may be preferable at the CommandStore level. - static class SafeRedundantBefore + public static class SafeRedundantBefore { final long ticket; final RedundantBefore redundantBefore; @@ -912,6 +870,15 @@ public class AccordCommandStore extends CommandStore { return a.ticket >= b.ticket ? a : b; } + + public static Runnable updater(AccordCommandStore commandStore, RedundantBefore newRedundantBefore) + { + long ticket = nextSafeRedundantBeforeTicket.incrementAndGet(); + SafeRedundantBefore update = new SafeRedundantBefore(ticket, newRedundantBefore); + return () -> { + safeRedundantBeforeUpdater.accumulateAndGet(commandStore, update, SafeRedundantBefore::max); + }; + } } private @Nullable TableMetadata tableMetadata() diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index 14accb0fb8..228030ce39 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -54,13 +54,19 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.journal.Descriptor; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore.DurablyAppliedTo; -import org.apache.cassandra.service.accord.AccordExecutor.AccordExecutorFactory; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor.AccordExecutorFactory; +import org.apache.cassandra.service.accord.execution.AccordExecutorAsyncSubmit; +import org.apache.cassandra.service.accord.execution.AccordExecutorSemiSyncSubmit; +import org.apache.cassandra.service.accord.execution.AccordExecutorSignalLoop; +import org.apache.cassandra.service.accord.execution.AccordExecutorSimple; +import org.apache.cassandra.service.accord.execution.AccordExecutorSyncSubmit; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; -import static org.apache.cassandra.service.accord.AccordExecutor.constant; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.constant; import static org.apache.cassandra.service.accord.journal.ReplayMarkers.saveDirectory; import static org.apache.cassandra.utils.Clock.Global.nanoTime; diff --git a/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java b/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java index 6905b10bf9..554ea61cd2 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java @@ -34,8 +34,9 @@ import org.apache.cassandra.db.lifecycle.View; import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.accord.execution.Unstoppable; -class AccordDurableOnFlush implements BiConsumer +public class AccordDurableOnFlush implements BiConsumer { private static final Logger logger = LoggerFactory.getLogger(AccordDurableOnFlush.class); @@ -98,6 +99,11 @@ class AccordDurableOnFlush implements BiConsumer { return redundantBefore.toString(); } + + public static void reportMaybeTerminate(AccordCommandStore commandStore, int flags) + { + commandStore.maybeTerminated(ReportDurable.isCommandStoreFlush(flags), ReportDurable.isDataStoreFlush(flags)); + } } private Int2ObjectHashMap commandStores = new Int2ObjectHashMap<>(); @@ -185,7 +191,7 @@ class AccordDurableOnFlush implements BiConsumer static void notifyNow(CommandStore commandStore, ReportDurable report) { logger.debug("{} reporting flush with {}", commandStore, report); - commandStore.execute((AccordExecutor.Unstoppable) () -> "Report Durable", safeStore -> { + commandStore.execute((Unstoppable) () -> "Report Durable", safeStore -> { safeStore.reportDurable(report.redundantBefore, report.flags); }, commandStore.agent()); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java deleted file mode 100644 index 67a4d815c9..0000000000 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ /dev/null @@ -1,3563 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.service.accord; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.CancellationException; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.LockSupport; -import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.function.IntFunction; -import java.util.stream.Stream; - -import javax.annotation.Nullable; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import accord.api.Agent; -import accord.api.ExclusiveAsyncExecutor; -import accord.api.RoutingKey; -import accord.impl.AbstractAsyncExecutor; -import accord.local.Command; -import accord.local.ExecutionContext; -import accord.local.ExecutionContext.ExecutionSequence; -import accord.local.cfk.CommandsForKey; -import accord.messages.Accept; -import accord.messages.Commit; -import accord.messages.MessageType; -import accord.messages.MessageType.StandardMessage; -import accord.messages.Request; -import accord.primitives.Ballot; -import accord.primitives.SaveStatus; -import accord.primitives.TxnId; -import accord.utils.ArrayBuffers; -import accord.utils.IntrusiveHeapNode; -import accord.utils.IntrusivePriorityHeap; -import accord.utils.Invariants; -import accord.utils.QuadConsumer; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -import accord.utils.TinyEnumSet; -import accord.utils.TriConsumer; -import accord.utils.TriFunction; -import accord.utils.UnhandledEnum; -import accord.utils.async.AsyncCallbacks.CallAndCallback; -import accord.utils.async.AsyncCallbacks.RunOrFail; -import accord.utils.async.AsyncChain; -import accord.utils.async.AsyncChains; -import accord.utils.async.Cancellable; - -import org.apache.cassandra.cache.CacheSize; -import org.apache.cassandra.concurrent.CassandraThread; -import org.apache.cassandra.concurrent.DebuggableTask; -import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; -import org.apache.cassandra.concurrent.ExecutorLocals; -import org.apache.cassandra.concurrent.Shutdownable; -import org.apache.cassandra.config.AccordConfig; -import org.apache.cassandra.config.AccordConfig.QueueBalancingModel; -import org.apache.cassandra.config.AccordConfig.QueuePriorityModel; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.metrics.AccordCacheMetrics; -import org.apache.cassandra.metrics.AccordExecutorMetrics; -import org.apache.cassandra.metrics.AccordReplicaMetrics; -import org.apache.cassandra.metrics.AccordSystemMetrics; -import org.apache.cassandra.metrics.LogLinearDecayingHistograms; -import org.apache.cassandra.metrics.LogLinearDecayingHistograms.LogLinearDecayingHistogram; -import org.apache.cassandra.metrics.ShardedDecayingHistograms; -import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistogramsShard; -import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.UniqueSave; -import org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup; -import org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup; -import org.apache.cassandra.service.accord.AccordExecutor.Task.GroupKind; -import org.apache.cassandra.service.accord.AccordExecutor.Task.State; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExclusiveExecutor; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; -import org.apache.cassandra.utils.Clock; -import org.apache.cassandra.utils.Closeable; -import org.apache.cassandra.utils.WithResources; -import org.apache.cassandra.utils.concurrent.AsyncPromise; -import org.apache.cassandra.utils.concurrent.Condition; -import org.apache.cassandra.utils.concurrent.Future; - -import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY; -import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY_ATOMIC; -import static accord.primitives.Routable.Domain.Range; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.ORIG_HLC_FIFO; -import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER; -import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; -import static org.apache.cassandra.service.accord.AccordCache.registerJfrListener; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.COUNTER_LOWBITS; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.COUNTER_MASKS; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.minCounterValue; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.selectByOverflowBits; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.setOverflowWhenLessEqual; -import static org.apache.cassandra.service.accord.AccordExecutor.RunnableTaskQueue.RUNNABLE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.ACCEPT; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.APPLY; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.COMMIT; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.PREACCEPT; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.RANGE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.STABLE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.COMMAND_STORE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.OTHER; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_SCAN; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.SAVE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.MAX_TRANCHE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.CANCELLED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.EXECUTED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FAILED_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_OPTIONAL; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_REQUIRED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING_OR_EXECUTED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_OPTIONAL; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_REQUIRED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; - -/** - * NOTE: We assume that NO BLOCKING TASKS are submitted to this executor AND WAITED ON by another task executing on this executor. - * (as we do not immediately schedule additional threads for submitted tasks, but schedule new threads only if necessary when the submitting execution completes) - */ -public abstract class AccordExecutor implements CacheSize, LoadExecutor, Boolean>, SaveExecutor, Shutdownable, AbstractAsyncExecutor -{ - private static final Logger logger = LoggerFactory.getLogger(AccordExecutor.class); - - private static final QueuePriorityModel PRIORITY_MODEL; - private static final QueueBalancingModel BALANCING_MODEL; - private static final long AGE_TO_FIFO; - // PRIORITY_FAIR blends two strategies (flow: least fairly serviced; age: earliest-queued work) by deficit - // round-robin; weights of BLEND_TOTAL come from a single imbalance ramp (onset..onset+width) trading age->flow. - private static final int BLEND_SHIFT = 6, BLEND_TOTAL = 1 << BLEND_SHIFT; - private static final int FLOW_ONSET, FLOW_WIDTH_SHIFT; - private static final boolean BALANCE_BY_POSITION; - private static final long GLOBAL_QUEUE_LIMITS, EXCLUSIVE_QUEUE_LIMITS; - static final int NONSYNC_MIN_BATCH_SIZE, NONSYNC_MAX_BATCH_SIZE, NONSYNC_BLOCKED_LIMIT; - static final boolean NONSYNC_ENABLED; - - static - { - AccordConfig config = DatabaseDescriptor.getAccord(); - AGE_TO_FIFO = config.queue_priority_age_to_fifo.to(TimeUnit.MICROSECONDS); - PRIORITY_MODEL = config.queue_priority_model != null ? config.queue_priority_model : QueuePriorityModel.HLC_FIFO; - BALANCING_MODEL = config.queue_balancing_model != null ? config.queue_balancing_model : QueueBalancingModel.BLENDED_PRIORITY_PHASE_FAIR; - FLOW_ONSET = config.queue_flow_imbalance_onset == null ? 4 : config.queue_flow_imbalance_onset; - FLOW_WIDTH_SHIFT = config.queue_flow_imbalance_width_shift == null ? 5 : config.queue_flow_imbalance_width_shift; - NONSYNC_MIN_BATCH_SIZE = config.queue_nonsync_min_batch_size == null ? 16 : config.queue_nonsync_min_batch_size; - NONSYNC_MAX_BATCH_SIZE = config.queue_nonsync_max_batch_size == null ? 64 : config.queue_nonsync_max_batch_size; - NONSYNC_ENABLED = config.queue_nonsync_enabled == null || config.queue_nonsync_enabled; - NONSYNC_BLOCKED_LIMIT = config.queue_nonsync_blocked_limit == null ? 8 : config.queue_nonsync_blocked_limit; - Invariants.require(FLOW_ONSET >= 0 && FLOW_WIDTH_SHIFT >= 0); - switch (BALANCING_MODEL) - { - default: throw new UnhandledEnum(BALANCING_MODEL); - case PRIORITY_ONLY: - case BLENDED_PRIORITY_PHASE_FAIR: - BALANCE_BY_POSITION = true; - break; - case PHASE_ONLY: - case PHASE_FAIR: - BALANCE_BY_POSITION = false; - } - - { - // TODO (required): pick default max loads/saves/range loads based on number of threads - long global = COUNTER_MASKS, exclusive = COUNTER_MASKS; - global ^= (0x7fL ^ 1) << (RANGE_SCAN.ordinal() * 8); - if (config.queue_active_limits != null) - { - long[] limits = parseEnumParams(config.queue_active_limits, "queue_active_limits"); - global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), global, limits[0]); - exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), global, limits[1]); - } - GLOBAL_QUEUE_LIMITS = global; - EXCLUSIVE_QUEUE_LIMITS = exclusive; - } - - } - - public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); - - public interface AccordExecutorFactory - { - AccordExecutor get(int executorId, Mode mode, int threads, IntFunction name, Agent agent); - } - - public enum Mode { RUN_WITH_LOCK, RUN_WITHOUT_LOCK } - - public interface AccordTaskRunner - { - AccordExecutor accordActiveExecutor(); - void setAccordActiveExecutor(AccordExecutor newExecutor); - - AccordExecutor accordLockedExecutor(); - boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor); - void exitAccordLockedExecutor(); - - AccordExecutor.Task accordActiveTask(); - // to be called only by the thread itself, so can (eventually) avoid any memory barriers - AccordExecutor.Task accordActiveSelfTask(); - void setAccordActiveTask(AccordExecutor.Task newActiveTask); - - static AccordTaskRunner get() - { - return get(Thread.currentThread()); - } - - static AccordTaskRunner get(Thread thread) - { - return thread instanceof CassandraThread ? (CassandraThread)thread : ThreadLocalAccordTaskRunner.threadLocal.get(); - } - } - - public static final class ThreadLocalAccordTaskRunner implements AccordTaskRunner - { - private AccordExecutor lockedExecutor; - private AccordExecutor activeExecutor; - volatile Task activeTask; - - private static final ThreadLocal threadLocal = ThreadLocal.withInitial(ThreadLocalAccordTaskRunner::new); - - @Override - public AccordExecutor accordActiveExecutor() - { - return activeExecutor; - } - - @Override - public void setAccordActiveExecutor(AccordExecutor newExecutor) - { - activeExecutor = newExecutor; - } - - @Override - public AccordExecutor accordLockedExecutor() - { - return lockedExecutor; - } - - @Override - public boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) - { - if (lockedExecutor != null) - return false; - lockedExecutor = newLockedExecutor; - return true; - } - - @Override - public void exitAccordLockedExecutor() - { - lockedExecutor = null; - } - - @Override - public void setAccordActiveTask(Task newActiveTask) - { - activeTask = newActiveTask; - } - - @Override - public Task accordActiveTask() - { - return activeTask; - } - - @Override - public Task accordActiveSelfTask() - { - return activeTask; - } - } - - // WARNING: this is a shared object, so close is NOT idempotent - public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable - { - final AccordExecutor executor; - - public ExclusiveGlobalCaches(AccordExecutor executor, AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) - { - super(global, commands, commandsForKey); - this.executor = executor; - } - - @Override - public void close() - { - executor.beforeUnlockExternal(); - global.tryShrinkOrEvict(executor.lock); - executor.unlock(AccordTaskRunner.get()); - } - } - - public static class GlobalCaches - { - public final AccordCache global; - public final AccordCache.Type commands; - public final AccordCache.Type commandsForKey; - - public GlobalCaches(AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) - { - this.global = global; - this.commands = commands; - this.commandsForKey = commandsForKey; - } - } - - /** - * Tasks are separated into tranches based on their position. - * - * When we want to wait for all submitted tasks and their consequences to complete, we create a new tranche - * and track the number of tasks still extant for all prior tranches - once these reach zero we can signal - * the completion of the work. - */ - private static final class Tranches - { - class WithDeferred implements Runnable - { - final Runnable run; - final ArrayDeque deferred; - - WithDeferred(Runnable run, Runnable deferred) - { - this.run = run; - this.deferred = new ArrayDeque<>(1); - this.deferred.add(deferred); - } - - WithDeferred(Runnable run, ArrayDeque deferred) - { - this.run = run; - this.deferred = deferred; - } - - @Override - public void run() - { - Runnable register = deferred.poll(); - if (!deferred.isEmpty()) - { - Invariants.require(runs[firstIndex] != null); - Invariants.require(!(runs[firstIndex] instanceof WithDeferred)); - runs[firstIndex] = new WithDeferred(runs[firstIndex], deferred); - } - registerWait(register); - try { run.run(); } - catch (Throwable t) { owner.agent.onException(t); } - } - } - - final AccordExecutor owner; - - int firstTranche; - int firstIndex; - long[] mins = new long[8]; - int[] counts = new int[8]; - Runnable[] runs = new Runnable[8]; - - // the next tranche, that all new work is being collected against - // (and will move to the array accounting once a new wait is registered) - long nextMin; - int nextTranche; - int nextCount; - - private Tranches(AccordExecutor owner) - { - this.owner = owner; - } - - private int trancheToIndex(int tranche) - { - return firstIndex + trancheToIndexOffset(tranche); - } - - private int trancheToIndexOffset(int tranche) - { - int offset = tranche - firstTranche; - if (offset < 0) - offset += MAX_TRANCHE + 1; - return offset; - } - - int size() - { - return trancheToIndexOffset(nextTranche); - } - - int capacity() - { - return counts.length; - } - - int addNew(long position) - { - Invariants.require(position >= nextMin); - ++nextCount; - return nextTranche; - } - - void addInherited(int tranche, long position) - { - if (tranche == nextTranche) - { - Invariants.require(position >= nextMin); - ++nextCount; - } - else - { - int index = trancheToIndex(tranche); - Invariants.require(counts[index] > 0); - Invariants.require(mins[index] <= position); - ++counts[index]; - } - } - - void complete(int tranche) - { - if (tranche == nextTranche) - { - --nextCount; - } - else - { - - if (counts[trancheToIndex(tranche)] == 1) - owner.drainUnqueuedNewWorkExclusive(); // make sure we don't have any pending - - if (--counts[trancheToIndex(tranche)] == 0 && tranche == firstTranche) - { - do advance(); - while (firstTranche != nextTranche && counts[firstIndex] == 0); - } - - if (firstIndex >= counts.length / 2) - compact(); - } - } - - public void finishAll(long nextPosition) - { - while (firstTranche != nextTranche) - { - logger.warn("{} processed all pending tasks (<{}) but found {} waiting for {}", this, - nextPosition, counts[firstIndex], size() == 1 ? nextMin : mins[firstIndex + 1]); - advance(); - } - } - - private void advance() - { - Runnable run = runs[firstIndex]; - runs[firstIndex] = null; - ++firstIndex; - if (firstTranche == MAX_TRANCHE) firstTranche = 0; - else ++firstTranche; - try { run.run(); } - catch (Throwable t) { owner.agent.onException(t); } - } - - public void registerWait(Runnable run) - { - int newNextTranche = (nextTranche + 1) % (MAX_TRANCHE + 1); - if (newNextTranche == firstTranche) - { - Runnable cur = runs[firstIndex]; - if (cur instanceof WithDeferred) ((WithDeferred) cur).deferred.add(run); - else runs[firstIndex] = new WithDeferred(runs[firstIndex], run); - return; - } - - if ((firstIndex + size()) == capacity()) - growOrCompact(); - - int index = firstIndex + size(); - mins[index] = nextMin; - counts[index] = nextCount; - runs[index] = run; - nextMin = owner.minPosition = owner.nextPosition; - nextCount = 0; - nextTranche = newNextTranche; - } - - private void compact() - { - if (size() <= capacity()/4 && capacity() > 8) resize(capacity()/2); - else compact(mins, counts, runs); - } - - private void growOrCompact() - { - if (size() > capacity()/2) resize(capacity() * 2); - else compact(); - } - - private void resize(int newSize) - { - Invariants.require(newSize > 0); - long[] newMins = new long[newSize]; - int[] newCounts = new int[newSize]; - Runnable[] newRuns = new Runnable[newSize]; - compact(newMins, newCounts, newRuns); - mins = newMins; - counts = newCounts; - runs = newRuns; - } - - private void compact(long[] newMins, int[] newCounts, Runnable[] newRuns) - { - int size = size(); - System.arraycopy(mins, firstIndex, newMins, 0, size); - System.arraycopy(counts, firstIndex, newCounts, 0, size); - System.arraycopy(runs, firstIndex, newRuns, 0, size); - firstIndex = 0; - } - } - - final LogLinearDecayingHistograms histograms; - final LogLinearDecayingHistogram elapsedPreparingToRun; - final LogLinearDecayingHistogram elapsedWaitingToRun; - final LogLinearDecayingHistogram elapsedRunning; - final LogLinearDecayingHistogram elapsed; - final LogLinearDecayingHistogram keys; - public final AccordReplicaMetrics.Shard replicaMetrics; - - private final Lock lock; - final Agent agent; - final int executorId; - private final AccordCache cache; - private final ExclusiveGlobalCaches caches; - final AtomicLong uniqueCreatedAt = new AtomicLong(); - final DebugExecutor debug = DebugExecutor.maybeDebug(); - - private long maxWorkingSetSizeInBytes; - private long maxWorkingCapacityInBytes; - - final StandaloneTaskQueue> scanningRanges = new StandaloneTaskQueue<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning - final StandaloneTaskQueue> loading = new StandaloneTaskQueue<>(TinyEnumSet.encode(LOADING_REQUIRED, LOADING_OPTIONAL)); - final StandaloneTaskQueue> waitingOnCacheQueues = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL)); - final RunnableTaskQueue runnable = new RunnableTaskQueue<>(); - - private final Tranches tranches = new Tranches(this); - - /** - * Newly submitted work must take a position >= minPosition, but this condition does not apply to consequences of - * previously submitted work; this inherits the originating operation's position and tranche. - * This is to ensure afterSubmittedAndConsequences functions correctly. - */ - private long minPosition = 1; - private long nextPosition = 1; - int tasks; - - private boolean hasPausedLoading; - - private List waitingForQuiescence; - - AccordExecutor(Lock lock, int executorId, Agent agent) - { - this.lock = lock; - this.executorId = executorId; - this.cache = new AccordCache(this, 0); - this.agent = agent; - - final AccordCache.Type commands; - final AccordCache.Type commandsForKey; - commands = cache.newType(TxnId.class, COMMAND_ADAPTER, AccordCacheMetrics.CommandsCacheMetrics.newShard(lock)); - registerJfrListener(executorId, commands, "Command"); - - commandsForKey = cache.newType(RoutingKey.class, CFK_ADAPTER, AccordCacheMetrics.CommandsForKeyCacheMetrics.newShard(lock)); - registerJfrListener(executorId, commandsForKey, "CommandsForKey"); - - this.caches = new ExclusiveGlobalCaches(this, cache, commands, commandsForKey); - - DecayingHistogramsShard histogramsShard = HISTOGRAMS.newShard(lock); - this.histograms = histogramsShard.unsafeGetInternal(); - this.elapsedPreparingToRun = AccordExecutorMetrics.INSTANCE.elapsedPreparingToRun.forShard(histogramsShard); - this.elapsedWaitingToRun = AccordExecutorMetrics.INSTANCE.elapsedWaitingToRun.forShard(histogramsShard); - this.elapsedRunning = AccordExecutorMetrics.INSTANCE.elapsedRunning.forShard(histogramsShard); - this.elapsed = AccordExecutorMetrics.INSTANCE.elapsed.forShard(histogramsShard); - this.keys = AccordExecutorMetrics.INSTANCE.keys.forShard(histogramsShard); - this.replicaMetrics = new AccordReplicaMetrics.Shard(histogramsShard); - } - - public int executorId() - { - return executorId; - } - - public ExclusiveGlobalCaches lockCaches() - { - lock(AccordTaskRunner.get(Thread.currentThread())); - return caches; - } - - abstract boolean isInLoop(); - - public final Lock unsafeLock() - { - return lock; - } - - final void lock(AccordTaskRunner self) - { - long startAt = DEBUG_EXECUTION ? 0 : Clock.Global.nanoTime(); - if (!self.tryEnterAccordLockedExecutor(this)) - throw new UnsupportedOperationException("To ensure system performance, it is not permitted to lock multiple AccordExecutor simultaneously with the same thread"); - //noinspection LockAcquiredButNotSafelyReleased - lock.lock(); - if (DEBUG_EXECUTION) debug.onEnterLock(startAt); - } - - final void unlock(AccordTaskRunner self) - { - self.exitAccordLockedExecutor(); - if (DEBUG_EXECUTION) debug.onExitLock(); - lock.unlock(); - } - - final boolean tryLock(AccordTaskRunner self) - { - AccordExecutor locked = self.accordLockedExecutor(); - if (locked != null) - return false; - - return onTryLock(self, lock.tryLock()); - } - - final boolean onTryLock(AccordTaskRunner self, boolean result) - { - if (result && !self.tryEnterAccordLockedExecutor(this)) - { - // shouldn't be possible, we should have checked this already - lock.unlock(); - throw new IllegalStateException(); - } - return result; - } - - public AccordCache cacheExclusive() - { - Invariants.require(isOwningThread()); - return cache; - } - - public AccordCache cacheUnsafe() - { - return cache; - } - - final boolean hasAlreadyWaitingToRun() - { - return runnable.hasWaitingToRun(); - } - - void updateWaitingToRunExclusive() - { - // TODO (expected): this should not be invoked on every update of waiting to run - maybeUnpauseLoading(); - } - - // drain only new work; specifically leave anything that would call completeTask queued. - // this is to maintain invariants in Tranches.complete, where we may have some consequence of some earlier task - // pending but unqueued, so that we have not incremented its tranche count, and in the interim we set the tranche - // count to zero - void drainUnqueuedNewWorkExclusive() - { - } - - final Task pollAlreadyWaitingToRunExclusive() - { - Task next = runnable.poll(); - if (DEBUG_EXECUTION && next != null) DebugTask.get(next).onPolled(); - return next; - } - - public Stream active() - { - return Stream.of(); - } - - public void waitForQuiescence() - { - AccordTaskRunner self = AccordTaskRunner.get(); - Condition condition; - lock(self); - try - { - if (tasks == 0) - return; - - if (waitingForQuiescence == null) - waitingForQuiescence = new ArrayList<>(); - condition = Condition.newOneTimeCondition(); - waitingForQuiescence.add(condition); - } - finally - { - unlock(self); - } - condition.awaitThrowUncheckedOnInterrupt(); - } - - protected void notifyQuiescentExclusive() - { - if (waitingForQuiescence != null) - { - waitingForQuiescence.forEach(Condition::signalAll); - waitingForQuiescence = null; - } - tranches.finishAll(nextPosition); - } - - public void afterSubmittedAndConsequences(Runnable run) - { - AccordTaskRunner self = AccordTaskRunner.get(); - - lock(self); - try - { - drainUnqueuedNewWorkExclusive(); - if (tasks == 0) - { - run.run(); - return; - } - - tranches.registerWait(run); - } - finally - { - unlock(self); - } - } - - final void maybeUnpauseLoading() - { - if (!hasPausedLoading) - return; - - if (cache.weightedSize() < maxWorkingCapacityInBytes && !runnable.hasWaitingToRun()) - { - hasPausedLoading = false; - runnable.restart(RANGE_SCAN.ordinal()); - runnable.restart(LOAD.ordinal()); - runnable.restart(RANGE_LOAD.ordinal()); - } - } - - final void maybePauseLoading() - { - if (hasPausedLoading) - return; - - if (cache.weightedSize() >= maxWorkingCapacityInBytes || !runnable.hasWaitingToRun()) - { - AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); - hasPausedLoading = true; - runnable.stop(RANGE_SCAN.ordinal()); - runnable.stop(LOAD.ordinal()); - runnable.stop(RANGE_LOAD.ordinal()); - } - } - - public abstract boolean hasTasks(); - abstract void beforeUnlockExternal(); - abstract boolean isOwningThread(); - - public ExclusiveExecutor executor() - { - return new ExclusiveExecutor(this); - } - - public ExclusiveExecutor executor(int commandStoreId) - { - return new ExclusiveExecutor(this, commandStoreId); - } - - public ExclusiveAsyncExecutor newExclusiveExecutor() - { - return new ExclusiveExecutor(this); - } - - public void cancel(AccordTask task) - { - Invariants.require(task.commandStore.executor() == this, - "%s is a wrong command store for %s, should be %s", - this, task, task); - submit(AccordExecutor::cancelExclusive, CancelTask::new, task); - } - - @Override - public LoadRunnable load(AccordTask parent, Boolean isForRange, AccordCacheEntry entry) - { - LoadRunnable load = newLoad(entry, isForRange); - return submitPlainExclusive(parent, load); - } - - @Override - public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) - { - return submitPlainExclusive(null, new SaveRunnable(entry, identity, save)); - } - - private void submit(BiConsumer sync, Function async, P1 p1) - { - submit((e, c, p1a, p2a, p3) -> c.accept(e, p1a), (f, p1a, p2a, p3) -> f.apply(p1a), sync, async, p1, null, null); - } - - private void submit(TriConsumer sync, BiFunction async, P1 p1, P2 p2) - { - submit((e, c, p1a, p2a, p3) -> c.accept(e, p1a, p2a), (f, p1a, p2a, p3) -> f.apply(p1a, p2a), sync, async, p1, p2, null); - } - - private void submit(QuadConsumer sync, TriFunction async, P1 p1, P2 p2, P3 p3) - { - submit((e, c, p1a, p2a, p3a) -> c.accept(e, p1a, p2a, p3a), TriFunction::apply, sync, async, p1, p2, p3); - } - - private void submit(QuintConsumer sync, QuadFunction async, P1 p1, P2 p2, P3 p3, P4 p4) - { - submit(sync, async, p1, p1, p2, p3, p4); - } - - abstract void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4); - - void submit(AccordTask operation) - { - submit((self, task) -> task.submitExclusive(self), i -> i, operation); - } - - public void submitExclusive(Runnable runnable) - { - submitPlainExclusive(new PlainRunnable(null, runnable, OTHER)); - } - - private Cancellable submitPlain(Plain task) - { - submit(AccordExecutor::submitPlainExclusive, i -> i, task); - return task; - } - - private void submitPlainExclusive(Plain task) - { - registerExclusive(task); - task.onLoaded(); - ExclusiveExecutor executor = task.executor(); - if (executor == null) runnable.enqueue(task, true); - else executor.enqueue(task, true); - } - - Cancellable submitPlainExclusive(Task parent, GlobalGroup group, AbstractIOTask task) - { - return submitPlainExclusive(parent, new WrappedIOTask(task, group, parent.position, parent.tranche())); - } - - final T submitPlainExclusive(Task parent, T task) - { - Invariants.require(isOwningThread()); - task.setStateExclusive(WAITING_TO_RUN); - if (parent == null) registerExclusive(task); - else registerConsequenceExclusive(parent, task); - task.onLoaded(); - runnable.enqueue(task, true); - return task; - } - - private void registerConsequenceExclusive(Task parent, Task task) - { - ++tasks; - int tranche = parent.tranche(); - tranches.addInherited(tranche, parent.position); - task.position = parent.position; - task.setInheritedWithTranche(tranche); - } - - void registerExclusive(Task task) - { - ++tasks; - if (task.hasInherited()) - { - tranches.addInherited(task.tranche(), task.position); - } - else - { - long position = task.position; - if (position == 0) - { - task.position = position = nextPosition++; - } - else - { - long delta = nextPosition - position; - if (delta >= AGE_TO_FIFO) - task.position = position = nextPosition++; - else if (delta <= 0) - nextPosition = position + 1; - else if (position < minPosition) - task.position = position = minPosition; - } - task.setTranche(tranches.addNew(position)); - } - } - - final void cleanupTaskExclusive(Task task, boolean executed) - { - runnable.cleanup(task); - try { task.cleanupExclusive(this, executed); } - finally - { - cache.tryShrinkOrEvict(lock); - maybeUnpauseLoading(); - } - } - - final void unregisterExclusive(Task task) - { - int tranch = task.tranche(); - --tasks; - tranches.complete(tranch); - } - - final void cancelExclusive(AccordTask task) - { - State state = task.state(); - switch (state) - { - default: throw new UnhandledEnum(state); - case SCANNING_RANGES: - case LOADING_REQUIRED: - case LOADING_OPTIONAL: - case WAITING_ON_REQUIRED: - case WAITING_ON_OPTIONAL: - case WAITING_TO_RUN: - if (!task.hasIncrementalStarted()) - { - task.unqueueIfQueued(); - try { task.cancelExclusive(); } - finally { cleanupTaskExclusive(task, false); } - break; - } - case UNINITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase - case RUNNING: - case INCOMPLETE: - case EXECUTED: - case CANCELLED: - case FAILED_TO_LOAD: - // cannot safely cancel - } - } - - void onScannedRangesExclusive(AccordTask task, Throwable fail) - { - // the task may have already been cancelled, in which case we don't need to fail it - if (task.state().isExecuted()) - return; - - if (fail != null) failExclusive(task, fail, FAILED_TO_LOAD); - else task.rangeScanner().scannedExclusive(); - } - - private void failExclusive(AccordTask task, Throwable fail, State newState) - { - if (task.state().isExecuted()) - return; - - try { task.failExclusive(fail, newState); } - catch (Throwable t) { agent.onException(t); } - finally - { - task.unqueueIfQueued(); - cleanupTaskExclusive(task, false); - } - } - - private void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) - { - cache.saved(state, identity, fail); - } - - private void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail) - { - if (loaded.status() == EVICTED) - return; - - try (ArrayBuffers.BufferList> tasks = loaded.drainWaitingToLoad()) - { - if (fail != null) - { - for (AccordTask task : tasks) - failExclusive(task, fail, FAILED_TO_LOAD); - cache.failedToLoad(loaded); - } - else - { - cache.loaded(loaded, value); - for (AccordTask task : tasks) - task.onLoadOneExclusive(loaded); - } - } - - maybePauseLoading(); - } - - private Task inherit() - { - Thread thread = Thread.currentThread(); - AccordTaskRunner self = AccordTaskRunner.get(thread); - - if (self.accordActiveExecutor() != this) - return null; - - Task task = self.accordActiveSelfTask(); - if (task instanceof ExclusiveExecutorTask) - task = ((ExclusiveExecutorTask)task).queue.task; - return task; - } - - public Future submit(Runnable run) - { - Task inherit = inherit(); - PlainRunnable task = inherit == null ? new PlainRunnable(new AsyncPromise<>(), run, OTHER) - : new PlainRunnable(new AsyncPromise<>(), run, OTHER, inherit.position, inherit.tranche()); - submitPlain(task); - return task.result; - } - - public void execute(Runnable run) - { - Task inherit = inherit(); - PlainRunnable task = inherit == null ? new PlainRunnable(null, run, OTHER) - : new PlainRunnable(null, run, OTHER, inherit.position, inherit.tranche()); - submitPlain(task); - } - - @Override - public Cancellable execute(RunOrFail runOrFail) - { - Task inherit = inherit(); - PlainChain submit = inherit == null ? new PlainChain(runOrFail, null, ExclusiveGroup.OTHER) - : new PlainChain(runOrFail, null, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - return submitPlain(submit); - } - - public AsyncChain buildDebuggable(Callable call, Object describe) - { - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - Task inherit = inherit(); - return submitPlain(inherit == null ? new DebuggableChain(new CallAndCallback<>(call, callback), null, describe) - : new DebuggableChain(new CallAndCallback<>(call, callback), null, inherit.position, inherit.tranche(), describe)); - } - }; - } - - public void executeDirectlyWithLock(Runnable command) - { - AccordTaskRunner self = AccordTaskRunner.get(); - lock(self); - try - { - self.setAccordActiveExecutor(this); - command.run(); - } - finally - { - beforeUnlockExternal(); - self.exitAccordLockedExecutor(); - unlock(self); - } - } - - @Override - public void setCapacity(long bytes) - { - Invariants.require(isOwningThread()); - cache.setCapacity(bytes); - maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; - } - - public void setWorkingSetSize(long bytes) - { - Invariants.require(isOwningThread()); - maxWorkingSetSizeInBytes = bytes; - maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; - if (maxWorkingCapacityInBytes < maxWorkingSetSizeInBytes) - maxWorkingCapacityInBytes = Long.MAX_VALUE; - } - - @Override - public long capacity() - { - return cache.capacity(); - } - - @Override - public int size() - { - return cache.size(); - } - - @Override - public long weightedSize() - { - return cache.weightedSize(); - } - - protected static abstract class TaskRunner implements DebuggableTaskRunner - { - private volatile AccordTaskRunner wrapped; - - protected void setWrapped(AccordTaskRunner wrapped) - { - this.wrapped = wrapped; - } - - @Override - public DebuggableTask running() - { - Task running = wrapped.accordActiveTask(); - return running == null ? null : running.debuggable(); - } - } - - public static abstract class Task extends IntrusiveHeapNode - { - private static final int WAITING_ON_OPTIONAL_BIT = 1 << 5; - private static final int WAITING_TO_RUN_BIT = 1 << 6; - private static final int INCOMPLETE_BIT = 1 << 9; - - enum State - { - UNINITIALIZED(), - SCANNING_RANGES(UNINITIALIZED), - LOADING_REQUIRED(UNINITIALIZED, SCANNING_RANGES), - LOADING_OPTIONAL(UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED), - WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), - WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), - WAITING_TO_RUN(INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), - RUNNING(WAITING_TO_RUN), - EXECUTED(RUNNING), - INCOMPLETE(RUNNING), - FAILED_TO_LOAD(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), - FAILED_OTHER(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), - CANCELLED(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, RUNNING), - ; - - private final int permittedFrom; - public static final int WAITING = TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN); - public static final int WAITING_OR_RUNNING = WAITING | TinyEnumSet.encode(RUNNING); - public static final int RUNNING_OR_EXECUTED = WAITING | TinyEnumSet.encode(RUNNING, EXECUTED); - static final State[] VALUES = values(); - - static - { - // hack to allow us to create loops in our enum transition declarations - Invariants.require(INCOMPLETE_BIT == 1 << INCOMPLETE.ordinal()); - } - - State() - { - this.permittedFrom = 0; - } - - State(State ... permittedFroms) - { - this(0, permittedFroms); - } - - State(int additional, State ... permittedFroms) - { - int permittedFrom = additional; - for (State state : permittedFroms) - permittedFrom |= 1 << state.ordinal(); - this.permittedFrom = permittedFrom; - } - - boolean isPermittedFrom(int prevOrdinal) - { - return (permittedFrom & (1 << prevOrdinal)) != 0; - } - - boolean isExecuted() - { - return this.compareTo(EXECUTED) >= 0; - } - - boolean hasStarted() - { - return this.compareTo(RUNNING) >= 0; - } - - static State forOrdinal(int ordinal) - { - return VALUES[ordinal]; - } - } - - enum RunState - { - NONE, PERSISTING, SUCCESS, FAILED; - - private static final RunState[] VALUES = values(); - static RunState forOrdinal(int ordinal) - { - return VALUES[ordinal]; - } - } - - enum GlobalGroup - { - COMMAND_STORE, - LOAD, - SAVE, - OTHER, - RANGE_LOAD, - RANGE_SCAN, - } - - enum ExclusiveGroup - { - APPLY, - STABLE, - COMMIT, - ACCEPT, - OTHER, - RECOVER, - PREACCEPT, - RANGE, - } - - public enum GroupKind - { - EXCLUSIVE(ExclusiveGroup.values().length, EXCLUSIVE_GROUP_SHIFT), - GLOBAL(GlobalGroup.values().length, GLOBAL_GROUP_SHIFT), - NONE(0, 0); - - final int count; - final byte shift; - GroupKind(int count, int shift) - { - this.count = count; - this.shift = (byte) shift; - } - } - - private static final int STATE_MASK = 0xf; - private static final int GROUP_MASK = 0x7; - private static final int EXCLUSIVE_GROUP_SHIFT = 4; - private static final int GLOBAL_GROUP_SHIFT = 7; - - private static final int NONSYNC_BIT = 1 << 10; - private static final int CACHE_QUEUED_BIT = 1 << 11; - - private static final int INCREMENTAL_MASK = 0x3 << 12; - private static final int INCREMENTAL = 0x1 << 12; - private static final int INCREMENTAL_STARTED = 0x2 << 12; - private static final int INCREMENTAL_FINISHING = 0x3 << 12; - - private static final int SEQUENCED_SHIFT = 14; - private static final int SEQUENCED_MASK = 0x3 << SEQUENCED_SHIFT; - private static final int SEQUENCED_PRIORITY = 0x1 << SEQUENCED_SHIFT; - private static final int SEQUENCED_ATOMIC = 0x2 << SEQUENCED_SHIFT; - private static final int SEQUENCED_ATOMIC_AND_QUEUED = 0x3 << SEQUENCED_SHIFT; - - // spare two bits - - private static final int HAS_TRANCHE_BIT = 1 << 18; - private static final int HAS_INHERITED_BIT = 1 << 19; - private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 20; - - private static final int TRANCHE_SHIFT = 22; - static final int MAX_TRANCHE = 0x3ff; - - static - { - Invariants.require(SEQUENCED_PRIORITY == BY_PRIORITY.ordinal() << SEQUENCED_SHIFT); - Invariants.require(SEQUENCED_ATOMIC == BY_PRIORITY_ATOMIC.ordinal() << SEQUENCED_SHIFT); - Invariants.require(ExecutionSequence.values().length <= 3); - } - - public final WithResources resources; - Task next; - - long position; - private int info; - - // TODO (expected): do we need this? we should be able to determine the queue from state() if needed for e.g. cancellation - private TaskQueue queued; - - public final long createdAt; - // TODO (expected): expose via executors vtable - // TODO (expected): use just one long and some flag bits to indicate which point it represents, and report incrementally - public long loadedAt, runningAt, completeAt; - private byte runState; - - Task(GlobalGroup group) - { - resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - info = init(group, ExclusiveGroup.OTHER); - createdAt = nanoTime(); - } - - Task(ExclusiveGroup group) - { - resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - info = init(GlobalGroup.OTHER, group); - createdAt = nanoTime(); - } - - Task(GlobalGroup group, long position, int tranche) - { - this(group); - this.position = position; - setInheritedWithTranche(tranche); - } - - Task(ExclusiveGroup group, long position, int tranche) - { - this(group); - this.position = position; - setInheritedWithTranche(tranche); - } - - protected Task(ExecutionContext context, AtomicLong lastCreatedAt) - { - resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next < prev ? prev + 1 : next); - ExclusiveGroup group = ExclusiveGroup.OTHER; - TxnId txnId = context.primaryTxnId(); - if (txnId != null) - { - if (txnId.is(Range)) group = ExclusiveGroup.RANGE; - else - { - switch (PRIORITY_MODEL) - { - case HLC_FIFO: - case ORIG_HLC_FIFO: - { - // TODO (expected): port to ExecutionKind; also we aren't consistent about using Ballot - if (context instanceof Request) - { - MessageType type = ((Request) context).type(); - if (type instanceof StandardMessage) - { - switch ((StandardMessage)type) - { - case APPLY_REQ: - { - group = APPLY; - break; - } - case READ_EPHEMERAL_REQ: - case READ_REQ: - case STABLE_THEN_READ_REQ: - { - group = STABLE; - break; - } - case COMMIT_REQ: - { - Commit commit = (Commit) context; - if (PRIORITY_MODEL == ORIG_HLC_FIFO && !commit.ballot.equals(Ballot.ZERO)) - txnId = null; - if (commit.kind.saveStatus == SaveStatus.Stable) group = STABLE; - else group = COMMIT; - break; - } - case ACCEPT_REQ: - { - Accept accept = (Accept) context; - if (PRIORITY_MODEL == ORIG_HLC_FIFO && !accept.ballot.equals(Ballot.ZERO)) - txnId = null; - group = ExclusiveGroup.ACCEPT; - break; - } - case GET_EPHEMERAL_READ_DEPS_REQ: - case PRE_ACCEPT_REQ: - { - group = ExclusiveGroup.PREACCEPT; - break; - } - default: - { - txnId = null; - } - } - } - } - else - { - txnId = null; - } - break; - } - case FIFO: - { - txnId = null; - break; - } - } - } - } - - this.info = init(GlobalGroup.OTHER, group); - if (txnId != null) - this.position = txnId.hlc(); - } - - public final Task unwrap() - { - if (this instanceof ExclusiveExecutorTask) - return ((ExclusiveExecutorTask) this).queue.task; - return this; - } - - static Task unwrap(Task task) - { - return task == null ? null : task.unwrap(); - } - - public DebuggableTask debuggable() { return null; } - - abstract String toDescription(); - - abstract void submitExclusive(AccordExecutor owner); - - /** - * Prepare to run while holding the state cache lock - */ - void preRunExclusive() { setStateExclusive(RUNNING); } - - /** - * Run the command; the state cache lock may or may not be held depending on the executor implementation - */ - abstract void run(); - - /** - * Fail the command; the state cache lock may or may not be held depending on the executor implementation - */ - abstract void reportFailure(Throwable fail); - - final void failExclusive(Throwable fail, State newState) - { - try { setStateExclusive(newState); } - finally { reportFailure(fail); } - - } - - final void failExecution(Throwable fail) - { - Invariants.require(is(RUNNING)); - try { setRunState(RunState.FAILED); } - finally { reportFailure(fail); } - - } - - abstract boolean isNewWork(); - - /** - * Cleanup the command while holding the state cache lock - */ - void cleanupExclusive(AccordExecutor executor, boolean executed) - { - if (executed) setStateExclusive(EXECUTED); - else Invariants.require(state().isExecuted()); - executor.unregisterExclusive(this); - completeAt = nanoTime(); - if (runningAt != 0) - { - if (loadedAt == 0) - loadedAt = runningAt; - executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); - executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); - executor.elapsedRunning.increment(completeAt - runningAt, completeAt); - executor.elapsed.increment(completeAt - createdAt, completeAt); - } - if (DEBUG_EXECUTION) DebugTask.get(this).onCompleted(executor.debug); - } - - void cancelExclusive(AccordExecutor owner) {} - - @Nullable - final TaskQueue queued() - { - return queued; - } - - final void unqueueIfQueued() - { - if (queued != null) - { - queued.unqueue(this); - queued = null; - } - } - - final void unqueue(TaskQueue expected) - { - Invariants.require(queued == expected, "%s != %s", queued, expected); - queued.unqueue(this); - queued = null; - } - - final void unsetQueue(TaskQueue expected) - { - Invariants.require(queued == expected, "%s != %s", queued, expected); - queued = null; - } - - final void setQueue(TaskQueue queue) - { - Invariants.require(queued == null); - Invariants.require(isCompatible(queue)); - queued = queue; - } - - final void onRunning() - { - runningAt = nanoTime(); - if (DEBUG_EXECUTION) ((DebugTask)resources).onRunning(); - } - - final void onRunComplete() - { - if (DEBUG_EXECUTION) ((DebugTask)resources).onRunComplete(); - } - - final void onLoaded() - { - loadedAt = nanoTime(); - } - - final State state() - { - return State.forOrdinal(stateOrdinal()); - } - - final RunState runState() - { - return RunState.forOrdinal(runState); - } - - final Enum describeState() - { - State state = state(); - if (state == RUNNING || state == EXECUTED) - { - RunState runState = runState(); - if (runState == RunState.NONE) - return state; - return runState; - } - return State.forOrdinal(stateOrdinal()); - } - - private int stateOrdinal() - { - return info & STATE_MASK; - } - - final boolean is(State state) - { - return stateOrdinal() == state.ordinal(); - } - - final boolean isState(int stateBitSet) - { - return TinyEnumSet.contains(stateBitSet, stateOrdinal()); - } - - final boolean is(GlobalGroup group) - { - return globalGroupOrdinal() == group.ordinal(); - } - - final boolean is(ExclusiveGroup group) - { - return exclusiveGroupOrdinal() == group.ordinal(); - } - - final void override(GlobalGroup group) - { - info = (info & ~(GROUP_MASK << GLOBAL_GROUP_SHIFT)) | (group.ordinal() << GLOBAL_GROUP_SHIFT); - } - - final int compareTo(State state) - { - return stateOrdinal() - state.ordinal(); - } - - final void setStateExclusive(State state) - { - Invariants.require(state.isPermittedFrom(stateOrdinal()), "%s forbidden from %s", state, this, Task::reportBadStateTransition); - setStateUnsafe(state); - } - - final void setRunState(RunState state) - { - Invariants.require(isState(RUNNING_OR_EXECUTED)); - runState = (byte) state.ordinal(); - } - - private static String reportBadStateTransition(Task task) - { - return task.state() + " for " + task.toDescription(); - } - - final void setStateUnsafe(State state) - { - info = (info & ~STATE_MASK) | state.ordinal(); - } - - final int globalGroupOrdinal() - { - return (info >>> GLOBAL_GROUP_SHIFT) & GROUP_MASK; - } - - final int exclusiveGroupOrdinal() - { - return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; - } - - private boolean isCompatible(TaskQueue queue) - { - int self = stateOrdinal(); - return TinyEnumSet.contains(queue.states, self); - } - - final boolean isSync() - { - return 0 == (info & NONSYNC_BIT); - } - - final boolean isNonSync() - { - return !isSync(); - } - - final void setNonSyncExclusive() - { - info |= NONSYNC_BIT; - } - - final boolean isIncremental() - { - return 0 != (info & INCREMENTAL_MASK); - } - - final void setIncrementalExclusive() - { - info |= INCREMENTAL | NONSYNC_BIT; - } - - final boolean hasIncrementalStarted() - { - return (info & INCREMENTAL_MASK) >= INCREMENTAL_STARTED; - } - - final void setIncrementalStartedExclusive() - { - Invariants.require(isIncremental()); - if (!isIncrementalFinishing()) - info = (info & ~INCREMENTAL_MASK) | INCREMENTAL_STARTED; - } - - final boolean isIncrementalFinishing() - { - return (info & INCREMENTAL_MASK) >= INCREMENTAL_FINISHING; - } - - final void setIncrementalFinishingExclusive() - { - Invariants.require(isIncremental()); - info |= INCREMENTAL_FINISHING; - } - - final void setSequencedExclusive(ExecutionSequence sequence) - { - Invariants.require(isUnsequenced()); - info |= sequence.ordinal() << SEQUENCED_SHIFT; - } - - final boolean isUnsequenced() - { - return (info & SEQUENCED_MASK) == 0; - } - - final boolean isSequencedByPriority() - { - return (info & SEQUENCED_MASK) == SEQUENCED_PRIORITY; - } - - final boolean isSequencedByPriorityAtomic() - { - return (info & SEQUENCED_MASK) >= SEQUENCED_ATOMIC; - } - - final boolean isCacheQueuedFifo() - { - return (info & SEQUENCED_MASK) == SEQUENCED_ATOMIC_AND_QUEUED; - } - - final boolean isCacheQueued() - { - return 0 != (info & CACHE_QUEUED_BIT); - } - - // supersedes priority, in whichever order they're called - final void setCacheQueuedFifoExclusive() - { - Invariants.require(isSequencedByPriorityAtomic()); - info |= SEQUENCED_ATOMIC_AND_QUEUED | CACHE_QUEUED_BIT; - } - - final void setCacheQueuedExclusive() - { - info |= CACHE_QUEUED_BIT; - } - - final int tranche() - { - Invariants.require((info & HAS_TRANCHE_BIT) != 0); - return info >>> TRANCHE_SHIFT; - } - - final void setTranche(int tranche) - { - Invariants.require(tranche <= MAX_TRANCHE); - info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; - } - - final void setInheritedWithTranche(int tranche) - { - Invariants.require(tranche <= MAX_TRANCHE); - info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; - } - - final boolean hasInherited() - { - return (info & HAS_INHERITED_BIT) != 0; - } - - final void setInheritedRangeScan() - { - info = info | HAS_INHERITED_RANGE_SCAN_BIT; - } - - final boolean hasInheritedRangeScan() - { - return (info & HAS_INHERITED_RANGE_SCAN_BIT) != 0; - } - - static int init(GlobalGroup global, ExclusiveGroup exclusive) - { - return (global.ordinal() << GLOBAL_GROUP_SHIFT) | (exclusive.ordinal() << EXCLUSIVE_GROUP_SHIFT); - } - - static Task reverse(Task unqueued) - { - Task prev = null; - Task cur = unqueued; - while (cur != null) - { - Task next = cur.next; - cur.next = prev; - prev = cur; - cur = next; - } - return prev; - } - } - - // run the task even on a stopped commandStore - public interface Unstoppable extends ExecutionContext.Empty - { - } - - // run the task even on a terminated commandStore - public interface Unterminatable extends Unstoppable - { - } - - static final class ExclusiveExecutorTask extends Task - { - private final ExclusiveExecutor queue; - - ExclusiveExecutorTask(ExclusiveExecutor queue) - { - super(COMMAND_STORE); - this.queue = queue; - } - - @Override - String toDescription() - { - return queue.task.toDescription(); - } - - @Override void submitExclusive(AccordExecutor owner) { throw new UnsupportedOperationException(); } - - @Override - void preRunExclusive() - { - queue.preRunTask(); - } - - @Override - void run() - { - queue.runTask(); - } - - @Override - void reportFailure(Throwable t) - { - queue.task.reportFailure(t); - } - - @Override - boolean isNewWork() - { - throw new UnsupportedOperationException(); - } - - @Override - void cleanupExclusive(AccordExecutor executor, boolean executed) - { - queue.cleanupTask(executed); - } - - protected boolean isInHeap() - { - return super.isInHeap(); - } - } - - private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(ExclusiveExecutor.class, Thread.class, "owner"); - public final class ExclusiveExecutor extends MultiTaskQueue implements ExclusiveAsyncExecutor - { - final int commandStoreId; - final ExclusiveExecutorTask selfTask; - private Task task; - volatile Thread owner, waiting; - private boolean stopped; - private volatile boolean visibleStopped; - private boolean terminated; - - final DebugExclusiveExecutor debug; - - ExclusiveExecutor(AccordExecutor executor) - { - this(executor, -1); - } - - ExclusiveExecutor(AccordExecutor executor, int commandStoreId) - { - super(RUNNABLE, commandStoreId < 0 ? GroupKind.NONE : GroupKind.EXCLUSIVE, EXCLUSIVE_QUEUE_LIMITS); - this.commandStoreId = commandStoreId; - this.selfTask = new ExclusiveExecutorTask(this); - this.debug = DebugExclusiveExecutor.maybeDebug(executor.debug, commandStoreId); - } - - void preRunTask() - { - task.preRunExclusive(); - } - - void runTask() - { - Thread self = Thread.currentThread(); - if (!ownerUpdater.compareAndSet(this, null, self)) - { - if (DEBUG_EXECUTION) debug.onWaiting(); - Invariants.require(self == Thread.currentThread()); - waiting = self; - outer: do - { - while (true) - { - Thread owner = this.owner; - if (owner == self) break outer; - if (owner == null) continue outer; - LockSupport.park(); - } - } - while (!ownerUpdater.compareAndSet(this, null, self)); - } - waiting = null; - - if (stopped && reject(task)) - task.reportFailure(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).executionContext())); - else - task.run(); - - // NOTE: we can ONLY safely release owner here due to AccordCacheEntry locking, which remains in place until AccordTask.releaseResourcesExclusive - // this also relies on AccordSafeCommandStore$ExclusiveCaches.acquireIfLoaded returning false when the entry is locked - owner = null; - } - - private boolean reject(Task task) - { - if (!(task instanceof AccordTask)) - return true; - - ExecutionContext context = ((AccordTask) task).executionContext(); - return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); - } - - void cleanupTask(boolean executed) - { - try - { - task.unsetQueue(this); - task.cleanupExclusive(AccordExecutor.this, executed); - } - finally - { - active = 0; - task = super.pollMulti(); - if (DEBUG_EXECUTION) debug.onSetTask(task); - if (task != null) - { - selfTask.position = task.position; - selfTask.setStateUnsafe(WAITING_TO_RUN); - runnable.enqueue(selfTask, false); - } - } - } - - void enqueue(Task newTask, boolean incrementArrivals) - { - - if (task != null) - { - if (incrementArrivals) - runnable.incrementArrivals(selfTask); - // TODO (expected): restore some invariant here -// Invariants.require(selfTask.isInHeap() || selfTask.is(RUNNING)); - super.enqueueMulti(newTask, incrementArrivals); - } - else - { - Invariants.require(isEmptySingle()); - if (incrementArrivals) - incrementArrivals(newTask); - incrementDispatches(newTask); - task = newTask; - task.setQueue(this); - selfTask.position = newTask.position; - selfTask.setStateUnsafe(WAITING_TO_RUN); - runnable.enqueue(selfTask, incrementArrivals); - if (DEBUG_EXECUTION) debug.onSetTask(newTask); - } - } - - @Override - void unqueue(Task remove) - { - if (remove == task) removeCurrentTask(remove); - else super.unqueueMulti(remove); - } - - boolean tryUnqueueWaiting(Task remove) - { - if (remove == task) return tryRemoveCurrentTask(remove); - else return super.tryUnqueueWaiting(remove); - } - - private boolean tryRemoveCurrentTask(IntrusiveHeapNode remove) - { - if (runnable.isAssigned(selfTask)) - return false; - - removeCurrentTask(remove); - return true; - } - - private void removeCurrentTask(IntrusiveHeapNode remove) - { - Invariants.require(remove == task); - // cannot overwrite task while it is being executed - this cannot happen for AccordTask - // but can for other tasks that don't track their own state - - decrementDispatches(task); - task.unsetQueue(this); - task = pollMulti(); - if (DEBUG_EXECUTION) debug.onSetTask(task); - if (runnable.isWaiting(selfTask)) - { - if (task == null) runnable.unqueue(selfTask); - else - { - selfTask.position = task.position; - runnable.requeue(selfTask); - } - } - else - { - Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove); - if (task != null) - { - selfTask.position = task.position; - selfTask.setStateUnsafe(WAITING_TO_RUN); - runnable.enqueue(selfTask, false); - } - } - Invariants.require(task == null || runnable.isWaiting(selfTask)); - } - - public boolean inExecutor() - { - return owner == Thread.currentThread(); - } - - public boolean stopped() - { - return visibleStopped; - } - - void stop() - { - Invariants.require(inExecutor()); - this.stopped = true; - this.visibleStopped = true; - } - - void terminate() - { - Invariants.require(inExecutor()); - this.visibleStopped = this.terminated = this.stopped = true; - } - - @Override - public AsyncChain chain(Runnable run) - { - return AsyncChains.chain(this, run); - } - - @Override - public AsyncChain chain(Callable call) - { - return AsyncChains.chain(this, call); - } - - @Override - public AsyncChain flatChain(Callable> call) - { - return AsyncChains.flatChain(this, call); - } - - Task inherit() - { - Thread thread = Thread.currentThread(); - if (thread == owner) - return Task.unwrap(task); - return Task.unwrap(AccordTaskRunner.get(thread).accordActiveSelfTask()); - } - - @Override - public void execute(Runnable run) - { - Task inherit = inherit(); - PlainRunnable submit = inherit == null ? new PlainRunnable(null, run, this, ExclusiveGroup.OTHER) - : new PlainRunnable(null, run, this, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - submitPlain(submit); - } - - @Override - public Cancellable execute(RunOrFail runOrFail) - { - Task inherit = inherit(); - PlainChain submit = inherit == null ? new PlainChain(runOrFail, ExclusiveExecutor.this, ExclusiveGroup.OTHER) - : new PlainChain(runOrFail, ExclusiveExecutor.this, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - return submitPlain(submit); - } - - @Override - public boolean tryExecuteImmediately(Runnable run) - { - Thread thread = Thread.currentThread(); - Thread owner = this.owner; - if (owner != null && owner != thread) - return false; - - AccordTaskRunner self = AccordTaskRunner.get(thread); - AccordExecutor active = self.accordActiveExecutor(); - if (active != null && active != AccordExecutor.this) - return false; // prevent cross-executor locking/execution - - if (owner == null && !ownerUpdater.compareAndSet(this, null, thread)) - return false; - - if (active == null) - self.setAccordActiveExecutor(AccordExecutor.this); - - try { run.run(); } - catch (Throwable t) { agent.onException(t); } - finally - { - if (active == null) - self.setAccordActiveExecutor(null); - - if (owner == null) - { - Thread waiting = this.waiting; - Invariants.require(waiting != self); - this.owner = waiting; - if (waiting == null) // recheck, to ensure happens-before relation with a new waiter that expects any non-null owner to notify it - waiting = this.waiting; - if (waiting != null) - LockSupport.unpark(waiting); - } - } - return true; - } - } - - // has xSingle methods to distinguish from within MultiTaskQueue whether we're invoking the multi or single variation - static class TaskQueue extends IntrusivePriorityHeap - { - final int states; - public TaskQueue(int states) {this.states = states; } - - void unqueue(T task) - { - throw new UnsupportedOperationException(); - } - - @Override - public final int compare(T o1, T o2) - { - return Long.compare(o1.position, o2.position); - } - - @Override - protected final void ensureHeapified() - { - super.ensureHeapified(); - } - - protected final boolean requeueSingle(T requeue) - { - int oldIndex = updateNode(requeue); - if (oldIndex < 0) - return false; - - int newIndex = heapIndex(requeue); - return Math.min(oldIndex, newIndex) == 0 && heapifiedSize() > 0; - } - - final T peekSingle() - { - ensureHeapified(); - return peekNode(); - } - - final T pollSingle() - { - ensureHeapified(); - return pollNode(); - } - - final boolean isEmptySingle() - { - return isEmptyInternal(); - } - - final int enqueueSingle(T enqueue) - { - return insertNode(enqueue); - } - - final boolean unqueueSingle(T unqueue) - { - int heapIndex = heapIndex(unqueue); - removeNode(unqueue); - return heapIndex == 0; - } - - final boolean tryUnqueueSingle(T remove) - { - return removeNodeIfContains(remove); - } - - final T getSingle(int index) - { - return super.getNode(index); - } - - final boolean isQueuedSingle(T test) - { - return containsNode(test); - } - - @Override - public String toString() - { - return TinyEnumSet.toString(states, State::forOrdinal); - } - } - - static final class StandaloneTaskQueue extends TaskQueue - { - StandaloneTaskQueue(int states) - { - super(states); - } - - StandaloneTaskQueue(State state) - { - super(TinyEnumSet.encode(state)); - } - - void enqueue(T enqueue) - { - enqueue.setQueue(this); - enqueueSingle(enqueue); - } - - void unqueue(T unqueue) - { - unqueue.unsetQueue(this); - removeNode(unqueue); - } - - T peek() - { - return peekSingle(); - } - - T poll() - { - return pollSingle(); - } - - boolean isEmpty() - { - return isEmptySingle(); - } - } - - /** - * A {@link TaskQueue} sub-divided into up to eight per-group sub-queues (groups are phases for the exclusive - * executor, or work classes globally) plus a policy for choosing which sub-queue to serve next. The policy balances - * three competing concerns: ordering/priority (run the oldest / highest-priority work), fairness (do not let one - * group monopolise the executor), and throughput (do not let an even split leave a busy group's backlog growing - * without bound). - * - *

Packed counters

- * Per-group state is held in {@code long}s, one 7-bit lane per group; bit 7 of each byte is a spare "guard" bit used - * to run branch-free min/compare across all eight lanes at once (see {@code minCounters}). The lanes are: - *
    - *
  • {@code positions[]} - the head task's position (HLC, or submission order) per group; drives FIFO/priority.
    • - *
  • {@code recent} - a windowed count of recently served tasks per group (service received).
    • - *
  • {@code arrivals} - a windowed, size-biased count of recent arrivals per group (demand offered).
    • - *
  • {@code current} - tasks currently in flight per group, used to enforce {@code queue_active_limits}.
    • - *
  • {@code hasWork}/{@code dirty} - guard-bit masks: which groups have queued work / need a position refresh.
    • - *
- * - *

Windowing: a bounded memory of the past

- * {@code recent} and {@code arrivals} are not running totals. Whenever incrementing {@code recent} tips any lane to - * its maximum (0x80), both {@code recent} and {@code arrivals} are halved (see {@code incrementRecents}), - * and each lane also saturates at 0x7f. They are therefore an exponentially-decaying window keyed on service/time. - * This is what stops a one-off burst of arrivals granting a group a permanent boost: the burst saturates the lane - * briefly and then decays away as other work is served. - * - *

The fair selection counter

- * The fair models pick the group with the smallest {@code effective} counter, where per lane - * 
effective = min(0x7f, max(0, recent - arrivals) + bias)
- *
    - *
  • {@code max(0, recent - arrivals)} - a group whose arrivals have outpaced its recent service clamps to 0 and - * is therefore preferred; in steady state each group's service tracks its arrival rate, which bounds backlog. - * The clamp is symmetric, so a group cannot bank "credit" during a lull and later use it to monopolise service.
    • - *
- * - *

Choosing a group ({@code minGroup} / {@code pollByBlend})

- *
    - *
  • {@code PRIORITY_ONLY} - always the lowest {@code positions} (oldest / highest priority), ignoring fairness.
    • - *
  • {@code PHASE_OVERRIDE} - strict phase order: the lowest-index group with work, always.
    • - *
  • {@code PHASE_FAIR} - pure fairness: the smallest {@code effective}; ties broken by index, within a group by - * position. It has no priority fallback, so it is deliberately completing-first under contention.
    • - *
  • {@code PRIORITY_FAIR} (default) - a deficit round-robin blend of two strategies, chosen per poll: flow - * (smallest {@code effective}, i.e. least fairly serviced) and age (oldest {@code positions}, i.e. FIFO). - * The flow weight ramps up with the flow imbalance ({@code max-min} of {@code effective}) over - * {@code queue_flow_imbalance_onset..+width}, trading against age zero-sum; when balanced it is pure age. The - * ramp is smooth (not a hard mode switch), so there is no boundary to oscillate around and no hysteresis is - * needed. Age also drains stale/standing backlogs for free: a backlog's items are the oldest work, so age - * clears them without a separate backlog-aware strategy (the {@code effective} flow counter tracks arrival - * rate and is deliberately blind to standing stock).
    • - *
- * - *

Concurrency limits and eligibility

- * A group whose in-flight {@code current} count has reached its {@code queue_active_limits} limit is - * {@code saturated} and excluded from selection ({@code disabled}), as is any group with no queued work. - * - *

Note on {@code positions} freshness

- * {@code positions} is refreshed lazily from the sub-queue heads via the {@code dirty} mask; {@code dirty} must use - * the same guard-bit layout as {@code hasWork} so the refresh loop in {@code minGroupByPriority} actually runs - - * otherwise it silently degenerates to lowest-index-with-work and starves high-index / new-work groups. - * - *

NB it extends {@link TaskQueue} to keep the type hierarchy simple for method dispatch, and for efficiency for - * anonymous ExclusiveExecutors which do not use multiple queues, while letting ExclusiveExecutor share a parent - * class for both use cases. - */ - static abstract class MultiTaskQueue extends TaskQueue - { - private static final TaskQueue[] NO_QUEUES = new TaskQueue[0]; - private static final long[] NO_POSITIONS = new long[0]; - static final long COUNTER_OVERFLOWS = 0x8080808080808080L; - static final long COUNTER_MASKS = 0x7f7f7f7f7f7f7f7fL; - static final long COUNTER_LOWBITS = 0x0101010101010101L; - - final TaskQueue[] queues; - final long[] positions; - final byte groupShift; - final long limits; - - /** sets overflow bits for a queue that has been stopped */ - long stopped; - /** sets overflow bits for each counter when it needs its position updated */ - long dirty; - /** sets overflow bits for each counter when there's associated work */ - long hasWork; - /** Stores recent dequeue counts for up to 8 sub queues. */ - long dispatches; - // TODO (required): increment arrivals based on internal queue for ExclusiveExecutors - // also: experiment with decaying on arrival schedule rather than poll schedule, since this should respond to work growth more accurately - /** Stores recent enqueue counts for up to 8 sub queues. */ - long arrivals; - /** Stores currently-active counts for up to 8 sub queues. We can use this to impose limits on specific queues. */ - long active; - /** deficit-round-robin credits for the two PRIORITY_FAIR strategies (flow/age). */ - int creditFlow, creditAge; - - int waitingCount; - - MultiTaskQueue(int waitingStates, GroupKind groups, long limits) - { - super(waitingStates); - this.limits = limits; - int queueCount = groups.count; - Invariants.require(queueCount <= 8); - queues = queueCount == 0 ? NO_QUEUES : new TaskQueue[queueCount]; - positions = queueCount > 0 && BALANCE_BY_POSITION ? new long[queueCount] : NO_POSITIONS; - groupShift = groups.shift; - } - - final int group(Task task) - { - if (groupShift == 0) - return -1; - - return (task.info >>> groupShift) & Task.GROUP_MASK; - } - - void stop(int group) - { - stopped |= overflowBit(group); - } - - void restart(int group) - { - stopped &= ~overflowBit(group); - } - - final TaskQueue queue(Task task) - { - int group = group(task); - if (group < 0) - return this; - - return queue(group); - } - - final TaskQueue queue(int group) - { - TaskQueue queue = queues[group]; - if (queue == null) - queues[group] = queue = new TaskQueue<>(0); - - return queue; - } - - private int pollGroup() - { - if (hasWork == 0) - return -1; - - switch (BALANCING_MODEL) - { - default: throw new UnhandledEnum(PRIORITY_MODEL); - case PRIORITY_ONLY: return pollGroupByPriority(); - case PHASE_ONLY: return pollGroupByIndex(); - case PHASE_FAIR: return pollGroupByPhaseFair(); - case BLENDED_PRIORITY_PHASE_FAIR: return pollGroupByBlended(); - } - } - - private int pollGroupByPriority() - { - return pollGroupByPriority(unsaturatedWithWork()); - } - - private int pollGroupByPriority(long enabled) - { - long refresh = dirty & hasWork; - while (refresh != 0) - { - int bitIndex = Long.numberOfTrailingZeros(refresh); - int group = bitIndex / 8; - positions[group] = queues[group].peekSingle().position; - refresh ^= 1L << bitIndex; - } - dirty = 0; - - long minPosition = Long.MAX_VALUE; - int minGroup = -1; - long visit = enabled >>> 7; - while (visit != 0) - { - int bitIndex = Long.numberOfTrailingZeros(visit); - int group = bitIndex / 8; - long position = positions[group]; - if (position < minPosition) - { - minGroup = group; - minPosition = position; - } - visit ^= 1L << bitIndex; - } - - return minGroup; - } - - private int pollGroupByIndex() - { - long visit = (hasWork & unsaturated()) >>> 7; - if (visit == 0) - return -1; - - int bitIndex = Long.numberOfTrailingZeros(visit); - return bitIndex / 8; - } - - private int pollGroupByPhaseFair() - { - return minCounterIndex(recentFlowImbalances()); - } - - // PRIORITY_FAIR selection: a deficit round-robin blend of two strategies, chosen per poll: - // flow -> minCounterIndex(recent - arrivals + bias) (least fairly serviced) - // age -> minGroupByPriority() (earliest-queued work) - // wFlow = ramp(flow imbalance F = max-min of the selection counters); wAge = BLEND_TOTAL - wFlow. As flow gets - // uneven, polls trade from age to flow zero-sum; when balanced it is pure age (FIFO). A ramp (not a hard - // threshold) means there is no mode cliff to oscillate around, so no anti-oscillation penalty is needed. Age is - // also what drains a stale/standing backlog -- its items are the oldest -- so no separate stock strategy is needed. - private int pollGroupByBlended() - { - return pollGroupByBlended(saturatedOrWithoutWork()); - } - - private int pollGroupByBlended(long disabled) - { - long withoutWork = hasWork ^ COUNTER_OVERFLOWS; - long counters = recentFlowImbalances(); - long minMax = minMaxCounterValue(counters, withoutWork); - long min = minMax & 0x7f; - long max = minMax >>> 8; - int flowImbalance = (int) (max - min); - - int flowWeight = flowWeight(flowImbalance); - int priorityWeight = BLEND_TOTAL - flowWeight; - - creditFlow += flowWeight; - creditAge += priorityWeight; - - if (creditFlow >= creditAge) - { - creditFlow -= BLEND_TOTAL; - if (disabled != withoutWork) - min = minCounterValue(counters, disabled); - - return minCounterIndex(counters, min, disabled); - } - else - { - creditAge -= BLEND_TOTAL; - return pollGroupByPriority(disabled ^ COUNTER_OVERFLOWS); - } - } - - private long saturated() - { - return ((active | COUNTER_OVERFLOWS) - limits) & COUNTER_OVERFLOWS; - } - - private long unsaturated() - { - return saturated() ^ COUNTER_OVERFLOWS; - } - - private long unsaturatedWithWork() - { - return hasWork & unsaturated() & ~stopped; - } - - private long saturatedOrWithoutWork() - { - return (hasWork ^ COUNTER_OVERFLOWS) | saturated() | stopped; - } - - // arrivals is a windowed measure of ARRIVAL (incremented on enqueue, size-biased; decays on recent's overflow). - // Combined with recent (service) as effective = max(0, recent - arrivals): a queue whose arrivals outpace its - // service clamps to 0 and is preferred, so service converges to arrival rate (bounding the busy queue's backlog). - private long recentFlowImbalances() - { - return clampedSubtract(dispatches, arrivals); - } - - static long minCounterValue(long counters, long disabled) - { - long mins = counters; - mins |= overflowsToLowMasks(disabled); - mins = minCounters(mins, mins >>> 8); // each slot is min of slots [i..i+1] - mins = minCounters(mins, mins >>> 16); // each slot is min of slots [i..i+3] - mins = minCounters(mins, mins >>> 32); // each slot is min of slots [i..i+7] - return mins & 0x7f; - } - - static long minMaxCounterValue(long counters, long disabled) - { - long mins = counters; - long maxs = counters ^ COUNTER_MASKS; - long overflowMasks = overflowsToLowMasks(disabled); - mins |= overflowMasks; - maxs |= overflowMasks; - mins = minCounters(mins, mins >>> 8) & 0x007f007f007f007fL; // each slot is min of slots [i..i+1] - maxs = (minCounters(maxs, maxs << 8) & 0x7f007f007f007f00L); // each slot is min of slots ~[i..i+1] - long minmaxs = mins | maxs; - minmaxs = minCounters(minmaxs, minmaxs >>> 16); // each slot is min of slots [i..i+3] - minmaxs = minCounters(minmaxs, minmaxs >>> 32); // each slot is min of slots [i..i+7] - return (minmaxs ^ 0x7f00) & 0x7f7f; - } - - /** - * If provided two counters (containing 8 7 bit counters each), - * returns the minimum of each matching counter - */ - private static long minCounters(long a, long b) - { - // set overflow bits where a <= b - long selecta = setOverflowWhenLessEqual (a, b); - return selectByOverflowBits(selecta, a, b); - } - - static long setOverflowWhenLessEqual(long a, long b) - { - return ((b | COUNTER_OVERFLOWS) - a) & COUNTER_OVERFLOWS; - } - - // select a if overflow bit is set; b if it is unset - static long selectByOverflowBits(long selecta, long a, long b) - { - selecta = overflowsToLowMasks(selecta); - a &= selecta; - b &= ~selecta; - return a | b; - } - - static long overflowsToLowMasks(long v) - { - return v - (v >>> 7); - } - - private static int flowWeight(int flowImbalance) - { - if (flowImbalance <= FLOW_ONSET) return 0; - return Math.min(BLEND_TOTAL, ((flowImbalance - FLOW_ONSET) << BLEND_SHIFT) >>> FLOW_WIDTH_SHIFT); - } - - // per-lane max(0, a - b), carry-free: zero both a and b in lanes where a <= b, then subtract - private static long clampedSubtract(long a, long b) - { - long keep = ~overflowsToLowMasks(setOverflowWhenLessEqual(a, b)); - return (a & keep) - (b & keep); - } - - private int minCounterIndex(long counters) - { - return minCounterIndex(counters, saturatedOrWithoutWork()); - } - - private int minCounterIndex(long counters, long disabled) - { - return minCounterIndex(counters, minCounterValue(counters, disabled), disabled); - } - - private int minCounterIndex(long counters, long minCounterValue, long disabled) - { - long mins = minCounterValue * COUNTER_LOWBITS; - long select = ((mins | COUNTER_OVERFLOWS) - counters) & COUNTER_OVERFLOWS; - // now unset those overflow bits associated with disabled queues - select &= ~disabled; - if (select == 0) - return -1; - return (Long.numberOfTrailingZeros(select) - 7) / 8; - } - - final T pollMulti() - { - int group = pollGroup(); - if (group < 0) - { - // group < 0 can mean EITHER we don't have any nested queues OR those queues are either empty or DISABLED - T result = pollSingle(); - if (result != null) - --waitingCount; - return result; - } - - --waitingCount; - incrementActive(group); - incrementDispatches(group); - - TaskQueue queue = queues[group]; - T head = queue.pollSingle(); - // NOTE: must clear dirty when emptied, symmetrically with unqueue(): the fair selection paths - // never consume the dirty bit, so a group drained during a fairness episode would otherwise - // retain a stale dirty bit and NPE in minGroupByPriority.peekSingle() when balance is restored. - if (queue.isEmptySingle()) { unsetHasWork(group); unsetDirty(group); } - else setDirty(group); - return head; - } - - final void enqueueMulti(T task, boolean incrementArrivals) - { - task.setQueue(this); - int group = group(task); - if (group < 0) - { - enqueueSingle(task); - } - else - { - TaskQueue queue = queue(group); - int result = queue.enqueueSingle(task); - if (incrementArrivals) - incrementArrivals(group); - if (result < 0) setHasWork(group); - if (result != 0) setDirty(group); - } - ++waitingCount; - } - - final void requeue(T task) - { - int group = group(task); - if (group < 0) requeueSingle(task); - else - { - TaskQueue queue = queue(group); - Invariants.require(queue != null && queue.isQueuedSingle(task)); - if (queue.requeueSingle(task)) - setDirty(group); - } - } - - final void unqueueMulti(T task) - { - int group = group(task); - TaskQueue queue = group < 0 ? this : queue(task); - Invariants.require(queue.isQueuedSingle(task)); - unqueue(task, group, queue); - } - - // if there is an active collection, we return false and do not remove ourselves from it - boolean tryUnqueueWaiting(T task) - { - int group = group(task); - TaskQueue queue = group < 0 ? this : queue(task); - if (!queue.isQueuedSingle(task)) - return false; - - unqueue(task, group, queue); - return true; - } - - private void unqueue(T task, int group, TaskQueue queue) - { - task.unsetQueue(this); - boolean dirty = queue.unqueueSingle(task); - --waitingCount; - if (group >= 0) - { - if (queue.isEmptySingle()) - { - unsetHasWork(group); - unsetDirty(group); - } - else if (dirty) setDirty(group); - } - } - - final void incrementActive(int group) - { - active += lowBit(group); - } - - final void decrementActive(int group) - { - active -= lowBit(group); - } - - final void incrementDispatches(Task task) - { - int group = group(task); - if (group >= 0) - incrementDispatches(group); - } - - final void incrementDispatches(int group) - { - dispatches += lowBit(group); - if ((dispatches & COUNTER_OVERFLOWS) != 0) - { - dispatches = (dispatches >>> 1) & COUNTER_MASKS; - arrivals = (arrivals >>> 1) & COUNTER_MASKS; // arrivals (arrival) decays on the service/time clock - } - } - - final void decrementDispatches(Task task) - { - int group = group(task); - if (group >= 0) - decrementDispatches(group); - } - - final void decrementDispatches(int group) - { - long lowBit = lowBit(group); - dispatches -= lowBit; - dispatches += (dispatches >>> 7) & lowBit; - } - - final void incrementArrivals(Task task) - { - int group = group(task); - if (group >= 0) - incrementArrivals(group); - } - - final void incrementArrivals(int group) - { - int shift = group * 8; - long overflowBit = 0x80L << shift; - arrivals += 1L << shift; - // if we overflow, unset the overflow bit and set all other bits for the counter - long overflow = arrivals & overflowBit; - arrivals ^= overflow; - arrivals |= overflow - (overflow >>> 7); - } - - final void setHasWork(int group) - { - hasWork |= overflowBit(group); - } - - final void unsetHasWork(int group) - { - hasWork &= ~overflowBit(group); - } - - final void setDirty(int group) - { - dirty |= overflowBit(group); - } - - final void unsetDirty(int group) - { - dirty &= ~overflowBit(group); - } - - final boolean hasWaitingToRun() - { - return unsaturatedWithWork() != 0; - } - - final boolean isWaiting(T task) - { - return queue(task).isQueuedSingle(task); - } - - final int waitingCount() - { - return waitingCount; - } - - final long lowBit(int group) - { - return 1L << (group * 8); - } - - final long overflowBit(int group) - { - return 0x80L << (group * 8); - } - } - - static final class RunnableTaskQueue extends MultiTaskQueue - { - static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, RUNNING); - - final TaskQueue assigned; - - RunnableTaskQueue() - { - super(RUNNABLE, GroupKind.GLOBAL, GLOBAL_QUEUE_LIMITS); - this.assigned = new TaskQueue<>(0); - } - - T poll() - { - T next = pollMulti(); - if (next == null) - return null; - - assigned.enqueueSingle(next); - return next; - } - - void enqueue(T enqueue, boolean incrementArrivals) - { - enqueueMulti(enqueue, incrementArrivals); - } - - void unqueue(T unqueue) - { - if (assigned.isQueuedSingle(unqueue)) - { - int group = group(unqueue); - if (group >= 0) - decrementActive(group); - - unqueue.unsetQueue(this); - assigned.unqueueSingle(unqueue); - } - else - { - super.unqueueMulti(unqueue); - } - } - - int waitingOrAssignedCount() - { - return waitingCount + assigned.size(); - } - - boolean hasAssignedOrWaiting() - { - return waitingCount > 0 || !assigned.isEmptySingle(); - } - - boolean hasAssigned() - { - return !assigned.isEmptySingle(); - } - - boolean isAssigned(T task) - { - return assigned.isQueuedSingle(task); - } - - void cleanup(T task) - { - if (assigned.tryUnqueueSingle(task)) - { - int group = group(task); - if (group >= 0) - decrementActive(group); - task.unsetQueue(this); - } - } - } - - static class CancelTask extends Task - { - final Task cancel; - private CancelTask(Task cancel) - { - super(GlobalGroup.OTHER); - this.cancel = cancel; - } - - @Override void submitExclusive(AccordExecutor owner) { cancel.cancelExclusive(owner); } - @Override void preRunExclusive() { throw new UnsupportedOperationException(); } - @Override void run() { throw new UnsupportedOperationException(); } - @Override void reportFailure(Throwable fail) { throw new UnsupportedOperationException(); } - @Override boolean isNewWork() { return false; } - @Override String toDescription() { return "Cancel " + cancel.toDescription(); } - } - - static IntFunction constant(O out) - { - return ignore -> out; - } - - abstract class Plain extends Task implements Cancellable - { - Plain(GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - } - - Plain(ExclusiveGroup group, long position, int tranche) - { - super(group, position, tranche); - } - - Plain(GlobalGroup group) - { - super(group); - } - - Plain(ExclusiveGroup group) - { - super(group); - } - - abstract ExclusiveExecutor executor(); - - @Override - public void cancel() - { - submit((e, c) -> c.cancelExclusive(e), CancelTask::new, this); - } - - void cancelExclusive(AccordExecutor owner) - { - ExclusiveExecutor executor = executor(); - if ((executor == null ? runnable : executor).tryUnqueueWaiting(this)) - { - try { failExclusive(new CancellationException(), CANCELLED); } - catch (Throwable t) { agent.onException(t); } - finally { cleanupTaskExclusive(this, false); } - } - } - - @Override - final void submitExclusive(AccordExecutor owner) - { - setStateExclusive(WAITING_TO_RUN); - owner.submitPlainExclusive(this); - } - - @Override - protected boolean isNewWork() - { - return true; - } - } - - class PlainRunnable extends Plain implements Cancellable - { - final @Nullable AsyncPromise result; - final Runnable run; - final @Nullable ExclusiveExecutor executor; - - PlainRunnable(AsyncPromise result, Runnable run, GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - this.result = result; - this.run = run; - this.executor = null; - } - - PlainRunnable(AsyncPromise result, Runnable run, GlobalGroup group) - { - super(group); - this.result = result; - this.run = run; - this.executor = null; - } - - PlainRunnable(AsyncPromise result, Runnable run, ExclusiveExecutor executor, ExclusiveGroup group, long position, int tranche) - { - super(group, position, tranche); - this.result = result; - this.run = run; - this.executor = executor; - } - - PlainRunnable(AsyncPromise result, Runnable run, ExclusiveExecutor executor, ExclusiveGroup group) - { - super(group); - this.result = result; - this.run = run; - this.executor = executor; - } - - @Override - String toDescription() - { - // TODO (expected): ensure this is usefully descriptive, or accept a separate description - return run.toString(); - } - - @Override - protected void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - if (result != null) - result.trySuccess(null); - onRunComplete(); - } - - @Override - protected void reportFailure(Throwable t) - { - if (result != null) - result.tryFailure(t); - agent.onException(t); - } - - @Override - ExclusiveExecutor executor() - { - return executor; - } - } - - // a task that may be submitted to this executor or another - public abstract class IOTask extends Plain implements Cancellable, DebuggableTask - { - IOTask(GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - } - - IOTask(GlobalGroup group) - { - super(group); - } - - abstract void postRunExclusive(); - - @Override - void cleanupExclusive(AccordExecutor executor, boolean executed) - { - super.cleanupExclusive(executor, executed); - postRunExclusive(); - } - - @Override - ExclusiveExecutor executor() - { - return null; - } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } - } - - static class FailureHolder - { - static final FailureHolder NOT_STARTED = new FailureHolder(new RuntimeException("Not started")); - - final Throwable fail; - - FailureHolder(Throwable fail) - { - this.fail = fail; - } - } - - LoadRunnable newLoad(AccordCacheEntry entry, boolean isForRange) - { - return new LoadRunnable<>(entry, isForRange ? RANGE_LOAD : LOAD); - } - - public class LoadRunnable extends IOTask - { - final AccordCacheEntry entry; - Object result = FailureHolder.NOT_STARTED; - - LoadRunnable(AccordCacheEntry entry, GlobalGroup group) - { - super(group); - Invariants.require(group == LOAD || group == RANGE_LOAD); - this.entry = entry; - } - - void postRunExclusive() - { - if (!(result instanceof FailureHolder)) onLoadedExclusive(entry, (V)result, null); - else onLoadedExclusive(entry, null, ((FailureHolder)result).fail); - } - - @Override - String toDescription() - { - return "Load " + entry.key(); - } - - @Override - public void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); - } - onRunComplete(); - } - - @Override - void reportFailure(Throwable t) - { - result = new FailureHolder(t); - } - - @Override - public String description() - { - return "Loading " + entry; - } - } - - static abstract class AbstractIOTask - { - abstract void runInternal(); - abstract void postRunExclusive(); - abstract void fail(Throwable t); - abstract String description(); - } - - class WrappedIOTask extends IOTask - { - final AbstractIOTask wrapped; - - WrappedIOTask(AbstractIOTask wrap, GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - this.wrapped = wrap; - } - - @Override - String toDescription() - { - return wrapped.description(); - } - - @Override - protected void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - wrapped.runInternal(); - } - onRunComplete(); - } - - @Override - void postRunExclusive() - { - wrapped.postRunExclusive(); - } - - @Override - public String description() - { - return wrapped.description(); - } - - @Override - protected void reportFailure(Throwable fail) - { - wrapped.fail(fail); - } - } - - private static final Throwable NOT_STARTED = new Throwable(); - class SaveRunnable extends IOTask - { - final AccordCacheEntry entry; - final UniqueSave identity; - final Runnable run; - Throwable failure = NOT_STARTED; - - SaveRunnable(AccordCacheEntry entry, UniqueSave identity, Runnable run) - { - super(SAVE); - this.entry = entry; - this.identity = identity; - this.run = run; - } - - @Override - void postRunExclusive() - { - onSavedExclusive(entry, identity, failure); - } - - @Override - String toDescription() - { - return "Save " + entry.key(); - } - - @Override - public void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - onRunComplete(); - failure = null; - } - - @Override - protected void reportFailure(Throwable t) - { - failure = t; - } - - @Override - public String description() - { - return "Save " + entry; - } - } - - class PlainChain extends Plain - { - final RunOrFail runOrFail; - final @Nullable ExclusiveExecutor executor; - - PlainChain(RunOrFail runOrFail, ExclusiveExecutor executor, ExclusiveGroup group) - { - super(group); - this.runOrFail = runOrFail; - this.executor = executor; - } - - PlainChain(RunOrFail runOrFail, ExclusiveExecutor executor, ExclusiveGroup group, long position, int tranche) - { - super(group, position, tranche); - this.runOrFail = runOrFail; - this.executor = executor; - } - - @Override - ExclusiveExecutor executor() - { - return executor; - } - - @Override - String toDescription() - { - return runOrFail.toString(); - } - - @Override - protected void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - runOrFail.run(); - } - catch (Throwable t) - { - // shouldn't throw exceptions - agent.onException(t); - } - onRunComplete(); - } - - @Override - protected void reportFailure(Throwable fail) - { - try - { - runOrFail.fail(fail); - } - catch (Throwable t) - { - fail.addSuppressed(t); - agent.onException(fail); - } - } - } - - class DebuggableChain extends PlainChain implements DebuggableTask - { - final Object describe; - - DebuggableChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, Object describe) - { - super(runOrFail, executor, ExclusiveGroup.OTHER); - this.describe = Invariants.nonNull(describe); - } - - DebuggableChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, long position, int tranche, Object describe) - { - super(runOrFail, executor, ExclusiveGroup.OTHER, position, tranche); - this.describe = Invariants.nonNull(describe); - } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } - - @Override - public String description() - { - return describe.toString(); - } - - @Override - public DebuggableTask debuggable() - { - return this; - } - } - - - public static class TaskInfo implements Comparable - { - // sorted in name order for reporting to virtual tables - public enum Status { LOADING, RUNNING, SCANNING_RANGES, WAITING_TO_LOAD, WAITING_TO_RUN } - - final Status status; - final int commandStoreId; - - final Task task; - - public TaskInfo(Status status, int commandStoreId, Task task) - { - this.status = status; - this.commandStoreId = commandStoreId; - this.task = task; - } - - public Status status() - { - return status; - } - - public Integer commandStoreId() - { - return commandStoreId >= 0 ? commandStoreId : null; - } - - public long position() - { - return task.position; - } - - public @Nullable String describe() - { - if (task instanceof AccordTask) - return ((AccordTask) task).executionContext().reason(); - - if (task instanceof DebuggableTask) - return ((DebuggableTask) task).description(); - - return null; - } - - public @Nullable ExecutionContext preLoadContext() - { - if (task instanceof AccordTask) - return ((AccordTask) task).executionContext(); - if (task instanceof WrappedIOTask && ((WrappedIOTask) task).wrapped instanceof AccordTask.RangeTxnScanner) - return ((AccordTask.RangeTxnScanner) ((WrappedIOTask) task).wrapped).preLoadContext(); - return null; - } - - @Override - public int compareTo(TaskInfo that) - { - int c = this.status.compareTo(that.status); - if (c == 0) c = Long.compare(this.position(), that.position()); - return c; - } - } - - public List taskSnapshot() - { - List result = new ArrayList<>(); - AccordTaskRunner self = AccordTaskRunner.get(); - lock(self); - try - { - addToSnapshot(result, scanningRanges, TaskInfo.Status.SCANNING_RANGES, TaskInfo.Status.SCANNING_RANGES); - addToSnapshot(result, loading, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); - for (TaskQueue queue : runnable.queues) - { - if (queue != null) - addToSnapshot(result, queue, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); - } - addToSnapshot(result, runnable.assigned, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); - } - finally - { - unlock(self); - } - result.sort(TaskInfo::compareTo); - return result; - } - - private static void addToSnapshot(List snapshot, TaskQueue queue, TaskInfo.Status ifCurrent, TaskInfo.Status ifQueued) - { - for (int i = 0 ; i < queue.size() ; ++i) - { - Task t = queue.getSingle(i); - if (t instanceof ExclusiveExecutorTask) - { - ExclusiveExecutor q = ((ExclusiveExecutorTask) t).queue; - snapshot.add(new TaskInfo(ifCurrent, q.commandStoreId, q.task)); - for (TaskQueue q0 : q.queues) - { - if (q0 != null) - { - for (int j = 0 ; j < q0.size() ; ++j) - snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q0.getSingle(j))); - } - } - } - else - { - int commmandStoreId = t instanceof AccordTask ? ((AccordTask) t).commandStore.id() : -1; - snapshot.add(new TaskInfo(ifCurrent, commmandStoreId, t)); - } - } - } - - public int unsafePreparingToRunCount() - { - return loading.size() + scanningRanges.size(); - } - - public int unsafeWaitingToRunCount() - { - return runnable.waitingCount(); - } - - public int unsafeRunningCount() - { - return runnable.assigned.size(); - } - - private static long[] parseEnumParams(String input, String describe) - { - String[] specs = input.split(";"); - if (specs.length != 2) - throw new IllegalArgumentException("Invalid specifiers in " + describe + "; expect [GlobalGroup];[ExclusiveGroup] but got: " + input); - long[] result = new long[2]; - result[0] = parseEnumParams(GlobalGroup::valueOf, specs[0], describe + " for GlobalGroup"); - result[1] = parseEnumParams(ExclusiveGroup::valueOf, specs[1], describe + " for ExclusiveGroup"); - return result; - } - - private static long parseEnumParams(Function> get, String input, String describe) - { - long result = 0; - for (String spec : input.split(",")) - { - if (spec.trim().isEmpty()) continue; - - String[] split = spec.split(":"); - if (split.length != 2) - throw new IllegalArgumentException("Invalid specifier " + spec + " in " + describe + ": " + input); - - try - { - Enum queue = get.apply(split[0]); - long value = Long.parseLong(split[1]); - if (value <= 0 || value >= 128) - throw new IllegalArgumentException("Invalid limit " + value + " in queue_active_limits: " + input); - - result |= value << (queue.ordinal() * 8); - } - catch (Throwable t) - { - throw new IllegalArgumentException("Invalid queue identifier " + split[0] + " in " + describe + ": " + input); - } - } - - return result; - } - - private static long encodeCounters(Map, Long> counters) - { - long result = 0; - for (Map.Entry, Long> e : counters.entrySet()) - result |= e.getValue() << (e.getKey().ordinal() * 8); - return result; - } - -} diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index e268113b86..a14d9b3cd4 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -245,7 +245,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements logger.info("Reporting failure of plan {} for bootstrap of {} from {}", planId, range, from, fail); fail(from, Ranges.of(range), fail); } - }, ((AccordCommandStore) commandStore()).taskExecutor()); + }, ((AccordCommandStore) commandStore()).exclusiveExecutor()); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java index 5123471acd..b0efd33b1b 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java +++ b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import com.google.common.annotations.VisibleForTesting; @@ -122,6 +123,7 @@ import static org.apache.cassandra.config.AccordConfig.RangeIndexMode.journal_sa import static org.apache.cassandra.db.partitions.PartitionUpdate.singleRowUpdate; import static org.apache.cassandra.db.rows.BTreeRow.singleCellRow; import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; public class AccordKeyspace { @@ -353,9 +355,15 @@ public class AccordKeyspace BufferCell.live(CFKAccessor.data, timestampMicros, bytes))); } - public static Runnable systemTableUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized, long timestampMicros) + private static final AtomicLong lastSystemTimestampMicros = new AtomicLong(); + private static long nextSystemTimestampMicros() { - PartitionUpdate upd = makeUpdate(storeId, key, update, serialized, timestampMicros); + return lastSystemTimestampMicros.accumulateAndGet(currentTimeMillis(), (a, b) -> Math.max(a + 1, b)); + } + + public static Runnable systemTableUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized) + { + PartitionUpdate upd = makeUpdate(storeId, key, update, serialized, nextSystemTimestampMicros()); return () -> { ColumnFamilyStore cfs = AccordColumnFamilyStores.commandsForKey; try (OpOrder.Group group = Keyspace.writeOrder.start()) diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java b/src/java/org/apache/cassandra/service/accord/AccordSafeState.java deleted file mode 100644 index ca8baafbed..0000000000 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service.accord; - -import accord.local.SafeState; - -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; - -public interface AccordSafeState & AccordSafeState> -{ - AccordCacheEntry global(); - void postExecute(AccordTask owner); - void preExecute(AccordTask owner, LockMode lockMode); - - static AccordCacheEntry global(SafeState safeState) - { - return safeState.getClass() == AccordSafeCommand.class ? ((AccordSafeCommand) safeState).global() - : ((AccordSafeCommandsForKey) safeState).global(); - } - - static void postExecute(SafeState safeState, AccordTask owner) - { - if (safeState.getClass() == AccordSafeCommand.class) ((AccordSafeCommand) safeState).postExecute(owner); - else ((AccordSafeCommandsForKey) safeState).postExecute(owner); - } - - static void preExecute(SafeState safeState, AccordTask owner, LockMode lockMode) - { - if (safeState.getClass() == AccordSafeCommand.class) ((AccordSafeCommand) safeState).preExecute(owner, lockMode); - else ((AccordSafeCommandsForKey) safeState).preExecute(owner, lockMode); - } -} diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 98e1b239f4..994a553e8e 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -126,6 +126,7 @@ import org.apache.cassandra.service.accord.api.AccordTopologySorter; import org.apache.cassandra.service.accord.api.AccordViolationHandler; import org.apache.cassandra.service.accord.api.CompositeTopologySorter; import org.apache.cassandra.service.accord.api.TokenKey.KeyspaceSplitter; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.journal.ReplayMarkers; diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java index 77296264e0..abf7336b22 100644 --- a/src/java/org/apache/cassandra/service/accord/IAccordService.java +++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java @@ -63,6 +63,7 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.api.AccordScheduler; import org.apache.cassandra.service.accord.api.AccordTopologySorter; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.topology.AccordEndpointMapper; import org.apache.cassandra.service.accord.topology.AccordSyncPropagator; import org.apache.cassandra.service.accord.topology.AccordSyncPropagator.Notification; diff --git a/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java b/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java index 1af4dc96a0..27da1e7324 100644 --- a/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java @@ -44,6 +44,7 @@ import accord.primitives.Unseekables; import accord.utils.async.Cancellable; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import org.apache.cassandra.service.accord.serializers.CommandStoreSerializers; import static org.apache.cassandra.io.util.CompressedFrameDataInputPlus.readList; diff --git a/src/java/org/apache/cassandra/service/accord/RangeIndex.java b/src/java/org/apache/cassandra/service/accord/RangeIndex.java index 228aad3781..55f39c5bab 100644 --- a/src/java/org/apache/cassandra/service/accord/RangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/RangeIndex.java @@ -41,6 +41,7 @@ import accord.utils.Invariants; import org.apache.cassandra.exceptions.UnknownTableException; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import org.apache.cassandra.service.accord.journal.CommandChanges; import org.apache.cassandra.service.accord.serializers.Version; @@ -59,10 +60,10 @@ public interface RangeIndex protected abstract AccordCommandStore commandStore(); - protected abstract void loadExclusive(Map into, AccordCommandStore.Caches caches); - protected abstract void load(Map into, BooleanSupplier abort); - protected abstract void finish(Map into); - protected abstract void cleanupExclusive(AccordCommandStore.Caches caches); + public abstract void loadExclusive(Map into, AccordCommandStore.Caches caches); + public abstract void load(Map into, BooleanSupplier abort); + public abstract void finish(Map into); + public abstract void cleanupExclusive(AccordCommandStore.Caches caches); protected CommandSummaries.Summary loadFromDisk(TxnId txnId) { diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java index e5ff023c80..15b34ca23f 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java @@ -53,8 +53,6 @@ import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.utils.concurrent.Future; -import static accord.local.LoadKeys.SYNC; -import static accord.local.LoadKeysFor.READ_WRITE; import static java.util.Collections.emptyList; public class DebugBlockedTxns diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java index 037a09f1c4..99aa8fe0ce 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java @@ -29,7 +29,7 @@ import accord.local.Command; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.metrics.LogLinearHistogram; -import org.apache.cassandra.service.accord.AccordExecutor.Task; +import org.apache.cassandra.service.accord.execution.Task; import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.WithResources; diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java similarity index 88% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java rename to src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java index d7594025cd..ac7e4390a0 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java @@ -16,26 +16,26 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.locks.Lock; +import java.util.function.Consumer; +import java.util.function.Function; import accord.api.Agent; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; import org.apache.cassandra.concurrent.CassandraThread; -import org.apache.cassandra.service.accord.AccordExecutorLoops.LoopTask; +import org.apache.cassandra.service.accord.execution.Loops.LoopTask; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; -abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop +abstract class AbstractLockLoop extends AbstractLoop { int runningThreads; boolean shutdown; - AccordExecutorAbstractLockLoop(Lock lock, int executorId, Agent agent) + AbstractLockLoop(Lock lock, int executorId, Agent agent) { super(lock, executorId, agent); } @@ -43,17 +43,17 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop abstract void notifyWork(); abstract void notifyWorkExclusive(); abstract void awaitExclusive() throws InterruptedException; - abstract void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4); + abstract void submitExternal(Consumer sync, Function async, P1 p1); - final void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + final void submit(Consumer sync, Function async, P1 p1) { // if we're a loop thread, we will poll the waitingToRun queue when we come around // NOTE: this assumes no synchronous blocking tasks are submitted to this executor - if (isInLoop() || isOwningThread()) push(async.apply(p1a, p2, p3, p4)); - else submitExternal(sync, async, p1s, p1a, p2, p3, p4); + if (isInLoop() || isOwningThread()) push(async.apply(p1)); + else submitExternal(sync, async, p1); } - final void submitExternalExclusive(QuintConsumer sync, P1s p1s, P2 p2, P3 p3, P4 p4) + final void submitExternalExclusive(Consumer sync, Function async, P1 p1) { try { @@ -63,11 +63,11 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop } catch (Throwable t) { - try { sync.accept(this, p1s, p2, p3, p4); } + try { sync.accept(p1); } catch (Throwable t2) { t.addSuppressed(t2); } throw t; } - sync.accept(this, p1s, p2, p3, p4); + sync.accept(p1); } finally { @@ -142,7 +142,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop while (cur != null) { Task next = cur.next; - cur.submitExclusive(this); + cur.submitExclusive(); cur.next = null; cur = next; } @@ -195,8 +195,8 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop public void run() { Thread thread = Thread.currentThread(); - AccordTaskRunner self = AccordTaskRunner.get(thread); - self.setAccordActiveExecutor(AccordExecutorAbstractLockLoop.this); + TaskRunner self = TaskRunner.get(thread); + self.setAccordActiveExecutor(AbstractLockLoop.this); setWrapped(self); Task task; @@ -271,7 +271,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop public void run() { CassandraThread self = (CassandraThread) Thread.currentThread(); - self.setAccordActiveExecutor(AccordExecutorAbstractLockLoop.this); + self.setAccordActiveExecutor(AbstractLockLoop.this); setWrapped(self); Task task = null; diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java similarity index 84% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java rename to src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java index 5c93a8a0cc..684160b870 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.Lock; @@ -27,17 +27,17 @@ import accord.utils.Invariants; import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; -abstract class AccordExecutorAbstractLoop extends AccordExecutor +abstract class AbstractLoop extends AccordExecutor { volatile Task unqueued; - static final AtomicReferenceFieldUpdater unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordExecutorAbstractLoop.class, Task.class, "unqueued"); + static final AtomicReferenceFieldUpdater unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AbstractLoop.class, Task.class, "unqueued"); - AccordExecutorAbstractLoop(Lock lock, int executorId, Agent agent) + AbstractLoop(Lock lock, int executorId, Agent agent) { super(lock, executorId, agent); } - abstract AccordExecutorLoops loops(); + abstract Loops loops(); boolean hasUnqueued() { @@ -62,7 +62,7 @@ abstract class AccordExecutorAbstractLoop extends AccordExecutor if (hasUnqueued() || tasks > 0) return true; - AccordTaskRunner self = AccordTaskRunner.get(); + TaskRunner self = TaskRunner.get(); lock(self); try { @@ -84,7 +84,7 @@ abstract class AccordExecutorAbstractLoop extends AccordExecutor Invariants.require(cur != null); Task next = cur.next; cur.next = null; - if (cur.is(Task.State.UNINITIALIZED)) cur.submitExclusive(this); + if (cur.is(Task.State.UNINITIALIZED)) cur.submitExclusive(); else cleanupTaskExclusive(cur, true); return next; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractSemiSyncSubmit.java similarity index 65% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AbstractSemiSyncSubmit.java index dce92f103d..79f6736ffa 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractSemiSyncSubmit.java @@ -16,26 +16,26 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.locks.Lock; +import java.util.function.Consumer; +import java.util.function.Function; import accord.api.Agent; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -abstract class AccordExecutorAbstractSemiSyncSubmit extends AccordExecutorAbstractLockLoop +abstract class AbstractSemiSyncSubmit extends AbstractLockLoop { - AccordExecutorAbstractSemiSyncSubmit(Lock lock, int executorId, Agent agent) + AbstractSemiSyncSubmit(Lock lock, int executorId, Agent agent) { super(lock, executorId, agent); } abstract void awaitExclusive() throws InterruptedException; - void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submitExternal(Consumer sync, Function async, P1 p1) { - if (push(async.apply(p1a, p2, p3, p4)) == null && !isInLoop()) + if (push(async.apply(p1)) == null && !isInLoop()) notifyWork(); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCache.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java similarity index 90% rename from src/java/org/apache/cassandra/service/accord/AccordCache.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordCache.java index a75e6a6e27..8a5c2f6991 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCache.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.io.IOException; import java.nio.ByteBuffer; @@ -26,6 +26,7 @@ import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; @@ -44,6 +45,7 @@ import org.slf4j.LoggerFactory; import accord.api.RoutingKey; import accord.local.Command; +import accord.local.RedundantBefore; import accord.local.SafeState; import accord.local.cfk.CommandsForKey; import accord.local.cfk.Serialize; @@ -65,12 +67,17 @@ import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.metrics.AccordCacheMetrics; import org.apache.cassandra.metrics.LogLinearHistogram; import org.apache.cassandra.metrics.ShardedHitRate; -import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; -import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.Loading; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; -import org.apache.cassandra.service.accord.AccordSafeCommandsForKey.CommandsForKeyCacheEntry; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordKeyspace; +import org.apache.cassandra.service.accord.AccordObjectSizes; +import org.apache.cassandra.service.accord.OrderedKeys; +import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.events.CacheEvents; +import org.apache.cassandra.service.accord.execution.AccordCache.Adapter.Shrink; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Loading; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.execution.SaferCommandsForKey.CommandsForKeyCacheEntry; import org.apache.cassandra.service.accord.journal.CommandChangeWriter; import org.apache.cassandra.service.accord.journal.CommandChanges; import org.apache.cassandra.service.accord.serializers.CommandSerializers; @@ -81,11 +88,11 @@ import org.apache.cassandra.utils.ObjectSizes; import static accord.utils.Invariants.illegalState; import static accord.utils.Invariants.require; -import static org.apache.cassandra.service.accord.AccordCacheEntry.AGE_MASK; -import static org.apache.cassandra.service.accord.AccordCacheEntry.GENERATION_MASK; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.AGE_MASK; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.GENERATION_MASK; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.EVICTED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.LOADED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.MODIFIED; /** * Cache for AccordCommand and AccordCommandsForKey, available memory is shared between the two object types. @@ -102,15 +109,15 @@ public class AccordCache implements CacheSize // Debug mode to verify that loading from journal + system tables results in // functionally identical (or superceding) command to the one we've just evicted. - private static boolean VALIDATE_LOAD_ON_EVICT = false; + private static boolean DEBUG_VALIDATE_LOAD_ON_EVICT = false; @VisibleForTesting public static void validateLoadOnEvict(boolean value) { - VALIDATE_LOAD_ON_EVICT = value; + DEBUG_VALIDATE_LOAD_ON_EVICT = value; } - public interface Adapter & AccordSafeState> + public interface Adapter & SaferState> { enum Shrink { EVICT, DONE, PERFORM_WITHOUT_LOCK } @@ -194,7 +201,7 @@ public class AccordCache implements CacheSize /** * Make sure we don't have any items lingering too long in the no evict queue, to avoid cache memory leaks */ - void processNoEvictQueue() + public void processNoEvictQueue() { noEvictGeneration = (noEvictGeneration + 1) & GENERATION_MASK; if (noEvictQueue.isEmpty()) @@ -223,7 +230,7 @@ public class AccordCache implements CacheSize * Roughly respects LRU semantics when evicting. Might consider prioritising keeping MODIFIED nodes around * for longer to maximise the chances of hitting system tables fewer times (or not at all). */ - void tryShrinkOrEvict(Lock lock) + public void tryShrinkOrEvict(Lock lock) { if (!tryShrinkOrEvict) return; @@ -336,7 +343,7 @@ public class AccordCache implements CacheSize --parent.size; // TODO (expected): use listeners - if (node.status() == LOADED && VALIDATE_LOAD_ON_EVICT) + if (node.status() == LOADED && DEBUG_VALIDATE_LOAD_ON_EVICT) owner.validateLoadEvicted(node); AccordCacheEntry self = node.owner.remove(node.key()); @@ -376,19 +383,19 @@ public class AccordCache implements CacheSize evictQueue.addFirst(node); // add to front since we have just saved, so we were eligible for eviction } - public & AccordSafeState> void release(S safeRef, AccordTask owner) + public & SaferState> void release(S safeRef, SafeTask owner) { safeRef.global().owner.release(safeRef, owner); } - public & AccordSafeState> Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) + public & SaferState> Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) { Type instance = new Type<>(keyClass, adapter, metrics); types.add(instance); return instance; } - public & AccordSafeState> Type newType( + public & SaferState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, @@ -401,7 +408,7 @@ public class AccordCache implements CacheSize return newType(keyClass, loadFunction, saveFunction, quickShrink, (i, j) -> j, (c, i, j) -> (V)j, validateFunction, heapEstimator, i -> 0, safeRefFactory, metrics); } - public & AccordSafeState> Type newType( + public & SaferState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, @@ -433,7 +440,7 @@ public class AccordCache implements CacheSize default void onEvict(AccordCacheEntry state) {} } - public class Type & AccordSafeState> implements CacheSize + public class Type & SaferState> implements CacheSize { public class Instance implements Iterable> { @@ -534,7 +541,7 @@ public class AccordCache implements CacheSize return node; } - public final void release(S safeRef, AccordTask owner) + public final void release(S safeRef, SafeTask owner) { K key = safeRef.global().key(); logger.trace("Releasing resources for {}: {}", key, safeRef); @@ -566,8 +573,7 @@ public class AccordCache implements CacheSize { evict = node.is(LOADED) && node.isNull(); } - node.remove(owner, safeRef.isSafe()); - safeRef.setReleased(); + node.remove(owner, safeRef.setReleased()); if (node.decrement() == 0) { @@ -666,21 +672,21 @@ public class AccordCache implements CacheSize } @VisibleForTesting - final boolean keyIsReferenced(Object key, Class> valClass) + final boolean keyIsReferenced(Object key, Class> valClass) { AccordCacheEntry node = cache.get(key); return node != null && node.references() > 0; } @VisibleForTesting - final boolean keyIsCached(Object key, Class> valClass) + final boolean keyIsCached(Object key, Class> valClass) { AccordCacheEntry node = cache.get(key); return node != null; } @VisibleForTesting - final int references(Object key, Class> valClass) + final int references(Object key, Class> valClass) { AccordCacheEntry node = cache.get(key); return node != null ? node.references() : 0; @@ -769,7 +775,7 @@ public class AccordCache implements CacheSize } // can be safely garbage collected if empty - Instance newInstance(AccordCommandStore commandStore) + public Instance newInstance(AccordCommandStore commandStore) { return keyClass == RoutingKey.class ? new KeyInstance(commandStore) : new Instance(commandStore); } @@ -908,7 +914,7 @@ public class AccordCache implements CacheSize return size() == 0; } - Iterable> evictionQueue() + public Iterable> evictionQueue() { return evictQueue::iterator; } @@ -1007,7 +1013,7 @@ public class AccordCache implements CacheSize event.update(); } - static class FunctionalAdapter & AccordSafeState> implements Adapter + static class FunctionalAdapter & SaferState> implements Adapter { final BiFunction load; final QuadFunction save; @@ -1120,7 +1126,7 @@ public class AccordCache implements CacheSize } } - static class SettableWrapper & AccordSafeState> extends FunctionalAdapter + static class SettableWrapper & SaferState> extends FunctionalAdapter { volatile BiFunction load; @@ -1130,7 +1136,7 @@ public class AccordCache implements CacheSize this.load = super.load; } - public static & AccordSafeState> Adapter loadOnly(BiFunction load) + public static & SaferState> Adapter loadOnly(BiFunction load) { SettableWrapper result = new SettableWrapper<>(new NoOpAdapter()); result.load = load; @@ -1144,7 +1150,7 @@ public class AccordCache implements CacheSize } } - static class NoOpAdapter & AccordSafeState> implements Adapter + static class NoOpAdapter & SaferState> implements Adapter { @Override public V load(AccordCommandStore commandStore, K key) { return null; } @Override public Runnable save(AccordCommandStore commandStore, K key, @Nullable V value, @Nullable Object shrunk) { return null; } @@ -1158,7 +1164,7 @@ public class AccordCache implements CacheSize @Override public S safeRef(AccordCacheEntry node) { return null; } } - public static class CommandsForKeyAdapter implements Adapter + public static class CommandsForKeyAdapter implements Adapter { public static final CommandsForKeyAdapter CFK_ADAPTER = new CommandsForKeyAdapter(); private static int SHRINK_WITHOUT_LOCK = -1; @@ -1168,7 +1174,13 @@ public class AccordCache implements CacheSize @Override public CommandsForKey load(AccordCommandStore commandStore, RoutingKey key) { - return commandStore.loadCommandsForKey(key); + CommandsForKey cfk = AccordKeyspace.CommandsForKeyAccessor.load(commandStore.id(), (TokenKey) key); + if (cfk == null) + return null; + RedundantBefore.QuickBounds bounds = commandStore.safeGetRedundantBefore().get(key); + if (!Invariants.expect(bounds != null, "No RedundantBefore information found when loading key %s", key)) + return cfk; + return cfk.withCleanCfkBeforeAtLeast(bounds.cleanCfkBefore(), false); } @Override @@ -1179,7 +1191,7 @@ public class AccordCache implements CacheSize ByteBuffer bb = (ByteBuffer)serialized; serialized = bb.duplicate().position(prefixBytes(bb)); } - return commandStore.saveCommandsForKey(key, value, serialized); + return AccordKeyspace.CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey) key, value, serialized); } @Override @@ -1255,13 +1267,17 @@ public class AccordCache implements CacheSize @Override public boolean validate(AccordCommandStore commandStore, RoutingKey key, CommandsForKey value) { - return commandStore.validateCommandsForKey(key, value); + if (!Invariants.isParanoid()) + return true; + + CommandsForKey reloaded = AccordKeyspace.CommandsForKeyAccessor.load(commandStore.id(), (TokenKey) key); + return Objects.equals(value, reloaded); } @Override - public AccordSafeCommandsForKey safeRef(AccordCacheEntry node) + public SaferCommandsForKey safeRef(AccordCacheEntry node) { - return new AccordSafeCommandsForKey(node); + return new SaferCommandsForKey(node); } @Override @@ -1271,7 +1287,7 @@ public class AccordCache implements CacheSize } @Override - public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) + public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) { CommandsForKeyCacheEntry entry = new CommandsForKeyCacheEntry(key, owner); entry.readyToLoad(); @@ -1279,7 +1295,7 @@ public class AccordCache implements CacheSize } } - public static class CommandAdapter implements Adapter + public static class CommandAdapter implements Adapter { private static final int SHRINK_WITHOUT_LOCK = -1; @@ -1392,19 +1408,23 @@ public class AccordCache implements CacheSize @Override public boolean validate(AccordCommandStore commandStore, TxnId key, Command value) { - return commandStore.validateCommand(key, value); + if (!Invariants.isParanoid()) + return true; + + Command reloaded = commandStore.loadCommand(key); + return Objects.equals(value, reloaded); } @Override - public AccordSafeCommand safeRef(AccordCacheEntry node) + public SaferCommand safeRef(AccordCacheEntry node) { - return new AccordSafeCommand(node); + return new SaferCommand(node); } @Override - public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) + public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) { - AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); + AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); if (txnId.is(Txn.Kind.EphemeralRead)) { node.initialize(null); diff --git a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java similarity index 60% rename from src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java index 11d8a2b105..fefaad70a6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java @@ -15,13 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.function.BiConsumer; import java.util.function.BiPredicate; @@ -36,39 +33,38 @@ import accord.utils.ArrayBuffers.BufferList; import accord.utils.IntrusiveLinkedList; import accord.utils.IntrusiveLinkedListNode; import accord.utils.Invariants; -import accord.utils.SortedArrays; -import accord.utils.TriConsumer; import accord.utils.UnhandledEnum; import accord.utils.async.Cancellable; -import org.apache.cassandra.service.accord.AccordCache.Adapter; -import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; -import org.apache.cassandra.service.accord.AccordExecutor.IOTask; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.execution.AccordCache.Adapter; +import org.apache.cassandra.service.accord.execution.AccordCache.Adapter.Shrink; import org.apache.cassandra.utils.ObjectSizes; import static accord.utils.Invariants.nonNull; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.HOLD_QUEUE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.UNLOCKED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Queue.compare; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_SAVE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADING; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.SAVING; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.WAITING_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.WAITING_TO_SAVE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.HOLD_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.UNLOCKED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.EVICTED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.FAILED_TO_LOAD; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.FAILED_TO_SAVE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.LOADED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.LOADING; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.MODIFIED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.SAVING; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.WAITING_TO_SAVE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntryQueue.PRIORITY_START_INDEX; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntryQueue.compare; /** * Global (per CommandStore) state of a cached entity (Command or CommandsForKey). */ -public class AccordCacheEntry & AccordSafeState> extends IntrusiveLinkedListNode +public class AccordCacheEntry & SaferState> extends IntrusiveLinkedListNode { public enum Status { @@ -162,492 +158,6 @@ public class AccordCacheEntry & AccordSafeState[] tasks; - // TODO (expected): use bytes/shorts for indexes to keep size down, and have an expanded version of the Queue - // with better algorithmic complexity (e.g. Hash -> IntrusivePriorityHeap) - private int priorityHead, priorityTail, fifoHead, fifoTail; - private int unsequencedSize; - - public Queue() - { - tasks = new AccordTask[DEFAULT_CAPACITY]; - priorityHead = priorityTail = PRIORITY_START_INDEX; - fifoHead = fifoTail = DEFAULT_CAPACITY - 1; - } - - private Queue(Queue copy) - { - tasks = copy.tasks.clone(); - priorityHead = copy.priorityHead; - priorityTail = copy.priorityTail; - fifoHead = copy.fifoHead; - fifoTail = copy.fifoTail; - } - - // returns true if no fifo tasks already queue (i.e. so we become head) - boolean addFifo(AccordTask task) - { - ensureCapacity(); - boolean isHead = fifoHead == fifoTail; - if (unsequencedSize > 0) // simply displace the unsequence task, as they're an unordered list - tasks[fifoTail - unsequencedSize] = tasks[fifoTail]; - tasks[fifoTail--] = task; - validate(); - return isHead; - } - - boolean addUnsequenced(AccordTask task) - { - Invariants.require(task.isUnsequenced()); - ensureCapacity(); - tasks[fifoTail - unsequencedSize++] = task; - return unsequencedSize == 1 && sequencedSize() == 1; - } - - boolean isLocked(AccordTask task) - { - return tasks[LOCKED_INDEX] == task; - } - - AccordTask lockedBy() - { - return tasks[LOCKED_INDEX]; - } - - boolean removeIfHead(AccordTask task) - { - return removeIfFifoHead(task) || removeIfPriorityHead(task); - } - - boolean removeIfFifoHead(AccordTask task) - { - if (!hasFifo()) - return false; - - if (task != tasks[fifoHead]) - return false; - tasks[fifoHead--] = null; - return true; - } - - boolean removeIfPriorityHead(AccordTask task) - { - if (!hasPriority()) - return false; - - if (task != tasks[priorityHead]) - return false; - tasks[priorityHead++] = null; - return true; - } - - void lock(AccordTask task) - { - tasks[LOCKED_INDEX] = task; - } - - void unlock(AccordTask task) - { - Invariants.require(tasks[LOCKED_INDEX] == task); - tasks[LOCKED_INDEX] = null; - } - - // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue - boolean addPrioritised(AccordTask task) - { - ensureCapacity(); - int insertPos = Arrays.binarySearch(tasks, priorityHead, priorityTail, task, Queue::compare); - if (insertPos < 0) - insertPos = -1 - insertPos; - - if (priorityHead == PRIORITY_START_INDEX || insertPos > (priorityTail + priorityHead)/2) - { - System.arraycopy(tasks, insertPos, tasks, insertPos + 1, priorityTail - insertPos); - tasks[insertPos] = task; - priorityTail++; - } - else - { - System.arraycopy(tasks, priorityHead, tasks, priorityHead - 1, insertPos - priorityHead); - tasks[insertPos - 1] = task; - priorityHead--; - } - - validate(); - return fifoHead == fifoTail && tasks[priorityHead] == task; - } - - // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue - void addWaitingToLoad(AccordTask task) - { - ensureCapacity(); - tasks[priorityTail++] = task; - } - - private boolean hasTailRoom() - { - if (priorityTail + unsequencedSize <= fifoTail) - return true; - Invariants.require(priorityTail + unsequencedSize == 1 + fifoTail); - return false; - } - - private void ensureCapacity() - { - if (!hasTailRoom()) - { - if (fifoHead == fifoTail && unsequencedSize == 0 && fifoTail < tasks.length - 1) fifoHead = fifoTail = tasks.length - 1; - else if (priorityHead == priorityTail && priorityHead > PRIORITY_START_INDEX) priorityHead = priorityTail = PRIORITY_START_INDEX; - else if (totalSize() >= (tasks.length - 1) / 2) compact(new AccordTask[tasks.length * 2]); - else compact(tasks); - Invariants.require(hasTailRoom()); - } - } - - private void compact(AccordTask[] into) - { - if (priorityHead == priorityTail) priorityHead = priorityTail = PRIORITY_START_INDEX; - else - { - int priorityLength = priorityTail - priorityHead; - System.arraycopy(tasks, priorityHead, into, PRIORITY_START_INDEX, priorityLength); - int newTail = PRIORITY_START_INDEX + priorityLength; - Invariants.require(newTail <= priorityTail); - if (into == tasks) - Arrays.fill(into, newTail, priorityTail, null); - priorityHead = PRIORITY_START_INDEX; - priorityTail = newTail; - } - - if (fifoHead == fifoTail && unsequencedSize == 0) fifoHead = fifoTail = into.length - 1; - else - { - int fifoLength = fifoHead - fifoTail; - int copyLength = fifoLength + unsequencedSize; - int copyFrom = (fifoTail - unsequencedSize) + 1; - int copyTo = into.length - copyLength; - Invariants.require(copyTo >= copyFrom); - System.arraycopy(tasks, copyFrom, into, copyTo, copyLength); - if (into == tasks) - Arrays.fill(into, copyFrom, copyTo, null); - fifoHead = into.length - 1; - fifoTail = fifoHead - fifoLength; - } - - if (tasks != into) - { - into[LOCKED_INDEX] = tasks[LOCKED_INDEX]; - tasks = into; - } - validate(); - } - - private void validate() - { - for (int i = PRIORITY_START_INDEX ; i < priorityHead ; ++i) - Invariants.require(tasks[i] == null); - for (int i = priorityHead ; i < priorityTail ; ++i) - Invariants.require(tasks[i] != null); - for (int i = priorityTail; i <= fifoTail - unsequencedSize; ++i) - Invariants.require(tasks[i] == null); - for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoHead ; ++i) - Invariants.require(tasks[i] != null); - for (int i = fifoHead + 1 ; i < tasks.length ; ++i) - Invariants.require(tasks[i] == null); - } - - AccordTask peek() - { - if (hasFifo()) return tasks[fifoHead]; - if (hasPriority()) return tasks[priorityHead]; - return null; - } - - AccordTask peekFifo() - { - return hasFifo() ? tasks[fifoHead] : null; - } - - // second task - AccordTask peekBehind() - { - int fifoSize = fifoSize(); - if (fifoSize > 1) - return tasks[fifoHead - 1]; - int priorityIndex = priorityHead + (1 - fifoSize); - if (priorityIndex < priorityTail) - return tasks[priorityIndex]; - return null; - } - - boolean hasFifo() - { - return fifoHead != fifoTail; - } - - boolean hasPriority() - { - return priorityHead != priorityTail; - } - - boolean hasUnsequenced() - { - return unsequencedSize > 0; - } - - int sequencedSize() - { - return prioritySize() + fifoSize(); - } - - int unsequencedSize() - { - return unsequencedSize; - } - - int totalSize() - { - return sequencedSize() + unsequencedSize; - } - - int prioritySize() - { - return priorityTail - priorityHead; - } - - int fifoSize() - { - return fifoHead - fifoTail; - } - - // true iff was head - boolean removeFifoOrPriority(AccordTask task, boolean permitMissing) - { - int fifoIndex = fifoIndexOf(task); - if (fifoIndex >= 0) - { - if (fifoIndex == fifoHead) - { - tasks[fifoHead--] = null; - validate(); - return true; - } - else - { - if (remove(fifoIndex, fifoTail + 1, fifoHead + 1)) ++fifoTail; - else --fifoHead; - validate(); - return false; - } - } - - int priorityIndex = priorityIndexOf(task); - Invariants.require(priorityIndex >= 0 || permitMissing); - if (priorityIndex >= 0) - { - if (priorityIndex == priorityHead) - { - tasks[priorityHead++] = null; - return !hasFifo(); - } - - if (remove(priorityIndex, priorityHead, priorityTail)) ++priorityHead; - else --priorityTail; - return false; - } - - return false; - } - - boolean removeUnsequenced(AccordTask task) - { - int unsequencedIndex = unsequencedIndexOf(task); - if (unsequencedIndex < 0) - return false; - - --unsequencedSize; - tasks[unsequencedIndex] = tasks[fifoTail - unsequencedSize]; - tasks[fifoTail - unsequencedSize] = null; - return true; - } - - // return true IFF was head - private boolean removePriority(AccordTask task, boolean permitAbsent) - { - int i = priorityIndexOf(task); - if (i < 0) - { - Invariants.require(permitAbsent); - return false; - } - - boolean wasHead = i == priorityHead; - if (remove(i, priorityHead, priorityTail)) ++priorityHead; - else --priorityTail; - return wasHead; - } - - // return true if we move the start forwards, false if we moved the end back - private boolean remove(int i, int start, int end) - { - if (i < (start + end)/2) - { - System.arraycopy(tasks, start, tasks, start + 1, i - start); - tasks[start] = null; - return true; - } - else - { - System.arraycopy(tasks, i + 1, tasks, i, end - (i + 1)); - tasks[end - 1] = null; - return false; - } - } - - boolean contains(AccordTask task) - { - return indexOf(task) >= 0; - } - - private int indexOf(AccordTask task) - { - if (tasks[priorityHead] == task) - return priorityHead; - - if (tasks[fifoHead] == task) - return fifoHead; - - int i = priorityIndexOf(task); - if (i >= 0) - return i; - - return fifoIndexOf(task); - } - - private int priorityIndexOf(AccordTask task) - { - if (priorityTail - priorityHead > 16) - { - if (tasks[priorityHead] == task) - return priorityHead; - - int i = SortedArrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, Queue::compare, SortedArrays.Search.CEIL); - if (i < 0) - return -1; - - while (i < priorityTail) - { - if (tasks[i] == task) - return i; - if (compare(task, tasks[i]) != 0) - break; - ++i; - } - } - - for (int i = priorityHead ; i < priorityTail ; ++i) - { - if (tasks[i] == task) - return i; - } - return -1; - } - - private int fifoIndexOf(AccordTask task) - { - for (int i = fifoHead ; i > fifoTail ; --i) - { - if (tasks[i] == task) - return i; - } - return -1; - } - - private int unsequencedIndexOf(AccordTask task) - { - for (int i = (fifoTail - unsequencedSize) + 1 ; i <= fifoTail ; ++i) - { - if (tasks[i] == task) - return i; - } - return -1; - } - - int drainUnsequenced(TriConsumer, P1, P2> forEach, P1 p1, P2 p2) - { - for (int i = (fifoTail - unsequencedSize) + 1 ; i <= fifoTail ; ++i) - { - AccordTask task = tasks[i]; - tasks[i] = null; - // should not be reentrant - forEach.accept(task, p1, p2); - } - int count = unsequencedSize; - unsequencedSize = 0; - return count; - } - - RunnableStatus ensureHeadFifo(AccordTask task) - { - if (hasFifo()) - { - Invariants.require(tasks[fifoHead] == task); - return NOT_RUNNABLE; - } - - if (tasks[priorityHead] == task) - { - tasks[priorityHead++] = null; - addFifo(task); - return STILL_RUNNABLE; - } - else - { - boolean wasPriorityHead = removePriority(task, false); - boolean isFifoHead = addFifo(task); - if (!isFifoHead) - return NOT_RUNNABLE; - if (wasPriorityHead) - return STILL_RUNNABLE; - if (hasPriority() || hasUnsequenced()) - return NEWLY_BLOCKING_RUNNABLE; - return NEWLY_RUNNABLE; - } - } - - static int compare(AccordTask a, AccordTask b) - { - Invariants.require(a != null && b != null); - int c = Long.compare(a.position, b.position); - if (c == 0) - c = a.executionContext().executionKind().compareTo(b.executionContext().executionKind()); - if (c == 0) - c = Long.compare(a.createdAt, b.createdAt); - if (c == 0) - c = Long.compare(a.loadedAt, b.loadedAt); - if (c == 0) - c = a.loggingId().compareTo(b.loggingId()); - return c; - } - - public boolean hasQueued() - { - return hasFifo() || hasPriority(); - } - } - static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCacheEntry<>(null, null)); private final K key; @@ -676,13 +186,13 @@ public class AccordCacheEntry & AccordSafeState head = queue instanceof Queue ? ((Queue) queue).peek() : (AccordTask) queue; + SafeTask head = queue instanceof AccordCacheEntryQueue ? ((AccordCacheEntryQueue) queue).peek() : (SafeTask) queue; Invariants.require(isRunnable(head) || status == NOT_RUNNABLE); return status; } // TODO (expected): don't unwrap when only one entry, since this may cause us to flap when locking unsequenced tasks - private void maybeUnwrap(Queue q) + private void maybeUnwrap(AccordCacheEntryQueue q) { int size = q.sequencedSize(); switch (size) @@ -699,7 +209,7 @@ public class AccordCacheEntry & AccordSafeState lockedBy) + private boolean maybeUnwrap(AccordCacheEntryQueue q, SafeTask lockedBy) { if (q.sequencedSize() == 0) { @@ -711,11 +221,11 @@ public class AccordCacheEntry & AccordSafeState task) + final RunnableStatus moveToFifo(SafeTask task) { if (queue != task) { - Queue q = (Queue) queue; + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; RunnableStatus status = q.ensureHeadFifo(task); if (status == NEWLY_BLOCKING_RUNNABLE && isLoaded()) onChangedHead(q, null, q.peekBehind()); @@ -725,22 +235,22 @@ public class AccordCacheEntry & AccordSafeState> drainWaitingToLoad() + public final BufferList> drainWaitingToLoad() { Invariants.require(isLoading()); Invariants.require(!isLocked()); - BufferList> list = new BufferList<>(); + BufferList> list = new BufferList<>(); if (queue != null) { - if (queue instanceof Queue) + if (queue instanceof AccordCacheEntryQueue) { - Queue q = (Queue) queue; + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; for (int i = q.priorityHead ; i < q.priorityTail ; ++i) { list.add(q.tasks[i]); q.tasks[i] = null; } - q.priorityHead = q.priorityTail = Queue.PRIORITY_START_INDEX; + q.priorityHead = q.priorityTail = PRIORITY_START_INDEX; for (int i = q.fifoHead ; i > q.fifoTail ; --i) list.add(q.tasks[i]); @@ -748,7 +258,7 @@ public class AccordCacheEntry & AccordSafeState task = (AccordTask) queue; + SafeTask task = (SafeTask) queue; list.add(task); Invariants.require(!isLocked()); if (!task.isCacheQueuedFifo()) @@ -758,14 +268,14 @@ public class AccordCacheEntry & AccordSafeState task, boolean ownsLock) + final void remove(SafeTask task, boolean ownsLock) { - if (queue instanceof Queue) + if (queue instanceof AccordCacheEntryQueue) { - Queue q = (Queue) queue; + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; boolean remove; boolean isLocked = isLocked() && q.isLocked(task); - Invariants.require(isLocked == ownsLock); + Invariants.require(isLocked || !ownsLock); if (isLocked) { // if locked, we've already released unsequenced/pririty/fifo positions unless isLockedHoldingQueue @@ -789,7 +299,7 @@ public class AccordCacheEntry & AccordSafeState & AccordSafeState & AccordSafeState task) + final RunnableStatus headStatus(SafeTask task) { if (queue == task) return validate(isRunnable(task) ? NEWLY_RUNNABLE : NOT_RUNNABLE); - Queue q = (Queue) queue; + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; if (q.peek() != task || !isRunnable(task)) return NOT_RUNNABLE; return validate(q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); } - private Queue ensureQueue() + private AccordCacheEntryQueue ensureQueue() { - if (queue instanceof Queue) - return (Queue) queue; + if (queue instanceof AccordCacheEntryQueue) + return (AccordCacheEntryQueue) queue; - AccordTask head = (AccordTask) this.queue; - Queue q = new Queue(); + SafeTask head = (SafeTask) this.queue; + AccordCacheEntryQueue q = new AccordCacheEntryQueue(); if (isLocked()) { if (isLockedHoldingQueue()) @@ -856,32 +366,32 @@ public class AccordCacheEntry & AccordSafeState task) + final void addWaitingToLoad(SafeTask task) { Invariants.require(isLoading()); if (queue == null) queue = task; else ensureQueue().addWaitingToLoad(task); } - final AccordTask head() + final SafeTask head() { if (queue == null) return null; - if (queue instanceof Queue) - return ((Queue) queue).peek(); + if (queue instanceof AccordCacheEntryQueue) + return ((AccordCacheEntryQueue) queue).peek(); if (isLocked() && !isLockedHoldingQueue()) return null; - return (AccordTask) queue; + return (SafeTask) queue; } - final RunnableStatus addUnsequenced(AccordTask task) + final RunnableStatus addUnsequenced(SafeTask task) { Invariants.require(isLoaded()); - AccordTask head = head(); + SafeTask head = head(); if (head != null && head.holdsLocksBetweenRuns()) { boolean wait = compare(task, head) > 0 || (unsequenced == 0 && head.hasIncrementalStarted()); @@ -907,8 +417,8 @@ public class AccordCacheEntry & AccordSafeState & AccordSafeState head) + private boolean isRunnable(SafeTask head) { return !head.holdsLocksBetweenRuns() || unsequenced == 0; } - private RunnableStatus add(AccordTask task, BiPredicate> add) + private RunnableStatus add(SafeTask task, BiPredicate> add) { Object prev = this.queue; if (prev == null) @@ -931,12 +441,12 @@ public class AccordCacheEntry & AccordSafeState head = q.peek(); + SafeTask head = q.peek(); if (isRunnable(head)) head.onChangeHeadStatus(this, STILL_RUNNABLE_NEWLY_BLOCKING); } @@ -955,18 +465,18 @@ public class AccordCacheEntry & AccordSafeState task) + final RunnableStatus addPrioritised(SafeTask task) { Invariants.require(!isLoading()); - return add(task, Queue::addPrioritised); + return add(task, AccordCacheEntryQueue::addPrioritised); } - final RunnableStatus addFifo(AccordTask task) + final RunnableStatus addFifo(SafeTask task) { - return add(task, Queue::addFifo); + return add(task, AccordCacheEntryQueue::addFifo); } - private void onChangedHead(Queue q, @Nullable AccordTask notifyNewHead, @Nullable AccordTask notifyPrevHead) + private void onChangedHead(AccordCacheEntryQueue q, @Nullable SafeTask notifyNewHead, @Nullable SafeTask notifyPrevHead) { if (notifyNewHead != null && isRunnable(notifyNewHead)) notifyNewHead.onChangeHeadStatus(this, q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); @@ -1003,7 +513,7 @@ public class AccordCacheEntry & AccordSafeState owner, LockMode lockMode) + public final V lockExclusive(SafeTask owner, LockMode lockMode) { Invariants.require(!isLocked()); Invariants.require(isRunnable(owner) || owner.isUnsequenced(this)); @@ -1028,7 +538,7 @@ public class AccordCacheEntry & AccordSafeState & AccordSafeState & AccordSafeState release) + private void releaseUnsequenced(AccordCacheEntryQueue q, SafeTask release) { Invariants.require(release.isCacheQueued()); if (--unsequenced == 0) { - AccordTask head = q.peek(); + SafeTask head = q.peek(); if (head != null && head.holdsLocksBetweenRuns()) onChangedHead(q, head, null); } @@ -1086,10 +596,10 @@ public class AccordCacheEntry & AccordSafeState) - return ((AccordTask) queue).isCacheQueuedFifo(); + if (queue instanceof SafeTask) + return ((SafeTask) queue).isCacheQueuedFifo(); - return ((Queue)queue).hasFifo(); + return ((AccordCacheEntryQueue)queue).hasFifo(); } final void unlink() @@ -1137,7 +647,7 @@ public class AccordCacheEntry & AccordSafeState= MODIFIED.ordinal(); } @@ -1342,7 +852,7 @@ public class AccordCacheEntry & AccordSafeState task) + public final void releaseExclusive(S safeState, SafeTask task) { owner.release(safeState, task); } @@ -1644,7 +1154,7 @@ public class AccordCacheEntry & AccordSafeState & AccordSafeState> AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) + public static & SaferState> AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) { AccordCacheEntry node = new AccordCacheEntry<>(key, owner); node.readyToLoad(); diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java new file mode 100644 index 0000000000..9e4b60df7d --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java @@ -0,0 +1,518 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.Arrays; + +import accord.utils.Invariants; +import accord.utils.SortedArrays; +import accord.utils.TriConsumer; + +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; + +class AccordCacheEntryQueue +{ + private static final int DEFAULT_CAPACITY = 4; + static final int LOCKED_INDEX = 0; + static final int PRIORITY_START_INDEX = LOCKED_INDEX + 1; + /** + * [priorityHead..priorityTail) stores a priority-sorted list of tasks + * (fifoTail...fifoHead] stores a fifo queue that runs ahead of any priority tasks + * (fifoTail-unsequencedCount...fifoTail] stores unsequenced tasks that are waiting for a queued incremental task. + * This only happens for TxnId cache entries, since they may lockAndHoldQueue. Once the lock is released, + * any pending unsequenced tasks are notified and immediately made (irrevocably) runnable for this entry. + */ + SafeTask[] tasks; + // TODO (expected): use bytes/shorts for indexes to keep size down, and have an expanded version of the Queue + // with better algorithmic complexity (e.g. Hash -> IntrusivePriorityHeap) + int priorityHead, priorityTail, fifoHead, fifoTail; + int unsequencedSize; + + public AccordCacheEntryQueue() + { + tasks = new SafeTask[DEFAULT_CAPACITY]; + priorityHead = priorityTail = PRIORITY_START_INDEX; + fifoHead = fifoTail = DEFAULT_CAPACITY - 1; + } + + AccordCacheEntryQueue(AccordCacheEntryQueue copy) + { + tasks = copy.tasks.clone(); + priorityHead = copy.priorityHead; + priorityTail = copy.priorityTail; + fifoHead = copy.fifoHead; + fifoTail = copy.fifoTail; + } + + // returns true if no fifo tasks already queue (i.e. so we become head) + boolean addFifo(SafeTask task) + { + ensureCapacity(); + boolean isHead = fifoHead == fifoTail; + if (unsequencedSize > 0) // simply displace the unsequence task, as they're an unordered list + tasks[fifoTail - unsequencedSize] = tasks[fifoTail]; + tasks[fifoTail--] = task; + validate(); + return isHead; + } + + boolean addUnsequenced(SafeTask task) + { + Invariants.require(task.isUnsequenced()); + ensureCapacity(); + tasks[fifoTail - unsequencedSize++] = task; + return unsequencedSize == 1 && sequencedSize() == 1; + } + + boolean isLocked(SafeTask task) + { + return tasks[LOCKED_INDEX] == task; + } + + SafeTask lockedBy() + { + return tasks[LOCKED_INDEX]; + } + + boolean removeIfHead(SafeTask task) + { + return removeIfFifoHead(task) || removeIfPriorityHead(task); + } + + boolean removeIfFifoHead(SafeTask task) + { + if (!hasFifo()) + return false; + + if (task != tasks[fifoHead]) + return false; + tasks[fifoHead--] = null; + return true; + } + + boolean removeIfPriorityHead(SafeTask task) + { + if (!hasPriority()) + return false; + + if (task != tasks[priorityHead]) + return false; + tasks[priorityHead++] = null; + return true; + } + + void lock(SafeTask task) + { + tasks[LOCKED_INDEX] = task; + } + + void unlock(SafeTask task) + { + Invariants.require(tasks[LOCKED_INDEX] == task); + tasks[LOCKED_INDEX] = null; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + boolean addPrioritised(SafeTask task) + { + ensureCapacity(); + int insertPos = Arrays.binarySearch(tasks, priorityHead, priorityTail, task, AccordCacheEntryQueue::compare); + if (insertPos < 0) + insertPos = -1 - insertPos; + + if (priorityHead == PRIORITY_START_INDEX || insertPos > (priorityTail + priorityHead) / 2) + { + System.arraycopy(tasks, insertPos, tasks, insertPos + 1, priorityTail - insertPos); + tasks[insertPos] = task; + priorityTail++; + } + else + { + System.arraycopy(tasks, priorityHead, tasks, priorityHead - 1, insertPos - priorityHead); + tasks[insertPos - 1] = task; + priorityHead--; + } + + validate(); + return fifoHead == fifoTail && tasks[priorityHead] == task; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + void addWaitingToLoad(SafeTask task) + { + ensureCapacity(); + tasks[priorityTail++] = task; + } + + private boolean hasTailRoom() + { + if (priorityTail + unsequencedSize <= fifoTail) + return true; + Invariants.require(priorityTail + unsequencedSize == 1 + fifoTail); + return false; + } + + private void ensureCapacity() + { + if (!hasTailRoom()) + { + if (fifoHead == fifoTail && unsequencedSize == 0 && fifoTail < tasks.length - 1) + fifoHead = fifoTail = tasks.length - 1; + else if (priorityHead == priorityTail && priorityHead > PRIORITY_START_INDEX) + priorityHead = priorityTail = PRIORITY_START_INDEX; + else if (totalSize() >= (tasks.length - 1) / 2) compact(new SafeTask[tasks.length * 2]); + else compact(tasks); + Invariants.require(hasTailRoom()); + } + } + + private void compact(SafeTask[] into) + { + if (priorityHead == priorityTail) priorityHead = priorityTail = PRIORITY_START_INDEX; + else + { + int priorityLength = priorityTail - priorityHead; + System.arraycopy(tasks, priorityHead, into, PRIORITY_START_INDEX, priorityLength); + int newTail = PRIORITY_START_INDEX + priorityLength; + Invariants.require(newTail <= priorityTail); + if (into == tasks) + Arrays.fill(into, newTail, priorityTail, null); + priorityHead = PRIORITY_START_INDEX; + priorityTail = newTail; + } + + if (fifoHead == fifoTail && unsequencedSize == 0) fifoHead = fifoTail = into.length - 1; + else + { + int fifoLength = fifoHead - fifoTail; + int copyLength = fifoLength + unsequencedSize; + int copyFrom = (fifoTail - unsequencedSize) + 1; + int copyTo = into.length - copyLength; + Invariants.require(copyTo >= copyFrom); + System.arraycopy(tasks, copyFrom, into, copyTo, copyLength); + if (into == tasks) + Arrays.fill(into, copyFrom, copyTo, null); + fifoHead = into.length - 1; + fifoTail = fifoHead - fifoLength; + } + + if (tasks != into) + { + into[LOCKED_INDEX] = tasks[LOCKED_INDEX]; + tasks = into; + } + validate(); + } + + private void validate() + { + for (int i = PRIORITY_START_INDEX; i < priorityHead; ++i) + Invariants.require(tasks[i] == null); + for (int i = priorityHead; i < priorityTail; ++i) + Invariants.require(tasks[i] != null); + for (int i = priorityTail; i <= fifoTail - unsequencedSize; ++i) + Invariants.require(tasks[i] == null); + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoHead; ++i) + Invariants.require(tasks[i] != null); + for (int i = fifoHead + 1; i < tasks.length; ++i) + Invariants.require(tasks[i] == null); + } + + SafeTask peek() + { + if (hasFifo()) return tasks[fifoHead]; + if (hasPriority()) return tasks[priorityHead]; + return null; + } + + SafeTask peekFifo() + { + return hasFifo() ? tasks[fifoHead] : null; + } + + // second task + SafeTask peekBehind() + { + int fifoSize = fifoSize(); + if (fifoSize > 1) + return tasks[fifoHead - 1]; + int priorityIndex = priorityHead + (1 - fifoSize); + if (priorityIndex < priorityTail) + return tasks[priorityIndex]; + return null; + } + + boolean hasFifo() + { + return fifoHead != fifoTail; + } + + boolean hasPriority() + { + return priorityHead != priorityTail; + } + + boolean hasUnsequenced() + { + return unsequencedSize > 0; + } + + int sequencedSize() + { + return prioritySize() + fifoSize(); + } + + int unsequencedSize() + { + return unsequencedSize; + } + + int totalSize() + { + return sequencedSize() + unsequencedSize; + } + + int prioritySize() + { + return priorityTail - priorityHead; + } + + int fifoSize() + { + return fifoHead - fifoTail; + } + + // true iff was head + boolean removeFifoOrPriority(SafeTask task, boolean permitMissing) + { + int fifoIndex = fifoIndexOf(task); + if (fifoIndex >= 0) + { + if (fifoIndex == fifoHead) + { + tasks[fifoHead--] = null; + validate(); + return true; + } + else + { + if (remove(fifoIndex, fifoTail + 1, fifoHead + 1)) ++fifoTail; + else --fifoHead; + validate(); + return false; + } + } + + int priorityIndex = priorityIndexOf(task); + Invariants.require(priorityIndex >= 0 || permitMissing); + if (priorityIndex >= 0) + { + if (priorityIndex == priorityHead) + { + tasks[priorityHead++] = null; + return !hasFifo(); + } + + if (remove(priorityIndex, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return false; + } + + return false; + } + + boolean removeUnsequenced(SafeTask task) + { + int unsequencedIndex = unsequencedIndexOf(task); + if (unsequencedIndex < 0) + return false; + + --unsequencedSize; + tasks[unsequencedIndex] = tasks[fifoTail - unsequencedSize]; + tasks[fifoTail - unsequencedSize] = null; + return true; + } + + // return true IFF was head + private boolean removePriority(SafeTask task, boolean permitAbsent) + { + int i = priorityIndexOf(task); + if (i < 0) + { + Invariants.require(permitAbsent); + return false; + } + + boolean wasHead = i == priorityHead; + if (remove(i, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return wasHead; + } + + // return true if we move the start forwards, false if we moved the end back + private boolean remove(int i, int start, int end) + { + if (i < (start + end) / 2) + { + System.arraycopy(tasks, start, tasks, start + 1, i - start); + tasks[start] = null; + return true; + } + else + { + System.arraycopy(tasks, i + 1, tasks, i, end - (i + 1)); + tasks[end - 1] = null; + return false; + } + } + + boolean contains(SafeTask task) + { + return indexOf(task) >= 0; + } + + private int indexOf(SafeTask task) + { + if (tasks[priorityHead] == task) + return priorityHead; + + if (tasks[fifoHead] == task) + return fifoHead; + + int i = priorityIndexOf(task); + if (i >= 0) + return i; + + return fifoIndexOf(task); + } + + private int priorityIndexOf(SafeTask task) + { + if (priorityTail - priorityHead > 16) + { + if (tasks[priorityHead] == task) + return priorityHead; + + int i = SortedArrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, AccordCacheEntryQueue::compare, SortedArrays.Search.CEIL); + if (i < 0) + return -1; + + while (i < priorityTail) + { + if (tasks[i] == task) + return i; + if (compare(task, tasks[i]) != 0) + break; + ++i; + } + } + + for (int i = priorityHead; i < priorityTail; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int fifoIndexOf(SafeTask task) + { + for (int i = fifoHead; i > fifoTail; --i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int unsequencedIndexOf(SafeTask task) + { + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoTail; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + int drainUnsequenced(TriConsumer, P1, P2> forEach, P1 p1, P2 p2) + { + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoTail; ++i) + { + SafeTask task = tasks[i]; + tasks[i] = null; + // should not be reentrant + forEach.accept(task, p1, p2); + } + int count = unsequencedSize; + unsequencedSize = 0; + return count; + } + + AccordCacheEntry.RunnableStatus ensureHeadFifo(SafeTask task) + { + if (hasFifo()) + { + Invariants.require(tasks[fifoHead] == task); + return NOT_RUNNABLE; + } + + if (tasks[priorityHead] == task) + { + tasks[priorityHead++] = null; + addFifo(task); + return STILL_RUNNABLE; + } + else + { + boolean wasPriorityHead = removePriority(task, false); + boolean isFifoHead = addFifo(task); + if (!isFifoHead) + return NOT_RUNNABLE; + if (wasPriorityHead) + return STILL_RUNNABLE; + if (hasPriority() || hasUnsequenced()) + return NEWLY_BLOCKING_RUNNABLE; + return NEWLY_RUNNABLE; + } + } + + static int compare(SafeTask a, SafeTask b) + { + Invariants.require(a != null && b != null); + int c = Long.compare(a.position, b.position); + if (c == 0) + c = a.executionContext().executionKind().compareTo(b.executionContext().executionKind()); + if (c == 0) + c = Long.compare(a.createdAt, b.createdAt); + if (c == 0) + c = Long.compare(a.loadedAt, b.loadedAt); + if (c == 0) + c = a.loggingId().compareTo(b.loggingId()); + return c; + } + + public boolean hasQueued() + { + return hasFifo() || hasPriority(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java new file mode 100644 index 0000000000..a87e405dd2 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java @@ -0,0 +1,823 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.stream.Stream; + +import accord.api.Agent; +import accord.api.ExclusiveAsyncExecutor; +import accord.api.RoutingKey; +import accord.impl.AbstractAsyncExecutor; +import accord.local.Command; +import accord.local.cfk.CommandsForKey; +import accord.primitives.TxnId; +import accord.utils.ArrayBuffers; +import accord.utils.Invariants; +import accord.utils.TinyEnumSet; +import accord.utils.UnhandledEnum; +import accord.utils.async.AsyncCallbacks.CallAndCallback; +import accord.utils.async.AsyncCallbacks.RunOrFail; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.cache.CacheSize; +import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; +import org.apache.cassandra.concurrent.Shutdownable; +import org.apache.cassandra.config.AccordConfig; +import org.apache.cassandra.config.AccordConfig.QueueBalancingModel; +import org.apache.cassandra.config.AccordConfig.QueuePriorityModel; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.metrics.AccordCacheMetrics; +import org.apache.cassandra.metrics.AccordExecutorMetrics; +import org.apache.cassandra.metrics.AccordReplicaMetrics; +import org.apache.cassandra.metrics.AccordSystemMetrics; +import org.apache.cassandra.metrics.LogLinearDecayingHistograms; +import org.apache.cassandra.metrics.LogLinearDecayingHistograms.LogLinearDecayingHistogram; +import org.apache.cassandra.metrics.ShardedDecayingHistograms; +import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistogramsShard; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.SaveExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.UniqueSave; +import org.apache.cassandra.service.accord.execution.ExclusiveExecutor.ExclusiveExecutorTask; +import org.apache.cassandra.service.accord.execution.IOTaskWrapper.WrappableIOTask; +import org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup; +import org.apache.cassandra.service.accord.execution.Task.GlobalGroup; +import org.apache.cassandra.service.accord.execution.Task.State; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Condition; +import org.apache.cassandra.utils.concurrent.Future; + +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.AccordCache.CommandAdapter.COMMAND_ADAPTER; +import static org.apache.cassandra.service.accord.execution.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; +import static org.apache.cassandra.service.accord.execution.AccordCache.registerJfrListener; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.EVICTED; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.OTHER; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_SCAN; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED_TO_LOAD; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.COUNTER_MASKS; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.selectByOverflowBits; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.setOverflowWhenLessEqual; + +/** + * NOTE: We assume that NO BLOCKING TASKS are submitted to this executor AND WAITED ON by another task executing on this executor. + * (as we do not immediately schedule additional threads for submitted tasks, but schedule new threads only if necessary when the submitting execution completes) + */ +public abstract class AccordExecutor implements CacheSize, LoadExecutor, Boolean>, SaveExecutor, Shutdownable, AbstractAsyncExecutor +{ + static final QueuePriorityModel PRIORITY_MODEL; + static final QueueBalancingModel BALANCING_MODEL; + static final long AGE_TO_FIFO; + // PRIORITY_FAIR blends two strategies (flow: least fairly serviced; age: earliest-queued work) by deficit + // round-robin; weights of BLEND_TOTAL come from a single imbalance ramp (onset..onset+width) trading age->flow. + static final int BLEND_SHIFT = 6, BLEND_TOTAL = 1 << BLEND_SHIFT; + static final int FLOW_ONSET, FLOW_WIDTH_SHIFT; + static final boolean BALANCE_BY_POSITION; + static final long GLOBAL_QUEUE_LIMITS, EXCLUSIVE_QUEUE_LIMITS; + static final int NONSYNC_MIN_BATCH_SIZE, NONSYNC_MAX_BATCH_SIZE, NONSYNC_BLOCKED_LIMIT; + static final boolean NONSYNC_ENABLED; + + static + { + AccordConfig config = DatabaseDescriptor.getAccord(); + AGE_TO_FIFO = config.queue_priority_age_to_fifo.to(TimeUnit.MICROSECONDS); + PRIORITY_MODEL = config.queue_priority_model != null ? config.queue_priority_model : QueuePriorityModel.HLC_FIFO; + BALANCING_MODEL = config.queue_balancing_model != null ? config.queue_balancing_model : QueueBalancingModel.BLENDED_PRIORITY_PHASE_FAIR; + FLOW_ONSET = config.queue_flow_imbalance_onset == null ? 4 : config.queue_flow_imbalance_onset; + FLOW_WIDTH_SHIFT = config.queue_flow_imbalance_width_shift == null ? 5 : config.queue_flow_imbalance_width_shift; + NONSYNC_MIN_BATCH_SIZE = config.queue_nonsync_min_batch_size == null ? 16 : config.queue_nonsync_min_batch_size; + NONSYNC_MAX_BATCH_SIZE = config.queue_nonsync_max_batch_size == null ? 64 : config.queue_nonsync_max_batch_size; + NONSYNC_ENABLED = config.queue_nonsync_enabled == null || config.queue_nonsync_enabled; + NONSYNC_BLOCKED_LIMIT = config.queue_nonsync_blocked_limit == null ? 8 : config.queue_nonsync_blocked_limit; + Invariants.require(FLOW_ONSET >= 0 && FLOW_WIDTH_SHIFT >= 0); + switch (BALANCING_MODEL) + { + default: throw new UnhandledEnum(BALANCING_MODEL); + case PRIORITY_ONLY: + case BLENDED_PRIORITY_PHASE_FAIR: + BALANCE_BY_POSITION = true; + break; + case PHASE_ONLY: + case PHASE_FAIR: + BALANCE_BY_POSITION = false; + } + + { + // TODO (required): pick default max loads/saves/range loads based on number of threads + long global = COUNTER_MASKS, exclusive = COUNTER_MASKS; + global ^= (0x7fL ^ 1) << (RANGE_SCAN.ordinal() * 8); + if (config.queue_active_limits != null) + { + long[] limits = parseEnumParams(config.queue_active_limits, "queue_active_limits"); + global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), global, limits[0]); + exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), global, limits[1]); + } + GLOBAL_QUEUE_LIMITS = global; + EXCLUSIVE_QUEUE_LIMITS = exclusive; + } + + } + + public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); + + public interface AccordExecutorFactory + { + AccordExecutor get(int executorId, Mode mode, int threads, IntFunction name, Agent agent); + } + + public enum Mode { RUN_WITH_LOCK, RUN_WITHOUT_LOCK } + + // WARNING: this is a shared object, so close is NOT idempotent + public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable + { + final AccordExecutor executor; + + public ExclusiveGlobalCaches(AccordExecutor executor, AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) + { + super(global, commands, commandsForKey); + this.executor = executor; + } + + @Override + public void close() + { + executor.beforeUnlockExternal(); + global.tryShrinkOrEvict(executor.lock); + executor.unlock(TaskRunner.get()); + } + } + + public static class GlobalCaches + { + public final AccordCache global; + public final AccordCache.Type commands; + public final AccordCache.Type commandsForKey; + + public GlobalCaches(AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) + { + this.global = global; + this.commands = commands; + this.commandsForKey = commandsForKey; + } + } + + final LogLinearDecayingHistograms histograms; + final LogLinearDecayingHistogram elapsedPreparingToRun; + final LogLinearDecayingHistogram elapsedWaitingToRun; + final LogLinearDecayingHistogram elapsedRunning; + final LogLinearDecayingHistogram elapsed; + final LogLinearDecayingHistogram keys; + public final AccordReplicaMetrics.Shard replicaMetrics; + + private final Lock lock; + final Agent agent; + public final int executorId; + private final AccordCache cache; + private final ExclusiveGlobalCaches caches; + final AtomicLong uniqueCreatedAt = new AtomicLong(); + final DebugExecutor debug = DebugExecutor.maybeDebug(); + + private long maxWorkingSetSizeInBytes; + private long maxWorkingCapacityInBytes; + + final TaskQueueStandalone> scanningRanges = new TaskQueueStandalone<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning + final TaskQueueStandalone> loading = new TaskQueueStandalone<>(TinyEnumSet.encode(LOADING_REQUIRED, LOADING_OPTIONAL)); + final TaskQueueStandalone> waitingOnCacheQueues = new TaskQueueStandalone<>(TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL)); + final TaskQueueRunnable runnable = new TaskQueueRunnable<>(); + + private final Tranches tranches = new Tranches(this); + + /** + * Newly submitted work must take a position >= minPosition, but this condition does not apply to consequences of + * previously submitted work; this inherits the originating task's position and tranche. + * This is to ensure afterSubmittedAndConsequences functions correctly. + */ + long minPosition = 1; + long nextPosition = 1; + int tasks; + + private boolean hasPausedLoading; + + private List waitingForQuiescence; + + AccordExecutor(Lock lock, int executorId, Agent agent) + { + this.lock = lock; + this.executorId = executorId; + this.cache = new AccordCache(this, 0); + this.agent = agent; + + final AccordCache.Type commands; + final AccordCache.Type commandsForKey; + commands = cache.newType(TxnId.class, COMMAND_ADAPTER, AccordCacheMetrics.CommandsCacheMetrics.newShard(lock)); + registerJfrListener(executorId, commands, "Command"); + + commandsForKey = cache.newType(RoutingKey.class, CFK_ADAPTER, AccordCacheMetrics.CommandsForKeyCacheMetrics.newShard(lock)); + registerJfrListener(executorId, commandsForKey, "CommandsForKey"); + + this.caches = new ExclusiveGlobalCaches(this, cache, commands, commandsForKey); + + DecayingHistogramsShard histogramsShard = HISTOGRAMS.newShard(lock); + this.histograms = histogramsShard.unsafeGetInternal(); + this.elapsedPreparingToRun = AccordExecutorMetrics.INSTANCE.elapsedPreparingToRun.forShard(histogramsShard); + this.elapsedWaitingToRun = AccordExecutorMetrics.INSTANCE.elapsedWaitingToRun.forShard(histogramsShard); + this.elapsedRunning = AccordExecutorMetrics.INSTANCE.elapsedRunning.forShard(histogramsShard); + this.elapsed = AccordExecutorMetrics.INSTANCE.elapsed.forShard(histogramsShard); + this.keys = AccordExecutorMetrics.INSTANCE.keys.forShard(histogramsShard); + this.replicaMetrics = new AccordReplicaMetrics.Shard(histogramsShard); + } + + abstract boolean isInLoop(); + abstract void beforeUnlockExternal(); + abstract void submit(Consumer sync, Function async, P1 p1); + public abstract boolean hasTasks(); + public abstract boolean isOwningThread(); + + public final Lock unsafeLock() + { + return lock; + } + + public final void lock(TaskRunner self) + { + long startAt = DEBUG_EXECUTION ? Clock.Global.nanoTime() : 0; + if (!self.tryEnterAccordLockedExecutor(this)) + throw new UnsupportedOperationException("To ensure system performance, it is not permitted to utilise multiple AccordExecutor simultaneously with the same thread"); + //noinspection LockAcquiredButNotSafelyReleased + lock.lock(); + if (DEBUG_EXECUTION) debug.onEnterLock(startAt); + } + + public final void unlock(TaskRunner self) + { + self.exitAccordLockedExecutor(); + if (DEBUG_EXECUTION) debug.onExitLock(); + lock.unlock(); + } + + public final boolean tryLock(TaskRunner self) + { + AccordExecutor active = self.accordActiveExecutor(); + if (active != null && active != this) + return false; + + return onTryLock(self, lock.tryLock()); + } + + final boolean onTryLock(TaskRunner self, boolean result) + { + if (result && !self.tryEnterAccordLockedExecutor(this)) + { + // shouldn't be possible, we should have checked this already + lock.unlock(); + throw new IllegalStateException(); + } + return result; + } + + public ExclusiveGlobalCaches lockCaches() + { + lock(TaskRunner.get(Thread.currentThread())); + return caches; + } + + public AccordCache cacheExclusive() + { + Invariants.require(isOwningThread()); + return cache; + } + + public AccordCache cacheUnsafe() + { + return cache; + } + + final boolean hasAlreadyWaitingToRun() + { + return runnable.hasWaitingToRun(); + } + + void updateWaitingToRunExclusive() + { + // TODO (expected): this should not be invoked on every update of waiting to run + maybeUnpauseLoading(); + } + + // drain only new work; specifically leave anything that would call completeTask queued. + // this is to maintain invariants in Tranches.complete, where we may have some consequence of some earlier task + // pending but unqueued, so that we have not incremented its tranche count, and in the interim we set the tranche + // count to zero + void drainUnqueuedNewWorkExclusive() + { + } + + final Task pollAlreadyWaitingToRunExclusive() + { + Task next = runnable.poll(); + if (DEBUG_EXECUTION && next != null) DebugTask.get(next).onPolled(); + return next; + } + + public void waitForQuiescence() + { + TaskRunner self = TaskRunner.get(); + Condition condition; + lock(self); + try + { + if (tasks == 0) + return; + + if (waitingForQuiescence == null) + waitingForQuiescence = new ArrayList<>(); + condition = Condition.newOneTimeCondition(); + waitingForQuiescence.add(condition); + } + finally + { + unlock(self); + } + condition.awaitThrowUncheckedOnInterrupt(); + } + + protected void notifyQuiescentExclusive() + { + if (waitingForQuiescence != null) + { + waitingForQuiescence.forEach(Condition::signalAll); + waitingForQuiescence = null; + } + tranches.finishAll(nextPosition); + } + + public void afterSubmittedAndConsequences(Runnable run) + { + TaskRunner self = TaskRunner.get(); + + lock(self); + try + { + drainUnqueuedNewWorkExclusive(); + if (tasks == 0) + { + run.run(); + return; + } + + tranches.registerWait(run); + } + finally + { + unlock(self); + } + } + + final void maybeUnpauseLoading() + { + if (!hasPausedLoading) + return; + + if (cache.weightedSize() < maxWorkingCapacityInBytes && !runnable.hasWaitingToRun()) + { + hasPausedLoading = false; + runnable.restart(RANGE_SCAN.ordinal()); + runnable.restart(LOAD.ordinal()); + runnable.restart(RANGE_LOAD.ordinal()); + } + } + + final void maybePauseLoading() + { + if (hasPausedLoading) + return; + + if (cache.weightedSize() >= maxWorkingCapacityInBytes || !runnable.hasWaitingToRun()) + { + AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); + hasPausedLoading = true; + runnable.stop(RANGE_SCAN.ordinal()); + runnable.stop(LOAD.ordinal()); + runnable.stop(RANGE_LOAD.ordinal()); + } + } + + public ExclusiveExecutor newExclusiveExecutor(int commandStoreId) + { + return new ExclusiveExecutor(this, commandStoreId); + } + + public ExclusiveAsyncExecutor newExclusiveExecutor() + { + return new ExclusiveExecutor(this); + } + + @Override + public IOTaskLoad load(SafeTask parent, Boolean isForRange, AccordCacheEntry entry) + { + IOTaskLoad result = newLoad(entry, isForRange); + result.submitExclusive(null, parent); + return result; + } + + @Override + public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) + { + return new IOTaskSave(this, entry, identity, save).submitExclusive(null, null); + } + + Cancellable submit(Task task) + { + submit(Task::submitExclusive, i -> i, task); + return task; + } + + public Future submit(Runnable run) + { + Task inherit = inherit(); + PlainRunnable task = inherit == null ? new PlainRunnable(this, new AsyncPromise<>(), run, OTHER) + : new PlainRunnable(this, new AsyncPromise<>(), run, OTHER, inherit.position, inherit.tranche()); + submit(task); + return task.result; + } + + public void execute(Runnable run) + { + Task inherit = inherit(); + PlainRunnable task = inherit == null ? new PlainRunnable(this, null, run, OTHER) + : new PlainRunnable(this, null, run, OTHER, inherit.position, inherit.tranche()); + submit(task); + } + + @Override + public Cancellable execute(RunOrFail runOrFail) + { + Task inherit = inherit(); + PlainChain submit = inherit == null ? new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER) + : new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + return submit(submit); + } + + public AsyncChain buildDebuggable(Callable call, Object describe) + { + return new AsyncChains.Head<>() + { + @Override + protected Cancellable start(BiConsumer callback) + { + Task inherit = inherit(); + return submit(inherit == null ? new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, describe) + : new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, inherit.position, inherit.tranche(), describe)); + } + }; + } + + public void submitExclusive(Runnable runnable) + { + new PlainRunnable(this, null, runnable, OTHER).submitExclusive(null, null); + } + + Cancellable submitExclusive(Task parent, GlobalGroup group, WrappableIOTask task) + { + return new IOTaskWrapper(this, task, group, parent.position, parent.tranche()).submitExclusive(null, parent); + } + + Task inherit() + { + return inherit(Thread.currentThread()); + } + + Task inherit(Thread thread) + { + TaskRunner self = TaskRunner.get(thread); + + if (self.accordActiveExecutor() != this) + return null; + + Task task = self.accordActiveSelfTask(); + if (task instanceof ExclusiveExecutorTask) + task = ((ExclusiveExecutorTask)task).queue.task; + return task; + } + + void registerConsequenceExclusive(Task parent, Task task) + { + ++tasks; + int tranche = parent.tranche(); + tranches.addInherited(tranche, parent.position); + task.position = parent.position; + task.setInheritedWithTranche(tranche); + } + + void registerExclusive(Task task) + { + ++tasks; + if (task.hasInherited()) + { + tranches.addInherited(task.tranche(), task.position); + } + else + { + long position = task.position; + if (position == 0) + { + task.position = position = nextPosition++; + } + else + { + long delta = nextPosition - position; + if (delta >= AGE_TO_FIFO) + task.position = position = nextPosition++; + else if (delta <= 0) + nextPosition = position + 1; + else if (position < minPosition) + task.position = position = minPosition; + } + task.setTranche(tranches.addNew(position)); + } + } + + final void cleanupTaskExclusive(Task task, boolean executed) + { + runnable.cleanup(task); + try { task.cleanupExclusive(this, executed); } + finally + { + cache.tryShrinkOrEvict(lock); + maybeUnpauseLoading(); + } + } + + final void unregisterExclusive(Task task) + { + int tranch = task.tranche(); + --tasks; + tranches.complete(tranch); + } + + void onScannedRangesExclusive(SafeTask task, Throwable fail) + { + // the task may have already been cancelled, in which case we don't need to fail it + if (task.state().isExecuted()) + return; + + if (fail != null) failExclusive(task, fail, FAILED_TO_LOAD); + else task.rangeScanner().scannedExclusive(); + } + + private void failExclusive(SafeTask task, Throwable fail, State newState) + { + if (task.state().isExecuted()) + return; + + try { task.failExclusive(fail, newState); } + catch (Throwable t) { agent.onException(t); } + finally + { + task.unqueueIfQueued(); + cleanupTaskExclusive(task, false); + } + } + + void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) + { + cache.saved(state, identity, fail); + } + + void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail) + { + if (loaded.status() == EVICTED) + return; + + try (ArrayBuffers.BufferList> tasks = loaded.drainWaitingToLoad()) + { + if (fail != null) + { + for (SafeTask task : tasks) + failExclusive(task, fail, FAILED_TO_LOAD); + cache.failedToLoad(loaded); + } + else + { + cache.loaded(loaded, value); + for (SafeTask task : tasks) + task.onLoadOneExclusive(loaded); + } + } + + maybePauseLoading(); + } + + public void executeDirectlyWithLock(Runnable command) + { + TaskRunner self = TaskRunner.get(); + lock(self); + try + { + self.setAccordActiveExecutor(this); + command.run(); + } + finally + { + beforeUnlockExternal(); + self.setAccordActiveExecutor(null); + unlock(self); + } + } + + @Override + public void setCapacity(long bytes) + { + Invariants.require(isOwningThread()); + cache.setCapacity(bytes); + maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; + } + + public void setWorkingSetSize(long bytes) + { + Invariants.require(isOwningThread()); + maxWorkingSetSizeInBytes = bytes; + maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; + if (maxWorkingCapacityInBytes < maxWorkingSetSizeInBytes) + maxWorkingCapacityInBytes = Long.MAX_VALUE; + } + + @Override + public long capacity() + { + return cache.capacity(); + } + + @Override + public int size() + { + return cache.size(); + } + + @Override + public long weightedSize() + { + return cache.weightedSize(); + } + + public int unsafePreparingToRunCount() + { + return loading.size() + scanningRanges.size(); + } + + public int unsafeWaitingToRunCount() + { + return runnable.waitingCount(); + } + + public int unsafeRunningCount() + { + return runnable.assigned.size(); + } + + public Stream active() + { + return Stream.of(); + } + + public int executorId() + { + return executorId; + } + + public static IntFunction constant(O out) + { + return ignore -> out; + } + + IOTaskLoad newLoad(AccordCacheEntry entry, boolean isForRange) + { + return new IOTaskLoad<>(this, entry, isForRange ? RANGE_LOAD : LOAD); + } + + public List taskSnapshot() + { + List result = new ArrayList<>(); + TaskRunner self = TaskRunner.get(); + lock(self); + try + { + addToSnapshot(result, scanningRanges, TaskInfo.Status.SCANNING_RANGES, TaskInfo.Status.SCANNING_RANGES); + addToSnapshot(result, loading, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); + for (TaskQueue queue : runnable.queues) + { + if (queue != null) + addToSnapshot(result, queue, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); + } + addToSnapshot(result, runnable.assigned, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); + } + finally + { + unlock(self); + } + result.sort(TaskInfo::compareTo); + return result; + } + + private static void addToSnapshot(List snapshot, TaskQueue queue, TaskInfo.Status ifCurrent, TaskInfo.Status ifQueued) + { + for (int i = 0 ; i < queue.size() ; ++i) + { + Task t = queue.getSingle(i); + if (t instanceof ExclusiveExecutorTask) + { + ExclusiveExecutor q = ((ExclusiveExecutorTask) t).queue; + snapshot.add(new TaskInfo(ifCurrent, q.commandStoreId, q.task)); + for (TaskQueue q0 : q.queues) + { + if (q0 != null) + { + for (int j = 0 ; j < q0.size() ; ++j) + snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q0.getSingle(j))); + } + } + } + else + { + int commmandStoreId = t instanceof SafeTask ? ((SafeTask) t).commandStore.id() : -1; + snapshot.add(new TaskInfo(ifCurrent, commmandStoreId, t)); + } + } + } + + private static long[] parseEnumParams(String input, String describe) + { + String[] specs = input.split(";"); + if (specs.length != 2) + throw new IllegalArgumentException("Invalid specifiers in " + describe + "; expect [GlobalGroup];[ExclusiveGroup] but got: " + input); + long[] result = new long[2]; + result[0] = parseEnumParams(GlobalGroup::valueOf, specs[0], describe + " for GlobalGroup"); + result[1] = parseEnumParams(ExclusiveGroup::valueOf, specs[1], describe + " for ExclusiveGroup"); + return result; + } + + private static long parseEnumParams(Function> get, String input, String describe) + { + long result = 0; + for (String spec : input.split(",")) + { + if (spec.trim().isEmpty()) continue; + + String[] split = spec.split(":"); + if (split.length != 2) + throw new IllegalArgumentException("Invalid specifier " + spec + " in " + describe + ": " + input); + + try + { + Enum queue = get.apply(split[0]); + long value = Long.parseLong(split[1]); + if (value <= 0 || value >= 128) + throw new IllegalArgumentException("Invalid limit " + value + " in queue_active_limits: " + input); + + result |= value << (queue.ordinal() * 8); + } + catch (Throwable t) + { + throw new IllegalArgumentException("Invalid queue identifier " + split[0] + " in " + describe + ": " + input); + } + } + + return result; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorAsyncSubmit.java similarity index 88% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorAsyncSubmit.java index a2c57a4556..55a0086cf1 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorAsyncSubmit.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.function.IntFunction; @@ -26,9 +26,9 @@ import accord.api.Agent; import org.apache.cassandra.utils.concurrent.LockWithAsyncSignal; // WARNING: experimental - needs more testing -public class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSubmit +public class AccordExecutorAsyncSubmit extends AbstractSemiSyncSubmit { - private final AccordExecutorLoops loops; + private final Loops loops; private final LockWithAsyncSignal lock; public AccordExecutorAsyncSubmit(int executorId, Mode mode, int threads, IntFunction name, Agent agent) @@ -40,7 +40,7 @@ public class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSub { super(lock, executorId, agent); this.lock = lock; - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); } @Override @@ -52,7 +52,7 @@ public class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSub } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -76,7 +76,7 @@ public class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSub } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isOwner(Thread.currentThread()); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java similarity index 89% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java index 67e897d479..2bafc5b4b6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; @@ -26,9 +26,9 @@ import java.util.function.IntFunction; import accord.api.Agent; // WARNING: experimental - needs more testing -public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSyncSubmit +public class AccordExecutorSemiSyncSubmit extends AbstractSemiSyncSubmit { - private final AccordExecutorLoops loops; + private final Loops loops; private final ReentrantLock lock; private final Condition hasWork; private int waiting; @@ -43,7 +43,7 @@ public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSync super(lock, executorId, agent); this.lock = lock; this.hasWork = lock.newCondition(); - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); } @Override @@ -61,7 +61,7 @@ public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSync } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -87,7 +87,7 @@ public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSync } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isHeldByCurrentThread(); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java similarity index 92% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java index 392ce2a015..acb5eb7202 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java @@ -16,33 +16,33 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import javax.annotation.Nullable; import accord.api.Agent; import accord.utils.Invariants; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; import org.apache.cassandra.utils.concurrent.SignalLock; import static accord.utils.Invariants.nonNull; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.UNINITIALIZED; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.Task.State.UNINITIALIZED; -public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop +public class AccordExecutorSignalLoop extends AbstractLoop { private static class ShutdownException extends RuntimeException {} private static final int MAX_LOOPS = 1000; // limit the amount of time we hold the lock for private final SignalLock lock; - private final AccordExecutorLoops loops; + private final Loops loops; private int readyToRunTarget = 1; private final int readyToRunLimit; // TODO (desired): intrusive queue using Task.next, but a little challenging because we reuse SequentialQueueTask so have ABA problem @@ -64,13 +64,13 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop super(lock, executorId, agent); Invariants.require(threads <= SignalLock.MAX_THREADS); this.lock = lock; - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); this.readyToRunLimit = Math.min(threads * 4, SignalLock.MAX_SIGNAL_COUNT); } - void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submit(Consumer sync, Function async, P1 p1) { - Task next = async.apply(p1a, p2, p3, p4); + Task next = async.apply(p1); Task prev = push(next); if (prev == null) lock.signalLockWork(); @@ -134,7 +134,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop { Task next = submit.next; submit.next = null; - submit.submitExclusive(this); + submit.submitExclusive(); submit = next; } } @@ -159,7 +159,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop pendingNewHead = destructiveNext(submit); if (pendingNewHead == null) pendingNewTail = null; - submit.submitExclusive(this); + submit.submitExclusive(); } return true; } @@ -315,7 +315,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop return hasAlreadyWaitingToRun(); } - class LoopTask extends AccordExecutorLoops.LoopTask + class LoopTask extends Loops.LoopTask { final int index; @@ -325,7 +325,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop this.index = index; } - private Task awaitWork(AccordTaskRunner self) + private Task awaitWork(TaskRunner self) { while (true) { @@ -353,7 +353,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop } } - private Task executedAndMaybeGetWork(AccordTaskRunner self, @Nullable Task executed) + private Task executedAndMaybeGetWork(TaskRunner self, @Nullable Task executed) { if (lock.tryAcquireAsyncWork()) { @@ -394,7 +394,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop return result; } - final boolean tryLock(AccordTaskRunner self) + final boolean tryLock(TaskRunner self) { if (self.accordLockedExecutor() != null) return false; @@ -402,7 +402,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop return onTryLock(self, lock.tryLock(index)); } - final boolean unlockAndAcquire(AccordTaskRunner self) + final boolean unlockAndAcquire(TaskRunner self) { self.exitAccordLockedExecutor(); if (DEBUG_EXECUTION) debug.onExitLock(); @@ -413,7 +413,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop public void run() { Thread thread = Thread.currentThread(); - AccordTaskRunner self = AccordTaskRunner.get(thread); + TaskRunner self = TaskRunner.get(thread); self.setAccordActiveExecutor(AccordExecutorSignalLoop.this); setWrapped(self); @@ -482,7 +482,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -494,7 +494,7 @@ public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isOwner(); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java similarity index 90% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java index cadd7a766e..4d2508b87f 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java @@ -16,22 +16,22 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import accord.api.Agent; import accord.utils.Invariants; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; import org.apache.cassandra.concurrent.ExecutorPlus; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -class AccordExecutorSimple extends AccordExecutor +public class AccordExecutorSimple extends AccordExecutor { final ExecutorPlus executor; final ReentrantLock lock; @@ -82,7 +82,7 @@ class AccordExecutorSimple extends AccordExecutor protected void run() { - AccordTaskRunner self = AccordTaskRunner.get(); + TaskRunner self = TaskRunner.get(); self.setAccordActiveExecutor(AccordExecutorSimple.this); lock.lock(); try @@ -126,12 +126,12 @@ class AccordExecutorSimple extends AccordExecutor } @Override - void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submit(Consumer sync, Function async, P1 p1) { lock.lock(); try { - sync.accept(this, p1s, p2, p3, p4); + sync.accept(p1); } finally { @@ -155,7 +155,7 @@ class AccordExecutorSimple extends AccordExecutor } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isHeldByCurrentThread(); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSyncSubmit.java similarity index 79% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSyncSubmit.java index e4efac3ad4..e6d47f8910 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSyncSubmit.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import accord.api.Agent; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop +public class AccordExecutorSyncSubmit extends AbstractLockLoop { - private final AccordExecutorLoops loops; + private final Loops loops; private final ReentrantLock lock; private final Condition hasWork; @@ -48,7 +48,7 @@ public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop super(lock, executorId, agent); this.lock = lock; this.hasWork = lock.newCondition(); - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); } @Override @@ -58,7 +58,7 @@ public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -70,7 +70,7 @@ public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isHeldByCurrentThread(); } @@ -89,13 +89,13 @@ public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop hasWork.signal(); } - void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submitExternal(Consumer sync, Function async, P1 p1) { - AccordTaskRunner self = AccordTaskRunner.get(); + TaskRunner self = TaskRunner.get(); lock(self); try { - submitExternalExclusive(sync, p1s, p2, p3, p4); + submitExternalExclusive(sync, async, p1); } finally { diff --git a/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java new file mode 100644 index 0000000000..c171ed08de --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +final class CancelTask extends Task +{ + final Task cancel; + + CancelTask(Task cancel) + { + super(GlobalGroup.OTHER); + this.cancel = cancel; + } + + @Override + void submitExclusive() + { + cancel.cancelExclusive(); + } + + @Override + boolean isNewWork() + { + return false; + } + + @Override + String toDescription() + { + return "Cancel " + cancel.toDescription(); + } + + @Override void preRunExclusive() { throw new UnsupportedOperationException(); } + @Override void run() { throw new UnsupportedOperationException(); } + @Override void reportFailure(Throwable fail) { throw new UnsupportedOperationException(); } + @Override public void cancel() { throw new UnsupportedOperationException(); } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java new file mode 100644 index 0000000000..b0bd8d37e6 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.Callable; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.concurrent.locks.LockSupport; + +import accord.api.ExclusiveAsyncExecutor; +import accord.local.ExecutionContext; +import accord.utils.IntrusiveHeapNode; +import accord.utils.Invariants; +import accord.utils.async.AsyncCallbacks; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.service.accord.debug.DebugExecution; + +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.COMMAND_STORE; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.execution.TaskQueueRunnable.RUNNABLE; + +public final class ExclusiveExecutor extends TaskQueueMulti implements ExclusiveAsyncExecutor +{ + private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(ExclusiveExecutor.class, Thread.class, "owner"); + + static final class ExclusiveExecutorTask extends Task + { + final ExclusiveExecutor queue; + + ExclusiveExecutorTask(ExclusiveExecutor queue) + { + super(COMMAND_STORE); + this.queue = queue; + } + + @Override void submitExclusive() { throw new UnsupportedOperationException(); } + @Override boolean isNewWork() { throw new UnsupportedOperationException(); } + @Override public void cancel() { throw new UnsupportedOperationException(); } + + @Override + void preRunExclusive() + { + super.preRunExclusive(); + queue.preRunTask(); + } + + @Override + void run() + { + queue.runTask(); + } + + @Override + void cleanupExclusive(AccordExecutor executor, boolean executed) + { + unsafeSetStateExclusive(WAITING_TO_RUN); + queue.cleanupTask(executed); + } + + @Override + void reportFailure(Throwable t) + { + queue.task.reportFailure(t); + } + + @Override + String toDescription() + { + return queue.task.toDescription(); + } + + protected boolean isInHeap() + { + return super.isInHeap(); + } + } + + private final AccordExecutor executor; + final int commandStoreId; + final ExclusiveExecutorTask selfTask; + Task task; + volatile Thread owner, waiting; + private boolean stopped; + private volatile boolean visibleStopped; + private boolean terminated; + + final DebugExecution.DebugExclusiveExecutor debug; + + ExclusiveExecutor(AccordExecutor executor) + { + this(executor, -1); + } + + ExclusiveExecutor(AccordExecutor executor, int commandStoreId) + { + super(RUNNABLE, commandStoreId < 0 ? Task.GroupKind.NONE : Task.GroupKind.EXCLUSIVE, AccordExecutor.EXCLUSIVE_QUEUE_LIMITS); + this.executor = executor; + this.commandStoreId = commandStoreId; + this.selfTask = new ExclusiveExecutorTask(this); + this.debug = DebugExecution.DebugExclusiveExecutor.maybeDebug(executor.debug, commandStoreId); + } + + void preRunTask() + { + task.preRunExclusive(); + } + + void runTask() + { + Thread self = Thread.currentThread(); + if (!ownerUpdater.compareAndSet(this, null, self)) + { + if (DEBUG_EXECUTION) debug.onWaiting(); + Invariants.require(waiting == null); + waiting = self; + outer: + do + { + while (true) + { + Thread owner = this.owner; + if (owner == self) break outer; + if (owner == null) continue outer; + LockSupport.park(); + } + } + while (!ownerUpdater.compareAndSet(this, null, self)); + Invariants.require(waiting == self); + waiting = null; + } + + try + { + if (stopped && reject(task)) + task.reportFailure(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((SafeTask) task).executionContext())); + else + task.run(); + } + finally + { + // NOTE: we can ONLY safely release owner here due to AccordCacheEntry locking, which remains in place until AccordTask.releaseResourcesExclusive + // this also relies on AccordSafeCommandStore$ExclusiveCaches.acquireIfLoaded returning false when the entry is locked + owner = null; + } + } + + private boolean reject(Task task) + { + if (!(task instanceof SafeTask)) + return true; + + ExecutionContext context = ((SafeTask) task).executionContext(); + return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); + } + + void cleanupTask(boolean executed) + { + try + { + task.unsetQueue(this); + task.cleanupExclusive(executor, executed); + } + finally + { + active = 0; + task = super.pollMulti(); + if (DEBUG_EXECUTION) debug.onSetTask(task); + if (task != null) + { + selfTask.position = task.position; + selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); + executor.runnable.enqueue(selfTask, false); + } + } + } + + void enqueue(Task newTask, boolean incrementArrivals) + { + + if (task != null) + { + if (incrementArrivals) + executor.runnable.incrementArrivals(selfTask); + // TODO (expected): restore some invariant here +// Invariants.require(selfTask.isInHeap() || selfTask.is(RUNNING)); + super.enqueueMulti(newTask, incrementArrivals); + } + else + { + Invariants.require(isEmptySingle()); + if (incrementArrivals) + incrementArrivals(newTask); + incrementDispatches(newTask); + task = newTask; + task.setQueue(this); + selfTask.position = newTask.position; + selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); + executor.runnable.enqueue(selfTask, incrementArrivals); + if (DEBUG_EXECUTION) debug.onSetTask(newTask); + } + } + + @Override + void unqueue(Task remove) + { + if (remove == task) removeCurrentTask(remove); + else super.unqueueMulti(remove); + } + + boolean tryUnqueueWaiting(Task remove) + { + if (remove == task) return tryRemoveCurrentTask(remove); + else return super.tryUnqueueWaiting(remove); + } + + private boolean tryRemoveCurrentTask(IntrusiveHeapNode remove) + { + if (executor.runnable.isAssigned(selfTask)) + return false; + + removeCurrentTask(remove); + return true; + } + + private void removeCurrentTask(IntrusiveHeapNode remove) + { + Invariants.require(remove == task); + // cannot overwrite task while it is being executed - this cannot happen for AccordTask + // but can for other tasks that don't track their own state + + decrementDispatches(task); + task.unsetQueue(this); + task = pollMulti(); + if (DEBUG_EXECUTION) debug.onSetTask(task); + if (executor.runnable.isWaiting(selfTask)) + { + if (task == null) executor.runnable.unqueue(selfTask); + else + { + selfTask.position = task.position; + executor.runnable.requeue(selfTask); + } + } + else + { + Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove); + if (task != null) + { + selfTask.position = task.position; + selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); + executor.runnable.enqueue(selfTask, false); + } + } + Invariants.require(task == null || executor.runnable.isWaiting(selfTask)); + } + + public boolean inExecutor() + { + return owner == Thread.currentThread(); + } + + public boolean stopped() + { + return visibleStopped; + } + + public void stop() + { + Invariants.require(inExecutor()); + this.stopped = true; + this.visibleStopped = true; + } + + public void terminate() + { + Invariants.require(inExecutor()); + this.visibleStopped = this.terminated = this.stopped = true; + } + + @Override + public AsyncChain chain(Runnable run) + { + return AsyncChains.chain(this, run); + } + + @Override + public AsyncChain chain(Callable call) + { + return AsyncChains.chain(this, call); + } + + @Override + public AsyncChain flatChain(Callable> call) + { + return AsyncChains.flatChain(this, call); + } + + Task inherit() + { + Thread thread = Thread.currentThread(); + if (thread == owner) + return Task.unwrap(task); + + return executor.inherit(thread); + } + + @Override + public void execute(Runnable run) + { + Task inherit = inherit(); + PlainRunnable submit = inherit == null ? new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER) + : new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + executor.submit(submit); + } + + @Override + public Cancellable execute(AsyncCallbacks.RunOrFail runOrFail) + { + Task inherit = inherit(); + PlainChain submit = inherit == null ? new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER) + : new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + return executor.submit(submit); + } + + @Override + public boolean tryExecuteImmediately(Runnable run) + { + Thread thread = Thread.currentThread(); + Thread owner = this.owner; + if (owner != null && owner != thread) + return false; + + TaskRunner self = TaskRunner.get(thread); + AccordExecutor active = self.accordActiveExecutor(); + if (active != null && active != executor) + return false; // prevent cross-executor locking/execution + + if (owner == null && !ownerUpdater.compareAndSet(this, null, thread)) + return false; + + try + { + if (active == null) + self.setAccordActiveExecutor(executor); + + run.run(); + } + catch (Throwable t) + { + executor.agent.onException(t); + } + finally + { + if (owner == null) + { + Thread waiting = this.waiting; + Invariants.require(waiting != self); + this.owner = waiting; + if (waiting == null) // recheck, to ensure happens-before relation with a new waiter that expects any non-null owner to notify it + waiting = this.waiting; + if (waiting != null) + LockSupport.unpark(waiting); + } + + if (active == null) + self.setAccordActiveExecutor(null); + } + return true; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTask.java b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java new file mode 100644 index 0000000000..68c976eb7b --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.DebuggableTask; + +// a task that may be submitted to this executor or another +public abstract class IOTask extends Plain implements Cancellable, DebuggableTask +{ + IOTask(AccordExecutor executor, GlobalGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + } + + IOTask(AccordExecutor executor, GlobalGroup group) + { + super(executor, group); + } + + abstract void postRunExclusive(); + + @Override + void cleanupExclusive(AccordExecutor executor, boolean executed) + { + super.cleanupExclusive(executor, executed); + postRunExclusive(); + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return null; + } + + @Override + public long creationTimeNanos() + { + return createdAt; + } + + @Override + public long startTimeNanos() + { + return runningAt; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java new file mode 100644 index 0000000000..aa54cfcf2a --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.Invariants; + +import org.apache.cassandra.utils.Closeable; + +import static org.apache.cassandra.service.accord.execution.IOTaskLoad.FailureHolder.NOT_STARTED; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; + +public class IOTaskLoad extends IOTask +{ + static class FailureHolder + { + static final FailureHolder NOT_STARTED = new FailureHolder(new RuntimeException("Not started")); + + final Throwable fail; + + FailureHolder(Throwable fail) + { + this.fail = fail; + } + } + + final AccordCacheEntry entry; + Object result = NOT_STARTED; + + IOTaskLoad(AccordExecutor executor, AccordCacheEntry entry, GlobalGroup group) + { + super(executor, group); + Invariants.require(group == LOAD || group == RANGE_LOAD); + this.entry = entry; + } + + void postRunExclusive() + { + if (!(result instanceof FailureHolder)) + executor.onLoadedExclusive(entry, (V) result, null); + else + executor.onLoadedExclusive(entry, null, ((FailureHolder) result).fail); + } + + @Override + String toDescription() + { + return "Load " + entry.key(); + } + + @Override + public void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); + } + onRunComplete(); + } + + @Override + void reportFailure(Throwable t) + { + result = new FailureHolder(t); + } + + @Override + public String description() + { + return "Loading " + entry; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java new file mode 100644 index 0000000000..d010b6e1f1 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.apache.cassandra.utils.Closeable; + +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.SAVE; + +class IOTaskSave extends IOTask +{ + private static final Throwable NOT_STARTED = new Throwable(); + + final AccordCacheEntry entry; + final AccordCacheEntry.UniqueSave identity; + final Runnable run; + Throwable failure = NOT_STARTED; + + IOTaskSave(AccordExecutor executor, AccordCacheEntry entry, AccordCacheEntry.UniqueSave identity, Runnable run) + { + super(executor, SAVE); + this.entry = entry; + this.identity = identity; + this.run = run; + } + + @Override + void postRunExclusive() + { + executor.onSavedExclusive(entry, identity, failure); + } + + @Override + String toDescription() + { + return "Save " + entry.key(); + } + + @Override + public void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + run.run(); + } + onRunComplete(); + failure = null; + } + + @Override + protected void reportFailure(Throwable t) + { + failure = t; + } + + @Override + public String description() + { + return "Save " + entry; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java new file mode 100644 index 0000000000..ce6d60d530 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.apache.cassandra.utils.Closeable; + +class IOTaskWrapper extends IOTask +{ + static abstract class WrappableIOTask + { + abstract void runInternal(); + abstract void postRunExclusive(); + abstract void fail(Throwable t); + abstract String description(); + } + + final WrappableIOTask wrapped; + + IOTaskWrapper(AccordExecutor executor, WrappableIOTask wrap, GlobalGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + this.wrapped = wrap; + } + + @Override + String toDescription() + { + return wrapped.description(); + } + + @Override + protected void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + wrapped.runInternal(); + } + onRunComplete(); + } + + @Override + void postRunExclusive() + { + wrapped.postRunExclusive(); + } + + @Override + public String description() + { + return wrapped.description(); + } + + @Override + protected void reportFailure(Throwable fail) + { + wrapped.fail(fail); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorLoops.java b/src/java/org/apache/cassandra/service/accord/execution/Loops.java similarity index 87% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorLoops.java rename to src/java/org/apache/cassandra/service/accord/execution/Loops.java index 3eddaaf9e2..d15afc51c6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorLoops.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Loops.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -29,19 +29,19 @@ import accord.utils.Invariants; import accord.utils.TriFunction; import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; -import org.apache.cassandra.service.accord.AccordExecutor.Mode; -import org.apache.cassandra.service.accord.AccordExecutor.TaskRunner; +import org.apache.cassandra.service.accord.execution.AccordExecutor.Mode; +import org.apache.cassandra.service.accord.execution.TaskRunner.DebuggableWrapper; import org.apache.cassandra.utils.concurrent.Condition; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorThreadTag.INFINITE_LOOP; import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.NON_DAEMON; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -class AccordExecutorLoops +class Loops { - static abstract class LoopTask extends TaskRunner implements Runnable + static abstract class LoopTask extends DebuggableWrapper implements Runnable { final String id; LoopTask(String id) { this.id = id; } @@ -54,7 +54,7 @@ class AccordExecutorLoops private final AtomicInteger running = new AtomicInteger(); private final Condition terminated = Condition.newOneTimeCondition(); - public AccordExecutorLoops(Mode mode, int threads, IntFunction loopName, TriFunction loopFactory) + public Loops(Mode mode, int threads, IntFunction loopName, TriFunction loopFactory) { Invariants.require(mode == RUN_WITH_LOCK ? threads == 1 : threads >= 1); running.addAndGet(threads); diff --git a/src/java/org/apache/cassandra/service/accord/execution/Plain.java b/src/java/org/apache/cassandra/service/accord/execution/Plain.java new file mode 100644 index 0000000000..aff55550e2 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Plain.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.CancellationException; + +import accord.utils.Invariants; +import accord.utils.async.Cancellable; + +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; + +abstract class Plain extends Task implements Cancellable +{ + final AccordExecutor executor; + + Plain(AccordExecutor executor, GlobalGroup group, long position, int tranche) + { + super(group, position, tranche); + this.executor = executor; + } + + Plain(AccordExecutor executor, ExclusiveGroup group, long position, int tranche) + { + super(group, position, tranche); + this.executor = executor; + } + + Plain(AccordExecutor executor, GlobalGroup group) + { + super(group); + this.executor = executor; + } + + Plain(AccordExecutor executor, ExclusiveGroup group) + { + super(group); + this.executor = executor; + } + + abstract ExclusiveExecutor exclusiveExecutor(); + + @Override + public void cancel() + { + executor.submit(Task::cancelExclusive, CancelTask::new, this); + } + + void cancelExclusive() + { + ExclusiveExecutor exclusiveExecutor = exclusiveExecutor(); + if ((exclusiveExecutor == null ? executor.runnable : exclusiveExecutor).tryUnqueueWaiting(this)) + { + try + { + failExclusive(new CancellationException(), CANCELLED); + } + catch (Throwable t) + { + executor.agent.onException(t); + } + finally + { + executor.cleanupTaskExclusive(this, false); + } + } + } + + @Override + final void submitExclusive() + { + submitExclusive(exclusiveExecutor(), null); + } + + final Cancellable submitExclusive(ExclusiveExecutor exclusiveExecutor, Task parent) + { + Invariants.require(executor.isOwningThread()); + setStateExclusive(WAITING_TO_RUN); + + if (parent == null) executor.registerExclusive(this); + else executor.registerConsequenceExclusive(parent, this); + onLoaded(); + + if (exclusiveExecutor == null) executor.runnable.enqueue(this, true); + else exclusiveExecutor.enqueue(this, true); + return this; + } + + @Override + protected boolean isNewWork() + { + return true; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java new file mode 100644 index 0000000000..c729f58e84 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.utils.async.AsyncCallbacks; + +import org.apache.cassandra.utils.Closeable; + +class PlainChain extends Plain +{ + final AsyncCallbacks.RunOrFail runOrFail; + final @Nullable ExclusiveExecutor exclusiveExecutor; + + PlainChain(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group) + { + super(executor, group); + this.runOrFail = runOrFail; + this.exclusiveExecutor = exclusiveExecutor; + } + + PlainChain(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + this.runOrFail = runOrFail; + this.exclusiveExecutor = exclusiveExecutor; + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return exclusiveExecutor; + } + + @Override + String toDescription() + { + return runOrFail.toString(); + } + + @Override + protected void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + runOrFail.run(); + } + catch (Throwable t) + { + // shouldn't throw exceptions + executor.agent.onException(t); + } + onRunComplete(); + } + + @Override + protected void reportFailure(Throwable fail) + { + try + { + runOrFail.fail(fail); + } + catch (Throwable t) + { + fail.addSuppressed(t); + executor.agent.onException(fail); + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java new file mode 100644 index 0000000000..6a2ab7950b --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.utils.Invariants; +import accord.utils.async.AsyncCallbacks; + +import org.apache.cassandra.concurrent.DebuggableTask; + +class PlainChainDebuggable extends PlainChain implements DebuggableTask +{ + final Object describe; + + PlainChainDebuggable(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, @Nullable ExclusiveExecutor exclusiveExecutor, Object describe) + { + super(executor, runOrFail, exclusiveExecutor, ExclusiveGroup.OTHER); + this.describe = Invariants.nonNull(describe); + } + + PlainChainDebuggable(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, @Nullable ExclusiveExecutor exclusiveExecutor, long position, int tranche, Object describe) + { + super(executor, runOrFail, exclusiveExecutor, ExclusiveGroup.OTHER, position, tranche); + this.describe = Invariants.nonNull(describe); + } + + @Override + public long creationTimeNanos() + { + return createdAt; + } + + @Override + public long startTimeNanos() + { + return runningAt; + } + + @Override + public String description() + { + return describe.toString(); + } + + @Override + public DebuggableTask debuggable() + { + return this; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java new file mode 100644 index 0000000000..594ff107bf --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.utils.async.Cancellable; + +import org.apache.cassandra.utils.Closeable; +import org.apache.cassandra.utils.concurrent.AsyncPromise; + +class PlainRunnable extends Plain implements Cancellable +{ + final @Nullable AsyncPromise result; + final Runnable run; + final @Nullable ExclusiveExecutor exclusiveExecutor; + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, GlobalGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + this.result = result; + this.run = run; + this.exclusiveExecutor = null; + } + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, GlobalGroup group) + { + super(executor, group); + this.result = result; + this.run = run; + this.exclusiveExecutor = null; + } + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + this.result = result; + this.run = run; + this.exclusiveExecutor = exclusiveExecutor; + } + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group) + { + super(executor, group); + this.result = result; + this.run = run; + this.exclusiveExecutor = exclusiveExecutor; + } + + @Override + String toDescription() + { + // TODO (expected): ensure this is usefully descriptive, or accept a separate description + return run.toString(); + } + + @Override + protected void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + run.run(); + } + if (result != null) + result.trySuccess(null); + onRunComplete(); + } + + @Override + protected void reportFailure(Throwable t) + { + if (result != null) + result.tryFailure(t); + executor.agent.onException(t); + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return exclusiveExecutor; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java similarity index 84% rename from src/java/org/apache/cassandra/service/accord/AccordTask.java rename to src/java/org/apache/cassandra/service/accord/execution/SafeTask.java index 30e6998396..77c61ecf40 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -69,15 +69,16 @@ import accord.utils.async.Cancellable; import org.apache.cassandra.concurrent.DebuggableTask; import org.apache.cassandra.metrics.LogLinearDecayingHistograms; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; -import org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus; -import org.apache.cassandra.service.accord.AccordCacheEntry.Loading; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStore.Caches; -import org.apache.cassandra.service.accord.AccordExecutor.Task; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; +import org.apache.cassandra.service.accord.RangeIndex; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Loading; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; import org.apache.cassandra.service.accord.serializers.CommandSerializers; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.Closeable; @@ -90,50 +91,50 @@ import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.RECOVERY; import static accord.local.LoadKeysFor.WRITE; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.HOLD_QUEUE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; -import static org.apache.cassandra.service.accord.AccordExecutor.NONSYNC_BLOCKED_LIMIT; -import static org.apache.cassandra.service.accord.AccordExecutor.NONSYNC_ENABLED; -import static org.apache.cassandra.service.accord.AccordExecutor.NONSYNC_MIN_BATCH_SIZE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.RunState.PERSISTING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.CANCELLED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.INCOMPLETE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_OPTIONAL; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_REQUIRED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_OPTIONAL; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_REQUIRED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_OR_RUNNING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.AccordSafeState.global; -import static org.apache.cassandra.service.accord.AccordSafeState.postExecute; -import static org.apache.cassandra.service.accord.AccordSafeState.preExecute; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask.SANITY_CHECK; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.HOLD_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.NONSYNC_BLOCKED_LIMIT; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.NONSYNC_ENABLED; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.NONSYNC_MIN_BATCH_SIZE; +import static org.apache.cassandra.service.accord.execution.SaferState.global; +import static org.apache.cassandra.service.accord.execution.SaferState.postExecute; +import static org.apache.cassandra.service.accord.execution.SaferState.preExecute; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; +import static org.apache.cassandra.service.accord.execution.Task.RunState.PERSISTING; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.INCOMPLETE; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_OR_RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; -public final class AccordTask extends Task implements Cancellable, DebuggableTask +public final class SafeTask extends Task implements Cancellable, DebuggableTask { - private static final Logger logger = LoggerFactory.getLogger(AccordTask.class); + private static final Logger logger = LoggerFactory.getLogger(SafeTask.class); private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES); - public static AccordTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) + public static SafeTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) { - return new AccordTask<>((AccordCommandStore) commandStore, context, safeStore -> { + return new SafeTask<>((AccordCommandStore) commandStore, context, safeStore -> { consumer.accept(safeStore); return null; }); } - public static AccordTask create(CommandStore commandStore, ExecutionContext context, Function function) + public static SafeTask create(CommandStore commandStore, ExecutionContext context, Function function) { - return new AccordTask<>((AccordCommandStore) commandStore, context, function); + return new SafeTask<>((AccordCommandStore) commandStore, context, function); } final class NonSyncState extends ExecutionContext.Wrapped implements ExecutionContext @@ -221,7 +222,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable if ((blocking == null || !populate(keys, blocking)) && notBlocking != null) populate(keys, notBlocking); - keys.forEach(key -> preExecute(refs.get(key), AccordTask.this, RELEASE_QUEUE)); + keys.forEach(key -> preExecute(refs.get(key), SafeTask.this, RELEASE_QUEUE)); keys.sort(RoutingKey::compareTo); active = RoutingKeys.of(keys); } @@ -258,7 +259,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable if (active != null) { for (RoutingKey key : active) - postExecute(refs.remove(key), AccordTask.this); + postExecute(refs.remove(key), SafeTask.this); active = null; } ready = readyCount(); @@ -301,7 +302,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable private BiConsumer callback; - public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext, Function function) + public SafeTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext, Function function) { super(executionContext, commandStore.executor().uniqueCreatedAt); this.commandStore = commandStore; @@ -332,7 +333,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable return toBriefString() + ": " + (queued() == null ? "unqueued" : describeState()) + ", primaryTxnId: " + executionContext.primaryTxnId() - + ", state: " + summarise(refs, AccordSafeState::global); + + ", state: " + summarise(refs, SaferState::global); } @@ -383,8 +384,8 @@ public final class AccordTask extends Task implements Cancellable, Debuggable protected Cancellable start(BiConsumer callback) { preSetup(callback); - executor().submit(AccordTask.this); - return AccordTask.this; + executor().submit(SafeTask.this); + return SafeTask.this; } }; } @@ -393,11 +394,18 @@ public final class AccordTask extends Task implements Cancellable, Debuggable { Invariants.require(this.callback == null); this.callback = callback; - commandStore.tryPreSetup(this); + TaskRunner self = TaskRunner.get(); + Task task = self.accordActiveSelfTask(); + if (task instanceof SafeTask) + { + SafeTask parent = (SafeTask) task; + if (parent.commandStore == commandStore) + preSetup(parent); + } } // to be invoked only by the CommandStore owning thread, to take references to objects already in use by the current execution - public void preSetup(AccordTask parent) + public void preSetup(SafeTask parent) { this.position = parent.position; setInheritedWithTranche(parent.tranche()); @@ -441,7 +449,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable { // TODO (desired): custom map we can more cheaply fork/copy parent.refs.forEach((key, val) -> { - if (val instanceof AccordSafeCommandsForKey) + if (val instanceof SaferCommandsForKey) preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); }); } @@ -457,7 +465,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable case Range: AbstractRanges ranges = (AbstractRanges) keysOrRanges; parent.refs.forEach((key, val) -> { - if (val instanceof AccordSafeCommandsForKey && ranges.contains((RoutingKey) key)) + if (val instanceof SaferCommandsForKey && ranges.contains((RoutingKey) key)) preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); }); break; @@ -470,14 +478,10 @@ public final class AccordTask extends Task implements Cancellable, Debuggable } @Override - void submitExclusive(AccordExecutor owner) - { - owner.registerExclusive(this); - setupInternal(commandStore.cachesExclusive()); - } - - private void setupInternal(Caches caches) + void submitExclusive() { + Caches caches = commandStore.cachesExclusive(); + executor().registerExclusive(this); boolean hasPreSetup = hasInherited(); LoadKeys loadKeys = loadKeys(executionContext); if (loadKeys != NONE) @@ -496,8 +500,8 @@ public final class AccordTask extends Task implements Cancellable, Debuggable case Range: if (!hasInheritedRangeScan()) setupRangeLoadsExclusive(caches); else refs.forEach((k, v) -> { - if (v instanceof AccordSafeCommandsForKey) - completePresetupExclusive((AccordSafeCommandsForKey)v); + if (v instanceof SaferCommandsForKey) + completePresetupExclusive((SaferCommandsForKey)v); }); break; case Key: @@ -551,7 +555,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable } // expects mutual exclusivity only on the command store - private & AccordSafeState> void preSetup(K k, Map> parentMap, AccordCache.Type.Instance cache) + private & SaferState> void preSetup(K k, Map> parentMap, AccordCache.Type.Instance cache) { S ref = (S) parentMap.get(k); if (ref == null) @@ -566,7 +570,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable keys++; } - private & AccordSafeState> boolean completePresetupExclusive(K k) + private & SaferState> boolean completePresetupExclusive(K k) { S preacquired = (S) refs.get(k); if (preacquired != null) @@ -577,7 +581,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable return false; } - private & AccordSafeState> void completePresetupExclusive(S preacquired) + private & SaferState> void completePresetupExclusive(S preacquired) { AccordCacheEntry entry = preacquired.global(); if (entry.isLoaded()) completeSetupOfLoaded(entry); @@ -585,7 +589,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable } // expects to hold lock - private & AccordSafeState> void setupExclusive(K k, AccordCache.Type.Instance cache, int waitForIncrement) + private & SaferState> void setupExclusive(K k, AccordCache.Type.Instance cache, int waitForIncrement) { S safeRef = cache.acquire(k); AccordCacheEntry entry = safeRef.global(); @@ -706,7 +710,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable Invariants.require(waitingForState == 0); onLoaded(); executor().runnable.incrementArrivals(this); - commandStore.exclusiveExecutor.incrementArrivals(this); + commandStore.exclusiveExecutor().incrementArrivals(this); this.refs.forEach((key, safeState) -> { AccordCacheEntry entry = global(safeState); @@ -838,7 +842,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable else { // TODO (expected): this is potentially costly; maybe we don't want to swap these in and out (but harder to maintain invariants) - unqueue(commandStore.exclusiveExecutor); + unqueue(commandStore.exclusiveExecutor()); setStateExclusive(WAITING_ON_REQUIRED); executor().waitingOnCacheQueues.enqueue(this); } @@ -875,7 +879,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable nonSync.onNotHead(entry); if (is(WAITING_TO_RUN) && !nonSync.isWaitReady()) { - unqueue(commandStore.exclusiveExecutor); + unqueue(commandStore.exclusiveExecutor()); setStateExclusive(WAITING_ON_OPTIONAL); executor().waitingOnCacheQueues.enqueue(this); } @@ -904,7 +908,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable void waitToRunExclusive() { setStateExclusive(WAITING_TO_RUN); - commandStore.exclusiveExecutor.enqueue(this, false); + commandStore.exclusiveExecutor().enqueue(this, false); } public ExecutionContext executionContext() @@ -979,13 +983,13 @@ public final class AccordTask extends Task implements Cancellable, Debuggable public void run() { onRunning(); - AccordSafeCommandStore safeStore = null; + SaferCommandStore safeStore = null; try (Closeable close = resources.get()) { if (Tracing.isTracing()) Tracing.trace(executionContext.describe()); - commandStore.begin(safeStore = new AccordSafeCommandStore(this, isSync() ? executionContext : nonSync)); + commandStore.begin(safeStore = new SaferCommandStore(this, isSync() ? executionContext : nonSync)); R result = function.apply(safeStore); @@ -995,9 +999,9 @@ public final class AccordTask extends Task implements Cancellable, Debuggable List changes = new ArrayList<>(); // TODO (expected): save any TxnId we add so that we don't need to iterate all of refs refs.forEach((key, value) -> { - if (value instanceof AccordSafeCommand) + if (value instanceof SaferCommand) { - AccordSafeCommand safeCommand = (AccordSafeCommand) value; + SaferCommand safeCommand = (SaferCommand) value; Journal.CommandUpdate diff = safeCommand.update(); if (diff != null) { @@ -1060,7 +1064,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable } } - private void maybeSanityCheck(AccordSafeCommand safeCommand) + private void maybeSanityCheck(SaferCommand safeCommand) { if (SANITY_CHECK) { @@ -1112,27 +1116,52 @@ public final class AccordTask extends Task implements Cancellable, Debuggable public void cancel() { if (!state().hasStarted()) - executor().cancel(this); + executor().submit(Task::cancelExclusive, CancelTask::new, this); } void cancelExclusive() { - logger.info("Cancelling {}", executionContext); - setStateExclusive(CANCELLED); - if (rangeScanner != null) - rangeScanner.cancelled = true; - if (callback != null) + State state = state(); + switch (state) { - if (executor().isInLoop()) callback.accept(null, new CancellationException()); - else executor().submit(() -> callback.accept(null, new CancellationException())); + default: throw new UnhandledEnum(state); + case SCANNING_RANGES: + case LOADING_REQUIRED: + case LOADING_OPTIONAL: + case WAITING_ON_REQUIRED: + case WAITING_ON_OPTIONAL: + case WAITING_TO_RUN: + if (!hasIncrementalStarted()) + { + unqueueIfQueued(); + setStateExclusive(CANCELLED); + if (rangeScanner != null) + rangeScanner.cancelled = true; + try + { + logger.info("Cancelling {}", executionContext); + if (callback != null) + { + if (executor().isInLoop()) callback.accept(null, new CancellationException()); + else executor().submit(() -> callback.accept(null, new CancellationException())); + } + } + finally + { + executor().cleanupTaskExclusive(this, false); + } + break; + } + case UNINITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase + case RUNNING: + case INCOMPLETE: + case EXECUTED: + case CANCELLED: + case FAILED_TO_LOAD: + // cannot safely cancel } } - void cancelExclusive(AccordExecutor owner) - { - owner.cancelExclusive(this); - } - private void finish(R result, Throwable failure) { setRunState(failure == null ? RunState.SUCCESS : RunState.FAILED); @@ -1155,7 +1184,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable } refs.forEach((key, safeState) -> { - AccordSafeState.postExecute(safeState, this); + SaferState.postExecute(safeState, this); }); if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedState(); } @@ -1178,7 +1207,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable refs.forEach((k, safeState) -> { if (!safeState.isReleased()) { - try { AccordSafeState.postExecute(safeState, this); } + try { SaferState.postExecute(safeState, this); } catch (Throwable t) { suppressedBy.addSuppressed(t); } } }); @@ -1193,16 +1222,16 @@ public final class AccordTask extends Task implements Cancellable, Debuggable public void onUpdate(AccordCacheEntry state) { if (ranges.contains(state.key())) - reference((AccordCacheEntry) state); + reference((AccordCacheEntry) state); } } final Set intersectingKeys = new ObjectHashSet<>(); final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); - final AccordCache.Type.Instance commandsForKeyCache; + final AccordCache.Type.Instance commandsForKeyCache; KeyWatcher keyWatcher = new KeyWatcher(); - public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) + public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) { this.commandsForKeyCache = commandsForKeyCache; } @@ -1223,7 +1252,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable super.runInternal(); } - private void reference(AccordCacheEntry entry) + private void reference(AccordCacheEntry entry) { switch (entry.status()) { @@ -1306,7 +1335,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable } } - public class RangeTxnScanner extends AccordExecutor.AbstractIOTask + public class RangeTxnScanner extends IOTaskWrapper.WrappableIOTask { final Map summaries = new HashMap<>(); final Map guardedSummaries = Collections.synchronizedMap(summaries); @@ -1330,7 +1359,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable @Override protected void postRunExclusive() { - executor().onScannedRangesExclusive(AccordTask.this, failure); + executor().onScannedRangesExclusive(SafeTask.this, failure); } @Override @@ -1343,9 +1372,9 @@ public final class AccordTask extends Task implements Cancellable, Debuggable { Caches caches = commandStore.cachesExclusive(); setStateExclusive(SCANNING_RANGES); - executor().scanningRanges.enqueue(AccordTask.this); + executor().scanningRanges.enqueue(SafeTask.this); startInternal(caches); - executor().submitPlainExclusive(AccordTask.this, GlobalGroup.RANGE_SCAN, this); + executor().submitExclusive(SafeTask.this, GlobalGroup.RANGE_SCAN, this); } void startInternal(Caches caches) @@ -1356,7 +1385,7 @@ public final class AccordTask extends Task implements Cancellable, Debuggable public void scannedExclusive() { - Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", AccordTask.this, AccordTask::toDescription); + Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", SafeTask.this, SafeTask::toDescription); scanned = true; scannedInternal(); unqueue(executor().scanningRanges); diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommand.java similarity index 77% rename from src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java rename to src/java/org/apache/cassandra/service/accord/execution/SaferCommand.java index 1263a3e1ec..3d066aa425 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommand.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.Objects; @@ -25,14 +25,14 @@ import accord.local.Command; import accord.local.SafeCommand; import accord.primitives.TxnId; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; -public class AccordSafeCommand extends SafeCommand implements AccordSafeState +public class SaferCommand extends SafeCommand implements SaferState { - private final AccordCacheEntry global; + private final AccordCacheEntry global; private Command original; - public AccordSafeCommand(AccordCacheEntry global) + public SaferCommand(AccordCacheEntry global) { super(global.key()); this.global = global; @@ -43,7 +43,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState global() + public AccordCacheEntry global() { return global; } @Override - public void postExecute(AccordTask owner) + public void postExecute(SafeTask owner) { global.releaseExclusive(this, owner); } @@ -80,7 +80,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState owner, LockMode lockMode) + public void preExecute(SafeTask owner, LockMode lockMode) { requireUninitialised(); original = global.lockExclusive(owner, lockMode); diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java similarity index 85% rename from src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java rename to src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java index 6b02b5ab9d..e1b73cf15e 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.function.Predicate; @@ -46,19 +46,20 @@ import accord.utils.Invariants; import accord.utils.UnhandledEnum; import org.apache.cassandra.metrics.LogLinearDecayingHistograms; +import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches; import org.apache.cassandra.service.accord.AccordCommandStore.SafeRedundantBefore; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; import org.apache.cassandra.service.paxos.PaxosState; import static accord.utils.Invariants.illegalState; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.UNQUEUED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.UNQUEUED; -public final class AccordSafeCommandStore extends AbstractSafeCommandStore +public final class SaferCommandStore extends AbstractSafeCommandStore { - final AccordTask task; + final SafeTask task; - AccordSafeCommandStore(AccordTask task, ExecutionContext context) + SaferCommandStore(SafeTask task, ExecutionContext context) { super(context); this.task = task; @@ -73,9 +74,9 @@ public final class AccordSafeCommandStore extends AbstractSafeCommandStore { - AccordCommandStore.safeRedundantBeforeUpdater.accumulateAndGet(commandStore(), update, SafeRedundantBefore::max); - }; + Runnable reportRedundantBefore = SafeRedundantBefore.updater(commandStore(), updates.newRedundantBefore); Runnable prevOnDone = onDone; onDone = prevOnDone == null ? reportRedundantBefore : () -> { try { reportRedundantBefore.run(); } @@ -136,7 +133,7 @@ public final class AccordSafeCommandStore extends AbstractSafeCommandStore skip = context.keys().without(keysOrRanges); for (SafeState safeState : task.refs.values()) { - if (!(safeState instanceof AccordSafeCommandsForKey)) + if (!(safeState instanceof SaferCommandsForKey)) continue; SafeCommandsForKey safeCfk = (SafeCommandsForKey) safeState; diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandsForKey.java similarity index 72% rename from src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java rename to src/java/org/apache/cassandra/service/accord/execution/SaferCommandsForKey.java index fc38874caa..ea188159be 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandsForKey.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.Objects; @@ -24,25 +24,24 @@ import accord.api.RoutingKey; import accord.local.cfk.CommandsForKey; import accord.local.cfk.NotifySink; import accord.local.cfk.SafeCommandsForKey; -import accord.utils.Invariants; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; -public class AccordSafeCommandsForKey extends SafeCommandsForKey implements AccordSafeState +public class SaferCommandsForKey extends SafeCommandsForKey implements SaferState { - public static class CommandsForKeyCacheEntry extends AccordCacheEntry + public static class CommandsForKeyCacheEntry extends AccordCacheEntry { private NotifySink overrideSink; - CommandsForKeyCacheEntry(RoutingKey key, AccordCache.Type.Instance owner) + CommandsForKeyCacheEntry(RoutingKey key, AccordCache.Type.Instance owner) { super(key, owner); } } - private final AccordCacheEntry global; + private final AccordCacheEntry global; - public AccordSafeCommandsForKey(AccordCacheEntry global) + public SaferCommandsForKey(AccordCacheEntry global) { super(global.key()); this.global = global; @@ -53,7 +52,7 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - AccordSafeCommandsForKey that = (AccordSafeCommandsForKey) o; + SaferCommandsForKey that = (SaferCommandsForKey) o; return Objects.equals(current, that.current); } @@ -73,13 +72,13 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco '}'; } - public final AccordCacheEntry global() + public final AccordCacheEntry global() { return global; } @Override - public void postExecute(AccordTask owner) + public void postExecute(SafeTask owner) { global.releaseExclusive(this, owner); } @@ -96,7 +95,7 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco return ((CommandsForKeyCacheEntry)global).overrideSink; } - public void preExecute(AccordTask owner, LockMode lockMode) + public void preExecute(SafeTask owner, LockMode lockMode) { requireUninitialised(); current = global.lockExclusive(owner, lockMode); diff --git a/src/java/org/apache/cassandra/service/accord/execution/SaferState.java b/src/java/org/apache/cassandra/service/accord/execution/SaferState.java new file mode 100644 index 0000000000..6708bdaa05 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferState.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.accord.execution; + +import accord.local.SafeState; + +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; + +public interface SaferState & SaferState> +{ + AccordCacheEntry global(); + void postExecute(SafeTask owner); + void preExecute(SafeTask owner, LockMode lockMode); + + static AccordCacheEntry global(SafeState safeState) + { + return safeState.getClass() == SaferCommand.class ? ((SaferCommand) safeState).global() + : ((SaferCommandsForKey) safeState).global(); + } + + static void postExecute(SafeState safeState, SafeTask owner) + { + if (safeState.getClass() == SaferCommand.class) ((SaferCommand) safeState).postExecute(owner); + else ((SaferCommandsForKey) safeState).postExecute(owner); + } + + static void preExecute(SafeState safeState, SafeTask owner, LockMode lockMode) + { + if (safeState.getClass() == SaferCommand.class) ((SaferCommand) safeState).preExecute(owner, lockMode); + else ((SaferCommandsForKey) safeState).preExecute(owner, lockMode); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Task.java b/src/java/org/apache/cassandra/service/accord/execution/Task.java new file mode 100644 index 0000000000..0d2395f8de --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Task.java @@ -0,0 +1,724 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.atomic.AtomicLong; + +import javax.annotation.Nullable; + +import accord.local.ExecutionContext; +import accord.messages.Accept; +import accord.messages.Commit; +import accord.messages.MessageType; +import accord.messages.Request; +import accord.primitives.Ballot; +import accord.primitives.SaveStatus; +import accord.primitives.TxnId; +import accord.utils.IntrusiveHeapNode; +import accord.utils.Invariants; +import accord.utils.TinyEnumSet; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.DebuggableTask; +import org.apache.cassandra.concurrent.ExecutorLocals; +import org.apache.cassandra.service.accord.debug.DebugExecution; +import org.apache.cassandra.utils.WithResources; + +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY; +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY_ATOMIC; +import static accord.primitives.Routable.Domain.Range; +import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.ORIG_HLC_FIFO; +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.APPLY; +import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.COMMIT; +import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.STABLE; +import static org.apache.cassandra.service.accord.execution.Task.State.EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING_OR_EXECUTED; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +public abstract class Task extends IntrusiveHeapNode implements Cancellable +{ + private static final int WAITING_ON_OPTIONAL_BIT = 1 << 5; + private static final int WAITING_TO_RUN_BIT = 1 << 6; + private static final int INCOMPLETE_BIT = 1 << 9; + + enum State + { + UNINITIALIZED(), + SCANNING_RANGES(UNINITIALIZED), + LOADING_REQUIRED(UNINITIALIZED, SCANNING_RANGES), + LOADING_OPTIONAL(UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED), + WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), + WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), + WAITING_TO_RUN(INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), + RUNNING(WAITING_TO_RUN), + EXECUTED(RUNNING), + INCOMPLETE(RUNNING), + FAILED_TO_LOAD(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), + FAILED_OTHER(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), + CANCELLED(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, RUNNING), + ; + + private final int permittedFrom; + public static final int WAITING = TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN); + public static final int WAITING_OR_RUNNING = WAITING | TinyEnumSet.encode(RUNNING); + public static final int RUNNING_OR_EXECUTED = WAITING | TinyEnumSet.encode(RUNNING, EXECUTED); + static final State[] VALUES = values(); + + static + { + // hack to allow us to create loops in our enum transition declarations + Invariants.require(INCOMPLETE_BIT == 1 << INCOMPLETE.ordinal()); + } + + State() + { + this.permittedFrom = 0; + } + + State(State... permittedFroms) + { + this(0, permittedFroms); + } + + State(int additional, State... permittedFroms) + { + int permittedFrom = additional; + for (State state : permittedFroms) + permittedFrom |= 1 << state.ordinal(); + this.permittedFrom = permittedFrom; + } + + boolean isPermittedFrom(int prevOrdinal) + { + return (permittedFrom & (1 << prevOrdinal)) != 0; + } + + boolean isExecuted() + { + return this.compareTo(EXECUTED) >= 0; + } + + boolean hasStarted() + { + return this.compareTo(RUNNING) >= 0; + } + + static State forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + + enum RunState + { + NONE, PERSISTING, SUCCESS, FAILED; + + private static final RunState[] VALUES = values(); + + static RunState forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + + enum GlobalGroup + { + COMMAND_STORE, + LOAD, + SAVE, + OTHER, + RANGE_LOAD, + RANGE_SCAN, + } + + enum ExclusiveGroup + { + APPLY, + STABLE, + COMMIT, + ACCEPT, + OTHER, + RECOVER, + PREACCEPT, + RANGE, + } + + public enum GroupKind + { + EXCLUSIVE(ExclusiveGroup.values().length, EXCLUSIVE_GROUP_SHIFT), + GLOBAL(GlobalGroup.values().length, GLOBAL_GROUP_SHIFT), + NONE(0, 0); + + final int count; + final byte shift; + + GroupKind(int count, int shift) + { + this.count = count; + this.shift = (byte) shift; + } + } + + private static final int STATE_MASK = 0xf; + static final int GROUP_MASK = 0x7; + private static final int EXCLUSIVE_GROUP_SHIFT = 4; + private static final int GLOBAL_GROUP_SHIFT = 7; + + private static final int NONSYNC_BIT = 1 << 10; + private static final int CACHE_QUEUED_BIT = 1 << 11; + + private static final int INCREMENTAL_MASK = 0x3 << 12; + private static final int INCREMENTAL = 0x1 << 12; + private static final int INCREMENTAL_STARTED = 0x2 << 12; + private static final int INCREMENTAL_FINISHING = 0x3 << 12; + + private static final int SEQUENCED_SHIFT = 14; + private static final int SEQUENCED_MASK = 0x3 << SEQUENCED_SHIFT; + private static final int SEQUENCED_PRIORITY = 0x1 << SEQUENCED_SHIFT; + private static final int SEQUENCED_ATOMIC = 0x2 << SEQUENCED_SHIFT; + private static final int SEQUENCED_ATOMIC_AND_QUEUED = 0x3 << SEQUENCED_SHIFT; + + // spare two bits + + private static final int HAS_TRANCHE_BIT = 1 << 18; + private static final int HAS_INHERITED_BIT = 1 << 19; + private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 20; + + private static final int TRANCHE_SHIFT = 22; + static final int MAX_TRANCHE = 0x3ff; + + static + { + Invariants.require(SEQUENCED_PRIORITY == BY_PRIORITY.ordinal() << SEQUENCED_SHIFT); + Invariants.require(SEQUENCED_ATOMIC == BY_PRIORITY_ATOMIC.ordinal() << SEQUENCED_SHIFT); + Invariants.require(ExecutionContext.ExecutionSequence.values().length <= 3); + } + + public final WithResources resources; + Task next; + + long position; + int info; + + // TODO (expected): do we need this? we should be able to determine the queue from state() if needed for e.g. cancellation + private TaskQueue queued; + + public final long createdAt; + // TODO (expected): expose via executors vtable + // TODO (expected): use just one long and some flag bits to indicate which point it represents, and report incrementally + public long loadedAt, runningAt, completeAt; + private byte runState; + + Task(GlobalGroup group) + { + resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(group, ExclusiveGroup.OTHER); + createdAt = nanoTime(); + } + + Task(ExclusiveGroup group) + { + resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(GlobalGroup.OTHER, group); + createdAt = nanoTime(); + } + + Task(GlobalGroup group, long position, int tranche) + { + this(group); + this.position = position; + setInheritedWithTranche(tranche); + } + + Task(ExclusiveGroup group, long position, int tranche) + { + this(group); + this.position = position; + setInheritedWithTranche(tranche); + } + + protected Task(ExecutionContext context, AtomicLong lastCreatedAt) + { + resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next < prev ? prev + 1 : next); + ExclusiveGroup group = ExclusiveGroup.OTHER; + TxnId txnId = context.primaryTxnId(); + if (txnId != null) + { + if (txnId.is(Range)) group = ExclusiveGroup.RANGE; + else + { + switch (AccordExecutor.PRIORITY_MODEL) + { + case HLC_FIFO: + case ORIG_HLC_FIFO: + { + // TODO (expected): port to ExecutionKind; also we aren't consistent about using Ballot + if (context instanceof Request) + { + MessageType type = ((Request) context).type(); + if (type instanceof MessageType.StandardMessage) + { + switch ((MessageType.StandardMessage) type) + { + case APPLY_REQ: + { + group = APPLY; + break; + } + case READ_EPHEMERAL_REQ: + case READ_REQ: + case STABLE_THEN_READ_REQ: + { + group = STABLE; + break; + } + case COMMIT_REQ: + { + Commit commit = (Commit) context; + if (AccordExecutor.PRIORITY_MODEL == ORIG_HLC_FIFO && !commit.ballot.equals(Ballot.ZERO)) + txnId = null; + if (commit.kind.saveStatus == SaveStatus.Stable) group = STABLE; + else group = COMMIT; + break; + } + case ACCEPT_REQ: + { + Accept accept = (Accept) context; + if (AccordExecutor.PRIORITY_MODEL == ORIG_HLC_FIFO && !accept.ballot.equals(Ballot.ZERO)) + txnId = null; + group = ExclusiveGroup.ACCEPT; + break; + } + case GET_EPHEMERAL_READ_DEPS_REQ: + case PRE_ACCEPT_REQ: + { + group = ExclusiveGroup.PREACCEPT; + break; + } + default: + { + txnId = null; + } + } + } + } + else + { + txnId = null; + } + break; + } + case FIFO: + { + txnId = null; + break; + } + } + } + } + + this.info = init(GlobalGroup.OTHER, group); + if (txnId != null) + this.position = txnId.hlc(); + } + + public final Task unwrap() + { + if (this instanceof ExclusiveExecutor.ExclusiveExecutorTask) + return ((ExclusiveExecutor.ExclusiveExecutorTask) this).queue.task; + return this; + } + + static Task unwrap(Task task) + { + return task == null ? null : task.unwrap(); + } + + public DebuggableTask debuggable() + { + return null; + } + + abstract String toDescription(); + + abstract void submitExclusive(); + + /** + * Prepare to run while holding the state cache lock + */ + void preRunExclusive() + { + setStateExclusive(RUNNING); + } + + /** + * Run the command; the state cache lock may or may not be held depending on the executor implementation + */ + abstract void run(); + + /** + * Fail the command; the state cache lock may or may not be held depending on the executor implementation + */ + abstract void reportFailure(Throwable fail); + + final void failExclusive(Throwable fail, State newState) + { + try + { + setStateExclusive(newState); + } + finally + { + reportFailure(fail); + } + } + + final void failExecution(Throwable fail) + { + Invariants.require(is(RUNNING)); + try + { + setRunState(RunState.FAILED); + } + finally + { + reportFailure(fail); + } + } + + abstract boolean isNewWork(); + + /** + * Cleanup the command while holding the state cache lock + */ + void cleanupExclusive(AccordExecutor executor, boolean executed) + { + if (executed) setStateExclusive(EXECUTED); + else Invariants.require(state().isExecuted()); + executor.unregisterExclusive(this); + completeAt = nanoTime(); + if (runningAt != 0) + { + if (loadedAt == 0) + loadedAt = runningAt; + executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); + executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); + executor.elapsedRunning.increment(completeAt - runningAt, completeAt); + executor.elapsed.increment(completeAt - createdAt, completeAt); + } + if (DEBUG_EXECUTION) DebugExecution.DebugTask.get(this).onCompleted(executor.debug); + } + + void cancelExclusive() + { + } + + @Nullable + final TaskQueue queued() + { + return queued; + } + + final void unqueueIfQueued() + { + if (queued != null) + { + queued.unqueue(this); + queued = null; + } + } + + final void unqueue(TaskQueue expected) + { + Invariants.require(queued == expected, "%s != %s", queued, expected); + queued.unqueue(this); + queued = null; + } + + final void unsetQueue(TaskQueue expected) + { + Invariants.require(queued == expected, "%s != %s", queued, expected); + queued = null; + } + + final void setQueue(TaskQueue queue) + { + Invariants.require(queued == null); + Invariants.require(isCompatible(queue)); + queued = queue; + } + + final void onRunning() + { + runningAt = nanoTime(); + if (DEBUG_EXECUTION) ((DebugExecution.DebugTask) resources).onRunning(); + } + + final void onRunComplete() + { + if (DEBUG_EXECUTION) ((DebugExecution.DebugTask) resources).onRunComplete(); + } + + final void onLoaded() + { + loadedAt = nanoTime(); + } + + final State state() + { + return State.forOrdinal(stateOrdinal()); + } + + final RunState runState() + { + return RunState.forOrdinal(runState); + } + + final Enum describeState() + { + State state = state(); + if (state == RUNNING || state == EXECUTED) + { + RunState runState = runState(); + if (runState == RunState.NONE) + return state; + return runState; + } + return State.forOrdinal(stateOrdinal()); + } + + private int stateOrdinal() + { + return info & STATE_MASK; + } + + final boolean is(State state) + { + return stateOrdinal() == state.ordinal(); + } + + final boolean isState(int stateBitSet) + { + return TinyEnumSet.contains(stateBitSet, stateOrdinal()); + } + + final boolean is(GlobalGroup group) + { + return globalGroupOrdinal() == group.ordinal(); + } + + final boolean is(ExclusiveGroup group) + { + return exclusiveGroupOrdinal() == group.ordinal(); + } + + final void override(GlobalGroup group) + { + info = (info & ~(GROUP_MASK << GLOBAL_GROUP_SHIFT)) | (group.ordinal() << GLOBAL_GROUP_SHIFT); + } + + final int compareTo(State state) + { + return stateOrdinal() - state.ordinal(); + } + + final void setStateExclusive(State state) + { + Invariants.require(state.isPermittedFrom(stateOrdinal()), "%s forbidden from %s", state, this, Task::reportBadStateTransition); + unsafeSetStateExclusive(state); + } + + final void setRunState(RunState state) + { + Invariants.require(isState(RUNNING_OR_EXECUTED)); + runState = (byte) state.ordinal(); + } + + private static String reportBadStateTransition(Task task) + { + return task.state() + " for " + task.toDescription(); + } + + final void unsafeSetStateExclusive(State state) + { + info = (info & ~STATE_MASK) | state.ordinal(); + } + + final int globalGroupOrdinal() + { + return (info >>> GLOBAL_GROUP_SHIFT) & GROUP_MASK; + } + + final int exclusiveGroupOrdinal() + { + return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; + } + + private boolean isCompatible(TaskQueue queue) + { + int self = stateOrdinal(); + return TinyEnumSet.contains(queue.states, self); + } + + final boolean isSync() + { + return 0 == (info & NONSYNC_BIT); + } + + final boolean isNonSync() + { + return !isSync(); + } + + final void setNonSyncExclusive() + { + info |= NONSYNC_BIT; + } + + final boolean isIncremental() + { + return 0 != (info & INCREMENTAL_MASK); + } + + final void setIncrementalExclusive() + { + info |= INCREMENTAL | NONSYNC_BIT; + } + + final boolean hasIncrementalStarted() + { + return (info & INCREMENTAL_MASK) >= INCREMENTAL_STARTED; + } + + final void setIncrementalStartedExclusive() + { + Invariants.require(isIncremental()); + if (!isIncrementalFinishing()) + info = (info & ~INCREMENTAL_MASK) | INCREMENTAL_STARTED; + } + + final boolean isIncrementalFinishing() + { + return (info & INCREMENTAL_MASK) >= INCREMENTAL_FINISHING; + } + + final void setIncrementalFinishingExclusive() + { + Invariants.require(isIncremental()); + info |= INCREMENTAL_FINISHING; + } + + final void setSequencedExclusive(ExecutionContext.ExecutionSequence sequence) + { + Invariants.require(isUnsequenced()); + info |= sequence.ordinal() << SEQUENCED_SHIFT; + } + + final boolean isUnsequenced() + { + return (info & SEQUENCED_MASK) == 0; + } + + final boolean isSequencedByPriority() + { + return (info & SEQUENCED_MASK) == SEQUENCED_PRIORITY; + } + + final boolean isSequencedByPriorityAtomic() + { + return (info & SEQUENCED_MASK) >= SEQUENCED_ATOMIC; + } + + final boolean isCacheQueuedFifo() + { + return (info & SEQUENCED_MASK) == SEQUENCED_ATOMIC_AND_QUEUED; + } + + final boolean isCacheQueued() + { + return 0 != (info & CACHE_QUEUED_BIT); + } + + // supersedes priority, in whichever order they're called + final void setCacheQueuedFifoExclusive() + { + Invariants.require(isSequencedByPriorityAtomic()); + info |= SEQUENCED_ATOMIC_AND_QUEUED | CACHE_QUEUED_BIT; + } + + final void setCacheQueuedExclusive() + { + info |= CACHE_QUEUED_BIT; + } + + final int tranche() + { + Invariants.require((info & HAS_TRANCHE_BIT) != 0); + return info >>> TRANCHE_SHIFT; + } + + final void setTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; + } + + final void setInheritedWithTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; + } + + final boolean hasInherited() + { + return (info & HAS_INHERITED_BIT) != 0; + } + + final void setInheritedRangeScan() + { + info = info | HAS_INHERITED_RANGE_SCAN_BIT; + } + + final boolean hasInheritedRangeScan() + { + return (info & HAS_INHERITED_RANGE_SCAN_BIT) != 0; + } + + static int init(GlobalGroup global, ExclusiveGroup exclusive) + { + return (global.ordinal() << GLOBAL_GROUP_SHIFT) | (exclusive.ordinal() << EXCLUSIVE_GROUP_SHIFT); + } + + static Task reverse(Task unqueued) + { + Task prev = null; + Task cur = unqueued; + while (cur != null) + { + Task next = cur.next; + cur.next = prev; + prev = cur; + cur = next; + } + return prev; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java new file mode 100644 index 0000000000..d9163d5a58 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.local.ExecutionContext; + +import org.apache.cassandra.concurrent.DebuggableTask; + +public class TaskInfo implements Comparable +{ + // sorted in name order for reporting to virtual tables + public enum Status + { + LOADING, RUNNING, SCANNING_RANGES, WAITING_TO_LOAD, WAITING_TO_RUN + } + + final Status status; + final int commandStoreId; + + final Task task; + + public TaskInfo(Status status, int commandStoreId, Task task) + { + this.status = status; + this.commandStoreId = commandStoreId; + this.task = task; + } + + public Status status() + { + return status; + } + + public Integer commandStoreId() + { + return commandStoreId >= 0 ? commandStoreId : null; + } + + public long position() + { + return task.position; + } + + public @Nullable String describe() + { + if (task instanceof SafeTask) + return ((SafeTask) task).executionContext().reason(); + + if (task instanceof DebuggableTask) + return ((DebuggableTask) task).description(); + + return null; + } + + public @Nullable ExecutionContext preLoadContext() + { + if (task instanceof SafeTask) + return ((SafeTask) task).executionContext(); + if (task instanceof IOTaskWrapper && ((IOTaskWrapper) task).wrapped instanceof SafeTask.RangeTxnScanner) + return ((SafeTask.RangeTxnScanner) ((IOTaskWrapper) task).wrapped).preLoadContext(); + return null; + } + + @Override + public int compareTo(TaskInfo that) + { + int c = this.status.compareTo(that.status); + if (c == 0) c = Long.compare(this.position(), that.position()); + return c; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java new file mode 100644 index 0000000000..ec483dd640 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.IntrusivePriorityHeap; +import accord.utils.TinyEnumSet; + +// has xSingle methods to distinguish from within MultiTaskQueue whether we're invoking the multi or single variation +class TaskQueue extends IntrusivePriorityHeap +{ + final int states; + + public TaskQueue(int states) + { + this.states = states; + } + + void unqueue(T task) + { + throw new UnsupportedOperationException(); + } + + @Override + public final int compare(T o1, T o2) + { + return Long.compare(o1.position, o2.position); + } + + @Override + protected final void ensureHeapified() + { + super.ensureHeapified(); + } + + protected final boolean requeueSingle(T requeue) + { + int oldIndex = updateNode(requeue); + if (oldIndex < 0) + return false; + + int newIndex = heapIndex(requeue); + return Math.min(oldIndex, newIndex) == 0 && heapifiedSize() > 0; + } + + final T peekSingle() + { + ensureHeapified(); + return peekNode(); + } + + final T pollSingle() + { + ensureHeapified(); + return pollNode(); + } + + final boolean isEmptySingle() + { + return isEmptyInternal(); + } + + final int enqueueSingle(T enqueue) + { + return insertNode(enqueue); + } + + final boolean unqueueSingle(T unqueue) + { + int heapIndex = heapIndex(unqueue); + removeNode(unqueue); + return heapIndex == 0; + } + + final boolean tryUnqueueSingle(T remove) + { + return removeNodeIfContains(remove); + } + + final T getSingle(int index) + { + return super.getNode(index); + } + + final boolean isQueuedSingle(T test) + { + return containsNode(test); + } + + @Override + public String toString() + { + return TinyEnumSet.toString(states, Task.State::forOrdinal); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java new file mode 100644 index 0000000000..9784e51ad9 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java @@ -0,0 +1,612 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.Invariants; +import accord.utils.UnhandledEnum; + +/** + * A {@link TaskQueue} sub-divided into up to eight per-group sub-queues (groups are phases for the exclusive + * executor, or work classes globally) plus a policy for choosing which sub-queue to serve next. The policy balances + * three competing concerns: ordering/priority (run the oldest / highest-priority work), fairness (do not let one + * group monopolise the executor), and throughput (do not let an even split leave a busy group's backlog growing + * without bound). + * + *

Packed counters

+ * Per-group state is held in {@code long}s, one 7-bit lane per group; bit 7 of each byte is a spare "guard" bit used + * to run branch-free min/compare across all eight lanes at once (see {@code minCounters}). The lanes are: + *
    + *
  • {@code positions[]} - the head task's position (HLC, or submission order) per group; drives FIFO/priority.
    • + *
  • {@code recent} - a windowed count of recently served tasks per group (service received).
    • + *
  • {@code arrivals} - a windowed, size-biased count of recent arrivals per group (demand offered).
    • + *
  • {@code current} - tasks currently in flight per group, used to enforce {@code queue_active_limits}.
    • + *
  • {@code hasWork}/{@code dirty} - guard-bit masks: which groups have queued work / need a position refresh.
    • + *
+ * + *

Windowing: a bounded memory of the past

+ * {@code recent} and {@code arrivals} are not running totals. Whenever incrementing {@code recent} tips any lane to + * its maximum (0x80), both {@code recent} and {@code arrivals} are halved (see {@code incrementRecents}), + * and each lane also saturates at 0x7f. They are therefore an exponentially-decaying window keyed on service/time. + * This is what stops a one-off burst of arrivals granting a group a permanent boost: the burst saturates the lane + * briefly and then decays away as other work is served. + * + *

The fair selection counter

+ * The fair models pick the group with the smallest {@code effective} counter, where per lane + * 
effective = min(0x7f, max(0, recent - arrivals) + bias)
+ *
    + *
  • {@code max(0, recent - arrivals)} - a group whose arrivals have outpaced its recent service clamps to 0 and + * is therefore preferred; in steady state each group's service tracks its arrival rate, which bounds backlog. + * The clamp is symmetric, so a group cannot bank "credit" during a lull and later use it to monopolise service.
    • + *
+ * + *

Choosing a group ({@code minGroup} / {@code pollByBlend})

+ *
    + *
  • {@code PRIORITY_ONLY} - always the lowest {@code positions} (oldest / highest priority), ignoring fairness.
    • + *
  • {@code PHASE_OVERRIDE} - strict phase order: the lowest-index group with work, always.
    • + *
  • {@code PHASE_FAIR} - pure fairness: the smallest {@code effective}; ties broken by index, within a group by + * position. It has no priority fallback, so it is deliberately completing-first under contention.
    • + *
  • {@code PRIORITY_FAIR} (default) - a deficit round-robin blend of two strategies, chosen per poll: flow + * (smallest {@code effective}, i.e. least fairly serviced) and age (oldest {@code positions}, i.e. FIFO). + * The flow weight ramps up with the flow imbalance ({@code max-min} of {@code effective}) over + * {@code queue_flow_imbalance_onset..+width}, trading against age zero-sum; when balanced it is pure age. The + * ramp is smooth (not a hard mode switch), so there is no boundary to oscillate around and no hysteresis is + * needed. Age also drains stale/standing backlogs for free: a backlog's items are the oldest work, so age + * clears them without a separate backlog-aware strategy (the {@code effective} flow counter tracks arrival + * rate and is deliberately blind to standing stock).
    • + *
+ * + *

Concurrency limits and eligibility

+ * A group whose in-flight {@code current} count has reached its {@code queue_active_limits} limit is + * {@code saturated} and excluded from selection ({@code disabled}), as is any group with no queued work. + * + *

Note on {@code positions} freshness

+ * {@code positions} is refreshed lazily from the sub-queue heads via the {@code dirty} mask; {@code dirty} must use + * the same guard-bit layout as {@code hasWork} so the refresh loop in {@code minGroupByPriority} actually runs - + * otherwise it silently degenerates to lowest-index-with-work and starves high-index / new-work groups. + * + *

NB it extends {@link TaskQueue} to keep the type hierarchy simple for method dispatch, and for efficiency for + * anonymous ExclusiveExecutors which do not use multiple queues, while letting ExclusiveExecutor share a parent + * class for both use cases. + */ +abstract class TaskQueueMulti extends TaskQueue +{ + private static final TaskQueue[] NO_QUEUES = new TaskQueue[0]; + private static final long[] NO_POSITIONS = new long[0]; + static final long COUNTER_OVERFLOWS = 0x8080808080808080L; + static final long COUNTER_MASKS = 0x7f7f7f7f7f7f7f7fL; + static final long COUNTER_LOWBITS = 0x0101010101010101L; + + final TaskQueue[] queues; + final long[] positions; + final byte groupShift; + final long limits; + + /** + * sets overflow bits for a queue that has been stopped + */ + long stopped; + /** + * sets overflow bits for each counter when it needs its position updated + */ + long dirty; + /** + * sets overflow bits for each counter when there's associated work + */ + long hasWork; + /** + * Stores recent dequeue counts for up to 8 sub queues. + */ + long dispatches; + // TODO (required): increment arrivals based on internal queue for ExclusiveExecutors + // also: experiment with decaying on arrival schedule rather than poll schedule, since this should respond to work growth more accurately + /** + * Stores recent enqueue counts for up to 8 sub queues. + */ + long arrivals; + /** + * Stores currently-active counts for up to 8 sub queues. We can use this to impose limits on specific queues. + */ + long active; + /** + * deficit-round-robin credits for the two PRIORITY_FAIR strategies (flow/age). + */ + int creditFlow, creditAge; + + int waitingCount; + + TaskQueueMulti(int waitingStates, Task.GroupKind groups, long limits) + { + super(waitingStates); + this.limits = limits; + int queueCount = groups.count; + Invariants.require(queueCount <= 8); + queues = queueCount == 0 ? NO_QUEUES : new TaskQueue[queueCount]; + positions = queueCount > 0 && AccordExecutor.BALANCE_BY_POSITION ? new long[queueCount] : NO_POSITIONS; + groupShift = groups.shift; + } + + final int group(Task task) + { + if (groupShift == 0) + return -1; + + return (task.info >>> groupShift) & Task.GROUP_MASK; + } + + void stop(int group) + { + stopped |= overflowBit(group); + } + + void restart(int group) + { + stopped &= ~overflowBit(group); + } + + final TaskQueue queue(Task task) + { + int group = group(task); + if (group < 0) + return this; + + return queue(group); + } + + final TaskQueue queue(int group) + { + TaskQueue queue = queues[group]; + if (queue == null) + queues[group] = queue = new TaskQueue<>(0); + + return queue; + } + + private int pollGroup() + { + if (hasWork == 0) + return -1; + + switch (AccordExecutor.BALANCING_MODEL) + { + default: + throw new UnhandledEnum(AccordExecutor.PRIORITY_MODEL); + case PRIORITY_ONLY: + return pollGroupByPriority(); + case PHASE_ONLY: + return pollGroupByIndex(); + case PHASE_FAIR: + return pollGroupByPhaseFair(); + case BLENDED_PRIORITY_PHASE_FAIR: + return pollGroupByBlended(); + } + } + + private int pollGroupByPriority() + { + return pollGroupByPriority(unsaturatedWithWork()); + } + + private int pollGroupByPriority(long enabled) + { + long refresh = dirty & hasWork; + while (refresh != 0) + { + int bitIndex = Long.numberOfTrailingZeros(refresh); + int group = bitIndex / 8; + positions[group] = queues[group].peekSingle().position; + refresh ^= 1L << bitIndex; + } + dirty = 0; + + long minPosition = Long.MAX_VALUE; + int minGroup = -1; + long visit = enabled >>> 7; + while (visit != 0) + { + int bitIndex = Long.numberOfTrailingZeros(visit); + int group = bitIndex / 8; + long position = positions[group]; + if (position < minPosition) + { + minGroup = group; + minPosition = position; + } + visit ^= 1L << bitIndex; + } + + return minGroup; + } + + private int pollGroupByIndex() + { + long visit = (hasWork & unsaturated()) >>> 7; + if (visit == 0) + return -1; + + int bitIndex = Long.numberOfTrailingZeros(visit); + return bitIndex / 8; + } + + private int pollGroupByPhaseFair() + { + return minCounterIndex(recentFlowImbalances()); + } + + // PRIORITY_FAIR selection: a deficit round-robin blend of two strategies, chosen per poll: + // flow -> minCounterIndex(recent - arrivals + bias) (least fairly serviced) + // age -> minGroupByPriority() (earliest-queued work) + // wFlow = ramp(flow imbalance F = max-min of the selection counters); wAge = BLEND_TOTAL - wFlow. As flow gets + // uneven, polls trade from age to flow zero-sum; when balanced it is pure age (FIFO). A ramp (not a hard + // threshold) means there is no mode cliff to oscillate around, so no anti-oscillation penalty is needed. Age is + // also what drains a stale/standing backlog -- its items are the oldest -- so no separate stock strategy is needed. + private int pollGroupByBlended() + { + return pollGroupByBlended(saturatedOrWithoutWork()); + } + + private int pollGroupByBlended(long disabled) + { + long withoutWork = hasWork ^ COUNTER_OVERFLOWS; + long counters = recentFlowImbalances(); + long minMax = minMaxCounterValue(counters, withoutWork); + long min = minMax & 0x7f; + long max = minMax >>> 8; + int flowImbalance = (int) (max - min); + + int flowWeight = flowWeight(flowImbalance); + int priorityWeight = AccordExecutor.BLEND_TOTAL - flowWeight; + + creditFlow += flowWeight; + creditAge += priorityWeight; + + if (creditFlow >= creditAge) + { + creditFlow -= AccordExecutor.BLEND_TOTAL; + if (disabled != withoutWork) + min = minCounterValue(counters, disabled); + + return minCounterIndex(counters, min, disabled); + } + else + { + creditAge -= AccordExecutor.BLEND_TOTAL; + return pollGroupByPriority(disabled ^ COUNTER_OVERFLOWS); + } + } + + private long saturated() + { + return ((active | COUNTER_OVERFLOWS) - limits) & COUNTER_OVERFLOWS; + } + + private long unsaturated() + { + return saturated() ^ COUNTER_OVERFLOWS; + } + + private long unsaturatedWithWork() + { + return hasWork & unsaturated() & ~stopped; + } + + private long saturatedOrWithoutWork() + { + return (hasWork ^ COUNTER_OVERFLOWS) | saturated() | stopped; + } + + // arrivals is a windowed measure of ARRIVAL (incremented on enqueue, size-biased; decays on recent's overflow). + // Combined with recent (service) as effective = max(0, recent - arrivals): a queue whose arrivals outpace its + // service clamps to 0 and is preferred, so service converges to arrival rate (bounding the busy queue's backlog). + private long recentFlowImbalances() + { + return clampedSubtract(dispatches, arrivals); + } + + static long minCounterValue(long counters, long disabled) + { + long mins = counters; + mins |= overflowsToLowMasks(disabled); + mins = minCounters(mins, mins >>> 8); // each slot is min of slots [i..i+1] + mins = minCounters(mins, mins >>> 16); // each slot is min of slots [i..i+3] + mins = minCounters(mins, mins >>> 32); // each slot is min of slots [i..i+7] + return mins & 0x7f; + } + + static long minMaxCounterValue(long counters, long disabled) + { + long mins = counters; + long maxs = counters ^ COUNTER_MASKS; + long overflowMasks = overflowsToLowMasks(disabled); + mins |= overflowMasks; + maxs |= overflowMasks; + mins = minCounters(mins, mins >>> 8) & 0x007f007f007f007fL; // each slot is min of slots [i..i+1] + maxs = (minCounters(maxs, maxs << 8) & 0x7f007f007f007f00L); // each slot is min of slots ~[i..i+1] + long minmaxs = mins | maxs; + minmaxs = minCounters(minmaxs, minmaxs >>> 16); // each slot is min of slots [i..i+3] + minmaxs = minCounters(minmaxs, minmaxs >>> 32); // each slot is min of slots [i..i+7] + return (minmaxs ^ 0x7f00) & 0x7f7f; + } + + /** + * If provided two counters (containing 8 7 bit counters each), + * returns the minimum of each matching counter + */ + private static long minCounters(long a, long b) + { + // set overflow bits where a <= b + long selecta = setOverflowWhenLessEqual(a, b); + return selectByOverflowBits(selecta, a, b); + } + + static long setOverflowWhenLessEqual(long a, long b) + { + return ((b | COUNTER_OVERFLOWS) - a) & COUNTER_OVERFLOWS; + } + + // select a if overflow bit is set; b if it is unset + static long selectByOverflowBits(long selecta, long a, long b) + { + selecta = overflowsToLowMasks(selecta); + a &= selecta; + b &= ~selecta; + return a | b; + } + + static long overflowsToLowMasks(long v) + { + return v - (v >>> 7); + } + + private static int flowWeight(int flowImbalance) + { + if (flowImbalance <= AccordExecutor.FLOW_ONSET) return 0; + return Math.min(AccordExecutor.BLEND_TOTAL, ((flowImbalance - AccordExecutor.FLOW_ONSET) << AccordExecutor.BLEND_SHIFT) >>> AccordExecutor.FLOW_WIDTH_SHIFT); + } + + // per-lane max(0, a - b), carry-free: zero both a and b in lanes where a <= b, then subtract + private static long clampedSubtract(long a, long b) + { + long keep = ~overflowsToLowMasks(setOverflowWhenLessEqual(a, b)); + return (a & keep) - (b & keep); + } + + private int minCounterIndex(long counters) + { + return minCounterIndex(counters, saturatedOrWithoutWork()); + } + + private int minCounterIndex(long counters, long disabled) + { + return minCounterIndex(counters, minCounterValue(counters, disabled), disabled); + } + + private int minCounterIndex(long counters, long minCounterValue, long disabled) + { + long mins = minCounterValue * COUNTER_LOWBITS; + long select = ((mins | COUNTER_OVERFLOWS) - counters) & COUNTER_OVERFLOWS; + // now unset those overflow bits associated with disabled queues + select &= ~disabled; + if (select == 0) + return -1; + return (Long.numberOfTrailingZeros(select) - 7) / 8; + } + + final T pollMulti() + { + int group = pollGroup(); + if (group < 0) + { + // group < 0 can mean EITHER we don't have any nested queues OR those queues are either empty or DISABLED + T result = pollSingle(); + if (result != null) + --waitingCount; + return result; + } + + --waitingCount; + incrementActive(group); + incrementDispatches(group); + + TaskQueue queue = queues[group]; + T head = queue.pollSingle(); + // NOTE: must clear dirty when emptied, symmetrically with unqueue(): the fair selection paths + // never consume the dirty bit, so a group drained during a fairness episode would otherwise + // retain a stale dirty bit and NPE in minGroupByPriority.peekSingle() when balance is restored. + if (queue.isEmptySingle()) + { + unsetHasWork(group); + unsetDirty(group); + } + else setDirty(group); + return head; + } + + final void enqueueMulti(T task, boolean incrementArrivals) + { + task.setQueue(this); + int group = group(task); + if (group < 0) + { + enqueueSingle(task); + } + else + { + TaskQueue queue = queue(group); + int result = queue.enqueueSingle(task); + if (incrementArrivals) + incrementArrivals(group); + if (result < 0) setHasWork(group); + if (result != 0) setDirty(group); + } + ++waitingCount; + } + + final void requeue(T task) + { + int group = group(task); + if (group < 0) requeueSingle(task); + else + { + TaskQueue queue = queue(group); + Invariants.require(queue != null && queue.isQueuedSingle(task)); + if (queue.requeueSingle(task)) + setDirty(group); + } + } + + final void unqueueMulti(T task) + { + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + Invariants.require(queue.isQueuedSingle(task)); + unqueue(task, group, queue); + } + + // if there is an active collection, we return false and do not remove ourselves from it + boolean tryUnqueueWaiting(T task) + { + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + if (!queue.isQueuedSingle(task)) + return false; + + unqueue(task, group, queue); + return true; + } + + private void unqueue(T task, int group, TaskQueue queue) + { + task.unsetQueue(this); + boolean dirty = queue.unqueueSingle(task); + --waitingCount; + if (group >= 0) + { + if (queue.isEmptySingle()) + { + unsetHasWork(group); + unsetDirty(group); + } + else if (dirty) setDirty(group); + } + } + + final void incrementActive(int group) + { + active += lowBit(group); + } + + final void decrementActive(int group) + { + active -= lowBit(group); + } + + final void incrementDispatches(Task task) + { + int group = group(task); + if (group >= 0) + incrementDispatches(group); + } + + final void incrementDispatches(int group) + { + dispatches += lowBit(group); + if ((dispatches & COUNTER_OVERFLOWS) != 0) + { + dispatches = (dispatches >>> 1) & COUNTER_MASKS; + arrivals = (arrivals >>> 1) & COUNTER_MASKS; // arrivals (arrival) decays on the service/time clock + } + } + + final void decrementDispatches(Task task) + { + int group = group(task); + if (group >= 0) + decrementDispatches(group); + } + + final void decrementDispatches(int group) + { + long lowBit = lowBit(group); + dispatches -= lowBit; + dispatches += (dispatches >>> 7) & lowBit; + } + + final void incrementArrivals(Task task) + { + int group = group(task); + if (group >= 0) + incrementArrivals(group); + } + + final void incrementArrivals(int group) + { + int shift = group * 8; + long overflowBit = 0x80L << shift; + arrivals += 1L << shift; + // if we overflow, unset the overflow bit and set all other bits for the counter + long overflow = arrivals & overflowBit; + arrivals ^= overflow; + arrivals |= overflow - (overflow >>> 7); + } + + final void setHasWork(int group) + { + hasWork |= overflowBit(group); + } + + final void unsetHasWork(int group) + { + hasWork &= ~overflowBit(group); + } + + final void setDirty(int group) + { + dirty |= overflowBit(group); + } + + final void unsetDirty(int group) + { + dirty &= ~overflowBit(group); + } + + final boolean hasWaitingToRun() + { + return unsaturatedWithWork() != 0; + } + + final boolean isWaiting(T task) + { + return queue(task).isQueuedSingle(task); + } + + final int waitingCount() + { + return waitingCount; + } + + final long lowBit(int group) + { + return 1L << (group * 8); + } + + final long overflowBit(int group) + { + return 0x80L << (group * 8); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java new file mode 100644 index 0000000000..8aad433fb6 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.TinyEnumSet; + +import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; + +final class TaskQueueRunnable extends TaskQueueMulti +{ + static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, RUNNING); + + final TaskQueue assigned; + + TaskQueueRunnable() + { + super(RUNNABLE, Task.GroupKind.GLOBAL, AccordExecutor.GLOBAL_QUEUE_LIMITS); + this.assigned = new TaskQueue<>(0); + } + + T poll() + { + T next = pollMulti(); + if (next == null) + return null; + + assigned.enqueueSingle(next); + return next; + } + + void enqueue(T enqueue, boolean incrementArrivals) + { + enqueueMulti(enqueue, incrementArrivals); + } + + void unqueue(T unqueue) + { + if (assigned.isQueuedSingle(unqueue)) + { + int group = group(unqueue); + if (group >= 0) + decrementActive(group); + + unqueue.unsetQueue(this); + assigned.unqueueSingle(unqueue); + } + else + { + super.unqueueMulti(unqueue); + } + } + + int waitingOrAssignedCount() + { + return waitingCount + assigned.size(); + } + + boolean hasAssignedOrWaiting() + { + return waitingCount > 0 || !assigned.isEmptySingle(); + } + + boolean hasAssigned() + { + return !assigned.isEmptySingle(); + } + + boolean isAssigned(T task) + { + return assigned.isQueuedSingle(task); + } + + void cleanup(T task) + { + if (assigned.tryUnqueueSingle(task)) + { + int group = group(task); + if (group >= 0) + decrementActive(group); + task.unsetQueue(this); + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java new file mode 100644 index 0000000000..7dd8b1def0 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.TinyEnumSet; + +final class TaskQueueStandalone extends TaskQueue +{ + TaskQueueStandalone(int states) + { + super(states); + } + + TaskQueueStandalone(Task.State state) + { + super(TinyEnumSet.encode(state)); + } + + void enqueue(T enqueue) + { + enqueue.setQueue(this); + enqueueSingle(enqueue); + } + + void unqueue(T unqueue) + { + unqueue.unsetQueue(this); + removeNode(unqueue); + } + + T peek() + { + return peekSingle(); + } + + T poll() + { + return pollSingle(); + } + + boolean isEmpty() + { + return isEmptySingle(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java new file mode 100644 index 0000000000..2dffb11e18 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.apache.cassandra.concurrent.CassandraThread; +import org.apache.cassandra.concurrent.DebuggableTask; + +public interface TaskRunner +{ + AccordExecutor accordActiveExecutor(); + void setAccordActiveExecutor(AccordExecutor newExecutor); + + AccordExecutor accordLockedExecutor(); + boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor); + void exitAccordLockedExecutor(); + + // to be called only by the thread itself, so can (eventually) avoid any memory barriers + Task accordActiveSelfTask(); + Task accordActiveTask(); + void setAccordActiveTask(Task newActiveTask); + + static TaskRunner get() + { + return get(Thread.currentThread()); + } + + static TaskRunner get(Thread thread) + { + return thread instanceof CassandraThread ? (CassandraThread) thread : ThreadLocalTaskRunner.threadLocal.get(); + } + + final class ThreadLocalTaskRunner implements TaskRunner + { + private AccordExecutor lockedExecutor; + private AccordExecutor activeExecutor; + volatile Task activeTask; + + private static final ThreadLocal threadLocal = ThreadLocal.withInitial(ThreadLocalTaskRunner::new); + + @Override + public AccordExecutor accordActiveExecutor() + { + return activeExecutor; + } + + @Override + public void setAccordActiveExecutor(AccordExecutor newExecutor) + { + activeExecutor = newExecutor; + } + + @Override + public AccordExecutor accordLockedExecutor() + { + return lockedExecutor; + } + + @Override + public boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) + { + if (lockedExecutor != null) + return false; + lockedExecutor = newLockedExecutor; + return true; + } + + @Override + public void exitAccordLockedExecutor() + { + lockedExecutor = null; + } + + @Override + public void setAccordActiveTask(Task newActiveTask) + { + activeTask = newActiveTask; + } + + @Override + public Task accordActiveTask() + { + return activeTask; + } + + @Override + public Task accordActiveSelfTask() + { + return activeTask; + } + } + + abstract class DebuggableWrapper implements DebuggableTask.DebuggableTaskRunner + { + private volatile TaskRunner wrapped; + + protected void setWrapped(TaskRunner wrapped) + { + this.wrapped = wrapped; + } + + @Override + public DebuggableTask running() + { + Task running = wrapped.accordActiveTask(); + return running == null ? null : running.debuggable(); + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Tranches.java b/src/java/org/apache/cassandra/service/accord/execution/Tranches.java new file mode 100644 index 0000000000..5b5b3f2668 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Tranches.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayDeque; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.utils.Invariants; + +import static org.apache.cassandra.service.accord.execution.Task.MAX_TRANCHE; + +/** + * Tasks are separated into tranches based on their position. + *

+ * When we want to wait for all submitted tasks and their consequences to complete, we create a new tranche + * and track the number of tasks still extant for all prior tranches - once these reach zero we can signal + * the completion of the work. + */ +final class Tranches +{ + private static final Logger logger = LoggerFactory.getLogger(Tranches.class); + + class WithDeferred implements Runnable + { + final Runnable run; + final ArrayDeque deferred; + + WithDeferred(Runnable run, Runnable deferred) + { + this.run = run; + this.deferred = new ArrayDeque<>(1); + this.deferred.add(deferred); + } + + WithDeferred(Runnable run, ArrayDeque deferred) + { + this.run = run; + this.deferred = deferred; + } + + @Override + public void run() + { + Runnable register = deferred.poll(); + if (!deferred.isEmpty()) + { + Invariants.require(runs[firstIndex] != null); + Invariants.require(!(runs[firstIndex] instanceof WithDeferred)); + runs[firstIndex] = new WithDeferred(runs[firstIndex], deferred); + } + registerWait(register); + try + { + run.run(); + } + catch (Throwable t) + { + owner.agent.onException(t); + } + } + } + + final AccordExecutor owner; + + int firstTranche; + int firstIndex; + long[] mins = new long[8]; + int[] counts = new int[8]; + Runnable[] runs = new Runnable[8]; + + // the next tranche, that all new work is being collected against + // (and will move to the array accounting once a new wait is registered) + long nextMin; + int nextTranche; + int nextCount; + + Tranches(AccordExecutor owner) + { + this.owner = owner; + } + + private int trancheToIndex(int tranche) + { + return firstIndex + trancheToIndexOffset(tranche); + } + + private int trancheToIndexOffset(int tranche) + { + int offset = tranche - firstTranche; + if (offset < 0) + offset += MAX_TRANCHE + 1; + return offset; + } + + int size() + { + return trancheToIndexOffset(nextTranche); + } + + int capacity() + { + return counts.length; + } + + int addNew(long position) + { + Invariants.require(position >= nextMin); + ++nextCount; + return nextTranche; + } + + void addInherited(int tranche, long position) + { + if (tranche == nextTranche) + { + Invariants.require(position >= nextMin); + ++nextCount; + } + else + { + int index = trancheToIndex(tranche); + Invariants.require(counts[index] > 0); + Invariants.require(mins[index] <= position); + ++counts[index]; + } + } + + void complete(int tranche) + { + if (tranche == nextTranche) + { + --nextCount; + } + else + { + + if (counts[trancheToIndex(tranche)] == 1) + owner.drainUnqueuedNewWorkExclusive(); // make sure we don't have any pending + + if (--counts[trancheToIndex(tranche)] == 0 && tranche == firstTranche) + { + do + { + advance(); + } + while (firstTranche != nextTranche && counts[firstIndex] == 0); + } + + if (firstIndex >= counts.length / 2) + compact(); + } + } + + public void finishAll(long nextPosition) + { + while (firstTranche != nextTranche) + { + logger.warn("{} processed all pending tasks (<{}) but found {} waiting for {}", this, + nextPosition, counts[firstIndex], size() == 1 ? nextMin : mins[firstIndex + 1]); + advance(); + } + } + + private void advance() + { + Runnable run = runs[firstIndex]; + runs[firstIndex] = null; + ++firstIndex; + if (firstTranche == MAX_TRANCHE) firstTranche = 0; + else ++firstTranche; + try + { + run.run(); + } + catch (Throwable t) + { + owner.agent.onException(t); + } + } + + public void registerWait(Runnable run) + { + int newNextTranche = (nextTranche + 1) % (MAX_TRANCHE + 1); + if (newNextTranche == firstTranche) + { + Runnable cur = runs[firstIndex]; + if (cur instanceof WithDeferred) ((WithDeferred) cur).deferred.add(run); + else runs[firstIndex] = new WithDeferred(runs[firstIndex], run); + return; + } + + if ((firstIndex + size()) == capacity()) + growOrCompact(); + + int index = firstIndex + size(); + mins[index] = nextMin; + counts[index] = nextCount; + runs[index] = run; + nextMin = owner.minPosition = owner.nextPosition; + nextCount = 0; + nextTranche = newNextTranche; + } + + private void compact() + { + if (size() <= capacity() / 4 && capacity() > 8) resize(capacity() / 2); + else compact(mins, counts, runs); + } + + private void growOrCompact() + { + if (size() > capacity() / 2) resize(capacity() * 2); + else compact(); + } + + private void resize(int newSize) + { + Invariants.require(newSize > 0); + long[] newMins = new long[newSize]; + int[] newCounts = new int[newSize]; + Runnable[] newRuns = new Runnable[newSize]; + compact(newMins, newCounts, newRuns); + mins = newMins; + counts = newCounts; + runs = newRuns; + } + + private void compact(long[] newMins, int[] newCounts, Runnable[] newRuns) + { + int size = size(); + System.arraycopy(mins, firstIndex, newMins, 0, size); + System.arraycopy(counts, firstIndex, newCounts, 0, size); + System.arraycopy(runs, firstIndex, newRuns, 0, size); + firstIndex = 0; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java b/src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java new file mode 100644 index 0000000000..d4851481a6 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.local.ExecutionContext; + +// run the task even on a stopped commandStore +public interface Unstoppable extends ExecutionContext.Empty +{ +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java b/src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java new file mode 100644 index 0000000000..20badc4eda --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +// run the task even on a terminated commandStore +public interface Unterminatable extends Unstoppable +{ +} diff --git a/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java b/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java index c3be7c87a0..3755750324 100644 --- a/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java @@ -54,13 +54,13 @@ import accord.utils.UnhandledEnum; import accord.utils.btree.BTree; import accord.utils.btree.IntervalBTree; -import org.apache.cassandra.service.accord.AccordCache; -import org.apache.cassandra.service.accord.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStore.Caches; import org.apache.cassandra.service.accord.RangeIndex; import org.apache.cassandra.service.accord.TokenRange; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import static accord.local.CommandSummaries.Relevance.IRRELEVANT; import static accord.local.LoadKeysFor.RECOVERY; @@ -199,7 +199,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements } @Override - protected void finish(Map into) + public void finish(Map into) { } @@ -233,7 +233,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements } @Override - protected void cleanupExclusive(Caches caches) + public void cleanupExclusive(Caches caches) { if (commandWatcher != null) { diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java b/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java index 29f0220790..87ce533638 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java @@ -58,10 +58,10 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.TokenRange; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; import org.apache.cassandra.service.accord.serializers.Version; diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java index bddb2b6fba..3c72c34ba1 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java @@ -55,8 +55,8 @@ import org.apache.cassandra.io.ParameterisedVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; import org.apache.cassandra.service.accord.serializers.Version; diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java index 719eb0f590..90cd9898c9 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java @@ -64,8 +64,8 @@ import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; import org.apache.cassandra.service.accord.serializers.Version; diff --git a/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java b/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java index 88a785d9a9..d82bd4ab83 100644 --- a/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java +++ b/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java @@ -24,6 +24,7 @@ import java.util.function.Function; import org.junit.Test; import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.utils.concurrent.Semaphore; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java index 7e91f5a7fb..4b573b2551 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java +++ b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java @@ -83,7 +83,7 @@ import org.apache.cassandra.distributed.shared.DistributedTestBase; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.Verb; -import org.apache.cassandra.service.accord.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCache; import static java.lang.System.currentTimeMillis; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java index 506407cb13..107488f0d5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java @@ -46,7 +46,7 @@ import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.streaming.StreamEvent; import org.apache.cassandra.streaming.StreamEventHandler; @@ -262,7 +262,7 @@ public class AccordBootstrapTest extends AccordBootstrapTestBase getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { if (safeStore.ranges().currentRanges().contains(partitionKey)) { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + SaferCommandStore ss = (SaferCommandStore) safeStore; Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java index edff67507a..97f98f319b 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java @@ -24,7 +24,6 @@ import org.assertj.core.api.Assertions; import accord.local.CommandStores; import accord.local.ExecutionContext; -import accord.local.LoadKeys; import accord.local.Node; import accord.local.cfk.CommandsForKey; import accord.local.cfk.SafeCommandsForKey; @@ -42,11 +41,10 @@ import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.TokenRange; -import static accord.local.LoadKeysFor.READ_WRITE; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; import static org.apache.cassandra.service.accord.AccordService.getBlocking; @@ -140,7 +138,7 @@ public class AccordDropTableBase extends TestBaseImpl { AccordCommandStore store = (AccordCommandStore) stores.forId(storeId); getBlocking(store.chain(ctx, input -> { - AccordSafeCommandStore safe = (AccordSafeCommandStore) input; + SaferCommandStore safe = (SaferCommandStore) input; for (SafeCommandsForKey safeCfk : safe.safeCommandsForKeys()) { CommandsForKey cfk = safeCfk.current(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java index 95d79b6fa7..d3c1fadfc4 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java @@ -62,7 +62,7 @@ import org.apache.cassandra.harry.checker.ModelChecker; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.IAccordService.DelegatingAccordService; @@ -209,7 +209,7 @@ public class AccordIncrementalRepairTest extends TestBaseImpl Node node = accordService().node(); AtomicReference waitFor = new AtomicReference<>(null); getBlocking(node.commandStores().forEach("Test", RoutingKeys.of(key), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { - AccordSafeCommandStore store = (AccordSafeCommandStore) safeStore; + SaferCommandStore store = (SaferCommandStore) safeStore; SafeCommandsForKey safeCfk = store.ifLoadedAndInitialised(key); if (safeCfk == null) return; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java index dff776cbf2..6e6bb30ab7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java @@ -54,7 +54,7 @@ import org.apache.cassandra.metrics.DefaultNameFactory; import org.apache.cassandra.metrics.RatioGaugeSet; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.exceptions.AccordReadPreemptedException; import org.apache.cassandra.service.accord.exceptions.AccordWritePreemptedException; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java index 4d600f0c71..881728882f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java @@ -38,7 +38,7 @@ import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.api.PartitionKey; import static com.google.common.collect.Iterables.getOnlyElement; @@ -112,7 +112,7 @@ public class AccordMoveTest extends AccordBootstrapTestBase getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), moveMax, moveMax, safeStore -> { if (!safeStore.ranges().allAt(preMove).contains(partitionKey)) { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + SaferCommandStore ss = (SaferCommandStore) safeStore; Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java index 79cf938c7e..4f4b7942f0 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java @@ -47,7 +47,7 @@ import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordOperations; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; @@ -187,7 +187,7 @@ public class AccordRecoverFromAvailabilityLossTest extends AccordBootstrapTestBa getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { if (safeStore.ranges().currentRanges().contains(partitionKey)) { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + SaferCommandStore ss = (SaferCommandStore) safeStore; Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java index 1b864aea1b..67493f5a2c 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java @@ -36,9 +36,9 @@ import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.accord.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommand; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.utils.ByteBufferUtil; @@ -87,7 +87,7 @@ public class AccordJournalConsistentExpungeTest extends TestBaseImpl Node node = service.node(); AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable()); - Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); + Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); TxnId txnId = TxnId.NONE; diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java index d5a8f79c63..640ce44740 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java @@ -74,7 +74,7 @@ public class AccordRebootstrapTest extends FuzzTestBase HashSet downInstances = new HashSet<>(); AtomicInteger nextId = new AtomicInteger(); withRandom(rng -> { - Generator schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bootstrap_fuzz" + (nextId.incrementAndGet()), POPULATION, + Generator schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bsfuzz" + (nextId.incrementAndGet()), POPULATION, SchemaSpec.optionsBuilder() .addWriteTimestamps(false) .withTransactionalMode(TransactionalMode.full) diff --git a/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java b/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java index fcdc7deb41..f6262a7db2 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java @@ -45,8 +45,8 @@ public class AbstractPairOfSequencesPaxosSimulationTest "\tat accord.impl.RequestCallbacks$CallbackStripe$RegisteredCallback.lambda$safeInvoke$0(RequestCallbacks.java:140)\n" + "\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n" + "\tat accord.utils.async.AsyncChains.lambda$encapsulate$0(AsyncChains.java:498)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + "\tat org.apache.cassandra.service.accord.AccordExecutorSimple.run(AccordExecutorSimple.java:95)\n" + "\tat org.apache.cassandra.concurrent.FutureTask$2.call(FutureTask.java:124)\n" + "\tat org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)\n" + @@ -77,8 +77,8 @@ public class AbstractPairOfSequencesPaxosSimulationTest "\tat accord.impl.RequestCallbacks$CallbackStripe$RegisteredCallback.lambda$safeInvoke$0(RequestCallbacks.java:140)\n" + "\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n" + "\tat accord.utils.async.AsyncChains.lambda$encapsulate$0(AsyncChains.java:498)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + "\tat org.apache.cassandra.service.accord.AccordExecutorSimple.run(AccordExecutorSimple.java:95)\n" + "\tat org.apache.cassandra.concurrent.FutureTask$2.call(FutureTask.java:124)\n" + "\tat org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)\n" + diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java deleted file mode 100644 index 6bf1f46dfd..0000000000 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.simulator.test; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.LockSupport; -import java.util.function.BooleanSupplier; -import java.util.function.Consumer; - -import javax.annotation.Nullable; - -import org.junit.Test; - -import accord.api.AsyncExecutor; -import accord.api.ExclusiveAsyncExecutor; -import accord.utils.async.Cancellable; - -import org.apache.cassandra.concurrent.ExecutorPlus; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; -import org.apache.cassandra.service.accord.AccordExecutor; -import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; -import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; -import org.apache.cassandra.service.accord.api.AccordAgent; -import org.apache.cassandra.utils.concurrent.AsyncPromise; -import org.apache.cassandra.utils.concurrent.SignalLock; - -import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; - -// TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit -public class AccordExecutorAndCacheTest extends SimulationTestBase -{ - @Test - public void signalLoopTest() - { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new AccordAgent()), - 16); - } - - static class Submitted - { - final AtomicInteger nextId = new AtomicInteger(); - final AtomicInteger doneBefore = new AtomicInteger(); - final ConcurrentHashMap>> consequences = new ConcurrentHashMap<>(); - - boolean isDone() - { - return isDoneBetween(0, nextId.get()); - } - - boolean isDoneBefore(int before) - { - if (!isDoneBetween(doneBefore.get(), before)) - return false; - doneBefore.accumulateAndGet(before, Integer::max); - return true; - } - - boolean isDoneBetween(int from, int before) - { - for (int id = from ; id < before ; ++id) - { - for (Future future : consequences.get(id)) - { - if (!future.isDone()) - return false; - } - } - return true; - } - - Collection> start() - { - ConcurrentLinkedQueue> result = new ConcurrentLinkedQueue<>(); - - while (true) - { - int id = nextId.get(); - Object prev = consequences.putIfAbsent(id, result); - nextId.compareAndSet(id, id + 1); - if (prev == null) - return result; - } - } - } - - static class Control extends ConcurrentLinkedQueue - { - final Submitted submitted; - final AtomicInteger count = new AtomicInteger(); - final float cancelChance; - float processChance; - - Control(float cancelChance, Submitted submitted) - { - this(submitted, cancelChance, ThreadLocalRandom.current().nextFloat() * 0.5f); - } - - Control(Submitted submitted, float cancelChance, float processChance) - { - this.submitted = submitted; - this.cancelChance = cancelChance; - this.processChance = processChance; - } - - void submit(AsyncExecutor executor, Collection> consequences, Consumer>> run) - { - AsyncPromise future = new AsyncPromise<>(); - Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { - if (fail == null) future.trySuccess(null); - else - { - future.tryFailure(fail); - if (fail instanceof CancellationException) - run.accept(submitted.start()); - } - }); - consequences.add(future); - - if (cancel != null && ThreadLocalRandom.current().nextFloat() <= cancelChance) - { - add(cancel); - count.incrementAndGet(); - } - - if (count.get() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance) - { - int cancelCount = 0; - do - { - ++cancelCount; - - float delta = ThreadLocalRandom.current().nextFloat() - 0.5f; - if (delta < 0) processChance /= delta; - else processChance *= -delta; - if (processChance < 0.001f || processChance > 0.999f) - processChance = cancelChance; - } while (count.decrementAndGet() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance); - - // do outside of loop to avoid reentry - while (cancelCount-- > 0) - remove().cancel(); - } - } - } - - public void executorTest(SerializableSupplier supplier, int submissionThreads) - { - simulate(arr(() -> { - try - { - DatabaseDescriptor.daemonInitialization(); - ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); - AccordExecutor executor = supplier.get(); - Lock lock = executor.unsafeLock(); - ExclusiveAsyncExecutor sequentialExecutor = executor.newExclusiveExecutor(); - Executor lockExecutor = executorFactory().sequential("lock"); - - for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) - { - for (float lockChance : new float[] { 0f, 0.01f, 0.1f }) - { - for (float cancelChance : new float[] { 0f, 0.01f, 0.1f }) - { - System.out.println(String.format("sleepChance %.2f, lockChance %.2f, cancelChance %.2f", sleepChance, lockChance, cancelChance)); - List> done = new ArrayList<>(); - Submitted submitted = new Submitted(); - for (int i = 0; i < submissionThreads; ++i) - { - int id = i; - done.add(submit.submit(() -> { - try - { - submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance, new Control(cancelChance, submitted), submitted); - } - catch (ExecutionException | InterruptedException e) - { - throw new RuntimeException(e); - } - })); - } - for (Future f : done) - f.get(); - - if (!submitted.isDone()) - throw new AssertionError(); - } - } - } - } - catch (Throwable t) - { - throw new RuntimeException(t); - } - }), - () -> {}, 1L); - } - - private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance, Control control, Submitted submitted) throws ExecutionException, InterruptedException - { - ConcurrentLinkedQueue> awaitConsequences = new ConcurrentLinkedQueue<>(); - while (outerLoop-- > 0) - { - List>> allAwaitSubmitted = new ArrayList<>(); - for (int i = 0; i < innerLoop; ++i) - { - Collection> awaitSubmitted = submitted.start(); - allAwaitSubmitted.add(awaitSubmitted); - submitRecursive(lock, executor, sequentialExecutor, 1 + i, awaitSubmitted, awaitConsequences, submitted, sleepChance, lockChance, control); - } - - AtomicBoolean done = new AtomicBoolean(); - submitUntil(lock, lockExecutor, sleepChance, done::get); - for (Collection> awaitSubmitted : allAwaitSubmitted) - await(awaitSubmitted, CancellationException.class); - await(awaitConsequences, null); - done.set(true); - System.out.println("Loop " + id + '.' + (1 + outerLoop)); - } - } - - private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) - { - AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; - - control.submit(submitTo, consequences, nextConsequences -> { - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - boolean locked = false; - if (rnd.nextFloat() < lockChance) - { - if (rnd.nextBoolean()) locked = lock.tryLock(); - else { locked = true; lock.lock(); } - } - if (ThreadLocalRandom.current().nextFloat() < 0.01f) - { - int expectDoneBefore = submitted.nextId.get(); - AsyncPromise afterConsequences = new AsyncPromise<>(); - executor.afterSubmittedAndConsequences(() -> { - if (!submitted.isDoneBefore(expectDoneBefore)) - throw new AssertionError(); - afterConsequences.setSuccess(null); - }); - awaitConsequences.add(afterConsequences); - } - try - { - if (count > 1) - submitRecursive(lock, executor, sequentialExecutor, count -1, nextConsequences, awaitConsequences, submitted, sleepChance, lockChance, control); - if (rnd.nextFloat() < sleepChance) - LockSupport.parkNanos(rnd.nextInt(10000, 100000)); - } - finally - { - if (locked) - lock.unlock(); - } - }); - } - - private static void submitUntil(Lock lock, Executor executor, float sleepChance, BooleanSupplier done) - { - if (done.getAsBoolean()) - return; - - executor.execute(() -> { - - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - boolean tryLock = rnd.nextBoolean(); - boolean locked = !tryLock; - if (tryLock) locked = lock.tryLock(); - else lock.lock(); - try - { - if (rnd.nextFloat() < sleepChance) - LockSupport.parkNanos(rnd.nextInt(10000, 100000)); - - submitUntil(lock, executor, sleepChance, done); - } - finally - { - if (locked) - lock.unlock(); - } - }); - } - - private static void await(Collection> await, @Nullable Class ignore) throws InterruptedException, ExecutionException - { - for (Future future : await) - { - try { future.get(); } - catch (ExecutionException e) - { - if (ignore == null || !(ignore.isInstance(e.getCause()))) - throw e; - } - } - } -} diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index 52a2b85409..3d441ae04e 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -47,15 +47,16 @@ import accord.utils.async.Cancellable; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; -import org.apache.cassandra.service.accord.AccordExecutor; -import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; -import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutorAsyncSubmit; +import org.apache.cassandra.service.accord.execution.AccordExecutorSignalLoop; import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.simulator.test.AccordExecutorTest.Submitted.Consequences; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.SignalLock; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; // TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit public class AccordExecutorTest extends SimulationTestBase @@ -114,19 +115,38 @@ public class AccordExecutorTest extends SimulationTestBase return true; } - Collection> start() + class Consequences extends ConcurrentLinkedQueue> { - ConcurrentLinkedQueue> result = new ConcurrentLinkedQueue<>(); + volatile boolean started; - while (true) + void ensureStarted() { - int id = nextId.get(); - Object prev = consequences.putIfAbsent(id, result); - nextId.compareAndSet(id, id + 1); - if (prev == null) - return result; + if (started) + return; + + synchronized (this) + { + if (started) + return; + + while (true) + { + int id = nextId.get(); + Object prev = consequences.putIfAbsent(id, this); + nextId.compareAndSet(id, id + 1); + if (prev == null) + break; + } + + started = true; + } } } + + Consequences allocate() + { + return new Consequences(); + } } static class Control extends ConcurrentLinkedQueue @@ -148,19 +168,20 @@ public class AccordExecutorTest extends SimulationTestBase this.processChance = processChance; } - void submit(AsyncExecutor executor, Collection> consequences, Consumer>> run) + void submit(AsyncExecutor executor, Consequences consequences, Consumer run) { AsyncPromise future = new AsyncPromise<>(); + consequences.add(future); Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { if (fail == null) future.trySuccess(null); else { future.tryFailure(fail); if (fail instanceof CancellationException) - run.accept(submitted.start()); + run.accept(submitted.allocate()); } }); - consequences.add(future); + consequences.ensureStarted(); if (cancel != null && ThreadLocalRandom.current().nextFloat() <= cancelChance) { @@ -249,7 +270,7 @@ public class AccordExecutorTest extends SimulationTestBase List>> allAwaitSubmitted = new ArrayList<>(); for (int i = 0; i < innerLoop; ++i) { - Collection> awaitSubmitted = submitted.start(); + Consequences awaitSubmitted = submitted.allocate(); allAwaitSubmitted.add(awaitSubmitted); submitRecursive(lock, executor, sequentialExecutor, 1 + i, awaitSubmitted, awaitConsequences, submitted, sleepChance, lockChance, control); } @@ -264,7 +285,7 @@ public class AccordExecutorTest extends SimulationTestBase } } - private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) + private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Consequences consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) { AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index 6f836c0e86..cca5292f81 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -188,7 +188,7 @@ import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.accord.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCache; import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.transport.Event; diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index 69f3722bd0..97a9fb22ff 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -83,17 +83,14 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTestUtils; import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.utils.FBUtilities; -import static accord.local.ExecutionContext.contextFor; import static accord.local.ExecutionContext.unsequencedReadWrite; -import static accord.local.LoadKeys.SYNC; -import static accord.local.LoadKeysFor.READ_WRITE; import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE; import static accord.primitives.Routable.Domain.Range; import static accord.primitives.Timestamp.Flag.HLC_BOUND; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java index a8bbec8fdf..92081f8918 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java @@ -62,6 +62,9 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SaferCommand; +import org.apache.cassandra.service.accord.execution.SaferCommandsForKey; +import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializerTest.TestSafeCommandStore; import org.apache.cassandra.service.accord.txn.TxnDataResult; import org.apache.cassandra.service.accord.txn.TxnUpdate; @@ -71,15 +74,15 @@ import org.apache.cassandra.utils.Pair; import static accord.primitives.Status.Durability.AllQuorums; import static com.google.common.collect.Iterables.getOnlyElement; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; import static org.apache.cassandra.service.accord.AccordService.getBlocking; import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.preaccepted; import static org.apache.cassandra.service.accord.AccordTestUtils.ballot; import static org.apache.cassandra.service.accord.AccordTestUtils.createAccordCommandStore; import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn; -import static org.apache.cassandra.service.accord.AccordTestUtils.loaded; import static org.apache.cassandra.service.accord.AccordTestUtils.timestamp; import static org.apache.cassandra.service.accord.AccordTestUtils.txnId; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; public class AccordCommandStoreTest { @@ -138,8 +141,8 @@ public class AccordCommandStoreTest Command expected = Command.Executed.executed(txnId, SaveStatus.Applied, AllQuorums, StoreParticipants.all(route), promised, executeAt, txn, dependencies, accepted, waitingOn, result.left, TxnDataResult.PERSISTABLE); - AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null)); - safeCommand.preExecute(new AccordTask<>(null, ExecutionContext.unsequenced(txnId, "Test"), null), RELEASE_QUEUE); + SaferCommand safeCommand = new SaferCommand(loaded(txnId, null)); + safeCommand.preExecute(new SafeTask<>(null, ExecutionContext.unsequenced(txnId, "Test"), null), RELEASE_QUEUE); safeCommand.set(expected); // In practice we should never need to save it with the condition boolean set // Not sure why this test does that @@ -167,13 +170,13 @@ public class AccordCommandStoreTest Command command1 = preaccepted(txnId1, txn, timestamp(1, clock.incrementAndGet(), 1)); Command command2 = preaccepted(txnId2, txn, timestamp(1, clock.incrementAndGet(), 1)); - AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null)); - cfk.preExecute(new AccordTask<>(null, ExecutionContext.unsequenced(txnId1, "Test"), null), RELEASE_QUEUE); + SaferCommandsForKey cfk = new SaferCommandsForKey(loaded(key, null)); + cfk.preExecute(new SafeTask<>(null, ExecutionContext.unsequenced(txnId1, "Test"), null), RELEASE_QUEUE); cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command1).cfk()); cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command2).cfk()); - CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run(); + CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null).run(); logger.info("E: {}", cfk); CommandsForKey actual = CommandsForKeyAccessor.load(commandStore.id(), key); logger.info("A: {}", actual); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java b/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java index 66514033a5..2353fd5ac2 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java @@ -48,6 +48,7 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.journal.TestParams; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.journal.CommandChanges; import org.apache.cassandra.utils.AccordGenerators; @@ -70,7 +71,7 @@ import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.p * *

Mechanism under test: when the FULL-input cleanup path decides EXPUNGE, * {@link accord.impl.CommandChange.Builder#construct} returns {@code null}. - * Downstream, {@link AccordSafeCommand#preExecute} maps {@code null} to a + * Downstream, {@link SaferCommand#preExecute} maps {@code null} to a * {@code Command.NotDefined.uninitialised(...)} — which is exactly the bogus * NotDefined the user observed. The journal load API itself should never collapse * "this txnId has been erased" into the same answer as "we have never heard of this diff --git a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java index b467edce53..d73fdaa62b 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java @@ -70,6 +70,7 @@ import org.apache.cassandra.schema.SchemaProvider; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.utils.CassandraGenerators; import static accord.local.Command.Committed.committed; @@ -82,6 +83,7 @@ import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME; import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor.findAllKeysBetween; import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor.makeSystemTableKeyBytes; import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport; import static org.apache.cassandra.utils.AccordGenerators.fromQT; @@ -120,7 +122,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory Command.Committed committed = committed(id, SaveStatus.Committed, Status.Durability.NotDurable, participants, Ballot.ZERO, id, partialTxn, deps.intersecting(scope), Ballot.ZERO, waitingOn); - AccordSafeCommand safeCommand = new AccordSafeCommand(AccordTestUtils.loaded(id, null)); + SaferCommand safeCommand = new SaferCommand(loaded(id, null)); safeCommand.set(committed); AccordTestUtils.appendCommandsBlocking(store, null, committed); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index 297d13a79d..c8f9d7cb7b 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -57,7 +57,6 @@ import accord.local.Node; import accord.local.Node.Id; import accord.local.NodeCommandStoreService; import accord.local.SafeCommandStore; -import accord.local.SafeState; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.durability.DurabilityService; @@ -85,8 +84,6 @@ import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; import org.apache.cassandra.ServerTestUtils; -import org.apache.cassandra.concurrent.ExecutorPlus; -import org.apache.cassandra.concurrent.ManualExecutor; import org.apache.cassandra.config.AccordConfig; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; @@ -103,10 +100,12 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; -import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; -import org.apache.cassandra.service.accord.AccordExecutor.IOTask; import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutorSyncSubmit; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; @@ -123,9 +122,9 @@ import static accord.primitives.SaveStatus.PreAccepted; import static accord.primitives.Status.Durability.NotDurable; import static accord.primitives.Txn.Kind.Write; import static java.lang.String.format; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.service.accord.AccordService.getBlocking; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; public class AccordTestUtils { @@ -163,17 +162,10 @@ public class AccordTestUtils } } - public static & AccordSafeState> AccordCacheEntry loaded(K key, V value) + public static SaferCommand safeCommand(Command command) { - AccordCacheEntry global = new AccordCacheEntry<>(key, null); - global.initialize(value); - return global; - } - - public static AccordSafeCommand safeCommand(Command command) - { - AccordCacheEntry global = loaded(command.txnId(), command); - return new AccordSafeCommand(global); + AccordCacheEntry global = loaded(command.txnId(), command); + return new SaferCommand(global); } public static Function testableLoad(K key, V val) @@ -184,39 +176,6 @@ public class AccordTestUtils }; } - private static LoadExecutor loadExecutor(ExecutorPlus executor) - { - return new LoadExecutor<>() - { - @Override - public IOTask load(P1 p1, P2 p2, AccordCacheEntry entry) - { - executor.submit(() -> { - V v; - try { v = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); } - catch (Throwable t) - { - entry.failedToLoad(); - throw t; - } - entry.loaded(v); - }); - return null; - } - }; - } - - public static & AccordSafeState> void testLoad(ManualExecutor executor, S safeState, V val) - { - Assert.assertEquals(AccordCacheEntry.Status.WAITING_TO_LOAD, safeState.global().status()); - safeState.global().load(loadExecutor(executor), null, null); - Assert.assertEquals(AccordCacheEntry.Status.LOADING, safeState.global().status()); - executor.runOne(); - Assert.assertEquals(AccordCacheEntry.Status.LOADED, safeState.global().status()); - safeState.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); - Assert.assertEquals(val, safeState.current()); - } - public static TxnId txnId(long epoch, long hlc, int node) { return txnId(epoch, hlc, node, Write); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java similarity index 95% rename from test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java rename to test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java index 9ca2b11678..2da44ed37f 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java @@ -77,10 +77,15 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches; -import org.apache.cassandra.service.accord.AccordExecutor.ExclusiveGlobalCaches; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor.ExclusiveGlobalCaches; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SaferCommand; +import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Condition; @@ -95,12 +100,12 @@ import static org.apache.cassandra.service.accord.AccordService.getBlocking; import static org.apache.cassandra.service.accord.AccordTestUtils.createAccordCommandStore; import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn; import static org.apache.cassandra.service.accord.AccordTestUtils.keys; -import static org.apache.cassandra.service.accord.AccordTestUtils.loaded; import static org.apache.cassandra.service.accord.AccordTestUtils.txnId; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; -public class AccordTaskTest +public class SafeTaskTest { - private static final Logger logger = LoggerFactory.getLogger(AccordTaskTest.class); + private static final Logger logger = LoggerFactory.getLogger(SafeTaskTest.class); private static final AtomicLong clock = new AtomicLong(0); @BeforeClass @@ -173,7 +178,7 @@ public class AccordTaskTest private static Command createStableAndPersist(AccordCommandStore commandStore, TxnId txnId, Timestamp executeAt) { Command command = AccordTestUtils.Commands.stable(txnId, createPartialTxn(0), executeAt); - AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null)); + SaferCommand safeCommand = new SaferCommand(loaded(txnId, null)); safeCommand.set(command); appendDiffToLog(commandStore).accept(null, command); @@ -313,7 +318,7 @@ public class AccordTaskTest throw new NullPointerException("txn_id " + txnId); }); } - AccordTask o1 = AccordTask.create(commandStore, ctx, consumer); + SafeTask o1 = SafeTask.create(commandStore, ctx, consumer); AssertionUtils.assertThatThrownBy(() -> getBlocking(o1.chain())) .hasRootCause() .isInstanceOf(NullPointerException.class) @@ -334,7 +339,7 @@ public class AccordTaskTest return cmd; }); } - AccordTask o2 = AccordTask.create(commandStore, ctx, store -> { + SafeTask o2 = SafeTask.create(commandStore, ctx, store -> { ids.forEach(id -> { store.ifInitialised(id).readyToExecute(store); }); @@ -370,7 +375,7 @@ public class AccordTaskTest String errorMsg = "txn_ids " + ids; Mockito.doThrow(new NullPointerException(errorMsg)).when(consumer).accept(Mockito.any()); - AccordTask operation = AccordTask.create(commandStore, ctx, consumer); + SafeTask operation = SafeTask.create(commandStore, ctx, consumer); AssertionUtils.assertThatThrownBy(() -> getBlocking(operation.chain())) .hasRootCause() diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 00705915dd..165115d8fd 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -100,6 +100,10 @@ import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutorSimple; import org.apache.cassandra.service.accord.journal.RangeSearcher; import org.apache.cassandra.service.accord.journal.SegmentRangeSearcher; import org.apache.cassandra.service.accord.topology.AccordTopology; diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java index fcd7ccdde6..5da64a83c2 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java @@ -53,6 +53,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.accord.SimulatedAccordCommandStore.FunctionWrapper; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.utils.Pair; import static accord.utils.Property.qt; @@ -179,12 +180,12 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase private enum Action { SUCCESS, FAILURE, LOAD_FAILURE } - private static AccordTask operation(SimulatedAccordCommandStore instance, ExecutionContext ctx, Action action, BooleanSupplier delay) + private static SafeTask operation(SimulatedAccordCommandStore instance, ExecutionContext ctx, Action action, BooleanSupplier delay) { Function function = action == Action.FAILURE ? safeStore -> { throw new SimulatedFault("Operation failed for keys " + ctx.keys()); } : safeStore -> null; - return AccordTask.create(instance.commandStore, ctx, function); + return SafeTask.create(instance.commandStore, ctx, function); } private static class Counter implements BiConsumer diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheEntryTest.java similarity index 88% rename from test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java rename to test/unit/org/apache/cassandra/service/accord/execution/AccordCacheEntryTest.java index 4c174a37b7..4f61205a65 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java +++ b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheEntryTest.java @@ -15,24 +15,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import org.junit.Assert; import org.junit.Test; import accord.local.SafeState; -import org.apache.cassandra.service.accord.AccordCache.Type; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.execution.AccordCache.Type; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; public class AccordCacheEntryTest { - static class TestSafeState extends SafeState implements AccordSafeState + static class TestSafeState extends SafeState implements SaferState { @Override public AccordCacheEntry global() { return null; } - @Override public void preExecute(AccordTask owner, LockMode lockMode) {} - @Override public void postExecute(AccordTask owner) {} + @Override public void preExecute(SafeTask owner, LockMode lockMode) {} + @Override public void postExecute(SafeTask owner) {} } static class CacheEntry extends AccordCacheEntry diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheTest.java similarity index 95% rename from test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java rename to test/unit/org/apache/cassandra/service/accord/execution/AccordCacheTest.java index aedf100df0..669d25fe30 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java +++ b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.function.Function; @@ -30,12 +30,12 @@ import org.apache.cassandra.cache.CacheSize; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.ManualExecutor; import org.apache.cassandra.metrics.AccordCacheMetrics; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; -import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.SaveExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; -import static org.apache.cassandra.service.accord.AccordTestUtils.testLoad; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.testLoad; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.mockito.Mockito.mock; @@ -45,7 +45,7 @@ public class AccordCacheTest { private static final long DEFAULT_NODE_SIZE = nodeSize(0); - private static abstract class TestSafeState & AccordSafeState> extends SafeState implements AccordSafeState + private static abstract class TestSafeState & SaferState> extends SafeState implements SaferState { protected final AccordCacheEntry global; @@ -61,7 +61,7 @@ public class AccordCacheTest public final T key() { return global.key(); } - public void preExecute(AccordTask owner, LockMode lockMode) + public void preExecute(SafeTask owner, LockMode lockMode) { requireUninitialised(); current = global.lockExclusive(owner, lockMode); @@ -77,7 +77,7 @@ public class AccordCacheTest } @Override - public void postExecute(AccordTask owner) + public void postExecute(SafeTask owner) { global.releaseExclusive(this, owner); } @@ -91,7 +91,7 @@ public class AccordCacheTest } @Override - public void postExecute(AccordTask owner) + public void postExecute(SafeTask owner) { global.releaseExclusive(this, owner); } @@ -240,7 +240,7 @@ public class AccordCacheTest assertCacheMetrics(cacheMetrics, 0, 3, 3, 3); SafeString safeString = instance.acquire("1"); - safeString.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); + safeString.preExecute(new SafeTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals(Status.LOADED, safeString.global.status()); assertCacheState(cache, 1, 3, nodeSize(1) * 3); @@ -369,7 +369,7 @@ public class AccordCacheTest assertCacheState(cache, 1, 1, nodeSize(1)); SafeString safeString2 = instance.acquire("0"); - safeString2.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); + safeString2.preExecute(new SafeTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals("0", safeString2.current()); Assert.assertEquals(Status.LOADED, safeString1.global.status()); Assert.assertEquals(2, instance.references("0", SafeString.class)); diff --git a/test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java b/test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java new file mode 100644 index 0000000000..1a92a205d9 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.junit.Assert; + +import accord.local.ExecutionContext; +import accord.local.SafeState; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.concurrent.ManualExecutor; + +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; + +public class AccordExecutionTestUtils +{ + public static & SaferState> AccordCacheEntry loaded(K key, V value) + { + AccordCacheEntry global = new AccordCacheEntry<>(key, null); + global.initialize(value); + return global; + } + + private static AccordCacheEntry.LoadExecutor loadExecutor(ExecutorPlus executor) + { + return new AccordCacheEntry.LoadExecutor<>() + { + @Override + public IOTask load(P1 p1, P2 p2, AccordCacheEntry entry) + { + executor.submit(() -> { + V v; + try { v = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); } + catch (Throwable t) + { + entry.failedToLoad(); + throw t; + } + entry.loaded(v); + }); + return null; + } + }; + } + + public static & SaferState> void testLoad(ManualExecutor executor, S safeState, V val) + { + Assert.assertEquals(AccordCacheEntry.Status.WAITING_TO_LOAD, safeState.global().status()); + safeState.global().load(loadExecutor(executor), null, null); + Assert.assertEquals(AccordCacheEntry.Status.LOADING, safeState.global().status()); + executor.runOne(); + Assert.assertEquals(AccordCacheEntry.Status.LOADED, safeState.global().status()); + safeState.preExecute(new SafeTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); + Assert.assertEquals(val, safeState.current()); + } +} From c4b0257b3879f4010d31b3d05d438965fc74b107 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Tue, 28 Jul 2026 23:29:38 +0100 Subject: [PATCH 06/10] Refactor AccordExecutor/Task responsibilities, in particular: - fix race conditions when updating Task.info (between running and Exclusive methods) - do not run consequences of actions that fail and do not become durable (e.g. applying to data store/post-applying a transaction that we failed to complete the PreApplied/Applying step for) --- modules/accord | 2 +- .../db/virtual/AccordDebugKeyspace.java | 2 +- .../apache/cassandra/gms/FailureDetector.java | 4 + .../cassandra/metrics/AccordCacheMetrics.java | 2 +- .../metrics/LogLinearDecayingHistograms.java | 5 + .../service/accord/AccordCommandStores.java | 8 +- .../service/accord/AccordDataStore.java | 2 +- .../service/accord/AccordService.java | 10 +- .../service/accord/api/AccordAgent.java | 12 +- .../service/accord/debug/DebugExecution.java | 23 +- .../accord/execution/AbstractLockLoop.java | 76 +- .../accord/execution/AbstractLoop.java | 6 +- .../service/accord/execution/AccordCache.java | 13 +- .../accord/execution/AccordCacheEntry.java | 14 +- .../execution/AccordCacheEntryQueue.java | 28 +- .../accord/execution/AccordExecutor.java | 160 ++-- .../AccordExecutorSemiSyncSubmit.java | 5 +- .../execution/AccordExecutorSignalLoop.java | 48 +- .../execution/AccordExecutorSimple.java | 20 +- .../service/accord/execution/CancelTask.java | 24 +- .../accord/execution/ExclusiveExecutor.java | 122 ++-- .../service/accord/execution/IOTask.java | 26 - .../service/accord/execution/IOTaskLoad.java | 32 +- .../service/accord/execution/IOTaskSave.java | 29 +- .../accord/execution/IOTaskWrapper.java | 33 +- .../service/accord/execution/Plain.java | 83 +-- .../service/accord/execution/PlainChain.java | 64 +- .../execution/PlainChainDebuggable.java | 24 - .../accord/execution/PlainRunnable.java | 45 +- .../service/accord/execution/SafeTask.java | 681 ++++++++++-------- .../accord/execution/SaferCommandStore.java | 16 +- .../service/accord/execution/Task.java | 568 ++++++++++----- .../service/accord/execution/TaskInfo.java | 8 +- .../service/accord/execution/TaskQueue.java | 11 +- .../accord/execution/TaskQueueMulti.java | 37 +- .../accord/execution/TaskQueueRunnable.java | 13 +- .../accord/execution/TaskQueueStandalone.java | 24 +- .../apache/cassandra/utils/NoSpamLogger.java | 161 ++++- .../CompactionAccordIteratorsTest.java | 4 +- .../service/accord/AccordCommandTest.java | 4 +- .../service/accord/AccordTestUtils.java | 2 +- .../service/accord/SafeTaskTest.java | 15 +- .../accord/execution/TaskLifecycleTest.java | 529 ++++++++++++++ 43 files changed, 1914 insertions(+), 1081 deletions(-) create mode 100644 test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java diff --git a/modules/accord b/modules/accord index 8d2cdb35ae..4d091037c7 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 8d2cdb35ae96205b44da16582797bf42aa6fddcc +Subproject commit 4d091037c7b751631d6ce204e0c192e2301162da diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 6e2fad78b4..5b3e9f79e5 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -1795,7 +1795,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace SafeCommand safeCommand = safeStore.unsafeGet(txnId); Command command = safeCommand.current(); if (command.saveStatus() == SaveStatus.Applying) - return Commands.applyChain(safeStore, command); + return Commands.applyChain(safeStore, command.asExecuted()); Commands.maybeExecute(safeStore, safeCommand, command, true, true, NotifyWaitingOnPlus.adapter(ignore -> {}, true, true)); return AsyncChains.success(null); }); diff --git a/src/java/org/apache/cassandra/gms/FailureDetector.java b/src/java/org/apache/cassandra/gms/FailureDetector.java index 13d6266ca0..eaa73d09cc 100644 --- a/src/java/org/apache/cassandra/gms/FailureDetector.java +++ b/src/java/org/apache/cassandra/gms/FailureDetector.java @@ -377,6 +377,10 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean logger.debug("Still not marking nodes down due to local pause"); return; } + + if (!isAlive(ep)) + return; // don't convict nodes that are already down - this helps Accord on startup which doesn't report itself alive in Gossip until ready to serve traffic + double phi = hbWnd.phi(now); logger.trace("PHI for {} : {}", ep, phi); diff --git a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java index 00627e019a..a9c5c4c28e 100644 --- a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java @@ -46,7 +46,7 @@ public class AccordCacheMetrics public AccordCacheGlobalMetrics() { DefaultNameFactory factory = new DefaultNameFactory(ACCORD_CACHE); - this.usedBytes = Metrics.gauge(factory.createMetricName("UsedBytes"), fromAccordService(sumExecutors(executor -> executor.cacheUnsafe().weightedSize()), 0L)); + this.usedBytes = Metrics.gauge(factory.createMetricName("UsedBytes"), fromAccordService(sumExecutors(AccordExecutor::weightedSize), 0L)); this.unreferencedBytes = Metrics.gauge(factory.createMetricName("UnreferencedBytes"), fromAccordService(sumExecutors(executor -> executor.cacheUnsafe().unreferencedBytes()), 0L)); } diff --git a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java index 823346917a..4a17794827 100644 --- a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java +++ b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java @@ -73,6 +73,11 @@ public class LogLinearDecayingHistograms buffer[bufferCount++] = value; } + public void clear() + { + bufferCount = 0; + } + public void flush(long at) { for (int i = 0 ; i < bufferCount ; ++i) diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index 228030ce39..576fdf482b 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -38,6 +38,7 @@ import accord.api.ProgressLog; import accord.local.CommandStores; import accord.local.NodeCommandStoreService; import accord.local.ShardDistributor; +import accord.utils.Invariants; import accord.utils.RandomSource; import accord.utils.Reduce; import accord.utils.async.AsyncChain; @@ -63,6 +64,7 @@ import org.apache.cassandra.service.accord.execution.AccordExecutorSimple; import org.apache.cassandra.service.accord.execution.AccordExecutorSyncSubmit; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD; +import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD_SYNC_QUEUE; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; @@ -123,19 +125,21 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu break; case EXEC_ST: factory = AccordExecutorSimple::new; + Invariants.require(executors.length == 1, "Executors that hold their lock during execution cannot support multiple executor shards, as they cannot lock multiple executors at once"); maxThreads = 1; break; } + QueueShardModel shardModel = config.queue_shard_model; + Invariants.require(shardModel != THREAD_PER_SHARD_SYNC_QUEUE || executors.length == 1, "Executors that hold their lock during execution cannot support multiple executor shards, as they cannot lock multiple executors at once"); for (int id = 0; id < executors.length; id++) { - QueueShardModel shardModel = config.queue_shard_model; String baseName = AccordExecutor.class.getSimpleName() + '[' + id; switch (shardModel) { case THREAD_PER_SHARD: case THREAD_PER_SHARD_SYNC_QUEUE: - executors[id] = factory.get(id, shardModel == THREAD_PER_SHARD ? RUN_WITHOUT_LOCK : RUN_WITH_LOCK, 1, constant(baseName + ']'), agent); + executors[id] = factory.get(id, shardModel == THREAD_PER_SHARD_SYNC_QUEUE ? RUN_WITH_LOCK : RUN_WITHOUT_LOCK, 1, constant(baseName + ']'), agent); break; case THREAD_POOL_PER_SHARD: int threads = Math.min(maxThreads, Math.max(DatabaseDescriptor.getAccordConcurrentOps() / executors.length, 1)); diff --git a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java index 7eba765c5b..0e757ba418 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java @@ -165,7 +165,7 @@ public class AccordDataStore implements DataStore break; case ABORT: case ERROR: - IllegalStateException ex = new IllegalStateException(String.format("Repair failed: %s", event)); + RuntimeException ex = new RuntimeException(String.format("Repair failed (%s):", event.getType(), event.getMessage())); callback.fail(ranges, ex); syncResult.tryFailure(ex); break; diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 994a553e8e..19a0f9da09 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -385,6 +385,11 @@ public class AccordService implements IAccordService, Shutdownable AccordService as = new AccordService(tcmIdToAccord(tcmId)); unsafeInstance = replyInstance = as; as.localStartup(); + + AccordReplicaMetrics.touch(); + AccordSystemMetrics.touch(); + AccordExecutorMetrics.touch(); + AccordViolationHandler.setup(); } } @@ -398,11 +403,6 @@ public class AccordService implements IAccordService, Shutdownable return as; as.distributedStartupInternal(); - - AccordReplicaMetrics.touch(); - AccordSystemMetrics.touch(); - AccordExecutorMetrics.touch(); - AccordViolationHandler.setup(); return as; } diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index 454fa9edbe..3188fe0029 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -43,6 +43,7 @@ import accord.coordinate.Exhausted; import accord.coordinate.Preempted; import accord.coordinate.Timeout; import accord.local.Command; +import accord.local.CommandStore; import accord.local.LogUnavailableException; import accord.local.Node; import accord.local.SafeCommand; @@ -84,6 +85,7 @@ import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.NoSpamLogger; +import org.apache.cassandra.utils.NoSpamLogger.NoDuplicateSpamLogStatement; import static accord.primitives.Routable.Domain.Key; import static accord.utils.SortedArrays.SortedArrayList.ofSorted; @@ -107,12 +109,14 @@ import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retry import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowRead; import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowTxnPreaccept; import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.txn_data; +import static org.apache.cassandra.utils.NoSpamLogger.NoDuplicateSpamLogStatement.exceptionId; // TODO (expected): merge with AccordService public class AccordAgent implements Agent, OwnershipEventListener { private static final Logger logger = LoggerFactory.getLogger(AccordAgent.class); private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, MINUTES); + private static final NoDuplicateSpamLogStatement noSpamException = new NoDuplicateSpamLogStatement(logger, "", 1L, MINUTES); private static final ReplicaEventListener replicaEventListener = new AccordReplicaMetrics.Listener(); private static BiConsumer onFailedBarrier; @@ -155,6 +159,12 @@ public class AccordAgent implements Agent, OwnershipEventListener config = DatabaseDescriptor.getAccord(); } + @Override + public void onSuccessfulBootstrap(CommandStore commandStore, int attempt, long epoch, Ranges ranges) + { + logger.info("{}: Completed bootstrap of {} on epoch {}", commandStore, ranges, epoch); + } + @Override public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Runnable fail, Throwable failure) { @@ -214,7 +224,7 @@ public class AccordAgent implements Agent, OwnershipEventListener AccordSystemMetrics.metrics.errors.inc(); if (t instanceof CancellationException || t instanceof TimeoutException || t instanceof Timeout || t instanceof Preempted || t instanceof Exhausted || t instanceof LogUnavailableException) // TODO (required): leaky logger, permitting multiple messages per time period and reporting how many were dropped - noSpamLogger.warn("", t); + noSpamException.warn(exceptionId(t), t); else JVMStabilityInspector.uncaughtException(Thread.currentThread(), t); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java index 99aa8fe0ce..e01a8363d8 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java @@ -132,15 +132,16 @@ public class DebugExecution public void onComplete(Task completed) { long readyAt = setTaskAt; + long runningAt = DebugTask.get(completed).runningAt; if (waitingAt > setTaskAt) { readyAt = waitingAt; - long waitingMicros = (completed.runningAt - waitingAt)/1000; + long waitingMicros = (runningAt - waitingAt)/1000; owner.sequentialExecutorWaitingToRunLatency.increment(waitingMicros); if (waitingMicros > REPORT_MAX_LATENCY_MICROS) report("{} spent {}us blocked by a direct execution on queue {}", completed, waitingMicros, commandStoreId); } - long atHeadMicros = (completed.runningAt - readyAt)/1000; + long atHeadMicros = (runningAt - readyAt)/1000; owner.sequentialExecutorSetHeadToRunLatency.increment(atHeadMicros); if (atHeadMicros > REPORT_MAX_LATENCY_MICROS) { @@ -172,7 +173,7 @@ public class DebugExecution } public List sanityCheck; // for AccordTask only - long polledAt, preRunAt, runCompleteAt, completedAt; + long polledAt, preRunAt, runningAt, runCompleteAt, completeAt, completedAt; long releasedRangeScannerAt, releasedStateAt; long runningAtCpu, runCompleteAtCpu; Thread thread; @@ -191,6 +192,7 @@ public class DebugExecution { thread = Thread.currentThread(); runningAtCpu = nowCpu(); + runningAt = nanoTime(); } public void onRunComplete() @@ -209,22 +211,27 @@ public class DebugExecution releasedStateAt = nanoTime(); } + public void onComplete() + { + completeAt = nanoTime(); + } + public void onCompleted(DebugExecutor owner) { completedAt = nanoTime(); - if (task.runningAt > 0 && polledAt > 0) + if (runningAt > 0 && polledAt > 0) { - long pollToRunMicros = (task.runningAt - polledAt)/1000; + long pollToRunMicros = (runningAt - polledAt)/1000; owner.pollToRun.increment(pollToRunMicros); long runningMicros = -1; if (runCompleteAt > 0) { - runningMicros = (runCompleteAt - task.runningAt) / 1000; + runningMicros = (runCompleteAt - runningAt) / 1000; owner.running.increment(runningMicros); } - long runToCleanMicros = (task.completeAt - runCompleteAt) / 1000; + long runToCleanMicros = (completeAt - runCompleteAt) / 1000; owner.runToCleanup.increment(runToCleanMicros); - long cleanupMicros = (completedAt - task.completeAt) / 1000; + long cleanupMicros = (completedAt - completeAt) / 1000; owner.cleanup.increment(cleanupMicros); long totalMicros = (completedAt - polledAt)/1000; owner.taskTotal.increment(totalMicros); diff --git a/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java index ac7e4390a0..41738b8dd3 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java @@ -142,7 +142,7 @@ abstract class AbstractLockLoop extends AbstractLoop while (cur != null) { Task next = cur.next; - cur.submitExclusive(); + cur.submitExclusiveNoExcept(); cur.next = null; cur = next; } @@ -209,28 +209,7 @@ abstract class AbstractLockLoop extends AbstractLoop while (true) { task = pollWaitingToRunExclusive(); - - if (task != null) - { - self.setAccordActiveTask(task); - boolean executed = false; - try - { - task.preRunExclusive(); - executed = true; - task.run(); - } - catch (Throwable t) - { - executed = false; - task.failExecution(t); - } - finally - { - cleanupTaskExclusive(task, executed); - self.setAccordActiveTask(null); - } - } + if (task != null) prepareRunComplete(self, task); else { if (shutdown) @@ -250,7 +229,6 @@ abstract class AbstractLockLoop extends AbstractLoop catch (Throwable t) { exitLockLoop(); - try { agent.onException(t); } catch (Throwable t2) { } } @@ -285,18 +263,20 @@ abstract class AbstractLockLoop extends AbstractLoop { Task tmp = task; task = null; - cleanupTaskExclusive(tmp, true); - self.setAccordActiveTask(null); + tmp.completeExclusiveNoExcept(); } while (true) { task = pollWaitingToRunExclusive(); - if (task != null) { - self.setAccordActiveTask(task); - task.preRunExclusive(); + if (!task.prepareExclusiveNoExcept()) + { + task = null; + continue; + } + if (DEBUG_EXECUTION) debug.onExitLock(); exitLockLoop(); break; @@ -319,21 +299,8 @@ abstract class AbstractLockLoop extends AbstractLoop } catch (Throwable t) { - if (task != null) - { - try { task.failExecution(t); } - catch (Throwable t2) { t.addSuppressed(t2); } - try { cleanupTaskExclusive(task, false); } - catch (Throwable t2) { t.addSuppressed(t2); } - try { agent.onException(t); } - catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ } - task = null; - } - else - { - try { agent.onException(t); } - catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ } - } + try { agent.onException(t); } + catch (Throwable t2) { } exitLockLoop(); continue; } @@ -343,26 +310,7 @@ abstract class AbstractLockLoop extends AbstractLoop unlock(self); } - try - { - task.run(); - } - catch (Throwable t) - { - try { task.failExecution(t); } - catch (Throwable t2) - { - try - { - t2.addSuppressed(t); - agent.onException(t2); - } - catch (Throwable t3) - { - // empty to ensure we definitely loop so we cleanup the task - } - } - } + task.runNoExcept(self); } } }; diff --git a/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java index 684160b870..de6d8020f7 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java @@ -27,6 +27,8 @@ import accord.utils.Invariants; import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; +import static org.apache.cassandra.service.accord.execution.Task.State.REGISTERED; + abstract class AbstractLoop extends AccordExecutor { volatile Task unqueued; @@ -84,8 +86,8 @@ abstract class AbstractLoop extends AccordExecutor Invariants.require(cur != null); Task next = cur.next; cur.next = null; - if (cur.is(Task.State.UNINITIALIZED)) cur.submitExclusive(); - else cleanupTaskExclusive(cur, true); + if (cur.compareTo(REGISTERED) < 0) cur.submitExclusiveNoExcept(); + else cur.completeExclusiveNoExcept(); return next; } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java index 8a5c2f6991..21248920c6 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java @@ -102,7 +102,7 @@ import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Sta * * TODO (required): we only iterate over unreferenced entries */ -public class AccordCache implements CacheSize +public class AccordCache { private static final Logger logger = LoggerFactory.getLogger(AccordCache.class); private static final NoSpamLogStatement evictNoEvict = NoSpamLogger.getStatement(logger, "Found and expired {} marked no evict, with age {}, exceeding its expected max age of {}", 1L, TimeUnit.MINUTES); @@ -180,8 +180,8 @@ public class AccordCache implements CacheSize } // note: only affects current contents after lock is released - @Override - public void setCapacity(long sizeInBytes) + // should NOT be called directly, should be called via AccordExecutor.setCapacity, so it may refresh its derived values + void setCapacity(long sizeInBytes) { maxSizeInBytes = sizeInBytes; tryShrinkOrEvict = true; @@ -192,7 +192,6 @@ public class AccordCache implements CacheSize this.shrinkingOn = shrinkingOn; } - @Override public long capacity() { return maxSizeInBytes; @@ -945,14 +944,12 @@ public class AccordCache implements CacheSize return unreferencedBytes; } - @Override - public int size() + int size() { return cacheSize(); } - @Override - public long weightedSize() + long weightedSize() { return bytesCached; } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java index fefaad70a6..43ad91fedc 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java @@ -283,6 +283,7 @@ public class AccordCacheEntry & SaferState status &= ~LOCKED_MASK; q.unlock(task); } + else if (isLoading()) remove = true; else if (task.isUnsequenced(this)) { if (task.isCacheQueued()) @@ -326,7 +327,14 @@ public class AccordCacheEntry & SaferState { Invariants.require(!ownsLock); if (task.isUnsequenced(this) && task.isCacheQueued()) - --unsequenced; // nothing to release if we hit zero + { + if (--unsequenced == 0 && queue != null && isLoaded()) + { + SafeTask head = (SafeTask) queue; + if (head.holdsLocksBetweenRuns()) + head.onChangeOptionalHeadStatus(this, NEWLY_RUNNABLE); + } + } } } @@ -479,7 +487,7 @@ public class AccordCacheEntry & SaferState private void onChangedHead(AccordCacheEntryQueue q, @Nullable SafeTask notifyNewHead, @Nullable SafeTask notifyPrevHead) { if (notifyNewHead != null && isRunnable(notifyNewHead)) - notifyNewHead.onChangeHeadStatus(this, q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); + notifyNewHead.onChangeHeadStatus(this, q.totalSize() <= 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); if (notifyPrevHead != null && isRunnable(notifyPrevHead)) notifyPrevHead.onChangeHeadStatus(this, NOT_RUNNABLE); } @@ -583,7 +591,7 @@ public class AccordCacheEntry & SaferState if (--unsequenced == 0) { SafeTask head = q.peek(); - if (head != null && head.holdsLocksBetweenRuns()) + if (head != null && head.holdsLocksBetweenRuns() && isLoaded()) onChangedHead(q, head, null); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java index 9e4b60df7d..06025e49c7 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java @@ -31,6 +31,8 @@ import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Run class AccordCacheEntryQueue { + static final AccordCacheEntryQueue EMPTY = new AccordCacheEntryQueue(0); + private static final int DEFAULT_CAPACITY = 4; static final int LOCKED_INDEX = 0; static final int PRIORITY_START_INDEX = LOCKED_INDEX + 1; @@ -49,9 +51,14 @@ class AccordCacheEntryQueue public AccordCacheEntryQueue() { - tasks = new SafeTask[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + public AccordCacheEntryQueue(int capacity) + { + tasks = new SafeTask[capacity]; priorityHead = priorityTail = PRIORITY_START_INDEX; - fifoHead = fifoTail = DEFAULT_CAPACITY - 1; + fifoHead = fifoTail = capacity - 1; } AccordCacheEntryQueue(AccordCacheEntryQueue copy) @@ -412,18 +419,7 @@ class AccordCacheEntryQueue if (tasks[priorityHead] == task) return priorityHead; - int i = SortedArrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, AccordCacheEntryQueue::compare, SortedArrays.Search.CEIL); - if (i < 0) - return -1; - - while (i < priorityTail) - { - if (tasks[i] == task) - return i; - if (compare(task, tasks[i]) != 0) - break; - ++i; - } + return Arrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, AccordCacheEntryQueue::compare); } for (int i = priorityHead; i < priorityTail; ++i) @@ -504,10 +500,6 @@ class AccordCacheEntryQueue c = a.executionContext().executionKind().compareTo(b.executionContext().executionKind()); if (c == 0) c = Long.compare(a.createdAt, b.createdAt); - if (c == 0) - c = Long.compare(a.loadedAt, b.loadedAt); - if (c == 0) - c = a.loggingId().compareTo(b.loggingId()); return c; } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java index a87e405dd2..37ba493d32 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java @@ -21,6 +21,7 @@ package org.apache.cassandra.service.accord.execution; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; @@ -39,7 +40,6 @@ import accord.local.cfk.CommandsForKey; import accord.primitives.TxnId; import accord.utils.ArrayBuffers; import accord.utils.Invariants; -import accord.utils.TinyEnumSet; import accord.utils.UnhandledEnum; import accord.utils.async.AsyncCallbacks.CallAndCallback; import accord.utils.async.AsyncCallbacks.RunOrFail; @@ -70,8 +70,8 @@ import org.apache.cassandra.service.accord.execution.AccordCacheEntry.UniqueSave import org.apache.cassandra.service.accord.execution.ExclusiveExecutor.ExclusiveExecutorTask; import org.apache.cassandra.service.accord.execution.IOTaskWrapper.WrappableIOTask; import org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup; +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; import org.apache.cassandra.service.accord.execution.Task.GlobalGroup; -import org.apache.cassandra.service.accord.execution.Task.State; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Condition; @@ -86,13 +86,11 @@ import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOA import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.OTHER; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_SCAN; -import static org.apache.cassandra.service.accord.execution.Task.State.FAILED_TO_LOAD; -import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; -import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; -import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; -import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED; +import static org.apache.cassandra.service.accord.execution.Task.State.UNREGISTERED; import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.COUNTER_MASKS; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.overflowBit; import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.selectByOverflowBits; import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.setOverflowWhenLessEqual; @@ -113,6 +111,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor> scanningRanges = new TaskQueueStandalone<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning - final TaskQueueStandalone> loading = new TaskQueueStandalone<>(TinyEnumSet.encode(LOADING_REQUIRED, LOADING_OPTIONAL)); - final TaskQueueStandalone> waitingOnCacheQueues = new TaskQueueStandalone<>(TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL)); + final TaskQueueStandalone> loading = new TaskQueueStandalone<>(ExecutorQueue.LOADING); + final TaskQueueStandalone> waiting = new TaskQueueStandalone<>(ExecutorQueue.WAITING); final TaskQueueRunnable runnable = new TaskQueueRunnable<>(); private final Tranches tranches = new Tranches(this); @@ -409,17 +407,17 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor= maxWorkingCapacityInBytes || !runnable.hasWaitingToRun()) + if (!hasPausedLoading && cache.weightedSize() >= maxWorkingCapacityInBytes && hasNonLoadingWaitingToRun()) { AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); hasPausedLoading = true; - runnable.stop(RANGE_SCAN.ordinal()); - runnable.stop(LOAD.ordinal()); - runnable.stop(RANGE_LOAD.ordinal()); + runnable.stop(LOADING_GROUPS); } } @@ -452,46 +448,48 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor IOTaskLoad load(SafeTask parent, Boolean isForRange, AccordCacheEntry entry) { IOTaskLoad result = newLoad(entry, isForRange); - result.submitExclusive(null, parent); + result.inherit(parent).submitExclusiveNoExcept(); return result; } @Override public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) { - return new IOTaskSave(this, entry, identity, save).submitExclusive(null, null); + IOTaskSave task = new IOTaskSave(this, entry, identity, save); + task.submitExclusiveNoExcept(); + return task; } - Cancellable submit(Task task) + void submitTask(Task task) { - submit(Task::submitExclusive, i -> i, task); - return task; + submit(Task::submitExclusiveNoExcept, i -> i, task); } public Future submit(Runnable run) { Task inherit = inherit(); - PlainRunnable task = inherit == null ? new PlainRunnable(this, new AsyncPromise<>(), run, OTHER) - : new PlainRunnable(this, new AsyncPromise<>(), run, OTHER, inherit.position, inherit.tranche()); - submit(task); + PlainRunnable task = new PlainRunnable(this, new AsyncPromise<>(), run, OTHER); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); return task.result; } public void execute(Runnable run) { Task inherit = inherit(); - PlainRunnable task = inherit == null ? new PlainRunnable(this, null, run, OTHER) - : new PlainRunnable(this, null, run, OTHER, inherit.position, inherit.tranche()); - submit(task); + PlainRunnable task = new PlainRunnable(this, null, run, OTHER); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); } @Override public Cancellable execute(RunOrFail runOrFail) { Task inherit = inherit(); - PlainChain submit = inherit == null ? new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER) - : new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - return submit(submit); + PlainChain task = new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); + return task; } public AsyncChain buildDebuggable(Callable call, Object describe) @@ -502,30 +500,29 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor callback) { Task inherit = inherit(); - return submit(inherit == null ? new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, describe) - : new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, inherit.position, inherit.tranche(), describe)); + PlainChainDebuggable task = new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, describe); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); + return task; } }; } public void submitExclusive(Runnable runnable) { - new PlainRunnable(this, null, runnable, OTHER).submitExclusive(null, null); + new PlainRunnable(this, null, runnable, OTHER).submitExclusiveNoExcept(); } - Cancellable submitExclusive(Task parent, GlobalGroup group, WrappableIOTask task) + Cancellable submitExclusive(Task parent, GlobalGroup group, WrappableIOTask wrap) { - return new IOTaskWrapper(this, task, group, parent.position, parent.tranche()).submitExclusive(null, parent); + IOTaskWrapper task = new IOTaskWrapper(this, wrap, group); + task.inherit(parent).submitExclusiveNoExcept(); + return task; } Task inherit() { - return inherit(Thread.currentThread()); - } - - Task inherit(Thread thread) - { - TaskRunner self = TaskRunner.get(thread); + TaskRunner self = TaskRunner.get(); if (self.accordActiveExecutor() != this) return null; @@ -536,17 +533,10 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor= 0); + if (DEBUG_EXECUTION) DebugTask.get(task).onCompleted(debug); + unregisterExclusive(task); runnable.cleanup(task); - try { task.cleanupExclusive(this, executed); } - finally - { - cache.tryShrinkOrEvict(lock); - maybeUnpauseLoading(); - } + cache.tryShrinkOrEvict(lock); + maybeUnpauseLoading(); } final void unregisterExclusive(Task task) @@ -594,25 +583,13 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor task, Throwable fail) { // the task may have already been cancelled, in which case we don't need to fail it - if (task.state().isExecuted()) + if (task.state().isDone()) return; - if (fail != null) failExclusive(task, fail, FAILED_TO_LOAD); - else task.rangeScanner().scannedExclusive(); - } - - private void failExclusive(SafeTask task, Throwable fail, State newState) - { - if (task.state().isExecuted()) - return; - - try { task.failExclusive(fail, newState); } - catch (Throwable t) { agent.onException(t); } - finally - { - task.unqueueIfQueued(); - cleanupTaskExclusive(task, false); - } + SafeTask.RangeTxnScanner scanner = task.rangeScanner(); + if (scanner == null && fail == null) fail = new CancellationException(); + if (fail != null) task.tryFailAndCompleteExclusive(fail, FAILED); + else scanner.scannedExclusive(); } void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) @@ -630,7 +607,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor task : tasks) - failExclusive(task, fail, FAILED_TO_LOAD); + task.tryFailAndCompleteExclusive(fail, FAILED); cache.failedToLoad(loaded); } else @@ -666,13 +643,18 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor name, Agent agent) { @@ -55,9 +54,7 @@ public class AccordExecutorSemiSyncSubmit extends AbstractSemiSyncSubmit private void awaitWork() throws InterruptedException { - waiting++; - try { hasWork.await(); } - finally { waiting--; } + hasWork.await(); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java index acb5eb7202..51275ee4fe 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java @@ -34,7 +34,7 @@ import org.apache.cassandra.utils.concurrent.SignalLock; import static accord.utils.Invariants.nonNull; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; -import static org.apache.cassandra.service.accord.execution.Task.State.UNINITIALIZED; +import static org.apache.cassandra.service.accord.execution.Task.State.REGISTERED; public class AccordExecutorSignalLoop extends AbstractLoop { @@ -134,7 +134,7 @@ public class AccordExecutorSignalLoop extends AbstractLoop { Task next = submit.next; submit.next = null; - submit.submitExclusive(); + submit.submitExclusiveNoExcept(); submit = next; } } @@ -151,7 +151,7 @@ public class AccordExecutorSignalLoop extends AbstractLoop pendingExecutedHead = destructiveNext(executed); if (pendingExecutedHead == null) pendingExecutedTail = null; - cleanupTaskExclusive(executed, true); + executed.completeExclusiveNoExcept(); } else { @@ -159,7 +159,7 @@ public class AccordExecutorSignalLoop extends AbstractLoop pendingNewHead = destructiveNext(submit); if (pendingNewHead == null) pendingNewTail = null; - submit.submitExclusive(); + submit.submitExclusiveNoExcept(); } return true; } @@ -227,19 +227,9 @@ public class AccordExecutorSignalLoop extends AbstractLoop return; } } - else + else if (task.prepareExclusiveNoExcept()) { - try { task.preRunExclusive(); } - catch (Throwable t) - { - try { task.failExclusive(t, Task.State.FAILED_OTHER); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - try { cleanupTaskExclusive(task, false); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - continue; - } if (DEBUG_EXECUTION) DebugTask.get(task).onPreRun(); - addReadyToRun(task); boolean incremented = lock.incrementAsyncWork(false); Invariants.require(incremented); @@ -260,7 +250,7 @@ public class AccordExecutorSignalLoop extends AbstractLoop while (cur != null) { Task next = cur.next; - if (cur.is(UNINITIALIZED)) + if (cur.compareTo(REGISTERED) < 0) { if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); else addNewHead = reverseOne(addNewHead, cur); @@ -368,7 +358,8 @@ public class AccordExecutorSignalLoop extends AbstractLoop try { - cleanupTaskExclusive(executed, true); + if (executed != null) + executed.completeExclusiveNoExcept(); fetchWorkExclusive(); } catch (Throwable t) @@ -431,24 +422,7 @@ public class AccordExecutorSignalLoop extends AbstractLoop if (task == null) task = awaitWork(self); - try - { - self.setAccordActiveTask(task); - task.run(); - } - catch (Throwable t) - { - try { task.failExecution(t); } - catch (Throwable t2) - { - try - { - t2.addSuppressed(t); - agent.onException(t2); - } - catch (Throwable t3) { /* empty to ensure we definitely loop so we cleanup the task */ } - } - } + task.runNoExcept(self); } catch (ShutdownException ignore) { @@ -458,10 +432,6 @@ public class AccordExecutorSignalLoop extends AbstractLoop { agent.onException(t); } - finally - { - self.setAccordActiveTask(null); - } } } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java index 4d2508b87f..ee8e79e535 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java @@ -96,25 +96,7 @@ public class AccordExecutorSimple extends AccordExecutor return; } - // TODO (expected): dedup with AbstractLockLoop, and cleanup (executed flag is a bit ugly) - self.setAccordActiveTask(task); - boolean executed = false; - try - { - task.preRunExclusive(); - executed = true; - task.run(); - } - catch (Throwable t) - { - executed = false; - task.failExecution(t); - } - finally - { - cleanupTaskExclusive(task, executed); - self.setAccordActiveTask(null); - } + prepareRunComplete(self, task); } } finally diff --git a/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java index c171ed08de..e93ae0c084 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java @@ -29,9 +29,9 @@ final class CancelTask extends Task } @Override - void submitExclusive() + void submitExclusiveMayThrow() { - cancel.cancelExclusive(); + cancel.tryCancelExclusive(); } @Override @@ -41,13 +41,23 @@ final class CancelTask extends Task } @Override - String toDescription() + public String description() { - return "Cancel " + cancel.toDescription(); + return "Cancel " + cancel.description(); } - @Override void preRunExclusive() { throw new UnsupportedOperationException(); } - @Override void run() { throw new UnsupportedOperationException(); } - @Override void reportFailure(Throwable fail) { throw new UnsupportedOperationException(); } + @Override + public String briefDescription() + { + return "Cancel " + cancel.briefDescription(); + } + + @Override void unqueueIfQueued() { throw new UnsupportedOperationException(); } + @Override + void maybeCompleteExclusiveMayThrow() { throw new UnsupportedOperationException(); } + @Override void tryCancelExclusive() { throw new UnsupportedOperationException(); } + @Override void reportFailureMayThrow(Throwable fail) { throw new UnsupportedOperationException(); } + @Override boolean runMayThrow() { throw new UnsupportedOperationException(); } @Override public void cancel() { throw new UnsupportedOperationException(); } + @Override AccordExecutor executor() { throw new UnsupportedOperationException(); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java index b0bd8d37e6..5181f11e18 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java @@ -33,11 +33,13 @@ import accord.utils.async.AsyncChains; import accord.utils.async.Cancellable; import org.apache.cassandra.service.accord.debug.DebugExecution; +import org.apache.cassandra.service.accord.execution.Task.GroupKind; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.EXCLUSIVE_QUEUE_LIMITS; +import static org.apache.cassandra.service.accord.execution.Task.ExecutorQueue.RUNNABLE; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.COMMAND_STORE; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.execution.TaskQueueRunnable.RUNNABLE; public final class ExclusiveExecutor extends TaskQueueMulti implements ExclusiveAsyncExecutor { @@ -53,40 +55,59 @@ public final class ExclusiveExecutor extends TaskQueueMulti implements Exc this.queue = queue; } - @Override void submitExclusive() { throw new UnsupportedOperationException(); } + @Override AccordExecutor executor() { return queue.executor; } + @Override void submitExclusiveMayThrow() { throw new UnsupportedOperationException(); } @Override boolean isNewWork() { throw new UnsupportedOperationException(); } @Override public void cancel() { throw new UnsupportedOperationException(); } + @Override boolean runMayThrow() { throw new UnsupportedOperationException(); } + @Override void unqueueIfQueued() {} + @Override void reportFailureMayThrow(Throwable t) { throw new UnsupportedOperationException(); } + @Override void tryCancelExclusive() { throw new UnsupportedOperationException(); } - @Override - void preRunExclusive() + boolean prepareTask() { - super.preRunExclusive(); - queue.preRunTask(); + Task task = queue.task; + try + { + task.prepareExclusiveMayThrow(); + task.setStateExclusive(State.PREPARED); + setStateExclusive(State.PREPARED); + return true; + } + catch (Throwable t) + { + task.setStateExclusive(State.FAILED); + task.reportFailureNoExcept(t); + maybeCompleteExclusiveMayThrow(); + return false; + } } @Override - void run() + void maybeCompleteExclusiveMayThrow() { - queue.runTask(); + try + { + unsafeSetStateExclusive(WAITING_TO_RUN); + executor().runnable.cleanup(this); + queue.completeTask(); + } + catch (Throwable t) + { + onException(t); + } } @Override - void cleanupExclusive(AccordExecutor executor, boolean executed) + public String description() { - unsafeSetStateExclusive(WAITING_TO_RUN); - queue.cleanupTask(executed); + return queue.task.description(); } @Override - void reportFailure(Throwable t) + public String briefDescription() { - queue.task.reportFailure(t); - } - - @Override - String toDescription() - { - return queue.task.toDescription(); + return queue.task.briefDescription(); } protected boolean isInHeap() @@ -113,48 +134,43 @@ public final class ExclusiveExecutor extends TaskQueueMulti implements Exc ExclusiveExecutor(AccordExecutor executor, int commandStoreId) { - super(RUNNABLE, commandStoreId < 0 ? Task.GroupKind.NONE : Task.GroupKind.EXCLUSIVE, AccordExecutor.EXCLUSIVE_QUEUE_LIMITS); + super(RUNNABLE, commandStoreId < 0 ? GroupKind.NONE : GroupKind.EXCLUSIVE, EXCLUSIVE_QUEUE_LIMITS); this.executor = executor; this.commandStoreId = commandStoreId; this.selfTask = new ExclusiveExecutorTask(this); this.debug = DebugExecution.DebugExclusiveExecutor.maybeDebug(executor.debug, commandStoreId); } - void preRunTask() + void runTask(TaskRunner self) { - task.preRunExclusive(); - } - - void runTask() - { - Thread self = Thread.currentThread(); - if (!ownerUpdater.compareAndSet(this, null, self)) + Thread thread = Thread.currentThread(); + if (!ownerUpdater.compareAndSet(this, null, thread)) { if (DEBUG_EXECUTION) debug.onWaiting(); Invariants.require(waiting == null); - waiting = self; + waiting = thread; outer: do { while (true) { Thread owner = this.owner; - if (owner == self) break outer; + if (owner == thread) break outer; if (owner == null) continue outer; LockSupport.park(); } } - while (!ownerUpdater.compareAndSet(this, null, self)); - Invariants.require(waiting == self); + while (!ownerUpdater.compareAndSet(this, null, thread)); + Invariants.require(waiting == thread); waiting = null; } try { if (stopped && reject(task)) - task.reportFailure(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((SafeTask) task).executionContext())); + task.rejectAtRuntime(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + task.description())); else - task.run(); + task.runNoExcept(self); } finally { @@ -173,12 +189,12 @@ public final class ExclusiveExecutor extends TaskQueueMulti implements Exc return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); } - void cleanupTask(boolean executed) + void completeTask() { try { - task.unsetQueue(this); - task.cleanupExclusive(executor, executed); + task.unsetQueue(kind); + task.completeExclusiveNoExcept(); } finally { @@ -212,7 +228,7 @@ public final class ExclusiveExecutor extends TaskQueueMulti implements Exc incrementArrivals(newTask); incrementDispatches(newTask); task = newTask; - task.setQueue(this); + task.setQueue(kind); selfTask.position = newTask.position; selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); executor.runnable.enqueue(selfTask, incrementArrivals); @@ -249,7 +265,7 @@ public final class ExclusiveExecutor extends TaskQueueMulti implements Exc // but can for other tasks that don't track their own state decrementDispatches(task); - task.unsetQueue(this); + task.unsetQueue(kind); task = pollMulti(); if (DEBUG_EXECUTION) debug.onSetTask(task); if (executor.runnable.isWaiting(selfTask)) @@ -315,31 +331,23 @@ public final class ExclusiveExecutor extends TaskQueueMulti implements Exc return AsyncChains.flatChain(this, call); } - Task inherit() - { - Thread thread = Thread.currentThread(); - if (thread == owner) - return Task.unwrap(task); - - return executor.inherit(thread); - } - @Override public void execute(Runnable run) { - Task inherit = inherit(); - PlainRunnable submit = inherit == null ? new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER) - : new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - executor.submit(submit); + Task inherit = executor.inherit(); + PlainRunnable task = new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER); + if (inherit != null) inherit.addConsequence(task); + else executor.submitTask(task); } @Override public Cancellable execute(AsyncCallbacks.RunOrFail runOrFail) { - Task inherit = inherit(); - PlainChain submit = inherit == null ? new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER) - : new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - return executor.submit(submit); + Task inherit = executor.inherit(); + PlainChain task = new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER); + if (inherit != null) inherit.addConsequence(task); + else executor.submitTask(task); + return task; } @Override @@ -367,7 +375,7 @@ public final class ExclusiveExecutor extends TaskQueueMulti implements Exc } catch (Throwable t) { - executor.agent.onException(t); + selfTask.onException(t); } finally { diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTask.java b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java index 68c976eb7b..b3607151fe 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/IOTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java @@ -25,40 +25,14 @@ import org.apache.cassandra.concurrent.DebuggableTask; // a task that may be submitted to this executor or another public abstract class IOTask extends Plain implements Cancellable, DebuggableTask { - IOTask(AccordExecutor executor, GlobalGroup group, long position, int tranche) - { - super(executor, group, position, tranche); - } - IOTask(AccordExecutor executor, GlobalGroup group) { super(executor, group); } - abstract void postRunExclusive(); - - @Override - void cleanupExclusive(AccordExecutor executor, boolean executed) - { - super.cleanupExclusive(executor, executed); - postRunExclusive(); - } - @Override ExclusiveExecutor exclusiveExecutor() { return null; } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java index aa54cfcf2a..759ff32ba7 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java @@ -20,8 +20,6 @@ package org.apache.cassandra.service.accord.execution; import accord.utils.Invariants; -import org.apache.cassandra.utils.Closeable; - import static org.apache.cassandra.service.accord.execution.IOTaskLoad.FailureHolder.NOT_STARTED; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; @@ -50,33 +48,25 @@ public class IOTaskLoad extends IOTask this.entry = entry; } - void postRunExclusive() + @Override + void maybeCompleteExclusiveMayThrow() { if (!(result instanceof FailureHolder)) executor.onLoadedExclusive(entry, (V) result, null); else executor.onLoadedExclusive(entry, null, ((FailureHolder) result).fail); + super.maybeCompleteExclusiveMayThrow(); } @Override - String toDescription() + public boolean runMayThrow() { - return "Load " + entry.key(); + result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); + return true; } @Override - public void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); - } - onRunComplete(); - } - - @Override - void reportFailure(Throwable t) + void reportFailureMayThrow(Throwable t) { result = new FailureHolder(t); } @@ -84,6 +74,12 @@ public class IOTaskLoad extends IOTask @Override public String description() { - return "Loading " + entry; + return "Load " + entry.key(); + } + + @Override + public String briefDescription() + { + return description(); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java index d010b6e1f1..cfe80ee817 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java @@ -18,8 +18,6 @@ package org.apache.cassandra.service.accord.execution; -import org.apache.cassandra.utils.Closeable; - import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.SAVE; class IOTaskSave extends IOTask @@ -40,31 +38,22 @@ class IOTaskSave extends IOTask } @Override - void postRunExclusive() + void maybeCompleteExclusiveMayThrow() { executor.onSavedExclusive(entry, identity, failure); + super.maybeCompleteExclusiveMayThrow(); } @Override - String toDescription() + public boolean runMayThrow() { - return "Save " + entry.key(); - } - - @Override - public void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - onRunComplete(); + run.run(); failure = null; + return true; } @Override - protected void reportFailure(Throwable t) + void reportFailureMayThrow(Throwable t) { failure = t; } @@ -74,4 +63,10 @@ class IOTaskSave extends IOTask { return "Save " + entry; } + + @Override + String briefDescription() + { + return description(); + } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java index ce6d60d530..3476dbde2d 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java @@ -18,8 +18,6 @@ package org.apache.cassandra.service.accord.execution; -import org.apache.cassandra.utils.Closeable; - class IOTaskWrapper extends IOTask { static abstract class WrappableIOTask @@ -32,33 +30,24 @@ class IOTaskWrapper extends IOTask final WrappableIOTask wrapped; - IOTaskWrapper(AccordExecutor executor, WrappableIOTask wrap, GlobalGroup group, long position, int tranche) + IOTaskWrapper(AccordExecutor executor, WrappableIOTask wrap, GlobalGroup group) { - super(executor, group, position, tranche); + super(executor, group); this.wrapped = wrap; } @Override - String toDescription() + protected boolean runMayThrow() { - return wrapped.description(); + wrapped.runInternal(); + return true; } @Override - protected void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - wrapped.runInternal(); - } - onRunComplete(); - } - - @Override - void postRunExclusive() + void maybeCompleteExclusiveMayThrow() { wrapped.postRunExclusive(); + super.maybeCompleteExclusiveMayThrow(); } @Override @@ -68,7 +57,13 @@ class IOTaskWrapper extends IOTask } @Override - protected void reportFailure(Throwable fail) + public String briefDescription() + { + return description(); + } + + @Override + void reportFailureMayThrow(Throwable fail) { wrapped.fail(fail); } diff --git a/src/java/org/apache/cassandra/service/accord/execution/Plain.java b/src/java/org/apache/cassandra/service/accord/execution/Plain.java index aff55550e2..cc5a3c0ffc 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/Plain.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Plain.java @@ -20,28 +20,16 @@ package org.apache.cassandra.service.accord.execution; import java.util.concurrent.CancellationException; -import accord.utils.Invariants; import accord.utils.async.Cancellable; import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; abstract class Plain extends Task implements Cancellable { final AccordExecutor executor; - Plain(AccordExecutor executor, GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - this.executor = executor; - } - - Plain(AccordExecutor executor, ExclusiveGroup group, long position, int tranche) - { - super(group, position, tranche); - this.executor = executor; - } - Plain(AccordExecutor executor, GlobalGroup group) { super(group); @@ -57,49 +45,48 @@ abstract class Plain extends Task implements Cancellable abstract ExclusiveExecutor exclusiveExecutor(); @Override - public void cancel() + public final void cancel() { - executor.submit(Task::cancelExclusive, CancelTask::new, this); + executor.submit(Task::tryCancelExclusive, CancelTask::new, this); } - void cancelExclusive() + @Override + final void tryCancelExclusive() { + tryFailAndCompleteExclusive(new CancellationException(), CANCELLED); + } + + @Override + final void submitExclusiveMayThrow() + { + executor.registerExclusive(this); + setStateExclusive(WAITING_TO_RUN); ExclusiveExecutor exclusiveExecutor = exclusiveExecutor(); - if ((exclusiveExecutor == null ? executor.runnable : exclusiveExecutor).tryUnqueueWaiting(this)) + if (exclusiveExecutor == null) executor.runnable.enqueue(this, true); + else exclusiveExecutor.enqueue(this, true); + } + + @Override + void maybeCompleteExclusiveMayThrow() + { + if (completeState()) { - try - { - failExclusive(new CancellationException(), CANCELLED); - } - catch (Throwable t) - { - executor.agent.onException(t); - } - finally - { - executor.cleanupTaskExclusive(this, false); - } + long completeAt = nanoTime(); + executor.elapsedWaitingToRun.increment(runningAt - createdAt, runningAt); + executor.elapsedRunning.increment(completeAt - runningAt, completeAt); + executor.elapsed.increment(completeAt - createdAt, completeAt); } } @Override - final void submitExclusive() + void unqueueIfQueued() { - submitExclusive(exclusiveExecutor(), null); - } - - final Cancellable submitExclusive(ExclusiveExecutor exclusiveExecutor, Task parent) - { - Invariants.require(executor.isOwningThread()); - setStateExclusive(WAITING_TO_RUN); - - if (parent == null) executor.registerExclusive(this); - else executor.registerConsequenceExclusive(parent, this); - onLoaded(); - - if (exclusiveExecutor == null) executor.runnable.enqueue(this, true); - else exclusiveExecutor.enqueue(this, true); - return this; + if (isQueued()) + { + ExclusiveExecutor exclusiveExecutor = exclusiveExecutor(); + if (exclusiveExecutor != null) exclusiveExecutor.unqueue(this); + else executor.runnable.unqueue(this); + } } @Override @@ -107,4 +94,10 @@ abstract class Plain extends Task implements Cancellable { return true; } + + @Override + AccordExecutor executor() + { + return executor; + } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java index c729f58e84..dda6a54e2c 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java @@ -22,8 +22,6 @@ import javax.annotation.Nullable; import accord.utils.async.AsyncCallbacks; -import org.apache.cassandra.utils.Closeable; - class PlainChain extends Plain { final AsyncCallbacks.RunOrFail runOrFail; @@ -36,11 +34,29 @@ class PlainChain extends Plain this.exclusiveExecutor = exclusiveExecutor; } - PlainChain(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group, long position, int tranche) + @Override + public String description() { - super(executor, group, position, tranche); - this.runOrFail = runOrFail; - this.exclusiveExecutor = exclusiveExecutor; + return runOrFail.toString(); + } + + @Override + public String briefDescription() + { + return description(); + } + + @Override + boolean runMayThrow() + { + runOrFail.run(); + return true; + } + + @Override + void reportFailureMayThrow(Throwable fail) + { + runOrFail.fail(fail); } @Override @@ -48,40 +64,4 @@ class PlainChain extends Plain { return exclusiveExecutor; } - - @Override - String toDescription() - { - return runOrFail.toString(); - } - - @Override - protected void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - runOrFail.run(); - } - catch (Throwable t) - { - // shouldn't throw exceptions - executor.agent.onException(t); - } - onRunComplete(); - } - - @Override - protected void reportFailure(Throwable fail) - { - try - { - runOrFail.fail(fail); - } - catch (Throwable t) - { - fail.addSuppressed(t); - executor.agent.onException(fail); - } - } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java index 6a2ab7950b..efcef616fd 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java @@ -35,33 +35,9 @@ class PlainChainDebuggable extends PlainChain implements DebuggableTask this.describe = Invariants.nonNull(describe); } - PlainChainDebuggable(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, @Nullable ExclusiveExecutor exclusiveExecutor, long position, int tranche, Object describe) - { - super(executor, runOrFail, exclusiveExecutor, ExclusiveGroup.OTHER, position, tranche); - this.describe = Invariants.nonNull(describe); - } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } - @Override public String description() { return describe.toString(); } - - @Override - public DebuggableTask debuggable() - { - return this; - } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java index 594ff107bf..e51c83c964 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java @@ -22,7 +22,6 @@ import javax.annotation.Nullable; import accord.utils.async.Cancellable; -import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.concurrent.AsyncPromise; class PlainRunnable extends Plain implements Cancellable @@ -31,14 +30,6 @@ class PlainRunnable extends Plain implements Cancellable final Runnable run; final @Nullable ExclusiveExecutor exclusiveExecutor; - PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, GlobalGroup group, long position, int tranche) - { - super(executor, group, position, tranche); - this.result = result; - this.run = run; - this.exclusiveExecutor = null; - } - PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, GlobalGroup group) { super(executor, group); @@ -47,14 +38,6 @@ class PlainRunnable extends Plain implements Cancellable this.exclusiveExecutor = null; } - PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group, long position, int tranche) - { - super(executor, group, position, tranche); - this.result = result; - this.run = run; - this.exclusiveExecutor = exclusiveExecutor; - } - PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group) { super(executor, group); @@ -64,31 +47,35 @@ class PlainRunnable extends Plain implements Cancellable } @Override - String toDescription() + protected boolean runMayThrow() + { + run.run(); + if (result != null) + { + try { result.trySuccess(null); } + catch (Throwable t) { onException(t); } + } + return true; + } + + @Override + public String description() { // TODO (expected): ensure this is usefully descriptive, or accept a separate description return run.toString(); } @Override - protected void run() + public String briefDescription() { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - if (result != null) - result.trySuccess(null); - onRunComplete(); + return description(); } @Override - protected void reportFailure(Throwable t) + void reportFailureMayThrow(Throwable t) { if (result != null) result.tryFailure(t); - executor.agent.onException(t); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java index 77c61ecf40..7027adb478 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java @@ -56,6 +56,7 @@ import accord.primitives.AbstractRanges; import accord.primitives.AbstractUnseekableKeys; import accord.primitives.Range; import accord.primitives.Ranges; +import accord.primitives.Routable; import accord.primitives.RoutingKeys; import accord.primitives.Timestamp; import accord.primitives.TxnId; @@ -81,7 +82,6 @@ import org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableSt import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; import org.apache.cassandra.service.accord.serializers.CommandSerializers; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.NoSpamLogger; import org.apache.cassandra.utils.concurrent.Condition; @@ -90,6 +90,7 @@ import static accord.local.LoadKeys.NONE; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.RECOVERY; import static accord.local.LoadKeysFor.WRITE; +import static accord.primitives.Routable.Domain.Key; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask.SANITY_CHECK; @@ -106,18 +107,26 @@ import static org.apache.cassandra.service.accord.execution.SaferState.postExecu import static org.apache.cassandra.service.accord.execution.SaferState.preExecute; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; -import static org.apache.cassandra.service.accord.execution.Task.RunState.PERSISTING; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_FAILED; +import static org.apache.cassandra.service.accord.execution.Task.RunState.NOT_YET_RUN; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_PERSISTING; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_INCOMPLETE; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUNNING; import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED_UNREGISTERED; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED; import static org.apache.cassandra.service.accord.execution.Task.State.INCOMPLETE; import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; -import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.PREPARED; +import static org.apache.cassandra.service.accord.execution.Task.State.REGISTERED; import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; -import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_OR_RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_OR_PREPARED; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; public final class SafeTask extends Task implements Cancellable, DebuggableTask { @@ -137,47 +146,46 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa return new SafeTask<>((AccordCommandStore) commandStore, context, function); } - final class NonSyncState extends ExecutionContext.Wrapped implements ExecutionContext + static class NonSyncState extends ExecutionContext.Wrapped implements ExecutionContext { RoutingKeys active; - ObjectHashSet notBlocking; - ObjectHashSet blocking; // cache entries on which we're blocking work + ObjectHashSet blocking, notBlocking; int loaded, processed; - int ready; + boolean alwaysReady; - public NonSyncState() + public NonSyncState(ExecutionContext context) { - super(executionContext); + super(context); } @Override - public Unseekables keys() + public final Unseekables keys() { return active; } - void addLoaded() + final void addLoaded() { ++loaded; } - void onNotHead(AccordCacheEntry entry) + final void onNotHead(AccordCacheEntry entry) { if ((notBlocking == null || !notBlocking.remove((RoutingKey) entry.key())) && blocking != null) blocking.remove((RoutingKey) entry.key()); } - void onNewHead(AccordCacheEntry entry) + final void onNewHead(AccordCacheEntry entry) { ensureNotBlocking().add((RoutingKey) entry.key()); } - void onNewBlockingHead(AccordCacheEntry entry) + final void onNewBlockingHead(AccordCacheEntry entry) { ensureBlocking().add((RoutingKey) entry.key()); } - void onStillHeadNewBlocking(AccordCacheEntry entry) + final void onStillHeadNewBlocking(AccordCacheEntry entry) { notBlocking.remove((RoutingKey) entry.key()); ensureBlocking().add((RoutingKey) entry.key()); @@ -202,33 +210,33 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa return (blocking == null ? 0 : blocking.size()) + (notBlocking == null ? 0 : notBlocking.size()); } - boolean isLoaded() + final boolean isLoaded(SafeTask owner) { - return loaded >= Math.min(keys, NONSYNC_MIN_BATCH_SIZE); + return loaded >= Math.min(owner.keys, alwaysReady ? 1 : NONSYNC_MIN_BATCH_SIZE); } - boolean isWaitReady() + final boolean isWaitReady(SafeTask owner) { - if (readyCount() >= Math.min(keys - processed, NONSYNC_MIN_BATCH_SIZE)) + if (readyCount() >= Math.min(owner.keys - processed, alwaysReady ? 1 : NONSYNC_MIN_BATCH_SIZE)) return true; return blocking != null && blocking.size() >= NONSYNC_BLOCKED_LIMIT; } - void preRunExclusive() + void prepareExclusive(SafeTask owner) { try (BufferList keys = new BufferList<>()) { if ((blocking == null || !populate(keys, blocking)) && notBlocking != null) populate(keys, notBlocking); - keys.forEach(key -> preExecute(refs.get(key), SafeTask.this, RELEASE_QUEUE)); + keys.forEach(key -> preExecute(owner.refs.get(key), owner, RELEASE_QUEUE)); keys.sort(RoutingKey::compareTo); active = RoutingKeys.of(keys); } processed += active.size(); - if (processed == keys && isIncremental()) - setIncrementalFinishingExclusive(); + if (processed == owner.keys && owner.isIncremental()) + owner.setIncrementalFinishingExclusive(); } private boolean populate(List keys, ObjectHashSet from) @@ -254,20 +262,35 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa return false; } - void postRunExclusive() + void postRunExclusive(SafeTask owner) { if (active != null) { for (RoutingKey key : active) - postExecute(refs.remove(key), SafeTask.this); + postExecute(owner.refs.remove(key), owner); active = null; } - ready = readyCount(); + } + } + + static class IncrementalState extends NonSyncState + { + long waiting, running; + + public IncrementalState(ExecutionContext context) + { + super(context); + } + + void updateMetrics(long waiting, long running) + { + this.running += running; + this.waiting += waiting; } } final AccordCommandStore commandStore; - private final ExecutionContext executionContext; + private final ExecutionContext context; private final Function function; // TODO (expected): simple custom map that allows (at least): @@ -296,17 +319,17 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa int keys; // TODO (expected): not counting keys we add during execution LogLinearDecayingHistograms.Buffer histogramBuffer; - @Nullable RangeTxnScanner rangeScanner; - @Nullable CommandSummaries commandsForRanges; - byte runState; + + @Nullable Object ranges; + long loadedAt; private BiConsumer callback; - public SafeTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext, Function function) + public SafeTask(@Nonnull AccordCommandStore commandStore, ExecutionContext context, Function function) { - super(executionContext, commandStore.executor().uniqueCreatedAt); + super(context, commandStore.executor().uniqueCreatedAt); this.commandStore = commandStore; - this.executionContext = executionContext; + this.context = context; this.function = function; if (logger.isTraceEnabled()) logger.trace("Created {} on {}", this, commandStore); @@ -320,21 +343,19 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa @Override public String toString() { - return "@[" + commandStore.id() + ',' + commandStore.node().id() + "] " + executionContext.describe() + ' ' + toBriefString(); + return "@[" + commandStore.id() + ',' + commandStore.node().id() + "] " + context.describe() + ' ' + toBriefString(); } public String toBriefString() { - return '{' + loggingId() + ',' + describeState() + '}'; + return '{' + loggingId() + ',' + currentState() + '}'; } - public String toDescription() + public String summarise() { - return toBriefString() + ": " - + (queued() == null ? "unqueued" : describeState()) - + ", primaryTxnId: " + executionContext.primaryTxnId() + return loggingId() + ' ' + context.executionKind() + + ": primaryTxnId: " + context.primaryTxnId() + ", state: " + summarise(refs, SaferState::global); - } private static String summarise(Map map, Function transform) @@ -383,66 +404,79 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa @Override protected Cancellable start(BiConsumer callback) { - preSetup(callback); - executor().submit(SafeTask.this); + if (!preSetup(callback)) + executor().submitTask(SafeTask.this); return SafeTask.this; } }; } - private void preSetup(BiConsumer callback) + private boolean preSetup(BiConsumer callback) { Invariants.require(this.callback == null); this.callback = callback; - TaskRunner self = TaskRunner.get(); - Task task = self.accordActiveSelfTask(); - if (task instanceof SafeTask) + Task inherit = executor().inherit(); + if (inherit == null) + return false; + + Invariants.require(inherit.is(PREPARED)); + if (!inherit.is(RUNNING)) + return false; + + if (inherit instanceof SafeTask) { - SafeTask parent = (SafeTask) task; - if (parent.commandStore == commandStore) - preSetup(parent); + SafeTask inheritRefs = (SafeTask) inherit; + if (inheritRefs.commandStore == commandStore) + preSetup(inheritRefs); } + inherit.addConsequence(this); + return true; } // to be invoked only by the CommandStore owning thread, to take references to objects already in use by the current execution - public void preSetup(SafeTask parent) + private void preSetup(SafeTask parent) { - this.position = parent.position; - setInheritedWithTranche(parent.tranche()); - // note we use the caches "unsafely" here deliberately, as we only reference commands we already have references to // so we do not mutate anything, except the atomic counter of references - - LoadKeys loadKeys = loadKeys(executionContext); + LoadKeys loadKeys = loadKeys(context); if (loadKeys != NONE) { - Unseekables parentKeysOrRanges = parent.executionContext.keys(); - Unseekables keysOrRanges = executionContext.keys(); + Unseekables parentKeysOrRanges = parent.context.keys(); + Unseekables keysOrRanges = context.keys(); boolean isKeySubset = parent.isIncremental() ? parent.nonSync.active.containsAll(keysOrRanges) : parentKeysOrRanges.containsAll(keysOrRanges); if (isKeySubset) setInheritedRangeScan(); - setSequencedExclusive(executionContext.executionSequence()); - if (isSequencedByPriorityAtomic()) - { - boolean isTxnIdSubset = executionContext.isTxnIdSubsetOf(parent.executionContext); - Invariants.require(isKeySubset, "Must start ATOMIC tasks from a task declaring a superset of the required keys (for ASYNC/INCR tasks this means the keys active for the batch in question)"); - Invariants.require(isTxnIdSubset, "Must start ATOMIC tasks from a task declaring a superset of the required TxnIds"); - // TODO (required): we're appending to the fifo queue - does this maintain correct order? - setCacheQueuedFifoExclusive(); - } - + setSequencedExclusive(context.executionSequence()); if (loadKeys != SYNC) { setNonSyncExclusive(); - nonSync = new NonSyncState(); if (loadKeys == INCR) { + nonSync = new IncrementalState(context); setIncrementalExclusive(); // forbid BY_PRIORITY sequencing to avoid priority inversion deadlocks on INCR tasks that lock a TxnId but await some key that has a higher priority task (that is waiting on our locked TxnId) - solvable in future if necessary Invariants.require(isSequencedByPriorityAtomic() || isUnsequenced(), "INCR tasks may currently only be ATOMIC or UNSEQUENCED"); } + else + { + nonSync = new NonSyncState(context); + } + } + + if (isSequencedByPriorityAtomic()) + { + boolean isTxnIdSubset = context.isTxnIdSubsetOf(parent.context); + if (!isKeySubset) + { + Invariants.require(keysOrRanges.domain() == Key, "ATOMIC tasks over ranges must declare a subset of their parent's task keys() to avoid a range scan across which we would not impose sequencing"); + // to avoid priority inversion deadlocks, if we are not a strict subset of the parent task we permit running with a single key ready + nonSync.alwaysReady = true; + } + Invariants.require(isTxnIdSubset, "ATOMIC tasks must declare a subset of their parent's task txnIds() to avoid a priority inversion deadlock"); + // TODO (required): we're appending to the fifo queue - does this maintain correct order? + setCacheQueuedFifoExclusive(); } if (keysOrRanges.equals(parentKeysOrRanges)) @@ -473,28 +507,32 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa } } - for (TxnId txnId : executionContext.txnIds()) + for (TxnId txnId : context.txnIds()) preSetup(txnId, parent.refs, commandStore.cachesUnsafe().commands()); } @Override - void submitExclusive() + void submitExclusiveMayThrow() { Caches caches = commandStore.cachesExclusive(); executor().registerExclusive(this); + setStateExclusive(REGISTERED); boolean hasPreSetup = hasInherited(); - LoadKeys loadKeys = loadKeys(executionContext); + LoadKeys loadKeys = loadKeys(context); if (loadKeys != NONE) { if (loadKeys != SYNC && !hasPreSetup) { setNonSyncExclusive(); - nonSync = new NonSyncState(); - if (loadKeys == INCR) + if (loadKeys != INCR) nonSync = new NonSyncState(context); + else + { + nonSync = new IncrementalState(context); setIncrementalExclusive(); + } } - Unseekables keysOrRanges = executionContext.keys(); + Unseekables keysOrRanges = context.keys(); switch (keysOrRanges.domain()) { case Range: @@ -510,7 +548,7 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa } } - for (TxnId txnId : executionContext.txnIds()) + for (TxnId txnId : context.txnIds()) { if (hasPreSetup && completePresetupExclusive(txnId)) continue; @@ -525,14 +563,15 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa private void setupKeyLoadsExclusive(boolean hasPreSetup, Caches caches, Iterable setupKeys, boolean doNotScanRanges) { - if (executionContext.loadKeys() == NONE) + if (context.loadKeys() == NONE) return; - if (!doNotScanRanges && executionContext.loadKeysFor() == RECOVERY) + if (!doNotScanRanges && context.loadKeysFor() == RECOVERY) { - Invariants.require(rangeScanner == null); - rangeScanner = new RangeTxnScanner(); - rangeScanner.start(); + Invariants.require(ranges == null); + RangeTxnScanner scanner = new RangeTxnScanner(); + ranges = scanner; + scanner.start(); } int waitsForIncrement = isSync() ? 1 : 0; @@ -547,11 +586,12 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa private void setupRangeLoadsExclusive(Caches caches) { - if (executionContext.loadKeysFor() == WRITE) + if (context.loadKeysFor() == WRITE) return; - rangeScanner = new RangeTxnAndKeyScanner(caches.commandsForKeys()); - rangeScanner.start(); + RangeTxnAndKeyScanner scanner = new RangeTxnAndKeyScanner(caches.commandsForKeys()); + ranges = scanner; + scanner.start(); } // expects mutual exclusivity only on the command store @@ -679,7 +719,7 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa private void onLoadedRequiredExclusive() { - if (isSync() || nonSync.isLoaded()) + if (isSync() || nonSync.isLoaded(this) || isCacheQueuedFifo()) { waitOnCacheQueuesExclusive(); } @@ -702,13 +742,13 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa boolean holdsLocksBetweenRuns() { // TODO (desired): encode as a state bit - return isIncremental() && executionContext.primaryTxnId() != null; + return isIncremental() && context.primaryTxnId() != null; } private void waitOnCacheQueuesExclusive() { Invariants.require(waitingForState == 0); - onLoaded(); + loadedAt = nanoTime(); executor().runnable.incrementArrivals(this); commandStore.exclusiveExecutor().incrementArrivals(this); @@ -731,17 +771,17 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa else { setStateExclusive(WAITING_ON_REQUIRED); - executor().waitingOnCacheQueues.enqueue(this); + executor().waiting.enqueue(this); } } private void waitOnOptionalCacheQueuesExclusive() { - if (isSync() || nonSync.isWaitReady()) waitToRunExclusive(); + if (isSync() || nonSync.isWaitReady(this)) waitToRunExclusive(); else { setStateExclusive(WAITING_ON_OPTIONAL); - executor().waitingOnCacheQueues.enqueue(this); + executor().waiting.enqueue(this); } } @@ -765,7 +805,7 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa case WAITING_ON_REQUIRED: case WAITING_ON_OPTIONAL: case WAITING_TO_RUN: - case RUNNING: + case PREPARED: RunnableStatus status = addToCacheQueue(loaded, false); if (status != NOT_RUNNABLE) addQueuedOptionalKey(loaded, status); @@ -796,7 +836,7 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa void addLoadedOptionalKey() { nonSync.addLoaded(); - if (is(LOADING_OPTIONAL) && nonSync.isLoaded()) + if (is(LOADING_OPTIONAL) && nonSync.isLoaded(this)) { unqueue(executor().loading); waitOnCacheQueuesExclusive(); @@ -820,16 +860,16 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa break; } - if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady()) + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady(this)) { - unqueue(executor().waitingOnCacheQueues); + unqueue(executor().waiting); waitToRunExclusive(); } } void onChangeHeadStatus(AccordCacheEntry entry, RunnableStatus status) { - if (isSync() || !entry.isCommandsForKey()) onChangeRequiredHeadStatus(entry, status); + if (isSync() || !entry.isCommandsForKey()) onChangeRequiredHeadStatus(status); if (isNonSync() && entry.isCommandsForKey()) onChangeOptionalHeadStatus(entry, status); } @@ -844,14 +884,14 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa // TODO (expected): this is potentially costly; maybe we don't want to swap these in and out (but harder to maintain invariants) unqueue(commandStore.exclusiveExecutor()); setStateExclusive(WAITING_ON_REQUIRED); - executor().waitingOnCacheQueues.enqueue(this); + executor().waiting.enqueue(this); } } Invariants.require(waitingForState < refs.size()); ++waitingForState; } - void onChangeRequiredHeadStatus(AccordCacheEntry entry, RunnableStatus newStatus) + void onChangeRequiredHeadStatus(RunnableStatus newStatus) { if (newStatus == NOT_RUNNABLE) { @@ -862,7 +902,7 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa Invariants.require(is(WAITING_ON_REQUIRED)); if (--waitingForState == 0) { - unqueue(executor().waitingOnCacheQueues); + unqueue(executor().waiting); waitOnOptionalCacheQueuesExclusive(); } } @@ -870,18 +910,18 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa void onChangeOptionalHeadStatus(AccordCacheEntry entry, RunnableStatus status) { - Invariants.require(isState(WAITING_OR_RUNNING)); + Invariants.require(isState(WAITING_OR_PREPARED)); switch (status) { default: throw UnhandledEnum.unknown(status); case STILL_RUNNABLE: throw UnhandledEnum.invalid(STILL_RUNNABLE); // onChange -> changed (but this means no change) case NOT_RUNNABLE: nonSync.onNotHead(entry); - if (is(WAITING_TO_RUN) && !nonSync.isWaitReady()) + if (is(WAITING_TO_RUN) && !nonSync.isWaitReady(this)) { unqueue(commandStore.exclusiveExecutor()); setStateExclusive(WAITING_ON_OPTIONAL); - executor().waitingOnCacheQueues.enqueue(this); + executor().waiting.enqueue(this); } return; @@ -898,9 +938,9 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa break; } - if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady()) + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady(this)) { - unqueue(executor().waitingOnCacheQueues); + unqueue(executor().waiting); waitToRunExclusive(); } } @@ -913,135 +953,134 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa public ExecutionContext executionContext() { - return executionContext; + return context; } @Override - protected void preRunExclusive() + protected void prepareExclusiveMayThrow() { - super.preRunExclusive(); - if (rangeScanner != null) + try { - commandsForRanges = rangeScanner.finish(commandStore.cachesExclusive()); - rangeScanner = null; - } + if (ranges instanceof SafeTask.RangeTxnScanner) + ranges = ((SafeTask.RangeTxnScanner)ranges).finish(commandStore.cachesExclusive()); - if (isSync()) - { - refs.forEach((k, v) -> { - preExecute(v, this, RELEASE_QUEUE); - }); - } - else - { - if (!hasIncrementalStarted()) + if (isSync()) { - TxnId primaryTxnId = executionContext.primaryTxnId(); - if (primaryTxnId != null) - { - LockMode lockMode = holdsLocksBetweenRuns() ? HOLD_QUEUE : RELEASE_QUEUE; - preExecute(refs.get(primaryTxnId), this, lockMode); - TxnId additionalTxnId = executionContext.additionalTxnId(); - if (additionalTxnId != null) - preExecute(refs.get(additionalTxnId), this, lockMode); - } - - if (isIncremental() && isSequencedByPriorityAtomic() && !isCacheQueuedFifo()) - { - setCacheQueuedFifoExclusive(); - refs.forEach((key, safeState) -> { - AccordCacheEntry entry = global(safeState); - RunnableStatus status = entry.moveToFifo(this); - if (entry.isLoaded()) - { - switch (status) - { - default: throw UnhandledEnum.unknown(status); - case NOT_RUNNABLE: - case STILL_RUNNABLE: - case STILL_RUNNABLE_NEWLY_BLOCKING: - break; - case NEWLY_RUNNABLE: - nonSync.onNewHead(entry); - break; - case NEWLY_BLOCKING_RUNNABLE: - nonSync.onNewBlockingHead(entry); - break; - } - } - }); - } - - if (isIncremental()) - setIncrementalStartedExclusive(); - } - nonSync.preRunExclusive(); - } - } - - @Override - public void run() - { - onRunning(); - SaferCommandStore safeStore = null; - try (Closeable close = resources.get()) - { - if (Tracing.isTracing()) - Tracing.trace(executionContext.describe()); - - commandStore.begin(safeStore = new SaferCommandStore(this, isSync() ? executionContext : nonSync)); - - R result = function.apply(safeStore); - - boolean finished = !isIncremental() || isIncrementalFinishing(); - if (finished) - { - List changes = new ArrayList<>(); - // TODO (expected): save any TxnId we add so that we don't need to iterate all of refs - refs.forEach((key, value) -> { - if (value instanceof SaferCommand) - { - SaferCommand safeCommand = (SaferCommand) value; - Journal.CommandUpdate diff = safeCommand.update(); - if (diff != null) - { - changes.add(diff); - maybeSanityCheck(safeCommand); - } - } + refs.forEach((k, v) -> { + preExecute(v, this, RELEASE_QUEUE); }); - - boolean flush = !changes.isEmpty() || safeStore.fieldUpdates() != null; - if (flush) - { - setRunState(PERSISTING); - Runnable onFlush = () -> finish(result, null); - safeStore.persistFieldUpdatesInternal(changes.isEmpty() ? onFlush : null); - if (!changes.isEmpty()) - save(changes, onFlush); - finished = false; - } } + else + { + if (!hasIncrementalStarted()) + { + TxnId primaryTxnId = context.primaryTxnId(); + if (primaryTxnId != null) + { + LockMode lockMode = holdsLocksBetweenRuns() ? HOLD_QUEUE : RELEASE_QUEUE; + preExecute(refs.get(primaryTxnId), this, lockMode); + TxnId additionalTxnId = context.additionalTxnId(); + if (additionalTxnId != null) + preExecute(refs.get(additionalTxnId), this, lockMode); + } - // TODO (required): exception handling here needs improving - safeStore.postExecute(); - commandStore.complete(safeStore); - safeStore = null; + if (isIncremental() && isSequencedByPriorityAtomic() && !isCacheQueuedFifo()) + { + setCacheQueuedFifoExclusive(); + refs.forEach((key, safeState) -> { + AccordCacheEntry entry = global(safeState); + RunnableStatus status = entry.moveToFifo(this); + if (entry.isLoaded()) + { + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case NOT_RUNNABLE: + case STILL_RUNNABLE: + case STILL_RUNNABLE_NEWLY_BLOCKING: + break; + case NEWLY_RUNNABLE: + nonSync.onNewHead(entry); + break; + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(entry); + break; + } + } + }); + } - if (finished) - finish(result, null); + if (isIncremental()) + setIncrementalStartedExclusive(); + } + nonSync.prepareExclusive(this); + } } catch (Throwable t) { - if (safeStore != null) - refs.forEach((k, v) -> v.setAbandoned()); + refs.forEach((k, v) -> { v.setAbandoned(); }); throw t; } - finally + } + + @Override + public boolean runMayThrow() + { + try { - if (safeStore != null) + SaferCommandStore safeStore = new SaferCommandStore(this, isSync() ? context : nonSync); + commandStore.begin(safeStore); + try + { + if (Tracing.isTracing()) + Tracing.trace(context.describe()); + + R result = function.apply(safeStore); + boolean finished = !isIncremental() || isIncrementalFinishing(); + if (!finished) setRunState(RUN_INCOMPLETE); + else + { + List changes = new ArrayList<>(); + // TODO (expected): save any TxnId we add so that we don't need to iterate all of refs + refs.forEach((key, value) -> { + if (value instanceof SaferCommand) + { + SaferCommand safeCommand = (SaferCommand) value; + Journal.CommandUpdate diff = safeCommand.update(); + if (diff != null) + { + changes.add(diff); + maybeSanityCheck(safeCommand); + } + } + }); + + boolean flush = !changes.isEmpty() || safeStore.fieldUpdates() != null; + if (flush) + { + setRunState(RUN_PERSISTING); + Runnable onFlush = () -> finish(result); + safeStore.persistFieldUpdatesInternal(changes.isEmpty() ? onFlush : null); + if (!changes.isEmpty()) + save(changes, onFlush); + finished = false; + } + } + + safeStore.postExecute(); + if (finished) + finish(result); + return finished; + } + finally + { commandStore.complete(safeStore); - onRunComplete(); + } + } + catch (Throwable t) + { + refs.forEach((k, v) -> v.setAbandoned()); + throw t; } } @@ -1075,40 +1114,77 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa } } - public void reportFailure(Throwable throwable) + void reportFailureMayThrow(Throwable failure) { - try + if (callback == null) executor().agent.onException(failure); + else { - if (callback != null) - callback.accept(null, throwable); - } - finally - { - commandStore.agent().onException(throwable); + BiConsumer reportTo = callback; + callback = null; + + if (executor().isInLoop()) reportTo.accept(null, failure); + else executor().submit(() -> reportTo.accept(null, failure)); } } @Override - protected void cleanupExclusive(AccordExecutor executor, boolean executed) + void maybeCompleteExclusiveMayThrow() { - if (is(RUNNING) && isNonSync()) + if (isNonSync() && !isEither(NOT_YET_RUN, RUN_FAILED)) { - nonSync.postRunExclusive(); - if (isIncremental() && !isIncrementalFinishing()) + if (is(PREPARED)) { - setStateExclusive(INCOMPLETE); - waitOnOptionalCacheQueuesExclusive(); - return; + nonSync.postRunExclusive(this); + if (isIncremental()) + { + long now = nanoTime(); + ((IncrementalState)nonSync).updateMetrics(now - runningAt, runningAt - loadedAt); + if (!isIncrementalFinishing()) + { + setStateExclusive(INCOMPLETE); + waitOnOptionalCacheQueuesExclusive(); + return; + } + } + } + else + { + Invariants.expect(is(FAILED)); + Invariants.expect(isIncremental()); + setRunState(RUN_FAILED); + refs.forEach((k, v) -> v.setAbandoned()); } } - executor.keys.increment(keys, runningAt); releaseResourcesExclusive(commandStore.cachesExclusive()); - super.cleanupExclusive(executor, executed); - if (histogramBuffer != null) + + AccordExecutor executor = executor(); + if (completeState()) { - histogramBuffer.flush(completeAt); - histogramBuffer = null; + long completeAt = nanoTime(); + executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); + if (isIncremental()) + { + IncrementalState incrementalState = (IncrementalState) nonSync; + executor.elapsedWaitingToRun.increment(incrementalState.waiting, runningAt); + executor.elapsedRunning.increment(incrementalState.running, completeAt); + } + else + { + executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); + executor.elapsedRunning.increment(completeAt - runningAt, completeAt); + } + executor.elapsed.increment(completeAt - createdAt, completeAt); + executor.keys.increment(keys, runningAt); + if (histogramBuffer != null) + { + histogramBuffer.flush(runningAt); + histogramBuffer = null; + } + } + else if (histogramBuffer != null) + { + histogramBuffer.clear(); } } @@ -1116,15 +1192,20 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa public void cancel() { if (!state().hasStarted()) - executor().submit(Task::cancelExclusive, CancelTask::new, this); + executor().submit(Task::tryCancelExclusive, CancelTask::new, this); } - void cancelExclusive() + void tryCancelExclusive() { State state = state(); switch (state) { default: throw new UnhandledEnum(state); + case UNREGISTERED: + failExclusive(new CancellationException(), CANCELLED_UNREGISTERED); + break; + + case REGISTERED: case SCANNING_RANGES: case LOADING_REQUIRED: case LOADING_OPTIONAL: @@ -1132,41 +1213,53 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa case WAITING_ON_OPTIONAL: case WAITING_TO_RUN: if (!hasIncrementalStarted()) - { - unqueueIfQueued(); - setStateExclusive(CANCELLED); - if (rangeScanner != null) - rangeScanner.cancelled = true; - try - { - logger.info("Cancelling {}", executionContext); - if (callback != null) - { - if (executor().isInLoop()) callback.accept(null, new CancellationException()); - else executor().submit(() -> callback.accept(null, new CancellationException())); - } - } - finally - { - executor().cleanupTaskExclusive(this, false); - } - break; - } - case UNINITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase - case RUNNING: + cancelExclusive(); + break; + + case CANCELLED_UNREGISTERED: case INCOMPLETE: + case PREPARED: case EXECUTED: case CANCELLED: - case FAILED_TO_LOAD: + case FAILED: // cannot safely cancel } } - private void finish(R result, Throwable failure) + void cancelExclusive() + { + if (ranges instanceof SafeTask.RangeTxnScanner) + ((SafeTask.RangeTxnScanner)ranges).cancelled = true; // TODO (expected): should we try to cancel this directly? + failAndCompleteExclusive(new CancellationException(), CANCELLED); + } + + void unqueueIfQueued() + { + TaskQueue expected; + switch (queued()) + { + default: throw UnhandledEnum.unknown(queued()); + case NONE: return; + case LOADING: + expected = executor().loading; + break; + case WAITING: + expected = executor().waiting; + break; + case RUNNABLE: + expected = commandStore.exclusiveExecutor(); + break; + } + expected.unqueue(this); + } + + private void finish(R result) { - setRunState(failure == null ? RunState.SUCCESS : RunState.FAILED); if (callback != null) - callback.accept(result, failure); + { + try { callback.accept(result, null); } + catch (Throwable t) { commandStore.agent().onException(t); } + } } void releaseResourcesExclusive(Caches caches) @@ -1176,12 +1269,12 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa try { - if (rangeScanner != null) + if (ranges instanceof SafeTask.RangeTxnScanner) { - rangeScanner.cleanup(caches); - rangeScanner = null; + ((SafeTask.RangeTxnScanner)ranges).cleanup(caches); if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedRangeScanner(); } + ranges = null; refs.forEach((key, safeState) -> { SaferState.postExecute(safeState, this); @@ -1227,7 +1320,7 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa } final Set intersectingKeys = new ObjectHashSet<>(); - final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); + final Ranges ranges = ((AbstractRanges) context.keys()).toRanges(); final AccordCache.Type.Instance commandsForKeyCache; KeyWatcher keyWatcher = new KeyWatcher(); @@ -1353,7 +1446,7 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa ExecutionContext preLoadContext() { - return executionContext; + return context; } @Override @@ -1371,24 +1464,24 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa public void start() { Caches caches = commandStore.cachesExclusive(); - setStateExclusive(SCANNING_RANGES); - executor().scanningRanges.enqueue(SafeTask.this); + SafeTask.this.setStateExclusive(SCANNING_RANGES); + executor().loading.enqueue(SafeTask.this); startInternal(caches); executor().submitExclusive(SafeTask.this, GlobalGroup.RANGE_SCAN, this); } void startInternal(Caches caches) { - loader = commandStore.rangeIndex().loader(executionContext.primaryTxnId(), executionContext.executeAt(), executionContext.loadKeysFor(), executionContext.keys()); + loader = commandStore.rangeIndex().loader(context.primaryTxnId(), context.executeAt(), context.loadKeysFor(), context.keys()); loader.loadExclusive(guardedSummaries, caches); } public void scannedExclusive() { - Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", SafeTask.this, SafeTask::toDescription); + Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", SafeTask.this, SafeTask::description); scanned = true; scannedInternal(); - unqueue(executor().scanningRanges); + unqueue(executor().loading); // likely to be requeued to same queue, but simpler invariants if we remove here onSetupOrScannedExclusive(); } @@ -1414,7 +1507,7 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa @Override public String description() { - return "Scanning range intersections for " + executionContext.reason() + ' ' + toBriefString(); + return "Scanning range intersections for " + context.reason() + ' ' + toBriefString(); } @Override @@ -1424,31 +1517,19 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa } } - @Override - public DebuggableTask debuggable() - { - return this; - } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } - @Override public String description() { - return executionContext.describe(); + return context.describe(); } - private AccordExecutor executor() + @Override + public String briefDescription() + { + return context.reason(); + } + + final AccordExecutor executor() { return commandStore.executor(); } @@ -1459,10 +1540,18 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa return true; } - @Nullable - public RangeTxnScanner rangeScanner() + public @Nullable SafeTask.RangeTxnScanner rangeScanner() { - return rangeScanner; + if (ranges instanceof SafeTask.RangeTxnScanner) + return ((SafeTask.RangeTxnScanner) ranges); + return null; + } + + public @Nullable CommandSummaries commandsForRanges() + { + if (ranges instanceof CommandSummaries) + return (CommandSummaries) ranges; + return null; } private static LoadKeys loadKeys(ExecutionContext context) diff --git a/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java index e1b73cf15e..89c3f4b46b 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java @@ -30,6 +30,7 @@ import accord.api.RoutingKey; import accord.impl.AbstractSafeCommandStore; import accord.local.Command; import accord.local.CommandStores; +import accord.local.CommandSummaries; import accord.local.ExecutionContext; import accord.local.NodeCommandStoreService; import accord.local.RedundantBefore; @@ -214,7 +215,8 @@ public final class SaferCommandStore extends AbstractSafeCommandStore void visit(Unseekables keysOrRanges, Timestamp startedBefore, Kinds testKind, ActiveCommandVisitor visitor, P1 p1, P2 p2) { visitForKey(keysOrRanges, cfk -> { cfk.visit(startedBefore, testKind, visitor, p1, p2); return true; }); - if (task.commandsForRanges != null) - task.commandsForRanges.visit(keysOrRanges, startedBefore, testKind, visitor, p1, p2); + CommandSummaries commandsForRanges = task.commandsForRanges(); + if (commandsForRanges != null) + commandsForRanges.visit(keysOrRanges, startedBefore, testKind, visitor, p1, p2); } @Override public boolean visit(Unseekables keysOrRanges, TxnId testTxnId, Kinds testKind, SupersedingCommandVisitor visit) { - return visitForKey(keysOrRanges, cfk -> cfk.visit(testTxnId, testKind, visit)) - && (task.commandsForRanges == null || task.commandsForRanges.visit(keysOrRanges, testTxnId, testKind, visit)); + if (!visitForKey(keysOrRanges, cfk -> cfk.visit(testTxnId, testKind, visit))) + return false; + + CommandSummaries commandsForRanges = task.commandsForRanges(); + return commandsForRanges == null || commandsForRanges.visit(keysOrRanges, testTxnId, testKind, visit); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/execution/Task.java b/src/java/org/apache/cassandra/service/accord/execution/Task.java index 0d2395f8de..bd0af0d472 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/Task.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Task.java @@ -18,10 +18,10 @@ package org.apache.cassandra.service.accord.execution; +import java.util.concurrent.CancellationException; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; -import javax.annotation.Nullable; - import accord.local.ExecutionContext; import accord.messages.Accept; import accord.messages.Commit; @@ -33,11 +33,14 @@ import accord.primitives.TxnId; import accord.utils.IntrusiveHeapNode; import accord.utils.Invariants; import accord.utils.TinyEnumSet; +import accord.utils.UnhandledEnum; import accord.utils.async.Cancellable; import org.apache.cassandra.concurrent.DebuggableTask; import org.apache.cassandra.concurrent.ExecutorLocals; -import org.apache.cassandra.service.accord.debug.DebugExecution; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.service.accord.execution.ExclusiveExecutor.ExclusiveExecutorTask; +import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.WithResources; import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY; @@ -48,44 +51,61 @@ import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXE import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.APPLY; import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.COMMIT; import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.STABLE; +import static org.apache.cassandra.service.accord.execution.Task.RunState.NOT_YET_RUN; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_FAILED; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_INCOMPLETE; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED_UNREGISTERED; import static org.apache.cassandra.service.accord.execution.Task.State.EXECUTED; -import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; -import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING_OR_EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.PREPARED; +import static org.apache.cassandra.service.accord.execution.Task.State.PREPARED_OR_EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.execution.Task.State.UNREGISTERED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -public abstract class Task extends IntrusiveHeapNode implements Cancellable +public abstract class Task extends IntrusiveHeapNode implements Cancellable, DebuggableTask { - private static final int WAITING_ON_OPTIONAL_BIT = 1 << 5; - private static final int WAITING_TO_RUN_BIT = 1 << 6; - private static final int INCOMPLETE_BIT = 1 << 9; + private static final int WAITING_ON_OPTIONAL_BIT = 1 << 7; + private static final int WAITING_TO_RUN_BIT = 1 << 8; + private static final int INCOMPLETE_BIT = 1 << 10; enum State { - UNINITIALIZED(), - SCANNING_RANGES(UNINITIALIZED), - LOADING_REQUIRED(UNINITIALIZED, SCANNING_RANGES), - LOADING_OPTIONAL(UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED), - WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), - WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), - WAITING_TO_RUN(INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), - RUNNING(WAITING_TO_RUN), - EXECUTED(RUNNING), - INCOMPLETE(RUNNING), - FAILED_TO_LOAD(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), - FAILED_OTHER(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), - CANCELLED(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, RUNNING), + UNREGISTERED(), + CANCELLED_UNREGISTERED(UNREGISTERED), + REGISTERED(UNREGISTERED), + SCANNING_RANGES(REGISTERED), + LOADING_REQUIRED(REGISTERED, SCANNING_RANGES), + LOADING_OPTIONAL(REGISTERED, SCANNING_RANGES, LOADING_REQUIRED), + WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), + WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), + WAITING_TO_RUN(INCOMPLETE_BIT, UNREGISTERED, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), + PREPARED(WAITING_TO_RUN), + INCOMPLETE(PREPARED), + EXECUTED(PREPARED), + FAILED(UNREGISTERED, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, PREPARED, INCOMPLETE), + CANCELLED(UNREGISTERED, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), ; private final int permittedFrom; public static final int WAITING = TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN); - public static final int WAITING_OR_RUNNING = WAITING | TinyEnumSet.encode(RUNNING); - public static final int RUNNING_OR_EXECUTED = WAITING | TinyEnumSet.encode(RUNNING, EXECUTED); + public static final int WAITING_OR_PREPARED = WAITING | TinyEnumSet.encode(PREPARED); + public static final int PREPARED_OR_EXECUTED = TinyEnumSet.encode(PREPARED, EXECUTED); static final State[] VALUES = values(); static { // hack to allow us to create loops in our enum transition declarations Invariants.require(INCOMPLETE_BIT == 1 << INCOMPLETE.ordinal()); + Invariants.require(WAITING_TO_RUN_BIT == 1 << WAITING_TO_RUN.ordinal()); + Invariants.require(WAITING_ON_OPTIONAL_BIT == 1 << WAITING_ON_OPTIONAL.ordinal()); } State() @@ -111,14 +131,14 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable return (permittedFrom & (1 << prevOrdinal)) != 0; } - boolean isExecuted() + boolean isDone() { return this.compareTo(EXECUTED) >= 0; } boolean hasStarted() { - return this.compareTo(RUNNING) >= 0; + return this.compareTo(PREPARED) >= 0; } static State forOrdinal(int ordinal) @@ -129,7 +149,7 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable enum RunState { - NONE, PERSISTING, SUCCESS, FAILED; + NOT_YET_RUN, RUNNING, RUN_INCOMPLETE, RUN_PERSISTING, RUN_SUCCESS, RUN_FAILED; private static final RunState[] VALUES = values(); @@ -177,6 +197,28 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable } } + enum ExecutorQueue + { + NONE(), + LOADING(SCANNING_RANGES, LOADING_OPTIONAL, LOADING_REQUIRED), + WAITING(WAITING_ON_OPTIONAL, WAITING_ON_REQUIRED), + RUNNABLE(WAITING_TO_RUN); + + private static final ExecutorQueue[] VALUES = values(); + + final int permittedStates; + + ExecutorQueue(State ... states) + { + this.permittedStates = TinyEnumSet.encode(states); + } + + public static ExecutorQueue forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + private static final int STATE_MASK = 0xf; static final int GROUP_MASK = 0x7; private static final int EXCLUSIVE_GROUP_SHIFT = 4; @@ -185,23 +227,26 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable private static final int NONSYNC_BIT = 1 << 10; private static final int CACHE_QUEUED_BIT = 1 << 11; - private static final int INCREMENTAL_MASK = 0x3 << 12; - private static final int INCREMENTAL = 0x1 << 12; - private static final int INCREMENTAL_STARTED = 0x2 << 12; - private static final int INCREMENTAL_FINISHING = 0x3 << 12; + private static final int HAS_TRANCHE_BIT = 1 << 12; + private static final int HAS_INHERITED_BIT = 1 << 13; + private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 14; - private static final int SEQUENCED_SHIFT = 14; + private static final int EXECUTOR_QUEUE_SHIFT = 16; + private static final int EXECUTOR_QUEUE_UNSHIFTED_MASK = 0x3; + private static final int EXECUTOR_QUEUE_SHIFTED_MASK = EXECUTOR_QUEUE_UNSHIFTED_MASK << EXECUTOR_QUEUE_SHIFT; + + private static final int INCREMENTAL_SHIFT = 18; + private static final int INCREMENTAL_MASK = 0x3 << INCREMENTAL_SHIFT; + private static final int INCREMENTAL = 0x1 << INCREMENTAL_SHIFT; + private static final int INCREMENTAL_STARTED = 0x2 << INCREMENTAL_SHIFT; + private static final int INCREMENTAL_FINISHING = 0x3 << INCREMENTAL_SHIFT; + + private static final int SEQUENCED_SHIFT = 20; private static final int SEQUENCED_MASK = 0x3 << SEQUENCED_SHIFT; private static final int SEQUENCED_PRIORITY = 0x1 << SEQUENCED_SHIFT; private static final int SEQUENCED_ATOMIC = 0x2 << SEQUENCED_SHIFT; private static final int SEQUENCED_ATOMIC_AND_QUEUED = 0x3 << SEQUENCED_SHIFT; - // spare two bits - - private static final int HAS_TRANCHE_BIT = 1 << 18; - private static final int HAS_INHERITED_BIT = 1 << 19; - private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 20; - private static final int TRANCHE_SHIFT = 22; static final int MAX_TRANCHE = 0x3ff; @@ -212,53 +257,40 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable Invariants.require(ExecutionContext.ExecutionSequence.values().length <= 3); } + // TODO (desired): quite heavy to pass-through tracing session state we mostly don't use public final WithResources resources; Task next; + Task consequences; + // the queue position until run is invoked, at which point it is assigned a nanoTime() value long position; int info; - // TODO (expected): do we need this? we should be able to determine the queue from state() if needed for e.g. cancellation - private TaskQueue queued; - public final long createdAt; - // TODO (expected): expose via executors vtable - // TODO (expected): use just one long and some flag bits to indicate which point it represents, and report incrementally - public long loadedAt, runningAt, completeAt; - private byte runState; + long runningAt; + // TODO (desired): expose via executors vtable + private volatile int runState; + + private static final AtomicIntegerFieldUpdater runStateUpdater = AtomicIntegerFieldUpdater.newUpdater(Task.class, "runState"); Task(GlobalGroup group) { - resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); info = init(group, ExclusiveGroup.OTHER); createdAt = nanoTime(); } Task(ExclusiveGroup group) { - resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); info = init(GlobalGroup.OTHER, group); createdAt = nanoTime(); } - Task(GlobalGroup group, long position, int tranche) - { - this(group); - this.position = position; - setInheritedWithTranche(tranche); - } - - Task(ExclusiveGroup group, long position, int tranche) - { - this(group); - this.position = position; - setInheritedWithTranche(tranche); - } - protected Task(ExecutionContext context, AtomicLong lastCreatedAt) { - resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next < prev ? prev + 1 : next); + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next <= prev ? prev + 1 : next); ExclusiveGroup group = ExclusiveGroup.OTHER; TxnId txnId = context.primaryTxnId(); if (txnId != null) @@ -343,144 +375,241 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable public final Task unwrap() { - if (this instanceof ExclusiveExecutor.ExclusiveExecutorTask) - return ((ExclusiveExecutor.ExclusiveExecutorTask) this).queue.task; + if (this instanceof ExclusiveExecutorTask) + return ((ExclusiveExecutorTask) this).queue.task; return this; } - static Task unwrap(Task task) - { - return task == null ? null : task.unwrap(); - } - public DebuggableTask debuggable() { - return null; + return this; } - abstract String toDescription(); + public final long creationTimeNanos() + { + return createdAt; + } - abstract void submitExclusive(); + public final long startTimeNanos() + { + if (runState() == NOT_YET_RUN) + return 0; + return runningAt; + } + + abstract void submitExclusiveMayThrow(); + /** Return true if COMPLETED successfully. false indicates more work is being done. Should not throw any exceptions related to reporting success. */ + abstract boolean runMayThrow(); + abstract void maybeCompleteExclusiveMayThrow(); + abstract void tryCancelExclusive(); + abstract void reportFailureMayThrow(Throwable fail); + + abstract AccordExecutor executor(); + abstract void unqueueIfQueued(); + abstract boolean isNewWork(); + abstract String briefDescription(); + + final void submitExclusiveNoExcept() + { + if (is(UNREGISTERED)) + { + try { submitExclusiveMayThrow(); } + catch (Throwable t) + { + tryFailAndCompleteExclusive(t, State.FAILED); + onException(t); + } + } + } /** - * Prepare to run while holding the state cache lock + * Prepare to run while holding the state cache lock. + * If returns false, prepare failed and the task should be discarded with no further action. */ - void preRunExclusive() + final boolean prepareExclusiveNoExcept() + { + if (getClass() == ExclusiveExecutorTask.class) + { + return ((ExclusiveExecutorTask)this).prepareTask(); + } + else + { + try + { + prepareExclusiveMayThrow(); + setStateExclusive(State.PREPARED); + return true; + } + catch (Throwable t) + { + failAndCompleteExclusive(t, State.FAILED); + return false; + } + } + } + + void prepareExclusiveMayThrow() { - setStateExclusive(RUNNING); } /** * Run the command; the state cache lock may or may not be held depending on the executor implementation */ - abstract void run(); + final void runNoExcept(TaskRunner self) + { + if (getClass() == ExclusiveExecutorTask.class) + { + ((ExclusiveExecutorTask)this).queue.runTask(self); + } + else + { + onRunning(); + self.setAccordActiveTask(this); + try (Closeable close = resources.get()) + { + if (runMayThrow()) + onSuccess(); + else + Invariants.require(compareTo(RUNNING) > 0); + } + catch (Throwable t) + { + setRunState(RunState.RUN_FAILED); + reportFailureNoExcept(t); + } + finally + { + self.setAccordActiveTask(null); + } + } + } - /** - * Fail the command; the state cache lock may or may not be held depending on the executor implementation - */ - abstract void reportFailure(Throwable fail); + final void rejectAtRuntime(Throwable reject) + { + setRunState(RunState.RUN_FAILED); + reportFailureNoExcept(reject); + } + + final void completeExclusiveNoExcept() + { + try + { + if (DEBUG_EXECUTION) DebugTask.get(this).onComplete(); + maybeCompleteExclusiveMayThrow(); + } + catch (Throwable t) + { + onException(t); + if (compareTo(EXECUTED) < 0) + failExclusive(t, State.FAILED); + } + finally + { + if (compareTo(EXECUTED) >= 0) + { + try { submitConsequencesExclusive(is(EXECUTED)); } + catch (Throwable t) { onException(t); } + executor().completedTaskExclusive(this); + } + } + } + + final void onException(Throwable t) + { + try { executor().agent.onException(t); } + catch (Throwable t2) { } + } + + final void reportFailureNoExcept(Throwable fail) + { + try { reportFailureMayThrow(fail); } + catch (Throwable t) + { + try { fail.addSuppressed(t); } + catch (Throwable t2) { } + onException(fail); + } + } + + // propagate RunState to State + // true if task was executed successfully + final boolean completeState() + { + RunState runState = runState(); + boolean success; + switch (runState) + { + default: throw UnhandledEnum.unknown(runState); + case RUN_INCOMPLETE: throw UnhandledEnum.invalid(runState); + case NOT_YET_RUN: + Invariants.expect(state().isDone()); + Invariants.expect(consequences == null); + success = false; + break; + + case RUN_FAILED: + if (compareTo(EXECUTED) < 0) + setStateExclusive(State.FAILED); + success = false; + break; + + case RUNNING: + case RUN_PERSISTING: + case RUN_SUCCESS: + setStateExclusive(EXECUTED); + success = true; + break; + } + return success; + } + + final void tryFailAndCompleteExclusive(Throwable fail, State newState) + { + if (is(UNREGISTERED)) + failExclusive(fail, CANCELLED_UNREGISTERED); + else if (compareTo(WAITING_TO_RUN) <= 0) + failAndCompleteExclusive(fail, newState); + } final void failExclusive(Throwable fail, State newState) { - try - { - setStateExclusive(newState); - } - finally - { - reportFailure(fail); - } + unqueueIfQueued(); + setStateExclusive(newState); + reportFailureNoExcept(fail); } - final void failExecution(Throwable fail) + final void failAndCompleteExclusive(Throwable fail, State newState) { - Invariants.require(is(RUNNING)); - try - { - setRunState(RunState.FAILED); - } - finally - { - reportFailure(fail); - } - } - - abstract boolean isNewWork(); - - /** - * Cleanup the command while holding the state cache lock - */ - void cleanupExclusive(AccordExecutor executor, boolean executed) - { - if (executed) setStateExclusive(EXECUTED); - else Invariants.require(state().isExecuted()); - executor.unregisterExclusive(this); - completeAt = nanoTime(); - if (runningAt != 0) - { - if (loadedAt == 0) - loadedAt = runningAt; - executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); - executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); - executor.elapsedRunning.increment(completeAt - runningAt, completeAt); - executor.elapsed.increment(completeAt - createdAt, completeAt); - } - if (DEBUG_EXECUTION) DebugExecution.DebugTask.get(this).onCompleted(executor.debug); - } - - void cancelExclusive() - { - } - - @Nullable - final TaskQueue queued() - { - return queued; - } - - final void unqueueIfQueued() - { - if (queued != null) - { - queued.unqueue(this); - queued = null; - } + failExclusive(fail, newState); + completeExclusiveNoExcept(); } final void unqueue(TaskQueue expected) { - Invariants.require(queued == expected, "%s != %s", queued, expected); - queued.unqueue(this); - queued = null; + Invariants.require(queuedOrdinal() == expected.kind.ordinal()); + expected.unqueue(this); + info &= ~EXECUTOR_QUEUE_SHIFTED_MASK; } - final void unsetQueue(TaskQueue expected) + final void unsetQueue(ExecutorQueue expected) { - Invariants.require(queued == expected, "%s != %s", queued, expected); - queued = null; - } - - final void setQueue(TaskQueue queue) - { - Invariants.require(queued == null); - Invariants.require(isCompatible(queue)); - queued = queue; + Invariants.require(expected.ordinal() == queuedOrdinal()); + info &= ~EXECUTOR_QUEUE_SHIFTED_MASK; } final void onRunning() { + Invariants.require(is(PREPARED)); + Invariants.require(is(NOT_YET_RUN) || is(RUN_INCOMPLETE)); runningAt = nanoTime(); - if (DEBUG_EXECUTION) ((DebugExecution.DebugTask) resources).onRunning(); + setRunState(RunState.RUNNING); + if (DEBUG_EXECUTION) ((DebugTask) resources).onRunning(); } - final void onRunComplete() + final void onSuccess() { - if (DEBUG_EXECUTION) ((DebugExecution.DebugTask) resources).onRunComplete(); - } - - final void onLoaded() - { - loadedAt = nanoTime(); + setRunState(RunState.RUN_SUCCESS); + if (DEBUG_EXECUTION) ((DebugTask) resources).onRunComplete(); } final State state() @@ -493,20 +622,20 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable return RunState.forOrdinal(runState); } - final Enum describeState() + final Enum currentState() { State state = state(); - if (state == RUNNING || state == EXECUTED) + if (state == State.PREPARED || state == EXECUTED) { RunState runState = runState(); - if (runState == RunState.NONE) + if (runState == NOT_YET_RUN) return state; return runState; } return State.forOrdinal(stateOrdinal()); } - private int stateOrdinal() + final int stateOrdinal() { return info & STATE_MASK; } @@ -516,6 +645,27 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable return stateOrdinal() == state.ordinal(); } + final int compareTo(State state) + { + return stateOrdinal() - state.ordinal(); + } + + final int compareTo(RunState state) + { + return runState - state.ordinal(); + } + + final boolean is(RunState state) + { + return runState == state.ordinal(); + } + + final boolean isEither(RunState state1, RunState state2) + { + int runState = this.runState; + return runState == state1.ordinal() || runState == state2.ordinal(); + } + final boolean isState(int stateBitSet) { return TinyEnumSet.contains(stateBitSet, stateOrdinal()); @@ -536,11 +686,6 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable info = (info & ~(GROUP_MASK << GLOBAL_GROUP_SHIFT)) | (group.ordinal() << GLOBAL_GROUP_SHIFT); } - final int compareTo(State state) - { - return stateOrdinal() - state.ordinal(); - } - final void setStateExclusive(State state) { Invariants.require(state.isPermittedFrom(stateOrdinal()), "%s forbidden from %s", state, this, Task::reportBadStateTransition); @@ -549,13 +694,18 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable final void setRunState(RunState state) { - Invariants.require(isState(RUNNING_OR_EXECUTED)); - runState = (byte) state.ordinal(); + Invariants.require(isState(PREPARED_OR_EXECUTED) || (state == RUN_FAILED && is(FAILED))); + setRunState(state.ordinal()); + } + + final void setRunState(int newRunState) + { + runStateUpdater.lazySet(this, newRunState); } private static String reportBadStateTransition(Task task) { - return task.state() + " for " + task.toDescription(); + return task.state() + " for " + task.description(); } final void unsafeSetStateExclusive(State state) @@ -573,10 +723,10 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; } - private boolean isCompatible(TaskQueue queue) + private boolean isCompatible(ExecutorQueue queue) { int self = stateOrdinal(); - return TinyEnumSet.contains(queue.states, self); + return TinyEnumSet.contains(queue.permittedStates, self); } final boolean isSync() @@ -658,6 +808,28 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable return 0 != (info & CACHE_QUEUED_BIT); } + final boolean isQueued() + { + return 0 != (info & EXECUTOR_QUEUE_SHIFTED_MASK); + } + + final int queuedOrdinal() + { + return (info >>> EXECUTOR_QUEUE_SHIFT) & EXECUTOR_QUEUE_UNSHIFTED_MASK; + } + + final ExecutorQueue queued() + { + return ExecutorQueue.forOrdinal(queuedOrdinal()); + } + + final void setQueue(ExecutorQueue queue) + { + Invariants.require(isCompatible(queue)); + Invariants.require(!isQueued()); + info |= queue.ordinal() << EXECUTOR_QUEUE_SHIFT; + } + // supersedes priority, in whichever order they're called final void setCacheQueuedFifoExclusive() { @@ -682,6 +854,14 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; } + final Task inherit(Task parent) + { + Invariants.require(!hasInherited()); + position = parent.position; + setInheritedWithTranche(parent.tranche()); + return this; + } + final void setInheritedWithTranche(int tranche) { Invariants.require(tranche <= MAX_TRANCHE); @@ -703,6 +883,42 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable return (info & HAS_INHERITED_RANGE_SCAN_BIT) != 0; } + void addConsequence(Task task) + { + Task prev = consequences; + Invariants.require(prev == null || prev.is(UNREGISTERED)); + task.next = prev; + consequences = task; + } + + final void submitConsequencesExclusive(boolean success) + { + if (consequences == null) + return; + + Task cur = Task.reverse(consequences); + consequences = null; + + while (cur != null) + { + Task next = cur.next; + cur.next = null; + if (cur.is(UNREGISTERED)) + { + if (success || !(cur instanceof SafeTask || this instanceof SafeTask)) + { + cur.inherit(this); + cur.submitExclusiveNoExcept(); + } + else + { + cur.failExclusive(new CancellationException("Parent task failed"), CANCELLED); + } + } + cur = next; + } + } + static int init(GlobalGroup global, ExclusiveGroup exclusive) { return (global.ordinal() << GLOBAL_GROUP_SHIFT) | (exclusive.ordinal() << EXCLUSIVE_GROUP_SHIFT); diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java index d9163d5a58..7b9f2c352c 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java @@ -22,14 +22,12 @@ import javax.annotation.Nullable; import accord.local.ExecutionContext; -import org.apache.cassandra.concurrent.DebuggableTask; - public class TaskInfo implements Comparable { // sorted in name order for reporting to virtual tables public enum Status { - LOADING, RUNNING, SCANNING_RANGES, WAITING_TO_LOAD, WAITING_TO_RUN + RUNNING, LOADING, WAITING_TO_RUN } final Status status; @@ -64,8 +62,8 @@ public class TaskInfo implements Comparable if (task instanceof SafeTask) return ((SafeTask) task).executionContext().reason(); - if (task instanceof DebuggableTask) - return ((DebuggableTask) task).description(); + if (task != null) + return task.description(); return null; } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java index ec483dd640..1374742860 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java @@ -19,16 +19,17 @@ package org.apache.cassandra.service.accord.execution; import accord.utils.IntrusivePriorityHeap; -import accord.utils.TinyEnumSet; + +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; // has xSingle methods to distinguish from within MultiTaskQueue whether we're invoking the multi or single variation class TaskQueue extends IntrusivePriorityHeap { - final int states; + final ExecutorQueue kind; - public TaskQueue(int states) + public TaskQueue(ExecutorQueue kind) { - this.states = states; + this.kind = kind; } void unqueue(T task) @@ -105,6 +106,6 @@ class TaskQueue extends IntrusivePriorityHeap @Override public String toString() { - return TinyEnumSet.toString(states, Task.State::forOrdinal); + return kind.toString(); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java index 9784e51ad9..f3ec1f5e85 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java @@ -21,6 +21,11 @@ package org.apache.cassandra.service.accord.execution; import accord.utils.Invariants; import accord.utils.UnhandledEnum; +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; +import org.apache.cassandra.service.accord.execution.Task.GroupKind; + +import static org.apache.cassandra.service.accord.execution.Task.ExecutorQueue.RUNNABLE; + /** * A {@link TaskQueue} sub-divided into up to eight per-group sub-queues (groups are phases for the exclusive * executor, or work classes globally) plus a policy for choosing which sub-queue to serve next. The policy balances @@ -130,9 +135,9 @@ abstract class TaskQueueMulti extends TaskQueue int waitingCount; - TaskQueueMulti(int waitingStates, Task.GroupKind groups, long limits) + TaskQueueMulti(ExecutorQueue kind, GroupKind groups, long limits) { - super(waitingStates); + super(kind); this.limits = limits; int queueCount = groups.count; Invariants.require(queueCount <= 8); @@ -149,14 +154,14 @@ abstract class TaskQueueMulti extends TaskQueue return (task.info >>> groupShift) & Task.GROUP_MASK; } - void stop(int group) + void stop(long groupOverflowBits) { - stopped |= overflowBit(group); + stopped |= groupOverflowBits; } - void restart(int group) + void restart(long groupOverflowBits) { - stopped &= ~overflowBit(group); + stopped &= ~groupOverflowBits; } final TaskQueue queue(Task task) @@ -172,7 +177,7 @@ abstract class TaskQueueMulti extends TaskQueue { TaskQueue queue = queues[group]; if (queue == null) - queues[group] = queue = new TaskQueue<>(0); + queues[group] = queue = new TaskQueue<>(RUNNABLE); return queue; } @@ -439,7 +444,7 @@ abstract class TaskQueueMulti extends TaskQueue final void enqueueMulti(T task, boolean incrementArrivals) { - task.setQueue(this); + task.setQueue(kind); int group = group(task); if (group < 0) { @@ -492,7 +497,7 @@ abstract class TaskQueueMulti extends TaskQueue private void unqueue(T task, int group, TaskQueue queue) { - task.unsetQueue(this); + task.unsetQueue(kind); boolean dirty = queue.unqueueSingle(task); --waitingCount; if (group >= 0) @@ -565,6 +570,11 @@ abstract class TaskQueueMulti extends TaskQueue arrivals |= overflow - (overflow >>> 7); } + final boolean hasWaitingToRunExcluding(long groupOverflowBits) + { + return (unsaturatedWithWork() & ~groupOverflowBits) != 0; + } + final void setHasWork(int group) { hasWork |= overflowBit(group); @@ -600,13 +610,18 @@ abstract class TaskQueueMulti extends TaskQueue return waitingCount; } - final long lowBit(int group) + static long lowBit(int group) { return 1L << (group * 8); } - final long overflowBit(int group) + static long overflowBit(int group) { return 0x80L << (group * 8); } + + static long overflowBit(Enum group) + { + return overflowBit(group.ordinal()); + } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java index 8aad433fb6..ddd801b8e4 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java @@ -18,21 +18,16 @@ package org.apache.cassandra.service.accord.execution; -import accord.utils.TinyEnumSet; - -import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; -import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.execution.Task.ExecutorQueue.RUNNABLE; final class TaskQueueRunnable extends TaskQueueMulti { - static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, RUNNING); - final TaskQueue assigned; TaskQueueRunnable() { super(RUNNABLE, Task.GroupKind.GLOBAL, AccordExecutor.GLOBAL_QUEUE_LIMITS); - this.assigned = new TaskQueue<>(0); + this.assigned = new TaskQueue<>(RUNNABLE); } T poll() @@ -58,7 +53,7 @@ final class TaskQueueRunnable extends TaskQueueMulti if (group >= 0) decrementActive(group); - unqueue.unsetQueue(this); + unqueue.unsetQueue(kind); assigned.unqueueSingle(unqueue); } else @@ -94,7 +89,7 @@ final class TaskQueueRunnable extends TaskQueueMulti int group = group(task); if (group >= 0) decrementActive(group); - task.unsetQueue(this); + task.unsetQueue(kind); } } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java index 7dd8b1def0..4f7757dc25 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java @@ -18,29 +18,33 @@ package org.apache.cassandra.service.accord.execution; -import accord.utils.TinyEnumSet; +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; final class TaskQueueStandalone extends TaskQueue { - TaskQueueStandalone(int states) + TaskQueueStandalone(ExecutorQueue kind) { - super(states); - } - - TaskQueueStandalone(Task.State state) - { - super(TinyEnumSet.encode(state)); + super(kind); } void enqueue(T enqueue) { - enqueue.setQueue(this); + enqueue.setQueue(kind); enqueueSingle(enqueue); } + boolean tryUnqueue(T unqueue) + { + if (!containsNode(unqueue)) + return false; + + unqueue(unqueue); + return true; + } + void unqueue(T unqueue) { - unqueue.unsetQueue(this); + unqueue.unsetQueue(kind); removeNode(unqueue); } diff --git a/src/java/org/apache/cassandra/utils/NoSpamLogger.java b/src/java/org/apache/cassandra/utils/NoSpamLogger.java index a2bf815b30..b14088913c 100644 --- a/src/java/org/apache/cassandra/utils/NoSpamLogger.java +++ b/src/java/org/apache/cassandra/utils/NoSpamLogger.java @@ -17,6 +17,11 @@ */ package org.apache.cassandra.utils; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; @@ -64,16 +69,18 @@ public class NoSpamLogger CLOCK = clock; } - public class NoSpamLogStatement extends AtomicLong + public static class NoSpamLogStatement extends AtomicLong { private static final long serialVersionUID = 1L; + private final Logger wrapped; private final String statement; private final long minIntervalNanos; - public NoSpamLogStatement(String statement, long minIntervalNanos) + public NoSpamLogStatement(Logger wrapped, String statement, long minIntervalNanos) { super(Long.MIN_VALUE); + this.wrapped = wrapped; this.statement = statement; this.minIntervalNanos = minIntervalNanos; } @@ -159,6 +166,154 @@ public class NoSpamLogger } } + public static class NoDuplicateSpamLogStatement + { + private static final long serialVersionUID = 1L; + private static final int PRUNE_SIZE = 32; + + private final Logger wrapped; + private final String statement; + private final long minIntervalNanos; + private final ConcurrentHashMap lastLogged = new ConcurrentHashMap<>(); + private final AtomicLong nextPruneAt = new AtomicLong(); + + public NoDuplicateSpamLogStatement(Logger wrapped, String statement, long minInterval, TimeUnit units) + { + this(wrapped, statement, units.toNanos(minInterval)); + } + + public NoDuplicateSpamLogStatement(Logger wrapped, String statement, long minIntervalNanos) + { + this.wrapped = wrapped; + this.statement = statement; + this.minIntervalNanos = minIntervalNanos; + } + + private boolean shouldLog(long id, long nowNanos) + { + Long expected = lastLogged.getOrDefault(id, Long.MIN_VALUE); + if (nowNanos < expected || !lastLogged.replace(id, expected, nowNanos + minIntervalNanos)) + return false; + + if (lastLogged.size() >= PRUNE_SIZE) + { + long pruneAt = nextPruneAt.get(); + if (nowNanos >= pruneAt && nextPruneAt.compareAndSet(pruneAt, nowNanos + minIntervalNanos)) + { + for (Map.Entry e : lastLogged.entrySet()) + { + if (nowNanos < e.getValue()) + lastLogged.remove(e.getKey(), e.getValue()); + } + } + } + return true; + } + + public boolean log(Level l, long id, long nowNanos, Object... objects) + { + if (!shouldLog(id, nowNanos)) return false; + return logNoCheck(l, objects); + } + + private boolean logNoCheck(Level l, Object... objects) + { + switch (l) + { + case DEBUG: + wrapped.debug(statement, objects); + break; + case INFO: + wrapped.info(statement, objects); + break; + case WARN: + wrapped.warn(statement, objects); + break; + case ERROR: + wrapped.error(statement, objects); + break; + default: + throw new AssertionError(); + } + return true; + } + + public boolean debug(long id, long nowNanos, Object... objects) + { + return log(Level.DEBUG, id, nowNanos, objects); + } + + public boolean debug(long id, Object... objects) + { + return debug(id, CLOCK.nanoTime(), objects); + } + + public boolean info(long id, long nowNanos, Object... objects) + { + return log(Level.INFO, id, nowNanos, objects); + } + + public boolean info(long id, Object... objects) + { + return info(id, CLOCK.nanoTime(), objects); + } + + public boolean warn(long id, long nowNanos, Object... objects) + { + return log(Level.WARN, id, nowNanos, objects); + } + + public boolean warn(long id, Object... objects) + { + return warn(id, CLOCK.nanoTime(), objects); + } + + public boolean error(long id, long nowNanos, Object... objects) + { + return log(Level.ERROR, id, nowNanos, objects); + } + + public boolean error(long id, Object... objects) + { + return error(id, CLOCK.nanoTime(), objects); + } + + public static long exceptionId(Throwable throwable) + { + return exceptionId(throwable, Collections.newSetFromMap(new IdentityHashMap<>())); + } + + private static long exceptionId(Throwable throwable, Set visited) + { + long id = throwable.getClass().hashCode(); + for (StackTraceElement ste : throwable.getStackTrace()) + { + id *= 31; + id += ste.getClassName().hashCode(); + id *= 31; + id += ste.getLineNumber(); + } + + for (Throwable suppressed : throwable.getSuppressed()) + { + if (!visited.add(suppressed)) + continue; + + id *= 31; + id += exceptionId(suppressed, visited); + } + for (Throwable cause = throwable.getCause() ; cause != null ; cause = cause.getCause()) + { + if (!visited.add(cause)) + break; + + id *= 31; + id += exceptionId(cause, visited); + } + return id; + } + } + private static final NonBlockingHashMap wrappedLoggers = new NonBlockingHashMap<>(); @VisibleForTesting @@ -305,7 +460,7 @@ public class NoSpamLogger NoSpamLogStatement statement = lastMessage.get(key); if (statement == null) { - statement = new NoSpamLogStatement(s, minIntervalNanos); + statement = new NoSpamLogStatement(wrapped, s, minIntervalNanos); NoSpamLogStatement temp = lastMessage.putIfAbsent(key, statement); if (temp != null) statement = temp; diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index 97a9fb22ff..f0c32490b9 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -285,11 +285,11 @@ public class CompactionAccordIteratorsTest try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { cacheSize = cache.global.capacity(); - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { - cache.global.setCapacity(cacheSize); + commandStore.executor().setCapacity(cacheSize); } commandsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS); while (commandStore.executor().hasTasks()) diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index 189b08ed0d..539d58b268 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -98,7 +98,7 @@ public class AccordCommandTest public void basicCycleTest() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", unused -> commandStore.executor().setCapacity(0))); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(1); @@ -191,7 +191,7 @@ public class AccordCommandTest public void computeDeps() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((ExecutionContext.Empty)()->"Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)()->"Test", unused -> commandStore.executor().setCapacity(0))); TxnId txnId1 = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(2); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index c8f9d7cb7b..1f8f963ccc 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -379,7 +379,7 @@ public class AccordTestUtils Node.Id node = new Id(1); Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node))); AccordCommandStore store = createAccordCommandStore(node, now, topology); - store.execute((ExecutionContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20)); + store.execute((ExecutionContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().setCapacity(1 << 20)); return store; } diff --git a/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java index 2da44ed37f..9f88b78353 100644 --- a/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java @@ -84,7 +84,6 @@ import org.apache.cassandra.service.accord.execution.AccordExecutor.ExclusiveGlo import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; -import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.FBUtilities; @@ -101,7 +100,6 @@ import static org.apache.cassandra.service.accord.AccordTestUtils.createAccordCo import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn; import static org.apache.cassandra.service.accord.AccordTestUtils.keys; import static org.apache.cassandra.service.accord.AccordTestUtils.txnId; -import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; public class SafeTaskTest { @@ -178,9 +176,6 @@ public class SafeTaskTest private static Command createStableAndPersist(AccordCommandStore commandStore, TxnId txnId, Timestamp executeAt) { Command command = AccordTestUtils.Commands.stable(txnId, createPartialTxn(0), executeAt); - SaferCommand safeCommand = new SaferCommand(loaded(txnId, null)); - safeCommand.set(command); - appendDiffToLog(commandStore).accept(null, command); return command; } @@ -215,12 +210,12 @@ public class SafeTaskTest try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches()) { cacheSize = cache.global.capacity(); - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches()) { - cache.global.setCapacity(cacheSize); + commandStore.executor().setCapacity(cacheSize); } while (commandStore.executor().hasTasks()) @@ -265,11 +260,11 @@ public class SafeTaskTest try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { cacheSize = cache.global.capacity(); - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { - cache.global.setCapacity(cacheSize); + commandStore.executor().setCapacity(cacheSize); } while (commandStore.executor().hasTasks()) @@ -287,7 +282,7 @@ public class SafeTaskTest AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } Gen txnIdGen = rs -> txnId(1, clock.incrementAndGet(), 1); diff --git a/test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java b/test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java new file mode 100644 index 0000000000..a1a737317d --- /dev/null +++ b/test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java @@ -0,0 +1,529 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import com.google.common.collect.Sets; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; + +import accord.api.RoutingKey; +import accord.local.ExecutionContext; +import accord.local.Node; +import accord.local.Node.Id; +import accord.local.SafeCommandStore; +import accord.primitives.RoutingKeys; +import accord.primitives.TxnId; +import accord.topology.Shard; +import accord.topology.Topology; +import accord.utils.SortedArrays.SortedArrayList; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordTestUtils; +import org.apache.cassandra.service.accord.TokenRange; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.service.accord.execution.AccordExecutor.Mode; +import org.apache.cassandra.utils.concurrent.CountDownLatch; + +import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Lifecycle tests for {@link Task} and its interaction with {@link AccordExecutor}, covering the invariants the + * executor's callers rely on: + *

    + *
  • every task is notified exactly once, and every {@code chain()} eventually completes (success, failure or + * cancellation) - a task that is dropped silently hangs its caller forever;
    • + *
  • a task is completed exactly once, so {@code tasks}/{@link Tranches} accounting returns to zero and + * {@link AccordExecutor#hasTasks()} becomes false (otherwise {@code waitForQuiescence} and + * {@code afterSubmittedAndConsequences} never fire again);
    • + *
  • a task always releases its cache references, even when it fails;
    • + *
  • no failure is reported to the {@link accord.api.Agent} on any of these paths - an agent exception here means + * an internal invariant was broken, not that the user's operation failed.
    • + *
+ * + * Each test creates a private executor + command store so that failures cannot leak between tests. + */ +public class TaskLifecycleTest +{ + private static final long TIMEOUT_SECONDS = 30; + private static final AtomicLong clock = new AtomicLong(0); + + private final List executors = new CopyOnWriteArrayList<>(); + private final List stores = new CopyOnWriteArrayList<>(); + + @BeforeClass + public static void beforeClass() throws Throwable + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); + } + + @After + public void after() + { + stores.forEach(AccordCommandStore::shutdown); + stores.clear(); + executors.forEach(AccordExecutor::shutdown); + executors.clear(); + } + + /** + * An executor plus one command store, and a record of everything reported to the agent. + */ + private class Env + { + final List agentExceptions = new CopyOnWriteArrayList<>(); + final AccordExecutor executor; + final AccordCommandStore store; + + Env(Mode mode) + { + AccordAgent agent = new AccordAgent() + { + @Override + public void onException(Throwable t) + { + agentExceptions.add(t); + } + + @Override + public void onException(Throwable t, String context) + { + agentExceptions.add(t); + } + }; + agent.setup(Id.NONE); + this.executor = new AccordExecutorSyncSubmit(0, mode, "TaskLifecycleTest", agent); + executors.add(executor); + this.store = newStore(executor); + stores.add(store); + } + + void assertQuiescent() + { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (executor.hasTasks() && System.nanoTime() < deadline) + Thread.yield(); + assertThat(executor.hasTasks()).describedAs("executor still has registered tasks").isFalse(); + assertThat(executor.unsafeRunningCount()).describedAs("tasks still assigned to a runner").isZero(); + } + + void assertNoAgentExceptions() + { + assertThat(agentExceptions).describedAs("internal failures reported to the agent").isEmpty(); + } + } + + private static AccordCommandStore newStore(AccordExecutor executor) + { + TableMetadata metadata = Schema.instance.getTableMetadata("ks", "tbl"); + TokenRange range = TokenRange.fullRange(metadata.id, Murmur3Partitioner.instance); + Node.Id node = new Id(1); + Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[]{ node }), Sets.newHashSet(node))); + AccordCommandStore store = AccordTestUtils.createAccordCommandStore(node, clock::incrementAndGet, topology, executor); + // NOTE: capacity must be set via the executor, so that its derived maxWorkingCapacityInBytes is refreshed + executor.executeDirectlyWithLock(() -> { + executor.setCapacity(1 << 20); + executor.setWorkingSetSize(1 << 20); + }); + return store; + } + + private TxnId nextTxnId() + { + return AccordTestUtils.txnId(1, clock.incrementAndGet(), 1); + } + + private static Consumer noop() + { + return ignore -> {}; + } + + private static void await(CountDownLatch latch, String what) throws InterruptedException + { + assertThat(latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)).describedAs(what).isTrue(); + } + + /** + * A task that fails must cancel *every* consequence it accumulated, and must still be unregistered. + */ + @Test + public void failedTaskCancelsAllConsequences() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + int count = 4; + List> children = new ArrayList<>(); + CountDownLatch childrenDone = CountDownLatch.newCountDownLatch(count); + for (int i = 0 ; i < count ; ++i) + children.add(new AtomicReference<>()); + + AtomicReference parentFailure = new AtomicReference<>(); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + RuntimeException failure = new RuntimeException("deliberate failure"); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + for (int i = 0 ; i < count ; ++i) + { + AtomicReference child = children.get(i); + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "child" + i), noop(), + (r, f) -> { child.set(f == null ? new AssertionError("child should not have run") : f); childrenDone.decrement(); }); + } + throw failure; + }, (r, f) -> { parentFailure.set(f); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + assertThat(parentFailure.get()).isSameAs(failure); + await(childrenDone, "every consequence was notified"); + for (int i = 0 ; i < count ; ++i) + assertThat(children.get(i).get()).describedAs("child" + i).isInstanceOf(CancellationException.class); + + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * The happy path counterpart: consequences of a successful task all run. + */ + @Test + public void successfulTaskSubmitsAllConsequences() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + int count = 4; + AtomicInteger ran = new AtomicInteger(); + CountDownLatch childrenDone = CountDownLatch.newCountDownLatch(count); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + for (int i = 0 ; i < count ; ++i) + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "child" + i), + (Consumer) s -> ran.incrementAndGet(), + (r, f) -> { assertThat(f).isNull(); childrenDone.decrement(); }); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + await(childrenDone, "every consequence was notified"); + assertThat(ran.get()).isEqualTo(count); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * Cancelling one consequence must not disturb its siblings. Here the cancellation is submitted from within the + * parent, so it is applied after the parent has completed (i.e. once the consequence has been submitted). + */ + @Test + public void cancellingOneConsequenceDoesNotAffectSiblings() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + AtomicReference cancelled = new AtomicReference<>(); + CountDownLatch cancelledDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch siblingDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + Cancellable cancel = env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "cancelled"), noop(), + (r, f) -> { cancelled.set(f); cancelledDone.decrement(); }); + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "sibling"), noop(), + (r, f) -> { assertThat(f).isNull(); siblingDone.decrement(); }); + cancel.cancel(); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + await(siblingDone, "sibling of the cancelled consequence ran"); + await(cancelledDone, "cancelled consequence was notified"); + assertThat(cancelled.get()).isInstanceOf(CancellationException.class); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * As above, but the cancellation arrives from a foreign thread while the parent is still running, so the + * consequence is terminated before it has ever been submitted. + */ + @Test + public void externallyCancellingOneConsequenceDoesNotAffectSiblings() throws Throwable + { + Env env = new Env(RUN_WITHOUT_LOCK); // the parent blocks, so it must not hold the executor lock + AtomicReference toCancel = new AtomicReference<>(); + AtomicReference cancelled = new AtomicReference<>(); + CountDownLatch cancelledDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch siblingDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch consequencesAdded = CountDownLatch.newCountDownLatch(1); + CountDownLatch releaseParent = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + toCancel.set(env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "cancelled"), noop(), + (r, f) -> { cancelled.set(f); cancelledDone.decrement(); })); + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "sibling"), noop(), + (r, f) -> { assertThat(f).isNull(); siblingDone.decrement(); }); + consequencesAdded.decrement(); + releaseParent.awaitUninterruptibly(); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(consequencesAdded, "consequences were registered"); + toCancel.get().cancel(); + releaseParent.decrement(); + + await(parentDone, "parent was notified"); + await(siblingDone, "sibling of the cancelled consequence ran"); + await(cancelledDone, "cancelled consequence was notified"); + assertThat(cancelled.get()).isInstanceOf(CancellationException.class); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * A task whose prepare fails must be failed and completed, and the {@link ExclusiveExecutor} it was dispatched + * from must go on to dispatch the next task. + */ + @Test + public void prepareFailureCompletesTaskAndDispatchesNext() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + ExclusiveExecutor exclusive = env.executor.newExclusiveExecutor(0); + + CountDownLatch failed = CountDownLatch.newCountDownLatch(1); + CountDownLatch ran = CountDownLatch.newCountDownLatch(1); + RuntimeException failure = new RuntimeException("deliberate prepare failure"); + TestTask first = new TestTask(env.executor, exclusive, failure, failed); + TestTask second = new TestTask(env.executor, exclusive, null, ran); + + env.executor.executeDirectlyWithLock(() -> { + first.submitExclusiveNoExcept(); + second.submitExclusiveNoExcept(); + }); + + await(failed, "the task that failed to prepare was notified"); + assertThat(first.failure).isSameAs(failure); + await(ran, "the next queued task was dispatched"); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * A chain submitted to a command store hosted by a *different* executor must be submitted to that executor + * independently, not attached as a consequence of the running task (whose executor's lock we hold). + */ + @Test + public void consequenceOnAnotherExecutorIsSubmittedIndependently() throws Throwable + { + Env a = new Env(RUN_WITHOUT_LOCK); + Env b = new Env(RUN_WITHOUT_LOCK); + + CountDownLatch childDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + + a.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parentOnA"), (Consumer) safe -> { + b.store.execute(ExecutionContext.unsequenced(nextTxnId(), "childOnB"), noop(), + (r, f) -> { assertThat(f).isNull(); childDone.decrement(); }); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + await(childDone, "child on the other executor ran"); + a.assertNoAgentExceptions(); + b.assertNoAgentExceptions(); + a.assertQuiescent(); + b.assertQuiescent(); + } + + /** + * An incremental task with more keys than a single batch can hold must run several batches and then complete once. + */ + @Test + public void incrementalTaskRunsMultipleBatches() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + AtomicInteger batches = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + CountDownLatch done = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequencedIncrementalWrite(keys(0, 200), "incremental"), + (Consumer) safe -> batches.incrementAndGet(), + (r, f) -> { failure.set(f); done.decrement(); }); + + await(done, "incremental task completed"); + assertThat(failure.get()).isNull(); + assertThat(batches.get()).describedAs("expected several batches").isGreaterThan(1); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * An incremental task can be failed while it is parked between batches - this is what a late failure of one of + * its (optional) key loads does, see {@link AccordExecutor#onLoadedExclusive}. It must release its cache + * references, exactly as it would if it were failed before it first ran. + */ + @Test + public void incrementalTaskFailedBetweenBatchesReleasesResources() throws Throwable + { + Env env = new Env(RUN_WITHOUT_LOCK); + AtomicInteger batches = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + CountDownLatch done = CountDownLatch.newCountDownLatch(1); + RuntimeException loadFailure = new RuntimeException("simulated load failure"); + + Cancellable submitted = + env.store.execute(ExecutionContext.unsequencedIncrementalWrite(keys(1000, 1200), "incremental"), + (Consumer) safe -> { + batches.incrementAndGet(); + try { Thread.sleep(5); } catch (InterruptedException e) { throw new RuntimeException(e); } + }, + (r, f) -> { failure.set(f); done.decrement(); }); + + SafeTask task = (SafeTask) submitted; + AtomicReference failedIn = new AtomicReference<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (failedIn.get() == null && failure.get() == null && System.nanoTime() < deadline) + { + env.executor.executeDirectlyWithLock(() -> { + Task.State state = task.state(); + if (batches.get() > 0 && task.isState(Task.State.WAITING)) + { + failedIn.set(state); + task.tryFailAndCompleteExclusive(loadFailure, Task.State.FAILED); + } + }); + } + + assertThat(failedIn.get()).describedAs("did not observe the task parked between batches").isNotNull(); + await(done, "task was notified of the failure"); + assertThat(failure.get()).isSameAs(loadFailure); + int stillHeld = task.refs == null ? 0 : task.refs.size(); + assertThat(stillHeld).describedAs("cache references were not released").isZero(); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * The executor throttles loading while the cache is over its working-set budget, but must always be able to make + * *some* progress: if the only work that can run is loading, it must run regardless of the budget. + */ + @Test + public void progressesWithZeroCacheCapacity() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + env.executor.executeDirectlyWithLock(() -> { + env.executor.setCapacity(0); + env.executor.setWorkingSetSize(0); + }); + + for (int i = 0 ; i < 2 ; ++i) + { + AtomicReference failure = new AtomicReference<>(); + CountDownLatch done = CountDownLatch.newCountDownLatch(1); + env.store.execute(ExecutionContext.unsequencedIncrementalWrite(keys(2000 + i * 200, 2200 + i * 200), "zeroCapacity" + i), + noop(), (r, f) -> { failure.set(f); done.decrement(); }); + await(done, "task completed with a zero capacity cache"); + assertThat(failure.get()).isNull(); + } + + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + private static RoutingKeys keys(int from, int to) + { + TableMetadata metadata = Schema.instance.getTableMetadata("ks", "tbl"); + List keys = new ArrayList<>(to - from); + for (int i = from ; i < to ; ++i) + keys.add(AccordTestUtils.key(metadata, i).toUnseekable()); + return RoutingKeys.of(keys); + } + + /** + * A minimal {@link Plain} task: optionally fails during prepare, otherwise records that it ran. + */ + private static class TestTask extends Plain + { + final ExclusiveExecutor exclusiveExecutor; + final RuntimeException failPrepareWith; + final CountDownLatch notified; + volatile Throwable failure; + + TestTask(AccordExecutor executor, ExclusiveExecutor exclusiveExecutor, RuntimeException failPrepareWith, CountDownLatch notified) + { + super(executor, ExclusiveGroup.OTHER); + this.exclusiveExecutor = exclusiveExecutor; + this.failPrepareWith = failPrepareWith; + this.notified = notified; + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return exclusiveExecutor; + } + + @Override + void prepareExclusiveMayThrow() + { + if (failPrepareWith != null) + throw failPrepareWith; + } + + @Override + boolean runMayThrow() + { + if (failPrepareWith != null) + throw new AssertionError("should not have run"); + notified.decrement(); + return true; + } + + @Override + void reportFailureMayThrow(Throwable fail) + { + failure = fail; + notified.decrement(); + } + + @Override + public String description() + { + return "TestTask[failPrepare=" + (failPrepareWith != null) + ']'; + } + + @Override + String briefDescription() + { + return description(); + } + } +} From 988235c73607cda1b82aa9f7a62e2e12f1b42f21 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Thu, 30 Jul 2026 12:55:36 +0100 Subject: [PATCH 07/10] fix ThreadLocalTaskRunner reentrancy claude tests fix negative histogram values under simulation, and catch and refuse bad values in other cases --- modules/accord | 2 +- .../metrics/LogLinearDecayingHistograms.java | 7 + .../cassandra/metrics/LogLinearHistogram.java | 9 + .../service/accord/api/AccordAgent.java | 11 +- .../accord/execution/AbstractLockLoop.java | 4 +- .../service/accord/execution/SafeTask.java | 2 +- .../service/accord/execution/Task.java | 2 +- .../service/accord/execution/TaskRunner.java | 10 +- .../AccordCommandStoreExecutorTest.java | 441 ++++++++++++++++++ .../simulator/test/AccordExecutorTest.java | 77 ++- 10 files changed, 542 insertions(+), 23 deletions(-) create mode 100644 test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java diff --git a/modules/accord b/modules/accord index 4d091037c7..138e96dd0f 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 4d091037c7b751631d6ce204e0c192e2301162da +Subproject commit 138e96dd0f0b9e315be21a8659f1aa81ec48637a diff --git a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java index 4a17794827..46f5681450 100644 --- a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java +++ b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java @@ -59,6 +59,9 @@ public class LogLinearDecayingHistograms private void add(long histogramIndex, long value) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + Invariants.require(histogramIndex <= HISTOGRAM_INDEX_MASK); Invariants.require(value >= 0); if (value <= LARGE_VALUE) value <<= HISTOGRAM_INDEX_BITS; @@ -117,6 +120,9 @@ public class LogLinearDecayingHistograms public void increment(long value, long at) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + int index = index(value); if (bufferCount >= buffer.length) { @@ -127,6 +133,7 @@ public class LogLinearDecayingHistograms updateDecay(at); long v = Double.doubleToRawLongBits(increment) & VALUE_MASK; v |= histogramIndex | ((long)index << HISTOGRAM_INDEX_BITS); + Invariants.require(Double.longBitsToDouble(v & VALUE_MASK) > 0); buffer[bufferCount++] = v; } diff --git a/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java b/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java index 10c727c7c4..bae5779e66 100644 --- a/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java +++ b/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java @@ -54,6 +54,9 @@ public class LogLinearHistogram public void increment(long value) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + int index = index(value); buckets(index)[index]++; ++totalCount; @@ -61,6 +64,9 @@ public class LogLinearHistogram public void decrement(long value) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + int index = index(value); buckets(index)[index]--; --totalCount; @@ -68,6 +74,9 @@ public class LogLinearHistogram public void replace(long decrement, long increment) { + if (!Invariants.expect(decrement >= 0 && increment >= 0, "Negative value(s?) reported: %s and %s", decrement, increment)) + return; + int decrementIndex = index(decrement); int incrementIndex = index(increment); long[] buckets = buckets(Math.max(decrementIndex, incrementIndex)); diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index 3188fe0029..fc3ffa975b 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -39,6 +39,7 @@ import accord.api.Result; import accord.api.RoutingKey; import accord.api.Tracing; import accord.coordinate.Coordination; +import accord.coordinate.CoordinationFailed; import accord.coordinate.Exhausted; import accord.coordinate.Preempted; import accord.coordinate.Timeout; @@ -222,13 +223,19 @@ public class AccordAgent implements Agent, OwnershipEventListener return; AccordSystemMetrics.metrics.errors.inc(); - if (t instanceof CancellationException || t instanceof TimeoutException || t instanceof Timeout || t instanceof Preempted || t instanceof Exhausted || t instanceof LogUnavailableException) - // TODO (required): leaky logger, permitting multiple messages per time period and reporting how many were dropped + if (expectedException(t)) // TODO (required): leaky logger, permitting multiple messages per time period and reporting how many were dropped noSpamException.warn(exceptionId(t), t); else JVMStabilityInspector.uncaughtException(Thread.currentThread(), t); } + public static boolean expectedException(Throwable t) + { + if (t instanceof CancellationException) + return t.getCause() == null; + return t instanceof TimeoutException || t instanceof LogUnavailableException || t instanceof CoordinationFailed; + } + @Override public void onException(Throwable t) { diff --git a/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java index 41738b8dd3..7018ae6318 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java @@ -24,7 +24,6 @@ import java.util.function.Function; import accord.api.Agent; -import org.apache.cassandra.concurrent.CassandraThread; import org.apache.cassandra.service.accord.execution.Loops.LoopTask; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; @@ -248,7 +247,8 @@ abstract class AbstractLockLoop extends AbstractLoop @Override public void run() { - CassandraThread self = (CassandraThread) Thread.currentThread(); + Thread thread = Thread.currentThread(); + TaskRunner self = TaskRunner.get(thread); self.setAccordActiveExecutor(AbstractLockLoop.this); setWrapped(self); diff --git a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java index 7027adb478..5eee015214 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java @@ -748,7 +748,7 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa private void waitOnCacheQueuesExclusive() { Invariants.require(waitingForState == 0); - loadedAt = nanoTime(); + loadedAt = Math.max(createdAt, nanoTime()); executor().runnable.incrementArrivals(this); commandStore.exclusiveExecutor().incrementArrivals(this); diff --git a/src/java/org/apache/cassandra/service/accord/execution/Task.java b/src/java/org/apache/cassandra/service/accord/execution/Task.java index bd0af0d472..6a3e50309f 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/Task.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Task.java @@ -601,7 +601,7 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable, Deb { Invariants.require(is(PREPARED)); Invariants.require(is(NOT_YET_RUN) || is(RUN_INCOMPLETE)); - runningAt = nanoTime(); + runningAt = Math.max(createdAt, nanoTime()); setRunState(RunState.RUNNING); if (DEBUG_EXECUTION) ((DebugTask) resources).onRunning(); } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java index 2dffb11e18..0d9705a186 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java @@ -48,6 +48,7 @@ public interface TaskRunner final class ThreadLocalTaskRunner implements TaskRunner { private AccordExecutor lockedExecutor; + private int lockedExecutorDepth; private AccordExecutor activeExecutor; volatile Task activeTask; @@ -74,16 +75,17 @@ public interface TaskRunner @Override public boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) { - if (lockedExecutor != null) - return false; - lockedExecutor = newLockedExecutor; + if (lockedExecutor == null) lockedExecutor = newLockedExecutor; + else if (lockedExecutor != newLockedExecutor) return false; + ++lockedExecutorDepth; return true; } @Override public void exitAccordLockedExecutor() { - lockedExecutor = null; + if (--lockedExecutorDepth == 0) + lockedExecutor = null; } @Override diff --git a/test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java b/test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java new file mode 100644 index 0000000000..2d1a68a0d2 --- /dev/null +++ b/test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.LockSupport; +import java.util.function.LongSupplier; +import java.util.function.ToLongFunction; + +import org.junit.Test; + +import accord.api.AsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; +import accord.api.ProgressLog; +import accord.api.Result; +import accord.api.RoutingKey; +import accord.api.Scheduler; +import accord.coordinate.Coordinations; +import accord.impl.DefaultLocalListeners; +import accord.impl.TestAgent; +import accord.impl.DefaultLocalListeners.NotifySink; +import accord.impl.DefaultRemoteListeners; +import accord.impl.basic.InMemoryJournal; +import accord.local.CommandStores.RangesForEpoch; +import accord.local.DurableBefore; +import accord.local.TimeService; +import accord.local.durability.DurabilityService; +import accord.local.ExecutionContext; +import accord.local.LoadKeys; +import accord.local.LoadKeysFor; +import accord.local.Node.Id; +import accord.local.NodeCommandStoreService; +import accord.local.SafeCommandStore; +import accord.local.SafeState; +import accord.primitives.Ballot; +import accord.primitives.Range; +import accord.primitives.Ranges; +import accord.primitives.Route; +import accord.primitives.RoutingKeys; +import accord.primitives.Timestamp; +import accord.primitives.TxnId; +import accord.primitives.Writes; +import accord.topology.TopologyManager; +import accord.utils.DefaultRandom; +import accord.utils.Invariants; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.TokenRange; +import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.simulator.test.SimulationTestBase; +import org.apache.cassandra.utils.concurrent.CountDownLatch; +import org.apache.cassandra.utils.concurrent.Future; + +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; + +/** + * Simulator driven test of {@link SafeTask} execution on a real {@link AccordCommandStore}, i.e. of the parts of the + * executor {@link AccordExecutorTest} cannot reach: cache references, the per-key/per-txnId queues, and the ordering + * guarantees they provide. + * + *

The command store is real, but everything below it is synthetic: an in-memory journal, and cache load functions + * that return {@code null} (an uninitialised value) rather than reading from {@code system_accord}. So no schema, no + * cluster metadata, no commit log - only the executor and task machinery is under test. + * + *

Scope (phase 1)

+ * A single executor and command store; {@code SYNC} key-domain contexts; no nesting, no cancellation, no failures. + * Verifies: + *
    + *
  • liveness: every submission is notified exactly once, and successfully;
    • + *
  • declared access: while running, a task holds a reference for every key and txnId it declared;
    • + *
  • mutual exclusion: two tasks that declare the same key or txnId never execute concurrently - the + * central guarantee of the cache-entry queues;
    • + *
  • no leaks: once everything has been notified, no cache entry is still referenced and no task retains + * its references.
    • + *
+ * + *

Notes

+ *
    + *
  • only the {@code SIGNAL} and {@code ASYNC} submission models can be simulated; {@code SYNC}/{@code SEMI_SYNC} + * (the production default) need the simulator to intercept {@link java.util.concurrent.locks.ReentrantLock}, + * see the TODO in {@link AccordExecutorTest};
    • + *
  • only one test method may be run per JVM (pre-existing limitation of {@link AccordExecutorTest}); ant forks + * per test method.
    • + *
+ */ +public class AccordCommandStoreExecutorTest extends SimulationTestBase +{ + private static final int KEYS = 8; + private static final int TXN_IDS = 8; + private static final int SUBMIT_THREADS = 8; + private static final int OUTER_LOOP = 5; + private static final int INNER_LOOP = 10; + private static final int MAX_KEYS_PER_TASK = 3; + private static final int MAX_TXN_IDS_PER_TASK = 2; + private static final int MAX_TASKS = SUBMIT_THREADS * OUTER_LOOP * INNER_LOOP; + + @Test + public void signalLoopTest() + { + test(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, 4, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new RecordingAgent())); + } + + @Test + public void asyncSubmitTest() + { + test(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, 4, i -> "Loop" + i, new RecordingAgent())); + } + + private void test(SerializableSupplier executorSupplier) + { + simulate(arr(() -> { + try + { + DatabaseDescriptor.daemonInitialization(); + AccordExecutor executor = executorSupplier.get(); + Env env = new Env(executor); + + for (float loadDelayChance : new float[]{ 0f, 0.1f }) + { + for (float sleepChance : new float[]{ 0f, 0.1f }) + { + System.out.printf("loadDelayChance %.2f, sleepChance %.2f%n", loadDelayChance, sleepChance); + env.loadDelayChance = loadDelayChance; + env.round(sleepChance); + } + } + } + catch (Throwable t) + { + throw new RuntimeException(t); + } + }), + () -> {}, 1L); + } + + /** + * Records everything reported to the agent: on these paths any report indicates a broken internal invariant, + * not a failed operation. We do not use {@link AccordAgent}, as reporting an exception there touches + * {@code AccordSystemMetrics}, which requires a started {@code AccordService}. + */ + public static class RecordingAgent extends TestAgent + { + static final List exceptions = new CopyOnWriteArrayList<>(); + + @Override + public void onException(Throwable t) + { + exceptions.add(t); + System.out.println("### agent.onException: " + t); + t.printStackTrace(System.out); + } + + @Override + public void onException(Throwable t, String context) + { + onException(t); + } + } + + static class Env + { + final AccordExecutor executor; + final AccordCommandStore store; + final RoutingKey[] keys = new RoutingKey[KEYS]; + final TxnId[] txnIds = new TxnId[TXN_IDS]; + + /** key/txnId ordinal -> id of the task currently executing with it, or -1 */ + final AtomicIntegerArray keyOwner = new AtomicIntegerArray(KEYS); + final AtomicIntegerArray txnIdOwner = new AtomicIntegerArray(TXN_IDS); + + /** reset for every round: task ids are round-local */ + AtomicInteger nextTaskId = new AtomicInteger(); + /** task id -> number of times its callback has been invoked; must be exactly one for every submission */ + AtomicIntegerArray notifications = new AtomicIntegerArray(MAX_TASKS); + final List failures = new CopyOnWriteArrayList<>(); + SafeTask[] tasks = new SafeTask[MAX_TASKS]; + + volatile float loadDelayChance; + + Env(AccordExecutor executor) + { + this.executor = executor; + TableId tableId = TableId.fromUUID(new java.util.UUID(0, 1)); + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + this.store = newCommandStore(tableId, partitioner, executor); + for (int i = 0 ; i < KEYS ; ++i) + keys[i] = new TokenKey(tableId, partitioner.getToken(Int32Type.instance.decompose(i))); + for (int i = 0 ; i < TXN_IDS ; ++i) + txnIds[i] = TxnId.fromValues(1, 1 + i, 0, new Id(1)); + for (int i = 0 ; i < KEYS ; ++i) + keyOwner.set(i, -1); + for (int i = 0 ; i < TXN_IDS ; ++i) + txnIdOwner.set(i, -1); + + // synthetic loads: an uninitialised value, optionally after some simulated latency + executor.cacheUnsafe().types().forEach(type -> type.unsafeSetLoadFunction((ignoreStore, ignoreKey) -> { + maybePark(loadDelayChance); + return null; + })); + } + + void round(float sleepChance) throws ExecutionException, InterruptedException + { + nextTaskId = new AtomicInteger(); + notifications = new AtomicIntegerArray(MAX_TASKS); + tasks = new SafeTask[MAX_TASKS]; + ExecutorPlus submit = executorFactory().pooled("submit", SUBMIT_THREADS); + try + { + List> submitting = new ArrayList<>(); + for (int i = 0 ; i < SUBMIT_THREADS ; ++i) + { + int id = i; + submitting.add(submit.submit(() -> { + for (int outer = 0 ; outer < OUTER_LOOP ; ++outer) + { + CountDownLatch inner = CountDownLatch.newCountDownLatch(INNER_LOOP); + for (int j = 0 ; j < INNER_LOOP ; ++j) + submitOne(sleepChance, inner); + inner.awaitUninterruptibly(); + System.out.println("Loop " + id + '.' + outer); + } + })); + } + for (Future f : submitting) + f.get(); + } + finally + { + submit.shutdown(); + } + + verifyRoundComplete(); + } + + private void submitOne(float sleepChance, CountDownLatch done) + { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + int taskId = nextTaskId.getAndIncrement(); + int[] keyOrdinals = distinct(rnd, KEYS, 1 + rnd.nextInt(MAX_KEYS_PER_TASK)); + int[] txnIdOrdinals = distinct(rnd, TXN_IDS, rnd.nextInt(1 + MAX_TXN_IDS_PER_TASK)); + + RoutingKeys declaredKeys = keys(keyOrdinals); + TxnId primary = txnIdOrdinals.length > 0 ? txnIds[txnIdOrdinals[0]] : null; + TxnId additional = txnIdOrdinals.length > 1 ? txnIds[txnIdOrdinals[1]] : null; + ExecutionContext context = ExecutionContext.contextFor(primary, additional, declaredKeys, LoadKeys.SYNC, LoadKeysFor.READ_WRITE, "task" + taskId); + + Cancellable submitted = + store.execute(context, (java.util.function.Consumer) safeStore -> body(taskId, keyOrdinals, txnIdOrdinals, safeStore, sleepChance), + (success, fail) -> { + notifications.incrementAndGet(taskId); + if (fail != null) + failures.add(fail); + done.decrement(); + }); + tasks[taskId] = (SafeTask) submitted; + } + + private void body(int taskId, int[] keyOrdinals, int[] txnIdOrdinals, SafeCommandStore safeStore, float sleepChance) + { + SafeTask task = ((SaferCommandStore) safeStore).task; + + // declared access: we must hold a reference for everything we declared, and be able to look it up + for (int k : keyOrdinals) + { + SafeState ref = task.refs.get(keys[k]); + Invariants.require(ref != null, "task %d declared key %d but holds no reference for it", taskId, k); + Invariants.require(SaferState.global(ref).references() > 0, "task %d holds an unreferenced entry for key %d", taskId, k); + safeStore.ifLoadedAndInitialised(keys[k]); + } + for (int t : txnIdOrdinals) + { + SafeState ref = task.refs.get(txnIds[t]); + Invariants.require(ref != null, "task %d declared txnId %d but holds no reference for it", taskId, t); + Invariants.require(SaferState.global(ref).references() > 0, "task %d holds an unreferenced entry for txnId %d", taskId, t); + safeStore.ifInitialised(txnIds[t]); + } + + // mutual exclusion: nobody else may be executing with any key or txnId we declared + int keysTaken = 0, txnIdsTaken = 0; + try + { + while (keysTaken < keyOrdinals.length) + { + int k = keyOrdinals[keysTaken]; + Invariants.require(keyOwner.compareAndSet(k, -1, taskId), + "task %d ran concurrently with task %d, which also declared key %d", taskId, keyOwner.get(k), k); + ++keysTaken; + } + while (txnIdsTaken < txnIdOrdinals.length) + { + int t = txnIdOrdinals[txnIdsTaken]; + Invariants.require(txnIdOwner.compareAndSet(t, -1, taskId), + "task %d ran concurrently with task %d, which also declared txnId %d", taskId, txnIdOwner.get(t), t); + ++txnIdsTaken; + } + + maybePark(sleepChance); + } + finally + { + while (keysTaken > 0) + keyOwner.set(keyOrdinals[--keysTaken], -1); + while (txnIdsTaken > 0) + txnIdOwner.set(txnIdOrdinals[--txnIdsTaken], -1); + } + } + + private void verifyRoundComplete() + { + Invariants.require(failures.isEmpty(), "%s", failures); + Invariants.require(RecordingAgent.exceptions.isEmpty(), "%s", RecordingAgent.exceptions); + + int lastTaskId = nextTaskId.get(); + Invariants.require(lastTaskId == MAX_TASKS, "submitted %d tasks, expected %d", lastTaskId, MAX_TASKS); + for (int taskId = 0 ; taskId < lastTaskId ; ++taskId) + { + Invariants.require(notifications.get(taskId) == 1, "task %d was notified %d times", taskId, notifications.get(taskId)); + Invariants.require(tasks[taskId].refs == null, "task %d did not release its references", taskId); + } + for (int i = 0 ; i < KEYS ; ++i) + Invariants.require(keyOwner.get(i) == -1, "key %d is still owned by task %d", i, keyOwner.get(i)); + for (int i = 0 ; i < TXN_IDS ; ++i) + Invariants.require(txnIdOwner.get(i) == -1, "txnId %d is still owned by task %d", i, txnIdOwner.get(i)); + + try (AccordCommandStore.ExclusiveCaches caches = store.lockCaches()) + { + for (AccordCacheEntry entry : caches.commands()) + Invariants.require(entry.references() == 0, "%s is still referenced", entry); + for (AccordCacheEntry entry : caches.commandsForKeys()) + Invariants.require(entry.references() == 0, "%s is still referenced", entry); + } + } + + private RoutingKeys keys(int[] ordinals) + { + RoutingKey[] result = new RoutingKey[ordinals.length]; + for (int i = 0 ; i < ordinals.length ; ++i) + result[i] = keys[ordinals[i]]; + return RoutingKeys.of(result); + } + } + + private static void maybePark(float chance) + { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + if (chance > 0 && rnd.nextFloat() < chance) + LockSupport.parkNanos(rnd.nextInt(10000, 100000)); + } + + /** {@code count} distinct ordinals in [0..limit), in ascending order (so a task never declares a key twice) */ + private static int[] distinct(ThreadLocalRandom rnd, int limit, int count) + { + int mask = 0; + for (int i = 0 ; i < count ; ++i) + mask |= 1 << rnd.nextInt(limit); + int[] result = new int[Integer.bitCount(mask)]; + for (int i = 0, ordinal = 0 ; mask != 0 ; ++ordinal, mask >>>= 1) + { + if ((mask & 1) != 0) + result[i++] = ordinal; + } + return result; + } + + private static AccordCommandStore newCommandStore(TableId tableId, IPartitioner partitioner, AccordExecutor executor) + { + AtomicLong clock = new AtomicLong(); + LongSupplier now = clock::incrementAndGet; + Id nodeId = new Id(1); + NodeCommandStoreService node = new NodeCommandStoreService() + { + private final ToLongFunction elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MICROSECONDS, this::now); + private long stamp = 0; + + @Override public AsyncExecutor someExecutor() { return null; } + @Override public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } + @Override public accord.api.Timeouts timeouts() { return null; } + @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; } + @Override public DurabilityService durability() { return null; } + @Override public Id id() { return nodeId; } + @Override public long epoch() { return 1; } + @Override public long now() { return now.getAsLong(); } + @Override public long uniqueNow(long atLeast) { return now.getAsLong(); } + @Override public long elapsed(TimeUnit units) { return elapsed.applyAsLong(units); } + @Override public TopologyManager topology() { throw new UnsupportedOperationException(); } + @Override public Coordinations coordinations() { return new Coordinations(); } + @Override public Scheduler scheduler() { return null; } + @Override public long currentStamp() { return stamp; } + @Override public void updateStamp() { ++stamp; } + @Override public boolean isReplaying() { return false; } + @Override public void reportLocalExecution(TxnId txnId, Route route, Ballot ballot, Timestamp applyAt, Writes writes, Result result) {} + }; + + Range range = TokenRange.fullRange(tableId, partitioner); + RangesForEpoch rangesForEpoch = new RangesForEpoch(1, Ranges.of(range)); + AccordCommandStore store = new AccordCommandStore(0, node, new RecordingAgent(), null, + cs -> new ProgressLog.NoOpProgressLog(), + cs -> new DefaultLocalListeners(null, new DefaultRemoteListeners.NoOpRemoteListeners(), new NotifySink.NoOpNotifySink()), + rangesForEpoch, + new InMemoryJournal(nodeId, new DefaultRandom(1)), + executor); + executor.executeDirectlyWithLock(() -> { + executor.setCapacity(8 << 20); + executor.setWorkingSetSize(4 << 20); + }); + return store; + } +} diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index 3d441ae04e..e075a0da7c 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -87,7 +87,7 @@ public class AccordExecutorTest extends SimulationTestBase { final AtomicInteger nextId = new AtomicInteger(); final AtomicInteger doneBefore = new AtomicInteger(); - final ConcurrentHashMap>> consequences = new ConcurrentHashMap<>(); + final ConcurrentHashMap consequences = new ConcurrentHashMap<>(); boolean isDone() { @@ -106,11 +106,8 @@ public class AccordExecutorTest extends SimulationTestBase { for (int id = from ; id < before ; ++id) { - for (Future future : consequences.get(id)) - { - if (!future.isDone()) - return false; - } + if (!consequences.get(id).isDone()) + return false; } return true; } @@ -119,6 +116,34 @@ public class AccordExecutorTest extends SimulationTestBase { volatile boolean started; + /** + * The number of submissions in this group that have not yet invoked their callback. + *

+ * We cannot decide this by iterating the queue: each level of the recursion appends the next level's + * future while it runs, i.e. strictly before its own future completes, so the queue is still growing + * while it is being consumed - and {@link ConcurrentLinkedQueue}'s iterator prefetches, so an iterator + * that has reached the tail terminates and never sees the later additions. A counter is exact, because + * for the same reason it can only reach zero once the whole group has finished: a consequence is + * registered before its parent's future completes. + */ + final AtomicInteger outstanding = new AtomicInteger(); + + void submitted(Future future) + { + outstanding.incrementAndGet(); + add(future); + } + + void completed() + { + outstanding.decrementAndGet(); + } + + boolean isDone() + { + return outstanding.get() == 0; + } + void ensureStarted() { if (started) @@ -171,14 +196,29 @@ public class AccordExecutorTest extends SimulationTestBase void submit(AsyncExecutor executor, Consequences consequences, Consumer run) { AsyncPromise future = new AsyncPromise<>(); - consequences.add(future); + consequences.submitted(future); Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { - if (fail == null) future.trySuccess(null); - else + try { - future.tryFailure(fail); - if (fail instanceof CancellationException) - run.accept(submitted.allocate()); + if (fail == null) future.trySuccess(null); + else + { + future.tryFailure(fail); + // replace the cancelled work with an equivalent submission, so that cancelling does not + // simply reduce the amount of work we perform. + // + // NOTE: this callback may be invoked from inside the executor (e.g. while cancelling, which + // happens with the executor's lock held), so we must not run the body here: it re-enters the + // executor (afterSubmittedAndConsequences, and the lock itself), which is forbidden for a + // thread that already holds it. Submit it as ordinary work instead, so it runs in a legal + // context - this also widens the set of interleavings we explore. + if (fail instanceof CancellationException) + submit(executor, submitted.allocate(), run); + } + } + finally + { + consequences.completed(); } }); consequences.ensureStarted(); @@ -248,8 +288,16 @@ public class AccordExecutorTest extends SimulationTestBase for (Future f : done) f.get(); + // the awaits in submitLoop only wait for the submissions they can see, and the + // deeper levels of each group are only registered as they run, so work may still + // be in flight here; drain the executor before verifying that everything we + // recorded has in fact run + executor.waitForQuiescence(); if (!submitted.isDone()) throw new AssertionError(); + // nothing is running, so we can now safely inspect every future we recorded + for (int id = 0 ; id < submitted.nextId.get() ; ++id) + await(submitted.consequences.get(id), CancellationException.class); } } } @@ -350,6 +398,11 @@ public class AccordExecutorTest extends SimulationTestBase }); } + /** + * Waits for those submissions that are visible in {@code await}; note that this deliberately does not wait for + * the whole tree of consequences (which is still being appended to as each level runs), so that submission + * threads continue to overlap their outer loops - {@link Submitted#isDone} is verified once the executor drains. + */ private static void await(Collection> await, @Nullable Class ignore) throws InterruptedException, ExecutionException { for (Future future : await) From ba683a02fde1834e7f8c7ce5528827703bd1aaca Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Tue, 19 May 2026 15:07:59 +0100 Subject: [PATCH 08/10] Improve Rebootstrap: - Distinguish corrupt/incomplete cases; latter case only refuses uncertain data - Ensures quorum durability before marking own log faulty - Refuse unsafe operations more selectively once log marked faulty - Simplify and merge logic with standard bootstrap --- .gitmodules | 2 +- .../managing/configuration/configuration.adoc | 2 +- modules/accord | 2 +- .../apache/cassandra/concurrent/Stage.java | 2 +- .../apache/cassandra/config/AccordConfig.java | 42 +- .../cassandra/config/DatabaseDescriptor.java | 14 + .../db/virtual/AccordDebugKeyspace.java | 27 +- .../exceptions/ExceptionSerializer.java | 11 +- .../apache/cassandra/journal/Compactor.java | 7 +- .../cassandra/journal/SegmentCompactor.java | 6 +- .../service/accord/AccordCommandStore.java | 8 +- .../service/accord/AccordDataStore.java | 81 +- .../accord/AccordFetchCoordinator.java | 29 +- .../service/accord/AccordKeyspace.java | 3 +- .../service/accord/AccordMessageSink.java | 8 + .../service/accord/AccordResult.java | 4 +- .../service/accord/AccordService.java | 133 ++- .../service/accord/IAccordService.java | 8 +- .../service/accord/api/AccordAgent.java | 62 +- .../accord/api/AccordWaitStrategies.java | 8 +- .../accord/interop/AccordInteropAdapter.java | 20 +- .../interop/AccordInteropExecution.java | 16 +- .../journal/AbstractSegmentCompactor.java | 10 + .../serializers/CommandStoreSerializers.java | 5 +- .../accord/serializers/FetchSerializers.java | 20 +- .../tcm/sequences/BootstrapAndJoin.java | 3 +- test/conf/logback-info.xml | 56 + .../test/accord/AccordCQLTestBase.java | 27 +- .../accord/AccordIncrementalRepairTest.java | 4 +- .../test/accord/AccordTestBase.java | 7 +- .../test/accord/load/AccordLoadTestBase.java | 976 ++++++++++++++++++ .../load/AccordRebootstrapLoadTest.java | 66 ++ .../test/accord/load/AccordYcsbLoadTest.java | 137 +++ .../test/accord/load/LoadSettings.java | 338 ++++++ .../fuzz/topology/AccordRebootstrapTest.java | 2 +- .../service/accord/AccordMessageSinkTest.java | 2 +- .../service/accord/AccordTestUtils.java | 16 +- .../accord/SimulatedAccordCommandStore.java | 7 + .../CommandsForKeySerializerTest.java | 2 + 39 files changed, 1946 insertions(+), 227 deletions(-) create mode 100644 test/conf/logback-info.xml create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java diff --git a/.gitmodules b/.gitmodules index 4b1eea6020..c00814bf07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "modules/accord"] path = modules/accord url = https://github.com/belliottsmith/cassandra-accord.git - branch = executor-fair + branch = rerebootstrap diff --git a/doc/modules/cassandra/pages/managing/configuration/configuration.adoc b/doc/modules/cassandra/pages/managing/configuration/configuration.adoc index 9b4df248b2..0933da73fd 100644 --- a/doc/modules/cassandra/pages/managing/configuration/configuration.adoc +++ b/doc/modules/cassandra/pages/managing/configuration/configuration.adoc @@ -153,7 +153,7 @@ configuration changes in the near future. `@Replaces` is the annotation to be used when you make changes to any configuration parameters in `Config` class and `cassandra.yaml`, and you want to add backward compatibility with previous Cassandra versions. `Converters` class enumerates the different methods used for backward compatibility. `IDENTITY` is the one used for name change only. For more information about the other Converters, please, check the JavaDoc in the class. -For backward compatibility virtual table `Settings` contains both the old and the new +For backward compatibility virtual table `LoadSettings` contains both the old and the new parameters with the old and the new value format. Only exception at the moment are the following three parameters: `key_cache_save_period`, `row_cache_save_period` and `counter_cache_save_period` which appear only once with the new value format. The old names and value format still can be used at least until the next major release. Deprecation warning is emitted on startup. diff --git a/modules/accord b/modules/accord index 138e96dd0f..673cf2ff93 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 138e96dd0f0b9e315be21a8659f1aa81ec48637a +Subproject commit 673cf2ff93a4b32a392c91feacda09bb48ac16e7 diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java index 557c1ca0a4..4e99729e0f 100644 --- a/src/java/org/apache/cassandra/concurrent/Stage.java +++ b/src/java/org/apache/cassandra/concurrent/Stage.java @@ -47,7 +47,7 @@ public enum Stage MUTATION (true, "MutationStage", "request", DatabaseDescriptor::getConcurrentWriters, DatabaseDescriptor::setConcurrentWriters, Stage::multiThreadedLowSignalStage), COUNTER_MUTATION (true, "CounterMutationStage", "request", DatabaseDescriptor::getConcurrentCounterWriters, DatabaseDescriptor::setConcurrentCounterWriters, Stage::multiThreadedLowSignalStage), VIEW_MUTATION (true, "ViewMutationStage", "request", DatabaseDescriptor::getConcurrentViewWriters, DatabaseDescriptor::setConcurrentViewWriters, Stage::multiThreadedLowSignalStage), - ACCORD_MIGRATION (false, "AccordMigrationStage", "request", DatabaseDescriptor::getAccordConcurrentOps, DatabaseDescriptor::setConcurrentAccordOps, Stage::multiThreadedLowSignalStage), + ACCORD_MIGRATION (false, "AccordMigrationStage", "request", DatabaseDescriptor::getAccordConcurrentMigrationOps, DatabaseDescriptor::setConcurrentAccordMigrationOps, Stage::multiThreadedLowSignalStage), GOSSIP (true, "GossipStage", "internal", () -> 1, null, Stage::singleThreadedStage), REQUEST_RESPONSE (false, "RequestResponseStage", "request", FBUtilities::getAvailableProcessors, null, Stage::multiThreadedLowSignalStage), ANTI_ENTROPY (false, "AntiEntropyStage", "internal", () -> 1, null, Stage::singleThreadedStage), diff --git a/src/java/org/apache/cassandra/config/AccordConfig.java b/src/java/org/apache/cassandra/config/AccordConfig.java index 9813f3ffa0..50d00d2b6f 100644 --- a/src/java/org/apache/cassandra/config/AccordConfig.java +++ b/src/java/org/apache/cassandra/config/AccordConfig.java @@ -34,7 +34,7 @@ import org.apache.cassandra.journal.Params; import org.apache.cassandra.service.accord.serializers.Version; import org.apache.cassandra.service.consensus.TransactionalMode; -import static org.apache.cassandra.config.AccordConfig.CatchupMode.NORMAL; +import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.HLC_FIFO; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_POOL_PER_SHARD; import static org.apache.cassandra.config.AccordConfig.QueueSubmissionModel.SYNC; import static org.apache.cassandra.config.AccordConfig.RangeIndexMode.in_memory; @@ -214,6 +214,10 @@ public class AccordConfig * once this number of tasks are blocked behind it, regardless of batch_size. */ public Integer queue_nonsync_blocked_limit; + /** + * The number of threads that may be used to execute distributed requests for migration tasks + */ + public volatile OptionaldPositiveInt migration_concurrency = OptionaldPositiveInt.UNDEFINED; /** * If set, the signal loop does not match park/unpark pairs, but instead consumers perform timed-park spin waits @@ -258,6 +262,7 @@ public class AccordConfig public String expire_epoch_wait = "10s"; // we don't want to wait ages for durability as it blocks other durability progress; even this might be too long, as we can always retry public String expire_durability = "10s*attempts <= 30s"; + public String slow_durability = "10s"; public String slow_syncpoint_preaccept = "10s"; public String slow_txn_preaccept = "30ms <= p50*2 <= 1000ms"; public String slow_read = "30ms <= p50*2 <= 1000ms"; @@ -294,12 +299,12 @@ public class AccordConfig */ public volatile TransactionalRangeMigration range_migration = TransactionalRangeMigration.auto; - public enum CatchupMode + public enum CatchupFallbackMode { - DISABLED, - NORMAL, - FALLBACK_TO_HARD, - HARD + IGNORE, + EXIT, + REBOOTSTRAP, + REBOOTSTRAP_AND_CATCHUP } /** @@ -344,16 +349,19 @@ public class AccordConfig public int commands_for_key_prune_interval = 64; public DurationSpec.IntSecondsBound max_conflicts_prune_delta = new DurationSpec.IntSecondsBound(1); + public int catchup_on_start_max_attempts = 5; + public boolean catchup_on_start = true; public DurationSpec.IntSecondsBound catchup_on_start_success_latency = new DurationSpec.IntSecondsBound(60); public DurationSpec.IntSecondsBound catchup_on_start_fail_latency = new DurationSpec.IntSecondsBound(900); - public int catchup_on_start_max_attempts = 5; - // TODO (required): roll this back to catchup_on_start_exit_on_failure: true - public boolean catchup_on_start_exit_on_failure = false; - public CatchupMode catchup_on_start = NORMAL; + // TODO (required): default this to EXIT or REBOOTSTRAP + public CatchupFallbackMode catchup_on_start_on_timeout = CatchupFallbackMode.IGNORE; + public CatchupFallbackMode catchup_on_start_on_error = CatchupFallbackMode.IGNORE; + public CatchupFallbackMode catchup_on_start_on_rebootstrap_fallback = CatchupFallbackMode.IGNORE; + public DurationSpec.IntSecondsBound shutdown_grace_period = new DurationSpec.IntSecondsBound(15 * 60); + public boolean execute_waiting_on_start = true; public DurationSpec.IntSecondsBound execute_waiting_on_start_timeout = new DurationSpec.IntSecondsBound(0); public boolean execute_waiting_on_start_fail_on_timeout = false; - public DurationSpec.IntSecondsBound shutdown_grace_period = new DurationSpec.IntSecondsBound(15 * 60); public enum RangeIndexMode { in_memory, journal_sai } public RangeIndexMode range_index_mode = in_memory; @@ -391,7 +399,17 @@ public class AccordConfig * Replay journal entries for commands that are not durable to the data or command stores. * THIS MODE IS NOT YET SAFE TO RUN */ - NON_DURABLE + NON_DURABLE, + + /** + * Don't replay, simply rebootstrap, marking our log as incomplete. + */ + REBOOTSTRAP_INCOMPLETE, + + /** + * Don't replay, simply rebootstrap, marking our log as corrupted/unavailable. + */ + REBOOTSTRAP_RESET } public enum ReplaySavePoint diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 5bc6de7ac1..765ee10d45 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -2967,6 +2967,20 @@ public class DatabaseDescriptor conf.accord.queue_thread_count = new OptionaldPositiveInt(concurrent_operations); } + public static int getAccordConcurrentMigrationOps() + { + return conf.accord.migration_concurrency.or(2 * FBUtilities.getAvailableProcessors()); + } + + public static void setConcurrentAccordMigrationOps(int concurrent_operations) + { + if (concurrent_operations < 0) + { + throw new IllegalArgumentException("Concurrent accord operations must be non-negative"); + } + conf.accord.migration_concurrency = new OptionaldPositiveInt(concurrent_operations); + } + public static int getFlushWriters() { return conf.memtable_flush_writers; diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 5b3e9f79e5..9b149de9f2 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -62,7 +62,8 @@ import accord.impl.CommandChange; import accord.impl.progresslog.DefaultProgressLog; import accord.impl.progresslog.DefaultProgressLog.ModeFlag; import accord.impl.progresslog.TxnStateKind; -import accord.local.CatchupHard; +import accord.local.BootstrapReason; +import accord.local.Catchup; import accord.local.Cleanup; import accord.local.Command; import accord.local.CommandStore; @@ -166,6 +167,8 @@ import org.apache.cassandra.utils.concurrent.Future; import static accord.coordinate.Infer.InvalidIf.NotKnownToBeInvalid; import static accord.impl.CommandChange.Field.ACCEPTED; import static accord.impl.CommandChange.Field.PROMISED; +import static accord.local.BootstrapReason.LOG_CORRUPTED; +import static accord.local.BootstrapReason.LOG_INCOMPLETE; import static accord.local.RedundantStatus.Property.GC_BEFORE; import static accord.local.RedundantStatus.Property.LOCALLY_APPLIED; import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_COMMAND_STORE; @@ -183,6 +186,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; import static org.apache.cassandra.db.virtual.AbstractLazyVirtualTable.OnTimeout.BEST_EFFORT; import static org.apache.cassandra.db.virtual.AbstractLazyVirtualTable.OnTimeout.FAIL; +import static org.apache.cassandra.db.virtual.AccordDebugKeyspace.CommandStoreOpsTable.CommandStoreOp.REBOOTSTRAP_CORRUPTED; import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.ASC; import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.SORTED; import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.UNSORTED; @@ -1988,8 +1992,10 @@ public class AccordDebugKeyspace extends VirtualKeyspace UNSET_PROGRESS_LOG_MODE("Unset the specified progress log mode."), TRY_EXECUTE_LISTENING("Try to execute all of the transactions (and their dependencies) that have registered listeners on other transactions."), REPLAY("Run journal replay for all transactions"), - REBOOTSTRAP("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."), - HARD_CATCHUP("Hard catchup the command store. This invalidates the local journal for any ranges not up to date with some quorum, synchronises its data via data repair and rejoins the distributed state machine."), + // TODO (expected): specify ranges + REBOOTSTRAP_CORRUPTED("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."), + REBOOTSTRAP_INCOMPLETE("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."), + REBOOTSTRAP_IF_BEHIND("Hard catchup the command store. This invalidates the local journal for any ranges not up to date with some quorum, synchronises its data via data repair and rejoins the distributed state machine."), ; final String description; @@ -2082,13 +2088,15 @@ public class AccordDebugKeyspace extends VirtualKeyspace }; break; } - case REBOOTSTRAP: - allFunction = () -> node.commandStores().rebootstrap(node); - function = commandStore -> commandStore.rebootstrap(node); + case REBOOTSTRAP_CORRUPTED: + case REBOOTSTRAP_INCOMPLETE: + BootstrapReason reason = op == REBOOTSTRAP_CORRUPTED ? LOG_CORRUPTED : LOG_INCOMPLETE; + allFunction = () -> node.commandStores().rebootstrap(node, reason); + function = commandStore -> commandStore.rebootstrap(node, reason); break; - case HARD_CATCHUP: - allFunction = () -> CatchupHard.catchup(node, Arrays.asList(node.commandStores().all())).beginAsResult(); - function = commandStore -> CatchupHard.catchup(node, Collections.singletonList(commandStore)).beginAsResult(); + case REBOOTSTRAP_IF_BEHIND: + allFunction = () -> Catchup.rebootstrapIfBehind(node, Arrays.asList(node.commandStores().all())).beginAsResult(); + function = commandStore -> Catchup.rebootstrapIfBehind(node, Collections.singletonList(commandStore)).beginAsResult(); break; } @@ -2484,7 +2492,6 @@ public class AccordDebugKeyspace extends VirtualKeyspace { throw new InvalidRequestException("Unknown bucket_mode '" + value + '\''); } - } private static int checkNonNegative(Object value, String field, int ifNull) diff --git a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java index 834abca49d..50a5393d10 100644 --- a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java +++ b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java @@ -72,10 +72,13 @@ public class ExceptionSerializer static String getMessageWithOriginatingHost(Throwable t, boolean isFirstException) { - if (isFirstException) - return "Remote exception from host " + FBUtilities.getBroadcastAddressAndPort().toString() + (t.getLocalizedMessage() != null ? " - " + t.getLocalizedMessage() : ""); - else - return t.getLocalizedMessage(); + String message; + if (isFirstException) message = "Remote exception from host " + FBUtilities.getBroadcastAddressAndPort().toString() + (t.getLocalizedMessage() != null ? " - " + t.getLocalizedMessage() : ""); + else message = t.getLocalizedMessage(); + + if (message.length() > Short.MAX_VALUE) + message = message.substring(0, Short.MAX_VALUE); + return message; } private static final IVersionedSerializer stackTraceElementSerializer = new IVersionedSerializer() diff --git a/src/java/org/apache/cassandra/journal/Compactor.java b/src/java/org/apache/cassandra/journal/Compactor.java index ffc48ff5f2..428866098c 100644 --- a/src/java/org/apache/cassandra/journal/Compactor.java +++ b/src/java/org/apache/cassandra/journal/Compactor.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.journal; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -29,6 +28,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.concurrent.Shutdownable; +import org.apache.cassandra.journal.SegmentCompactor.Stopped; import org.apache.cassandra.utils.concurrent.WaitQueue; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; @@ -100,9 +100,9 @@ public final class Compactor implements Runnable, Shutdownable compacted.signalAll(); } - catch (IOException e) + catch (Stopped e) { - throw new RuntimeException("Could not compact segments: " + toCompact); + logger.info("Cancelling compaction of {}", toCompact); } } @@ -118,6 +118,7 @@ public final class Compactor implements Runnable, Shutdownable logger.debug("Shutting down " + executor); if (scheduled != null) scheduled.cancel(false); + segmentCompactor.stop(); executor.shutdown(); } diff --git a/src/java/org/apache/cassandra/journal/SegmentCompactor.java b/src/java/org/apache/cassandra/journal/SegmentCompactor.java index 7b84ea82e1..01897a52c1 100644 --- a/src/java/org/apache/cassandra/journal/SegmentCompactor.java +++ b/src/java/org/apache/cassandra/journal/SegmentCompactor.java @@ -17,18 +17,20 @@ */ package org.apache.cassandra.journal; -import java.io.IOException; import java.util.Collection; public interface SegmentCompactor { SegmentCompactor NOOP = (SegmentCompactor) (segments) -> segments; + class Stopped extends RuntimeException {} + static SegmentCompactor noop() { //noinspection unchecked return (SegmentCompactor) NOOP; } - Collection> compact(Collection> segments) throws IOException; + Collection> compact(Collection> segments) throws Stopped; + default void stop() {} } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index 2dbbc07e00..3fe7550439 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -254,8 +254,10 @@ public class AccordCommandStore extends CommandStore maybeLoadRedundantBefore(journal.loadRedundantBefore(id())); maybeLoadBootstrapBeganAt(journal.loadBootstrapBeganAt(id())); maybeLoadSafeToRead(journal.loadSafeToRead(id())); - - tableId = (TableId)rangesForEpoch.all().stream().map(r -> r.start().prefix()).reduce((a, b) -> { + maybeLoadRangesForEpoch(journal.loadRangesForEpoch(id())); + RangesForEpoch ranges = this.rangesForEpoch; + Invariants.require(ranges != null && !ranges.all().isEmpty(), "CommandStore %d created with no ranges", id); + tableId = (TableId)ranges.all().stream().map(r -> r.start().prefix()).reduce((a, b) -> { Invariants.require(a.equals(b), "CommandStore created with multiple distinct TableId (%s and %s)", a, b); return a; }).orElseThrow(() -> Invariants.illegalState("CommandStore %d created with no ranges", id)); @@ -824,7 +826,7 @@ public class AccordCommandStore extends CommandStore { File rjbf = new File(savePoint, "reject_before"); if (rjbf.exists()) - mxc = mxc.with(readOne(rjbf, rejectBefore)); + mxc = mxc.update(readOne(rjbf, rejectBefore)); } dll = readList(new File(savePoint, "listeners"), txnListener); dpl = readList(new File(savePoint, "progress_log"), progressLogState); diff --git a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java index 0e757ba418..9e227e5f5a 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java @@ -20,24 +20,23 @@ package org.apache.cassandra.service.accord; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.DataStore; import accord.local.CommandStore; +import accord.local.CommandStores.RestrictedStoreSelector; +import accord.local.MapReduceConsumeCommandStores; import accord.local.Node; import accord.local.RedundantBefore; import accord.local.SafeCommandStore; import accord.primitives.Ranges; -import accord.primitives.SyncPoint; +import accord.primitives.Timestamp; import accord.primitives.TxnId; import accord.utils.Invariants; -import accord.utils.Reduce; +import accord.utils.SortedArrays.SortedArrayList; import accord.utils.UnhandledEnum; -import accord.utils.async.AsyncResult; import accord.utils.async.AsyncResults; import org.apache.cassandra.concurrent.ScheduledExecutors; @@ -56,10 +55,9 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.progress.ProgressEvent; +import org.apache.cassandra.utils.progress.ProgressListener; -import static accord.local.durability.DurabilityService.SyncLocal.NoLocal; -import static accord.local.durability.DurabilityService.SyncRemote.Quorum; -import static accord.primitives.Txn.Kind.ExclusiveSyncPoint; import static accord.utils.Invariants.illegalArgument; public class AccordDataStore implements DataStore @@ -94,12 +92,12 @@ public class AccordDataStore implements DataStore AccordDurableOnFlush.notifyOnDurable(cfs, commandStore, ReportDurable.of(redundantBefore, flags)); } - public FetchResult image(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback) + public FetchResult image(Node node, SafeCommandStore safeStore, Ranges ranges, TxnId atLeast, SortedArrayList readable, FetchRanges callback) { AccordFetchCoordinator coordinator; try { - coordinator = new AccordFetchCoordinator(node, ranges, syncPoint, callback, safeStore.commandStore()); + coordinator = new AccordFetchCoordinator(node, ranges, atLeast, readable, callback, safeStore.commandStore()); } catch (Throwable t) { @@ -110,11 +108,10 @@ public class AccordDataStore implements DataStore return coordinator.result(); } - public FetchResult sync(Node node, SafeCommandStore safeStore, Map rangesById, FetchRanges callback) + @Override + public FetchResult sync(Node node, SafeCommandStore safeStore, Ranges ranges, TxnId atLeast, SortedArrayList readable, FetchRanges callback) { TableId tableId = ((AccordCommandStore)safeStore.commandStore()).tableId(); - Invariants.require(rangesById.values().stream().flatMap(Ranges::stream).allMatch(r -> tableId.equals(r.prefix()))); - Ranges ranges = rangesById.values().stream().reduce(Ranges::with).get(); ClusterMetadata cm = ClusterMetadata.current(); TableMetadata tableMetadata = cm.schema.getTableMetadata(tableId); @@ -137,27 +134,22 @@ public class AccordDataStore implements DataStore // TODO (expected): add some automatic slicing of ranges and retry/back-off logic; but for now, // since this is done at the command store level, and this is already a slice of a node, this should be fine SyncResult syncResult = new SyncResult(); + ProgressListener listener = new ProgressListener() + { + StartingRangeFetch starting = callback.starting(ranges); + { Invariants.require(starting != null); } - logger.info("Requesting quorum durability before initiating repair"); - List> syncs = new ArrayList<>(); - for (Map.Entry e : rangesById.entrySet()) - syncs.add(node.durability().sync("Sync Data Store", ExclusiveSyncPoint, e.getKey(), e.getValue(), NoLocal, Quorum, 1L, TimeUnit.HOURS)); - - AsyncResults.reduce(syncs, Reduce.toNull()).invoke((success, fail) -> { - if (fail != null) + @Override + public void progress(String tag, ProgressEvent event) { - logger.error("{} failed to achieve quorum durability before repair for rebootstrap of {}", safeStore.commandStore(), ranges); - syncResult.tryFailure(fail); - return; - } - - RepairCoordinator coord = StorageService.instance.newRepairCoordinator(tableMetadata.keyspace, options(tableMetadata, ranges)); - coord.addProgressListener((tag, event) -> { switch (event.getType()) { default: throw new UnhandledEnum(event.getType()); + case SUCCESS: + callback.fetched(ranges); + syncResult.trySuccess(null); case START: - callback.starting(ranges); + reportStarted(); break; case PROGRESS: case COMPLETE: @@ -169,17 +161,32 @@ public class AccordDataStore implements DataStore callback.fail(ranges, ex); syncResult.tryFailure(ex); break; - case SUCCESS: - callback.fetched(ranges); - syncResult.trySuccess(null); - break; } - }); + } - ScheduledExecutors.optionalTasks.submit(coord).addCallback((s, f) -> { - if (f != null) - syncResult.tryFailure(f); - }); + private void reportStarted() + { + StartingRangeFetch start = this.starting; + if (start == null) + return; + this.starting = null; + node.commandStores().mapReduceConsume(new RestrictedStoreSelector(ranges, 0, Long.MAX_VALUE), new MapReduceConsumeCommandStores(ranges) + { + @Override public Timestamp reduce(Timestamp o1, Timestamp o2) { return Timestamp.max(o1, o2); } + @Override public void accept(Timestamp result, Throwable failure) { start.started(result); } + @Override public TxnId primaryTxnId() { return null; } + @Override public String reason() { return "Compute MaxConflict to report for fetch"; } + @Override protected Timestamp applyInternal(SafeCommandStore safeStore) { return safeStore.commandStore().maxConflict(TxnId.NONE, ranges); } + }); + } + }; + + RepairCoordinator coord = StorageService.instance.newRepairCoordinator(tableMetadata.keyspace, options(tableMetadata, ranges)); + coord.addProgressListener(listener); + + ScheduledExecutors.optionalTasks.submit(coord).addCallback((s, f) -> { + if (f != null) + syncResult.tryFailure(f); }); return syncResult; diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index a14d9b3cd4..630f81ccbf 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -38,7 +38,6 @@ import accord.impl.AbstractFetchCoordinator; import accord.local.CommandStore; import accord.local.Node; import accord.local.SafeCommandStore; -import accord.primitives.PartialDeps; import accord.primitives.PartialTxn; import accord.primitives.Participants; import accord.primitives.Range; @@ -46,12 +45,12 @@ import accord.primitives.Ranges; import accord.primitives.Routable; import accord.primitives.Seekable; import accord.primitives.Seekables; -import accord.primitives.SyncPoint; import accord.primitives.Timestamp; import accord.primitives.Txn; import accord.primitives.TxnId; import accord.topology.TopologyException; import accord.utils.Invariants; +import accord.utils.SortedArrays.SortedArrayList; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; @@ -395,9 +394,9 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements private final Map streams = new HashMap<>(); - public AccordFetchCoordinator(Node node, Ranges ranges, SyncPoint syncPoint, DataStore.FetchRanges fetchRanges, CommandStore commandStore) throws TopologyException + public AccordFetchCoordinator(Node node, Ranges ranges, TxnId atLeast, SortedArrayList readable, DataStore.FetchRanges fetchRanges, CommandStore commandStore) throws TopologyException { - super(node, node.someExclusiveExecutor(), ranges, syncPoint, fetchRanges, commandStore); + super(node, node.someExclusiveExecutor(), ranges, atLeast, readable, fetchRanges, commandStore); } @Override @@ -431,13 +430,6 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements super.onDone(success, failure); } - @Override - protected PartialTxn rangeReadTxn(Ranges ranges) - { - StreamingRead read = new StreamingRead(FBUtilities.getBroadcastAddressAndPort(), ranges); - return new PartialTxn.InMemory(Txn.Kind.Read, ranges, read, noopQuery, null, TableMetadatasAndKeys.none(Routable.Domain.Range)); - } - @Override protected synchronized void onReadOk(Node.Id from, CommandStore commandStore, Data data, Ranges received) { @@ -463,9 +455,9 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements public static class AccordFetchRequest extends FetchRequest { - public AccordFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialDeps partialDeps, PartialTxn partialTxn) + public AccordFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialTxn partialTxn) { - super(sourceEpoch, syncId, ranges, partialDeps, partialTxn); + super(sourceEpoch, syncId, ranges, partialTxn); } @Override @@ -479,8 +471,15 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements } @Override - protected FetchRequest newFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialDeps partialDeps, PartialTxn partialTxn) + protected FetchRequest newFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges) { - return new AccordFetchRequest(sourceEpoch, syncId, ranges, partialDeps, partialTxn); + return new AccordFetchRequest(sourceEpoch, syncId, ranges, rangeReadTxn(ranges)); } + + private PartialTxn rangeReadTxn(Ranges ranges) + { + StreamingRead read = new StreamingRead(FBUtilities.getBroadcastAddressAndPort(), ranges); + return new PartialTxn.InMemory(Txn.Kind.Read, ranges, read, noopQuery, null, TableMetadatasAndKeys.none(Routable.Domain.Range)); + } + } diff --git a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java index b0efd33b1b..a7bdedefdb 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java +++ b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java @@ -106,6 +106,7 @@ import org.apache.cassandra.schema.Tables; import org.apache.cassandra.schema.Types; import org.apache.cassandra.schema.UserFunctions; import org.apache.cassandra.schema.Views; +import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.serializers.CommandSerializers; import org.apache.cassandra.utils.AbstractIterator; @@ -341,7 +342,7 @@ public class AccordKeyspace { ByteBuffer bytes; if (serialized instanceof ByteBuffer) bytes = (ByteBuffer) serialized; - else bytes = Serialize.toBytesWithoutKey(commandsForKey.maximalPrune()); // TODO (expected): we only need to strip pruned, not prune additional txns + else bytes = Serialize.toBytesWithoutKey(commandsForKey.maybePrune(0, AccordAgent.cfkHlcPruneDelta(DatabaseDescriptor.getAccord()))); // TODO (expected): we only need to strip pruned, not prune additional txns return makeUpdate(storeId, key, timestampMicros, bytes); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java index 18dafb1908..2ddac1b52c 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java +++ b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java @@ -254,6 +254,14 @@ public class AccordMessageSink implements MessageSink slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS); break; } + + case ACCORD_WAIT_UNTIL_APPLIED_REQ: + { + TimeoutStrategy slow = slowPreaccept(txnId); + if (slow != null) + slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS); + break; + } } Message message = Message.out(verb, request, expiresAtNanos); diff --git a/src/java/org/apache/cassandra/service/accord/AccordResult.java b/src/java/org/apache/cassandra/service/accord/AccordResult.java index 033ccdd88a..59440ee174 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordResult.java +++ b/src/java/org/apache/cassandra/service/accord/AccordResult.java @@ -201,7 +201,9 @@ public class AccordResult extends AsyncFuture implements BiConsumer cs.unsafeProgressLog().start()); - if (rebootstrap) + if (rebootstrap != null) { // rebootstrap expects the durability service to process visibility sync points for command stores node.durability().start(); - getBlocking(node.commandStores().rebootstrap(node)); + getBlocking(node.commandStores().rebootstrap(node, rebootstrap)); } else { @@ -707,81 +719,100 @@ public class AccordService implements IAccordService, Shutdownable void catchup() { AccordConfig spec = DatabaseDescriptor.getAccord(); - if (spec.catchup_on_start == CatchupMode.DISABLED) + if (!spec.catchup_on_start) { logger.info("Catchup disabled; continuing to startup"); return; } - CatchupMode mode = spec.catchup_on_start; BootstrapState bootstrapState = SystemKeyspace.getBootstrapState(); if (bootstrapState == COMPLETED) { node.commandStores().forAllUnsafe(commandStore -> ((DefaultProgressLog)commandStore.unsafeProgressLog()).setMode(CATCH_UP)); try { + AccordConfig.CatchupFallbackMode onError = spec.catchup_on_start_on_error; + AccordConfig.CatchupFallbackMode onTimeout = spec.catchup_on_start_on_timeout; + long maxLatencyNanos = spec.catchup_on_start_fail_latency.toNanoseconds(); int attempts = 1; while (true) { - logger.info("Catchup ({}) with quorum...", mode); + logger.info("Catchup with quorum..."); long start = nanoTime(); long failAt = start + maxLatencyNanos; - Future f; + Future f; { - AsyncChain submit; - if (mode == HARD) - { - if (!node.durability().isStarted()) - node.durability().start(); - submit = CatchupHard.catchup(node); - } - else submit = Catchup.catchup(node); + AsyncChain submit = Catchup.catchup(node, failAt, NANOSECONDS); f = toFuture(submit); } - if (!f.awaitUntilThrowUncheckedOnInterrupt(failAt)) - { - if (spec.catchup_on_start_exit_on_failure) - { - logger.error("Catchup exceeded maximum latency of {}ns; shutting down", maxLatencyNanos); - throw new RuntimeException("Could not catchup with peers"); - } - logger.error("Catchup exceeded maximum latency of {}ns; continuing to startup", maxLatencyNanos); - break; - } + f.awaitThrowUncheckedOnInterrupt(); Throwable failed = f.cause(); + Unsuccessful result = f.getNow(); + if (failed != null) { - if (spec.catchup_on_start_exit_on_failure) - throw new RuntimeException("Could not catchup with peers", failed); - - logger.error("Could not catchup with peers; continuing to startup"); - break; + switch (onError) + { + default: throw new UnhandledEnum(onError); + case EXIT: + logger.error("Could not catchup with peers; exiting", failed); + throw new RuntimeException("Could not catchup with peers", failed); + case IGNORE: + logger.error("Could not catchup with peers; continuing to startup", failed); + return; + case REBOOTSTRAP: + case REBOOTSTRAP_AND_CATCHUP: + logger.error("Could not catchup with peers; rebootstrapping", failed); + getBlocking(node.commandStores().rebootstrap(node, LOG_INCOMPLETE)); + if (onError == REBOOTSTRAP) + return; + ++attempts; + onError = onTimeout = spec.catchup_on_start_on_rebootstrap_fallback; + continue; + } } long end = nanoTime(); double seconds = NANOSECONDS.toMillis(end - start)/1000.0; logger.info("Finished catchup with all quorums. {}s elapsed.", String.format("%.2f", seconds)); - if (seconds <= spec.catchup_on_start_success_latency.toSeconds()) - break; - - if (++attempts > spec.catchup_on_start_max_attempts) + if (result == null) { - if (spec.catchup_on_start_exit_on_failure) - { - logger.error("Catchup was slow, aborting after {} attempts and shutting down", attempts); - throw new RuntimeException("Could not catchup with peers"); - } + if (seconds <= spec.catchup_on_start_success_latency.toSeconds()) + return; - logger.info("Catchup was slow; continuing to startup after {} attempts.", attempts - 1); - break; + if (attempts < spec.catchup_on_start_max_attempts) + { + logger.info("Catchup was slow, so we may behind again; retrying"); + ++attempts; + continue; + } } - logger.info("Catchup was slow, so we may behind again; retrying"); - if (mode == FALLBACK_TO_HARD) - mode = HARD; + switch (onTimeout) + { + default: throw new UnhandledEnum(onTimeout); + case EXIT: + if (result == null) logger.error("Catchup was slow; aborting after {} attempts and shutting down", attempts); + else logger.error("Catchup was incomplete for {}; aborting after {} attempts and shutting down", result.ranges, attempts); + throw new RuntimeException("Could not catchup with peers"); + case IGNORE: + if (result == null) logger.info("Catchup was slow, continuing to startup after {} attempts", attempts); + else logger.info("Catchup was incomplete for {}, continuing to startup after {} attempts", result.ranges, attempts); + return; + case REBOOTSTRAP: + case REBOOTSTRAP_AND_CATCHUP: + if (result == null) logger.info("Catchup was slow, rebootstrapping after {} attempts", attempts); + else logger.info("Catchup was incomplete for {}, rebootstrapping after {} attempts", result.ranges, attempts); + getBlocking(node.commandStores().rebootstrap(node, result == null ? null : result.ranges, LOG_INCOMPLETE)); + if (onTimeout == REBOOTSTRAP) + return; + ++attempts; + onError = onTimeout = spec.catchup_on_start_on_rebootstrap_fallback; + continue; + } } } finally @@ -954,7 +985,7 @@ public class AccordService implements IAccordService, Shutdownable } @Override - public AsyncResult sync(Object requestedBy, TxnId minBound, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) + public AsyncResult sync(Object requestedBy, TxnId minBound, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) { return node.durability().sync(requestedBy, ExclusiveSyncPoint, minBound, ranges, include, syncLocal, syncRemote, timeout, timeoutUnits); } diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java index abf7336b22..79bde24b57 100644 --- a/src/java/org/apache/cassandra/service/accord/IAccordService.java +++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java @@ -88,8 +88,8 @@ public interface IAccordService IVerbHandler requestHandler(); IVerbHandler responseHandler(); - AsyncResult sync(Object requestedBy, @Nullable TxnId minBound, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits); - AsyncChain sync(@Nullable TxnId minBound, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote); + AsyncResult sync(Object requestedBy, @Nullable TxnId minBound, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits); + AsyncChain sync(@Nullable TxnId minBound, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote); AsyncChain maxConflict(Ranges ranges); @Nonnull @@ -402,13 +402,13 @@ public interface IAccordService } @Override - public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) + public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) { return delegate.sync(requestedBy, onOrAfter, ranges, include, syncLocal, syncRemote, timeout, timeoutUnits); } @Override - public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote) + public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote) { return delegate.sync(onOrAfter, keys, syncLocal, syncRemote); } diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index fc3ffa975b..cabb3de079 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -40,9 +40,6 @@ import accord.api.RoutingKey; import accord.api.Tracing; import accord.coordinate.Coordination; import accord.coordinate.CoordinationFailed; -import accord.coordinate.Exhausted; -import accord.coordinate.Preempted; -import accord.coordinate.Timeout; import accord.local.Command; import accord.local.CommandStore; import accord.local.LogUnavailableException; @@ -51,6 +48,7 @@ import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.TimeService; import accord.messages.MessageType; +import accord.primitives.Ballot; import accord.primitives.Keys; import accord.primitives.Participants; import accord.primitives.Ranges; @@ -266,6 +264,11 @@ public class AccordAgent implements Agent, OwnershipEventListener // TODO (expected): we probably want additional configuration here so we can prune on shorter time horizons when we have a lot of transactions on a single key @Override public long cfkHlcPruneDelta() + { + return cfkHlcPruneDelta(config); + } + + public static long cfkHlcPruneDelta(AccordConfig config) { return config.commands_for_key_prune_delta.to(MICROSECONDS); } @@ -325,24 +328,12 @@ public class AccordAgent implements Agent, OwnershipEventListener @Override public long slowCoordinatorDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int attempt) { - SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId); - if (safeCommand == null) - { - noSpamLogger.warn("{} invoked slowCoordinatorDelay for {} without having it in cache", safeStore.commandStore(), txnId, new RuntimeException()); - return recover(txnId).computeWait(attempt, units); - } - - Command command = safeCommand.current(); - if (command == null) - { - noSpamLogger.warn("{} invoked slowCoordinatorDelay for {} without knowing the command", safeStore.commandStore(), txnId, new RuntimeException()); - return recover(txnId).computeWait(attempt, units); - } - + Command command = command(safeStore, txnId, "slowCoordinatorDelay"); + Ballot promised = promised(command); // TODO (expected): make this a configurable calculation on normal request latencies (like ContentionStrategy) long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()); - long mostRecentStart = mostRecentStart(command, nowMicros); + long mostRecentStart = mostRecentStart(txnId, promised, nowMicros); long waitMicros = recover(txnId).computeWait(attempt, MICROSECONDS); long startTime = mostRecentStart + waitMicros; if (startTime < nowMicros) @@ -355,7 +346,7 @@ public class AccordAgent implements Agent, OwnershipEventListener } RoutingKey homeKey = command.route().homeKey(); - Shard shard = node.topology().active().forEpochIfKnown(homeKey, command.txnId().epoch()); + Shard shard = node.topology().active().forEpochIfKnown(homeKey, txnId.epoch()); startTime = nonClashingStartTime(startTime, shard == null ? null : shard.nodes, node.id(), ONE_SECOND, random); long delayMicros = Math.max(1, startTime - nowMicros); @@ -363,15 +354,15 @@ public class AccordAgent implements Agent, OwnershipEventListener return units.convert(delayMicros, MICROSECONDS); } - private static long mostRecentStart(Command command, long nowMicros) + private static long mostRecentStart(TxnId txnId, Ballot promised, long nowMicros) { // TODO (expected): make this a configurable calculation on normal request latencies (like ContentionStrategy) - long promisedHlc = command.promised().hlc(); + long promisedHlc = promised.hlc(); if (promisedHlc > nowMicros + ONE_MINUTE) promisedHlc = 0; - long result = Math.max(command.txnId().hlc(), promisedHlc); + long result = Math.max(txnId.hlc(), promisedHlc); if (result > nowMicros + ONE_SECOND) - noSpamLogger.warn("max({},{})>{}", command.txnId(), command.promised(), nowMicros); + noSpamLogger.warn("max({},{})>{}", txnId, promised, nowMicros); return result; } @@ -405,25 +396,36 @@ public class AccordAgent implements Agent, OwnershipEventListener return newStartTime; } - @Override - public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil blockedUntil, TimeUnit units) + private static Ballot promised(@Nullable Command command) + { + return command == null ? Ballot.ZERO : command.promised(); + } + + private static Command command(SafeCommandStore safeStore, TxnId txnId, String source) { SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId); if (safeCommand == null) { - noSpamLogger.warn("{} invoked slowReplicaDelay for {} without having it in cache", safeStore.commandStore(), txnId, new RuntimeException()); - return fetch(txnId).computeWait(attempt, units); + noSpamLogger.warn("{} invoked {} for {} without having it in cache", safeStore.commandStore(), source, txnId, new RuntimeException()); + return null; } Command command = safeCommand.current(); if (command == null) { - noSpamLogger.warn("{} invoked slowReplicaDelay for {} without knowing the command", safeStore.commandStore(), txnId, new RuntimeException()); - return fetch(txnId).computeWait(attempt, units); + noSpamLogger.warn("{} invoked {} for {} without knowing the command", safeStore.commandStore(), source, txnId, new RuntimeException()); + return null; } + return command; + } + + @Override + public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil blockedUntil, TimeUnit units) + { + Ballot promised = promised(command(safeStore, txnId, "slowReplicaDelay")); long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()); - long mostRecentStart = mostRecentStart(command, nowMicros); + long mostRecentStart = mostRecentStart(txnId, promised, nowMicros); long waitMicros = fetch(txnId).computeWait(attempt, units); long startTime = mostRecentStart + waitMicros; if (startTime < nowMicros) diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java index cf7d8ff5ae..eb1fcbfbfa 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java @@ -38,7 +38,7 @@ import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory. public class AccordWaitStrategies { - static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead; + static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead, slowDurability; static TimeoutStrategy expireTxn, expireSyncPoint, expireDurability, expireEpochWait; static TimeoutStrategy fetchTxn, fetchSyncPoint; static RetryStrategy recoverTxn, recoverSyncPoint, retrySyncPoint, retryDurability, retryBootstrap, retryJoinBootstrap; @@ -88,6 +88,7 @@ public class AccordWaitStrategies setSlowRead(config.slow_read); setSlowTxnPreaccept(config.slow_txn_preaccept); setSlowSyncPointPreaccept(config.slow_syncpoint_preaccept); + setSlowDurability(config.slow_durability); setExpireTxn(config.expire_txn); setExpireSyncPoint(config.expire_syncpoint); setExpireDurability(config.expire_durability); @@ -119,6 +120,11 @@ public class AccordWaitStrategies slowSyncPointPreaccept = TimeoutStrategy.parse(spec, none()); } + public static void setSlowDurability(String spec) + { + slowDurability = TimeoutStrategy.parse(spec, none()); + } + public static void setExpireTxn(String spec) { expireTxn = TimeoutStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics)); diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java index 697a094e60..adaf531a5b 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java @@ -47,6 +47,7 @@ import org.apache.cassandra.service.accord.topology.AccordEndpointMapper; import org.apache.cassandra.service.accord.txn.AccordUpdate; import org.apache.cassandra.service.accord.txn.TxnRead; +import static accord.api.ProtocolModifiers.sendMinimal; import static accord.messages.Apply.Kind.Maximal; import static accord.messages.Apply.Kind.Minimal; @@ -55,12 +56,13 @@ public class AccordInteropAdapter extends TxnAdapter private static final Logger logger = LoggerFactory.getLogger(AccordInteropAdapter.class); public static final class AccordInteropFactory extends DefaultFactory { - final AccordInteropAdapter standard, recovery; + final AccordInteropAdapter minimal, maximal, recovery; public AccordInteropFactory(AccordEndpointMapper endpointMapper) { - standard = new AccordInteropAdapter(endpointMapper, Minimal); - recovery = new AccordInteropAdapter(endpointMapper, Maximal); + minimal = new AccordInteropAdapter(endpointMapper, Minimal, true); + maximal = new AccordInteropAdapter(endpointMapper, Maximal, true); + recovery = new AccordInteropAdapter(endpointMapper, Maximal, false); } @Override @@ -68,18 +70,18 @@ public class AccordInteropAdapter extends TxnAdapter { if (txnId.isSyncPoint()) return super.get(txnId, step); - return (CoordinationAdapter) (step == Kind.Recovery ? recovery : standard); + return (CoordinationAdapter) (step == Kind.Recovery ? recovery : sendMinimal() ? minimal : maximal); } }; private final AccordEndpointMapper endpointMapper; - private final Apply.Kind applyKind; + private final boolean isUserFacing; - private AccordInteropAdapter(AccordEndpointMapper endpointMapper, Apply.Kind applyKind) + private AccordInteropAdapter(AccordEndpointMapper endpointMapper, Apply.Kind applyKind, boolean isUserFacing) { - super(Minimal); + super(applyKind); this.endpointMapper = endpointMapper; - this.applyKind = applyKind; + this.isUserFacing = isUserFacing; } @Override @@ -92,7 +94,7 @@ public class AccordInteropAdapter extends TxnAdapter @Override public void persist(Node node, ExclusiveAsyncExecutor executor, Topologies any, Route require, Route sendTo, FullRoute route, Ballot ballot, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, boolean informDurableOnDone, BiConsumer callback) { - if (applyKind == Minimal && doInteropPersist(node, executor, any, require, sendTo, ballot, txnId, txn, executeAt, deps, writes, result, route, informDurableOnDone, callback)) + if (isUserFacing && doInteropPersist(node, executor, any, require, sendTo, ballot, txnId, txn, executeAt, deps, writes, result, route, informDurableOnDone, callback)) return; super.persist(node, executor, any, require, sendTo, route, ballot, flags, txnId, txn, executeAt, deps, writes, result, informDurableOnDone, callback); diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java index 262e98af4e..874c2e305a 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java @@ -94,6 +94,7 @@ import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.transport.Dispatcher; +import static accord.api.ProtocolModifiers.sendMinimal; import static accord.coordinate.CoordinationAdapter.Factory.Kind.Standard; import static accord.primitives.Txn.Kind.Write; import static accord.topology.SelectShards.LIVE; @@ -144,6 +145,7 @@ public class AccordInteropExecution implements ReadCoordinator ExclusiveAsyncExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper) throws TopologyException { requireArgument(!txn.read().keys().isEmpty() || updateKind == AccordUpdate.Kind.UNRECOVERABLE_REPAIR); + // TODO (required): this does not support privileged coordinator optimisation, as must apply local stable before any other message is sent this.node = node; this.txnId = txnId; this.txn = txn; @@ -217,8 +219,7 @@ public class AccordInteropExecution implements ReadCoordinator // TODO (desired): It would be better to use the re-use the command from the transaction but it's fragile // to try and figure out exactly what changed for things like read repair and short read protection // Also this read scope doesn't reflect the contents of this particular read and is larger than it needs to be - // TODO (required): understand interop and whether StableFastPath is appropriate - AccordInteropStableThenRead commit = new AccordInteropStableThenRead(id, allTopologies, txnId, Kind.StableFastPath, executeAt, txn, deps, route, message.payload); + AccordInteropStableThenRead commit = new AccordInteropStableThenRead(id, allTopologies, txnId, commitKind(), executeAt, txn, deps, route, message.payload); node.send(id, commit, executor, new AccordInteropRead.ReadCallback(id, to, message, callback, this), null); } @@ -376,7 +377,7 @@ public class AccordInteropExecution implements ReadCoordinator { InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(to); if (endpoint != null && !contacted.contains(endpoint)) - node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); + node.send(to, new Commit(commitKind(), to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); } } @@ -387,7 +388,7 @@ public class AccordInteropExecution implements ReadCoordinator for (Node.Id to : allTopologies.nodes()) { if (!executeTopology.contains(to)) - node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); + node.send(to, new Commit(commitKind(), to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); } } AsyncChain result; @@ -415,6 +416,11 @@ public class AccordInteropExecution implements ReadCoordinator }); } + private static Commit.Kind commitKind() + { + return sendMinimal() ? Kind.StableFastPath : Kind.StableWithTxnAndDeps; + } + private AsyncChain executeUnrecoverableRepairUpdate() { return AsyncChains.chain(Stage.ACCORD_MIGRATION.executor(), () -> { @@ -423,7 +429,7 @@ public class AccordInteropExecution implements ReadCoordinator // and can be extended similar to MessageType which allows additional types not from Accord to be added // This commit won't necessarily execute before the interop read repair message so there could be an insufficient which is fine for (Node.Id to : executeTopology.nodes()) - node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); + node.send(to, new Commit(commitKind(), to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); repairUpdate.runBRR(AccordInteropExecution.this); return new TxnData(); }); diff --git a/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java b/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java index f2caf35537..32f17625fd 100644 --- a/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java +++ b/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java @@ -93,6 +93,7 @@ public abstract class AbstractSegmentCompactor implements SegmentCompactor> compact(Collection> segments) @@ -129,6 +130,9 @@ public abstract class AbstractSegmentCompactor implements SegmentCompactor reader; while ((reader = readers.poll()) != null) { + if (stopped) + throw new Stopped(); + if (key == null || !reader.key().equals(key)) { maybeWritePartition(key, merger, serializer, firstDescriptor, firstOffset); @@ -232,6 +236,12 @@ public abstract class AbstractSegmentCompactor implements SegmentCompactor= (1 << RedundantStatus.Property.WAS_OWNED.ordinal())) + throw new IllegalStateException("Should not be serializing WAS_OWNED or NOT_OWNED statuses"); + if ((v & ~0xFFFF) != 0) // retain this throw new IllegalStateException("Cannot serialize RedundantStatus larger than 0xFFFF. Requires serialization changes."); return (short)v; } diff --git a/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java index 2acd8e2464..c413780546 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java @@ -25,7 +25,11 @@ import accord.impl.AbstractFetchCoordinator.FetchRequest; import accord.impl.AbstractFetchCoordinator.FetchResponse; import accord.messages.ReadData.CommitOrReadNack; import accord.messages.ReadData.ReadReply; +import accord.primitives.PartialDeps; +import accord.primitives.PartialTxn; import accord.primitives.Ranges; +import accord.primitives.TxnId; +import accord.utils.Invariants; import accord.utils.VIntCoding; import org.apache.cassandra.db.TypeSizes; @@ -51,18 +55,20 @@ public class FetchSerializers out.writeUnsignedVInt(request.executeAtEpoch); CommandSerializers.txnId.serialize(request.txnId, out); KeySerializers.ranges.serialize((Ranges) request.scope, out); - DepsSerializers.partialDeps.serialize(request.partialDeps, out); + DepsSerializers.partialDeps.serialize(PartialDeps.NONE, out); StreamingTxn.serializer.serialize(request.read, out, version); } @Override public FetchRequest deserialize(DataInputPlus in, Version version) throws IOException { - return new AccordFetchRequest(in.readUnsignedVInt(), - CommandSerializers.txnId.deserialize(in), - KeySerializers.ranges.deserialize(in), - DepsSerializers.partialDeps.deserialize(in), - StreamingTxn.serializer.deserialize(in, version)); + long epoch = in.readUnsignedVInt(); + TxnId txnId = CommandSerializers.txnId.deserialize(in); + Ranges ranges = KeySerializers.ranges.deserialize(in); + PartialDeps deps = DepsSerializers.partialDeps.deserialize(in); + Invariants.require(deps.isEmpty()); + PartialTxn txn = StreamingTxn.serializer.deserialize(in, version); + return new AccordFetchRequest(epoch, txnId, ranges, txn); } @Override @@ -71,7 +77,7 @@ public class FetchSerializers return TypeSizes.sizeofUnsignedVInt(request.executeAtEpoch) + CommandSerializers.txnId.serializedSize(request.txnId) + KeySerializers.ranges.serializedSize((Ranges) request.scope) - + DepsSerializers.partialDeps.serializedSize(request.partialDeps) + + DepsSerializers.partialDeps.serializedSize(PartialDeps.NONE) + StreamingTxn.serializer.serializedSize(request.read, version); } }; diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java index 07147fac17..1629b94fa3 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java +++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java @@ -74,6 +74,7 @@ import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.vint.VIntCoding; +import static accord.local.BootstrapReason.GAIN_OWNERSHIP; import static com.google.common.collect.ImmutableList.of; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.cassandra.tcm.MultiStepOperation.Kind.JOIN; @@ -396,7 +397,7 @@ public class BootstrapAndJoin extends MultiStepOperation Node node = service.node(); node.commandStores().forAllUnsafe(commandStore -> { - AsyncResult ready = commandStore.resumeBootstrap(node); + AsyncResult ready = commandStore.resumeBootstrap(node, GAIN_OWNERSHIP); bootstraps.add(AccordService.toFuture(ready.flatMap(e -> e.reads))); }); } diff --git a/test/conf/logback-info.xml b/test/conf/logback-info.xml new file mode 100644 index 0000000000..bc121ca649 --- /dev/null +++ b/test/conf/logback-info.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + ./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log + + %-5level [%thread] ${instance_id} %date{"yyyy-MM-dd'T'HH:mm:ss,SSS", UTC} %F:%L - %msg%n + + + INFO + + true + + + + + %-5level %date{"HH:mm:ss,SSS"} %msg%n + + + INFO + + + + + + + + + + diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java index f38b84ede2..03c16edd2a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java @@ -81,6 +81,7 @@ import org.apache.cassandra.distributed.api.ICoordinator; import org.apache.cassandra.distributed.api.QueryResults; import org.apache.cassandra.distributed.api.SimpleQueryResult; import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.distributed.shared.FutureUtils; import org.apache.cassandra.distributed.test.sai.SAIUtil; import org.apache.cassandra.distributed.util.QueryResultUtil; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -93,6 +94,7 @@ import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FailingConsumer; import org.apache.cassandra.utils.Pair; @@ -116,8 +118,10 @@ import static org.junit.Assert.fail; public abstract class AccordCQLTestBase extends AccordTestBase { private static final Logger logger = LoggerFactory.getLogger(AccordCQLTestBase.class); + private static final int CAS_SIMULATOR_LITE_CONCURRENCY = 20; - protected AccordCQLTestBase(TransactionalMode transactionalMode) { + protected AccordCQLTestBase(TransactionalMode transactionalMode) + { super(transactionalMode); } @@ -131,7 +135,9 @@ public abstract class AccordCQLTestBase extends AccordTestBase public static void setupClass() throws IOException { AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL) - .set("paxos_variant", PaxosVariant.v2.name())), 2); + .set("paxos_variant", PaxosVariant.v2.name()) + .set("accord.migration_concurrency", "" + (1 + CAS_SIMULATOR_LITE_CONCURRENCY)) + ), 2); SHARED_CLUSTER.schemaChange("CREATE TYPE " + KEYSPACE + ".person (height int, age int)"); } @@ -3332,15 +3338,18 @@ public abstract class AccordCQLTestBase extends AccordTestBase coordinator.execute("INSERT INTO " + qualifiedAccordTableName + " (pk, count, seq1, seq2) VALUES (1, 0, '', []) USING TIMESTAMP 0", ConsistencyLevel.ALL); ListType LIST_TYPE = ListType.getInstance(Int32Type.instance, true); - ExecutorService es = Executors.newCachedThreadPool(); - List> futures = new ArrayList<>(); - for (int ii = 0; ii < 10; ii++) + Future[] futures = new Future[CAS_SIMULATOR_LITE_CONCURRENCY]; + long[] starts = new long[CAS_SIMULATOR_LITE_CONCURRENCY]; + for (int id = 0; id < CAS_SIMULATOR_LITE_CONCURRENCY; id++) { - int id = ii; - futures.add(es.submit(() -> coordinator.execute("UPDATE " + qualifiedAccordTableName + " SET count = count + 1, seq1 = seq1 + ?, seq2 = seq2 + ? WHERE pk = ? IF EXISTS", ConsistencyLevel.ALL, id + ",", ByteBufferUtil.getArray(LIST_TYPE.decompose(singletonList(id))), 1))); + starts[id] = Clock.Global.nanoTime(); + futures[id] = FutureUtils.map(coordinator.asyncExecuteWithResult("UPDATE " + qualifiedAccordTableName + " SET count = count + 1, seq1 = seq1 + ?, seq2 = seq2 + ? WHERE pk = ? IF EXISTS", ConsistencyLevel.ALL, id + ",", ByteBufferUtil.getArray(LIST_TYPE.decompose(singletonList(id))), 1), SimpleQueryResult::toObjectArrays); + } + for (int id = 0; id < CAS_SIMULATOR_LITE_CONCURRENCY; id++) + { + futures[id].get(); + System.out.println(String.format("Completed in %.3fs", (Clock.Global.nanoTime() - starts[id]) * 0.000000001)); } - for (Future f : futures) - f.get(); Object[][] result = coordinator.execute("SELECT pk, count, seq1, seq2 FROM " + qualifiedAccordTableName + " WHERE pk = 1", ConsistencyLevel.SERIAL); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java index d3c1fadfc4..1802efa82a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java @@ -89,7 +89,7 @@ public class AccordIncrementalRepairTest extends TestBaseImpl } @Override - public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) + public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) { return delegate.sync(requestedBy, onOrAfter, ranges, include, syncLocal, syncRemote, 10L, TimeUnit.MINUTES).map(v -> { executedBarriers = true; @@ -98,7 +98,7 @@ public class AccordIncrementalRepairTest extends TestBaseImpl } @Override - public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) + public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) { return delegate.sync(onOrAfter, keys, syncLocal, syncRemote).map(v -> { executedBarriers = true; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java index 763cbd4d5a..c3ec06b6a7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java @@ -170,7 +170,12 @@ public abstract class AccordTestBase extends TestBaseImpl { SHARED_CLUSTER.filters().reset(); for (IInvokableInstance instance : SHARED_CLUSTER) - instance.runOnInstance(() -> AccordService.instance().node().commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start())); + { + instance.runOnInstance(() -> { + if (AccordService.isStarted()) + AccordService.instance().node().commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start()); + }); + } truncateSystemTables(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java new file mode 100644 index 0000000000..fc1763a273 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java @@ -0,0 +1,976 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import java.io.IOException; +import java.security.SecureRandom; +import java.time.Duration; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Snapshot; +import com.codahale.metrics.Timer; +import com.google.common.util.concurrent.RateLimiter; + +import org.agrona.collections.IntArrayList; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.local.Catchup; +import accord.local.CommandStore; +import accord.local.Node; +import accord.local.ExecutionContext; +import accord.local.SafeCommand; +import accord.primitives.PartialDeps; +import accord.primitives.TxnId; +import accord.utils.Functions; +import accord.utils.Invariants; +import accord.utils.UnhandledEnum; + +import org.apache.cassandra.concurrent.NamedThreadFactory; +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.distributed.api.IInstance; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.distributed.api.IMessage; +import org.apache.cassandra.distributed.api.IMessageFilters; +import org.apache.cassandra.distributed.test.accord.AccordTestBase; +import org.apache.cassandra.distributed.test.accord.load.LoadSettings.ClusterChaos; +import org.apache.cassandra.metrics.AccordCoordinatorMetrics; +import org.apache.cassandra.metrics.AccordExecutorMetrics; +import org.apache.cassandra.metrics.ShardedDecayingHistograms.ShardedDecayingHistogram; +import org.apache.cassandra.metrics.ShardedHistogram; +import org.apache.cassandra.metrics.SnapshottingTimer; +import org.apache.cassandra.net.ArtificialLatency; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.service.accord.AccordKeyspace; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.service.accord.debug.AccordTracing; +import org.apache.cassandra.service.accord.debug.AccordTracing.Message; +import org.apache.cassandra.service.accord.debug.CoordinationKinds; +import org.apache.cassandra.service.accord.debug.TxnKindsAndDomains; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import org.apache.cassandra.utils.concurrent.WaitQueue; + +import static accord.coordinate.Coordination.CoordinationKind.Client; +import static accord.coordinate.Coordination.CoordinationKind.Execute; +import static accord.local.BootstrapReason.LOG_CORRUPTED; +import static accord.local.BootstrapReason.LOG_INCOMPLETE; +import static java.lang.System.currentTimeMillis; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS; +import static org.apache.cassandra.distributed.test.accord.load.LoadSettings.ClusterChaos.REBOOTSTRAP_RESET; +import static org.apache.cassandra.service.accord.debug.AccordTracing.BucketMode.LEAKY; +import static org.apache.cassandra.service.accord.debug.AccordTracing.BucketMode.RING; +import static org.apache.cassandra.service.accord.debug.AccordTracing.BucketMode.SLOWEST; + +public class AccordLoadTestBase extends AccordTestBase +{ + private static long CHAOS_TIMEOUT_NANOS = TimeUnit.MINUTES.toNanos(10L); + private static final Logger logger = LoggerFactory.getLogger(AccordLoadTestBase.class); + + @Before + public void setup() + { + setupCluster(); + super.setup(); + } + + public void setupCluster() + { + setupCluster(5); + } + + public void setupCluster(int nodeCount) + { + setupCluster(nodeCount, config -> { + config.with(Feature.NETWORK, Feature.GOSSIP) + .set("accord.shard_durability_target_splits", "8") + .set("accord.shard_durability_max_splits", "16") + .set("accord.shard_durability_cycle", "1m") + .set("accord.queue_submission_model", "SIGNAL") + .set("accord.command_store_shard_count", "8") + .set("accord.queue_thread_count", "4") + .set("accord.queue_shard_count", "1") + .set("accord.replica_execution", "ALL") + .set("accord.send_stable", "TO_ALL_REPLICA_EXECUTABLE_ELSE_FOR_READS") + .set("accord.send_minimal", "false") + .set("accord.catchup_on_start_fail_latency", "2m"); + }); + } + + public void setupCluster(int nodeCount, Consumer configure) + { + Invariants.require(SHARED_CLUSTER == null); + try { SHARED_CLUSTER = createCluster(nodeCount, builder -> builder.withDCs(nodeCount).withConfig(configure)); } + catch (IOException e) { throw new RuntimeException(e); } + CassandraRelevantProperties.SIMULATOR_STARTED.setString(Long.toString(MILLISECONDS.toSeconds(currentTimeMillis()))); + } + + public void testLoad(final LoadSettings settings) throws Exception + { + Cluster cluster = SHARED_CLUSTER; + cluster.schemaChange("CREATE TABLE " + qualifiedAccordTableName + " (k int, v int, PRIMARY KEY(k)) WITH transactional_mode = 'full'"); +// long seed = new SecureRandom().nextLong(); + long seed = 3705626102508196273L; + try + { + final ConcurrentHashMap verbs = new ConcurrentHashMap<>(); + cluster.filters().outbound().messagesMatching(new IMessageFilters.Matcher() + { + @Override + public boolean matches(int i, int i1, IMessage iMessage) + { + verbs.computeIfAbsent(Verb.fromId(iMessage.verb()), ignore -> new AtomicInteger()).incrementAndGet(); + return false; + } + }).drop(); + + boolean waitForTransactions = settings.totalTransactions < Long.MAX_VALUE; + boolean waitForClusterChaos = settings.totalClusterChaos < Long.MAX_VALUE; + + int clientCount = settings.clients < 0 ? cluster.size() : settings.clients; + long nextRepairAt = settings.repairInterval; + long nextCompactionAt = settings.compactionInterval; + long nextJournalFlushAt = settings.journalFlushInterval; + long nextDataFlushAt = settings.dataFlushInterval; + long nextCfkFlushAt = settings.cfkFlushInterval; + long nextChaosAt = settings.clusterChaosInterval; + final ExecutorService chaosExecutor = Executors.newFixedThreadPool(settings.clusterChaosConcurrency, new NamedThreadFactory("ClusterChaos")); + final ExecutorService clientExecutor = Executors.newFixedThreadPool(clientCount, new NamedThreadFactory("Client")); + final BitSet initialised = new BitSet(); + final Supplier clusterChaos = settings.clusterChaos.apply(seed); + + List chaosActive = new ArrayList<>(); + cluster.get(1).nodetoolResult("cms", "reconfigure", "datacenter1:1", "datacenter2:1", "datacenter3:1").asserts().success(); + if (settings.cfkCompactionPeriodSeconds < Integer.MAX_VALUE && settings.cfkCompactionPeriodSeconds > 0) + { + cluster.forEach(i -> i.acceptOnInstance(period -> { + ((AccordService) AccordService.instance()).journal().compactor().updateCompactionPeriod(period, SECONDS); + }, settings.cfkCompactionPeriodSeconds)); + } + + if (settings.artificialLatencies != null) + { + for (int i = 0 ; i < cluster.size() ; ++i) + { + StringBuilder str = new StringBuilder(); + for (int j = 0 ; j < settings.artificialLatencies[i].length ; ++j) + { + if (j > 0) + str.append(","); + str.append("datacenter") + .append(j + 1) + .append(':') + .append(settings.artificialLatencies[i][j]) + .append("ms"); + } + cluster.get(i + 1).acceptOnInstance(latencies -> { + ArtificialLatency.setArtificialLatencies(latencies); + ArtificialLatency.setArtificialLatencyOnlyPermittedConsistencyLevels(false); + ArtificialLatency.setArtificialLatencyVerbs(ArtificialLatency.recommendedVerbs()); + ArtificialLatency.setEnabled(true); + }, str.toString()); + } + } + + if (settings.traceSlowest > 0f) + { + float traceSlowest = settings.traceSlowest; + for (int i = 0 ; i < cluster.size() ; ++i) + { + cluster.get(i + 1).runOnInstance(() -> { + AccordTracing tracing = ((AccordAgent) AccordService.unsafeInstance().agent()).tracing(); + tracing.setPattern(1, pattern -> pattern.withChance(traceSlowest) + .withKinds(TxnKindsAndDomains.parse("{K*}")) + .withTraceNew(CoordinationKinds.ALL), + SLOWEST, -1, 2, LEAKY, 10, 1, CoordinationKinds.ALL); + }); + } + } + + if (settings.traceLast > 0) + { + int traceLast = settings.traceLast; + for (int i = 0 ; i < cluster.size() ; ++i) + { + cluster.get(i + 1).runOnInstance(() -> { + AccordTracing tracing = ((AccordAgent) AccordService.unsafeInstance().agent()).tracing(); + tracing.setPattern(2, pattern -> pattern.withKinds(TxnKindsAndDomains.parse("{KW}")) + .withTraceNew(CoordinationKinds.ALL), + RING, -1, traceLast, LEAKY, 10, 1, CoordinationKinds.ALL); + }); + } + } + + final AtomicBoolean stop = new AtomicBoolean(); + final AtomicBoolean pauseOrStop = new AtomicBoolean(); + final WaitQueue waitQueue = WaitQueue.newWaitQueue(); + final Random random = new Random(); + final Semaphore completed = new Semaphore(0); + final AtomicIntegerArray coordinatorIndexes = new AtomicIntegerArray(clientCount); + final Set chaosCandidates = new ConcurrentSkipListSet<>(); + for (int i = 1; i <= cluster.size() ; ++i) + chaosCandidates.add(i); + final List> clients = new ArrayList<>(); + final AtomicReferenceArray rateLimiters = new AtomicReferenceArray<>(clientCount); + final AtomicReference readHistogram = new AtomicReference<>(new EstimatedHistogram(200)); + final AtomicReference writeHistogram = new AtomicReference<>(new EstimatedHistogram(200)); + final List chaosHistory = new ArrayList<>(); + if (settings.clients >= cluster.size()) + throw new IllegalArgumentException("Cannot have more clients than nodes"); + if (settings.clusterChaosInterval < Integer.MAX_VALUE && settings.clients + 1 >= cluster.size()) + throw new IllegalArgumentException("If restarting, cannot have as many clients as nodes, as must reroute client requests during restart"); + + int clientRatePerSecond = Math.min(settings.ratePerSecond, settings.minRatePerSecond) / clientCount; + for (int client = 0 ; client < clientCount ; ++client) + { + rateLimiters.set(client, RateLimiter.create(clientRatePerSecond)); + final int clientIndex = client; + coordinatorIndexes.set(client, client + 1); + clients.add(clientExecutor.submit(() -> { + final Semaphore inFlight = new Semaphore(settings.clientConcurrency); + while (true) + { + while (pauseOrStop.get()) + { + if (stop.get()) + break; + + WaitQueue.Signal signal = waitQueue.register(); + if (pauseOrStop.get()) signal.awaitThrowUncheckedOnInterrupt(); + else signal.cancel(); + } + + int coordinatorIdx = coordinatorIndexes.get(clientIndex); + ICoordinator coordinator = cluster.coordinator(coordinatorIdx); + try + { + rateLimiters.get(clientIndex).acquire(); + inFlight.acquire(); + long commandStart = System.nanoTime(); + IntArrayList keys = new IntArrayList(settings.keysPerOperation, -1); + for (int i = 0 ; i < settings.keysPerOperation ; ++i) + { + int k = settings.keySelector.getAsInt(); + if (!keys.containsInt(k)) + keys.add(k); + } + if (!keys.intStream().allMatch(initialised::get)) + { + coordinator.executeWithResult((success, fail) -> { + inFlight.release(); + completed.release(); + if (fail == null) + { + long elapsed = System.nanoTime() - commandStart; + writeHistogram.get().add(NANOSECONDS.toMicros(elapsed)); + synchronized (initialised) + { + keys.forEachInt(initialised::set); + } + } + else + { + logger.error("{}", fail.toString()); + } + }, "UPDATE " + qualifiedAccordTableName + " SET v = 0 WHERE k IN ?", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, keys); + } + else if (random.nextFloat() < settings.readRatio) + { + coordinator.executeWithResult((success, fail) -> { + inFlight.release(); + completed.release(); + if (fail == null) + readHistogram.get().add(NANOSECONDS.toMicros(System.nanoTime() - commandStart)); + }, "BEGIN TRANSACTION\n" + + "SELECT * FROM " + qualifiedAccordTableName + " WHERE k IN ?;\n" + + "COMMIT TRANSACTION;", ConsistencyLevel.SERIAL, keys + ); + } + else + { + coordinator.executeWithResult((success, fail) -> { + inFlight.release(); + completed.release(); + if (fail == null) + { + long elapsed = System.nanoTime() - commandStart; + writeHistogram.get().add(NANOSECONDS.toMicros(elapsed)); + } + else + logger.error("{}", fail.toString()); + }, "BEGIN TRANSACTION\n" + + // "UPDATE " + qualifiedAccordTableName + " SET v = ? WHERE k = ?;\n" + + "UPDATE " + qualifiedAccordTableName + " SET v += ? WHERE k IN ?;\n" + + "COMMIT TRANSACTION;", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, random.nextInt(100), keys); + } + } + catch (RejectedExecutionException e) + { + inFlight.release(); + } + catch (InterruptedException e) + { + throw new UncheckedInterruptedException(e); + } + } + })); + } + + int targetClientRatePerSecond = settings.ratePerSecond / clientCount; + int nextRateLimitIncrease = settings.increaseRatePerSecondInterval; + long remainingTransactions = settings.totalTransactions; + int remainingClusterChaos = settings.totalClusterChaos; + while (true) + { + long batchStart = System.nanoTime(); + int batchSize = 0; + + if (completed.tryAcquire(settings.batchSize, settings.batchPeriodNanos, NANOSECONDS)) + batchSize = settings.batchSize; + batchSize += completed.drainPermits(); + + if (clientRatePerSecond < targetClientRatePerSecond) + { + if ((nextRateLimitIncrease -= batchSize) <= 0) + { + clientRatePerSecond = Math.min(clientRatePerSecond * 2, targetClientRatePerSecond); + for (int i = 0 ; i < clientCount ; ++i) + rateLimiters.set(i, RateLimiter.create(clientRatePerSecond)); + nextRateLimitIncrease = settings.increaseRatePerSecondInterval; + } + } + + if ((nextRepairAt -= batchSize) <= 0) + { + nextRepairAt += settings.repairInterval; + System.out.println("repairing..."); + cluster.coordinator(1).instance().nodetool("repair", qualifiedAccordTableName); + } + + if ((nextCompactionAt -= batchSize) <= 0) + { + nextCompactionAt += settings.compactionInterval; + compactJournalCfs(cluster); + } + + if ((nextJournalFlushAt -= batchSize) <= 0) + { + nextJournalFlushAt += settings.journalFlushInterval; + flushJournal(cluster); + } + + if ((nextDataFlushAt -= batchSize) <= 0) + { + nextDataFlushAt += settings.dataFlushInterval; + flushData(cluster); + } + + if ((nextCfkFlushAt -= batchSize) <= 0) + { + nextCfkFlushAt += settings.cfkFlushInterval; + flushCfk(cluster); + } + + if ((nextChaosAt -= batchSize) <= 0) + { + Iterator iter = chaosActive.iterator(); + while (iter.hasNext()) + { + ChaosActive chaos = iter.next(); + if (chaos.future.isDone()) + { + chaos.future.get(); + iter.remove(); + Invariants.require(cluster.size() <= chaosActive.size() + chaosCandidates.size()); + } + else + { + long elapsedNanos = Clock.Global.nanoTime() - chaos.startedAt; + if (elapsedNanos >= CHAOS_TIMEOUT_NANOS) + throw new AssertionError("Chaos " + chaos.kind + " has been running for " + NANOSECONDS.toSeconds(elapsedNanos) + "s with seed " + seed); + } + } + + if (chaosActive.size() < settings.clusterChaosConcurrency) + { + if (remainingClusterChaos > 0) + { + --remainingClusterChaos; + nextChaosAt += settings.clusterChaosInterval; + ClusterChaos chaos = clusterChaos.get(); + chaosActive.add(new ChaosActive(chaos, chaos(cluster, coordinatorIndexes, chaosCandidates, chaosExecutor, random, chaos, chaosHistory))); + } + else if (chaosActive.isEmpty() && (remainingTransactions <= 0 || !waitForTransactions)) + { + break; + } + } + } + + if ((remainingTransactions -= batchSize) <= 0 && (remainingClusterChaos <= 0 || !waitForClusterChaos)) + break; + + long nowMillis = System.currentTimeMillis(); + EstimatedHistogram reads = readHistogram.getAndSet(new EstimatedHistogram(200)); + EstimatedHistogram writes = writeHistogram.getAndSet(new EstimatedHistogram(200)); + + maybePrintSlowestTraces(cluster, settings); + maybePrintLastTraces(cluster, pauseOrStop, settings, waitQueue); + printInstanceMetrics(nowMillis, cluster); + printRates(nowMillis, batchStart, batchSize, reads, writes); + printVerbs(nowMillis, verbs); + } + } + catch (Throwable t) + { + t.printStackTrace(); + System.exit(1); + } + + logger.info("Workload completed successfully"); + } + + static void safeForEach(Cluster cluster, IIsolatedExecutor.SerializableRunnable run) + { + safeForEach(cluster, ignore -> run.run(), null); + } + + static

void safeForEach(Cluster cluster, IIsolatedExecutor.SerializableConsumer

consumer, P param) + { + for (IInvokableInstance i : cluster) + { + try + { + if (!i.isShutdown()) + { + i.acceptOnInstance(consumer, param); + } + } + catch (Throwable t) + { + logger.error("", t); + } + } + } + + private void compactJournalCfs(Cluster cluster) + { + System.out.println("compacting journal cfs..."); + for (IInvokableInstance i : cluster) + { + try { i.nodetool("compact", "system_accord.journal"); } + catch (Throwable t) { logger.error("", t); } + } + } + + private void flushJournal(Cluster cluster) + { + System.out.println("flushing journal..."); + safeForEach(cluster, () -> { + if (AccordService.started()) + ((AccordService) AccordService.instance()).journal().closeCurrentSegmentForTestingIfNonEmpty(); + }); + } + + private void flushData(Cluster cluster) + { + System.out.println("flushing data..."); + safeForEach(cluster, name -> { + Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(KEYSPACE, name).id).forceFlush(UNIT_TESTS); + }, accordTableName); + } + + private void flushCfk(Cluster cluster) + { + System.out.println("flushing cfk..."); + safeForEach(cluster, () -> { + if (CommitLog.instance.isStarted()) + AccordKeyspace.AccordColumnFamilyStores.commandsForKey.forceFlush(UNIT_TESTS); + }); + } + + private static class ChaosActive + { + final ClusterChaos kind; + final Future future; + final long startedAt; + + private ChaosActive(ClusterChaos kind, Future future) + { + this.kind = kind; + this.future = future; + this.startedAt = Clock.Global.nanoTime(); + } + } + + private static Future chaos(Cluster cluster, AtomicIntegerArray coordinatorIndexes, Set candidates, ExecutorService chaosExecutor, Random random, ClusterChaos chaos, List history) + { + List snapshot = new ArrayList<>(candidates); + int nodeIdx; + { + int i = random.nextInt(snapshot.size()); + Integer remove = snapshot.get(i); + candidates.remove(remove); + snapshot = new ArrayList<>(candidates); + Invariants.require(snapshot.size() > 0); + nodeIdx = remove; + } + + for (int i = 0; i < coordinatorIndexes.length(); ++i) + { + if (nodeIdx == coordinatorIndexes.get(i)) + { + int j = random.nextInt(snapshot.size()); + int replaceIdx = snapshot.get(j); + coordinatorIndexes.set(j, replaceIdx); + } + } + + String describe = String.format("%s node %d...", chaos, nodeIdx); + history.add(describe); + System.out.println("========= BEGIN CHAOS =========="); + System.out.println(describe); + System.out.println(candidates); + System.out.println("========= BEGIN CHAOS =========="); + switch (chaos) + { + default: throw UnhandledEnum.unknown(chaos); + case REBOOTSTRAP_INCOMPLETE: + case REBOOTSTRAP_RESET: + { + IInvokableInstance node = cluster.get(nodeIdx); + return node.asyncAcceptsOnInstance((Set cnds) -> { + try + { + Node accordNode = AccordService.instance().node(); + AccordService.getBlocking(accordNode.commandStores().rebootstrap(accordNode, chaos == REBOOTSTRAP_RESET ? LOG_CORRUPTED : LOG_INCOMPLETE)); + } + finally + { + Invariants.require(cnds.add(nodeIdx)); + System.out.println("========== END CHAOS ==========="); + System.out.println(describe); + System.out.println("========== END CHAOS ==========="); + } + }).apply(candidates); + } + case REBOOTSTRAP_IF_BEHIND: + { + IInvokableInstance node = cluster.get(nodeIdx); + return node.asyncAcceptsOnInstance((Set cmds) -> { + try + { + AccordService.getBlocking(Catchup.rebootstrapIfBehind(AccordService.instance().node())); + } + finally + { + Invariants.require(cmds.add(nodeIdx)); + System.out.println("========== END CHAOS ==========="); + System.out.println(String.format("%s node %d...", chaos, nodeIdx)); + System.out.println("========== END CHAOS ==========="); + } + }).apply(candidates); + } + case RESTART: + case RESTART_AND_REBOOTSTRAP_INCOMPLETE: + case RESTART_AND_REBOOTSTRAP_RESET: + case RESTART_AND_REBOOTSTRAP_AFTER_TIMEOUT: + { + return chaosExecutor.submit(() -> { + IInvokableInstance node = cluster.get(nodeIdx); + try + { + node.shutdown().get(); + switch (chaos) + { + case RESTART_AND_REBOOTSTRAP_AFTER_TIMEOUT: + node.config().set("accord.catchup_on_start_on_timeout", "REBOOTSTRAP"); + node.config().set("accord.catchup_on_start_success_latency", "0s"); + node.config().set("accord.catchup_on_start_fail_latency", "0s"); + break; + case RESTART_AND_REBOOTSTRAP_INCOMPLETE: + node.config().set("accord.journal.replay", "REBOOTSTRAP_INCOMPLETE"); + break; + case RESTART_AND_REBOOTSTRAP_RESET: + node.config().set("accord.journal.replay", "REBOOTSTRAP_RESET"); + break; + } + node.startup(); + return null; + } + catch (InterruptedException | ExecutionException e) + { + throw new RuntimeException(e); + } + finally + { + Invariants.require(candidates.add(nodeIdx)); + node.config().set("accord.catchup_on_start_success_latency", "60s"); + node.config().set("accord.catchup_on_start_fail_latency", "120s"); + node.config().set("accord.catchup_on_start_on_timeout", "IGNORE"); + node.config().set("accord.journal.replay", "PART_NON_DURABLE"); + System.out.println("========== END CHAOS ==========="); + System.out.println(String.format("%s node %d...", chaos, nodeIdx)); + System.out.println("========== END CHAOS ==========="); + } + }); + } + } + } + + + private static void printRates(long nowMillis, long batchStartNanos, long batchSize, EstimatedHistogram reads, EstimatedHistogram writes) + { + System.out.println(String.format("%tT.%tL rate: %.2f/s (%d total)", nowMillis, nowMillis, (((float)batchSize * 1000) / NANOSECONDS.toMillis(System.nanoTime() - batchStartNanos)), batchSize)); + System.out.println(String.format("%tT.%tL reads : %d %d %d %d %d %d", nowMillis, nowMillis, reads.percentile(.25)/1000, reads.percentile(.5)/1000, reads.percentile(.95)/1000, reads.percentile(.99)/1000, reads.percentile(.999)/1000, reads.percentile(1)/1000)); + System.out.println(String.format("%tT.%tL writes: %d %d %d %d %d %d", nowMillis, nowMillis, writes.percentile(.25)/1000, writes.percentile(.5)/1000, writes.percentile(.95)/1000, writes.percentile(.99)/1000, writes.percentile(.999)/1000, writes.percentile(1)/1000)); + } + + private static void printInstanceMetrics(long nowMillis, Cluster cluster) + { + safeForEach(cluster, () -> { + refresh(AccordExecutorMetrics.INSTANCE.elapsedRunning); + refresh(AccordExecutorMetrics.INSTANCE.elapsed); + System.out.println(String.format("%tT.%tL (%d %d %d %d %d %d)ms (%d %d %d %d %d %d)ms (%d %d %d %d %.0f, %d %d %d)us %d %d %d", nowMillis, nowMillis, + getLatency(AccordCoordinatorMetrics.readMetrics.preacceptLatency, 0.5), + getLatency(AccordCoordinatorMetrics.readMetrics.executeLatency, 0.5), + getLatency(AccordCoordinatorMetrics.readMetrics.applyLatency, 0.5), + getLatency(AccordCoordinatorMetrics.readMetrics.preacceptLatency, 0.999), + getLatency(AccordCoordinatorMetrics.readMetrics.executeLatency, 0.999), + getLatency(AccordCoordinatorMetrics.readMetrics.applyLatency, 0.999), + getLatency(AccordCoordinatorMetrics.writeMetrics.preacceptLatency, 0.95), + getLatency(AccordCoordinatorMetrics.writeMetrics.executeLatency, 0.5), + getLatency(AccordCoordinatorMetrics.writeMetrics.applyLatency, 0.5), + getLatency(AccordCoordinatorMetrics.writeMetrics.preacceptLatency, 0.999), + getLatency(AccordCoordinatorMetrics.writeMetrics.executeLatency, 0.999), + getLatency(AccordCoordinatorMetrics.writeMetrics.applyLatency, 0.999), + getLatency(AccordExecutorMetrics.INSTANCE.elapsedRunning, 0.5), + getLatency(AccordExecutorMetrics.INSTANCE.elapsedRunning, 0.9), + getLatency(AccordExecutorMetrics.INSTANCE.elapsedRunning, 1.0), + getCount(AccordExecutorMetrics.INSTANCE.elapsedRunning), + getTotal(AccordExecutorMetrics.INSTANCE.elapsedRunning), + getLatency(AccordExecutorMetrics.INSTANCE.elapsed, 0.5), + getLatency(AccordExecutorMetrics.INSTANCE.elapsed, 0.9), + getLatency(AccordExecutorMetrics.INSTANCE.elapsed, 0.999), + AccordExecutorMetrics.INSTANCE.running.getValue(), + AccordExecutorMetrics.INSTANCE.waitingToRun.getValue(), + AccordExecutorMetrics.INSTANCE.preparingToRun.getValue() + )); + clear(AccordExecutorMetrics.INSTANCE.elapsedRunning); + clear(AccordExecutorMetrics.INSTANCE.elapsed); + }); + } + + private static void printVerbs(long nowMillis, Map verbs) + { + class VerbCount + { + final Verb verb; + final int count; + + VerbCount(Verb verb, int count) + { + this.verb = verb; + this.count = count; + } + } + List verbCounts = new ArrayList<>(); + for (Map.Entry e : verbs.entrySet()) + { + int count = e.getValue().getAndSet(0); + if (count != 0) verbCounts.add(new VerbCount(e.getKey(), count)); + } + verbCounts.sort(Comparator.comparing(v -> -v.count)); + + StringBuilder verbSummary = new StringBuilder(); + for (VerbCount vs : verbCounts) + { + { + if (verbSummary.length() > 0) + verbSummary.append(", "); + verbSummary.append(vs.verb); + verbSummary.append(": "); + verbSummary.append(vs.count); + } + } + System.out.println(String.format("%tT.%tL verbs: %s", nowMillis, nowMillis, verbSummary)); + } + + private static void maybePrintSlowestTraces(Cluster cluster, LoadSettings settings) + { + if (settings.traceSlowest > 0f) + { + safeForEach(cluster, () -> { + AccordTracing tracing = ((AccordAgent)AccordService.instance().agent()).tracing(); + + tracing.forEach(Functions.alwaysTrue(), (txnId, state) -> { + state.forEach(event -> { + if (event.elapsedNanos() < MILLISECONDS.toNanos(100)) + return; + + for (Message message : event.messages()) + { + long multiplier = message.atNanos < event.doneAtNanos() ? 1 : -1; + System.out.println(String.format("%s %s %s %s %s %s", txnId, event.kind, multiplier * (message.atNanos - event.atNanos)/1000000, message.nodeId, message.commandStoreId, message.message)); + } + }); + }); + tracing.eraseAll(); + }); + } + } + + private static void maybePrintLastTraces(Cluster cluster, AtomicBoolean pauseOrStop, LoadSettings settings, WaitQueue waitQueue) + { + if (settings.traceLast > 0) + { + pauseOrStop.set(true); + Map>> print = new HashMap<>(); + for (int i = 1 ; i <= cluster.size() ; ++i) + { + cluster.get(i).acceptOnInstance(out -> { + AccordService service = (AccordService)AccordService.instance(); + AccordTracing tracing = ((AccordAgent)AccordService.instance().agent()).tracing(); + PriorityQueue candidates = new PriorityQueue<>(Comparator.comparingLong(c -> -c.elapsedMicros)); + tracing.forEach(Functions.alwaysTrue(), (txnId, events) -> { + events.forEach(event -> { + if (event.kind == Client) + { + long doneAtMicros = event.doneAtMicros(); + long elapsedMicros = doneAtMicros - event.txnId().hlc(); + if (elapsedMicros > 350000 && elapsedMicros < 390000) + candidates.add(new SortedByElapsed(txnId, elapsedMicros)); + } + }); + }); + + AtomicInteger storeId = new AtomicInteger(); + while (!candidates.isEmpty()) + { + SortedByElapsed sortedCandidate = candidates.poll(); + if (sortedCandidate.elapsedMicros < 300000) + return; + + TxnId candidate = sortedCandidate.txnId; + storeId.lazySet(-1); + tracing.forEach(candidate, events -> { + events.forEach(event -> { + if (storeId.get() >= 0) + return; + for (Message message : event.messages()) + { + if (message.nodeId < 0 && message.commandStoreId >= 0) + { + storeId.set(message.commandStoreId); + break; + } + } + }); + }); + + if (storeId.get() >= 0) + { + CommandStore commandStore = service.node().commandStores().forId(storeId.get()); + List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.unsequenced(candidate, "LoadTest"), safeStore -> { + SafeCommand safeCommand = safeStore.unsafeGet(candidate); + PartialDeps deps = safeCommand.current().partialDeps(); + if (deps == null) + return null; + List> infos = new ArrayList<>(); + for (TxnId txnId : deps.txnIds()) + { + List info = new ArrayList<>(); + info.add(txnId.toString()); + infos.add(info); + } + List info = new ArrayList<>(); + info.add(candidate.toString()); + infos.add(info); + return infos; + })); + + if (result != null) + { + for (List info : result) + { + TxnId txnId = TxnId.parse(info.get(0)); + AccordService.getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "LoadTest"), safeStore -> { + SafeCommand safeCommand = safeStore.unsafeGet(txnId); + if (safeCommand.current().executeAt != null) + info.add(safeCommand.current().executeAt.toString()); + })); + } + + out.put(candidate.toString(), result); + return; + } + } + } + }, print); + } + + for (int i = 1 ; i <= cluster.size() ; ++i) + { + cluster.get(i).acceptOnInstance(out -> { + AccordTracing tracing = ((AccordAgent)AccordService.instance().agent()).tracing(); + for (Map.Entry>> e : out.entrySet()) + { + TxnId parentId = TxnId.parse(e.getKey()); + for (List infos : e.getValue()) + { + TxnId depId = TxnId.parse(infos.get(0)); + tracing.forEach(depId, events -> { + events.forEach(event -> { + infos.add(event.kind + ": [" + (event.idMicros - parentId.hlc()) + "..." + (event.doneAtMicros() - parentId.hlc()) + "][" + (event.idMicros - depId.hlc()) + "..." + (event.doneAtMicros() - depId.hlc()) + "]"); + if (event.kind == Execute) + { + for (Message message : event.messages()) + { + if (message.nodeId == parentId.node.id) + { + long atMicros = (event.idMicros + (message.atNanos - event.atNanos)/1000) - parentId.hlc(); + infos.add(atMicros + ": " + message.message); + } + } + } + }); + }); + } + } + }, print); + } + + for (Map.Entry>> e : print.entrySet()) + { + System.out.println("======" + e.getKey() + "======"); + for (List infos : e.getValue()) + System.out.println(infos); + } + if (!print.isEmpty()) + System.out.println(); + pauseOrStop.set(false); + waitQueue.signalAll(); + } + } + + private static void refresh(Histogram histogram) + { + if (histogram instanceof ShardedHistogram) + ((ShardedHistogram) histogram).refresh(); + if (histogram instanceof ShardedDecayingHistogram) + ((ShardedDecayingHistogram) histogram).refresh(); + } + + private static long getLatency(Histogram histogram, double percentile) + { + return (long)(histogram.getSnapshot().getValue(percentile) / 1000); + } + + private static long getCount(Histogram histogram) + { + return histogram.getSnapshot().size(); + } + + private static double getTotal(Histogram histogram) + { + Snapshot snapshot = histogram.getSnapshot(); + return (snapshot.getMean() * 0.0001d * snapshot.size()); + } + + private static void clear(Histogram histogram) + { + if (histogram instanceof ShardedHistogram) + ((ShardedHistogram) histogram).clear(); + if (histogram instanceof ShardedDecayingHistogram) + ((ShardedDecayingHistogram) histogram).clear(); + } + + private static long getLatency(Timer timer, double percentile) + { + if (timer instanceof SnapshottingTimer) + return (long) (((SnapshottingTimer) timer).getPercentileSnapshot().getValue(percentile) / 1000); + return (long)(timer.getSnapshot().getValue(0.999) / 1000); + } + + private static long getSize(Timer timer) + { + if (timer instanceof SnapshottingTimer) + return ((SnapshottingTimer) timer).getPercentileSnapshot().size(); + return timer.getSnapshot().size(); + } + + @Override + protected Logger logger() + { + return logger; + } + + static class SortedByElapsed + { + final TxnId txnId; + final long elapsedMicros; + + SortedByElapsed(TxnId txnId, long elapsedMicros) + { + this.txnId = txnId; + this.elapsedMicros = elapsedMicros; + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java new file mode 100644 index 0000000000..4ddbf72fe0 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.distributed.api.Feature; + +import static org.apache.cassandra.distributed.test.accord.load.LoadSettings.ycsbZipfian; + +public class AccordRebootstrapLoadTest extends AccordLoadTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(AccordRebootstrapLoadTest.class); + + @Override + protected Logger logger() + { + return logger; + } + + public void setupCluster(int nodeCount) + { + setupCluster(nodeCount, config -> { + config.with(Feature.NETWORK, Feature.GOSSIP) + .set("accord.shard_durability_target_splits", "8") + .set("accord.shard_durability_max_splits", "16") + .set("accord.shard_durability_cycle", "1m") + .set("accord.queue_submission_model", "SIGNAL") + .set("accord.command_store_shard_count", "8") + .set("accord.queue_thread_count", "4") + .set("accord.queue_shard_count", "1") + .set("accord.send_minimal", "false") // TODO (expected): only required because of misordering of preaccept, accept, commit if queued together + .set("accord.catchup_on_start_fail_latency", "2m"); + }); + } + + @Test + public void testLoad() throws Exception + { + testLoad(new LoadSettings.Builder() + .setKeySelector(ycsbZipfian(100_000)) + .setRatePerSecond(200) + .setClusterChaosInterval(10000) + .setClusterChaosConcurrency(2) + .setTotalClusterChaos(10) + .build()); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java new file mode 100644 index 0000000000..e72cd30aad --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import java.util.Arrays; + +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.distributed.shared.DistributedTestBase; + +import static org.apache.cassandra.distributed.test.accord.load.LoadSettings.ycsbZipfian; + +public class AccordYcsbLoadTest extends AccordLoadTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(AccordYcsbLoadTest.class); + + private static final int[][] LATENCIES = new int[][] { + new int[] { 0, 44, 64, 43, 84 }, + new int[] { 44, 0, 30, 3, 45 }, + new int[] { 64, 30, 0, 28, 37 }, + new int[] { 43, 3, 28, 0, 49 }, + new int[] { 84, 45, 37, 49, 0 } + }; + + private static LoadSettings.Builder withArtificialLatencies(LoadSettings.Builder builder) + { + return builder.setArtificialLatencies(LATENCIES); + } + + private static LoadSettings.Builder ycsbA(LoadSettings.Builder builder, int keyCount) + { + return builder.setKeySelector(ycsbZipfian(keyCount)) + .setReadRatio(0.5f); + } + + private static LoadSettings.Builder ycsbB(LoadSettings.Builder builder, int keyCount) + { + return builder.setKeySelector(ycsbZipfian(keyCount)) + .setReadRatio(0.95f); + } + + private static LoadSettings.Builder ycsbC(LoadSettings.Builder builder, int keyCount) + { + return builder.setKeySelector(ycsbZipfian(keyCount)) + .setReadRatio(1.0f); + + } + + @Override + protected Logger logger() + { + return logger; + } + + private static void computeWorstLatencies() + { + int[] qs = new int[LATENCIES.length]; + for (int i = 0 ; i < qs.length ; ++i) + { + int[] copy = LATENCIES[i].clone(); + Arrays.sort(copy); + qs[i] = copy[copy.length/2]; + } + int[] ws = new int[qs.length]; + for (int i = 0 ; i < qs.length ; ++i) + { + int iw = Integer.MIN_VALUE; + for (int j = 0; j < qs.length ; ++j) + iw = Math.max(iw, qs[i] + 3*qs[j] + LATENCIES[i][j]); + ws[i] = iw; + } + System.out.println(Arrays.toString(ws)); + Arrays.fill(ws, 0); + for (int i = 0 ; i < qs.length ; ++i) + { + for (int j = 0 ; j < qs.length ; ++j) + { + if (j == i) continue; + if (qs[j] > 2*qs[i]) continue; + int w = qs[i] + 4*qs[j] + LATENCIES[i][j]; + if (w > ws[i]) + ws[i] = w; + } + } + System.out.println(Arrays.toString(ws)); + } + + @Ignore + @Test + public void testLoad() throws Exception + { + testLoad(ycsbA(new LoadSettings.Builder(), 100_000) + .setRatePerSecond(1600).setMinRatePerSecond(200) + .setIncreaseRatePerSecondInterval(5000) + .build()); + } + + public static void main(String[] args) throws Throwable + { + computeWorstLatencies(); + + DistributedTestBase.beforeClass(); + AccordYcsbLoadTest test = new AccordYcsbLoadTest(); + try + { + test.setupCluster(); + test.setup(); + test.testLoad(withArtificialLatencies(ycsbA(new LoadSettings.Builder(), 100_000) + .setRatePerSecond(1600).setMinRatePerSecond(200) + .setIncreaseRatePerSecondInterval(5000) + ).build()); + } + finally + { + test.tearDown(); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java new file mode 100644 index 0000000000..004df82941 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntSupplier; +import java.util.function.LongFunction; +import java.util.function.Supplier; + +import org.apache.commons.math3.distribution.ZipfDistribution; +import org.apache.commons.math3.random.JDKRandomGenerator; + +import accord.utils.DefaultRandom; + +import static java.util.concurrent.TimeUnit.SECONDS; + +class LoadSettings +{ + enum ClusterChaos { + RESTART, + RESTART_AND_REBOOTSTRAP_INCOMPLETE, RESTART_AND_REBOOTSTRAP_RESET, + RESTART_AND_REBOOTSTRAP_AFTER_TIMEOUT, + + REBOOTSTRAP_INCOMPLETE, REBOOTSTRAP_RESET, + REBOOTSTRAP_IF_BEHIND + } + + final int repairInterval; + final int compactionInterval; + final int journalFlushInterval; + final int cfkFlushInterval; + final int cfkCompactionPeriodSeconds; + final int dataFlushInterval; + final int clusterChaosInterval; + final int clusterChaosDecay; + final int clusterChaosConcurrency; + final LongFunction> clusterChaos; + final int batchSize; + final long batchPeriodNanos; + final int clientConcurrency; + final int clients; + final int ratePerSecond; + final int minRatePerSecond; + final int increaseRatePerSecondInterval; + final int keysPerOperation; + final float readRatio; + final IntSupplier keySelector; + final boolean readBeforeWrite; + final float traceSlowest; + final int traceLast; + final long totalTransactions; + final int totalClusterChaos; + final int[][] artificialLatencies; + + LoadSettings(Builder builder) + { + this.repairInterval = builder.repairInterval; + this.compactionInterval = builder.compactionInterval; + this.journalFlushInterval = builder.journalFlushInterval; + this.cfkFlushInterval = builder.cfkFlushInterval; + this.cfkCompactionPeriodSeconds = builder.cfkCompactionPeriodSeconds; + this.dataFlushInterval = builder.dataFlushInterval; + this.clusterChaosInterval = builder.clusterChaosInterval; + this.clusterChaosDecay = builder.clusterChaosDecay; + this.clusterChaosConcurrency = builder.clusterChaosConcurrency; + this.clusterChaos = builder.clusterChaos; + this.batchSize = builder.batchSize; + this.batchPeriodNanos = builder.batchPeriodNanos; + this.clientConcurrency = builder.clientConcurrency; + this.clients = builder.clients; + this.ratePerSecond = builder.ratePerSecond; + this.minRatePerSecond = builder.minRatePerSecond; + this.increaseRatePerSecondInterval = builder.increaseRatePerSecondInterval; + this.keysPerOperation = builder.keysPerOperation; + this.readRatio = builder.readRatio; + this.keySelector = builder.keySelector; + this.readBeforeWrite = builder.readBeforeWrite; + this.artificialLatencies = builder.artificialLatencies; + this.traceSlowest = builder.traceSlowest; + this.traceLast = builder.traceLast; + this.totalTransactions = builder.totalTransactions; + this.totalClusterChaos = builder.totalClusterChaos; + } + + // interval is measured in terms of *operations* unless otherwise specified + public static class Builder + { + int repairInterval = Integer.MAX_VALUE; + int compactionInterval = Integer.MAX_VALUE; + int journalFlushInterval = Integer.MAX_VALUE; + int cfkFlushInterval = Integer.MAX_VALUE; + int cfkCompactionPeriodSeconds = 0; + int dataFlushInterval = Integer.MAX_VALUE; + int clusterChaosInterval = Integer.MAX_VALUE; + int clusterChaosDecay = 1; + int clusterChaosConcurrency = 1; + LongFunction> clusterChaos = seed -> new DefaultRandom(seed).randomWeightedPicker(LoadSettings.ClusterChaos.values()); + int batchSize = 1000; + long batchPeriodNanos = SECONDS.toNanos(10); + int clientConcurrency = 50; + int clients = -1; + int ratePerSecond = 1000; + int minRatePerSecond = 50; + int increaseRatePerSecondInterval = 1000; + int keysPerOperation = 1; + float readRatio = 0.5f; + IntSupplier keySelector; + boolean readBeforeWrite; + float traceSlowest; + int traceLast; + int[][] artificialLatencies; + long totalTransactions = Long.MAX_VALUE; + int totalClusterChaos = Integer.MAX_VALUE; + + public Builder setRepairInterval(int repairInterval) + { + this.repairInterval = repairInterval; + return this; + } + + public Builder setCompactionInterval(int compactionInterval) + { + this.compactionInterval = compactionInterval; + return this; + } + + public Builder setJournalFlushInterval(int journalFlushInterval) + { + this.journalFlushInterval = journalFlushInterval; + return this; + } + + public Builder setCfkFlushInterval(int cfkFlushInterval) + { + this.cfkFlushInterval = cfkFlushInterval; + return this; + } + + public Builder setCfkCompactionPeriodSeconds(int cfkCompactionPeriodSeconds) + { + this.cfkCompactionPeriodSeconds = cfkCompactionPeriodSeconds; + return this; + } + + public Builder setDataFlushInterval(int dataFlushInterval) + { + this.dataFlushInterval = dataFlushInterval; + return this; + } + + public Builder setClusterChaosInterval(int clusterChaosInterval) + { + this.clusterChaosInterval = clusterChaosInterval; + return this; + } + + public Builder setClusterChaosDecay(int clusterChaosDecay) + { + this.clusterChaosDecay = clusterChaosDecay; + return this; + } + + public Builder setClusterChaosConcurrency(int clusterChaosConcurrency) + { + this.clusterChaosConcurrency = clusterChaosConcurrency; + return this; + } + + public Builder setClusterChaos(LongFunction> clusterChaos) + { + this.clusterChaos = clusterChaos; + return this; + } + + public Builder setTotalTransactions(long totalTransactions) + { + this.totalTransactions = totalTransactions; + return this; + } + + public Builder setTotalClusterChaos(int totalClusterChaos) + { + this.totalClusterChaos = totalClusterChaos; + return this; + } + + public Builder setBatchSize(int batchSize) + { + this.batchSize = batchSize; + return this; + } + + public Builder setBatchPeriodNanos(long batchPeriodNanos) + { + this.batchPeriodNanos = batchPeriodNanos; + return this; + } + + public Builder setClientConcurrency(int clientConcurrency) + { + this.clientConcurrency = clientConcurrency; + return this; + } + + public Builder setClients(int clients) + { + this.clients = clients; + return this; + } + + public Builder setRatePerSecond(int ratePerSecond) + { + this.ratePerSecond = ratePerSecond; + return this; + } + + public Builder setMinRatePerSecond(int minRatePerSecond) + { + this.minRatePerSecond = minRatePerSecond; + return this; + } + + public Builder setIncreaseRatePerSecondInterval(int increaseRatePerSecondInterval) + { + this.increaseRatePerSecondInterval = increaseRatePerSecondInterval; + return this; + } + + public Builder setKeysPerOperation(int keysPerOperation) + { + this.keysPerOperation = keysPerOperation; + return this; + } + + public Builder setReadRatio(float readRatio) + { + this.readRatio = readRatio; + return this; + } + + public Builder setReadBeforeWrite(boolean readBeforeWrite) + { + this.readBeforeWrite = readBeforeWrite; + return this; + } + + public Builder setTraceSlowest(float traceSlowest) + { + this.traceSlowest = traceSlowest; + return this; + } + + public Builder setTraceLast(int traceLast) + { + this.traceLast = traceLast; + return this; + } + + public Builder setKeySelector(IntSupplier keySelector) + { + this.keySelector = keySelector; + return this; + } + + public Builder setArtificialLatencies(int[][] artificialLatencies) + { + this.artificialLatencies = artificialLatencies; + return this; + } + + public LoadSettings build() + { + return new LoadSettings(this); + } + } + + static IntSupplier ycsbZipfian(int keyCount) + { + ZipfDistribution distribution = new ZipfDistribution(new JDKRandomGenerator(), keyCount, 0.99); + int count = distribution.inverseCumulativeProbability(0.65f); + float[] probs = new float[count]; + for (int i = 0 ; i < probs.length ; ++i) + probs[i] = (float) distribution.cumulativeProbability(i); + // zipf is slow to compute, so we cache the first 65% of the distribution then use uniform probability; this is good enough for our purposes + float max = probs[probs.length - 1]; + float inv_incr = probs.length >= keyCount ? 0f : 1f / ((1f-max)/(keyCount - probs.length)); + Random random = new Random(); + return () -> { + float v = random.nextFloat(); + if (v < max) + { + int i = Arrays.binarySearch(probs, v); + if (i < 0) i = -1 - i; + return i; + } + else + { + return (int)((v - max)*inv_incr); + } + }; + } + + static IntSupplier roundrobin(int keyCount) + { + AtomicInteger next = new AtomicInteger(); + return () -> { + int v = next.incrementAndGet(); + if (v < keyCount) + return v; + return next.updateAndGet(i -> i > keyCount ? 0 : i + 1); + }; + } + + static IntSupplier uniform(int keyCount) + { + Random random = new Random(); + return () -> random.nextInt(keyCount); + } +} diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java index 640ce44740..04121c55d4 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java @@ -65,7 +65,7 @@ public class AccordRebootstrapTest extends FuzzTestBase try (Cluster cluster = builder.withNodes(3) .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(100)) .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(100, "dc0", "rack0")) - .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP)) + .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP).set("accord.catchup_on_start_fail_latency", "60s")) .start()) { IInvokableInstance cmsInstance = cluster.get(1); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java index b4af888878..8ab7e56bff 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java @@ -84,7 +84,7 @@ public class AccordMessageSinkTest TxnId id = nextTxnId(epoch, txn); Ranges ranges = Ranges.of(IntKey.range(40, 50)); PartialTxn partialTxn = txn.slice(ranges, true); - Request request = new AccordFetchRequest(epoch, id, ranges, PartialDeps.NONE, partialTxn); + Request request = new AccordFetchRequest(epoch, id, ranges, partialTxn); checkRequestReplies(request, new AbstractFetchCoordinator.FetchResponse(null, null, id), diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index 1f8f963ccc..c5689e813a 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -44,6 +44,7 @@ import accord.api.RemoteListeners.NoOpRemoteListeners; import accord.api.Result; import accord.api.Result.PersistableResult; import accord.api.RoutingKey; +import accord.api.Scheduler; import accord.api.Timeouts; import accord.coordinate.Coordinations; import accord.impl.DefaultLocalListeners; @@ -324,21 +325,11 @@ public class AccordTestUtils { NodeCommandStoreService time = new NodeCommandStoreService() { - @Override - public AsyncExecutor someExecutor() - { - return null; - } - - @Override - public ExclusiveAsyncExecutor someExclusiveExecutor() - { - return null; - } - private ToLongFunction elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MICROSECONDS, this::now); private long stamp = 0; + @Override public AsyncExecutor someExecutor() { return null; } + @Override public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } @Override public Timeouts timeouts() { return null; } @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; } @Override public DurabilityService durability() { return null; } @@ -349,6 +340,7 @@ public class AccordTestUtils @Override public long elapsed(TimeUnit timeUnit) { return elapsed.applyAsLong(timeUnit); } @Override public TopologyManager topology() { throw new UnsupportedOperationException(); } @Override public Coordinations coordinations() { return new Coordinations(); } + @Override public Scheduler scheduler() { return null; } @Override public long currentStamp() { return stamp; } @Override public void updateStamp() {++stamp;} @Override public boolean isReplaying() { return false; } diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 165115d8fd..ae5b6468e3 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -42,6 +42,7 @@ import accord.api.ProgressLog; import accord.api.RemoteListeners; import accord.api.Result; import accord.api.RoutingKey; +import accord.api.Scheduler; import accord.api.Timeouts; import accord.coordinate.Coordinations; import accord.impl.DefaultLocalListeners; @@ -268,6 +269,12 @@ public class SimulatedAccordCommandStore implements AutoCloseable } @Override public Coordinations coordinations() { return new Coordinations(); } + + @Override + public Scheduler scheduler() + { + return null; + } }; TestAgent.RethrowAgent agent = new TestAgent.RethrowAgent() diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 448583ac00..3b975207b7 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -56,6 +56,7 @@ import accord.api.OwnershipEventListener; import accord.api.ProgressLog; import accord.api.Result; import accord.api.RoutingKey; +import accord.api.Scheduler; import accord.api.Timeouts; import accord.coordinate.Coordinations; import accord.impl.AbstractReplayer; @@ -711,6 +712,7 @@ public class CommandsForKeySerializerTest @Override public long now() { return 0; } @Override public long elapsed(TimeUnit unit) { return 0; } @Override public Coordinations coordinations() { return new Coordinations(); } + @Override public Scheduler scheduler() { return null; } }; } @Override public boolean visit(Unseekables keysOrRanges, TxnId testTxnId, Kind.Kinds testKind, SupersedingCommandVisitor visit) { return false; } @Override public void visit(Unseekables keysOrRanges, Timestamp startedBefore, Kind.Kinds testKind, ActiveCommandVisitor visit, P1 p1, P2 p2) { } From 25f6d75be81dbb276f9beed1c1904fd82aa1cc69 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Thu, 30 Jul 2026 16:31:20 +0100 Subject: [PATCH 09/10] fixup --- .../managing/configuration/configuration.adoc | 2 +- modules/accord | 2 +- .../apache/cassandra/config/AccordConfig.java | 12 +++- .../exceptions/ExceptionSerializer.java | 10 ++- .../service/accord/AccordDataStore.java | 9 ++- .../accord/AccordFetchCoordinator.java | 9 --- .../service/accord/AccordService.java | 68 ++++++++++++++----- .../service/accord/execution/SafeTask.java | 5 +- .../service/accord/execution/Task.java | 1 - .../serializers/CommandStoreSerializers.java | 6 +- .../accord/AccordJournalIntegrationTest.java | 6 +- .../journal/AccordJournalReplayTest.java | 2 +- ...rnalAccessRouteIndexOnStartupRaceTest.java | 2 +- ...atchupTest.java => AccordCatchupTest.java} | 4 +- 14 files changed, 94 insertions(+), 44 deletions(-) rename test/distributed/org/apache/cassandra/fuzz/topology/{AccordHardCatchupTest.java => AccordCatchupTest.java} (99%) diff --git a/doc/modules/cassandra/pages/managing/configuration/configuration.adoc b/doc/modules/cassandra/pages/managing/configuration/configuration.adoc index 0933da73fd..9b4df248b2 100644 --- a/doc/modules/cassandra/pages/managing/configuration/configuration.adoc +++ b/doc/modules/cassandra/pages/managing/configuration/configuration.adoc @@ -153,7 +153,7 @@ configuration changes in the near future. `@Replaces` is the annotation to be used when you make changes to any configuration parameters in `Config` class and `cassandra.yaml`, and you want to add backward compatibility with previous Cassandra versions. `Converters` class enumerates the different methods used for backward compatibility. `IDENTITY` is the one used for name change only. For more information about the other Converters, please, check the JavaDoc in the class. -For backward compatibility virtual table `LoadSettings` contains both the old and the new +For backward compatibility virtual table `Settings` contains both the old and the new parameters with the old and the new value format. Only exception at the moment are the following three parameters: `key_cache_save_period`, `row_cache_save_period` and `counter_cache_save_period` which appear only once with the new value format. The old names and value format still can be used at least until the next major release. Deprecation warning is emitted on startup. diff --git a/modules/accord b/modules/accord index 673cf2ff93..f4c8ae02f9 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 673cf2ff93a4b32a392c91feacda09bb48ac16e7 +Subproject commit f4c8ae02f9393c5eb14720b5074bd45135ee5b00 diff --git a/src/java/org/apache/cassandra/config/AccordConfig.java b/src/java/org/apache/cassandra/config/AccordConfig.java index 50d00d2b6f..944b8c9d92 100644 --- a/src/java/org/apache/cassandra/config/AccordConfig.java +++ b/src/java/org/apache/cassandra/config/AccordConfig.java @@ -27,6 +27,7 @@ import accord.api.ProtocolModifiers.CoordinatorBacklogExecution; import accord.api.ProtocolModifiers.FastExecution; import accord.api.ProtocolModifiers.ReplicaExecution; import accord.api.ProtocolModifiers.SendStableMessages; +import accord.api.ProtocolModifiers.UniqueTimestampOnConflict; import accord.primitives.TxnId; import accord.utils.Invariants; @@ -34,7 +35,6 @@ import org.apache.cassandra.journal.Params; import org.apache.cassandra.service.accord.serializers.Version; import org.apache.cassandra.service.consensus.TransactionalMode; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.HLC_FIFO; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_POOL_PER_SHARD; import static org.apache.cassandra.config.AccordConfig.QueueSubmissionModel.SYNC; import static org.apache.cassandra.config.AccordConfig.RangeIndexMode.in_memory; @@ -173,6 +173,13 @@ public class AccordConfig BLENDED_PRIORITY_PHASE_FAIR, } + public enum UniqueTimestampReservations + { + NONE, + SMALL_SHARED, + HISTOGRAM + } + public QueueShardModel queue_shard_model = THREAD_POOL_PER_SHARD; public QueueSubmissionModel queue_submission_model = SYNC; @@ -334,6 +341,9 @@ public class AccordConfig public Boolean send_minimal; // note: simulator incompatible (for now) public Boolean precise_micros; + public UniqueTimestampReservations unique_timestamp_reservations; + public UniqueTimestampOnConflict unique_timestamp_on_conflict; + public DurationSpec.IntMillisecondsBound unique_timestamp_reservation_range; public boolean ephemeral_reads = true; public boolean state_cache_listener_jfr_enabled = false; diff --git a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java index 50a5393d10..63c70a5447 100644 --- a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java +++ b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java @@ -44,6 +44,9 @@ import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; */ public class ExceptionSerializer { + // allow plenty of room for UTF8 encoding + private static final int MAX_MESSAGE_CHAR_LENGTH = Short.MAX_VALUE / 4; + public static class RemoteException extends RuntimeException { public final String originalClass; @@ -76,8 +79,11 @@ public class ExceptionSerializer if (isFirstException) message = "Remote exception from host " + FBUtilities.getBroadcastAddressAndPort().toString() + (t.getLocalizedMessage() != null ? " - " + t.getLocalizedMessage() : ""); else message = t.getLocalizedMessage(); - if (message.length() > Short.MAX_VALUE) - message = message.substring(0, Short.MAX_VALUE); + if (message == null) + return message; + + if (message.length() > MAX_MESSAGE_CHAR_LENGTH) + message = message.substring(0, MAX_MESSAGE_CHAR_LENGTH); return message; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java index 9e227e5f5a..ebd2911673 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java @@ -148,6 +148,7 @@ public class AccordDataStore implements DataStore case SUCCESS: callback.fetched(ranges); syncResult.trySuccess(null); + // fall-through to ensure started case START: reportStarted(); break; @@ -157,7 +158,7 @@ public class AccordDataStore implements DataStore break; case ABORT: case ERROR: - RuntimeException ex = new RuntimeException(String.format("Repair failed (%s):", event.getType(), event.getMessage())); + RuntimeException ex = new RuntimeException(String.format("Repair failed (%s): %s", event.getType(), event.getMessage())); callback.fail(ranges, ex); syncResult.tryFailure(ex); break; @@ -173,7 +174,11 @@ public class AccordDataStore implements DataStore node.commandStores().mapReduceConsume(new RestrictedStoreSelector(ranges, 0, Long.MAX_VALUE), new MapReduceConsumeCommandStores(ranges) { @Override public Timestamp reduce(Timestamp o1, Timestamp o2) { return Timestamp.max(o1, o2); } - @Override public void accept(Timestamp result, Throwable failure) { start.started(result); } + @Override public void accept(Timestamp result, Throwable failure) + { + if (failure != null) syncResult.tryFailure(failure); + else start.started(result); + } @Override public TxnId primaryTxnId() { return null; } @Override public String reason() { return "Compute MaxConflict to report for fetch"; } @Override protected Timestamp applyInternal(SafeCommandStore safeStore) { return safeStore.commandStore().maxConflict(TxnId.NONE, ranges); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index 630f81ccbf..74266b0c15 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -459,15 +459,6 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements { super(sourceEpoch, syncId, ranges, partialTxn); } - - @Override - protected AsyncChain beginRead(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Participants execute) - { - AsyncChain result = super.beginRead(safeStore, executeAt, txn, execute); - // TODO (required): verify that streaming snapshots have all been created by now, so we won't stream any data that arrives after this - readStarted(safeStore); - return result; - } } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index e198670b5f..ff5752dca0 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -63,6 +63,9 @@ import accord.local.CommandStores; import accord.local.Node; import accord.local.Node.Id; import accord.local.ShardDistributor.EvenSplit; +import accord.local.TimeService; +import accord.local.UniqueTimeService; +import accord.local.UniqueTimeService.AtomicUniqueTime; import accord.local.UniqueTimeService.AtomicUniqueTimeWithStaleReservation; import accord.local.durability.DurabilityService; import accord.local.durability.ShardDurability; @@ -93,7 +96,9 @@ import accord.utils.async.AsyncResults; import org.apache.cassandra.concurrent.Shutdownable; import org.apache.cassandra.config.AccordConfig; +import org.apache.cassandra.config.AccordConfig.CatchupFallbackMode; import org.apache.cassandra.config.AccordConfig.JournalConfig.ReplaySavePoint; +import org.apache.cassandra.config.AccordConfig.UniqueTimestampReservations; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; @@ -175,16 +180,20 @@ import static accord.primitives.Txn.Kind.ExclusiveSyncPoint; import static accord.primitives.Txn.Kind.Write; import static accord.primitives.TxnId.MediumPath.NoMediumPath; import static accord.primitives.TxnId.MediumPath.TrackStable; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorThreadTag.JOB; import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.DAEMON; +import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.EXIT; import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.REBOOTSTRAP; +import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.REBOOTSTRAP_AND_CATCHUP; import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.REBOOTSTRAP_INCOMPLETE; import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.REBOOTSTRAP_RESET; import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.RESET; +import static org.apache.cassandra.config.AccordConfig.UniqueTimestampReservations.HISTOGRAM; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; import static org.apache.cassandra.config.DatabaseDescriptor.getAccordGlobalDurabilityCycle; import static org.apache.cassandra.config.DatabaseDescriptor.getAccordShardDurabilityCycle; @@ -479,7 +488,7 @@ public class AccordService implements IAccordService, Shutdownable this.node = new Node(localId, messageSink, topologyService, - time, new AtomicUniqueTimeWithStaleReservation(time), + time, uniqueTimeService(time), () -> dataStore, new KeyspaceSplitter(new EvenSplit<>(getAccord().commandStoreShardCount(), getPartitioner().accordSplitter())), agent, @@ -529,6 +538,8 @@ public class AccordService implements IAccordService, Shutdownable ProtocolModifiers.Configure.setSendStableMessages(config.send_stable); if (config.send_minimal != null) ProtocolModifiers.Configure.setSendMinimal(config.send_minimal); + if (config.unique_timestamp_on_conflict != null) + ProtocolModifiers.Configure.setUniqueTimestampOnConflict(config.unique_timestamp_on_conflict); } @Override @@ -718,8 +729,8 @@ public class AccordService implements IAccordService, Shutdownable void catchup() { - AccordConfig spec = DatabaseDescriptor.getAccord(); - if (!spec.catchup_on_start) + AccordConfig config = DatabaseDescriptor.getAccord(); + if (!config.catchup_on_start) { logger.info("Catchup disabled; continuing to startup"); return; @@ -731,22 +742,17 @@ public class AccordService implements IAccordService, Shutdownable node.commandStores().forAllUnsafe(commandStore -> ((DefaultProgressLog)commandStore.unsafeProgressLog()).setMode(CATCH_UP)); try { - AccordConfig.CatchupFallbackMode onError = spec.catchup_on_start_on_error; - AccordConfig.CatchupFallbackMode onTimeout = spec.catchup_on_start_on_timeout; + CatchupFallbackMode onError = config.catchup_on_start_on_error; + CatchupFallbackMode onTimeout = config.catchup_on_start_on_timeout; - long maxLatencyNanos = spec.catchup_on_start_fail_latency.toNanoseconds(); + long maxLatencyNanos = config.catchup_on_start_fail_latency.toNanoseconds(); int attempts = 1; while (true) { logger.info("Catchup with quorum..."); long start = nanoTime(); long failAt = start + maxLatencyNanos; - Future f; - { - AsyncChain submit = Catchup.catchup(node, failAt, NANOSECONDS); - f = toFuture(submit); - } - + Future f = toFuture(Catchup.catchup(node, failAt, NANOSECONDS)); f.awaitThrowUncheckedOnInterrupt(); Throwable failed = f.cause(); Unsuccessful result = f.getNow(); @@ -769,7 +775,7 @@ public class AccordService implements IAccordService, Shutdownable if (onError == REBOOTSTRAP) return; ++attempts; - onError = onTimeout = spec.catchup_on_start_on_rebootstrap_fallback; + onError = onTimeout = rebootstrapFallbackMode(config); continue; } } @@ -780,10 +786,10 @@ public class AccordService implements IAccordService, Shutdownable if (result == null) { - if (seconds <= spec.catchup_on_start_success_latency.toSeconds()) + if (seconds <= config.catchup_on_start_success_latency.toSeconds()) return; - if (attempts < spec.catchup_on_start_max_attempts) + if (attempts < config.catchup_on_start_max_attempts) { logger.info("Catchup was slow, so we may behind again; retrying"); ++attempts; @@ -810,7 +816,7 @@ public class AccordService implements IAccordService, Shutdownable if (onTimeout == REBOOTSTRAP) return; ++attempts; - onError = onTimeout = spec.catchup_on_start_on_rebootstrap_fallback; + onError = onTimeout = rebootstrapFallbackMode(config); continue; } } @@ -826,6 +832,18 @@ public class AccordService implements IAccordService, Shutdownable } } + private static CatchupFallbackMode rebootstrapFallbackMode(AccordConfig config) + { + CatchupFallbackMode mode = config.catchup_on_start_on_rebootstrap_fallback; + if (mode == REBOOTSTRAP_AND_CATCHUP) + { + CatchupFallbackMode newMode = EXIT; + logger.warn("Converting {} to {} after first failed rebootstrap, to avoid infinite loop", mode, newMode); + mode = newMode; + } + return mode; + } + /** * Startup is broken up in two phases: local and distributed startup. During local startup, we replay up to * the latest epoch known to the node prior to restart. After that, we replay journal itself, and only after @@ -1469,4 +1487,22 @@ public class AccordService implements IAccordService, Shutdownable { return journal.configuration(); } + + private static UniqueTimeService uniqueTimeService(TimeService time) + { + AccordConfig config = getAccord(); + UniqueTimestampReservations reservations = config.unique_timestamp_reservations; + if (reservations == null) + reservations = HISTOGRAM; + + switch (reservations) + { + default: throw UnhandledEnum.unknown(reservations); + case NONE: return new AtomicUniqueTime(time); + case SMALL_SHARED: return new AtomicUniqueTimeWithStaleReservation(time); + case HISTOGRAM: + long millis = config.unique_timestamp_reservation_range == null ? 50 : config.unique_timestamp_reservation_range.toMilliseconds(); + return new UniqueTimeService.AtomicUniqueAutoStaleTimes(time, millis, MILLISECONDS); + } + } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java index 5eee015214..3b4274a022 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java @@ -467,13 +467,16 @@ public final class SafeTask extends Task implements Cancellable, DebuggableTa if (isSequencedByPriorityAtomic()) { - boolean isTxnIdSubset = context.isTxnIdSubsetOf(parent.context); if (!isKeySubset) { Invariants.require(keysOrRanges.domain() == Key, "ATOMIC tasks over ranges must declare a subset of their parent's task keys() to avoid a range scan across which we would not impose sequencing"); // to avoid priority inversion deadlocks, if we are not a strict subset of the parent task we permit running with a single key ready nonSync.alwaysReady = true; } + + boolean isTxnIdSubset = true; + for (TxnId txnId : context.txnIds()) + isTxnIdSubset &= parent.refs.containsKey(txnId); Invariants.require(isTxnIdSubset, "ATOMIC tasks must declare a subset of their parent's task txnIds() to avoid a priority inversion deadlock"); // TODO (required): we're appending to the fifo queue - does this maintain correct order? setCacheQueuedFifoExclusive(); diff --git a/src/java/org/apache/cassandra/service/accord/execution/Task.java b/src/java/org/apache/cassandra/service/accord/execution/Task.java index 6a3e50309f..c377ea3346 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/Task.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Task.java @@ -262,7 +262,6 @@ public abstract class Task extends IntrusiveHeapNode implements Cancellable, Deb Task next; Task consequences; - // the queue position until run is invoked, at which point it is assigned a nanoTime() value long position; int info; diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java index 7c19d05d07..f394b68df3 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java @@ -273,13 +273,13 @@ public class CommandStoreSerializers } } - private short cast(long v) + private int cast(long v) { if (v < 0 || v >= (1 << RedundantStatus.Property.WAS_OWNED.ordinal())) throw new IllegalStateException("Should not be serializing WAS_OWNED or NOT_OWNED statuses"); if ((v & ~0xFFFF) != 0) // retain this throw new IllegalStateException("Cannot serialize RedundantStatus larger than 0xFFFF. Requires serialization changes."); - return (short)v; + return (int) v; } @Override @@ -301,7 +301,7 @@ public class CommandStoreSerializers bounds[i] = CommandSerializers.txnId.deserialize(in); int[] statuses = new int[count * 2]; for (int i = 0 ; i < statuses.length ; ++i) - statuses[i] = in.readShort(); + statuses[i] = in.readShort() & 0xFFFF; return new RedundantBefore.Bounds(range, startEpoch, endEpoch, bounds, statuses, staleUntilAtLeast); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java index f8e2f82d2d..c1bd8d6b48 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java @@ -47,7 +47,7 @@ public class AccordJournalIntegrationTest extends TestBaseImpl { try (WithProperties wp = new WithProperties().set(CassandraRelevantProperties.DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED, "true"); Cluster cluster = init(Cluster.build(1) - .withConfig(config -> config.set("accord.catchup_on_start", "DISABLED")) + .withConfig(config -> config.set("accord.catchup_on_start", "false")) .withoutVNodes() .start())) { @@ -96,7 +96,7 @@ public class AccordJournalIntegrationTest extends TestBaseImpl { try (Cluster cluster = Cluster.build(1) .withoutVNodes() - .withConfig(config -> config.set("accord.catchup_on_start", "DISABLED")) + .withConfig(config -> config.set("accord.catchup_on_start", "false")) .start()) { cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};"); @@ -128,7 +128,7 @@ public class AccordJournalIntegrationTest extends TestBaseImpl .withoutVNodes() .withConfig(c -> { c.with(GOSSIP).with(NETWORK); - c.set("accord.catchup_on_start", "DISABLED"); + c.set("accord.catchup_on_start", "false"); }) .start()) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java index 44bcc0f306..08779458fc 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java @@ -98,7 +98,7 @@ public class AccordJournalReplayTest extends TestBaseImpl .set("accord.retry_syncpoint", "1s*attempts") .set("accord.retry_durability", "1s*attempts") .set("accord.journal.replay_save_point", "NO") - .set("accord.catchup_on_start", "DISABLED") + .set("accord.catchup_on_start", "false") .with(NETWORK, GOSSIP)) .start()) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java index 04f53770c5..6ab7388879 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java @@ -66,7 +66,7 @@ public class JournalAccessRouteIndexOnStartupRaceTest extends TestBaseImpl public void test() throws IOException { try (Cluster cluster = Cluster.build(1) - .withConfig(config -> config.set("accord.catchup_on_start", "DISABLED")) + .withConfig(config -> config.set("accord.catchup_on_start", "false")) .withInstanceInitializer(BBHelper::install).start()) { IInvokableInstance node = cluster.get(1); diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordCatchupTest.java similarity index 99% rename from test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java rename to test/distributed/org/apache/cassandra/fuzz/topology/AccordCatchupTest.java index 746d3b5bb7..6f2135b794 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordCatchupTest.java @@ -46,7 +46,7 @@ import org.apache.cassandra.service.consensus.TransactionalMode; import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce; import static org.apache.cassandra.harry.checker.TestHelper.withRandom; -public class AccordHardCatchupTest extends FuzzTestBase +public class AccordCatchupTest extends FuzzTestBase { private static final int WRITES = 10; private static final int POPULATION = 1000; @@ -117,7 +117,7 @@ public class AccordHardCatchupTest extends FuzzTestBase writeAndValidate.run(); history.customThrowing(() -> { - cluster.get(2).config().set("accord.catchup_on_start", "HARD"); + cluster.get(2).config().set("accord.catchup_on_start", "true"); cluster.get(2).startup(); cluster.get(2).logs().watchFor(".*Catchup.*"); cluster.get(1).logs().watchFor("/127.0.0.2:.* is now UP"); From 9893f3b2fd5c19fa11fa6e30bfa34c16e483746f Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Fri, 31 Jul 2026 15:17:05 +0100 Subject: [PATCH 10/10] UNFINISHED.md - Claude generated summary of remaining work --- UNFINISHED.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 UNFINISHED.md diff --git a/UNFINISHED.md b/UNFINISHED.md new file mode 100644 index 0000000000..34fb6438c9 --- /dev/null +++ b/UNFINISHED.md @@ -0,0 +1,28 @@ +### D1. Partial log unavailability (the piece you're deferring) + + Current state, so it's recoverable later: + - Cleanup.isUnavailable (Cleanup.java:229-233) — any(LOG_UNAVAILABLE) or all(LOG_INCOMPLETE) && saveStatus < Stable, with your TODO (required): how does this interact with lost ownership?. + - SafeCommandStore.registerTransitiveDeps (~:565) — per-range subtraction via redundantBefore.removeLogUnavailableOrIncomplete(txnId, tmp), with TODO (required): if we only part-filter we're still going to have problems, as we won't be able to + update the transaction. + - RedundantBefore.Bounds.depBound — folds LOG_INCOMPLETE/LOG_UNAVAILABLE into the dep bound when UNREADY is ahead of the applied bound; this is what keeps deps sound after a restart, when the in-memory refuses map is gone. + - MapReduceCommandStores.Refuse{NONE,DEPS,ALL} + per-message abort() — the in-memory, coarse (whole-scope) refusal, only alive between unsafeRefuseRequests and unsafeAcceptRequests. + + These four must agree on one predicate. Concretely the questions to settle: + 1. Is any ever right for LOG_UNAVAILABLE? The dangerous arm isn't FULL (throwing is merely unavailability) but non-FULL, where logUnavailable(...) returns ERASE — a compaction decision that discards the record for ranges whose log is intact. + If nothing else changes, splitting just that arm (throw on any, erase only on all) is strictly safer and is a one-line intermediate. + 2. Lost ownership / retired ranges break all. Ranges that are WAS_OWNED or locally retired never carry LOG_INCOMPLETE, so all(...) is false and we don't refuse even though the owned remainder is incomplete. Your "treat pre-log-point ranges as + locally-retired-like" idea is exactly the missing normalisation, and note the patch already does the dual of it in SafeCommandStore.refuses(participants) (removeLocallyRetired(participants) before folding). A shared helper — "the + participants we must have a complete log for" = participants − retired − wasOwned − logIncomplete/Unavailable — would let Cleanup, refuses() and registerTransitive share one definition instead of three. + 3. Recovery cannot answer partially. BeginRecovery refuses on refuses.max != NONE, which only covers the in-memory window, not the post-restart window. There is no wire representation for "valid only for a sub-range" — but ReadOk.unavailable + already is exactly that for reads, so a RecoverNack{unavailable} (and the analogous field on the deps replies) is the smallest precedent-following addition; it would also let Commit/Apply stop depending on the coarse Refuse.MinMax. + 4. Future distinction you mentioned: if you do split further, the axis that matters to consumers looks like "records absent below the bound" (incomplete) vs "records possibly wrong below the bound" (corrupted) vs "records absent but decisions + recoverable from peers". Worth writing that down in BootstrapReason/RedundantStatus.Property javadoc before adding a third status, since the persisted bit space is nearly full (see N9). + + ### D2. Empty / short DurabilityResults + + Now benign-ish but still wrong: DurabilityService.submit's canUseDurableBefore && isAlreadyMet shortcut calls reportSuccess() while success == DurabilityResults.EMPTY, so markBootstrapping marks nothing and the attempt completes with missing + == valid → onFailedBootstrap → retry. Since each retry re-derives a fresh maxConflict and can be satisfied by durableBefore again, an idle range can retry indefinitely, with the unhelpful "Unknown failure" from complete()'s fail runnable. + Options: (a) populate DurabilityResults.of(ranges, request.min, ) on the shortcut path; (b) make the shortcut ineligible when the requester needs per-range bounds/readable (a flag on the request); (c) fail prepareToBootstrap explicitly + if byTxnId() doesn't cover the requested ranges, so the diagnostic names the cause. Also sync(...)/close(...) still return success(null) for empty ranges — an empty DurabilityResults would be safer for every caller that dereferences the value. + Worth adding an Invariants.require(success.byTxnId() covers commitRanges) at the markBootstrapping site so this class of shortfall is loud rather than a retry loop. +