diff --git a/.gitmodules b/.gitmodules index 616dacf610..c00814bf07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "modules/accord"] path = modules/accord - url = https://github.com/apache/cassandra-accord.git - branch = trunk + url = https://github.com/belliottsmith/cassandra-accord.git + branch = rerebootstrap diff --git a/UNFINISHED.md b/UNFINISHED.md new file mode 100644 index 0000000000..34fb6438c9 --- /dev/null +++ b/UNFINISHED.md @@ -0,0 +1,28 @@ +### D1. Partial log unavailability (the piece you're deferring) + + Current state, so it's recoverable later: + - Cleanup.isUnavailable (Cleanup.java:229-233) — any(LOG_UNAVAILABLE) or all(LOG_INCOMPLETE) && saveStatus < Stable, with your TODO (required): how does this interact with lost ownership?. + - SafeCommandStore.registerTransitiveDeps (~:565) — per-range subtraction via redundantBefore.removeLogUnavailableOrIncomplete(txnId, tmp), with TODO (required): if we only part-filter we're still going to have problems, as we won't be able to + update the transaction. + - RedundantBefore.Bounds.depBound — folds LOG_INCOMPLETE/LOG_UNAVAILABLE into the dep bound when UNREADY is ahead of the applied bound; this is what keeps deps sound after a restart, when the in-memory refuses map is gone. + - MapReduceCommandStores.Refuse{NONE,DEPS,ALL} + per-message abort() — the in-memory, coarse (whole-scope) refusal, only alive between unsafeRefuseRequests and unsafeAcceptRequests. + + These four must agree on one predicate. Concretely the questions to settle: + 1. Is any ever right for LOG_UNAVAILABLE? The dangerous arm isn't FULL (throwing is merely unavailability) but non-FULL, where logUnavailable(...) returns ERASE — a compaction decision that discards the record for ranges whose log is intact. + If nothing else changes, splitting just that arm (throw on any, erase only on all) is strictly safer and is a one-line intermediate. + 2. Lost ownership / retired ranges break all. Ranges that are WAS_OWNED or locally retired never carry LOG_INCOMPLETE, so all(...) is false and we don't refuse even though the owned remainder is incomplete. Your "treat pre-log-point ranges as + locally-retired-like" idea is exactly the missing normalisation, and note the patch already does the dual of it in SafeCommandStore.refuses(participants) (removeLocallyRetired(participants) before folding). A shared helper — "the + participants we must have a complete log for" = participants − retired − wasOwned − logIncomplete/Unavailable — would let Cleanup, refuses() and registerTransitive share one definition instead of three. + 3. Recovery cannot answer partially. BeginRecovery refuses on refuses.max != NONE, which only covers the in-memory window, not the post-restart window. There is no wire representation for "valid only for a sub-range" — but ReadOk.unavailable + already is exactly that for reads, so a RecoverNack{unavailable} (and the analogous field on the deps replies) is the smallest precedent-following addition; it would also let Commit/Apply stop depending on the coarse Refuse.MinMax. + 4. Future distinction you mentioned: if you do split further, the axis that matters to consumers looks like "records absent below the bound" (incomplete) vs "records possibly wrong below the bound" (corrupted) vs "records absent but decisions + recoverable from peers". Worth writing that down in BootstrapReason/RedundantStatus.Property javadoc before adding a third status, since the persisted bit space is nearly full (see N9). + + ### D2. Empty / short DurabilityResults + + Now benign-ish but still wrong: DurabilityService.submit's canUseDurableBefore && isAlreadyMet shortcut calls reportSuccess() while success == DurabilityResults.EMPTY, so markBootstrapping marks nothing and the attempt completes with missing + == valid → onFailedBootstrap → retry. Since each retry re-derives a fresh maxConflict and can be satisfied by durableBefore again, an idle range can retry indefinitely, with the unhelpful "Unknown failure" from complete()'s fail runnable. + Options: (a) populate DurabilityResults.of(ranges, request.min, ) on the shortcut path; (b) make the shortcut ineligible when the requester needs per-range bounds/readable (a flag on the request); (c) fail prepareToBootstrap explicitly + if byTxnId() doesn't cover the requested ranges, so the diagnostic names the cause. Also sync(...)/close(...) still return success(null) for empty ranges — an empty DurabilityResults would be safer for every caller that dereferences the value. + Worth adding an Invariants.require(success.byTxnId() covers commitRanges) at the markBootstrapping site so this class of shortfall is loud rather than a retry loop. + diff --git a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc index 4acb2395d6..ebe8631b77 100644 --- a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc +++ b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc @@ -87,7 +87,7 @@ finally { } .... -In other words, `CommandStore` collects the `PreLoadContext`, state +In other words, `CommandStore` collects the `ExecutionContext`, state required to be in memory for command execution (possible dependencies, such as `TxnId`s, and `Key`s of commands, but also `CommandsForKeys` that will be needed during execution). Once the context is collected and @@ -103,11 +103,11 @@ to ensure transactional integrity, changes to commands are tracked and are recorded into `Journal` for crash-recovery. `ProgressLog` and `CommandsForKey` are up -On Cassandra side, concurrent execution is controlled by `AccordTask`, +On Cassandra side, concurrent execution is controlled by `SafeTask`, which contains cache loading logic and persistence callbacks. Since Accord may potentially hold a large number of command states in memory, their states may be _shrunk_ to their binary representation to save some -memory, or they can get fully evicted. This also means that `AccordTask` +memory, or they can get fully evicted. This also means that `SafeTask` will have to reload relevant dependencies from preload context before command execution can begin. diff --git a/modules/accord b/modules/accord index fff32de2e9..f4c8ae02f9 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit fff32de2e915772fbc70d16b1c32346313877838 +Subproject commit f4c8ae02f9393c5eb14720b5074bd45135ee5b00 diff --git a/src/java/org/apache/cassandra/concurrent/CassandraThread.java b/src/java/org/apache/cassandra/concurrent/CassandraThread.java index 62cbcdb6a4..f2702ed7c0 100644 --- a/src/java/org/apache/cassandra/concurrent/CassandraThread.java +++ b/src/java/org/apache/cassandra/concurrent/CassandraThread.java @@ -18,14 +18,24 @@ package org.apache.cassandra.concurrent; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + import org.apache.cassandra.metrics.ThreadLocalMetrics; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.Task; +import org.apache.cassandra.service.accord.execution.TaskRunner; import io.netty.util.concurrent.FastThreadLocalThread; -public class CassandraThread extends FastThreadLocalThread +public class CassandraThread extends FastThreadLocalThread implements TaskRunner { private ThreadLocalMetrics threadLocalMetrics; private ExecutorLocals executorLocals; + private AccordExecutor accordActiveExecutor; + private AccordExecutor accordLockedExecutor; + private int accordLockedExecutorDepth; + private volatile Task accordActiveTask; + private static final AtomicReferenceFieldUpdater accordActiveTaskUpdater = AtomicReferenceFieldUpdater.newUpdater(CassandraThread.class, Task.class, "accordActiveTask"); private final ImmediateTaskHolder immediateTaskHolder; @@ -89,6 +99,55 @@ public class CassandraThread extends FastThreadLocalThread return current != null ? current : ExecutorLocals.none(); } + public final AccordExecutor accordActiveExecutor() + { + return accordActiveExecutor; + } + + public final void setAccordActiveExecutor(AccordExecutor newExecutor) + { + accordActiveExecutor = newExecutor; + } + + @Override + public final AccordExecutor accordLockedExecutor() + { + return accordLockedExecutor; + } + + @Override + public final boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) + { + if (accordLockedExecutor == null) accordLockedExecutor = newLockedExecutor; + else if (accordLockedExecutor != newLockedExecutor) return false; + ++accordLockedExecutorDepth; + return true; + } + + @Override + public final void exitAccordLockedExecutor() + { + if (--accordLockedExecutorDepth == 0) + accordLockedExecutor = null; + } + + public final Task accordActiveTask() + { + return accordActiveTask; + } + + // to be called only by the thread itself, so can (eventually) avoid any memory barriers + public final Task accordActiveSelfTask() + { + // TODO (expected): with newer JDK use accordActiveTaskUpdater.getPlain + return accordActiveTask; + } + + public final void setAccordActiveTask(Task newActiveTask) + { + accordActiveTaskUpdater.lazySet(this, newActiveTask); + } + // final to avoid skipping of the cleanup logic in child classes final public void run() { diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java index 557c1ca0a4..4e99729e0f 100644 --- a/src/java/org/apache/cassandra/concurrent/Stage.java +++ b/src/java/org/apache/cassandra/concurrent/Stage.java @@ -47,7 +47,7 @@ public enum Stage MUTATION (true, "MutationStage", "request", DatabaseDescriptor::getConcurrentWriters, DatabaseDescriptor::setConcurrentWriters, Stage::multiThreadedLowSignalStage), COUNTER_MUTATION (true, "CounterMutationStage", "request", DatabaseDescriptor::getConcurrentCounterWriters, DatabaseDescriptor::setConcurrentCounterWriters, Stage::multiThreadedLowSignalStage), VIEW_MUTATION (true, "ViewMutationStage", "request", DatabaseDescriptor::getConcurrentViewWriters, DatabaseDescriptor::setConcurrentViewWriters, Stage::multiThreadedLowSignalStage), - ACCORD_MIGRATION (false, "AccordMigrationStage", "request", DatabaseDescriptor::getAccordConcurrentOps, DatabaseDescriptor::setConcurrentAccordOps, Stage::multiThreadedLowSignalStage), + ACCORD_MIGRATION (false, "AccordMigrationStage", "request", DatabaseDescriptor::getAccordConcurrentMigrationOps, DatabaseDescriptor::setConcurrentAccordMigrationOps, Stage::multiThreadedLowSignalStage), GOSSIP (true, "GossipStage", "internal", () -> 1, null, Stage::singleThreadedStage), REQUEST_RESPONSE (false, "RequestResponseStage", "request", FBUtilities::getAvailableProcessors, null, Stage::multiThreadedLowSignalStage), ANTI_ENTROPY (false, "AntiEntropyStage", "internal", () -> 1, null, Stage::singleThreadedStage), diff --git a/src/java/org/apache/cassandra/config/AccordConfig.java b/src/java/org/apache/cassandra/config/AccordConfig.java index b9fe93b9a4..944b8c9d92 100644 --- a/src/java/org/apache/cassandra/config/AccordConfig.java +++ b/src/java/org/apache/cassandra/config/AccordConfig.java @@ -27,6 +27,7 @@ import accord.api.ProtocolModifiers.CoordinatorBacklogExecution; import accord.api.ProtocolModifiers.FastExecution; import accord.api.ProtocolModifiers.ReplicaExecution; import accord.api.ProtocolModifiers.SendStableMessages; +import accord.api.ProtocolModifiers.UniqueTimestampOnConflict; import accord.primitives.TxnId; import accord.utils.Invariants; @@ -34,8 +35,6 @@ import org.apache.cassandra.journal.Params; import org.apache.cassandra.service.accord.serializers.Version; import org.apache.cassandra.service.consensus.TransactionalMode; -import static org.apache.cassandra.config.AccordConfig.CatchupMode.NORMAL; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.HLC_FIFO; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_POOL_PER_SHARD; import static org.apache.cassandra.config.AccordConfig.QueueSubmissionModel.SYNC; import static org.apache.cassandra.config.AccordConfig.RangeIndexMode.in_memory; @@ -134,25 +133,51 @@ public class AccordConfig FIFO, /** - * If the work has an associated TxnId, prioritise by its HLC (and FIFO otherwise) + * If the work has an associated TxnId of Ballot, prioritise by the newest HLC (and FIFO otherwise) */ HLC_FIFO, /** - * Prioritise Apply, Stable, Commit, Accept, and PreAccept messages in that order. - * Within a given message type, prioritise by HLC. - * Other messages will be mixed with PreAccept messages, but using the next counter rather than the HLC of the TxnId. - * Note: this can have some performance edge cases for contended keys, as we may process Stable messages for later commands before - * we process earlier Accept/Commit, which may delay execution + * If the work has an associated TxnId, prioritise by its HLC (and FIFO otherwise) */ - PHASE_HLC_FIFO, + ORIG_HLC_FIFO + } + + public enum QueueBalancingModel + { + /** + * Always pick the task by priority + */ + PRIORITY_ONLY, /** - * Prioritise Apply, Stable, Commit, Accept, and PreAccept messages from the original coordinator only, in that order. - * Within a given message type, prioritise by HLC. - * Other messages will be mixed with PreAccept messages, but using the next counter rather than the HLC of the TxnId. + * Pick by phase first, so the highest phase with work ALWAYS runs. + * Within a phase, pick by priority. */ - ORIG_PHASE_HLC_FIFO + PHASE_ONLY, + + /** + * Pick by phase first, selecting the phase that has processed the least work recently relative to arrivals. + * Within a phase, pick by priority. + */ + PHASE_FAIR, + + /** + * While phases are within a threshold of imbalance, pick tasks by priority. + * Once the threshold is crossed, over-processed phases have a small penalty applied + * and work is prioritised by phase with the phase with the least recent work processed picked first, + * until the imbalance is resolved. + * + * Within a phase, pick by priority. + */ + BLENDED_PRIORITY_PHASE_FAIR, + } + + public enum UniqueTimestampReservations + { + NONE, + SMALL_SHARED, + HISTOGRAM } public QueueShardModel queue_shard_model = THREAD_POOL_PER_SHARD; @@ -168,7 +193,38 @@ public class AccordConfig */ public volatile OptionaldPositiveInt queue_thread_count = OptionaldPositiveInt.UNDEFINED; - public QueuePriorityModel queue_priority_model = HLC_FIFO; + public QueuePriorityModel queue_priority_model; + public QueueBalancingModel queue_balancing_model; + + public Integer queue_flow_imbalance_onset = null; + public Integer queue_flow_imbalance_width_shift = null; + + public String queue_active_limits; + + public Boolean queue_nonsync_enabled; + + /** + * Size at which we will begin processing a task that is ASYNC, INCR OR INCR_ATOMIC. + * Note that a size of zero will effectively give implicit priority to INCR_ATOMIC tasks, as they may immediately + * take a FIFO queue slot (which is processed preferentially). + */ + public Integer queue_nonsync_min_batch_size; + + /** + * If there are more than min_batch_size keys ready for an ASYNC, INCR or INCR_ATOMIC task, + * process up to this many keys at once. + */ + public Integer queue_nonsync_max_batch_size; + + /** + * An ASYNC, INCR or INCR_ATOMIC task that is ready to run but waiting for batch_size work will proceed + * once this number of tasks are blocked behind it, regardless of batch_size. + */ + public Integer queue_nonsync_blocked_limit; + /** + * The number of threads that may be used to execute distributed requests for migration tasks + */ + public volatile OptionaldPositiveInt migration_concurrency = OptionaldPositiveInt.UNDEFINED; /** * If set, the signal loop does not match park/unpark pairs, but instead consumers perform timed-park spin waits @@ -181,15 +237,6 @@ public class AccordConfig */ public DurationSpec.LongMicrosecondsBound queue_stop_check_interval; - /** - * If set, the signal loop reduces the number of threads it is using when the time spent parked exceeds real-time - * by this interval. - */ - public DurationSpec.LongMicrosecondsBound queue_signal_stop_check_interval_credit; - - // yield to other executor threads after executing this many tasks in a row, if there are waiting threads and tasks - public int queue_yield_interval = 100; - /** * If the HLC is older than this, queue by FIFO instead */ @@ -204,9 +251,6 @@ public class AccordConfig */ public volatile OptionaldPositiveInt command_store_shard_count = OptionaldPositiveInt.UNDEFINED; - public volatile OptionaldPositiveInt max_queued_loads = OptionaldPositiveInt.UNDEFINED; - public volatile OptionaldPositiveInt max_queued_range_loads = OptionaldPositiveInt.UNDEFINED; - public volatile OptionaldPositiveInt progress_log_concurrency = OptionaldPositiveInt.UNDEFINED; public DurationSpec.IntMillisecondsBound progress_log_query_fallback_timeout = new DurationSpec.IntMillisecondsBound("1m"); @@ -225,6 +269,7 @@ public class AccordConfig public String expire_epoch_wait = "10s"; // we don't want to wait ages for durability as it blocks other durability progress; even this might be too long, as we can always retry public String expire_durability = "10s*attempts <= 30s"; + public String slow_durability = "10s"; public String slow_syncpoint_preaccept = "10s"; public String slow_txn_preaccept = "30ms <= p50*2 <= 1000ms"; public String slow_read = "30ms <= p50*2 <= 1000ms"; @@ -261,12 +306,12 @@ public class AccordConfig */ public volatile TransactionalRangeMigration range_migration = TransactionalRangeMigration.auto; - public enum CatchupMode + public enum CatchupFallbackMode { - DISABLED, - NORMAL, - FALLBACK_TO_HARD, - HARD + IGNORE, + EXIT, + REBOOTSTRAP, + REBOOTSTRAP_AND_CATCHUP } /** @@ -296,6 +341,9 @@ public class AccordConfig public Boolean send_minimal; // note: simulator incompatible (for now) public Boolean precise_micros; + public UniqueTimestampReservations unique_timestamp_reservations; + public UniqueTimestampOnConflict unique_timestamp_on_conflict; + public DurationSpec.IntMillisecondsBound unique_timestamp_reservation_range; public boolean ephemeral_reads = true; public boolean state_cache_listener_jfr_enabled = false; @@ -311,16 +359,19 @@ public class AccordConfig public int commands_for_key_prune_interval = 64; public DurationSpec.IntSecondsBound max_conflicts_prune_delta = new DurationSpec.IntSecondsBound(1); + public int catchup_on_start_max_attempts = 5; + public boolean catchup_on_start = true; public DurationSpec.IntSecondsBound catchup_on_start_success_latency = new DurationSpec.IntSecondsBound(60); public DurationSpec.IntSecondsBound catchup_on_start_fail_latency = new DurationSpec.IntSecondsBound(900); - public int catchup_on_start_max_attempts = 5; - // TODO (required): roll this back to catchup_on_start_exit_on_failure: true - public boolean catchup_on_start_exit_on_failure = false; - public CatchupMode catchup_on_start = NORMAL; + // TODO (required): default this to EXIT or REBOOTSTRAP + public CatchupFallbackMode catchup_on_start_on_timeout = CatchupFallbackMode.IGNORE; + public CatchupFallbackMode catchup_on_start_on_error = CatchupFallbackMode.IGNORE; + public CatchupFallbackMode catchup_on_start_on_rebootstrap_fallback = CatchupFallbackMode.IGNORE; + public DurationSpec.IntSecondsBound shutdown_grace_period = new DurationSpec.IntSecondsBound(15 * 60); + public boolean execute_waiting_on_start = true; public DurationSpec.IntSecondsBound execute_waiting_on_start_timeout = new DurationSpec.IntSecondsBound(0); public boolean execute_waiting_on_start_fail_on_timeout = false; - public DurationSpec.IntSecondsBound shutdown_grace_period = new DurationSpec.IntSecondsBound(15 * 60); public enum RangeIndexMode { in_memory, journal_sai } public RangeIndexMode range_index_mode = in_memory; @@ -358,7 +409,17 @@ public class AccordConfig * Replay journal entries for commands that are not durable to the data or command stores. * THIS MODE IS NOT YET SAFE TO RUN */ - NON_DURABLE + NON_DURABLE, + + /** + * Don't replay, simply rebootstrap, marking our log as incomplete. + */ + REBOOTSTRAP_INCOMPLETE, + + /** + * Don't replay, simply rebootstrap, marking our log as corrupted/unavailable. + */ + REBOOTSTRAP_RESET } public enum ReplaySavePoint diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 5bc6de7ac1..765ee10d45 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -2967,6 +2967,20 @@ public class DatabaseDescriptor conf.accord.queue_thread_count = new OptionaldPositiveInt(concurrent_operations); } + public static int getAccordConcurrentMigrationOps() + { + return conf.accord.migration_concurrency.or(2 * FBUtilities.getAvailableProcessors()); + } + + public static void setConcurrentAccordMigrationOps(int concurrent_operations) + { + if (concurrent_operations < 0) + { + throw new IllegalArgumentException("Concurrent accord operations must be non-negative"); + } + conf.accord.migration_concurrency = new OptionaldPositiveInt(concurrent_operations); + } + public static int getFlushWriters() { return conf.memtable_flush_writers; diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 8c63b5767e..9b149de9f2 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -62,7 +62,8 @@ import accord.impl.CommandChange; import accord.impl.progresslog.DefaultProgressLog; import accord.impl.progresslog.DefaultProgressLog.ModeFlag; import accord.impl.progresslog.TxnStateKind; -import accord.local.CatchupHard; +import accord.local.BootstrapReason; +import accord.local.Catchup; import accord.local.Cleanup; import accord.local.Command; import accord.local.CommandStore; @@ -71,9 +72,9 @@ import accord.local.CommandStores.LatentStoreSelector; import accord.local.Commands; import accord.local.Commands.NotifyWaitingOnPlus; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.MaxConflicts; import accord.local.Node; -import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; @@ -132,11 +133,8 @@ import org.apache.cassandra.schema.Indexes; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.accord.AccordCache; -import org.apache.cassandra.service.accord.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStores; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.AccordKeyspace; import org.apache.cassandra.service.accord.AccordOperations; import org.apache.cassandra.service.accord.AccordService; @@ -154,6 +152,10 @@ import org.apache.cassandra.service.accord.debug.DebugTxnDepsAll; import org.apache.cassandra.service.accord.debug.DebugTxnDepsOrdered; import org.apache.cassandra.service.accord.debug.DebugTxnGraph; import org.apache.cassandra.service.accord.debug.TxnKindsAndDomains; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.TaskInfo; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.service.consensus.migration.TableMigrationState; @@ -165,6 +167,8 @@ import org.apache.cassandra.utils.concurrent.Future; import static accord.coordinate.Infer.InvalidIf.NotKnownToBeInvalid; import static accord.impl.CommandChange.Field.ACCEPTED; import static accord.impl.CommandChange.Field.PROMISED; +import static accord.local.BootstrapReason.LOG_CORRUPTED; +import static accord.local.BootstrapReason.LOG_INCOMPLETE; import static accord.local.RedundantStatus.Property.GC_BEFORE; import static accord.local.RedundantStatus.Property.LOCALLY_APPLIED; import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_COMMAND_STORE; @@ -182,6 +186,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; import static org.apache.cassandra.db.virtual.AbstractLazyVirtualTable.OnTimeout.BEST_EFFORT; import static org.apache.cassandra.db.virtual.AbstractLazyVirtualTable.OnTimeout.FAIL; +import static org.apache.cassandra.db.virtual.AccordDebugKeyspace.CommandStoreOpsTable.CommandStoreOp.REBOOTSTRAP_CORRUPTED; import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.ASC; import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.SORTED; import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.UNSORTED; @@ -331,22 +336,22 @@ public class AccordDebugKeyspace extends VirtualKeyspace int executorId = executor.executorId(); collector.partition(executorId).collect(rows -> { int uniquePos = 0; - AccordExecutor.TaskInfo prev = null; - for (AccordExecutor.TaskInfo info : executor.taskSnapshot()) + TaskInfo prev = null; + for (TaskInfo info : executor.taskSnapshot()) { if (prev != null && info.status() == prev.status() && info.position() == prev.position()) ++uniquePos; else uniquePos = 0; prev = info; - PreLoadContext preLoadContext = info.preLoadContext(); + ExecutionContext executionContext = info.preLoadContext(); rows.add(info.status().name(), info.position(), uniquePos) .lazyCollect(columns -> { columns.add("description", info.describe()) .add("command_store_id", info.commandStoreId()) - .add("txn_id", preLoadContext, PreLoadContext::primaryTxnId, TO_STRING) - .add("txn_id_additional", preLoadContext, PreLoadContext::additionalTxnId, TO_STRING) - .add("keys", preLoadContext, PreLoadContext::keys, TO_STRING) - .add("keys_loading", preLoadContext, PreLoadContext::loadKeys, TO_STRING) - .add("keys_loading_for", preLoadContext, PreLoadContext::loadKeysFor, TO_STRING); + .add("txn_id", executionContext, ExecutionContext::primaryTxnId, TO_STRING) + .add("txn_id_additional", executionContext, ExecutionContext::additionalTxnId, TO_STRING) + .add("keys", executionContext, ExecutionContext::keys, TO_STRING) + .add("keys_loading", executionContext, ExecutionContext::loadKeys, TO_STRING) + .add("keys_loading_for", executionContext, ExecutionContext::loadKeysFor, TO_STRING); }); } }); @@ -740,7 +745,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace collector.partition(commandStore.id()) .collect(rows -> { // TODO (desired): support maybe execute immediately with safeStore - Future future = toFuture(commandStore.chain((PreLoadContext.Empty) metadata::toString, safeStore -> { addRows(safeStore, rows); })); + Future future = toFuture(commandStore.chain((ExecutionContext.Empty) metadata::toString, safeStore -> { addRows(safeStore, rows); })); if (!future.awaitUntilThrowUncheckedOnInterrupt(collector.deadlineNanos())) throw new InternalTimeoutException(); }); @@ -1650,7 +1655,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace { try (AccordCommandStore.ExclusiveCaches caches = commandStore.lockCaches()) { - AccordCacheEntry entry = caches.commands().getUnsafe(txnId); + AccordCacheEntry entry = caches.commands().getUnsafe(txnId); return entry == null ? null : entry.getExclusive(); } } @@ -1794,7 +1799,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace SafeCommand safeCommand = safeStore.unsafeGet(txnId); Command command = safeCommand.current(); if (command.saveStatus() == SaveStatus.Applying) - return Commands.applyChain(safeStore, command); + return Commands.applyChain(safeStore, command.asExecuted()); Commands.maybeExecute(safeStore, safeCommand, command, true, true, NotifyWaitingOnPlus.adapter(ignore -> {}, true, true)); return AsyncChains.success(null); }); @@ -1875,7 +1880,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace Node node = AccordService.unsafeInstance().node(); if (Route.isFullRoute(route)) { - PrepareRecovery.recover(node, node.someSequentialExecutor(), txnId, NotKnownToBeInvalid, (FullRoute) route, null, LatentStoreSelector.standard(), (success, fail) -> { + PrepareRecovery.recover(node, node.someExclusiveExecutor(), txnId, NotKnownToBeInvalid, (FullRoute) route, null, LatentStoreSelector.standard(), (success, fail) -> { if (fail != null) result.setFailure(fail); else result.setSuccess(null); }); @@ -1895,7 +1900,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace AccordService.getBlocking(accord.node() .commandStores() .forId(commandStoreId) - .chain(PreLoadContext.contextFor(txnId, TXN_OPS), apply) + .chain(ExecutionContext.unsequenced(txnId, TXN_OPS), apply) .flatMap(i -> i)); } @@ -1987,8 +1992,10 @@ public class AccordDebugKeyspace extends VirtualKeyspace UNSET_PROGRESS_LOG_MODE("Unset the specified progress log mode."), TRY_EXECUTE_LISTENING("Try to execute all of the transactions (and their dependencies) that have registered listeners on other transactions."), REPLAY("Run journal replay for all transactions"), - REBOOTSTRAP("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."), - HARD_CATCHUP("Hard catchup the command store. This invalidates the local journal for any ranges not up to date with some quorum, synchronises its data via data repair and rejoins the distributed state machine."), + // TODO (expected): specify ranges + REBOOTSTRAP_CORRUPTED("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."), + REBOOTSTRAP_INCOMPLETE("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."), + REBOOTSTRAP_IF_BEHIND("Hard catchup the command store. This invalidates the local journal for any ranges not up to date with some quorum, synchronises its data via data repair and rejoins the distributed state machine."), ; final String description; @@ -2081,13 +2088,15 @@ public class AccordDebugKeyspace extends VirtualKeyspace }; break; } - case REBOOTSTRAP: - allFunction = () -> node.commandStores().rebootstrap(node); - function = commandStore -> commandStore.rebootstrap(node); + case REBOOTSTRAP_CORRUPTED: + case REBOOTSTRAP_INCOMPLETE: + BootstrapReason reason = op == REBOOTSTRAP_CORRUPTED ? LOG_CORRUPTED : LOG_INCOMPLETE; + allFunction = () -> node.commandStores().rebootstrap(node, reason); + function = commandStore -> commandStore.rebootstrap(node, reason); break; - case HARD_CATCHUP: - allFunction = () -> CatchupHard.catchup(node, Arrays.asList(node.commandStores().all())).beginAsResult(); - function = commandStore -> CatchupHard.catchup(node, Collections.singletonList(commandStore)).beginAsResult(); + case REBOOTSTRAP_IF_BEHIND: + allFunction = () -> Catchup.rebootstrapIfBehind(node, Arrays.asList(node.commandStores().all())).beginAsResult(); + function = commandStore -> Catchup.rebootstrapIfBehind(node, Collections.singletonList(commandStore)).beginAsResult(); break; } @@ -2483,7 +2492,6 @@ public class AccordDebugKeyspace extends VirtualKeyspace { throw new InvalidRequestException("Unknown bucket_mode '" + value + '\''); } - } private static int checkNonNegative(Object value, String field, int ifNull) diff --git a/src/java/org/apache/cassandra/db/virtual/QueriesTable.java b/src/java/org/apache/cassandra/db/virtual/QueriesTable.java index e58e874c14..673d7335ad 100644 --- a/src/java/org/apache/cassandra/db/virtual/QueriesTable.java +++ b/src/java/org/apache/cassandra/db/virtual/QueriesTable.java @@ -25,8 +25,8 @@ import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import static java.lang.Long.max; import static java.util.concurrent.TimeUnit.NANOSECONDS; diff --git a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java index 834abca49d..63c70a5447 100644 --- a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java +++ b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java @@ -44,6 +44,9 @@ import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; */ public class ExceptionSerializer { + // allow plenty of room for UTF8 encoding + private static final int MAX_MESSAGE_CHAR_LENGTH = Short.MAX_VALUE / 4; + public static class RemoteException extends RuntimeException { public final String originalClass; @@ -72,10 +75,16 @@ public class ExceptionSerializer static String getMessageWithOriginatingHost(Throwable t, boolean isFirstException) { - if (isFirstException) - return "Remote exception from host " + FBUtilities.getBroadcastAddressAndPort().toString() + (t.getLocalizedMessage() != null ? " - " + t.getLocalizedMessage() : ""); - else - return t.getLocalizedMessage(); + String message; + if (isFirstException) message = "Remote exception from host " + FBUtilities.getBroadcastAddressAndPort().toString() + (t.getLocalizedMessage() != null ? " - " + t.getLocalizedMessage() : ""); + else message = t.getLocalizedMessage(); + + if (message == null) + return message; + + if (message.length() > MAX_MESSAGE_CHAR_LENGTH) + message = message.substring(0, MAX_MESSAGE_CHAR_LENGTH); + return message; } private static final IVersionedSerializer stackTraceElementSerializer = new IVersionedSerializer() diff --git a/src/java/org/apache/cassandra/gms/FailureDetector.java b/src/java/org/apache/cassandra/gms/FailureDetector.java index 13d6266ca0..eaa73d09cc 100644 --- a/src/java/org/apache/cassandra/gms/FailureDetector.java +++ b/src/java/org/apache/cassandra/gms/FailureDetector.java @@ -377,6 +377,10 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean logger.debug("Still not marking nodes down due to local pause"); return; } + + if (!isAlive(ep)) + return; // don't convict nodes that are already down - this helps Accord on startup which doesn't report itself alive in Gossip until ready to serve traffic + double phi = hbWnd.phi(now); logger.trace("PHI for {} : {}", ep, phi); diff --git a/src/java/org/apache/cassandra/io/util/PathUtils.java b/src/java/org/apache/cassandra/io/util/PathUtils.java index 334763f271..780be3faaf 100644 --- a/src/java/org/apache/cassandra/io/util/PathUtils.java +++ b/src/java/org/apache/cassandra/io/util/PathUtils.java @@ -662,7 +662,6 @@ public final class PathUtils DeleteOnExit.clearOnExitThreads(); } - private static final class DeleteOnExit implements Runnable { private boolean isRegistered; diff --git a/src/java/org/apache/cassandra/journal/Compactor.java b/src/java/org/apache/cassandra/journal/Compactor.java index ffc48ff5f2..428866098c 100644 --- a/src/java/org/apache/cassandra/journal/Compactor.java +++ b/src/java/org/apache/cassandra/journal/Compactor.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.journal; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -29,6 +28,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.concurrent.Shutdownable; +import org.apache.cassandra.journal.SegmentCompactor.Stopped; import org.apache.cassandra.utils.concurrent.WaitQueue; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; @@ -100,9 +100,9 @@ public final class Compactor implements Runnable, Shutdownable compacted.signalAll(); } - catch (IOException e) + catch (Stopped e) { - throw new RuntimeException("Could not compact segments: " + toCompact); + logger.info("Cancelling compaction of {}", toCompact); } } @@ -118,6 +118,7 @@ public final class Compactor implements Runnable, Shutdownable logger.debug("Shutting down " + executor); if (scheduled != null) scheduled.cancel(false); + segmentCompactor.stop(); executor.shutdown(); } diff --git a/src/java/org/apache/cassandra/journal/SegmentCompactor.java b/src/java/org/apache/cassandra/journal/SegmentCompactor.java index 7b84ea82e1..01897a52c1 100644 --- a/src/java/org/apache/cassandra/journal/SegmentCompactor.java +++ b/src/java/org/apache/cassandra/journal/SegmentCompactor.java @@ -17,18 +17,20 @@ */ package org.apache.cassandra.journal; -import java.io.IOException; import java.util.Collection; public interface SegmentCompactor { SegmentCompactor NOOP = (SegmentCompactor) (segments) -> segments; + class Stopped extends RuntimeException {} + static SegmentCompactor noop() { //noinspection unchecked return (SegmentCompactor) NOOP; } - Collection> compact(Collection> segments) throws IOException; + Collection> compact(Collection> segments) throws Stopped; + default void stop() {} } diff --git a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java index 59efb88044..a9c5c4c28e 100644 --- a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java @@ -24,8 +24,8 @@ import java.util.function.ToLongFunction; import com.codahale.metrics.Gauge; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import static org.apache.cassandra.metrics.AccordMetricUtils.fromAccordService; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; @@ -46,7 +46,7 @@ public class AccordCacheMetrics public AccordCacheGlobalMetrics() { DefaultNameFactory factory = new DefaultNameFactory(ACCORD_CACHE); - this.usedBytes = Metrics.gauge(factory.createMetricName("UsedBytes"), fromAccordService(sumExecutors(executor -> executor.cacheUnsafe().weightedSize()), 0L)); + this.usedBytes = Metrics.gauge(factory.createMetricName("UsedBytes"), fromAccordService(sumExecutors(AccordExecutor::weightedSize), 0L)); this.unreferencedBytes = Metrics.gauge(factory.createMetricName("UnreferencedBytes"), fromAccordService(sumExecutors(executor -> executor.cacheUnsafe().unreferencedBytes()), 0L)); } diff --git a/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java b/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java index 3d5132131a..2c48fe7d3d 100644 --- a/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java @@ -23,10 +23,10 @@ import java.util.concurrent.TimeUnit; import com.codahale.metrics.Gauge; import org.apache.cassandra.metrics.ShardedDecayingHistograms.ShardedDecayingHistogram; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; -import static org.apache.cassandra.service.accord.AccordExecutor.HISTOGRAMS; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.HISTOGRAMS; public class AccordExecutorMetrics { diff --git a/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java b/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java index 6bc46f89c7..556ffb574d 100644 --- a/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java @@ -33,11 +33,11 @@ import org.apache.cassandra.metrics.LogLinearDecayingHistograms.LogLinearDecayin import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistogramsShard; import org.apache.cassandra.metrics.ShardedDecayingHistograms.ShardedDecayingHistogram; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.tracing.Tracing; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; -import static org.apache.cassandra.service.accord.AccordExecutor.HISTOGRAMS; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.HISTOGRAMS; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; public class AccordReplicaMetrics @@ -168,7 +168,7 @@ public class AccordReplicaMetrics private static LogLinearDecayingHistograms.Buffer buffer(SafeCommandStore safeStore) { - return ((AccordSafeCommandStore) safeStore).histogramBuffer(); + return ((SaferCommandStore) safeStore).histogramBuffer(); } @Override diff --git a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java index 823346917a..46f5681450 100644 --- a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java +++ b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java @@ -59,6 +59,9 @@ public class LogLinearDecayingHistograms private void add(long histogramIndex, long value) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + Invariants.require(histogramIndex <= HISTOGRAM_INDEX_MASK); Invariants.require(value >= 0); if (value <= LARGE_VALUE) value <<= HISTOGRAM_INDEX_BITS; @@ -73,6 +76,11 @@ public class LogLinearDecayingHistograms buffer[bufferCount++] = value; } + public void clear() + { + bufferCount = 0; + } + public void flush(long at) { for (int i = 0 ; i < bufferCount ; ++i) @@ -112,6 +120,9 @@ public class LogLinearDecayingHistograms public void increment(long value, long at) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + int index = index(value); if (bufferCount >= buffer.length) { @@ -122,6 +133,7 @@ public class LogLinearDecayingHistograms updateDecay(at); long v = Double.doubleToRawLongBits(increment) & VALUE_MASK; v |= histogramIndex | ((long)index << HISTOGRAM_INDEX_BITS); + Invariants.require(Double.longBitsToDouble(v & VALUE_MASK) > 0); buffer[bufferCount++] = v; } diff --git a/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java b/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java index 10c727c7c4..bae5779e66 100644 --- a/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java +++ b/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java @@ -54,6 +54,9 @@ public class LogLinearHistogram public void increment(long value) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + int index = index(value); buckets(index)[index]++; ++totalCount; @@ -61,6 +64,9 @@ public class LogLinearHistogram public void decrement(long value) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + int index = index(value); buckets(index)[index]--; --totalCount; @@ -68,6 +74,9 @@ public class LogLinearHistogram public void replace(long decrement, long increment) { + if (!Invariants.expect(decrement >= 0 && increment >= 0, "Negative value(s?) reported: %s and %s", decrement, increment)) + return; + int decrementIndex = index(decrement); int incrementIndex = index(increment); long[] buckets = buckets(Math.max(decrementIndex, incrementIndex)); diff --git a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java b/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java deleted file mode 100644 index 77b56f3203..0000000000 --- a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java +++ /dev/null @@ -1,764 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service.accord; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.function.BiConsumer; - -import javax.annotation.Nullable; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.primitives.Ints; - -import accord.utils.ArrayBuffers.BufferList; -import accord.utils.IntrusiveLinkedList; -import accord.utils.IntrusiveLinkedListNode; -import accord.utils.Invariants; -import accord.utils.async.Cancellable; - -import org.apache.cassandra.service.accord.AccordCache.Adapter; -import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; -import org.apache.cassandra.utils.ObjectSizes; - -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_SAVE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADING; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.SAVING; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.WAITING_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.WAITING_TO_SAVE; - -/** - * Global (per CommandStore) state of a cached entity (Command or CommandsForKey). - */ -public class AccordCacheEntry extends IntrusiveLinkedListNode -{ - public enum Status - { - UNINITIALIZED, - - UNUSED1, // spacing to permit easier bit masks - - WAITING_TO_LOAD(UNINITIALIZED), - LOADING(WAITING_TO_LOAD), - - /** - * Consumers should never see this state - */ - FAILED_TO_LOAD(LOADING), - - UNUSED2, // spacing to permit easier bit masks - UNUSED3, // spacing to permit easier bit masks - UNUSED4, // spacing to permit easier bit masks - - LOADED(true, false, UNINITIALIZED, LOADING), - MODIFIED(true, false, LOADED), - - UNUSED5, // spacing to permit easier bit masks - UNUSED6, // spacing to permit easier bit masks - - WAITING_TO_SAVE(true, true, MODIFIED), - SAVING(true, true, MODIFIED, WAITING_TO_SAVE), - - /** - * Attempted to save but failed. Shouldn't normally happen unless we have a bug in serialization, - * or commit log has been stopped. - */ - FAILED_TO_SAVE(true, true, SAVING), - - UNUSED7, // spacing to permit easier bit masks - - EVICTED(WAITING_TO_LOAD, LOADING, LOADED, FAILED_TO_LOAD), - ; - - static final Status[] VALUES = values(); - static - { - MODIFIED.permittedFrom |= 1 << MODIFIED.ordinal(); - MODIFIED.permittedFrom |= 1 << SAVING.ordinal(); - MODIFIED.permittedFrom |= 1 << FAILED_TO_SAVE.ordinal(); - LOADED.permittedFrom |= 1 << SAVING.ordinal(); - LOADED.permittedFrom |= 1 << MODIFIED.ordinal(); - for (Status status : VALUES) - { - if (status.name().startsWith("UNUSED")) continue; - Invariants.require((status.ordinal() & IS_LOADED) != 0 == status.loaded); - Invariants.require(((status.ordinal() & IS_LOADED) != 0 && (status.ordinal() & IS_NESTED) != 0) == status.nested); - Invariants.require(((status.ordinal() & IS_LOADING_OR_WAITING_MASK) == IS_LOADING_OR_WAITING) == (status == LOADING || status == WAITING_TO_LOAD)); - Invariants.require(((status.ordinal() & IS_SAVING_OR_WAITING_MASK) == IS_SAVING_OR_WAITING) == (status == SAVING || status == WAITING_TO_SAVE)); - } - } - - final boolean loaded; - final boolean nested; - int permittedFrom; - - Status(Status ... statuses) - { - this(false, false, statuses); - } - - Status(boolean loaded, boolean nested, Status ... statuses) - { - this.loaded = loaded; - this.nested = nested; - for (Status status : statuses) - permittedFrom |= 1 << status.ordinal(); - } - } - - static final int STATUS_MASK = 0x0000001F; - static final int SHRUNK = 0x00000040; - static final int NO_EVICT = 0x00000020; - static final int IS_NOT_EVICTED = 0xF; - static final int IS_LOADED = 0x8; - static final int IS_NESTED = 0x4; - static final int IS_LOADING_OR_WAITING_MASK = 0x6; - static final int IS_LOADING_OR_WAITING = 0x2; - static final int IS_SAVING_OR_WAITING_MASK = 0xE; - static final int IS_SAVING_OR_WAITING = 0xC; - static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCacheEntry<>(null, null)); - - private final K key; - final AccordCache.Type.Instance owner; - - private Object state; - private int status; - int sizeOnHeap; - private volatile int references; - private static final AtomicIntegerFieldUpdater referencesUpdater = AtomicIntegerFieldUpdater.newUpdater(AccordCacheEntry.class, "references"); - - AccordCacheEntry(K key, AccordCache.Type.Instance owner) - { - this.key = key; - this.owner = owner; - } - - void unlink() - { - remove(); - } - - boolean isUnqueued() - { - return isFree(); - } - - public K key() - { - return key; - } - - public int references() - { - return references; - } - - public int increment() - { - return referencesUpdater.incrementAndGet(this); - } - - public int decrement() - { - return referencesUpdater.decrementAndGet(this); - } - - boolean isLoaded() - { - return (status & IS_LOADED) != 0; - } - - boolean isModified() - { - return (status & IS_NOT_EVICTED) >= MODIFIED.ordinal(); - } - - boolean isNested() - { - Invariants.require(isLoaded()); - return (status & IS_NESTED) != 0; - } - - boolean isShrunk() - { - return (status & SHRUNK) != 0; - } - - public boolean is(Status status) - { - return (this.status & STATUS_MASK) == status.ordinal(); - } - - boolean isLoadingOrWaiting() - { - return (status & IS_LOADING_OR_WAITING_MASK) == IS_LOADING_OR_WAITING; - } - - boolean isSavingOrWaiting() - { - return (status & IS_SAVING_OR_WAITING_MASK) == IS_SAVING_OR_WAITING; - } - - public boolean isComplete() - { - return !is(LOADING) && !is(SAVING); - } - - int noEvictGeneration() - { - Invariants.require(isNoEvict()); - return (status >>> 8) & 0xffff; - } - - int noEvictMaxAge() - { - Invariants.require(isNoEvict()); - return status >>> 24; - } - - boolean isNoEvict() - { - return (status & NO_EVICT) != 0; - } - - int sizeOnHeap() - { - return sizeOnHeap; - } - - void updateSize(AccordCache.Type parent) - { - // TODO (expected): we aren't weighing the keys - int newSizeOnHeap = Ints.saturatedCast(EMPTY_SIZE + estimateOnHeapSize(parent.adapter())); - parent.updateSize(newSizeOnHeap, newSizeOnHeap - sizeOnHeap, references == 0, true); - sizeOnHeap = newSizeOnHeap; - } - - void initSize(AccordCache.Type parent) - { - // TODO (expected): we aren't weighing the keys - sizeOnHeap = Ints.saturatedCast(EMPTY_SIZE); - parent.updateSize(sizeOnHeap, sizeOnHeap, false, false); - parent.objectSize.increment(EMPTY_SIZE); - } - - @Override - public String toString() - { - return "Node{" + status() + - ", key=" + key() + - ", references=" + references + - "}@" + Integer.toHexString(System.identityHashCode(this)); - } - - public Status status() - { - return Status.VALUES[(status & STATUS_MASK)]; - } - - private void setStatus(Status newStatus) - { - Invariants.require((newStatus.permittedFrom & (1 << (status & STATUS_MASK))) != 0, "%s not permitted from %s", newStatus, status()); - setStatusUnsafe(newStatus); - } - - private void setStatusUnsafe(Status newStatus) - { - status &= ~STATUS_MASK; - status |= newStatus.ordinal(); - } - - public void initialize(V value) - { - Invariants.require(state == null); - setStatus(LOADED); - state = value; - } - - public void readyToLoad() - { - Invariants.require(state == null); - setStatus(WAITING_TO_LOAD); - state = new WaitingToLoad(); - } - - public void markNoEvict(int generation, int maxAge) - { - Invariants.require((maxAge & ~0xff) == 0); - Invariants.require((generation & ~0xffff) == 0); - status |= NO_EVICT; - status |= generation << 8; - status |= maxAge << 24; - } - - public LoadingOrWaiting loadingOrWaiting() - { - return (LoadingOrWaiting)state; - } - - void notifyListeners(BiConsumer, AccordCacheEntry> notify) - { - owner.notifyListeners(notify, this); - } - - public interface LoadExecutor - { - Cancellable load(P1 p1, P2 p2, AccordCacheEntry entry); - } - - // functions as both an identity object, and a register of listeners - public static class UniqueSave - { - @Nullable List onSuccess; - void onSuccess(Runnable onSuccess) - { - if (this.onSuccess == null) - this.onSuccess = new ArrayList<>(); - this.onSuccess.add(onSuccess); - } - - static void notify(List onSuccess) - { - if (onSuccess != null) - { - onSuccess.forEach(run -> { - try { run.run(); } - catch (Throwable t) - { - Thread thread = Thread.currentThread(); - thread.getUncaughtExceptionHandler().uncaughtException(thread, t); - } - }); - } - } - } - - public interface SaveExecutor - { - Cancellable save(AccordCacheEntry saving, UniqueSave identity, Runnable save); - } - - public Loading load(LoadExecutor loadExecutor, P1 p1, P2 p2) - { - Invariants.require(is(WAITING_TO_LOAD), "%s", this); - - WaitingToLoad cur = (WaitingToLoad)state; - Loading loading = cur.load(loadExecutor.load(p1, p2, this)); - setStatus(LOADING); - state = loading; - return loading; - } - - public Loading testLoad() - { - Invariants.require(is(WAITING_TO_LOAD)); - Loading loading = ((WaitingToLoad)state).load(() -> {}); - setStatus(LOADING); - state = loading; - return loading; - } - - public Loading loading() - { - Invariants.require(is(LOADING), "%s", this); - return (Loading) state; - } - - // must own the cache's lock when invoked. this is true of most methods in the class, - // but this one is less obvious so named as to draw attention - public V getExclusive() - { - Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); - Invariants.require(isLoaded(), "%s", this); - if (isShrunk()) - { - AccordCache.Type parent = owner.parent(); - inflate(owner.commandStore, key, parent.adapter()); - updateSize(parent); - } - - return (V)unwrap(); - } - - public Object getOrShrunkExclusive() - { - Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); - Invariants.require(isLoaded(), "%s", this); - return unwrap(); - } - - public V tryGetExclusive() - { - Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); - if (!isLoaded() || isShrunk()) - return null; - return (V)unwrap(); - } - - private Object unwrap() - { - return isNested() ? ((Nested)state).state : state; - } - - // must own the cache's lock when invoked - void setExclusive(V value) - { - if (value == state) - return; - - Saving cancel = is(SAVING) ? ((Saving)state) : null; - if (is(WAITING_TO_SAVE)) - { - ((WaitingToSave) state).state = value; - if (owner.parent().adapter().canSave(value, null)) - save(); - } - else - { - setStatus(MODIFIED); - state = value; - } - updateSize(owner.parent()); - // TODO (expected): do we want to cancel in-progress saving? - if (cancel != null && cancel.identity.onSuccess == null) - cancel.saving.cancel(); - } - - public void loaded(V value) - { - setStatus(LOADED); - state = value; - updateSize(owner.parent()); - } - - public void testLoaded(V value) - { - setStatus(LOADED); - state = value; - } - - public void failedToLoad() - { - setStatus(FAILED_TO_LOAD); - state = null; - } - - Shrink tryShrink() - { - if (!isLoaded()) - return Shrink.EVICT; - - AccordCache.Type parent = owner.parent(); - Adapter adapter = parent.adapter(); - if (isShrunk() || state == null) - return Shrink.EVICT; - - V cur = (V)unwrap(); - Shrink shrink = adapter.decideFullShrink(key, cur); - if (shrink == Shrink.PERFORM_WITHOUT_LOCK) - return Shrink.PERFORM_WITHOUT_LOCK; - - Object upd = adapter.fullShrink(key, cur); - if (upd == null || upd == cur) - return Shrink.EVICT; - applyShrink(parent, cur, upd); - return Shrink.DONE; - } - - V tryGetFull() - { - return isShrunk() ? null : (V)unwrap(); - } - - Object tryGetShrunk() - { - return isShrunk() ? unwrap() : null; - } - - boolean isNull() - { - return state == null; - } - - boolean saveWhenReady() - { - V full = isShrunk() ? null : (V)state; - Object shrunk = isShrunk() ? state : null; - if (owner.parent().adapter().canSave(full, shrunk)) - return save(); - - setStatus(WAITING_TO_SAVE); - UniqueSave identity = new UniqueSave(); - state = new WaitingToSave<>(identity, state); - return true; - } - - /** - * Submits a save runnable to the specified executor. When the runnable - * has completed, the state save will have either completed or failed. - */ - @VisibleForTesting - boolean save() - { - WaitingToSave waitingToSave = is(WAITING_TO_SAVE) ? (WaitingToSave)state : null; - Object state = waitingToSave == null ? this.state : waitingToSave.state; - V full = isShrunk() ? null : (V)state; - Object shrunk = isShrunk() ? state : null; - Runnable save = owner.parent().adapter().save(owner.commandStore, key, full, shrunk); - - UniqueSave identity = waitingToSave == null ? new UniqueSave() : waitingToSave.identity; - if (null == save) // null mutation -> null Runnable -> no change on disk - { - setStatus(LOADED); - if (waitingToSave != null) - this.state = state; - UniqueSave.notify(identity.onSuccess); - return false; - } - else - { - setStatus(SAVING); - Cancellable saving = owner.parent().parent().saveExecutor.save(this, identity, save); - this.state = new Saving(saving, identity, state); - return true; - } - } - - boolean saved(Object identity, Throwable fail) - { - if (identity instanceof UniqueSave) - UniqueSave.notify(((UniqueSave) identity).onSuccess); - - if (!is(SAVING)) - return false; - - Saving saving = (Saving) state; - if (saving.identity != identity) - return false; - - if (fail != null) - { - setStatus(FAILED_TO_SAVE); - state = new FailedToSave(fail, ((Saving)state).state); - return false; - } - else - { - setStatus(LOADED); - state = saving.state; - return true; - } - } - - protected void saved() - { - Invariants.require(is(MODIFIED)); - setStatus(LOADED); - } - - public SavingOrWaitingToSave savingOrWaitingToSave() - { - return (SavingOrWaitingToSave) state; - } - - public AccordCacheEntry evicted() - { - if (isNoEvict()) - setStatusUnsafe(EVICTED); - else setStatus(EVICTED); - state = null; - return this; - } - - public Throwable failure() - { - return ((FailedToSave)state).cause; - } - - void tryApplyShrink(Object cur, Object upd, IntrusiveLinkedList> queue) - { - if (references() > 0 || !isUnqueued()) - return; - - if (isLoaded() && unwrap() == cur && upd != cur && upd != null) - applyShrink(owner.parent(), cur, upd); - queue.addLast(this); - } - - private void applyShrink(AccordCache.Type parent, Object cur, Object upd) - { - if (isNested()) ((Nested)this.state).state = upd; - else this.state = upd; - status |= SHRUNK; - updateSize(parent); - } - - private void inflate(AccordCommandStore commandStore, K key, Adapter adapter) - { - Invariants.require(isShrunk()); - if (isNested()) - { - Nested nested = (Nested) state; - nested.state = adapter.inflate(commandStore, key, nested.state); - } - else - { - state = adapter.inflate(commandStore, key, state); - } - status &= ~SHRUNK; - } - - private long estimateOnHeapSize(Adapter adapter) - { - Object current = unwrap(); - if (current == null) return 0; - else if (isShrunk()) return adapter.estimateShrunkHeapSize(current); - return adapter.estimateHeapSize((V)current); - } - - public static abstract class LoadingOrWaiting - { - Collection> waiters; - - public LoadingOrWaiting() - { - } - - public LoadingOrWaiting(Collection> waiters) - { - this.waiters = waiters; - } - - public Collection> waiters() - { - return waiters != null ? waiters : Collections.emptyList(); - } - - public BufferList> copyWaiters() - { - BufferList> list = new BufferList<>(); - if (waiters != null) - list.addAll(waiters); - return list; - } - - public void add(AccordTask waiter) - { - if (waiters == null) - waiters = new ArrayList<>(); - waiters.add(waiter); - } - - public void remove(AccordTask waiter) - { - if (waiters != null) - { - waiters.remove(waiter); - if (waiters.isEmpty()) - waiters = null; - } - } - } - - static class WaitingToLoad extends LoadingOrWaiting - { - public Loading load(Cancellable loading) - { - Invariants.paranoid(waiters == null || !waiters.isEmpty()); - Loading result = new Loading(waiters, loading); - waiters = Collections.emptyList(); - return result; - } - } - - static class Loading extends LoadingOrWaiting - { - public final Cancellable loading; - - public Loading(Collection> waiters, Cancellable loading) - { - super(waiters); - this.loading = loading; - } - } - - static class Nested - { - Object state; - } - - static class SavingOrWaitingToSave extends Nested - { - final UniqueSave identity; - - SavingOrWaitingToSave(UniqueSave identity, Object state) - { - this.identity = identity; - this.state = state; - } - } - - static class Saving extends SavingOrWaitingToSave - { - final Cancellable saving; - - Saving(Cancellable saving, UniqueSave identity, Object state) - { - super(identity, state); - this.saving = saving; - } - } - - static class WaitingToSave extends SavingOrWaitingToSave - { - WaitingToSave(UniqueSave identity, Object state) - { - super(identity, state); - } - } - - static class FailedToSave extends Nested - { - final Throwable cause; - - FailedToSave(Throwable cause, Object state) - { - this.cause = cause; - this.state = state; - } - - public Throwable failure() - { - return cause; - } - } - - public static AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) - { - AccordCacheEntry node = new AccordCacheEntry<>(key, owner); - node.readyToLoad(); - return node; - } -} diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index f4dd59a9c7..3fe7550439 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -27,7 +27,6 @@ import java.util.Objects; import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.concurrent.locks.Lock; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.IntFunction; @@ -41,7 +40,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.Agent; -import accord.api.AsyncExecutor; import accord.api.DataStore; import accord.api.Journal; import accord.api.LocalListeners; @@ -57,14 +55,13 @@ import accord.impl.progresslog.TxnState; import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; -import accord.local.CommandSummaries; +import accord.local.ExecutionContext; +import accord.local.ExecutionContext.Empty; import accord.local.MaxConflicts; import accord.local.MaxDecidedRX; import accord.local.MinimalCommand; import accord.local.MinimalCommand.MinimalWithDeps; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; -import accord.local.PreLoadContext.Empty; import accord.local.RedundantBefore; import accord.local.RedundantBefore.Bounds; import accord.local.RedundantStatus.Property; @@ -74,7 +71,6 @@ import accord.local.cfk.CommandsForKey; import accord.primitives.PartialTxn; import accord.primitives.Range; import accord.primitives.Ranges; -import accord.primitives.RoutableKey; import accord.primitives.Route; import accord.primitives.SaveStatus; import accord.primitives.Status; @@ -100,9 +96,17 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; -import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfo; -import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.ExclusiveExecutor; +import org.apache.cassandra.service.accord.execution.SafeTask; +import org.apache.cassandra.service.accord.execution.SaferCommand; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandsForKey; +import org.apache.cassandra.service.accord.execution.TaskRunner; +import org.apache.cassandra.service.accord.execution.Unterminatable; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.journal.JournalRangeIndex; import org.apache.cassandra.service.accord.txn.TxnRead; @@ -139,10 +143,10 @@ public class AccordCommandStore extends CommandStore public static class Caches { private final AccordCache global; - private final AccordCache.Type.Instance commands; - private final AccordCache.Type.Instance commandsForKeys; + private final AccordCache.Type.Instance commands; + private final AccordCache.Type.Instance commandsForKeys; - Caches(AccordCache global, AccordCache.Type.Instance commandCache, AccordCache.Type.Instance commandsForKeyCache) + Caches(AccordCache global, AccordCache.Type.Instance commandCache, AccordCache.Type.Instance commandsForKeyCache) { this.global = global; this.commands = commandCache; @@ -154,44 +158,47 @@ public class AccordCommandStore extends CommandStore return global; } - public final AccordCache.Type.Instance commands() + public final AccordCache.Type.Instance commands() { return commands; } - public final AccordCache.Type.Instance commandsForKeys() + public final AccordCache.Type.Instance commandsForKeys() { return commandsForKeys; } } - public static final class ExclusiveCaches extends Caches implements CommandStoreCaches + public static final class ExclusiveCaches extends Caches implements CommandStoreCaches { - private final Lock lock; + private final AccordExecutor owner; - public ExclusiveCaches(Lock lock, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) + public ExclusiveCaches(AccordExecutor owner, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) { super(global, commands, commandsForKeys); - this.lock = lock; + this.owner = owner; } @Override - public AccordSafeCommand acquireIfLoaded(TxnId txnId) + public SaferCommand acquireIfLoaded(TxnId txnId) { - return commands().acquireIfLoaded(txnId); + // note: we must return false if the entry is locked to enforce ordering. + // note importantly that this is also coupled to the safety of synchronously releasing ExclusiveExecutor.owner, + // rather than waiting until the (potentially asynchronous) cleanup of the task completes + return commands().acquireIfLoadedAndPermitted(txnId); } @Override - public AccordSafeCommandsForKey acquireIfLoaded(RoutingKey key) + public SaferCommandsForKey acquireIfLoaded(RoutingKey key) { - return commandsForKeys().acquireIfLoaded(key); + return commandsForKeys().acquireIfLoadedAndPermitted(key); } @Override public void close() { - global().tryShrinkOrEvict(lock); - lock.unlock(); + global().tryShrinkOrEvict(owner.unsafeLock()); + owner.unlock(TaskRunner.get()); } } @@ -211,12 +218,10 @@ public class AccordCommandStore extends CommandStore = AtomicReferenceFieldUpdater.newUpdater(AccordCommandStore.class, Termination.class, "terminated"); static final AtomicLong nextSafeRedundantBeforeTicket = new AtomicLong(); - private static final AtomicLong lastSystemTimestampMicros = new AtomicLong(); - public final String loggingId; public final Journal journal; private final AccordExecutor sharedExecutor; - final AccordExecutor.SequentialExecutor exclusiveExecutor; + private final ExclusiveExecutor exclusiveExecutor; private final ExclusiveCaches caches; private final RangeIndex rangeIndex; private final TableId tableId; @@ -225,8 +230,8 @@ public class AccordCommandStore extends CommandStore volatile SafeRedundantBefore safeRedundantBefore; volatile Termination terminated; - private AccordSafeCommandStore current; - LogLinearDecayingHistograms.Buffer metricsBuffer; + private SaferCommandStore current; + public LogLinearDecayingHistograms.Buffer metricsBuffer; public AccordCommandStore(int id, NodeCommandStoreService node, @@ -249,21 +254,23 @@ public class AccordCommandStore extends CommandStore maybeLoadRedundantBefore(journal.loadRedundantBefore(id())); maybeLoadBootstrapBeganAt(journal.loadBootstrapBeganAt(id())); maybeLoadSafeToRead(journal.loadSafeToRead(id())); - - tableId = (TableId)rangesForEpoch.all().stream().map(r -> r.start().prefix()).reduce((a, b) -> { + maybeLoadRangesForEpoch(journal.loadRangesForEpoch(id())); + RangesForEpoch ranges = this.rangesForEpoch; + Invariants.require(ranges != null && !ranges.all().isEmpty(), "CommandStore %d created with no ranges", id); + tableId = (TableId)ranges.all().stream().map(r -> r.start().prefix()).reduce((a, b) -> { Invariants.require(a.equals(b), "CommandStore created with multiple distinct TableId (%s and %s)", a, b); return a; }).orElseThrow(() -> Invariants.illegalState("CommandStore %d created with no ranges", id)); - final AccordCache.Type.Instance commands; - final AccordCache.Type.Instance commandsForKey; + final AccordCache.Type.Instance commands; + final AccordCache.Type.Instance commandsForKey; try (AccordExecutor.ExclusiveGlobalCaches exclusive = sharedExecutor.lockCaches()) { commands = exclusive.commands.newInstance(this); commandsForKey = exclusive.commandsForKey.newInstance(this); - this.caches = new ExclusiveCaches(sharedExecutor.unsafeLock(), exclusive.global, commands, commandsForKey); + this.caches = new ExclusiveCaches(sharedExecutor, exclusive.global, commands, commandsForKey); } - this.exclusiveExecutor = sharedExecutor.executor(id); + this.exclusiveExecutor = sharedExecutor.newExclusiveExecutor(id); { AccordConfig.RangeIndexMode mode = getAccord().range_index_mode; @@ -296,12 +303,6 @@ public class AccordCommandStore extends CommandStore return exclusiveExecutor.inExecutor(); } - void tryPreSetup(AccordTask task) - { - if (inStore() && current != null) - task.presetup(current.task); - } - public final TableId tableId() { return tableId; @@ -315,7 +316,7 @@ public class AccordCommandStore extends CommandStore // TODO (desired): we use this for executing callbacks with mutual exclusivity, // but we don't need to block the actual CommandStore - could quite easily // inflate a separate queue dynamically in AccordExecutor - public AsyncExecutor taskExecutor() + public ExclusiveExecutor exclusiveExecutor() { return exclusiveExecutor; } @@ -323,13 +324,13 @@ public class AccordCommandStore extends CommandStore public ExclusiveCaches lockCaches() { //noinspection LockAcquiredButNotSafelyReleased - caches.lock.lock(); + caches.owner.lock(TaskRunner.get()); return caches; } public ExclusiveCaches tryLockCaches() { - if (caches.lock.tryLock()) + if (caches.owner.tryLock(TaskRunner.get())) return caches; return null; } @@ -357,128 +358,70 @@ public class AccordCommandStore extends CommandStore journal.saveCommand(id, new CommandUpdate(before, after), onFlush); } - boolean validateCommand(TxnId txnId, Command evicting) - { - if (!Invariants.isParanoid()) - return true; - - Command reloaded = loadCommand(txnId); - return Objects.equals(evicting, reloaded); - } - @VisibleForTesting public void sanityCheckCommand(RedundantBefore redundantBefore, Command command) { ((AccordJournal) journal).sanityCheck(id, redundantBefore, command); } - CommandsForKey loadCommandsForKey(RoutableKey key) - { - CommandsForKey cfk = CommandsForKeyAccessor.load(id, (TokenKey) key); - if (cfk == null) - return null; - RedundantBefore.QuickBounds bounds = safeGetRedundantBefore().get(key); - if (!Invariants.expect(bounds != null, "No RedundantBefore information found when loading key %s", key)) - return cfk; - return cfk.withCleanCfkBeforeAtLeast(bounds.cleanCfkBefore(), false); - } - - boolean validateCommandsForKey(RoutableKey key, CommandsForKey evicting) - { - if (!Invariants.isParanoid()) - return true; - - CommandsForKey reloaded = CommandsForKeyAccessor.load(id, (TokenKey) key); - return Objects.equals(evicting, reloaded); - } - - @Nullable - Runnable saveCommandsForKey(RoutingKey key, CommandsForKey after, Object serialized) - { - return CommandsForKeyAccessor.systemTableUpdater(id, (TokenKey) key, after, serialized, nextSystemTimestampMicros()); - } - - public long nextSystemTimestampMicros() - { - return lastSystemTimestampMicros.accumulateAndGet(node.now(), (a, b) -> Math.max(a + 1, b)); - } @Override - public AsyncChain chain(PreLoadContext loadCtx, Function function) + public AsyncChain chain(ExecutionContext context, Function function) { - return AccordTask.create(this, loadCtx, function).chain(); + return SafeTask.create(this, context, function).chain(); } @Override - public AsyncChain chain(PreLoadContext preLoadContext, Consumer consumer) + public AsyncChain chain(ExecutionContext context, Consumer consumer) { - return AccordTask.create(this, preLoadContext, consumer).chain(); - } - - @Override - public AsyncChain priorityChain(PreLoadContext preLoadContext, Consumer consumer) - { - return AccordTask.create(this, preLoadContext, consumer).priorityChain(); - } - - @Override - public AsyncChain priorityChain(PreLoadContext preLoadContext, Function function) - { - return AccordTask.create(this, preLoadContext, function).priorityChain(); + return SafeTask.create(this, context, consumer).chain(); } @Override public AsyncChain chain(Callable call) { - return taskExecutor().chain(call); + return exclusiveExecutor().chain(call); } @Override public void execute(Runnable run) { - taskExecutor().execute(run); + exclusiveExecutor().execute(run); } @Override public boolean tryExecuteImmediately(Runnable run) { - return taskExecutor().tryExecuteImmediately(run); + return exclusiveExecutor().tryExecuteImmediately(run); } - public AccordSafeCommandStore begin(AccordTask operation, @Nullable CommandSummaries commandsForRanges) + public SaferCommandStore begin(SaferCommandStore safeStore) { require(current == null); - current = AccordSafeCommandStore.create(operation, commandsForRanges, this); + current = safeStore; return current; } + public void complete(SaferCommandStore store) + { + require(current == store); + current = null; + } + public boolean hasSafeStore() { return current != null; } - DataStore dataStore() + public DataStore dataStore() { return dataStore; } - ProgressLog progressLog() + public ProgressLog progressLog() { return progressLog; } - public void complete(AccordSafeCommandStore store) - { - require(current == store); - current.postExecute(); - current = null; - } - - public void abort(AccordSafeCommandStore store) - { - Invariants.require(store == current); - current = null; - } - @Override public void shutdown() { @@ -662,7 +605,7 @@ public class AccordCommandStore extends CommandStore public Ready() { super(1); } @Override public void run() { decrement(); } - void maybeFlush(ExclusiveCaches caches, AccordCacheEntry e) + void maybeFlush(ExclusiveCaches caches, AccordCacheEntry e) { if (e.isModified()) { @@ -677,7 +620,7 @@ public class AccordCommandStore extends CommandStore { if (ranges == null) { - for (AccordCacheEntry e : caches.commandsForKeys()) + for (AccordCacheEntry e : caches.commandsForKeys()) ready.maybeFlush(caches, e); } else @@ -729,7 +672,7 @@ public class AccordCommandStore extends CommandStore if (!maybeShouldReplay(txnId)) return AsyncChains.success(null); - return commandStore.chain(PreLoadContext.contextFor(txnId, "Replay"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Replay"), safeStore -> { Replay replay = shouldReplay(txnId, safeStore.unsafeGet(txnId).current().participants()); if (replay == Replay.NONE) return null; @@ -776,7 +719,7 @@ public class AccordCommandStore extends CommandStore AsyncChain saveState(Descriptor descriptor) { - return chain((AccordExecutor.Unterminatable)() -> "Save State", safeStore -> { + return chain((Unterminatable)() -> "Save State", safeStore -> { File storeDir = storeSaveDir(); { File[] tmpDirs = listTmpSaveDirs(storeDir); @@ -883,7 +826,7 @@ public class AccordCommandStore extends CommandStore { File rjbf = new File(savePoint, "reject_before"); if (rjbf.exists()) - mxc = mxc.with(readOne(rjbf, rejectBefore)); + mxc = mxc.update(readOne(rjbf, rejectBefore)); } dll = readList(new File(savePoint, "listeners"), txnListener); dpl = readList(new File(savePoint, "progress_log"), progressLogState); @@ -914,7 +857,7 @@ public class AccordCommandStore extends CommandStore // TODO (expected): handle journal failures, and consider how we handle partial failures. // Very likely we will not be able to safely or cleanly handle partial failures of this logic, but decide and document. // TODO (desired): consider merging with PersistentField? This version is cheaper to manage which may be preferable at the CommandStore level. - static class SafeRedundantBefore + public static class SafeRedundantBefore { final long ticket; final RedundantBefore redundantBefore; @@ -929,6 +872,15 @@ public class AccordCommandStore extends CommandStore { return a.ticket >= b.ticket ? a : b; } + + public static Runnable updater(AccordCommandStore commandStore, RedundantBefore newRedundantBefore) + { + long ticket = nextSafeRedundantBeforeTicket.incrementAndGet(); + SafeRedundantBefore update = new SafeRedundantBefore(ticket, newRedundantBefore); + return () -> { + safeRedundantBeforeUpdater.accumulateAndGet(commandStore, update, SafeRedundantBefore::max); + }; + } } private @Nullable TableMetadata tableMetadata() diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index f362c960c4..576fdf482b 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -31,13 +31,14 @@ import org.slf4j.LoggerFactory; import accord.api.Agent; import accord.api.DataStore; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.LocalListeners; import accord.api.ProgressLog; import accord.local.CommandStores; import accord.local.NodeCommandStoreService; -import accord.local.SequentialAsyncExecutor; import accord.local.ShardDistributor; +import accord.utils.Invariants; import accord.utils.RandomSource; import accord.utils.Reduce; import accord.utils.async.AsyncChain; @@ -54,14 +55,20 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.journal.Descriptor; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore.DurablyAppliedTo; -import org.apache.cassandra.service.accord.AccordExecutor.AccordExecutorFactory; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor.AccordExecutorFactory; +import org.apache.cassandra.service.accord.execution.AccordExecutorAsyncSubmit; +import org.apache.cassandra.service.accord.execution.AccordExecutorSemiSyncSubmit; +import org.apache.cassandra.service.accord.execution.AccordExecutorSignalLoop; +import org.apache.cassandra.service.accord.execution.AccordExecutorSimple; +import org.apache.cassandra.service.accord.execution.AccordExecutorSyncSubmit; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD; +import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD_SYNC_QUEUE; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; -import static org.apache.cassandra.service.accord.AccordExecutor.constant; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.constant; import static org.apache.cassandra.service.accord.journal.ReplayMarkers.saveDirectory; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -73,7 +80,6 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu private final int mask; private long cacheSize, workingSetSize; - private int maxQueuedLoads, maxQueuedRangeLoads; private boolean shrinkingOn; AccordCommandStores(NodeCommandStoreService node, Agent agent, DataStore store, RandomSource random, @@ -87,12 +93,9 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu cacheSize = DatabaseDescriptor.getAccordCacheSizeInMiB() << 20; workingSetSize = DatabaseDescriptor.getAccordWorkingSetSizeInMiB() << 20; - AccordConfig config = DatabaseDescriptor.getAccord(); - maxQueuedLoads = maxQueuedLoads(config); - maxQueuedRangeLoads = maxQueuedRangeLoads(config); shrinkingOn = DatabaseDescriptor.getAccordCacheShrinkingOn(); refreshCapacities(); - ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(() -> { + ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(() -> { for (AccordExecutor executor : executors) { executor.executeDirectlyWithLock(() -> { @@ -122,19 +125,21 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu break; case EXEC_ST: factory = AccordExecutorSimple::new; + Invariants.require(executors.length == 1, "Executors that hold their lock during execution cannot support multiple executor shards, as they cannot lock multiple executors at once"); maxThreads = 1; break; } + QueueShardModel shardModel = config.queue_shard_model; + Invariants.require(shardModel != THREAD_PER_SHARD_SYNC_QUEUE || executors.length == 1, "Executors that hold their lock during execution cannot support multiple executor shards, as they cannot lock multiple executors at once"); for (int id = 0; id < executors.length; id++) { - QueueShardModel shardModel = config.queue_shard_model; String baseName = AccordExecutor.class.getSimpleName() + '[' + id; switch (shardModel) { case THREAD_PER_SHARD: case THREAD_PER_SHARD_SYNC_QUEUE: - executors[id] = factory.get(id, shardModel == THREAD_PER_SHARD ? RUN_WITHOUT_LOCK : RUN_WITH_LOCK, 1, constant(baseName + ']'), agent); + executors[id] = factory.get(id, shardModel == THREAD_PER_SHARD_SYNC_QUEUE ? RUN_WITH_LOCK : RUN_WITHOUT_LOCK, 1, constant(baseName + ']'), agent); break; case THREAD_POOL_PER_SHARD: int threads = Math.min(maxThreads, Math.max(DatabaseDescriptor.getAccordConcurrentOps() / executors.length, 1)); @@ -148,10 +153,10 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu } @Override - public SequentialAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someExclusiveExecutor() { int idx = ((int) Thread.currentThread().getId()) & mask; - return executors[idx].newSequentialExecutor(); + return executors[idx].newExclusiveExecutor(); } public synchronized void setCapacity(long bytes) @@ -173,13 +178,6 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu refreshCapacities(); } - public synchronized void setMaxQueuedLoads(int total, int range) - { - maxQueuedLoads = total; - maxQueuedRangeLoads = range; - refreshCapacities(); - } - public synchronized void setShrinking(boolean on) { shrinkingOn = on; @@ -213,14 +211,11 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu { long capacityPerExecutor = cacheSize / executors.length; long workingSetPerExecutor = workingSetSize < 0 ? Long.MAX_VALUE : workingSetSize / executors.length; - int maxLoadsPerExecutor = Math.max(1, (maxQueuedLoads + executors.length - 1) / executors.length); - int maxRangeLoadsPerExecutor = Math.max(1, (maxQueuedRangeLoads + executors.length - 1) / executors.length); for (AccordExecutor executor : executors) { executor.executeDirectlyWithLock(() -> { executor.setCapacity(capacityPerExecutor); executor.setWorkingSetSize(workingSetPerExecutor); - executor.setMaxQueuedLoads(maxLoadsPerExecutor, maxRangeLoadsPerExecutor); executor.cacheExclusive().setShrinkingOn(shrinkingOn); }); } @@ -341,21 +336,4 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu return Math.max(1, config.queue_shard_count.or(DatabaseDescriptor.getAvailableProcessors() / 8)); } } - - private static int threads(AccordConfig config) - { - return config.queue_thread_count.or(2 * FBUtilities.getAvailableProcessors()); - } - - public static int maxQueuedLoads(AccordConfig config) - { - return config.max_queued_loads.or(FBUtilities.getAvailableProcessors()); - } - - public static int maxQueuedRangeLoads(AccordConfig config) - { - return config.max_queued_range_loads.or(maxQueuedLoads(config) / 4); - } - - } diff --git a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java index 7eba765c5b..ebd2911673 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java @@ -20,24 +20,23 @@ package org.apache.cassandra.service.accord; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.DataStore; import accord.local.CommandStore; +import accord.local.CommandStores.RestrictedStoreSelector; +import accord.local.MapReduceConsumeCommandStores; import accord.local.Node; import accord.local.RedundantBefore; import accord.local.SafeCommandStore; import accord.primitives.Ranges; -import accord.primitives.SyncPoint; +import accord.primitives.Timestamp; import accord.primitives.TxnId; import accord.utils.Invariants; -import accord.utils.Reduce; +import accord.utils.SortedArrays.SortedArrayList; import accord.utils.UnhandledEnum; -import accord.utils.async.AsyncResult; import accord.utils.async.AsyncResults; import org.apache.cassandra.concurrent.ScheduledExecutors; @@ -56,10 +55,9 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.progress.ProgressEvent; +import org.apache.cassandra.utils.progress.ProgressListener; -import static accord.local.durability.DurabilityService.SyncLocal.NoLocal; -import static accord.local.durability.DurabilityService.SyncRemote.Quorum; -import static accord.primitives.Txn.Kind.ExclusiveSyncPoint; import static accord.utils.Invariants.illegalArgument; public class AccordDataStore implements DataStore @@ -94,12 +92,12 @@ public class AccordDataStore implements DataStore AccordDurableOnFlush.notifyOnDurable(cfs, commandStore, ReportDurable.of(redundantBefore, flags)); } - public FetchResult image(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback) + public FetchResult image(Node node, SafeCommandStore safeStore, Ranges ranges, TxnId atLeast, SortedArrayList readable, FetchRanges callback) { AccordFetchCoordinator coordinator; try { - coordinator = new AccordFetchCoordinator(node, ranges, syncPoint, callback, safeStore.commandStore()); + coordinator = new AccordFetchCoordinator(node, ranges, atLeast, readable, callback, safeStore.commandStore()); } catch (Throwable t) { @@ -110,11 +108,10 @@ public class AccordDataStore implements DataStore return coordinator.result(); } - public FetchResult sync(Node node, SafeCommandStore safeStore, Map rangesById, FetchRanges callback) + @Override + public FetchResult sync(Node node, SafeCommandStore safeStore, Ranges ranges, TxnId atLeast, SortedArrayList readable, FetchRanges callback) { TableId tableId = ((AccordCommandStore)safeStore.commandStore()).tableId(); - Invariants.require(rangesById.values().stream().flatMap(Ranges::stream).allMatch(r -> tableId.equals(r.prefix()))); - Ranges ranges = rangesById.values().stream().reduce(Ranges::with).get(); ClusterMetadata cm = ClusterMetadata.current(); TableMetadata tableMetadata = cm.schema.getTableMetadata(tableId); @@ -137,27 +134,23 @@ public class AccordDataStore implements DataStore // TODO (expected): add some automatic slicing of ranges and retry/back-off logic; but for now, // since this is done at the command store level, and this is already a slice of a node, this should be fine SyncResult syncResult = new SyncResult(); + ProgressListener listener = new ProgressListener() + { + StartingRangeFetch starting = callback.starting(ranges); + { Invariants.require(starting != null); } - logger.info("Requesting quorum durability before initiating repair"); - List> syncs = new ArrayList<>(); - for (Map.Entry e : rangesById.entrySet()) - syncs.add(node.durability().sync("Sync Data Store", ExclusiveSyncPoint, e.getKey(), e.getValue(), NoLocal, Quorum, 1L, TimeUnit.HOURS)); - - AsyncResults.reduce(syncs, Reduce.toNull()).invoke((success, fail) -> { - if (fail != null) + @Override + public void progress(String tag, ProgressEvent event) { - logger.error("{} failed to achieve quorum durability before repair for rebootstrap of {}", safeStore.commandStore(), ranges); - syncResult.tryFailure(fail); - return; - } - - RepairCoordinator coord = StorageService.instance.newRepairCoordinator(tableMetadata.keyspace, options(tableMetadata, ranges)); - coord.addProgressListener((tag, event) -> { switch (event.getType()) { default: throw new UnhandledEnum(event.getType()); + case SUCCESS: + callback.fetched(ranges); + syncResult.trySuccess(null); + // fall-through to ensure started case START: - callback.starting(ranges); + reportStarted(); break; case PROGRESS: case COMPLETE: @@ -165,21 +158,40 @@ public class AccordDataStore implements DataStore break; case ABORT: case ERROR: - IllegalStateException ex = new IllegalStateException(String.format("Repair failed: %s", event)); + RuntimeException ex = new RuntimeException(String.format("Repair failed (%s): %s", event.getType(), event.getMessage())); callback.fail(ranges, ex); syncResult.tryFailure(ex); break; - case SUCCESS: - callback.fetched(ranges); - syncResult.trySuccess(null); - break; } - }); + } - ScheduledExecutors.optionalTasks.submit(coord).addCallback((s, f) -> { - if (f != null) - syncResult.tryFailure(f); - }); + private void reportStarted() + { + StartingRangeFetch start = this.starting; + if (start == null) + return; + this.starting = null; + node.commandStores().mapReduceConsume(new RestrictedStoreSelector(ranges, 0, Long.MAX_VALUE), new MapReduceConsumeCommandStores(ranges) + { + @Override public Timestamp reduce(Timestamp o1, Timestamp o2) { return Timestamp.max(o1, o2); } + @Override public void accept(Timestamp result, Throwable failure) + { + if (failure != null) syncResult.tryFailure(failure); + else start.started(result); + } + @Override public TxnId primaryTxnId() { return null; } + @Override public String reason() { return "Compute MaxConflict to report for fetch"; } + @Override protected Timestamp applyInternal(SafeCommandStore safeStore) { return safeStore.commandStore().maxConflict(TxnId.NONE, ranges); } + }); + } + }; + + RepairCoordinator coord = StorageService.instance.newRepairCoordinator(tableMetadata.keyspace, options(tableMetadata, ranges)); + coord.addProgressListener(listener); + + ScheduledExecutors.optionalTasks.submit(coord).addCallback((s, f) -> { + if (f != null) + syncResult.tryFailure(f); }); return syncResult; diff --git a/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java b/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java index 6905b10bf9..554ea61cd2 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java @@ -34,8 +34,9 @@ import org.apache.cassandra.db.lifecycle.View; import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.accord.execution.Unstoppable; -class AccordDurableOnFlush implements BiConsumer +public class AccordDurableOnFlush implements BiConsumer { private static final Logger logger = LoggerFactory.getLogger(AccordDurableOnFlush.class); @@ -98,6 +99,11 @@ class AccordDurableOnFlush implements BiConsumer { return redundantBefore.toString(); } + + public static void reportMaybeTerminate(AccordCommandStore commandStore, int flags) + { + commandStore.maybeTerminated(ReportDurable.isCommandStoreFlush(flags), ReportDurable.isDataStoreFlush(flags)); + } } private Int2ObjectHashMap commandStores = new Int2ObjectHashMap<>(); @@ -185,7 +191,7 @@ class AccordDurableOnFlush implements BiConsumer static void notifyNow(CommandStore commandStore, ReportDurable report) { logger.debug("{} reporting flush with {}", commandStore, report); - commandStore.execute((AccordExecutor.Unstoppable) () -> "Report Durable", safeStore -> { + commandStore.execute((Unstoppable) () -> "Report Durable", safeStore -> { safeStore.reportDurable(report.redundantBefore, report.flags); }, commandStore.agent()); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java deleted file mode 100644 index b1aff08f13..0000000000 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ /dev/null @@ -1,2150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.service.accord; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.Callable; -import java.util.concurrent.CancellationException; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.LockSupport; -import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.function.IntFunction; -import java.util.stream.Stream; - -import javax.annotation.Nullable; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import accord.api.Agent; -import accord.api.RoutingKey; -import accord.impl.AbstractAsyncExecutor; -import accord.local.Command; -import accord.local.PreLoadContext; -import accord.local.SequentialAsyncExecutor; -import accord.local.cfk.CommandsForKey; -import accord.messages.Accept; -import accord.messages.Commit; -import accord.messages.MessageType; -import accord.messages.MessageType.StandardMessage; -import accord.messages.Request; -import accord.primitives.Ballot; -import accord.primitives.SaveStatus; -import accord.primitives.TxnId; -import accord.utils.ArrayBuffers.BufferList; -import accord.utils.IntrusivePriorityHeap; -import accord.utils.Invariants; -import accord.utils.QuadConsumer; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -import accord.utils.TriConsumer; -import accord.utils.TriFunction; -import accord.utils.UnhandledEnum; -import accord.utils.async.AsyncCallbacks.CallAndCallback; -import accord.utils.async.AsyncCallbacks.FlatCallAndCallback; -import accord.utils.async.AsyncCallbacks.RunAndCallback; -import accord.utils.async.AsyncCallbacks.RunOrFail; -import accord.utils.async.AsyncChain; -import accord.utils.async.AsyncChains; -import accord.utils.async.Cancellable; - -import org.apache.cassandra.cache.CacheSize; -import org.apache.cassandra.concurrent.DebuggableTask; -import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; -import org.apache.cassandra.concurrent.ExecutorLocals; -import org.apache.cassandra.concurrent.Shutdownable; -import org.apache.cassandra.config.AccordConfig.QueuePriorityModel; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.metrics.AccordCacheMetrics; -import org.apache.cassandra.metrics.AccordExecutorMetrics; -import org.apache.cassandra.metrics.AccordReplicaMetrics; -import org.apache.cassandra.metrics.AccordSystemMetrics; -import org.apache.cassandra.metrics.LogLinearDecayingHistograms; -import org.apache.cassandra.metrics.LogLinearDecayingHistograms.LogLinearDecayingHistogram; -import org.apache.cassandra.metrics.ShardedDecayingHistograms; -import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistogramsShard; -import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.UniqueSave; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugSequentialExecutor; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; -import org.apache.cassandra.utils.Closeable; -import org.apache.cassandra.utils.MonotonicClock; -import org.apache.cassandra.utils.WithResources; -import org.apache.cassandra.utils.concurrent.AsyncPromise; -import org.apache.cassandra.utils.concurrent.Condition; -import org.apache.cassandra.utils.concurrent.Future; - -import io.netty.util.concurrent.FastThreadLocal; - -import static accord.utils.Invariants.createIllegalState; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.PHASE_HLC_FIFO; -import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER; -import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; -import static org.apache.cassandra.service.accord.AccordCache.registerJfrListener; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordTask.State.LOADING; -import static org.apache.cassandra.service.accord.AccordTask.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordTask.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; - -/** - * NOTE: We assume that NO BLOCKING TASKS are submitted to this executor AND WAITED ON by another task executing on this executor. - * (as we do not immediately schedule additional threads for submitted tasks, but schedule new threads only if necessary when the submitting execution completes) - */ -public abstract class AccordExecutor implements CacheSize, LoadExecutor, Boolean>, SaveExecutor, Shutdownable, AbstractAsyncExecutor -{ - private static final Logger logger = LoggerFactory.getLogger(AccordExecutor.class); - - private static final long PRIORITY_BITS = 0x7000000000000000L; - private static final QueuePriorityModel PRIORITY_MODEL = DatabaseDescriptor.getAccord().queue_priority_model; - private static final long AGE_TO_FIFO = DatabaseDescriptor.getAccord().queue_priority_age_to_fifo.to(TimeUnit.MICROSECONDS); - public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); - private static final FastThreadLocal paranoidPriorityInversionCheck = new FastThreadLocal<>(); - - public interface AccordExecutorFactory - { - AccordExecutor get(int executorId, Mode mode, int threads, IntFunction name, Agent agent); - } - - public enum Mode { RUN_WITH_LOCK, RUN_WITHOUT_LOCK } - - // WARNING: this is a shared object, so close is NOT idempotent - public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable - { - final AccordExecutor executor; - - public ExclusiveGlobalCaches(AccordExecutor executor, AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) - { - super(global, commands, commandsForKey); - this.executor = executor; - } - - @Override - public void close() - { - executor.beforeUnlockExternal(); - global.tryShrinkOrEvict(executor.lock); - executor.unlock(); - } - } - - public static class GlobalCaches - { - public final AccordCache global; - public final AccordCache.Type commands; - public final AccordCache.Type commandsForKey; - - public GlobalCaches(AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) - { - this.global = global; - this.commands = commands; - this.commandsForKey = commandsForKey; - } - } - - private static class WaitForCompletion - { - final long position; - long maybeNotify; - final Runnable run; - - private WaitForCompletion(long position, Runnable run) - { - this.position = position; - this.maybeNotify = position - 1; - this.run = run; - } - - public String toString() - { - return run.toString() + " @" + position; - } - } - - private final Lock lock; - final Agent agent; - final int executorId; - private final AccordCache cache; - - private final TaskQueue> scanningRanges = new TaskQueue<>(SCANNING_RANGES); // never queried, just parked here while scanning - private final TaskQueue> loading = new TaskQueue<>(LOADING); // never queried, just parked here while loading - private final TaskQueue running = new TaskQueue<>(RUNNING); - - private final TaskQueue> waitingToLoadRangeTxns = new TaskQueue<>(WAITING_TO_LOAD); - private final TaskQueue> waitingToLoad = new TaskQueue<>(WAITING_TO_LOAD); - private final TaskQueue waitingToRun = new TaskQueue<>(WAITING_TO_RUN); - - private final ExclusiveGlobalCaches caches; - - private List waitingForQuiescence; - private Queue waitingForCompletion; - - final LogLinearDecayingHistograms histograms; - final LogLinearDecayingHistogram elapsedPreparingToRun; - final LogLinearDecayingHistogram elapsedWaitingToRun; - final LogLinearDecayingHistogram elapsedRunning; - final LogLinearDecayingHistogram elapsed; - final LogLinearDecayingHistogram keys; - public final AccordReplicaMetrics.Shard replicaMetrics; - - /** - * The maximum total number of loads we can queue at once - this includes loads for range transactions, - * which are subject to this limit as well as that imposed by {@link #maxQueuedRangeLoads} - */ - private int maxQueuedLoads = 64; - /** - * The maximum number of loads exclusively for range transactions we can queue at once; the {@link #maxQueuedLoads} limit also applies. - */ - private int maxQueuedRangeLoads = 8; - - private long maxWorkingSetSizeInBytes; - private long maxWorkingCapacityInBytes; - private long minPosition, nextPosition; - private int activeLoads, activeRangeLoads; - private boolean hasPausedLoading; - int tasks; - final DebugExecutor debug = DebugExecutor.maybeDebug(); - - AccordExecutor(Lock lock, int executorId, Agent agent) - { - this.lock = lock; - this.executorId = executorId; - this.cache = new AccordCache(this, 0); - this.agent = agent; - - final AccordCache.Type commands; - final AccordCache.Type commandsForKey; - commands = cache.newType(TxnId.class, COMMAND_ADAPTER, AccordCacheMetrics.CommandsCacheMetrics.newShard(lock)); - registerJfrListener(executorId, commands, "Command"); - - commandsForKey = cache.newType(RoutingKey.class, CFK_ADAPTER, AccordCacheMetrics.CommandsForKeyCacheMetrics.newShard(lock)); - registerJfrListener(executorId, commandsForKey, "CommandsForKey"); - - this.caches = new ExclusiveGlobalCaches(this, cache, commands, commandsForKey); - - DecayingHistogramsShard histogramsShard = HISTOGRAMS.newShard(lock); - this.histograms = histogramsShard.unsafeGetInternal(); - this.elapsedPreparingToRun = AccordExecutorMetrics.INSTANCE.elapsedPreparingToRun.forShard(histogramsShard); - this.elapsedWaitingToRun = AccordExecutorMetrics.INSTANCE.elapsedWaitingToRun.forShard(histogramsShard); - this.elapsedRunning = AccordExecutorMetrics.INSTANCE.elapsedRunning.forShard(histogramsShard); - this.elapsed = AccordExecutorMetrics.INSTANCE.elapsed.forShard(histogramsShard); - this.keys = AccordExecutorMetrics.INSTANCE.keys.forShard(histogramsShard); - this.replicaMetrics = new AccordReplicaMetrics.Shard(histogramsShard); - } - - public int executorId() - { - return executorId; - } - - public ExclusiveGlobalCaches lockCaches() - { - lock(); - return caches; - } - - abstract boolean isInLoop(); - - public final Lock unsafeLock() - { - return lock; - } - - final void lock() - { - if (Invariants.isParanoid()) paranoidLockExclusive(); - //noinspection LockAcquiredButNotSafelyReleased - lock.lock(); - if (DEBUG_EXECUTION) debug.onEnterLock(); - } - - final void unlock() - { - if (Invariants.isParanoid()) paranoidUnlockExclusive(); - if (DEBUG_EXECUTION) debug.onExitLock(); - lock.unlock(); - } - - final void paranoidLockExclusive() - { - Lock locked = paranoidPriorityInversionCheck.getAndSet(lock); - Invariants.require(locked == null || locked == lock, "Tried to take multiple AccordExecutor locks on same thread - this is dangerous for progress"); - } - - final void paranoidUnlockExclusive() - { - paranoidPriorityInversionCheck.set(null); - } - - final boolean tryLock() - { - return onTryLock(lock.tryLock()); - } - - final boolean onTryLock(boolean result) - { - if (DEBUG_EXECUTION && result) debug.onEnterLock(); - if (Invariants.isParanoid()) - { - if (result) - { - Lock locked = paranoidPriorityInversionCheck.getAndSet(lock); - if (locked != null && locked != lock) - { - lock.unlock(); - paranoidPriorityInversionCheck.set(locked); - Invariants.require(false, "Tried to take multiple AccordExecutor locks on same thread - this is dangerous for progress"); - return false; - } - } - else - { - Lock locked = paranoidPriorityInversionCheck.get(); - Invariants.require(locked == null || locked == lock, "Tried to take multiple AccordExecutor locks on same thread - this is dangerous for progress"); - } - } - return result; - } - - public AccordCache cacheExclusive() - { - Invariants.require(isOwningThread()); - return cache; - } - - public AccordCache cacheUnsafe() - { - return cache; - } - - final boolean hasWaitingToRun() - { - updateWaitingToRunExclusive(); - return hasAlreadyWaitingToRun(); - } - - final boolean hasAlreadyWaitingToRun() - { - return !waitingToRun.isEmpty(); - } - - void updateWaitingToRunExclusive() - { - // TODO (expected): this should not be invoked on every update of waiting to run - maybeUnpauseLoading(); - } - - final Task pollWaitingToRunExclusive() - { - updateWaitingToRunExclusive(); - return pollAlreadyWaitingToRunExclusive(); - } - - final Task pollAlreadyWaitingToRunExclusive() - { - Task next = waitingToRun.poll(); - if (next != null) - { - if (DEBUG_EXECUTION) DebugTask.get(next).onPolled(); - next.addToQueue(running); - } - return next; - } - - public Stream active() - { - return Stream.of(); - } - - public void waitForQuiescence() - { - Condition condition; - lock(); - try - { - if (tasks == 0) - return; - - if (waitingForQuiescence == null) - waitingForQuiescence = new ArrayList<>(); - condition = Condition.newOneTimeCondition(); - waitingForQuiescence.add(condition); - } - finally - { - unlock(); - } - condition.awaitThrowUncheckedOnInterrupt(); - } - - protected void notifyQuiescentExclusive() - { - if (waitingForQuiescence != null) - { - waitingForQuiescence.forEach(Condition::signalAll); - waitingForQuiescence = null; - } - if (waitingForCompletion != null) - { - logger.warn("{} processed all pending tasks (<{}) but found waiting: {}", this, nextPosition, waitingForCompletion); - waitingForCompletion.forEach(w -> w.run.run()); - waitingForCompletion = null; - } - } - - public void afterSubmittedAndConsequences(Runnable run) - { - lock(); - try - { - if (tasks == 0) - { - run.run(); - return; - } - - if (waitingForCompletion != null) // escape hatch for some bug that means we lose a notification for a given task's queue position - maybeNotifyWaitingForCompletion(); - if (waitingForCompletion == null) - waitingForCompletion = new ArrayDeque<>(); - - long position = nextPosition; - minPosition = position; - waitingForCompletion.add(new WaitForCompletion(position, run)); - } - finally - { - unlock(); - } - } - - void maybeUnpauseLoading() - { - if (!hasPausedLoading) - return; - - if (cache.weightedSize() < maxWorkingCapacityInBytes || (loading.isEmpty() && waitingToRun.isEmpty())) - { - hasPausedLoading = false; - enqueueLoadsExclusive(); - } - } - - public abstract boolean hasTasks(); - abstract void beforeUnlockExternal(); - abstract boolean isOwningThread(); - - private void enqueueLoadsExclusive() - { - outer: while (true) - { - TaskQueue> queue = waitingToLoadRangeTxns.isEmpty() || activeRangeLoads >= maxQueuedRangeLoads ? waitingToLoad : waitingToLoadRangeTxns; - AccordTask next = queue.peek(); - if (next == null) - return; - - if (hasPausedLoading || cache.weightedSize() >= maxWorkingCapacityInBytes) - { - // we have too much in memory already, and we have work waiting to run, so let that complete before queueing more - if (!loading.isEmpty() || !waitingToRun.isEmpty()) - { - AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); - hasPausedLoading = true; - return; - } - } - - switch (next.state()) - { - default: - { - failExclusive(next, createIllegalState("Unexpected state: " + next.toDescription())); - break; - } - case WAITING_TO_SCAN_RANGES: - if (activeRangeLoads >= maxQueuedRangeLoads) - { - parkRangeLoad(next); - } - else - { - ++activeRangeLoads; - ++activeLoads; - next.rangeScanner().start(this); - updateQueue(next); - } - break; - - case WAITING_TO_LOAD: - while (true) - { - AccordCacheEntry load = next.peekWaitingToLoad(); - boolean isForRange = isForRange(next, load); - if (isForRange && activeRangeLoads >= maxQueuedRangeLoads) - { - parkRangeLoad(next); - continue outer; - } - - Invariants.require(load != null); - ++activeLoads; - if (isForRange) - ++activeRangeLoads; - - for (AccordTask task : cache.load(this, next, isForRange, load)) - { - if (task == next) continue; - if (task.onLoading(load)) - updateQueue(task); - } - Object prev = next.pollWaitingToLoad(); - Invariants.require(prev == load); - if (next.peekWaitingToLoad() == null) - break; - - Invariants.require(next.state() == WAITING_TO_LOAD, "Invalid state: %s", next); - if (activeLoads >= maxQueuedLoads) - return; - } - Invariants.require(next.state().compareTo(LOADING) >= 0, "Invalid state: %s", next); - updateQueue(next); - } - } - } - - private boolean isForRange(AccordTask task, AccordCacheEntry load) - { - boolean isForRangeTxn = task.hasRanges(); - if (!isForRangeTxn) - return false; - - for (AccordTask t : load.loadingOrWaiting().waiters()) - { - if (!t.hasRanges()) - return false; - } - return true; - } - - @Override - public Cancellable execute(RunOrFail runOrFail) - { - PlainChain submit = new PlainChain(runOrFail); - return submit(submit); - } - - public AsyncChain buildDebuggable(Callable task, Object describe) - { - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return submit(new DebuggableChain(new CallAndCallback<>(task, callback), null, 0, describe)); - } - }; - } - - private void parkRangeLoad(AccordTask task) - { - if (task.queued() != waitingToLoadRangeTxns) - { - task.unqueueIfQueued(); - task.addToQueue(waitingToLoadRangeTxns); - } - } - - private void updateQueue(AccordTask task) - { - task.unqueueIfQueued(); - switch (task.state()) - { - default: throw new AssertionError("Unexpected state: " + task.toDescription()); - case WAITING_TO_SCAN_RANGES: - case WAITING_TO_LOAD: - task.addToQueue(waitingToLoad); - break; - case SCANNING_RANGES: - task.addToQueue(scanningRanges); - break; - case LOADING: - task.addToQueue(loading); - break; - case WAITING_TO_RUN: - waitingToRun(task); - break; - } - } - - private void waitingToRun(AccordTask task) - { - task.onWaitingToRun(); - task.addToQueue(task.commandStore.exclusiveExecutor); - } - - private void waitingToRun(Task task, @Nullable SequentialExecutor queue) - { - task.onWaitingToRun(); - task.addToQueue(queue == null ? waitingToRun : queue); - } - - public SequentialExecutor executor() - { - return new SequentialExecutor(this); - } - - public SequentialExecutor executor(int commandStoreId) - { - return new SequentialExecutor(this, commandStoreId); - } - - public SequentialAsyncExecutor newSequentialExecutor() - { - return new SequentialExecutor(this); - } - - public void cancel(AccordTask task) - { - Invariants.require(task.commandStore.executor() == this, - "%s is a wrong command store for %s, should be %s", - this, task, task); - submit(AccordExecutor::cancelExclusive, CancelTask::new, task); - } - - @Override - public Cancellable load(AccordTask parent, Boolean isForRange, AccordCacheEntry entry) - { - return submitPlainExclusive(parent, newLoad(entry, isForRange)); - } - - @Override - public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) - { - return submitPlainExclusive(null, new SaveRunnable(entry, identity, save)); - } - - private void submit(BiConsumer sync, Function async, P1 p1) - { - submit((e, c, p1a, p2a, p3) -> c.accept(e, p1a), (f, p1a, p2a, p3) -> f.apply(p1a), sync, async, p1, null, null); - } - - private void submit(TriConsumer sync, BiFunction async, P1 p1, P2 p2) - { - submit((e, c, p1a, p2a, p3) -> c.accept(e, p1a, p2a), (f, p1a, p2a, p3) -> f.apply(p1a, p2a), sync, async, p1, p2, null); - } - - private void submit(QuadConsumer sync, TriFunction async, P1 p1, P2 p2, P3 p3) - { - submit((e, c, p1a, p2a, p3a) -> c.accept(e, p1a, p2a, p3a), TriFunction::apply, sync, async, p1, p2, p3); - } - - private void submit(QuintConsumer sync, QuadFunction async, P1 p1, P2 p2, P3 p3, P4 p4) - { - submit(sync, async, p1, p1, p2, p3, p4); - } - - abstract void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4); - - void submit(AccordTask operation) - { - submit(AccordExecutor::submitExclusive, i -> i, operation); - } - - void submitPriority(AccordTask operation) - { - submit(AccordExecutor::submitPriorityExclusive, i -> i, operation); - } - - void submitExclusive(AccordTask task) - { - assignQueuePosition(task); - submitInternalExclusive(task); - } - - void submitPriorityExclusive(AccordTask task) - { - assignMinQueuePosition(task); - submitInternalExclusive(task); - } - - private void submitInternalExclusive(AccordTask task) - { - task.setupExclusive(); - ++tasks; - updateQueue(task); - enqueueLoadsExclusive(); - } - - public void submitExclusive(Runnable runnable) - { - submitPlainExclusive(new PlainRunnable(null, runnable)); - } - - private void submitPlainExclusive(Plain task) - { - ++tasks; - assignQueuePosition(task); - waitingToRun(task, task.executor()); - } - - Cancellable submitPlainExclusive(Task parent, AbstractIOTask task) - { - return submitPlainExclusive(parent, new WrappedIOTask(task)); - } - - T submitPlainExclusive(Task parent, T task) - { - Invariants.require(isOwningThread()); - ++tasks; - if (parent != null) inheritQueuePosition(parent, task); - else assignFifoQueuePosition(task); - task.onWaitingToRun(); - waitingToRun.append(task); - return task; - } - - private void assignQueuePosition(Task task) - { - if (task.queuePosition != 0) updateNextPosition(task); - else assignFifoQueuePosition(task); - } - - private void assignQueuePosition(AccordTask task) - { - if (task.queuePosition != 0) updateNextPosition(task); - else - { - long priority_bits = PRIORITY_BITS; - TxnId txnId = null; - switch (PRIORITY_MODEL) - { - case ORIG_PHASE_HLC_FIFO: - case PHASE_HLC_FIFO: - { - // TODO (expected): we should process messages for a TxnId together, to avoid processing delayed messages out of order - PreLoadContext context = task.preLoadContext(); - if (context instanceof Request) - { - MessageType type = ((Request) context).type(); - if (type instanceof StandardMessage) - { - TxnId txnId0 = context.primaryTxnId(); - switch ((StandardMessage)type) - { - case APPLY_REQ: - { - priority_bits = 0L; - txnId = txnId0; - break; - } - case READ_EPHEMERAL_REQ: - case READ_REQ: - case STABLE_THEN_READ_REQ: - { - priority_bits = 1000000000000000L; - txnId = txnId0; - break; - } - case COMMIT_REQ: - { - Commit commit = (Commit) context; - if (PRIORITY_MODEL == PHASE_HLC_FIFO || commit.ballot.equals(Ballot.ZERO)) - txnId = commit.txnId; - if (commit.kind.saveStatus == SaveStatus.Stable) priority_bits = 1000000000000L; - else priority_bits = 2000000000000L; - break; - } - case ACCEPT_REQ: - { - Accept accept = (Accept) context; - if (PRIORITY_MODEL == PHASE_HLC_FIFO || accept.ballot.equals(Ballot.ZERO)) - txnId = accept.txnId; - priority_bits = 3000000000000L; - break; - } - case GET_EPHEMERAL_READ_DEPS_REQ: - case PRE_ACCEPT_REQ: - { - txnId = txnId0; - break; - } - } - } - } - break; - } - case HLC_FIFO: - { - txnId = task.preLoadContext().primaryTxnId(); - break; - } - case FIFO: - { - break; - } - } - - if (txnId != null) - { - long hlc = txnId.hlc(); - long delta = nextPosition - hlc; - if (delta < AGE_TO_FIFO) - { - long position = hlc; - if (delta <= 0) nextPosition = position + 1; - else if (position < minPosition) position = minPosition; - position |= priority_bits; - task.queuePosition = position; - return; - } - } - - assignFifoQueuePosition(task); - } - - } - - private void assignMinQueuePosition(Task task) - { - task.queuePosition = minPosition | PRIORITY_BITS; - } - - private void assignFifoQueuePosition(Task task) - { - task.queuePosition = nextPosition++ | PRIORITY_BITS; - } - - private void updateNextPosition(Task task) - { - nextPosition = Math.max(nextPosition, (task.queuePosition & ~PRIORITY_BITS) + 1); - } - - private void inheritQueuePosition(Task parent, Task task) - { - task.queuePosition = parent.queuePosition; - } - - void completeTaskExclusive(Task task) - { - // for integration with SequentialExecutor, we must : - // - first take the position so that represents the just-executed task - // - call cleanup to submit any following task on the relevant sub-queue - // - remove the previous task from the running collection only if still present (SequentialExecutor will have removed it) - long position = task.queuePosition; - try - { - task.cleanupExclusive(this); - } - finally - { - --tasks; - if (running.contains(task)) - running.remove(task); - - if (waitingForCompletion != null && waitingForCompletion.peek().maybeNotify <= position) - maybeNotifyWaitingForCompletion(); - - cache.tryShrinkOrEvict(lock); - } - } - - private void maybeNotifyWaitingForCompletion() - { - long min = minPosition(waitingToRun.peek(), - minPosition(waitingToLoad.peek(), - minPosition(waitingToLoadRangeTxns.peek(), - minPosition(running.peek(), - minPosition(loading.peek(), - minPosition(scanningRanges.peek(), Long.MAX_VALUE)))))); - - while (!waitingForCompletion.isEmpty() && waitingForCompletion.peek().position - min <= 0) - waitingForCompletion.poll().run.run(); - if (waitingForCompletion.isEmpty()) - waitingForCompletion = null; - else - waitingForCompletion.peek().maybeNotify = min; - } - - private static long minPosition(@Nullable Task task, long min) - { - return task == null ? min : Long.min(task.queuePosition, min); - } - - void cancelExclusive(AccordTask task) - { - AccordTask.State state = task.state(); - switch (state) - { - default: throw new UnhandledEnum(state); - case SCANNING_RANGES: - case LOADING: - case WAITING_TO_LOAD: - case WAITING_TO_SCAN_RANGES: - case WAITING_TO_RUN: - task.unqueueIfQueued(); - try { task.cancelExclusive(); } - finally { completeTaskExclusive(task); } - break; - - case INITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase - case ASSIGNED: - case RUNNING: - case PERSISTING: - case FINISHED: - case CANCELLED: - case FAILED: - // cannot safely cancel - } - } - - void onScannedRangesExclusive(AccordTask task, Throwable fail) - { - --activeLoads; - --activeRangeLoads; - // the task may have already been cancelled, in which case we don't need to fail it - if (!task.state().isExecuted()) - { - if (fail != null) - { - failExclusive(task, fail); - } - else - { - task.rangeScanner().scannedExclusive(); - updateQueue(task); - } - } - enqueueLoadsExclusive(); - } - - private void failExclusive(AccordTask task, Throwable fail) - { - if (task.state().isExecuted()) - return; - - try { task.failExclusive(fail); } - catch (Throwable t) { agent.onException(t); } - finally - { - task.unqueueIfQueued(); - completeTaskExclusive(task); - } - } - - private void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) - { - cache.saved(state, identity, fail); - } - - private void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail, boolean isForRange) - { - --activeLoads; - if (isForRange) - --activeRangeLoads; - - if (loaded.status() != EVICTED) - { - try (BufferList> tasks = loaded.loading().copyWaiters()) - { - if (fail != null) - { - for (AccordTask task : tasks) - failExclusive(task, fail); - cache.failedToLoad(loaded); - } - else - { - cache.loaded(loaded, value); - for (AccordTask task : tasks) - { - if (task.onLoad(loaded)) - { - Invariants.require(task.queued() == loading); - task.unqueue(); - waitingToRun(task); - } - } - } - } - } - - enqueueLoadsExclusive(); - } - - public Future submit(Runnable run) - { - PlainRunnable task = new PlainRunnable(new AsyncPromise<>(), run); - submit(task); - return task.result; - } - - public void execute(Runnable command) - { - submit(new PlainRunnable(null, command)); - } - - private Cancellable submit(Plain task) - { - submit(AccordExecutor::submitPlainExclusive, i -> i, task); - return task; - } - - public void executeDirectlyWithLock(Runnable command) - { - lock(); - try - { - command.run(); - } - finally - { - beforeUnlockExternal(); - unlock(); - } - } - - @Override - public void setCapacity(long bytes) - { - Invariants.require(isOwningThread()); - cache.setCapacity(bytes); - maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; - } - - public void setWorkingSetSize(long bytes) - { - Invariants.require(isOwningThread()); - maxWorkingSetSizeInBytes = bytes; - maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; - if (maxWorkingCapacityInBytes < maxWorkingSetSizeInBytes) - maxWorkingCapacityInBytes = Long.MAX_VALUE; - } - - public void setMaxQueuedLoads(int total, int range) - { - Invariants.require(isOwningThread()); - Invariants.requireArgument(total >= 1, "Must permit at least one load"); - Invariants.requireArgument(range >= 1, "Must permit at least one range load"); - maxQueuedLoads = total; - maxQueuedRangeLoads = range; - } - - @Override - public long capacity() - { - return cache.capacity(); - } - - @Override - public int size() - { - return cache.size(); - } - - @Override - public long weightedSize() - { - return cache.weightedSize(); - } - - protected static abstract class TaskRunner implements DebuggableTaskRunner - { - // TODO (desired): this probably doesn't need to be volatile - private volatile Task running; - private static final AtomicReferenceFieldUpdater runningUpdater = AtomicReferenceFieldUpdater.newUpdater(TaskRunner.class, Task.class, "running"); - - @Override - public DebuggableTask running() - { - Task running = this.running; - return running == null ? null : running.debuggable(); - } - - Task runningTask() - { - return running; - } - - void setRunning(Task debuggable) - { - runningUpdater.lazySet(this, debuggable); - } - - void clearRunning() - { - runningUpdater.lazySet(this, null); - } - } - - public static abstract class Task extends IntrusivePriorityHeap.Node - { - public final WithResources resources; - Task next; - long queuePosition; - public long createdAt = nanoTime(), waitingToRunAt, runningAt, cleanupAt; - - protected Task() - { - resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - } - - public final Task unwrap() - { - if (this instanceof SequentialQueueTask) - return ((SequentialQueueTask) this).queue.task; - return this; - } - - final void setReadyToCleanup() - { - queuePosition |= Long.MIN_VALUE; - } - - final boolean isReadyToCleanup() - { - return 0 != (queuePosition & Long.MIN_VALUE); - } - - final void onRunning() - { - runningAt = nanoTime(); - if (DEBUG_EXECUTION) ((DebugTask)resources).onRunning(); - } - - final void onRunComplete() - { - if (DEBUG_EXECUTION) ((DebugTask)resources).onRunComplete(); - } - - final void onWaitingToRun() - { - waitingToRunAt = nanoTime(); - } - - static Task reverse(Task unqueued) - { - Task prev = null; - Task cur = unqueued; - while (cur != null) - { - Task next = cur.next; - cur.next = prev; - prev = cur; - cur = next; - } - return prev; - } - - public DebuggableTask debuggable() { return null; } - - abstract void submitExclusive(AccordExecutor owner); - - /** - * Prepare to run while holding the state cache lock - */ - abstract protected void preRunExclusive(); - - /** - * Run the command; the state cache lock may or may not be held depending on the executor implementation - */ - protected abstract void runInternal(); - - /** - * Fail the command; the state cache lock may or may not be held depending on the executor implementation - */ - abstract protected void fail(Throwable fail); - - /** - * Cleanup the command while holding the state cache lock - */ - protected void cleanupExclusive(AccordExecutor executor) - { - cleanupAt = nanoTime(); - if (runningAt != 0) - { - if (waitingToRunAt == 0) - waitingToRunAt = runningAt; - executor.elapsedWaitingToRun.increment(runningAt - waitingToRunAt, runningAt); - executor.elapsedPreparingToRun.increment(waitingToRunAt - createdAt, runningAt); - executor.elapsedRunning.increment(cleanupAt - runningAt, cleanupAt); - executor.elapsed.increment(cleanupAt - createdAt, cleanupAt); - } - if (DEBUG_EXECUTION) DebugTask.get(this).onCompleted(executor.debug); - } - - void cancelExclusive(AccordExecutor owner) {} - - abstract protected void addToQueue(TaskQueue queue); - } - - // run the task even on a stopped commandStore - public interface Unstoppable extends PreLoadContext.Empty - { - } - - // run the task even on a terminated commandStore - public interface Unterminatable extends Unstoppable - { - } - - static final class SequentialQueueTask extends Task - { - private final SequentialExecutor queue; - - SequentialQueueTask(SequentialExecutor queue) - { - super(); - this.queue = queue; - } - - @Override void submitExclusive(AccordExecutor owner) { throw new UnsupportedOperationException(); } - - @Override - protected void preRunExclusive() - { - queue.preRunTask(); - } - - @Override - protected void runInternal() - { - queue.runTask(); - } - - @Override - protected void fail(Throwable t) - { - queue.failTask(t); - } - - @Override - protected void cleanupExclusive(AccordExecutor executor) - { - queue.cleanupTask(); - } - - @Override - protected void addToQueue(TaskQueue queue) - { - Invariants.require(queue.kind == RUNNING); - queue.append(this); - } - - protected boolean isInHeap() - { - return super.isInHeap(); - } - } - - private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(SequentialExecutor.class, Thread.class, "owner"); - public class SequentialExecutor extends TaskQueue implements SequentialAsyncExecutor - { - final int commandStoreId; - final SequentialQueueTask selfTask; - private Task task; - private volatile Thread owner, waiting; - private boolean stopped; - private volatile boolean visibleStopped; - private boolean terminated; - - final DebugSequentialExecutor debug; - - SequentialExecutor(AccordExecutor executor) - { - this(executor, -1); - } - - SequentialExecutor(AccordExecutor executor, int commandStoreId) - { - super(WAITING_TO_RUN, commandStoreId < 0); - this.commandStoreId = commandStoreId; - this.selfTask = new SequentialQueueTask(this); - this.debug = DebugSequentialExecutor.maybeDebug(executor.debug, commandStoreId); - } - - void preRunTask() - { - Invariants.require(task != null); - task.preRunExclusive(); - } - - void runTask() - { - Thread self = Thread.currentThread(); - if (!ownerUpdater.compareAndSet(this, null, self)) - { - if (DEBUG_EXECUTION) debug.onWaiting(); - Invariants.require(self == Thread.currentThread()); - waiting = self; - outer: do - { - while (true) - { - Thread owner = this.owner; - if (owner == self) break outer; - if (owner == null) continue outer; - LockSupport.park(); - } - } - while (!ownerUpdater.compareAndSet(this, null, self)); - } - waiting = null; - - if (stopped && reject(task)) - task.fail(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).preLoadContext())); - else - task.runInternal(); - // NOTE: cannot safely release owner here, in case an immediate-execution runs before we can release our references and store their changes to the cache - } - - private boolean reject(Task task) - { - if (!(task instanceof AccordTask)) - return true; - - PreLoadContext context = ((AccordTask) task).preLoadContext(); - - return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); - } - - void failTask(Throwable t) - { - task.fail(t); - } - - void cleanupTask() - { - try { task.cleanupExclusive(AccordExecutor.this); } - finally - { - owner = null; - task = super.poll(); - if (DEBUG_EXECUTION) debug.onSetTask(task); - - // it should only be possible for this method to be invoked once we're on the running queue - AccordExecutor.this.running.remove(selfTask); - if (task != null) - { - selfTask.queuePosition = task.queuePosition; - waitingToRun.append(selfTask); - } - } - } - - // invoked by removeAndUpdateNext; expect to already be next - @Override - protected void append(Task newTask) - { - if (task != null) - { - Invariants.require(selfTask.isInHeap()); - super.append(newTask); - } - else - { - Invariants.require(isEmpty()); - task = newTask; - selfTask.queuePosition = newTask.queuePosition; - waitingToRun.append(selfTask); - if (DEBUG_EXECUTION) debug.onSetTask(newTask); - } - } - - @Override - protected void remove(Task remove) - { - if (remove == task) removeCurrentTask(remove); - else super.remove(remove); - } - - @Override - protected boolean removeIfContains(Task remove) - { - if (remove == task) return removeCurrentTask(remove); - else return super.removeIfContains(remove); - } - - private boolean removeCurrentTask(Node remove) - { - if (running.contains(selfTask)) - return false; - - Invariants.require(remove == task); - // cannot overwrite task while it is being executed - this cannot happen for AccordTask - // but can for other tasks that don't track their own state - - task = super.poll(); - if (DEBUG_EXECUTION) debug.onSetTask(task); - if (waitingToRun.contains(selfTask)) - { - if (task == null) waitingToRun.remove(selfTask); - else - { - selfTask.queuePosition = task.queuePosition; - waitingToRun.update(selfTask); - } - } - else - { - Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove); - if (task != null) - { - selfTask.queuePosition = task.queuePosition; - waitingToRun.append(selfTask); - } - } - Invariants.require(task == null || waitingToRun.contains(selfTask)); - return true; - } - - public boolean inExecutor() - { - return owner == Thread.currentThread(); - } - - public boolean stopped() - { - return visibleStopped; - } - - void stop() - { - Invariants.require(inExecutor()); - this.stopped = true; - this.visibleStopped = true; - } - - void terminate() - { - Invariants.require(inExecutor()); - this.visibleStopped = this.terminated = this.stopped = true; - } - - @Override - protected Task poll() - { - throw new UnsupportedOperationException(); - } - - @Override - protected Task peek() - { - throw new UnsupportedOperationException(); - } - - @Override - protected boolean contains(Task contains) - { - return super.contains(contains) || (task == contains && !running.contains(selfTask)); - } - - @Override - public AsyncChain chain(Runnable run) - { - long position = inheritQueuePosition(); - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return execute(new RunAndCallback(run, callback), position); - } - }; - } - - @Override - public AsyncChain chain(Callable call) - { - long position = inheritQueuePosition(); - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return execute(new CallAndCallback<>(call, callback), position); - } - }; - } - - @Override - public AsyncChain flatChain(Callable> call) - { - long position = inheritQueuePosition(); - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return execute(new FlatCallAndCallback<>(call, callback), position); - } - }; - } - - @Override - public Cancellable execute(RunOrFail runOrFail) - { - return execute(runOrFail, inheritQueuePosition()); - } - - private long inheritQueuePosition() - { - return inExecutor() && task != null ? task.queuePosition : 0; - } - - private Cancellable execute(RunOrFail runOrFail, long queuePosition) - { - PlainChain submit = new PlainChain(runOrFail, SequentialExecutor.this, queuePosition); - return AccordExecutor.this.submit(submit); - } - - @Override - public void execute(Runnable run) - { - PlainRunnable submit = new PlainRunnable(null, run, this, inheritQueuePosition()); - AccordExecutor.this.submit(submit); - } - - @Override - public boolean tryExecuteImmediately(Runnable run) - { - Thread self = Thread.currentThread(); - Thread owner = this.owner; - if (owner != null ? owner != self : !ownerUpdater.compareAndSet(this, null, self)) - return false; - - try { run.run(); } - catch (Throwable t) { agent.onException(t); } - finally - { - if (owner == null) - { - Thread waiting = this.waiting; - Invariants.require(waiting != self); - this.owner = waiting; - if (waiting == null) // recheck, to ensure happens-before relation with a new waiter that expects any non-null owner to notify it - waiting = this.waiting; - if (waiting != null) - LockSupport.unpark(waiting); - } - } - return true; - } - } - - static class TaskQueue extends IntrusivePriorityHeap - { - final AccordTask.State kind; - - TaskQueue(AccordTask.State kind) - { - this.kind = kind; - } - - TaskQueue(AccordTask.State kind, boolean tiny) - { - super(tiny); - this.kind = kind; - } - - @Override - public int compare(T o1, T o2) - { - return Long.compare(o1.queuePosition, o2.queuePosition); - } - - protected void append(T task) - { - super.append(task); - } - - protected void update(T task) - { - super.update(task); - } - - protected T poll() - { - ensureHeapified(); - return pollNode(); - } - - protected T peek() - { - ensureHeapified(); - return peekNode(); - } - - protected T get(int index) - { - return super.get(index); - } - - protected void remove(T remove) - { - super.remove(remove); - } - - @Override - protected boolean removeIfContains(T node) - { - return super.removeIfContains(node); - } - - protected boolean contains(T contains) - { - return super.contains(contains); - } - } - - static class CancelTask extends Task - { - final Task cancel; - private CancelTask(Task cancel) { this.cancel = cancel; } - @Override void submitExclusive(AccordExecutor owner) { cancel.cancelExclusive(owner); } - @Override protected void preRunExclusive() { throw new UnsupportedOperationException(); } - @Override protected void runInternal() { throw new UnsupportedOperationException(); } - @Override protected void fail(Throwable fail) { throw new UnsupportedOperationException(); } - @Override protected void addToQueue(TaskQueue queue) { throw new UnsupportedOperationException(); } - } - - static IntFunction constant(O out) - { - return ignore -> out; - } - - abstract class Plain extends Task implements Cancellable - { - abstract SequentialExecutor executor(); - - @Override - protected void preRunExclusive() {} - - @Override - protected final void addToQueue(TaskQueue queue) - { - Invariants.require(queue.kind == WAITING_TO_RUN || queue.kind == RUNNING); - queue.append(this); - } - - @Override - public void cancel() - { - submit((e, c) -> c.cancelExclusive(e), CancelTask::new, this); - } - - void cancelExclusive(AccordExecutor owner) - { - SequentialExecutor executor = executor(); - TaskQueue queue = executor == null ? waitingToRun : executor; - if (queue.contains(this)) - { - queue.remove(this); - completeTaskExclusive(this); - try { fail(new CancellationException()); } - catch (Throwable t) { agent.onException(t); } - } - } - - @Override - final void submitExclusive(AccordExecutor owner) - { - owner.submitPlainExclusive(this); - } - } - - class PlainRunnable extends Plain implements Cancellable - { - final @Nullable AsyncPromise result; - final Runnable run; - final @Nullable SequentialExecutor executor; - - PlainRunnable(Runnable run) - { - this(null, run); - } - - PlainRunnable(AsyncPromise result, Runnable run) - { - this(result, run, null, 0); - } - - PlainRunnable(AsyncPromise result, Runnable run, @Nullable SequentialExecutor executor, long queuePosition) - { - this.result = result; - this.run = run; - this.executor = executor; - this.queuePosition = queuePosition; - } - - @Override - protected void runInternal() - { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - if (result != null) - result.trySuccess(null); - onRunComplete(); - } - - @Override - protected void fail(Throwable t) - { - if (result != null) - result.tryFailure(t); - agent.onException(t); - } - - @Override - SequentialExecutor executor() - { - return executor; - } - } - - // a task that may be submitted to this executor or another - abstract class IOTask extends Plain implements Cancellable, DebuggableTask - { - final long createdAtNanos = MonotonicClock.Global.approxTime.now(); - long startedAtNanos; - - abstract void postRunExclusive(); - - @Override - protected void preRunExclusive() - { - startedAtNanos = MonotonicClock.Global.approxTime.now(); - } - - @Override - protected void cleanupExclusive(AccordExecutor executor) - { - super.cleanupExclusive(executor); - postRunExclusive(); - } - - @Override - SequentialExecutor executor() - { - return null; - } - - @Override - public long creationTimeNanos() - { - return createdAtNanos; - } - - @Override - public long startTimeNanos() - { - return startedAtNanos; - } - } - - static class FailureHolder - { - static final FailureHolder NOT_STARTED = new FailureHolder(new RuntimeException("Not started")); - - final Throwable fail; - - FailureHolder(Throwable fail) - { - this.fail = fail; - } - } - - LoadRunnable newLoad(AccordCacheEntry entry, boolean isForRange) - { - return isForRange ? new LoadRangeRunnable<>(entry) : new LoadRunnable<>(entry); - } - - class LoadRunnable extends IOTask - { - final AccordCacheEntry entry; - Object result = FailureHolder.NOT_STARTED; - - LoadRunnable(AccordCacheEntry entry) - { - this.entry = entry; - } - - boolean isForRange() { return false; } - - void postRunExclusive() - { - if (!(result instanceof FailureHolder)) onLoadedExclusive(entry, (V)result, null, isForRange()); - else onLoadedExclusive(entry, null, ((FailureHolder)result).fail, isForRange()); - } - - @Override - public void runInternal() - { - onRunning(); - try (Closeable close = resources.get()) - { - result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); - } - onRunComplete(); - } - - @Override - protected void fail(Throwable t) - { - result = new FailureHolder(t); - } - - @Override - public String description() - { - return "Loading " + entry; - } - } - - final class LoadRangeRunnable extends LoadRunnable - { - LoadRangeRunnable(AccordCacheEntry entry) { super(entry); } - @Override boolean isForRange() { return true; } - } - - static abstract class AbstractIOTask - { - abstract protected void runInternal(); - abstract protected void postRunExclusive(); - abstract protected void fail(Throwable t); - abstract protected String description(); - } - - class WrappedIOTask extends IOTask - { - final AbstractIOTask wrapped; - - WrappedIOTask(AbstractIOTask wrap) - { - this.wrapped = wrap; - } - - @Override - protected void runInternal() - { - onRunning(); - try (Closeable close = resources.get()) - { - wrapped.runInternal(); - } - onRunComplete(); - } - - @Override - void postRunExclusive() - { - wrapped.postRunExclusive(); - } - - @Override - public String description() - { - return wrapped.description(); - } - - @Override - protected void fail(Throwable fail) - { - wrapped.fail(fail); - } - } - - private static final Throwable NOT_STARTED = new Throwable(); - class SaveRunnable extends IOTask - { - final AccordCacheEntry entry; - final UniqueSave identity; - final Runnable run; - Throwable failure = NOT_STARTED; - - SaveRunnable(AccordCacheEntry entry, UniqueSave identity, Runnable run) - { - this.entry = entry; - this.identity = identity; - this.run = run; - } - - @Override - void postRunExclusive() - { - onSavedExclusive(entry, identity, failure); - } - - @Override - public void runInternal() - { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - onRunComplete(); - failure = null; - } - - @Override - protected void fail(Throwable t) - { - failure = t; - } - - @Override - public String description() - { - return "Save " + entry; - } - } - - class PlainChain extends Plain - { - final RunOrFail runOrFail; - final @Nullable SequentialExecutor executor; - - PlainChain(RunOrFail runOrFail) - { - this(runOrFail, null, 0); - } - - PlainChain(RunOrFail runOrFail, @Nullable SequentialExecutor executor, long queuePosition) - { - this.runOrFail = runOrFail; - this.executor = executor; - this.queuePosition = queuePosition; - } - - @Override - SequentialExecutor executor() - { - return executor; - } - - @Override - protected void runInternal() - { - onRunning(); - try (Closeable close = resources.get()) - { - runOrFail.run(); - } - catch (Throwable t) - { - // shouldn't throw exceptions - agent.onException(t); - } - onRunComplete(); - } - - @Override - protected void fail(Throwable fail) - { - try - { - runOrFail.fail(fail); - } - catch (Throwable t) - { - fail.addSuppressed(t); - agent.onException(fail); - } - } - } - - class DebuggableChain extends PlainChain implements DebuggableTask - { - final long createdAtNanos; - long startedAtNanos; - final Object describe; - - DebuggableChain(RunOrFail runOrFail, @Nullable SequentialExecutor executor, int queuePosition, Object describe) - { - super(runOrFail, executor, queuePosition); - this.createdAtNanos = MonotonicClock.Global.approxTime.now(); - this.describe = Invariants.nonNull(describe); - } - - @Override - public long creationTimeNanos() - { - return createdAtNanos; - } - - @Override - public long startTimeNanos() - { - return startedAtNanos; - } - - @Override - protected void preRunExclusive() - { - startedAtNanos = MonotonicClock.Global.approxTime.now(); - } - - @Override - public String description() - { - return describe.toString(); - } - - @Override - public DebuggableTask debuggable() - { - return this; - } - } - - - public static class TaskInfo implements Comparable - { - // sorted in name order for reporting to virtual tables - public enum Status { LOADING, RUNNING, SCANNING_RANGES, WAITING_TO_LOAD, WAITING_TO_RUN } - - final Status status; - final int commandStoreId; - - final Task task; - - public TaskInfo(Status status, int commandStoreId, Task task) - { - this.status = status; - this.commandStoreId = commandStoreId; - this.task = task; - } - - public Status status() - { - return status; - } - - public Integer commandStoreId() - { - return commandStoreId >= 0 ? commandStoreId : null; - } - - public long position() - { - return task.queuePosition; - } - - public @Nullable String describe() - { - if (task instanceof AccordTask) - return ((AccordTask) task).preLoadContext().reason(); - - if (task instanceof DebuggableTask) - return ((DebuggableTask) task).description(); - - return null; - } - - public @Nullable PreLoadContext preLoadContext() - { - if (task instanceof AccordTask) - return ((AccordTask) task).preLoadContext(); - if (task instanceof WrappedIOTask && ((WrappedIOTask) task).wrapped instanceof AccordTask.RangeTxnScanner) - return ((AccordTask.RangeTxnScanner) ((WrappedIOTask) task).wrapped).preLoadContext(); - return null; - } - - @Override - public int compareTo(TaskInfo that) - { - int c = this.status.compareTo(that.status); - if (c == 0) c = Long.compare(this.position(), that.position()); - return c; - } - } - - public List taskSnapshot() - { - List result = new ArrayList<>(); - lock(); - try - { - addToSnapshot(result, waitingToLoad, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); - addToSnapshot(result, waitingToLoadRangeTxns, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); - addToSnapshot(result, scanningRanges, TaskInfo.Status.SCANNING_RANGES, TaskInfo.Status.SCANNING_RANGES); - addToSnapshot(result, loading, TaskInfo.Status.LOADING, TaskInfo.Status.LOADING); - addToSnapshot(result, waitingToRun, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); - addToSnapshot(result, running, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); - } - finally - { - unlock(); - } - result.sort(TaskInfo::compareTo); - return result; - } - - private static void addToSnapshot(List snapshot, TaskQueue queue, TaskInfo.Status ifCurrent, TaskInfo.Status ifQueued) - { - for (int i = 0 ; i < queue.size() ; ++i) - { - Task t = queue.get(i); - if (t instanceof SequentialQueueTask) - { - SequentialExecutor q = ((SequentialQueueTask) t).queue; - snapshot.add(new TaskInfo(ifCurrent, q.commandStoreId, q.task)); - for (int j = 0 ; j < q.size() ; ++j) - snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q.get(j))); - } - else - { - int commmandStoreId = t instanceof AccordTask ? ((AccordTask) t).commandStore.id() : -1; - snapshot.add(new TaskInfo(ifCurrent, commmandStoreId, t)); - } - } - } - - public int unsafePreparingToRunCount() - { - return waitingToLoad.size() + waitingToLoadRangeTxns.size() + scanningRanges.size() + loading.size(); - } - - public int unsafeWaitingToRunCount() - { - return waitingToRun.size(); - } - - public int unsafeRunningCount() - { - return running.size(); - } - -} diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java deleted file mode 100644 index 034544de4e..0000000000 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java +++ /dev/null @@ -1,472 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.service.accord; - -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.TimeUnit; -import java.util.function.IntFunction; - -import javax.annotation.Nullable; - -import accord.api.Agent; -import accord.utils.Invariants; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; - -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; -import org.apache.cassandra.utils.concurrent.SignalLock; - -import static accord.utils.Invariants.nonNull; -import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; - -public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop -{ - private static class ShutdownException extends RuntimeException {} - - private final SignalLock lock; - private final AccordExecutorLoops loops; - private int readyToRunTarget = 1; - private final int readyToRunLimit; - // TODO (desired): intrusive queue using Task.next, but a little challenging because we reuse SequentialQueueTask so have ABA problem - private final ConcurrentLinkedQueue readyToRun = new ConcurrentLinkedQueue<>(); - - private Task pendingSequentialHead, pendingSequentialTail; - private Task pendingCleanupHead, pendingCleanupTail; - private Task pendingNewHead, pendingNewTail; - private int pendingCount; - - private boolean shutdown; - - public AccordExecutorSignalLoop(int executorId, Mode mode, int threads, long spinInterval, long stopCheckInterval, TimeUnit units, IntFunction name, Agent agent) - { - this(new SignalLock(threads, spinInterval, stopCheckInterval, units), executorId, mode, threads, name, agent); - } - - public AccordExecutorSignalLoop(SignalLock lock, int executorId, Mode mode, int threads, IntFunction name, Agent agent) - { - super(lock, executorId, agent); - Invariants.require(threads < SignalLock.MAX_THREADS); - this.lock = lock; - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); - this.readyToRunLimit = Math.min(threads * 4, SignalLock.MAX_SIGNAL_COUNT); - } - - void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) - { - Task next = async.apply(p1a, p2, p3, p4); - Task prev = push(next); - if (prev == null) - lock.signalLockWork(); - } - - @Override - final void beforeUnlockExternal() - { - } - - private Task pollReadyToRun() - { - return readyToRun.poll(); - } - - private void addReadyToRun(Task task) - { - readyToRun.add(task); - } - - private boolean hasReadyToRun() - { - return !readyToRun.isEmpty(); - } - - private LoopTask task(int index, String id, Mode mode) - { - return new LoopTask(index, id); - } - - class LoopTask extends AccordExecutorLoops.LoopTask - { - final int index; - - LoopTask(int index, String id) - { - super(id); - this.index = index; - } - - private Task awaitWork() - { - while (true) - { - if (lock.awaitAsyncOrLock(index)) - { - try - { - if (DEBUG_EXECUTION) debug.onEnterLock(); - fetchWorkExclusive(); - } - catch (Throwable t) - { - unlock(); - throw t; - } - - if (!unlockAndAcquire()) - continue; - } - - if (shutdown) - throw new ShutdownException(); - - return nonNull(pollReadyToRun()); - } - } - - private Task cleanupAndMaybeGetWork(@Nullable Task cleanup) - { - if (cleanup == null) - return null; - - if (lock.tryAcquireAsyncWork()) - { - if (shutdown) - throw new ShutdownException(); - - return pushCleanupAndReturn(cleanup, nonNull(pollReadyToRun())); - } - - if (!tryLock()) - return pushCleanupAndReturn(cleanup, null); - - try - { - completeTaskExclusive(cleanup); - clearRunning(); - fetchWorkExclusive(); - } - catch (Throwable t) - { - unlock(); - throw t; - } - - if (unlockAndAcquire()) - { - if (shutdown) - throw new ShutdownException(); - - return nonNull(pollReadyToRun()); - } - return null; - } - - private Task pushCleanupAndReturn(Task cleanup, Task result) - { - clearRunning(); - cleanup.setReadyToCleanup(); - if (push(cleanup) == null) - lock.signalLockWork(); - return result; - } - - final boolean tryLock() - { - return onTryLock(lock.tryLock(index)); - } - - final boolean unlockAndAcquire() - { - if (Invariants.isParanoid()) paranoidUnlockExclusive(); - if (DEBUG_EXECUTION) debug.onExitLock(); - return lock.unlockAndAcquireAsyncWork(); - } - - private boolean enqueueOnePending() - { - if (pendingCount == 0) - return false; - - if (pendingSequentialHead != null) - { - pendingSequentialHead = enqueueOneCleanup(pendingSequentialHead); - if (pendingSequentialHead == null) - pendingSequentialTail = null; - } - else if (pendingCleanupHead != null) - { - pendingCleanupHead = enqueueOneCleanup(pendingCleanupHead); - if (pendingCleanupHead == null) - pendingCleanupTail = null; - } - else - { - pendingNewHead = enqueueOneSubmit(pendingNewHead); - if (pendingNewHead == null) - pendingNewTail = null; - } - --pendingCount; - return true; - } - - private void fetchWorkExclusive() - { - boolean hasReadyToRun = hasReadyToRun(); - boolean hadWaitingToRun = hasAlreadyWaitingToRun(); - { - int prevPendingUnqueued = pendingCount; - updateAndEnqueuePendingUntilHasWaitingToRun(prevPendingUnqueued / 2); - } - - if (hadWaitingToRun && hasReadyToRun) - { - lock.addAndGetEnabledThreadCount(1); - } - else if (hadWaitingToRun) - { - readyToRunTarget = Math.min(readyToRunLimit, readyToRunTarget + (1+readyToRunTarget)/2); - } - else if (hasReadyToRun && readyToRunTarget > 1) - { - --readyToRunTarget; - } - - boolean hasDrainedSignal = false; - while (true) - { - long state = lock.state(); - int signals = SignalLock.asyncSignalCount(state); - int waiters = SignalLock.waitingEnabledThreadCount(state); - if (signals >= readyToRunTarget) - { - if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; - else if (hasDrainedSignal) - lock.signalLockWorkExclusive(); - return; - } - else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) - { - // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary - lock.propagateAsyncWorkSignals(1); - } - - Task task = pollAlreadyWaitingToRunExclusive(); - if (task == null) - { - if (updateAndEnqueuePendingUntilHasWaitingToRun(0)) - continue; - - lock.clearLockWork(); - hasDrainedSignal = true; - if (!updateAndEnqueuePendingUntilHasWaitingToRun(0)) - { - if (tasks == 0) - notifyQuiescentExclusive(); - return; - } - } - else - { - try { task.preRunExclusive(); } - catch (Throwable t) - { - try { task.fail(t); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - try { completeTaskExclusive(task); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - continue; - } - if (DEBUG_EXECUTION) DebugTask.get(task).onPreRun(); - - addReadyToRun(task); - boolean incremented = lock.incrementAsyncWork(false); - Invariants.require(incremented); - } - } - } - - private boolean updatePendingUnqueued() - { - if (!hasUnqueued()) - return false; - - int count = 0; - Task addSequentialHead = null, addSequentialTail = null; - Task addCleanupHead = null, addCleanupTail = null; - Task addNewHead = null, addNewTail = null; - { - Task cur = Task.reverse(acquireUnqueuedExclusive()); - while (cur != null) - { - Task next = cur.next; - if (!cur.isReadyToCleanup()) - { - if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); - else addNewHead = reverseOne(addNewHead, cur); - } - else if (cur instanceof SequentialQueueTask) - { - if (addSequentialHead == null) addSequentialHead = addSequentialTail = setNextNull(cur); - else addSequentialHead = reverseOne(addSequentialHead, cur); - } - else - { - if (addCleanupHead == null) addCleanupHead = addCleanupTail = setNextNull(cur); - else addCleanupHead = reverseOne(addCleanupHead, cur); - } - ++count; - cur = next; - } - } - - pendingCount += count; - if (addSequentialHead != null) - { - if (pendingSequentialHead == null) pendingSequentialHead = addSequentialHead; - else pendingSequentialTail.next = addSequentialHead; - pendingSequentialTail = addSequentialTail; - } - if (addCleanupHead != null) - { - if (pendingCleanupHead == null) pendingCleanupHead = addCleanupHead; - else pendingCleanupTail.next = addCleanupHead; - pendingCleanupTail = addCleanupTail; - } - if (addNewHead != null) - { - if (pendingNewHead == null) pendingNewHead = addNewHead; - else pendingNewTail.next = addNewHead; - pendingNewTail = addNewTail; - } - return true; - } - - private Task reverseOne(Task prev, Task cur) - { - cur.next = prev; - return cur; - } - - private Task setNextNull(Task cur) - { - cur.next = null; - return cur; - } - - private boolean updateAndEnqueuePendingUntilHasWaitingToRun(int processAtLeast) - { - updatePendingUnqueued(); - int count = 0; - while (enqueueOnePending()) - { - if (++count >= processAtLeast && hasAlreadyWaitingToRun()) - return true; - } - return hasAlreadyWaitingToRun(); - } - - @Override - public void run() - { - lock.register(index, Thread.currentThread()); - Task task = null; - while (true) - { - try - { - try { task = cleanupAndMaybeGetWork(task); } - catch (Throwable t) { task = null; throw t; } - if (task == null) - task = awaitWork(); - - try - { - setRunning(task); - task.runInternal(); - } - catch (Throwable t) - { - try { task.fail(t); } - catch (Throwable t2) - { - try - { - t2.addSuppressed(t); - agent.onException(t2); - } - catch (Throwable t3) { /* empty to ensure we definitely loop so we cleanup the task */ } - } - } - } - catch (ShutdownException ignore) - { - break; - } - catch (Throwable t) - { - agent.onException(t); - } - } - } - } - - @Override - public void shutdown() - { - lock.lock(); - try - { - shutdown = true; - lock.signalAllRegistered(); - } - finally - { - lock.unlock(); - } - } - - @Override - AccordExecutorLoops loops() - { - return loops; - } - - @Override - boolean isInLoop() - { - return loops.isInLoop(); - } - - @Override - boolean isOwningThread() - { - return lock.isOwner(); - } - - @Override - public boolean isTerminated() - { - return loops.isTerminated(); - } - - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException - { - return loops.awaitTermination(timeout, unit); - } -} diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index 45aeaa2f18..74266b0c15 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -38,7 +38,6 @@ import accord.impl.AbstractFetchCoordinator; import accord.local.CommandStore; import accord.local.Node; import accord.local.SafeCommandStore; -import accord.primitives.PartialDeps; import accord.primitives.PartialTxn; import accord.primitives.Participants; import accord.primitives.Range; @@ -46,12 +45,12 @@ import accord.primitives.Ranges; import accord.primitives.Routable; import accord.primitives.Seekable; import accord.primitives.Seekables; -import accord.primitives.SyncPoint; import accord.primitives.Timestamp; import accord.primitives.Txn; import accord.primitives.TxnId; import accord.topology.TopologyException; import accord.utils.Invariants; +import accord.utils.SortedArrays.SortedArrayList; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; @@ -245,7 +244,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements logger.info("Reporting failure of plan {} for bootstrap of {} from {}", planId, range, from, fail); fail(from, Ranges.of(range), fail); } - }, ((AccordCommandStore) commandStore()).taskExecutor()); + }, ((AccordCommandStore) commandStore()).exclusiveExecutor()); } } @@ -395,9 +394,9 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements private final Map streams = new HashMap<>(); - public AccordFetchCoordinator(Node node, Ranges ranges, SyncPoint syncPoint, DataStore.FetchRanges fetchRanges, CommandStore commandStore) throws TopologyException + public AccordFetchCoordinator(Node node, Ranges ranges, TxnId atLeast, SortedArrayList readable, DataStore.FetchRanges fetchRanges, CommandStore commandStore) throws TopologyException { - super(node, node.someSequentialExecutor(), ranges, syncPoint, fetchRanges, commandStore); + super(node, node.someExclusiveExecutor(), ranges, atLeast, readable, fetchRanges, commandStore); } @Override @@ -431,13 +430,6 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements super.onDone(success, failure); } - @Override - protected PartialTxn rangeReadTxn(Ranges ranges) - { - StreamingRead read = new StreamingRead(FBUtilities.getBroadcastAddressAndPort(), ranges); - return new PartialTxn.InMemory(Txn.Kind.Read, ranges, read, noopQuery, null, TableMetadatasAndKeys.none(Routable.Domain.Range)); - } - @Override protected synchronized void onReadOk(Node.Id from, CommandStore commandStore, Data data, Ranges received) { @@ -463,24 +455,22 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements public static class AccordFetchRequest extends FetchRequest { - public AccordFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialDeps partialDeps, PartialTxn partialTxn) + public AccordFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialTxn partialTxn) { - super(sourceEpoch, syncId, ranges, partialDeps, partialTxn); - } - - @Override - protected AsyncChain beginRead(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Participants execute) - { - AsyncChain result = super.beginRead(safeStore, executeAt, txn, execute); - // TODO (required): verify that streaming snapshots have all been created by now, so we won't stream any data that arrives after this - readStarted(safeStore); - return result; + super(sourceEpoch, syncId, ranges, partialTxn); } } @Override - protected FetchRequest newFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialDeps partialDeps, PartialTxn partialTxn) + protected FetchRequest newFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges) { - return new AccordFetchRequest(sourceEpoch, syncId, ranges, partialDeps, partialTxn); + return new AccordFetchRequest(sourceEpoch, syncId, ranges, rangeReadTxn(ranges)); } + + private PartialTxn rangeReadTxn(Ranges ranges) + { + StreamingRead read = new StreamingRead(FBUtilities.getBroadcastAddressAndPort(), ranges); + return new PartialTxn.InMemory(Txn.Kind.Read, ranges, read, noopQuery, null, TableMetadatasAndKeys.none(Routable.Domain.Range)); + } + } diff --git a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java index 5123471acd..a7bdedefdb 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java +++ b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import com.google.common.annotations.VisibleForTesting; @@ -105,6 +106,7 @@ import org.apache.cassandra.schema.Tables; import org.apache.cassandra.schema.Types; import org.apache.cassandra.schema.UserFunctions; import org.apache.cassandra.schema.Views; +import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.serializers.CommandSerializers; import org.apache.cassandra.utils.AbstractIterator; @@ -122,6 +124,7 @@ import static org.apache.cassandra.config.AccordConfig.RangeIndexMode.journal_sa import static org.apache.cassandra.db.partitions.PartitionUpdate.singleRowUpdate; import static org.apache.cassandra.db.rows.BTreeRow.singleCellRow; import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; public class AccordKeyspace { @@ -339,7 +342,7 @@ public class AccordKeyspace { ByteBuffer bytes; if (serialized instanceof ByteBuffer) bytes = (ByteBuffer) serialized; - else bytes = Serialize.toBytesWithoutKey(commandsForKey.maximalPrune()); // TODO (expected): we only need to strip pruned, not prune additional txns + else bytes = Serialize.toBytesWithoutKey(commandsForKey.maybePrune(0, AccordAgent.cfkHlcPruneDelta(DatabaseDescriptor.getAccord()))); // TODO (expected): we only need to strip pruned, not prune additional txns return makeUpdate(storeId, key, timestampMicros, bytes); } @@ -353,9 +356,15 @@ public class AccordKeyspace BufferCell.live(CFKAccessor.data, timestampMicros, bytes))); } - public static Runnable systemTableUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized, long timestampMicros) + private static final AtomicLong lastSystemTimestampMicros = new AtomicLong(); + private static long nextSystemTimestampMicros() { - PartitionUpdate upd = makeUpdate(storeId, key, update, serialized, timestampMicros); + return lastSystemTimestampMicros.accumulateAndGet(currentTimeMillis(), (a, b) -> Math.max(a + 1, b)); + } + + public static Runnable systemTableUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized) + { + PartitionUpdate upd = makeUpdate(storeId, key, update, serialized, nextSystemTimestampMicros()); return () -> { ColumnFamilyStore cfs = AccordColumnFamilyStores.commandsForKey; try (OpOrder.Group group = Keyspace.writeOrder.start()) diff --git a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java index 18dafb1908..2ddac1b52c 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java +++ b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java @@ -254,6 +254,14 @@ public class AccordMessageSink implements MessageSink slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS); break; } + + case ACCORD_WAIT_UNTIL_APPLIED_REQ: + { + TimeoutStrategy slow = slowPreaccept(txnId); + if (slow != null) + slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS); + break; + } } Message message = Message.out(verb, request, expiresAtNanos); diff --git a/src/java/org/apache/cassandra/service/accord/AccordResult.java b/src/java/org/apache/cassandra/service/accord/AccordResult.java index 033ccdd88a..59440ee174 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordResult.java +++ b/src/java/org/apache/cassandra/service/accord/AccordResult.java @@ -201,7 +201,9 @@ public class AccordResult extends AsyncFuture implements BiConsumer -{ - public static class DebugAccordSafeCommand extends AccordSafeCommand - { - final Ref selfRef; - public DebugAccordSafeCommand(AccordCacheEntry global) - { - super(global); - selfRef = new Ref<>(this, null); - selfRef.debug(global.key().toString()); - } - - @Override - public void markUnsafe() - { - super.markUnsafe(); - selfRef.release(); - } - - public static void trace(AccordSafeCommand safeCommand, String message) - { - ((DebugAccordSafeCommand)safeCommand).selfRef.debug(message); - } - } - - private boolean unsafe; - private final AccordCacheEntry global; - private Command original; - private Command current; - - public AccordSafeCommand(AccordCacheEntry global) - { - super(global.key()); - this.global = global; - this.original = null; - this.current = null; - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AccordSafeCommand that = (AccordSafeCommand) o; - return Objects.equals(original, that.original) && Objects.equals(current, that.current); - } - - @Override - public int hashCode() - { - throw new UnsupportedOperationException(); - } - - @Override - public String toString() - { - return "AccordSafeCommand{" + - "invalidated=" + unsafe + - ", global=" + global + - ", original=" + original + - ", current=" + current + - '}'; - } - - @Override - public AccordCacheEntry global() - { - checkNotInvalidated(); - return global; - } - - @Override - public Command current() - { - checkNotInvalidated(); - return current; - } - - @Override - @VisibleForTesting - public void set(Command command) - { - checkNotInvalidated(); - this.current = command; - } - - @Override - public Command original() - { - checkNotInvalidated(); - return original; - } - - public Journal.CommandUpdate update() - { - return new Journal.CommandUpdate(original, current); - } - - @Override - public void preExecute() - { - checkNotInvalidated(); - original = global.getExclusive(); - current = original; - if (isUnset()) - uninitialised(); - } - - @Override - public void markUnsafe() - { - unsafe = true; - } - - @Override - public boolean isUnsafe() - { - return unsafe; - } -} diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java deleted file mode 100644 index 74201823b1..0000000000 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.service.accord; - -import java.util.Objects; - -import com.google.common.annotations.VisibleForTesting; - -import accord.api.RoutingKey; -import accord.local.cfk.CommandsForKey; -import accord.local.cfk.NotifySink; -import accord.local.cfk.SafeCommandsForKey; - -public class AccordSafeCommandsForKey extends SafeCommandsForKey implements AccordSafeState -{ - public static class CommandsForKeyCacheEntry extends AccordCacheEntry - { - private NotifySink overrideSink; - - CommandsForKeyCacheEntry(RoutingKey key, AccordCache.Type.Instance owner) - { - super(key, owner); - } - } - - private boolean invalidated; - private final AccordCacheEntry global; - private CommandsForKey original; - private CommandsForKey current; - - public AccordSafeCommandsForKey(AccordCacheEntry global) - { - super(global.key()); - this.global = global; - this.original = null; - this.current = null; -// if (overrideSink() == null) -// overrideSink(new RecordingNotifySink()); - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AccordSafeCommandsForKey that = (AccordSafeCommandsForKey) o; - return Objects.equals(original, that.original) && Objects.equals(current, that.current); - } - - @Override - public int hashCode() - { - throw new UnsupportedOperationException(); - } - - @Override - public String toString() - { - return "AccordSafeCommandsForKey{" + - "invalidated=" + invalidated + - ", global=" + global + - ", original=" + original + - ", current=" + current + - '}'; - } - - @Override - public boolean hasUpdate() - { - boolean hasUpdate = AccordSafeState.super.hasUpdate(); - - // cfk initialization is legal, but doesn't need to be propagated to the cache (and would - // cause an exception to be thrown if it were). Making an exception on the cache side could - // throw away applied cfk updates as well, so it's special cased here - if (hasUpdate && original == null && current != null && current.size() == 0) - return false; - - return hasUpdate; - } - - @Override - public AccordCacheEntry global() - { - checkNotInvalidated(); - return global; - } - - @Override - public CommandsForKey current() - { - checkNotInvalidated(); - return current; - } - - @Override - @VisibleForTesting - public void set(CommandsForKey cfk) - { - checkNotInvalidated(); - this.current = cfk; - } - - @Override - public void overrideSink(NotifySink overrideSink) - { - ((CommandsForKeyCacheEntry)global).overrideSink = overrideSink; - } - - @Override - public NotifySink overrideSink() - { - return ((CommandsForKeyCacheEntry)global).overrideSink; - } - - public CommandsForKey original() - { - checkNotInvalidated(); - return original; - } - - @Override - public void preExecute() - { - checkNotInvalidated(); - original = global.getExclusive(); - current = original; - if (isUnset()) - initialize(); - } - - @Override - public void markUnsafe() - { - invalidated = true; - } - - @Override - public boolean isUnsafe() - { - return invalidated; - } -} diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 111e207474..ff5752dca0 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -56,12 +56,16 @@ import accord.impl.RequestCallbacks; import accord.impl.SizeOfIntersectionSorter; import accord.impl.progresslog.DefaultProgressLog; import accord.impl.progresslog.DefaultProgressLogs; +import accord.local.BootstrapReason; import accord.local.Catchup; -import accord.local.CatchupHard; +import accord.local.Catchup.Unsuccessful; import accord.local.CommandStores; import accord.local.Node; import accord.local.Node.Id; import accord.local.ShardDistributor.EvenSplit; +import accord.local.TimeService; +import accord.local.UniqueTimeService; +import accord.local.UniqueTimeService.AtomicUniqueTime; import accord.local.UniqueTimeService.AtomicUniqueTimeWithStaleReservation; import accord.local.durability.DurabilityService; import accord.local.durability.ShardDurability; @@ -92,8 +96,9 @@ import accord.utils.async.AsyncResults; import org.apache.cassandra.concurrent.Shutdownable; import org.apache.cassandra.config.AccordConfig; -import org.apache.cassandra.config.AccordConfig.CatchupMode; +import org.apache.cassandra.config.AccordConfig.CatchupFallbackMode; import org.apache.cassandra.config.AccordConfig.JournalConfig.ReplaySavePoint; +import org.apache.cassandra.config.AccordConfig.UniqueTimestampReservations; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; @@ -126,6 +131,7 @@ import org.apache.cassandra.service.accord.api.AccordTopologySorter; import org.apache.cassandra.service.accord.api.AccordViolationHandler; import org.apache.cassandra.service.accord.api.CompositeTopologySorter; import org.apache.cassandra.service.accord.api.TokenKey.KeyspaceSplitter; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.journal.ReplayMarkers; @@ -165,6 +171,8 @@ import static accord.api.Journal.TopologyUpdate; import static accord.api.ProtocolModifiers.FastExecution.MAY_BYPASS_SAFESTORE; import static accord.coordinate.Coordination.CoordinationKind.Client; import static accord.impl.progresslog.DefaultProgressLog.ModeFlag.CATCH_UP; +import static accord.local.BootstrapReason.LOG_CORRUPTED; +import static accord.local.BootstrapReason.LOG_INCOMPLETE; import static accord.local.durability.DurabilityService.SyncLocal.Self; import static accord.local.durability.DurabilityService.SyncRemote.All; import static accord.messages.SimpleReply.Ok; @@ -172,15 +180,20 @@ import static accord.primitives.Txn.Kind.ExclusiveSyncPoint; import static accord.primitives.Txn.Kind.Write; import static accord.primitives.TxnId.MediumPath.NoMediumPath; import static accord.primitives.TxnId.MediumPath.TrackStable; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorThreadTag.JOB; import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.DAEMON; -import static org.apache.cassandra.config.AccordConfig.CatchupMode.FALLBACK_TO_HARD; -import static org.apache.cassandra.config.AccordConfig.CatchupMode.HARD; +import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.EXIT; +import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.REBOOTSTRAP; +import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.REBOOTSTRAP_AND_CATCHUP; +import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.REBOOTSTRAP_INCOMPLETE; +import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.REBOOTSTRAP_RESET; import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.RESET; +import static org.apache.cassandra.config.AccordConfig.UniqueTimestampReservations.HISTOGRAM; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; import static org.apache.cassandra.config.DatabaseDescriptor.getAccordGlobalDurabilityCycle; import static org.apache.cassandra.config.DatabaseDescriptor.getAccordShardDurabilityCycle; @@ -384,6 +397,11 @@ public class AccordService implements IAccordService, Shutdownable AccordService as = new AccordService(tcmIdToAccord(tcmId)); unsafeInstance = replyInstance = as; as.localStartup(); + + AccordReplicaMetrics.touch(); + AccordSystemMetrics.touch(); + AccordExecutorMetrics.touch(); + AccordViolationHandler.setup(); } } @@ -397,11 +415,6 @@ public class AccordService implements IAccordService, Shutdownable return as; as.distributedStartupInternal(); - - AccordReplicaMetrics.touch(); - AccordSystemMetrics.touch(); - AccordExecutorMetrics.touch(); - AccordViolationHandler.setup(); return as; } @@ -464,6 +477,7 @@ public class AccordService implements IAccordService, Shutdownable agent.setup(localId); AccordTimeService time = new AccordTimeService(); this.scheduler = new AccordScheduler(); + // TODO (expected): can we pass ImmediateExecutor rather than Scheduler? final RequestCallbacks callbacks = new RequestCallbacks(time, scheduler); this.dataStore = new AccordDataStore(); this.journal = new AccordJournal(DatabaseDescriptor.getAccord().journal); @@ -474,7 +488,7 @@ public class AccordService implements IAccordService, Shutdownable this.node = new Node(localId, messageSink, topologyService, - time, new AtomicUniqueTimeWithStaleReservation(time), + time, uniqueTimeService(time), () -> dataStore, new KeyspaceSplitter(new EvenSplit<>(getAccord().commandStoreShardCount(), getPartitioner().accordSplitter())), agent, @@ -524,6 +538,8 @@ public class AccordService implements IAccordService, Shutdownable ProtocolModifiers.Configure.setSendStableMessages(config.send_stable); if (config.send_minimal != null) ProtocolModifiers.Configure.setSendMinimal(config.send_minimal); + if (config.unique_timestamp_on_conflict != null) + ProtocolModifiers.Configure.setUniqueTimestampOnConflict(config.unique_timestamp_on_conflict); } @Override @@ -532,7 +548,16 @@ public class AccordService implements IAccordService, Shutdownable if (state != State.INIT) return; - boolean rebootstrap = false; + BootstrapReason rebootstrap = null; + if (getAccord().journal.replay == REBOOTSTRAP_RESET) + { + rebootstrap = LOG_CORRUPTED; + } + else if (getAccord().journal.replay == REBOOTSTRAP_INCOMPLETE) + { + rebootstrap = LOG_INCOMPLETE; + } + else { long startMarker = ReplayMarkers.readStartMarker(); long stopMarker = ReplayMarkers.readStopMarker(); @@ -551,7 +576,7 @@ public class AccordService implements IAccordService, Shutdownable case REBOOTSTRAP: logger.info("Stop marker is older than start marker ({}<{}). Rebootstrapping.", stopMarker, startMarker); - rebootstrap = true; + rebootstrap = LOG_INCOMPLETE; } } } @@ -592,7 +617,7 @@ public class AccordService implements IAccordService, Shutdownable // restore save points before starting the journal so we can validate consistency between journal and save point state (where possible) Long2LongHashMap minSegments; - if (rebootstrap) minSegments = null; + if (rebootstrap != null) minSegments = null; else { switch (getAccord().journal.replaySavePoint) @@ -614,11 +639,11 @@ public class AccordService implements IAccordService, Shutdownable } node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start()); - if (rebootstrap) + if (rebootstrap != null) { // rebootstrap expects the durability service to process visibility sync points for command stores node.durability().start(); - getBlocking(node.commandStores().rebootstrap(node)); + getBlocking(node.commandStores().rebootstrap(node, rebootstrap)); } else { @@ -704,82 +729,96 @@ public class AccordService implements IAccordService, Shutdownable void catchup() { - AccordConfig spec = DatabaseDescriptor.getAccord(); - if (spec.catchup_on_start == CatchupMode.DISABLED) + AccordConfig config = DatabaseDescriptor.getAccord(); + if (!config.catchup_on_start) { logger.info("Catchup disabled; continuing to startup"); return; } - CatchupMode mode = spec.catchup_on_start; BootstrapState bootstrapState = SystemKeyspace.getBootstrapState(); if (bootstrapState == COMPLETED) { node.commandStores().forAllUnsafe(commandStore -> ((DefaultProgressLog)commandStore.unsafeProgressLog()).setMode(CATCH_UP)); try { - long maxLatencyNanos = spec.catchup_on_start_fail_latency.toNanoseconds(); + CatchupFallbackMode onError = config.catchup_on_start_on_error; + CatchupFallbackMode onTimeout = config.catchup_on_start_on_timeout; + + long maxLatencyNanos = config.catchup_on_start_fail_latency.toNanoseconds(); int attempts = 1; while (true) { - logger.info("Catchup ({}) with quorum...", mode); + logger.info("Catchup with quorum..."); long start = nanoTime(); long failAt = start + maxLatencyNanos; - Future f; - { - AsyncChain submit; - if (mode == HARD) - { - if (!node.durability().isStarted()) - node.durability().start(); - submit = CatchupHard.catchup(node); - } - else submit = Catchup.catchup(node); - f = toFuture(submit); - } - if (!f.awaitUntilThrowUncheckedOnInterrupt(failAt)) - { - if (spec.catchup_on_start_exit_on_failure) - { - logger.error("Catchup exceeded maximum latency of {}ns; shutting down", maxLatencyNanos); - throw new RuntimeException("Could not catchup with peers"); - } - logger.error("Catchup exceeded maximum latency of {}ns; continuing to startup", maxLatencyNanos); - break; - } - + Future f = toFuture(Catchup.catchup(node, failAt, NANOSECONDS)); + f.awaitThrowUncheckedOnInterrupt(); Throwable failed = f.cause(); + Unsuccessful result = f.getNow(); + if (failed != null) { - if (spec.catchup_on_start_exit_on_failure) - throw new RuntimeException("Could not catchup with peers", failed); - - logger.error("Could not catchup with peers; continuing to startup"); - break; + switch (onError) + { + default: throw new UnhandledEnum(onError); + case EXIT: + logger.error("Could not catchup with peers; exiting", failed); + throw new RuntimeException("Could not catchup with peers", failed); + case IGNORE: + logger.error("Could not catchup with peers; continuing to startup", failed); + return; + case REBOOTSTRAP: + case REBOOTSTRAP_AND_CATCHUP: + logger.error("Could not catchup with peers; rebootstrapping", failed); + getBlocking(node.commandStores().rebootstrap(node, LOG_INCOMPLETE)); + if (onError == REBOOTSTRAP) + return; + ++attempts; + onError = onTimeout = rebootstrapFallbackMode(config); + continue; + } } long end = nanoTime(); double seconds = NANOSECONDS.toMillis(end - start)/1000.0; logger.info("Finished catchup with all quorums. {}s elapsed.", String.format("%.2f", seconds)); - if (seconds <= spec.catchup_on_start_success_latency.toSeconds()) - break; - - if (++attempts > spec.catchup_on_start_max_attempts) + if (result == null) { - if (spec.catchup_on_start_exit_on_failure) - { - logger.error("Catchup was slow, aborting after {} attempts and shutting down", attempts); - throw new RuntimeException("Could not catchup with peers"); - } + if (seconds <= config.catchup_on_start_success_latency.toSeconds()) + return; - logger.info("Catchup was slow; continuing to startup after {} attempts.", attempts - 1); - break; + if (attempts < config.catchup_on_start_max_attempts) + { + logger.info("Catchup was slow, so we may behind again; retrying"); + ++attempts; + continue; + } } - logger.info("Catchup was slow, so we may behind again; retrying"); - if (mode == FALLBACK_TO_HARD) - mode = HARD; + switch (onTimeout) + { + default: throw new UnhandledEnum(onTimeout); + case EXIT: + if (result == null) logger.error("Catchup was slow; aborting after {} attempts and shutting down", attempts); + else logger.error("Catchup was incomplete for {}; aborting after {} attempts and shutting down", result.ranges, attempts); + throw new RuntimeException("Could not catchup with peers"); + case IGNORE: + if (result == null) logger.info("Catchup was slow, continuing to startup after {} attempts", attempts); + else logger.info("Catchup was incomplete for {}, continuing to startup after {} attempts", result.ranges, attempts); + return; + case REBOOTSTRAP: + case REBOOTSTRAP_AND_CATCHUP: + if (result == null) logger.info("Catchup was slow, rebootstrapping after {} attempts", attempts); + else logger.info("Catchup was incomplete for {}, rebootstrapping after {} attempts", result.ranges, attempts); + getBlocking(node.commandStores().rebootstrap(node, result == null ? null : result.ranges, LOG_INCOMPLETE)); + if (onTimeout == REBOOTSTRAP) + return; + ++attempts; + onError = onTimeout = rebootstrapFallbackMode(config); + continue; + } } } finally @@ -793,6 +832,18 @@ public class AccordService implements IAccordService, Shutdownable } } + private static CatchupFallbackMode rebootstrapFallbackMode(AccordConfig config) + { + CatchupFallbackMode mode = config.catchup_on_start_on_rebootstrap_fallback; + if (mode == REBOOTSTRAP_AND_CATCHUP) + { + CatchupFallbackMode newMode = EXIT; + logger.warn("Converting {} to {} after first failed rebootstrap, to avoid infinite loop", mode, newMode); + mode = newMode; + } + return mode; + } + /** * Startup is broken up in two phases: local and distributed startup. During local startup, we replay up to * the latest epoch known to the node prior to restart. After that, we replay journal itself, and only after @@ -952,7 +1003,7 @@ public class AccordService implements IAccordService, Shutdownable } @Override - public AsyncResult sync(Object requestedBy, TxnId minBound, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) + public AsyncResult sync(Object requestedBy, TxnId minBound, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) { return node.durability().sync(requestedBy, ExclusiveSyncPoint, minBound, ranges, include, syncLocal, syncRemote, timeout, timeoutUnits); } @@ -964,7 +1015,7 @@ public class AccordService implements IAccordService, Shutdownable return syncInternal(minBound, keys, syncLocal, syncRemote); return KeyBarriers.find(node, minBound, keys.get(0).toUnseekable(), syncLocal, syncRemote).chain() - .flatMap(found -> KeyBarriers.await(node, node.someSequentialExecutor(), found, syncLocal, syncRemote)) + .flatMap(found -> KeyBarriers.await(node, node.someExclusiveExecutor(), found, syncLocal, syncRemote)) .flatMap(success -> { if (success) return null; @@ -1436,4 +1487,22 @@ public class AccordService implements IAccordService, Shutdownable { return journal.configuration(); } + + private static UniqueTimeService uniqueTimeService(TimeService time) + { + AccordConfig config = getAccord(); + UniqueTimestampReservations reservations = config.unique_timestamp_reservations; + if (reservations == null) + reservations = HISTOGRAM; + + switch (reservations) + { + default: throw UnhandledEnum.unknown(reservations); + case NONE: return new AtomicUniqueTime(time); + case SMALL_SHARED: return new AtomicUniqueTimeWithStaleReservation(time); + case HISTOGRAM: + long millis = config.unique_timestamp_reservation_range == null ? 50 : config.unique_timestamp_reservation_range.toMilliseconds(); + return new UniqueTimeService.AtomicUniqueAutoStaleTimes(time, millis, MILLISECONDS); + } + } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/AccordTask.java deleted file mode 100644 index e3fe9656e0..0000000000 --- a/src/java/org/apache/cassandra/service/accord/AccordTask.java +++ /dev/null @@ -1,1179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service.accord; - -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.CancellationException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.function.Function; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import org.agrona.collections.Object2ObjectHashMap; -import org.agrona.collections.ObjectHashSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import accord.api.Journal; -import accord.api.RoutingKey; -import accord.local.Command; -import accord.local.CommandStore; -import accord.local.CommandSummaries; -import accord.local.CommandSummaries.Summary; -import accord.local.LoadKeys; -import accord.local.PreLoadContext; -import accord.local.SafeCommandStore; -import accord.local.cfk.CommandsForKey; -import accord.primitives.AbstractRanges; -import accord.primitives.AbstractUnseekableKeys; -import accord.primitives.Range; -import accord.primitives.Ranges; -import accord.primitives.Timestamp; -import accord.primitives.TxnId; -import accord.primitives.Unseekables; -import accord.utils.Invariants; -import accord.utils.UnhandledEnum; -import accord.utils.async.AsyncChain; -import accord.utils.async.AsyncChains; -import accord.utils.async.Cancellable; - -import org.apache.cassandra.concurrent.DebuggableTask; -import org.apache.cassandra.metrics.LogLinearDecayingHistograms; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; -import org.apache.cassandra.service.accord.AccordCommandStore.Caches; -import org.apache.cassandra.service.accord.AccordExecutor.Task; -import org.apache.cassandra.service.accord.AccordExecutor.TaskQueue; -import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; -import org.apache.cassandra.service.accord.api.TokenKey; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; -import org.apache.cassandra.service.accord.serializers.CommandSerializers; -import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.Clock; -import org.apache.cassandra.utils.Closeable; -import org.apache.cassandra.utils.NoSpamLogger; -import org.apache.cassandra.utils.concurrent.Condition; - -import static accord.local.LoadKeysFor.RECOVERY; -import static accord.local.LoadKeysFor.WRITE; -import static accord.primitives.Routable.Domain.Key; -import static accord.primitives.Txn.Kind.EphemeralRead; -import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; -import static org.apache.cassandra.service.accord.AccordTask.State.ASSIGNED; -import static org.apache.cassandra.service.accord.AccordTask.State.CANCELLED; -import static org.apache.cassandra.service.accord.AccordTask.State.FAILED; -import static org.apache.cassandra.service.accord.AccordTask.State.FINISHED; -import static org.apache.cassandra.service.accord.AccordTask.State.INITIALIZED; -import static org.apache.cassandra.service.accord.AccordTask.State.LOADING; -import static org.apache.cassandra.service.accord.AccordTask.State.PERSISTING; -import static org.apache.cassandra.service.accord.AccordTask.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordTask.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_SCAN_RANGES; -import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; -import static org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask.SANITY_CHECK; - -public abstract class AccordTask extends Task implements Function, Cancellable, DebuggableTask -{ - private static final Logger logger = LoggerFactory.getLogger(AccordTask.class); - private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES); - - static class ForFunction extends AccordTask - { - private final Function function; - - public ForFunction(AccordCommandStore commandStore, PreLoadContext loadCtx, Function function) - { - super(commandStore, loadCtx); - this.function = function; - } - - @Override - public R apply(SafeCommandStore commandStore) - { - return function.apply(commandStore); - } - } - - // TODO (desired): these anonymous ops are somewhat tricky to debug. We may want to at least give them names. - static class ForConsumer extends AccordTask - { - private final Consumer consumer; - - private ForConsumer(AccordCommandStore commandStore, PreLoadContext loadCtx, Consumer consumer) - { - super(commandStore, loadCtx); - this.consumer = consumer; - } - - @Override - public Void apply(SafeCommandStore commandStore) - { - consumer.accept(commandStore); - return null; - } - } - - public static AccordTask create(CommandStore commandStore, PreLoadContext ctx, Function function) - { - return new ForFunction<>((AccordCommandStore) commandStore, ctx, function); - } - - public static AccordTask create(CommandStore commandStore, PreLoadContext ctx, Consumer consumer) - { - return new ForConsumer((AccordCommandStore) commandStore, ctx, consumer); - } - - public enum State - { - INITIALIZED(), - WAITING_TO_SCAN_RANGES(INITIALIZED), - SCANNING_RANGES(WAITING_TO_SCAN_RANGES), - WAITING_TO_LOAD(INITIALIZED, SCANNING_RANGES), - LOADING(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD), - WAITING_TO_RUN(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD, LOADING), - ASSIGNED(WAITING_TO_RUN), - RUNNING(ASSIGNED), - PERSISTING(RUNNING), - FINISHED(RUNNING, PERSISTING), - CANCELLED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED), - FAILED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED, RUNNING, PERSISTING); - - private final int permittedFrom; - - State() - { - this.permittedFrom = 0; - } - - State(State ... permittedFroms) - { - int permittedFrom = 0; - for (State state : permittedFroms) - permittedFrom |= 1 << state.ordinal(); - this.permittedFrom = permittedFrom; - } - - boolean isPermittedFrom(State prev) - { - return (permittedFrom & (1 << prev.ordinal())) != 0; - } - - boolean isExecuted() - { - return this.compareTo(PERSISTING) >= 0; - } - - boolean isComplete() - { - return this.compareTo(FINISHED) >= 0; - } - } - - private State state = INITIALIZED; - final AccordCommandStore commandStore; - private final PreLoadContext preLoadContext; - private volatile String loggingId; - private static final AtomicLong nextLoggingId = new AtomicLong(Clock.Global.currentTimeMillis()); - private static final AtomicReferenceFieldUpdater loggingIdUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordTask.class, String.class, "loggingId"); - - // TODO (desired): merge all of these maps into one - @Nullable Object2ObjectHashMap commands; - @Nullable Object2ObjectHashMap commandsForKey; - @Nullable Object2ObjectHashMap> loading; - LogLinearDecayingHistograms.Buffer histogramBuffer; - // TODO (desired): collection supporting faster deletes but still fast poll (e.g. some ordered collection) - @Nullable ArrayDeque> waitingToLoad; - @Nullable RangeTxnScanner rangeScanner; - boolean hasRanges; - @Nullable CommandSummaries commandsForRanges; - @Nullable private TaskQueue queued; - - private BiConsumer callback; - - public AccordTask(@Nonnull AccordCommandStore commandStore, PreLoadContext preLoadContext) - { - this.commandStore = commandStore; - this.preLoadContext = preLoadContext; - this.loggingId = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); - - if (logger.isTraceEnabled()) - logger.trace("Created {} on {}", this, commandStore); - } - - private String loggingId() - { - String id = loggingId; - if (id == null) - { - id = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); - if (!loggingIdUpdater.compareAndSet(this, null, id)) - id = loggingId; - } - return id; - } - - @Override - public String toString() - { - return preLoadContext.describe() + ' ' + toBriefString(); - } - - public String toBriefString() - { - return '{' + loggingId() + ',' + state + '}'; - } - - public String toDescription() - { - return toBriefString() + ": " - + (queued == null ? "unqueued" : queued.kind) - + ", primaryTxnId: " + preLoadContext.primaryTxnId() - + ", waitingToLoad: " + summarise(waitingToLoad) - + ", loading:" + summarise(loading, AccordSafeState::global) - + ", cfks:" + summarise(commandsForKey, AccordSafeState::global) - + ", txns:" + summarise(commands, AccordSafeState::global); - - } - - private static String summarise(Map map, Function transform) - { - if (map == null) - return "null"; - - return summarise(map.values(), transform); - } - - private static String summarise(Collection collection) - { - return summarise(collection, Function.identity()); - } - - private static String summarise(Collection collection, Function transform) - { - if (collection == null) - return "null"; - - StringBuilder out = new StringBuilder("["); - int count = 0; - for (V v : collection) - { - if (count++ > 0) - { - out.append(','); - if (count >= 10) - { - out.append("...(*").append(collection.size() - 10).append(')'); - break; - } - } - out.append(transform.apply(v)); - } - out.append(']'); - return out.toString(); - } - - private void state(State state) - { - Invariants.require(state.isPermittedFrom(this.state), "%s forbidden from %s", state, this, AccordTask::toDescription); - this.state = state; - if (state == WAITING_TO_RUN) - { - Invariants.require(rangeScanner == null || rangeScanner.scanned); - Invariants.require(loading == null && waitingToLoad == null, "WAITING_TO_RUN => no loading or waiting; found %s", this, AccordTask::toDescription); - } - } - - Unseekables keys() - { - return preLoadContext.keys(); - } - - // TODO (expected): try to execute immediately BUT consider ordering requirements - // esp. with deferred actions on e.g. CommandsForKey (not yet supported but also important for performance) - public AsyncChain chain() - { - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - preSetup(callback); - commandStore.executor().submit(AccordTask.this); - return AccordTask.this; - } - }; - } - - public AsyncChain priorityChain() - { - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - preSetup(callback); - commandStore.executor().submitPriority(AccordTask.this); - return AccordTask.this; - } - }; - } - - private void preSetup(BiConsumer callback) - { - Invariants.require(this.callback == null); - this.callback = callback; - commandStore.tryPreSetup(this); - } - - // to be invoked only by the CommandStore owning thread, to take references to objects already in use by the current execution - public void presetup(AccordTask parent) - { - this.queuePosition = parent.queuePosition; - // note we use the caches "unsafely" here deliberately, as we only reference commands we already have references to - // so we do not mutate anything, except the atomic counter of references - if (parent.commands != null) - { - for (TxnId txnId : preLoadContext.txnIds()) - presetupExclusive(txnId, AccordTask::ensureCommands, parent.commands, commandStore.cachesUnsafe().commands()); - } - - if (parent.commandsForKey == null) return; - if (preLoadContext.keys().domain() != Key) return; - switch (preLoadContext.loadKeys()) - { - default: throw new UnhandledEnum(preLoadContext.loadKeys()); - case NONE: - break; - - case ASYNC: - case INCR: - case SYNC: - for (RoutingKey key : (AbstractUnseekableKeys)preLoadContext.keys()) - presetupExclusive(key, AccordTask::ensureCommandsForKey, parent.commandsForKey, commandStore.cachesUnsafe().commandsForKeys()); - break; - } - } - - @Override - void submitExclusive(AccordExecutor owner) - { - owner.submitExclusive(this); - } - - public void setupExclusive() - { - setupInternal(commandStore.cachesExclusive()); - state(rangeScanner != null ? WAITING_TO_SCAN_RANGES - : waitingToLoad != null ? State.WAITING_TO_LOAD - : loading != null ? LOADING : WAITING_TO_RUN); - } - - private void setupInternal(Caches caches) - { - { - boolean hasPreSetup = commands != null; - for (TxnId txnId : preLoadContext.txnIds()) - { - if (hasPreSetup && completePresetupExclusive(txnId, commands, caches.commands())) - continue; - setupExclusive(txnId, AccordTask::ensureCommands, caches.commands()); - } - } - - if (preLoadContext.keys().isEmpty()) - return; - - switch (preLoadContext.keys().domain()) - { - case Key: setupKeyLoadsExclusive(caches, (AbstractUnseekableKeys)preLoadContext.keys(), false); break; - case Range: setupRangeLoadsExclusive(caches); - } - } - - private void setupKeyLoadsExclusive(Caches caches, Iterable keys, boolean isToCompleteRangeScan) - { - if (preLoadContext.loadKeys() == LoadKeys.NONE) - return; - - if (!isToCompleteRangeScan && preLoadContext.loadKeysFor() == RECOVERY) - { - Invariants.require(rangeScanner == null); - rangeScanner = new RangeTxnScanner(); - } - - boolean hasPreSetup = commandsForKey != null; - for (RoutingKey key : keys) - { - if (hasPreSetup && completePresetupExclusive(key, commandsForKey, caches.commandsForKeys())) continue; - setupExclusive(key, AccordTask::ensureCommandsForKey, caches.commandsForKeys()); - } - } - - private void setupRangeLoadsExclusive(Caches caches) - { - if (preLoadContext.loadKeysFor() == WRITE) - return; - - switch (preLoadContext.loadKeys()) - { - default: throw new UnhandledEnum(preLoadContext.loadKeys()); - case NONE: - case ASYNC: - break; - - case INCR: - throw new AssertionError("Incremental mode should only be used with an explicit list of keys"); - - case SYNC: - hasRanges = true; - rangeScanner = new RangeTxnAndKeyScanner(caches.commandsForKeys()); - } - } - - // expects mutual exclusivity only on the command store - private > void presetupExclusive(K k, Function, Map> loaded, Map parentMap, AccordCache.Type.Instance cache) - { - AccordSafeState ref = parentMap.get(k); - if (ref == null) - return; - - AccordCacheEntry node = ref.global(); - int refs = node.increment(); - Invariants.require(refs > 1); - loaded.apply(this).put(k, cache.parent().adapter().safeRef(node)); - } - - // expects to hold lock - private > boolean completePresetupExclusive(K k, Map map, AccordCache.Type.Instance cache) - { - AccordSafeState preacquired = map.get(k); - if (preacquired != null) - { - cache.recordPreAcquired(preacquired); - return true; - } - return false; - } - - // expects to hold lock - private > void setupExclusive(K k, Function, Map> loaded, AccordCache.Type.Instance cache) - { - S safeRef = cache.acquire(k); - Status entryStatus = safeRef.global().status(); - Map map; - switch (entryStatus) - { - default: throw new UnhandledEnum(entryStatus); - case WAITING_TO_LOAD: - case LOADING: - map = ensureLoading(); - break; - case WAITING_TO_SAVE: - case SAVING: - case LOADED: - case MODIFIED: - case FAILED_TO_SAVE: - map = loaded.apply(this); - } - - Object prev = map.putIfAbsent(k, safeRef); - if (prev != null) - { - noSpamLogger.warn("PreLoadContext {} contained key {} more than once", map, k); - cache.release(safeRef, this); - } - else if (map == loading) - { - if (entryStatus == Status.WAITING_TO_LOAD) - ensureWaitingToLoad().add(safeRef.global()); - safeRef.global().loadingOrWaiting().add(this); - Invariants.paranoid(safeRef.global().loadingOrWaiting().waiters().size() == safeRef.global().references()); - } - } - - // expects to hold lock - public boolean onLoad(AccordCacheEntry state) - { - AccordSafeState safeRef = loading == null ? null : loading.remove(state.key()); - Invariants.require(safeRef != null && safeRef.global() == state, "Expected to find %s loading; found %s", state, this, AccordTask::toDescription); - if (safeRef.getClass() == AccordSafeCommand.class) - ensureCommands().put((TxnId)state.key(), (AccordSafeCommand) safeRef); - else - ensureCommandsForKey().put((RoutingKey) state.key(), (AccordSafeCommandsForKey) safeRef); - - if (!loading.isEmpty()) - return false; - - loading = null; - if (this.state.compareTo(State.WAITING_TO_LOAD) < 0) - return false; - - Invariants.require(waitingToLoad == null, "Invalid state: %s", this, AccordTask::toDescription); - state(WAITING_TO_RUN); - return true; - } - - // expects to hold lock - public boolean onLoading(AccordCacheEntry state) - { - boolean removed = waitingToLoad != null && waitingToLoad.remove(state); - Invariants.require(removed, "%s not found in waitingToLoad %s", state, this, AccordTask::toDescription); - if (!waitingToLoad.isEmpty()) - return false; - - return onEmptyWaitingToLoad(); - } - - private boolean onEmptyWaitingToLoad() - { - waitingToLoad = null; - if (this.state.compareTo(State.WAITING_TO_LOAD) < 0) - return false; - - state(loading == null ? WAITING_TO_RUN : LOADING); - return true; - } - - public PreLoadContext preLoadContext() - { - return preLoadContext; - } - - public Map commands() - { - return commands; - } - - public Map ensureCommands() - { - if (commands == null) - commands = new Object2ObjectHashMap<>(); - return commands; - } - - public Map commandsForKey() - { - return commandsForKey; - } - - public Map ensureCommandsForKey() - { - if (commandsForKey == null) - commandsForKey = new Object2ObjectHashMap<>(); - return commandsForKey; - } - - private Map> ensureLoading() - { - if (loading == null) - loading = new Object2ObjectHashMap<>(); - return loading; - } - - private ArrayDeque> ensureWaitingToLoad() - { - Invariants.require(state.compareTo(WAITING_TO_LOAD) <= 0, "Expected status to be on or before WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); - if (waitingToLoad == null) - waitingToLoad = new ArrayDeque<>(); - return waitingToLoad; - } - - public AccordCacheEntry pollWaitingToLoad() - { - Invariants.require(state == State.WAITING_TO_LOAD, "Expected status to be WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); - if (waitingToLoad == null) - return null; - - AccordCacheEntry next = waitingToLoad.poll(); - if (waitingToLoad.isEmpty()) - onEmptyWaitingToLoad(); - return next; - } - - public AccordCacheEntry peekWaitingToLoad() - { - return waitingToLoad == null ? null : waitingToLoad.peek(); - } - - private void maybeSanityCheck(AccordSafeCommand safeCommand) - { - if (SANITY_CHECK) - { - DebugTask debug = DebugTask.get(this); - if (debug.sanityCheck == null) - debug.sanityCheck = new ArrayList<>(commands.size()); - debug.sanityCheck.add(safeCommand.current()); - } - } - - private void save(List diffs, Runnable onFlush) - { - if (SANITY_CHECK && DebugTask.get(this).sanityCheck != null) - { - Condition condition = Condition.newOneTimeCondition(); - this.commandStore.appendCommands(diffs, condition::signal); - condition.awaitUninterruptibly(); - - for (Command check : DebugTask.get(this).sanityCheck) - this.commandStore.sanityCheckCommand(commandStore.unsafeGetRedundantBefore(), check); - - if (onFlush != null) onFlush.run(); - } - else - { - this.commandStore.appendCommands(diffs, onFlush); - } - } - - @Override - protected void preRunExclusive() - { - state(ASSIGNED); - queued = null; - if (rangeScanner != null) - { - commandsForRanges = rangeScanner.finish(commandStore.cachesExclusive()); - rangeScanner = null; - } - if (commands != null) - commands.forEach((k, v) -> v.preExecute()); - if (commandsForKey != null) - commandsForKey.forEach((k, v) -> v.preExecute()); - } - - @Override - public void runInternal() - { - onRunning(); - logger.trace("Running {} with state {}", this, state); - AccordSafeCommandStore safeStore = null; - try (Closeable close = resources.get()) - { - if (Tracing.isTracing()) - Tracing.trace(preLoadContext.describe()); - - state(RUNNING); - - safeStore = commandStore.begin(this, commandsForRanges); - R result = apply(safeStore); - - List changes = null; - if (commands != null) - { - for (AccordSafeCommand safeCommand : commands.values()) - { - if (safeCommand.txnId().is(EphemeralRead)) - continue; - - Journal.CommandUpdate diff = safeCommand.update(); - if (diff == null) - continue; - - if (changes == null) - changes = new ArrayList<>(commands.size()); - changes.add(diff); - - maybeSanityCheck(safeCommand); - } - } - - boolean flush = changes != null || safeStore.fieldUpdates() != null; - if (flush) - { - state(PERSISTING); - Runnable onFlush = () -> finish(result, null); - safeStore.persistFieldUpdatesInternal(changes == null ? onFlush : null); - if (changes != null) save(changes, onFlush); - } - - commandStore.complete(safeStore); - safeStore = null; - onRunComplete(); - if (!flush) - finish(result, null); - } - catch (Throwable t) - { - if (safeStore != null) - { - revert(); - commandStore.abort(safeStore); - } - throw t; - } - } - - public void fail(Throwable throwable) - { - if (state.isComplete()) - return; - - try - { - state(FAILED); - commandStore.agent().onException(throwable); - } - finally - { - if (callback != null) - callback.accept(null, throwable); - } - } - - public void failExclusive(Throwable throwable) - { - fail(throwable); - } - - @Override - protected void cleanupExclusive(AccordExecutor executor) - { - Invariants.expect(state.isExecuted()); - releaseResources(commandStore.cachesExclusive()); - super.cleanupExclusive(executor); - executor.keys.increment(commandsForKey == null ? 0 : commandsForKey.size(), runningAt); - if (histogramBuffer != null) - { - histogramBuffer.flush(cleanupAt); - histogramBuffer = null; - } - } - - @Nullable - public RangeTxnScanner rangeScanner() - { - return rangeScanner; - } - - public boolean hasRanges() - { - return hasRanges; - } - - @Override - public void cancel() - { - if (!state.isComplete()) - commandStore.executor().cancel(this); - } - - void cancelExclusive() - { - logger.info("Cancelling {}", preLoadContext); - state(CANCELLED); - if (rangeScanner != null) - rangeScanner.cancelled = true; - if (callback != null) - { - if (commandStore.executor().isInLoop()) callback.accept(null, new CancellationException()); - else commandStore.executor().submit(() -> callback.accept(null, new CancellationException())); - } - } - - void cancelExclusive(AccordExecutor owner) - { - owner.cancelExclusive(this); - } - - public State state() - { - return state; - } - - private void finish(R result, Throwable failure) - { - state(failure == null ? FINISHED : FAILED); - if (callback != null) - callback.accept(result, failure); - } - - void releaseResources(Caches caches) - { - try - { - // TODO (expected): we should destructively iterate to avoid invoking second time in fail; or else read and set to null - if (rangeScanner != null) - { - rangeScanner.cleanup(caches); - rangeScanner = null; - if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedRangeScanner(); - } - if (commands != null) - { - commands.forEach((k, v) -> caches.commands().release(v, this)); - commands.clear(); - commands = null; - if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedCommands(); - } - if (commandsForKey != null) - { - commandsForKey.forEach((k, v) -> caches.commandsForKeys().release(v, this)); - commandsForKey.clear(); - commandsForKey = null; - if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedCommandsForKeys(); - } - if (waitingToLoad != null) - { - while (!waitingToLoad.isEmpty()) - waitingToLoad.poll().loadingOrWaiting().remove(this); - waitingToLoad = null; - } - if (loading != null) - { - loading.forEach((k, v) -> caches.global().release(v, this)); - loading.clear(); - loading = null; - } - } - catch (Throwable t) - { - releaseResourcesSlow(caches, t); - commandStore.agent().onException(t); - } - } - - private void releaseResourcesSlow(Caches caches, Throwable suppressedBy) - { - if (commands != null) - { - safeRelease(commands, caches.commands(), suppressedBy); - commands.clear(); - commands = null; - } - if (commandsForKey != null) - { - safeRelease(commandsForKey, caches.commandsForKeys(), suppressedBy); - commandsForKey.clear(); - commandsForKey = null; - } - if (waitingToLoad != null) - { - while (!waitingToLoad.isEmpty()) - { - try { waitingToLoad.poll().loadingOrWaiting().remove(this); } - catch (Throwable t) { suppressedBy.addSuppressed(t); } - } - waitingToLoad = null; - } - if (loading != null) - { - safeRelease(loading, caches.global(), suppressedBy); - loading.clear(); - loading = null; - } - } - - private void safeRelease(Map> map, AccordCache.Type.Instance cache, Throwable suppressedBy) - { - for (AccordSafeState safeState : map.values()) - { - if (safeState.isUnsafe()) continue; - try { cache.release(safeState, this); } - catch (Throwable t) { suppressedBy.addSuppressed(t); } - } - } - - private void safeRelease(Map> map, AccordCache cache, Throwable suppressedBy) - { - for (AccordSafeState safeState : map.values()) - { - if (safeState.isUnsafe()) continue; - try { cache.release(safeState, this); } - catch (Throwable t) { suppressedBy.addSuppressed(t); } - } - } - - void revert() - { - if (commands != null) - commands.forEach((k, v) -> v.revert()); - if (commandsForKey != null) - commandsForKey.forEach((k, v) -> v.revert()); - } - - protected void addToQueue(TaskQueue queue) - { - if (state == CANCELLED) - return; - - Invariants.require(queue.kind == state || (queue.kind == State.WAITING_TO_LOAD && state == WAITING_TO_SCAN_RANGES), "Invalid queue type: %s vs %s", queue.kind, this, AccordTask::toDescription); - Invariants.require(this.queued == null, "Already queued with state: %s", this, AccordTask::toDescription); - queued = queue; - queue.append(this); - } - - @Nullable - TaskQueue queued() - { - return queued; - } - - TaskQueue unqueue() - { - TaskQueue wasQueued = queued; - queued.remove(this); - queued = null; - return wasQueued; - } - - TaskQueue unqueueIfQueued() - { - if (queued == null) - return null; - return unqueue(); - } - - public class RangeTxnAndKeyScanner extends RangeTxnScanner - { - class KeyWatcher implements AccordCache.Listener - { - @Override - public void onUpdate(AccordCacheEntry state) - { - if (ranges.contains(state.key())) - reference(state); - } - } - - // TODO (expected): produce key summaries to avoid locking all in memory - final Set intersectingKeys = new ObjectHashSet<>(); - final KeyWatcher keyWatcher = new KeyWatcher(); - final Ranges ranges = ((AbstractRanges) preLoadContext.keys()).toRanges(); - final AccordCache.Type.Instance commandsForKeyCache; - - public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) - { - this.commandsForKeyCache = commandsForKeyCache; - } - - protected void runInternal() - { - for (Range range : ranges) - { - CommandsForKeyAccessor.findAllKeysBetween(commandStore.id(), commandStore.tableId(), getPartitioner(), - (TokenKey) range.start(), range.startInclusive(), - (TokenKey) range.end(), range.endInclusive(), - key -> { - if (cancelled) - throw new CancellationException(); - intersectingKeys.add(key); - }); - } - super.runInternal(); - } - - private void reference(AccordCacheEntry entry) - { - if (loading != null && loading.containsKey(entry.key())) - return; - - switch (entry.status()) - { - default: throw new AssertionError("Unhandled Status: " + entry.status()); - case WAITING_TO_LOAD: - case LOADING: - return; - - case MODIFIED: - case WAITING_TO_SAVE: - case SAVING: - case LOADED: - case FAILED_TO_SAVE: - if (commandsForKey != null && commandsForKey.containsKey(entry.key())) - return; - - Object v = entry.getOrShrunkExclusive(); - if (v == null) return; - else if (v instanceof CommandsForKey) - { - if (!loader.isRelevant((CommandsForKey) v)) - return; - } - else - { - TxnId last = CommandSerializers.txnId.deserialize((ByteBuffer) v); - int position = (int)CommandSerializers.txnId.serializedSize(last); - TxnId minUndecided = CommandSerializers.txnId.deserialize((ByteBuffer) v, position); - if (!loader.isRelevant(entry.key(), last, minUndecided)) - return; - } - - ensureCommandsForKey().putIfAbsent(entry.key(), commandsForKeyCache.acquire(entry)); - } - } - - void startInternal(Caches caches) - { - for (Range range : ranges) - { - for (RoutingKey key : caches.commandsForKeys().keysBetween(range.start(), range.startInclusive(), range.end(), range.endInclusive())) - intersectingKeys.add((TokenKey) key); - } - caches.commandsForKeys().register(keyWatcher); - super.startInternal(caches); - } - - void scannedInternal() - { - if (commandsForKey != null) - intersectingKeys.removeAll(commandsForKey.keySet()); - if (loading != null) - intersectingKeys.removeAll(loading.keySet()); - setupKeyLoadsExclusive(commandStore.cachesExclusive(), intersectingKeys, true); - super.scannedInternal(); - } - - void cleanup(Caches caches) - { - caches.commandsForKeys().tryUnregister(keyWatcher); - super.cleanup(caches); - } - - CommandSummaries finish(Caches caches) - { - caches.commandsForKeys().unregister(keyWatcher); - return super.finish(caches); - } - } - - public class RangeTxnScanner extends AccordExecutor.AbstractIOTask - { - final Map summaries = new HashMap<>(); - final Map guardedSummaries = Collections.synchronizedMap(summaries); - - RangeIndex.Loader loader; - boolean scanned; - Throwable failure; - - volatile boolean cancelled; - - protected void runInternal() - { - loader.load(guardedSummaries, () -> cancelled); - } - - PreLoadContext preLoadContext() - { - return preLoadContext; - } - - @Override - protected void postRunExclusive() - { - commandStore.executor().onScannedRangesExclusive(AccordTask.this, failure); - } - - @Override - protected void fail(Throwable t) - { - this.failure = t; - } - - public void start(AccordExecutor executor) - { - Caches caches = commandStore.cachesExclusive(); - state(SCANNING_RANGES); - startInternal(caches); - executor.submitPlainExclusive(AccordTask.this, this); - } - - void startInternal(Caches caches) - { - loader = commandStore.rangeIndex().loader(preLoadContext.primaryTxnId(), preLoadContext.executeAt(), preLoadContext.loadKeysFor(), preLoadContext.keys()); - loader.loadExclusive(guardedSummaries, caches); - } - - public void scannedExclusive() - { - Invariants.require(state == SCANNING_RANGES, "Expected SCANNING_RANGES; found %s", AccordTask.this, AccordTask::toDescription); - scanned = true; - scannedInternal(); - if (loading == null) state(WAITING_TO_RUN); - else if (waitingToLoad == null) state(LOADING); - else state(State.WAITING_TO_LOAD); - } - - void scannedInternal() - { - } - - void cleanup(Caches caches) - { - if (loader != null) - loader.cleanupExclusive(caches); - } - - CommandSummaries finish(Caches caches) - { - loader.finish(summaries); - TreeMap byId = new TreeMap<>(summaries); - return (CommandSummaries.ByTxnIdSnapshot) () -> byId; - } - - @Override - public String description() - { - return "Scanning range intersections for " + preLoadContext.reason() + ' ' + toBriefString(); - } - - @Override - public String toString() - { - return description(); - } - } - - @Override - public DebuggableTask debuggable() - { - return this; - } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } - - @Override - public String description() - { - return preLoadContext.describe(); - } -} diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java index 77296264e0..79bde24b57 100644 --- a/src/java/org/apache/cassandra/service/accord/IAccordService.java +++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java @@ -63,6 +63,7 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.api.AccordScheduler; import org.apache.cassandra.service.accord.api.AccordTopologySorter; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.topology.AccordEndpointMapper; import org.apache.cassandra.service.accord.topology.AccordSyncPropagator; import org.apache.cassandra.service.accord.topology.AccordSyncPropagator.Notification; @@ -87,8 +88,8 @@ public interface IAccordService IVerbHandler requestHandler(); IVerbHandler responseHandler(); - AsyncResult sync(Object requestedBy, @Nullable TxnId minBound, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits); - AsyncChain sync(@Nullable TxnId minBound, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote); + AsyncResult sync(Object requestedBy, @Nullable TxnId minBound, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits); + AsyncChain sync(@Nullable TxnId minBound, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote); AsyncChain maxConflict(Ranges ranges); @Nonnull @@ -401,13 +402,13 @@ public interface IAccordService } @Override - public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) + public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) { return delegate.sync(requestedBy, onOrAfter, ranges, include, syncLocal, syncRemote, timeout, timeoutUnits); } @Override - public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote) + public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote) { return delegate.sync(onOrAfter, keys, syncLocal, syncRemote); } diff --git a/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java b/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java index b4b607ded3..27da1e7324 100644 --- a/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java @@ -44,6 +44,7 @@ import accord.primitives.Unseekables; import accord.utils.async.Cancellable; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import org.apache.cassandra.service.accord.serializers.CommandStoreSerializers; import static org.apache.cassandra.io.util.CompressedFrameDataInputPlus.readList; @@ -88,7 +89,7 @@ public class InMemoryRangeIndex extends InMemoryRangeSummaryIndex implements Ran { for (TxnId txnId : load) { - AccordCacheEntry entry = caches.commands().getUnsafe(txnId); + AccordCacheEntry entry = caches.commands().getUnsafe(txnId); if (entry == null) { loadFromDisk.add(txnId); @@ -115,7 +116,6 @@ public class InMemoryRangeIndex extends InMemoryRangeSummaryIndex implements Ran public void finish(Map into) { - cleanupExclusive(null); owner.search(this, into::put, null); } diff --git a/src/java/org/apache/cassandra/service/accord/RangeIndex.java b/src/java/org/apache/cassandra/service/accord/RangeIndex.java index a31179a4eb..55f39c5bab 100644 --- a/src/java/org/apache/cassandra/service/accord/RangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/RangeIndex.java @@ -41,6 +41,7 @@ import accord.utils.Invariants; import org.apache.cassandra.exceptions.UnknownTableException; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import org.apache.cassandra.service.accord.journal.CommandChanges; import org.apache.cassandra.service.accord.serializers.Version; @@ -59,10 +60,10 @@ public interface RangeIndex protected abstract AccordCommandStore commandStore(); - protected abstract void loadExclusive(Map into, AccordCommandStore.Caches caches); - protected abstract void load(Map into, BooleanSupplier abort); - protected abstract void finish(Map into); - protected abstract void cleanupExclusive(AccordCommandStore.Caches caches); + public abstract void loadExclusive(Map into, AccordCommandStore.Caches caches); + public abstract void load(Map into, BooleanSupplier abort); + public abstract void finish(Map into); + public abstract void cleanupExclusive(AccordCommandStore.Caches caches); protected CommandSummaries.Summary loadFromDisk(TxnId txnId) { @@ -82,7 +83,7 @@ public interface RangeIndex return null; } - public CommandSummaries.Summary ifRelevant(AccordCacheEntry state) + public CommandSummaries.Summary ifRelevant(AccordCacheEntry state) { if (state.key().domain() != Routable.Domain.Range) return null; diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index 454fa9edbe..cabb3de079 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -39,16 +39,16 @@ import accord.api.Result; import accord.api.RoutingKey; import accord.api.Tracing; import accord.coordinate.Coordination; -import accord.coordinate.Exhausted; -import accord.coordinate.Preempted; -import accord.coordinate.Timeout; +import accord.coordinate.CoordinationFailed; import accord.local.Command; +import accord.local.CommandStore; import accord.local.LogUnavailableException; import accord.local.Node; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.TimeService; import accord.messages.MessageType; +import accord.primitives.Ballot; import accord.primitives.Keys; import accord.primitives.Participants; import accord.primitives.Ranges; @@ -84,6 +84,7 @@ import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.NoSpamLogger; +import org.apache.cassandra.utils.NoSpamLogger.NoDuplicateSpamLogStatement; import static accord.primitives.Routable.Domain.Key; import static accord.utils.SortedArrays.SortedArrayList.ofSorted; @@ -107,12 +108,14 @@ import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retry import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowRead; import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowTxnPreaccept; import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.txn_data; +import static org.apache.cassandra.utils.NoSpamLogger.NoDuplicateSpamLogStatement.exceptionId; // TODO (expected): merge with AccordService public class AccordAgent implements Agent, OwnershipEventListener { private static final Logger logger = LoggerFactory.getLogger(AccordAgent.class); private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, MINUTES); + private static final NoDuplicateSpamLogStatement noSpamException = new NoDuplicateSpamLogStatement(logger, "", 1L, MINUTES); private static final ReplicaEventListener replicaEventListener = new AccordReplicaMetrics.Listener(); private static BiConsumer onFailedBarrier; @@ -155,6 +158,12 @@ public class AccordAgent implements Agent, OwnershipEventListener config = DatabaseDescriptor.getAccord(); } + @Override + public void onSuccessfulBootstrap(CommandStore commandStore, int attempt, long epoch, Ranges ranges) + { + logger.info("{}: Completed bootstrap of {} on epoch {}", commandStore, ranges, epoch); + } + @Override public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Runnable fail, Throwable failure) { @@ -212,13 +221,19 @@ public class AccordAgent implements Agent, OwnershipEventListener return; AccordSystemMetrics.metrics.errors.inc(); - if (t instanceof CancellationException || t instanceof TimeoutException || t instanceof Timeout || t instanceof Preempted || t instanceof Exhausted || t instanceof LogUnavailableException) - // TODO (required): leaky logger, permitting multiple messages per time period and reporting how many were dropped - noSpamLogger.warn("", t); + if (expectedException(t)) // TODO (required): leaky logger, permitting multiple messages per time period and reporting how many were dropped + noSpamException.warn(exceptionId(t), t); else JVMStabilityInspector.uncaughtException(Thread.currentThread(), t); } + public static boolean expectedException(Throwable t) + { + if (t instanceof CancellationException) + return t.getCause() == null; + return t instanceof TimeoutException || t instanceof LogUnavailableException || t instanceof CoordinationFailed; + } + @Override public void onException(Throwable t) { @@ -249,6 +264,11 @@ public class AccordAgent implements Agent, OwnershipEventListener // TODO (expected): we probably want additional configuration here so we can prune on shorter time horizons when we have a lot of transactions on a single key @Override public long cfkHlcPruneDelta() + { + return cfkHlcPruneDelta(config); + } + + public static long cfkHlcPruneDelta(AccordConfig config) { return config.commands_for_key_prune_delta.to(MICROSECONDS); } @@ -308,24 +328,12 @@ public class AccordAgent implements Agent, OwnershipEventListener @Override public long slowCoordinatorDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int attempt) { - SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId); - if (safeCommand == null) - { - noSpamLogger.warn("{} invoked slowCoordinatorDelay for {} without having it in cache", safeStore.commandStore(), txnId, new RuntimeException()); - return recover(txnId).computeWait(attempt, units); - } - - Command command = safeCommand.current(); - if (command == null) - { - noSpamLogger.warn("{} invoked slowCoordinatorDelay for {} without knowing the command", safeStore.commandStore(), txnId, new RuntimeException()); - return recover(txnId).computeWait(attempt, units); - } - + Command command = command(safeStore, txnId, "slowCoordinatorDelay"); + Ballot promised = promised(command); // TODO (expected): make this a configurable calculation on normal request latencies (like ContentionStrategy) long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()); - long mostRecentStart = mostRecentStart(command, nowMicros); + long mostRecentStart = mostRecentStart(txnId, promised, nowMicros); long waitMicros = recover(txnId).computeWait(attempt, MICROSECONDS); long startTime = mostRecentStart + waitMicros; if (startTime < nowMicros) @@ -338,7 +346,7 @@ public class AccordAgent implements Agent, OwnershipEventListener } RoutingKey homeKey = command.route().homeKey(); - Shard shard = node.topology().active().forEpochIfKnown(homeKey, command.txnId().epoch()); + Shard shard = node.topology().active().forEpochIfKnown(homeKey, txnId.epoch()); startTime = nonClashingStartTime(startTime, shard == null ? null : shard.nodes, node.id(), ONE_SECOND, random); long delayMicros = Math.max(1, startTime - nowMicros); @@ -346,15 +354,15 @@ public class AccordAgent implements Agent, OwnershipEventListener return units.convert(delayMicros, MICROSECONDS); } - private static long mostRecentStart(Command command, long nowMicros) + private static long mostRecentStart(TxnId txnId, Ballot promised, long nowMicros) { // TODO (expected): make this a configurable calculation on normal request latencies (like ContentionStrategy) - long promisedHlc = command.promised().hlc(); + long promisedHlc = promised.hlc(); if (promisedHlc > nowMicros + ONE_MINUTE) promisedHlc = 0; - long result = Math.max(command.txnId().hlc(), promisedHlc); + long result = Math.max(txnId.hlc(), promisedHlc); if (result > nowMicros + ONE_SECOND) - noSpamLogger.warn("max({},{})>{}", command.txnId(), command.promised(), nowMicros); + noSpamLogger.warn("max({},{})>{}", txnId, promised, nowMicros); return result; } @@ -388,25 +396,36 @@ public class AccordAgent implements Agent, OwnershipEventListener return newStartTime; } - @Override - public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil blockedUntil, TimeUnit units) + private static Ballot promised(@Nullable Command command) + { + return command == null ? Ballot.ZERO : command.promised(); + } + + private static Command command(SafeCommandStore safeStore, TxnId txnId, String source) { SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId); if (safeCommand == null) { - noSpamLogger.warn("{} invoked slowReplicaDelay for {} without having it in cache", safeStore.commandStore(), txnId, new RuntimeException()); - return fetch(txnId).computeWait(attempt, units); + noSpamLogger.warn("{} invoked {} for {} without having it in cache", safeStore.commandStore(), source, txnId, new RuntimeException()); + return null; } Command command = safeCommand.current(); if (command == null) { - noSpamLogger.warn("{} invoked slowReplicaDelay for {} without knowing the command", safeStore.commandStore(), txnId, new RuntimeException()); - return fetch(txnId).computeWait(attempt, units); + noSpamLogger.warn("{} invoked {} for {} without knowing the command", safeStore.commandStore(), source, txnId, new RuntimeException()); + return null; } + return command; + } + + @Override + public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil blockedUntil, TimeUnit units) + { + Ballot promised = promised(command(safeStore, txnId, "slowReplicaDelay")); long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()); - long mostRecentStart = mostRecentStart(command, nowMicros); + long mostRecentStart = mostRecentStart(txnId, promised, nowMicros); long waitMicros = fetch(txnId).computeWait(attempt, units); long startTime = mostRecentStart + waitMicros; if (startTime < nowMicros) diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java index cf7d8ff5ae..eb1fcbfbfa 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java @@ -38,7 +38,7 @@ import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory. public class AccordWaitStrategies { - static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead; + static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead, slowDurability; static TimeoutStrategy expireTxn, expireSyncPoint, expireDurability, expireEpochWait; static TimeoutStrategy fetchTxn, fetchSyncPoint; static RetryStrategy recoverTxn, recoverSyncPoint, retrySyncPoint, retryDurability, retryBootstrap, retryJoinBootstrap; @@ -88,6 +88,7 @@ public class AccordWaitStrategies setSlowRead(config.slow_read); setSlowTxnPreaccept(config.slow_txn_preaccept); setSlowSyncPointPreaccept(config.slow_syncpoint_preaccept); + setSlowDurability(config.slow_durability); setExpireTxn(config.expire_txn); setExpireSyncPoint(config.expire_syncpoint); setExpireDurability(config.expire_durability); @@ -119,6 +120,11 @@ public class AccordWaitStrategies slowSyncPointPreaccept = TimeoutStrategy.parse(spec, none()); } + public static void setSlowDurability(String spec) + { + slowDurability = TimeoutStrategy.parse(spec, none()); + } + public static void setExpireTxn(String spec) { expireTxn = TimeoutStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics)); diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java index af38a5c389..15b34ca23f 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java @@ -37,7 +37,7 @@ import accord.api.RoutingKey; import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommandStore; import accord.local.cfk.SafeCommandsForKey; import accord.primitives.RoutingKeys; @@ -53,8 +53,6 @@ import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.utils.concurrent.Future; -import static accord.local.LoadKeys.SYNC; -import static accord.local.LoadKeysFor.READ_WRITE; import static java.util.Collections.emptyList; public class DebugBlockedTxns @@ -178,7 +176,7 @@ public class DebugBlockedTxns private AsyncChain visitRootTxnAsync(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -188,7 +186,7 @@ public class DebugBlockedTxns private AsyncChain visitTxnAsync(CommandStore commandStore, TxnId txnId, Timestamp rootExecuteAt, @Nullable TokenKey byKey, int depth, boolean recurse) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -231,7 +229,7 @@ public class DebugBlockedTxns private AsyncChain visitKeysAsync(CommandStore commandStore, TokenKey key, Timestamp rootExecuteAt, int depth) { - return commandStore.chain(PreLoadContext.contextFor(RoutingKeys.of(key.toUnseekable()), SYNC, READ_WRITE, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequencedReadWrite(RoutingKeys.of(key.toUnseekable()), "Populate txn_blocked_by"), safeStore -> { visitKeysSync(safeStore, key, rootExecuteAt, depth); }); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java index d42f9bb338..e01a8363d8 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java @@ -29,7 +29,7 @@ import accord.local.Command; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.metrics.LogLinearHistogram; -import org.apache.cassandra.service.accord.AccordExecutor.Task; +import org.apache.cassandra.service.accord.execution.Task; import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.WithResources; @@ -40,10 +40,10 @@ public class DebugExecution { private static final Logger logger = LoggerFactory.getLogger(DebugExecution.class); public static final boolean DEBUG_EXECUTION = CassandraRelevantProperties.ACCORD_DEBUG_EXECUTION.getBoolean(false); - private static final long REPORT_MIN_LATENCY_MICROS = 20_000; + private static final long REPORT_MIN_LATENCY_MICROS = 50_000; private static final long REPORT_CPU_RATIO = 2; - private static final long REPORT_MAX_LATENCY_MICROS = 50_000; - private static final long REPORT_CPU_MICROS = 10_000; + private static final long REPORT_MAX_LATENCY_MICROS = 100_000; + private static final long REPORT_CPU_MICROS = 50_000; // TODO (expected): use sharded histogram so we can report global stats public static class DebugExecutor @@ -94,48 +94,21 @@ public class DebugExecution long lockedForCpuMicros = (unlockedAtCpu - lockedAtCpu)/1000; if (lockedForMicros >= REPORT_MAX_LATENCY_MICROS) { - report("Held lock for {}us (cpu:{}us)\n", lockedForMicros, lockedForCpuMicros); + report("Held lock for {}us (cpu:{}us)", lockedForMicros, lockedForCpuMicros); } else if (lockedForMicros >= REPORT_MIN_LATENCY_MICROS && (lockedForMicros / lockedForCpuMicros) >= REPORT_CPU_RATIO) { - report("Held lock for {}us with cpu time only {}us\n", lockedForMicros, lockedForCpuMicros); + report("Held lock for {}us with cpu time only {}us", lockedForMicros, lockedForCpuMicros); } locked.increment(lockedForMicros); } } - public static class DebugExecutorLoop + public static class DebugExclusiveExecutor { - final DebugExecutor owner; - long lockAt; - - public DebugExecutorLoop(DebugExecutor owner) + public static DebugExclusiveExecutor maybeDebug(DebugExecutor owner, int commandStoreId) { - this.owner = owner; - } - - public void onLock() - { - lockAt = nanoTime(); - } - - public void onEnterLock() - { - owner.onEnterLock(lockAt); - lockAt = 0; - } - - public void onExitLock() - { - owner.onExitLock(); - } - } - - public static class DebugSequentialExecutor - { - public static DebugSequentialExecutor maybeDebug(DebugExecutor owner, int commandStoreId) - { - return DEBUG_EXECUTION ? new DebugSequentialExecutor(owner, commandStoreId) : null; + return DEBUG_EXECUTION ? new DebugExclusiveExecutor(owner, commandStoreId) : null; } final DebugExecutor owner; @@ -144,7 +117,7 @@ public class DebugExecution long setTaskAt, waitingAt; Task prev; - public DebugSequentialExecutor(DebugExecutor owner, int commandStoreId) + public DebugExclusiveExecutor(DebugExecutor owner, int commandStoreId) { this.owner = owner; this.commandStoreId = commandStoreId; @@ -159,15 +132,16 @@ public class DebugExecution public void onComplete(Task completed) { long readyAt = setTaskAt; + long runningAt = DebugTask.get(completed).runningAt; if (waitingAt > setTaskAt) { readyAt = waitingAt; - long waitingMicros = (completed.runningAt - waitingAt)/1000; + long waitingMicros = (runningAt - waitingAt)/1000; owner.sequentialExecutorWaitingToRunLatency.increment(waitingMicros); if (waitingMicros > REPORT_MAX_LATENCY_MICROS) report("{} spent {}us blocked by a direct execution on queue {}", completed, waitingMicros, commandStoreId); } - long atHeadMicros = (completed.runningAt - readyAt)/1000; + long atHeadMicros = (runningAt - readyAt)/1000; owner.sequentialExecutorSetHeadToRunLatency.increment(atHeadMicros); if (atHeadMicros > REPORT_MAX_LATENCY_MICROS) { @@ -199,8 +173,8 @@ public class DebugExecution } public List sanityCheck; // for AccordTask only - long polledAt, preRunAt, runCompleteAt, completedAt; - long releasedRangeScannerAt, releasedCommandsAt, releasedCommandsForKeyAt; + long polledAt, preRunAt, runningAt, runCompleteAt, completeAt, completedAt; + long releasedRangeScannerAt, releasedStateAt; long runningAtCpu, runCompleteAtCpu; Thread thread; @@ -218,6 +192,7 @@ public class DebugExecution { thread = Thread.currentThread(); runningAtCpu = nowCpu(); + runningAt = nanoTime(); } public void onRunComplete() @@ -231,32 +206,32 @@ public class DebugExecution releasedRangeScannerAt = nanoTime(); } - public void onReleasedCommands() + public void onReleasedState() { - releasedCommandsAt = nanoTime(); + releasedStateAt = nanoTime(); } - public void onReleasedCommandsForKeys() + public void onComplete() { - releasedCommandsForKeyAt = nanoTime(); + completeAt = nanoTime(); } public void onCompleted(DebugExecutor owner) { completedAt = nanoTime(); - if (task.runningAt > 0 && polledAt > 0) + if (runningAt > 0 && polledAt > 0) { - long pollToRunMicros = (task.runningAt - polledAt)/1000; + long pollToRunMicros = (runningAt - polledAt)/1000; owner.pollToRun.increment(pollToRunMicros); long runningMicros = -1; if (runCompleteAt > 0) { - runningMicros = (runCompleteAt - task.runningAt) / 1000; + runningMicros = (runCompleteAt - runningAt) / 1000; owner.running.increment(runningMicros); } - long runToCleanMicros = (task.cleanupAt - runCompleteAt)/1000; + long runToCleanMicros = (completeAt - runCompleteAt) / 1000; owner.runToCleanup.increment(runToCleanMicros); - long cleanupMicros = (completedAt - task.cleanupAt)/1000; + long cleanupMicros = (completedAt - completeAt) / 1000; owner.cleanup.increment(cleanupMicros); long totalMicros = (completedAt - polledAt)/1000; owner.taskTotal.increment(totalMicros); @@ -266,7 +241,7 @@ public class DebugExecution String reason = ""; if (totalMicros > REPORT_MAX_LATENCY_MICROS) reason += "LONG TIME "; if (totalCpu > REPORT_CPU_MICROS) reason += "HIGH CPU "; - if ((totalMicros > REPORT_MIN_LATENCY_MICROS && (totalMicros/totalCpu) >= REPORT_CPU_RATIO)) reason += "LOW RATIO "; + if ((totalMicros > REPORT_MIN_LATENCY_MICROS && (totalCpu == 0 || (totalMicros/totalCpu) >= REPORT_CPU_RATIO))) reason += "LOW RATIO "; report("{}{}: total {}us cpu:{}us ({}), pollToRun {}us, running {}us, runToClean {}us, cleanup {}us", reason, task, totalMicros, totalCpu, thread, pollToRunMicros, runningMicros, runToCleanMicros, cleanupMicros); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java index 8bc6a6a53a..7807eff9bc 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java @@ -40,7 +40,7 @@ import javax.annotation.Nullable; import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommandStore; import accord.primitives.PartialDeps; import accord.primitives.Participants; @@ -221,7 +221,7 @@ public abstract class DebugTxnGraph private AsyncChain> submitRoot(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -232,7 +232,7 @@ public abstract class DebugTxnGraph private AsyncChain> submitParent(CommandStore commandStore, TxnId txnId, P param, Map infos, Set visitedParent, int depth) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -344,7 +344,7 @@ public abstract class DebugTxnGraph private AsyncChain populateTxnAsync(CommandStore commandStore, TxnId txnId, Map visited) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); visited.putIfAbsent(txnId, command == null || command.saveStatus() == SaveStatus.Uninitialised ? SaveInfo.NONE : new SaveInfo(command.saveStatus(), command.executeAtIfKnown())); }); diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java similarity index 56% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java rename to src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java index 5afeaf8a7b..7018ae6318 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java @@ -16,47 +16,43 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.locks.Lock; +import java.util.function.Consumer; +import java.util.function.Function; import accord.api.Agent; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.service.accord.AccordExecutorLoops.LoopTask; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutorLoop; +import org.apache.cassandra.service.accord.execution.Loops.LoopTask; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; -abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop +abstract class AbstractLockLoop extends AbstractLoop { - private static final int YIELD_INTERVAL = DatabaseDescriptor.getAccord().queue_yield_interval; int runningThreads; boolean shutdown; - AccordExecutorAbstractLockLoop(Lock lock, int executorId, Agent agent) + AbstractLockLoop(Lock lock, int executorId, Agent agent) { super(lock, executorId, agent); } abstract void notifyWork(); abstract void notifyWorkExclusive(); - void loopYieldExclusive() throws InterruptedException {} abstract void awaitExclusive() throws InterruptedException; - abstract void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4); + abstract void submitExternal(Consumer sync, Function async, P1 p1); - void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + final void submit(Consumer sync, Function async, P1 p1) { // if we're a loop thread, we will poll the waitingToRun queue when we come around // NOTE: this assumes no synchronous blocking tasks are submitted to this executor - if (isInLoop() || isOwningThread()) push(async.apply(p1a, p2, p3, p4)); - else submitExternal(sync, async, p1s, p1a, p2, p3, p4); + if (isInLoop() || isOwningThread()) push(async.apply(p1)); + else submitExternal(sync, async, p1); } - void submitExternalExclusive(QuintConsumer sync, P1s p1s, P2 p2, P3 p3, P4 p4) + final void submitExternalExclusive(Consumer sync, Function async, P1 p1) { try { @@ -66,11 +62,11 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop } catch (Throwable t) { - try { sync.accept(this, p1s, p2, p3, p4); } + try { sync.accept(p1); } catch (Throwable t2) { t.addSuppressed(t2); } throw t; } - sync.accept(this, p1s, p2, p3, p4); + sync.accept(p1); } finally { @@ -84,6 +80,73 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop notifyWorkExclusive(); } + final boolean hasWaitingToRun() + { + updateWaitingToRunExclusive(); + return hasAlreadyWaitingToRun(); + } + + final Task pollWaitingToRunExclusive() + { + updateWaitingToRunExclusive(); + return pollAlreadyWaitingToRunExclusive(); + } + + final void updateWaitingToRunExclusive() + { + drainUnqueuedExclusive(); + super.updateWaitingToRunExclusive(); + } + + final void drainUnqueuedExclusive() + { + Task cur = Task.reverse(acquireUnqueuedExclusive()); + while (cur != null) + cur = enqueueOneExclusive(cur); + } + + final void drainUnqueuedNewWorkExclusive() + { + Task cur = acquireUnqueuedExclusive(); + Task prev = null, requeue = null, requeueLast = null; + while (cur != null) + { + Task next = cur.next; + if (cur.isNewWork()) + { + cur.next = prev; + prev = cur; + } + else + { + if (requeue == null) requeue = cur; + else requeueLast.next = cur; + requeueLast = cur; + } + cur = next; + } + + if (requeue != null) + { + while (true) + { + Task next = unqueued; + requeueLast.next = next; + if (unqueuedUpdater.compareAndSet(this, next, requeue)) + break; + } + } + + cur = prev; + while (cur != null) + { + Task next = cur.next; + cur.submitExclusiveNoExcept(); + cur.next = null; + cur = next; + } + } + @Override final void beforeUnlockExternal() { @@ -130,36 +193,22 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop @Override public void run() { - Thread self = Thread.currentThread(); + Thread thread = Thread.currentThread(); + TaskRunner self = TaskRunner.get(thread); + self.setAccordActiveExecutor(AbstractLockLoop.this); + setWrapped(self); + Task task; while (true) { - lock(); + lock(self); try { enterLockLoop(); while (true) { task = pollWaitingToRunExclusive(); - - if (task != null) - { - setRunning(task); - try - { - task.preRunExclusive(); - task.runInternal(); - } - catch (Throwable t) - { - task.fail(t); - } - finally - { - completeTaskExclusive(task); - clearRunning(); - } - } + if (task != null) prepareRunComplete(self, task); else { if (shutdown) @@ -179,13 +228,12 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop catch (Throwable t) { exitLockLoop(); - try { agent.onException(t); } catch (Throwable t2) { } } finally { - unlock(); + unlock(self); } } } @@ -196,42 +244,39 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop { return new LoopTask(name) { - final DebugExecutorLoop debug = DEBUG_EXECUTION ? new DebugExecutorLoop(AccordExecutorAbstractLockLoop.this.debug) : null; @Override public void run() { - int count = 0; + Thread thread = Thread.currentThread(); + TaskRunner self = TaskRunner.get(thread); + self.setAccordActiveExecutor(AbstractLockLoop.this); + setWrapped(self); + Task task = null; while (true) { - if (DEBUG_EXECUTION) debug.onLock(); - lock(); + lock(self); try { - if (DEBUG_EXECUTION) debug.onEnterLock(); enterLockLoop(); if (task != null) { Task tmp = task; task = null; - completeTaskExclusive(tmp); - clearRunning(); - } - - if (count >= YIELD_INTERVAL) - { - loopYieldExclusive(); - count = 0; + tmp.completeExclusiveNoExcept(); } while (true) { task = pollWaitingToRunExclusive(); - if (task != null) { - setRunning(task); - task.preRunExclusive(); + if (!task.prepareExclusiveNoExcept()) + { + task = null; + continue; + } + if (DEBUG_EXECUTION) debug.onExitLock(); exitLockLoop(); break; @@ -250,56 +295,22 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop awaitExclusive(); if (DEBUG_EXECUTION) debug.onEnterLock(); resumeLoop(); - count = 0; } } catch (Throwable t) { - if (task != null) - { - try { task.fail(t); } - catch (Throwable t2) { t.addSuppressed(t2); } - try { completeTaskExclusive(task); } - catch (Throwable t2) { t.addSuppressed(t2); } - try { agent.onException(t); } - catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ } - task = null; - } - else - { - try { agent.onException(t); } - catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ } - } + try { agent.onException(t); } + catch (Throwable t2) { } exitLockLoop(); continue; } finally { if (DEBUG_EXECUTION) debug.onExitLock(); - unlock(); + unlock(self); } - try - { - ++count; - task.runInternal(); - } - catch (Throwable t) - { - try { task.fail(t); } - catch (Throwable t2) - { - try - { - t2.addSuppressed(t); - agent.onException(t2); - } - catch (Throwable t3) - { - // empty to ensure we definitely loop so we cleanup the task - } - } - } + task.runNoExcept(self); } } }; diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java similarity index 65% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java rename to src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java index a76d1bac18..de6d8020f7 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.Lock; @@ -27,28 +27,25 @@ import accord.utils.Invariants; import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; -abstract class AccordExecutorAbstractLoop extends AccordExecutor -{ - private volatile Task unqueued; - private static final AtomicReferenceFieldUpdater unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordExecutorAbstractLoop.class, Task.class, "unqueued"); +import static org.apache.cassandra.service.accord.execution.Task.State.REGISTERED; - AccordExecutorAbstractLoop(Lock lock, int executorId, Agent agent) +abstract class AbstractLoop extends AccordExecutor +{ + volatile Task unqueued; + static final AtomicReferenceFieldUpdater unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AbstractLoop.class, Task.class, "unqueued"); + + AbstractLoop(Lock lock, int executorId, Agent agent) { super(lock, executorId, agent); } - abstract AccordExecutorLoops loops(); + abstract Loops loops(); boolean hasUnqueued() { return unqueued != null; } - Task unqueued() - { - return unqueued; - } - final Task push(Task submit) { Invariants.require(submit.next == null); @@ -67,30 +64,18 @@ abstract class AccordExecutorAbstractLoop extends AccordExecutor if (hasUnqueued() || tasks > 0) return true; - lock(); + TaskRunner self = TaskRunner.get(); + lock(self); try { return hasUnqueued() || tasks > 0; } finally { - unlock(); + unlock(self); } } - final void updateWaitingToRunExclusive() - { - drainUnqueuedExclusive(); - super.updateWaitingToRunExclusive(); - } - - final void drainUnqueuedExclusive() - { - Task cur = Task.reverse(acquireUnqueuedExclusive()); - while (cur != null) - cur = enqueueOneExclusive(cur); - } - final Task acquireUnqueuedExclusive() { return unqueuedUpdater.getAndSet(this, null); @@ -101,26 +86,16 @@ abstract class AccordExecutorAbstractLoop extends AccordExecutor Invariants.require(cur != null); Task next = cur.next; cur.next = null; - if (cur.isReadyToCleanup()) completeTaskExclusive(cur); - else cur.submitExclusive(this); + if (cur.compareTo(REGISTERED) < 0) cur.submitExclusiveNoExcept(); + else cur.completeExclusiveNoExcept(); return next; } - final Task enqueueOneCleanup(Task cur) + final Task destructiveNext(Task cur) { Invariants.require(cur != null); Task next = cur.next; cur.next = null; - completeTaskExclusive(cur); - return next; - } - - final Task enqueueOneSubmit(Task cur) - { - Invariants.require(cur != null); - Task next = cur.next; - cur.next = null; - cur.submitExclusive(this); return next; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractSemiSyncSubmit.java similarity index 65% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AbstractSemiSyncSubmit.java index dce92f103d..79f6736ffa 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractSemiSyncSubmit.java @@ -16,26 +16,26 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.locks.Lock; +import java.util.function.Consumer; +import java.util.function.Function; import accord.api.Agent; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -abstract class AccordExecutorAbstractSemiSyncSubmit extends AccordExecutorAbstractLockLoop +abstract class AbstractSemiSyncSubmit extends AbstractLockLoop { - AccordExecutorAbstractSemiSyncSubmit(Lock lock, int executorId, Agent agent) + AbstractSemiSyncSubmit(Lock lock, int executorId, Agent agent) { super(lock, executorId, agent); } abstract void awaitExclusive() throws InterruptedException; - void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submitExternal(Consumer sync, Function async, P1 p1) { - if (push(async.apply(p1a, p2, p3, p4)) == null && !isInLoop()) + if (push(async.apply(p1)) == null && !isInLoop()) notifyWork(); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCache.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java similarity index 76% rename from src/java/org/apache/cassandra/service/accord/AccordCache.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordCache.java index 6b6c48b2fc..21248920c6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCache.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.io.IOException; import java.nio.ByteBuffer; @@ -26,6 +26,7 @@ import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; @@ -44,6 +45,8 @@ import org.slf4j.LoggerFactory; import accord.api.RoutingKey; import accord.local.Command; +import accord.local.RedundantBefore; +import accord.local.SafeState; import accord.local.cfk.CommandsForKey; import accord.local.cfk.Serialize; import accord.primitives.Routable; @@ -64,11 +67,17 @@ import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.metrics.AccordCacheMetrics; import org.apache.cassandra.metrics.LogLinearHistogram; import org.apache.cassandra.metrics.ShardedHitRate; -import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; -import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; -import org.apache.cassandra.service.accord.AccordSafeCommandsForKey.CommandsForKeyCacheEntry; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordKeyspace; +import org.apache.cassandra.service.accord.AccordObjectSizes; +import org.apache.cassandra.service.accord.OrderedKeys; +import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.events.CacheEvents; +import org.apache.cassandra.service.accord.execution.AccordCache.Adapter.Shrink; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Loading; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.execution.SaferCommandsForKey.CommandsForKeyCacheEntry; import org.apache.cassandra.service.accord.journal.CommandChangeWriter; import org.apache.cassandra.service.accord.journal.CommandChanges; import org.apache.cassandra.service.accord.serializers.CommandSerializers; @@ -79,9 +88,11 @@ import org.apache.cassandra.utils.ObjectSizes; import static accord.utils.Invariants.illegalState; import static accord.utils.Invariants.require; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.AGE_MASK; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.GENERATION_MASK; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.EVICTED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.LOADED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.MODIFIED; /** * Cache for AccordCommand and AccordCommandsForKey, available memory is shared between the two object types. @@ -91,22 +102,22 @@ import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFI * * TODO (required): we only iterate over unreferenced entries */ -public class AccordCache implements CacheSize +public class AccordCache { private static final Logger logger = LoggerFactory.getLogger(AccordCache.class); private static final NoSpamLogStatement evictNoEvict = NoSpamLogger.getStatement(logger, "Found and expired {} marked no evict, with age {}, exceeding its expected max age of {}", 1L, TimeUnit.MINUTES); // Debug mode to verify that loading from journal + system tables results in // functionally identical (or superceding) command to the one we've just evicted. - private static boolean VALIDATE_LOAD_ON_EVICT = false; + private static boolean DEBUG_VALIDATE_LOAD_ON_EVICT = false; @VisibleForTesting public static void validateLoadOnEvict(boolean value) { - VALIDATE_LOAD_ON_EVICT = value; + DEBUG_VALIDATE_LOAD_ON_EVICT = value; } - public interface Adapter + public interface Adapter & SaferState> { enum Shrink { EVICT, DONE, PERFORM_WITHOUT_LOCK } @@ -122,10 +133,10 @@ public class AccordCache implements CacheSize long estimateHeapSize(V value); long estimateShrunkHeapSize(Object shrunk); boolean validate(AccordCommandStore commandStore, K key, V value); - S safeRef(AccordCacheEntry node); + S safeRef(AccordCacheEntry node); default Comparator keyComparator() { return null; } - default AccordCacheEntry newEntry(K key, AccordCache.Type.Instance owner) + default AccordCacheEntry newEntry(K key, AccordCache.Type.Instance owner) { return AccordCacheEntry.createReadyToLoad(key, owner); } @@ -151,8 +162,8 @@ public class AccordCache implements CacheSize private final List> types = new CopyOnWriteArrayList<>(); final AccordCacheEntry.SaveExecutor saveExecutor; - private final IntrusiveLinkedList> evictQueue = new IntrusiveLinkedList<>(); - private final IntrusiveLinkedList> noEvictQueue = new IntrusiveLinkedList<>(); + private final IntrusiveLinkedList> evictQueue = new IntrusiveLinkedList<>(); + private final IntrusiveLinkedList> noEvictQueue = new IntrusiveLinkedList<>(); private long unreferencedBytes; private int unreferenced; @@ -169,8 +180,8 @@ public class AccordCache implements CacheSize } // note: only affects current contents after lock is released - @Override - public void setCapacity(long sizeInBytes) + // should NOT be called directly, should be called via AccordExecutor.setCapacity, so it may refresh its derived values + void setCapacity(long sizeInBytes) { maxSizeInBytes = sizeInBytes; tryShrinkOrEvict = true; @@ -181,7 +192,6 @@ public class AccordCache implements CacheSize this.shrinkingOn = shrinkingOn; } - @Override public long capacity() { return maxSizeInBytes; @@ -190,18 +200,18 @@ public class AccordCache implements CacheSize /** * Make sure we don't have any items lingering too long in the no evict queue, to avoid cache memory leaks */ - void processNoEvictQueue() + public void processNoEvictQueue() { - noEvictGeneration = (noEvictGeneration + 1) & 0xffff; + noEvictGeneration = (noEvictGeneration + 1) & GENERATION_MASK; if (noEvictQueue.isEmpty()) return; - Iterator> iter = noEvictQueue.iterator(); + Iterator> iter = noEvictQueue.iterator(); int skipCount = 3; while (skipCount > 0 && iter.hasNext()) { - AccordCacheEntry entry = iter.next(); - int age = (noEvictGeneration - entry.noEvictGeneration()) & 0xffff; + AccordCacheEntry entry = iter.next(); + int age = (noEvictGeneration - entry.noEvictGeneration()) & GENERATION_MASK; if (age >= entry.noEvictMaxAge()) { evictNoEvict.warn(entry, age, entry.noEvictMaxAge()); @@ -219,21 +229,21 @@ public class AccordCache implements CacheSize * Roughly respects LRU semantics when evicting. Might consider prioritising keeping MODIFIED nodes around * for longer to maximise the chances of hitting system tables fewer times (or not at all). */ - void tryShrinkOrEvict(Lock lock) + public void tryShrinkOrEvict(Lock lock) { if (!tryShrinkOrEvict) return; while (bytesCached > maxSizeInBytes && !evictQueue.isEmpty()) { - AccordCacheEntry node = evictQueue.peek(); + AccordCacheEntry node = evictQueue.peek(); shrinkOrEvict(lock, node); } tryShrinkOrEvict = false; } @VisibleForTesting - private void shrinkOrEvict(Lock lock, AccordCacheEntry node) + private void shrinkOrEvict(Lock lock, AccordCacheEntry node) { require(node.references() == 0); @@ -244,7 +254,7 @@ public class AccordCache implements CacheSize } else { - IntrusiveLinkedList> queue; + IntrusiveLinkedList> queue; queue = node.isNoEvict() ? noEvictQueue : evictQueue; node.unlink(); if (shrink == Shrink.DONE) @@ -272,7 +282,7 @@ public class AccordCache implements CacheSize } @VisibleForTesting - public void tryEvict(AccordCacheEntry node) + public void tryEvict(AccordCacheEntry node) { require(node.references() == 0); @@ -290,7 +300,6 @@ public class AccordCache implements CacheSize case LOADING: node.loading().loading.cancel(); case WAITING_TO_LOAD: - Invariants.paranoid(node.loadingOrWaiting().waiters == null); case LOADED: node.unlink(); evict(node, true); @@ -305,7 +314,7 @@ public class AccordCache implements CacheSize } } - public void saveWhenReadyExclusive(AccordCacheEntry entry, Runnable onSuccess) + public void saveWhenReadyExclusive(AccordCacheEntry entry, Runnable onSuccess) { if (!entry.isSavingOrWaiting() && !entry.saveWhenReady()) onSuccess.run(); @@ -313,7 +322,7 @@ public class AccordCache implements CacheSize entry.savingOrWaitingToSave().identity.onSuccess(onSuccess); } - private void evict(AccordCacheEntry node, boolean updateUnreferenced) + private void evict(AccordCacheEntry node, boolean updateUnreferenced) { if (logger.isTraceEnabled()) logger.trace("Evicting {}", node); @@ -333,29 +342,28 @@ public class AccordCache implements CacheSize --parent.size; // TODO (expected): use listeners - if (node.status() == LOADED && VALIDATE_LOAD_ON_EVICT) + if (node.status() == LOADED && DEBUG_VALIDATE_LOAD_ON_EVICT) owner.validateLoadEvicted(node); - AccordCacheEntry self = node.owner.remove(node.key()); + AccordCacheEntry self = node.owner.remove(node.key()); Invariants.require(self.references() == 0); require(self == node, "Leaked node detected; was attempting to remove %s but cache had %s", node, self); node.notifyListeners(Listener::onEvict); node.evicted(); } - Collection> load(LoadExecutor loadExecutor, P1 p1, P2 p2, AccordCacheEntry node) + Loading load(LoadExecutor loadExecutor, P1 p1, P2 p2, AccordCacheEntry node) { - Type parent = node.owner.parent(); - return node.load(loadExecutor, p1, p2).waiters(); + return node.load(loadExecutor, p1, p2); } - void loaded(AccordCacheEntry node, V value) + void loaded(AccordCacheEntry node, V value) { node.loaded(value); node.notifyListeners(Listener::onUpdate); } - void failedToLoad(AccordCacheEntry node) + void failedToLoad(AccordCacheEntry node) { Invariants.require(node.references() == 0); if (node.isUnqueued()) @@ -368,38 +376,38 @@ public class AccordCache implements CacheSize evict(node, true); } - void saved(AccordCacheEntry node, Object identity, Throwable fail) + void saved(AccordCacheEntry node, Object identity, Throwable fail) { if (node.saved(identity, fail) && node.references() == 0 && node.isUnqueued()) evictQueue.addFirst(node); // add to front since we have just saved, so we were eligible for eviction } - public > void release(S safeRef, AccordTask owner) + public & SaferState> void release(S safeRef, SafeTask owner) { safeRef.global().owner.release(safeRef, owner); } - public > Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) + public & SaferState> Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) { Type instance = new Type<>(keyClass, adapter, metrics); types.add(instance); return instance; } - public > Type newType( + public & SaferState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, Function quickShrink, TriFunction validateFunction, ToLongFunction heapEstimator, - Function, S> safeRefFactory, + Function, S> safeRefFactory, AccordCacheMetrics.Shard metrics) { return newType(keyClass, loadFunction, saveFunction, quickShrink, (i, j) -> j, (c, i, j) -> (V)j, validateFunction, heapEstimator, i -> 0, safeRefFactory, metrics); } - public > Type newType( + public & SaferState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, @@ -409,7 +417,7 @@ public class AccordCache implements CacheSize TriFunction validateFunction, ToLongFunction heapEstimator, ToLongFunction shrunkHeapEstimator, - Function, S> safeRefFactory, + Function, S> safeRefFactory, AccordCacheMetrics.Shard metrics) { return newType(keyClass, new FunctionalAdapter<>(loadFunction, saveFunction, quickShrink, @@ -426,18 +434,18 @@ public class AccordCache implements CacheSize public interface Listener { - default void onAdd(AccordCacheEntry state) {} - default void onUpdate(AccordCacheEntry state) {} - default void onEvict(AccordCacheEntry state) {} + default void onAdd(AccordCacheEntry state) {} + default void onUpdate(AccordCacheEntry state) {} + default void onEvict(AccordCacheEntry state) {} } - public class Type> implements CacheSize + public class Type & SaferState> implements CacheSize { - public class Instance implements Iterable> + public class Instance implements Iterable> { final AccordCommandStore commandStore; // TODO (desired): don't need to store key separately as stored in node; ideally use a hash set that allows us to get the current entry - private final Map> cache = new Object2ObjectHashMap<>(); + private final Map> cache = new Object2ObjectHashMap<>(); private List> listeners = null; // TODO (expected): update this after releasing the lock private OrderedKeys orderedKeys; @@ -447,50 +455,51 @@ public class AccordCache implements CacheSize this.commandStore = commandStore; } - public S acquire(K key) + public final S acquire(K key) { - AccordCacheEntry node = acquire(key, false); + AccordCacheEntry node = acquire(key, false); return adapter.safeRef(node); } - public S acquireIfLoaded(K key) + public final S acquireIfLoadedAndPermitted(K key) { - AccordCacheEntry node = acquire(key, true); + AccordCacheEntry node = acquire(key, true); if (node == null) return null; return adapter.safeRef(node); } - public S acquire(AccordCacheEntry node) + public final S acquire(AccordCacheEntry node) { Invariants.require(node.owner == this); acquireExisting(node, false); return adapter.safeRef(node); } - public void recordPreAcquired(AccordSafeState ref) + public final void recordPreAcquired(AccordCacheEntry entry) { - Invariants.require(ref.global().owner == this); - incrementCacheHits(); + Invariants.require(entry.owner == this); + if (entry.isLoaded()) incrementCacheHits(); + else incrementCacheMisses(); } - private AccordCacheEntry acquire(K key, boolean onlyIfLoaded) + private AccordCacheEntry acquire(K key, boolean onlyIfLoadedAndPermitted) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node == null - ? acquireAbsent(key, onlyIfLoaded) - : acquireExisting(node, onlyIfLoaded); + ? acquireAbsent(key, onlyIfLoadedAndPermitted) + : acquireExisting(node, onlyIfLoadedAndPermitted); } /* * Can only return a LOADING Node (or null) */ - private AccordCacheEntry acquireAbsent(K key, boolean onlyIfLoaded) + private AccordCacheEntry acquireAbsent(K key, boolean onlyIfLoaded) { incrementCacheMisses(); if (onlyIfLoaded) return null; - AccordCacheEntry node = adapter.newEntry(key, this); + AccordCacheEntry node = adapter.newEntry(key, this); node.increment(); Object prev = cache.put(key, node); @@ -507,7 +516,7 @@ public class AccordCache implements CacheSize /* * Can't return EVICTED or INITIALIZED */ - private AccordCacheEntry acquireExisting(AccordCacheEntry node, boolean onlyIfLoaded) + private AccordCacheEntry acquireExisting(AccordCacheEntry node, boolean onlyIfLoadedAndPermitted) { boolean isLoaded = node.isLoaded(); if (isLoaded) @@ -515,8 +524,11 @@ public class AccordCache implements CacheSize else incrementCacheMisses(); - if (onlyIfLoaded && !isLoaded) - return null; + if (onlyIfLoadedAndPermitted) + { + if (!isLoaded || node.hasFifoOrLocked()) + return null; + } if (node.increment() == 1) { @@ -528,21 +540,21 @@ public class AccordCache implements CacheSize return node; } - public void release(AccordSafeState safeRef, AccordTask owner) + public final void release(S safeRef, SafeTask owner) { K key = safeRef.global().key(); logger.trace("Releasing resources for {}: {}", key, safeRef); - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); - require(!safeRef.isUnsafe()); + require(!safeRef.isReleased()); require(safeRef.global() != null, "safeRef node is null for %s", key); require(safeRef.global() == node, "safeRef node not in map: %s != %s", safeRef.global(), node); require(node.references() > 0, "references (%d) are zero for %s (%s)", node.references(), key, node); require(node.isUnqueued()); boolean evict = false; - if (safeRef.hasUpdate()) + if (safeRef.isModified()) { V update = safeRef.current(); if (update != null) @@ -556,15 +568,11 @@ public class AccordCache implements CacheSize } node.notifyListeners(Listener::onUpdate); } - else if (node.isLoadingOrWaiting()) - { - node.loadingOrWaiting().remove(owner); - } else { evict = node.is(LOADED) && node.isNull(); } - safeRef.markUnsafe(); + node.remove(owner, safeRef.setReleased()); if (node.decrement() == 0) { @@ -597,9 +605,9 @@ public class AccordCache implements CacheSize tryShrinkOrEvict = true; } - AccordCacheEntry remove(K key) + final AccordCacheEntry remove(K key) { - AccordCacheEntry result = cache.remove(key); + AccordCacheEntry result = cache.remove(key); if (orderedKeys != null && result != null) orderedKeys.remove(key); return result; @@ -610,7 +618,7 @@ public class AccordCache implements CacheSize return Type.this; } - public Iterable keysBetween(K start, boolean startInclusive, K end, boolean endInclusive) + public final Iterable keysBetween(K start, boolean startInclusive, K end, boolean endInclusive) { if (orderedKeys == null) orderedKeys = new OrderedKeys<>(adapter.keyComparator(), cache.keySet()); @@ -619,15 +627,15 @@ public class AccordCache implements CacheSize } @Override - public Iterator> iterator() + public final Iterator> iterator() { return cache.values().iterator(); } - void validateLoadEvicted(AccordCacheEntry node) + final void validateLoadEvicted(AccordCacheEntry node) { @SuppressWarnings("unchecked") - AccordCacheEntry state = (AccordCacheEntry) node; + AccordCacheEntry state = (AccordCacheEntry) node; K key = state.key(); V evicted = state.tryGetFull(); if (evicted == null) @@ -650,46 +658,46 @@ public class AccordCache implements CacheSize } @VisibleForTesting - public AccordCacheEntry getUnsafe(K key) + public final AccordCacheEntry getUnsafe(K key) { return cache.get(key); } @VisibleForTesting - public boolean isReferenced(K key) + public final boolean isReferenced(K key) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null && node.references() > 0; } @VisibleForTesting - boolean keyIsReferenced(Object key, Class> valClass) + final boolean keyIsReferenced(Object key, Class> valClass) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null && node.references() > 0; } @VisibleForTesting - boolean keyIsCached(Object key, Class> valClass) + final boolean keyIsCached(Object key, Class> valClass) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null; } @VisibleForTesting - int references(Object key, Class> valClass) + final int references(Object key, Class> valClass) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null ? node.references() : 0; } - void notifyListeners(BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) + final void notifyListeners(BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) { notifyListeners(listeners, notify, node); notifyListeners(typeListeners, notify, node); } - void notifyListeners(List> listeners, BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) + final void notifyListeners(List> listeners, BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) { if (listeners != null) { @@ -699,20 +707,20 @@ public class AccordCache implements CacheSize } } - public void register(Listener l) + public final void register(Listener l) { if (listeners == null) listeners = new ArrayList<>(); listeners.add(l); } - public void unregister(Listener l) + public final void unregister(Listener l) { if (!tryUnregister(l)) throw illegalState("Listener was not registered"); } - public boolean tryUnregister(Listener l) + public final boolean tryUnregister(Listener l) { if (listeners == null || !listeners.remove(l)) return false; @@ -720,9 +728,23 @@ public class AccordCache implements CacheSize listeners = null; return true; } + + final boolean isCommandsForKey() + { + return getClass() == KeyInstance.class; + } } - private final Class keyClass; + // KeyInstance exists to provide us slightly easier discrimination about the Type an AccordCacheEntry is associated with + public final class KeyInstance extends Instance + { + public KeyInstance(AccordCommandStore commandStore) + { + super(commandStore); + } + } + + private final Class keyClass; // type of key, useful primarily for toString(), but also piggyback for deciding Instance type private Adapter adapter; private long bytesCached; private int size; @@ -735,6 +757,8 @@ public class AccordCache implements CacheSize public Type(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) { + // Integer and String permitted for testing, but the Invariant exists only to enforce that we construct the right kind of Instance + Invariants.require(keyClass == RoutingKey.class || keyClass == TxnId.class || keyClass == String.class || keyClass == Integer.class); this.keyClass = keyClass; this.adapter = adapter; this.objectSize = metrics.objectSize; @@ -750,9 +774,9 @@ public class AccordCache implements CacheSize } // can be safely garbage collected if empty - Instance newInstance(AccordCommandStore commandStore) + public Instance newInstance(AccordCommandStore commandStore) { - return new Instance(commandStore); + return keyClass == RoutingKey.class ? new KeyInstance(commandStore) : new Instance(commandStore); } private void incrementCacheHits() @@ -868,17 +892,17 @@ public class AccordCache implements CacheSize } @VisibleForTesting - AccordCacheEntry head() + AccordCacheEntry head() { - Iterator> iter = evictQueue.iterator(); + Iterator> iter = evictQueue.iterator(); return iter.hasNext() ? iter.next() : null; } @VisibleForTesting - AccordCacheEntry tail() + AccordCacheEntry tail() { - AccordCacheEntry last = null; - Iterator> iter = evictQueue.iterator(); + AccordCacheEntry last = null; + Iterator> iter = evictQueue.iterator(); while (iter.hasNext()) last = iter.next(); return last; @@ -889,7 +913,7 @@ public class AccordCache implements CacheSize return size() == 0; } - Iterable> evictionQueue() + public Iterable> evictionQueue() { return evictQueue::iterator; } @@ -920,14 +944,12 @@ public class AccordCache implements CacheSize return unreferencedBytes; } - @Override - public int size() + int size() { return cacheSize(); } - @Override - public long weightedSize() + long weightedSize() { return bytesCached; } @@ -938,10 +960,10 @@ public class AccordCache implements CacheSize return; type.register(new AccordCache.Listener<>() { - private final IdentityHashMap, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>(); + private final IdentityHashMap, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>(); @Override - public void onAdd(AccordCacheEntry state) + public void onAdd(AccordCacheEntry state) { CacheEvents.Add add = new CacheEvents.Add(); CacheEvents.Evict evict = new CacheEvents.Evict(); @@ -958,7 +980,7 @@ public class AccordCache implements CacheSize } @Override - public void onEvict(AccordCacheEntry state) + public void onEvict(AccordCacheEntry state) { CacheEvents.Evict event = pendingEvicts.remove(state); if (event == null) return; @@ -968,7 +990,7 @@ public class AccordCache implements CacheSize }); } - private static void updateMutable(AccordCache.Type type, AccordCacheEntry state, CacheEvents event) + private static void updateMutable(AccordCache.Type type, AccordCacheEntry state, CacheEvents event) { event.status = state.status().name(); @@ -988,7 +1010,7 @@ public class AccordCache implements CacheSize event.update(); } - static class FunctionalAdapter implements Adapter + static class FunctionalAdapter & SaferState> implements Adapter { final BiFunction load; final QuadFunction save; @@ -998,8 +1020,8 @@ public class AccordCache implements CacheSize final TriFunction validate; final ToLongFunction estimateHeapSize; final ToLongFunction estimateShrunkHeapSize; - final Function, S> newSafeRef; - final BiFunction.Instance, AccordCacheEntry> newNode; + final Function, S> newSafeRef; + final BiFunction.Instance, AccordCacheEntry> newNode; FunctionalAdapter(BiFunction load, QuadFunction save, @@ -1008,8 +1030,8 @@ public class AccordCache implements CacheSize TriFunction validate, ToLongFunction estimateHeapSize, ToLongFunction estimateShrunkHeapSize, - Function, S> newSafeRef, - BiFunction.Instance, AccordCacheEntry> newNode) + Function, S> newSafeRef, + BiFunction.Instance, AccordCacheEntry> newNode) { this.load = load; this.save = save; @@ -1083,13 +1105,13 @@ public class AccordCache implements CacheSize } @Override - public S safeRef(AccordCacheEntry node) + public S safeRef(AccordCacheEntry node) { return newSafeRef.apply(node); } @Override - public AccordCacheEntry newEntry(K key, Type.Instance owner) + public AccordCacheEntry newEntry(K key, Type.Instance owner) { return newNode.apply(key, owner); } @@ -1101,7 +1123,7 @@ public class AccordCache implements CacheSize } } - static class SettableWrapper extends FunctionalAdapter + static class SettableWrapper & SaferState> extends FunctionalAdapter { volatile BiFunction load; @@ -1111,9 +1133,9 @@ public class AccordCache implements CacheSize this.load = super.load; } - public static Adapter loadOnly(BiFunction load) + public static & SaferState> Adapter loadOnly(BiFunction load) { - SettableWrapper result = new SettableWrapper<>(new NoOpAdapter<>()); + SettableWrapper result = new SettableWrapper<>(new NoOpAdapter()); result.load = load; return result; } @@ -1125,7 +1147,7 @@ public class AccordCache implements CacheSize } } - static class NoOpAdapter implements Adapter + static class NoOpAdapter & SaferState> implements Adapter { @Override public V load(AccordCommandStore commandStore, K key) { return null; } @Override public Runnable save(AccordCommandStore commandStore, K key, @Nullable V value, @Nullable Object shrunk) { return null; } @@ -1136,10 +1158,10 @@ public class AccordCache implements CacheSize @Override public long estimateHeapSize(V value) { return 0; } @Override public long estimateShrunkHeapSize(Object shrunk) { return 0; } @Override public boolean validate(AccordCommandStore commandStore, K key, V value) { return false; } - @Override public S safeRef(AccordCacheEntry node) { return null; } + @Override public S safeRef(AccordCacheEntry node) { return null; } } - public static class CommandsForKeyAdapter implements Adapter + public static class CommandsForKeyAdapter implements Adapter { public static final CommandsForKeyAdapter CFK_ADAPTER = new CommandsForKeyAdapter(); private static int SHRINK_WITHOUT_LOCK = -1; @@ -1149,7 +1171,13 @@ public class AccordCache implements CacheSize @Override public CommandsForKey load(AccordCommandStore commandStore, RoutingKey key) { - return commandStore.loadCommandsForKey(key); + CommandsForKey cfk = AccordKeyspace.CommandsForKeyAccessor.load(commandStore.id(), (TokenKey) key); + if (cfk == null) + return null; + RedundantBefore.QuickBounds bounds = commandStore.safeGetRedundantBefore().get(key); + if (!Invariants.expect(bounds != null, "No RedundantBefore information found when loading key %s", key)) + return cfk; + return cfk.withCleanCfkBeforeAtLeast(bounds.cleanCfkBefore(), false); } @Override @@ -1160,7 +1188,7 @@ public class AccordCache implements CacheSize ByteBuffer bb = (ByteBuffer)serialized; serialized = bb.duplicate().position(prefixBytes(bb)); } - return commandStore.saveCommandsForKey(key, value, serialized); + return AccordKeyspace.CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey) key, value, serialized); } @Override @@ -1236,13 +1264,17 @@ public class AccordCache implements CacheSize @Override public boolean validate(AccordCommandStore commandStore, RoutingKey key, CommandsForKey value) { - return commandStore.validateCommandsForKey(key, value); + if (!Invariants.isParanoid()) + return true; + + CommandsForKey reloaded = AccordKeyspace.CommandsForKeyAccessor.load(commandStore.id(), (TokenKey) key); + return Objects.equals(value, reloaded); } @Override - public AccordSafeCommandsForKey safeRef(AccordCacheEntry node) + public SaferCommandsForKey safeRef(AccordCacheEntry node) { - return new AccordSafeCommandsForKey(node); + return new SaferCommandsForKey(node); } @Override @@ -1252,7 +1284,7 @@ public class AccordCache implements CacheSize } @Override - public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) + public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) { CommandsForKeyCacheEntry entry = new CommandsForKeyCacheEntry(key, owner); entry.readyToLoad(); @@ -1260,7 +1292,7 @@ public class AccordCache implements CacheSize } } - public static class CommandAdapter implements Adapter + public static class CommandAdapter implements Adapter { private static final int SHRINK_WITHOUT_LOCK = -1; @@ -1373,23 +1405,27 @@ public class AccordCache implements CacheSize @Override public boolean validate(AccordCommandStore commandStore, TxnId key, Command value) { - return commandStore.validateCommand(key, value); + if (!Invariants.isParanoid()) + return true; + + Command reloaded = commandStore.loadCommand(key); + return Objects.equals(value, reloaded); } @Override - public AccordSafeCommand safeRef(AccordCacheEntry node) + public SaferCommand safeRef(AccordCacheEntry node) { - return new AccordSafeCommand(node); + return new SaferCommand(node); } @Override - public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) + public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) { - AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); + AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); if (txnId.is(Txn.Kind.EphemeralRead)) { node.initialize(null); - int maxAge = (int)Math.min(0xff, 1 + DatabaseDescriptor.getReadRpcTimeout(TimeUnit.SECONDS)); + int maxAge = (int)Math.min(AGE_MASK, 1 + DatabaseDescriptor.getReadRpcTimeout(TimeUnit.SECONDS)); node.markNoEvict(owner.parent().parent().noEvictGeneration, maxAge); } else diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java new file mode 100644 index 0000000000..43ad91fedc --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java @@ -0,0 +1,1171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.function.BiConsumer; +import java.util.function.BiPredicate; + +import javax.annotation.Nullable; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.primitives.Ints; + +import accord.local.SafeState; +import accord.utils.ArrayBuffers.BufferList; +import accord.utils.IntrusiveLinkedList; +import accord.utils.IntrusiveLinkedListNode; +import accord.utils.Invariants; +import accord.utils.UnhandledEnum; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.execution.AccordCache.Adapter; +import org.apache.cassandra.service.accord.execution.AccordCache.Adapter.Shrink; +import org.apache.cassandra.utils.ObjectSizes; + +import static accord.utils.Invariants.nonNull; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.HOLD_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.UNLOCKED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.EVICTED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.FAILED_TO_LOAD; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.FAILED_TO_SAVE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.LOADED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.LOADING; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.MODIFIED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.SAVING; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.WAITING_TO_SAVE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntryQueue.PRIORITY_START_INDEX; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntryQueue.compare; + +/** + * Global (per CommandStore) state of a cached entity (Command or CommandsForKey). + */ +public class AccordCacheEntry & SaferState> extends IntrusiveLinkedListNode +{ + public enum Status + { + UNINITIALIZED, + + UNUSED1, // spacing to permit easier bit masks + + WAITING_TO_LOAD(UNINITIALIZED), + LOADING(WAITING_TO_LOAD), + + /** + * Consumers should never see this state + */ + FAILED_TO_LOAD(LOADING), + + UNUSED2, // spacing to permit easier bit masks + UNUSED3, // spacing to permit easier bit masks + UNUSED4, // spacing to permit easier bit masks + + LOADED(true, false, UNINITIALIZED, LOADING), + MODIFIED(true, false, LOADED), + + UNUSED5, // spacing to permit easier bit masks + UNUSED6, // spacing to permit easier bit masks + + WAITING_TO_SAVE(true, true, MODIFIED), + SAVING(true, true, MODIFIED, WAITING_TO_SAVE), + + /** + * Attempted to save but failed. Shouldn't normally happen unless we have a bug in serialization, + * or commit log has been stopped. + */ + FAILED_TO_SAVE(true, true, SAVING), + + UNUSED7, // spacing to permit easier bit masks + + EVICTED(WAITING_TO_LOAD, LOADING, LOADED, FAILED_TO_LOAD), + ; + + static final Status[] VALUES = values(); + static + { + MODIFIED.permittedFrom |= 1 << MODIFIED.ordinal(); + MODIFIED.permittedFrom |= 1 << SAVING.ordinal(); + MODIFIED.permittedFrom |= 1 << FAILED_TO_SAVE.ordinal(); + LOADED.permittedFrom |= 1 << SAVING.ordinal(); + LOADED.permittedFrom |= 1 << MODIFIED.ordinal(); + for (Status status : VALUES) + { + if (status.name().startsWith("UNUSED")) continue; + Invariants.require((status.ordinal() & IS_LOADED) != 0 == status.loaded); + Invariants.require(((status.ordinal() & IS_LOADED) != 0 && (status.ordinal() & IS_NESTED) != 0) == status.nested); + Invariants.require(((status.ordinal() & IS_LOADING_OR_WAITING_MASK) == IS_LOADING_OR_WAITING) == (status == LOADING || status == WAITING_TO_LOAD)); + Invariants.require(((status.ordinal() & IS_SAVING_OR_WAITING_MASK) == IS_SAVING_OR_WAITING) == (status == SAVING || status == WAITING_TO_SAVE)); + } + } + + final boolean loaded; + final boolean nested; + int permittedFrom; + + Status(Status ... statuses) + { + this(false, false, statuses); + } + + Status(boolean loaded, boolean nested, Status ... statuses) + { + this.loaded = loaded; + this.nested = nested; + for (Status status : statuses) + permittedFrom |= 1 << status.ordinal(); + } + } + + static final int STATUS_MASK = 0x0000001F; + static final int NO_EVICT = 0x00000020; + static final int SHRUNK = 0x00000040; + static final int LOCKED_MASK = 0x00000180; + static final int LOCKED_SHIFT = Integer.numberOfTrailingZeros(LOCKED_MASK); + static final int LOCKED_HOLDING_QUEUE = HOLD_QUEUE.ordinal() << LOCKED_SHIFT; + static final int IS_NOT_EVICTED = 0xF; + static final int IS_LOADED = 0x8; + static final int IS_NESTED = 0x4; + static final int IS_LOADING_OR_WAITING_MASK = 0x6; + static final int IS_LOADING_OR_WAITING = 0x2; + static final int IS_SAVING_OR_WAITING_MASK = 0xE; + static final int IS_SAVING_OR_WAITING = 0xC; + static final int GENERATION_SHIFT = 9; + static final int GENERATION_MASK = 0x7fff; + static final int AGE_SHIFT = 24; + static final int AGE_MASK = 0xff; + + static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCacheEntry<>(null, null)); + + private final K key; + final AccordCache.Type.Instance owner; + + private Object state; + /** + * Either a single AccordTask or a Queue object. The meaning of a single task is defined by various flags. + * If locked, then the task is not logically part of the queue unless LOCKED_HOLDING_QUEUE. + * If unlocked, or LOCKED_HOLDING_QUEUE, the task represents a single-item queue. + * If the task forms a single-item queue, whether it is FIFO or prioritised is determined by the task's isCacheQueuedFifo flag. + */ + private Object queue; // + private int status; + private int unsequenced; + int sizeOnHeap; + private volatile int references; + private static final AtomicIntegerFieldUpdater referencesUpdater = AtomicIntegerFieldUpdater.newUpdater(AccordCacheEntry.class, "references"); + + AccordCacheEntry(K key, AccordCache.Type.Instance owner) + { + this.key = key; + this.owner = owner; + } + + private RunnableStatus validate(RunnableStatus status) + { + Invariants.require(queue != null); + SafeTask head = queue instanceof AccordCacheEntryQueue ? ((AccordCacheEntryQueue) queue).peek() : (SafeTask) queue; + Invariants.require(isRunnable(head) || status == NOT_RUNNABLE); + return status; + } + + // TODO (expected): don't unwrap when only one entry, since this may cause us to flap when locking unsequenced tasks + private void maybeUnwrap(AccordCacheEntryQueue q) + { + int size = q.sequencedSize(); + switch (size) + { + case 0: + Invariants.require(q.unsequencedSize() == 0); + queue = isLocked() ? nonNull(q.lockedBy()) : null; + break; + + case 1: + if (isLocked() || q.unsequencedSize() > 0) + break; + queue = q.peek(); + } + } + + private boolean maybeUnwrap(AccordCacheEntryQueue q, SafeTask lockedBy) + { + if (q.sequencedSize() == 0) + { + Invariants.require(q.unsequencedSize() == 0); + queue = lockedBy; + return true; + } + return false; + } + + // assumes already queued with priority + final RunnableStatus moveToFifo(SafeTask task) + { + if (queue != task) + { + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; + RunnableStatus status = q.ensureHeadFifo(task); + if (status == NEWLY_BLOCKING_RUNNABLE && isLoaded()) + onChangedHead(q, null, q.peekBehind()); + return validate(isRunnable(task) ? status : NOT_RUNNABLE); + } + return validate(isRunnable(task) ? STILL_RUNNABLE : NOT_RUNNABLE); + } + + // drains ONLY those queued with addWaitingToLoad; addFifo are included in the result but are not removed from the collection + public final BufferList> drainWaitingToLoad() + { + Invariants.require(isLoading()); + Invariants.require(!isLocked()); + BufferList> list = new BufferList<>(); + if (queue != null) + { + if (queue instanceof AccordCacheEntryQueue) + { + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; + for (int i = q.priorityHead ; i < q.priorityTail ; ++i) + { + list.add(q.tasks[i]); + q.tasks[i] = null; + } + q.priorityHead = q.priorityTail = PRIORITY_START_INDEX; + for (int i = q.fifoHead ; i > q.fifoTail ; --i) + list.add(q.tasks[i]); + + maybeUnwrap(q); + } + else + { + SafeTask task = (SafeTask) queue; + list.add(task); + Invariants.require(!isLocked()); + if (!task.isCacheQueuedFifo()) + queue = null; + } + } + return list; + } + + final void remove(SafeTask task, boolean ownsLock) + { + if (queue instanceof AccordCacheEntryQueue) + { + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; + boolean remove; + boolean isLocked = isLocked() && q.isLocked(task); + Invariants.require(isLocked || !ownsLock); + if (isLocked) + { + // if locked, we've already released unsequenced/pririty/fifo positions unless isLockedHoldingQueue + remove = isLockedHoldingQueue(); + status &= ~LOCKED_MASK; + q.unlock(task); + } + else if (isLoading()) remove = true; + else if (task.isUnsequenced(this)) + { + if (task.isCacheQueued()) + { + if (!q.removeUnsequenced(task)) + releaseUnsequenced(q, task); + } + remove = false; + } + else remove = task.isCacheQueued(); + + if (remove) + { + boolean wasHead = remove && q.removeFifoOrPriority(task, false); + if (isLoaded() && wasHead) + { + unsequenced += q.drainUnsequenced(SafeTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); + onChangedHead(q, q.peek(), null); + } + } + + if (remove || isLocked) + maybeUnwrap(q); + } + else if (queue == task) + { + boolean isLocked = isLocked(); + Invariants.require(isLocked || !ownsLock); + if (isLocked) + { + status &= ~LOCKED_MASK; + } + else if (task.isUnsequenced(this)) + { + if (task.isCacheQueued()) --unsequenced; // nothing to release if we hit zero + else Invariants.require(isLoading()); + } + queue = null; + } + else + { + Invariants.require(!ownsLock); + if (task.isUnsequenced(this) && task.isCacheQueued()) + { + if (--unsequenced == 0 && queue != null && isLoaded()) + { + SafeTask head = (SafeTask) queue; + if (head.holdsLocksBetweenRuns()) + head.onChangeOptionalHeadStatus(this, NEWLY_RUNNABLE); + } + } + } + } + + final boolean isCommandsForKey() + { + return getClass() == SaferCommandsForKey.CommandsForKeyCacheEntry.class; + } + + final RunnableStatus headStatus(SafeTask task) + { + if (queue == task) + return validate(isRunnable(task) ? NEWLY_RUNNABLE : NOT_RUNNABLE); + + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; + if (q.peek() != task || !isRunnable(task)) + return NOT_RUNNABLE; + + return validate(q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); + } + + private AccordCacheEntryQueue ensureQueue() + { + if (queue instanceof AccordCacheEntryQueue) + return (AccordCacheEntryQueue) queue; + + SafeTask head = (SafeTask) this.queue; + AccordCacheEntryQueue q = new AccordCacheEntryQueue(); + if (isLocked()) + { + if (isLockedHoldingQueue()) + q.addFifo(head); + q.lock(head); + } + else if (head.isCacheQueuedFifo()) q.addFifo(head); + else q.addPrioritised(head); + this.queue = q; + return q; + } + + final void addWaitingToLoad(SafeTask task) + { + Invariants.require(isLoading()); + if (queue == null) queue = task; + else ensureQueue().addWaitingToLoad(task); + } + + final SafeTask head() + { + if (queue == null) + return null; + + if (queue instanceof AccordCacheEntryQueue) + return ((AccordCacheEntryQueue) queue).peek(); + + if (isLocked() && !isLockedHoldingQueue()) + return null; + + return (SafeTask) queue; + } + + final RunnableStatus addUnsequenced(SafeTask task) + { + Invariants.require(isLoaded()); + + SafeTask head = head(); + if (head != null && head.holdsLocksBetweenRuns()) + { + boolean wait = compare(task, head) > 0 || (unsequenced == 0 && head.hasIncrementalStarted()); + if (wait) + { + if (ensureQueue().addUnsequenced(task) && isLoaded() && unsequenced == 0) + head.onChangeHeadStatus(this, STILL_RUNNABLE_NEWLY_BLOCKING); + return NOT_RUNNABLE; + } + else + { + ++unsequenced; + if (unsequenced == 1 && isLoaded()) + head.onChangeHeadStatus(this, NOT_RUNNABLE); + return NEWLY_RUNNABLE; + } + } + + ++unsequenced; + return NEWLY_RUNNABLE; + } + + final int waitingCount() + { + Invariants.require(isLoading()); + return queue == null ? 0 : queue instanceof AccordCacheEntryQueue + ? ((AccordCacheEntryQueue)queue).sequencedSize() + : isLocked() == isLockedHoldingQueue() ? 1 : 0; + } + + public enum RunnableStatus + { + NOT_RUNNABLE, STILL_RUNNABLE, NEWLY_RUNNABLE, NEWLY_BLOCKING_RUNNABLE, STILL_RUNNABLE_NEWLY_BLOCKING + } + + private boolean isRunnable(SafeTask head) + { + return !head.holdsLocksBetweenRuns() || unsequenced == 0; + } + + private RunnableStatus add(SafeTask task, BiPredicate> add) + { + Object prev = this.queue; + if (prev == null) + { + queue = task; + return validate(isRunnable(task) ? NEWLY_RUNNABLE : NOT_RUNNABLE); + } + + AccordCacheEntryQueue q = ensureQueue(); + if (!add.test(q, task)) + { + if (isLoaded() && q.totalSize() == 2) + { + SafeTask head = q.peek(); + if (isRunnable(head)) + head.onChangeHeadStatus(this, STILL_RUNNABLE_NEWLY_BLOCKING); + } + return NOT_RUNNABLE; + } + + boolean isRunnable = isRunnable(task); + int sequencedSize = q.sequencedSize(); + int unsequencedSize = q.unsequencedSize(); + if (sequencedSize + unsequencedSize == 1) // could have one locked and one waiting + return validate(isRunnable ? NEWLY_RUNNABLE : NOT_RUNNABLE); + + if (isLoaded() && sequencedSize > 1) + onChangedHead(q, null, q.peekBehind()); + + return validate(isRunnable ? NEWLY_BLOCKING_RUNNABLE : NOT_RUNNABLE); + } + + final RunnableStatus addPrioritised(SafeTask task) + { + Invariants.require(!isLoading()); + return add(task, AccordCacheEntryQueue::addPrioritised); + } + + final RunnableStatus addFifo(SafeTask task) + { + return add(task, AccordCacheEntryQueue::addFifo); + } + + private void onChangedHead(AccordCacheEntryQueue q, @Nullable SafeTask notifyNewHead, @Nullable SafeTask notifyPrevHead) + { + if (notifyNewHead != null && isRunnable(notifyNewHead)) + notifyNewHead.onChangeHeadStatus(this, q.totalSize() <= 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); + if (notifyPrevHead != null && isRunnable(notifyPrevHead)) + notifyPrevHead.onChangeHeadStatus(this, NOT_RUNNABLE); + } + + public enum LockMode + { + /** + * Invalid as a parameter to lock methods, but represents the unlocked state + */ + UNLOCKED, + + /** + * If we're sequenced, remove ourselves from the relevant queue so that the next task can queue itself up. + */ + RELEASE_QUEUE, + + /** + * Hold onto our queue position (which we expect to be the head position, as we're queued to execute). + * This is used exclusively for INCR tasks that may hold onto their TxnId for multiple rounds of execution, + * and prevents later tasks from being scheduled when they will be unable to obtain the lock. + */ + HOLD_QUEUE, + + /** + * Skip all queue accounting (sequenced or unsequenced). This mode is used by optimistic referencing via + * tryLockCaches. + */ + UNQUEUED + } + + /** + * On lock we remove ourselves from the priority/fifo queues and notify the new head + */ + public final V lockExclusive(SafeTask owner, LockMode lockMode) + { + Invariants.require(!isLocked()); + Invariants.require(isRunnable(owner) || owner.isUnsequenced(this)); + + if (queue == owner) + { + Invariants.require(lockMode != UNLOCKED); + } + else if (queue == null) + { + queue = owner; + switch (lockMode) + { + default: throw UnhandledEnum.unknown(lockMode); + case UNLOCKED: throw UnhandledEnum.invalid(UNLOCKED); + case HOLD_QUEUE: throw UnhandledEnum.invalid(HOLD_QUEUE, "Must already be head of the queue"); + case RELEASE_QUEUE: + Invariants.require(owner.isUnsequenced(this) && owner.isCacheQueued(), "Must already be head of the queue"); + --unsequenced; + case UNQUEUED: + } + } + else + { + AccordCacheEntryQueue q = ensureQueue(); + switch (lockMode) + { + default: throw UnhandledEnum.unknown(lockMode); + case UNLOCKED: throw UnhandledEnum.invalid(UNLOCKED); + case HOLD_QUEUE: + Invariants.require(!owner.isUnsequenced(this)); + if (q.hasFifo()) Invariants.require(q.peekFifo() == owner); + else + { + boolean wasHead = q.removeIfPriorityHead(owner); + Invariants.require(wasHead); + q.addFifo(owner); + } + q.lock(owner); + break; + case RELEASE_QUEUE: + if (owner.isUnsequenced(this)) releaseUnsequenced(q, owner); + else + { + boolean wasHead = q.removeIfHead(owner); + Invariants.require(wasHead); + if (isLoaded()) + { + unsequenced += q.drainUnsequenced(SafeTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); + onChangedHead(q, q.peek(), null); + } + if (maybeUnwrap(q, owner)) + break; + } + case UNQUEUED: + q.lock(owner); + } + } + + status |= lockMode.ordinal() << LOCKED_SHIFT; + return getExclusive(); + } + + private void releaseUnsequenced(AccordCacheEntryQueue q, SafeTask release) + { + Invariants.require(release.isCacheQueued()); + if (--unsequenced == 0) + { + SafeTask head = q.peek(); + if (head != null && head.holdsLocksBetweenRuns() && isLoaded()) + onChangedHead(q, head, null); + } + } + + final boolean hasFifoOrLocked() + { + if (isLocked()) + return true; + + if (queue == null) + return false; + + if (queue instanceof SafeTask) + return ((SafeTask) queue).isCacheQueuedFifo(); + + return ((AccordCacheEntryQueue)queue).hasFifo(); + } + + final void unlink() + { + remove(); + } + + final boolean isUnqueued() + { + return isFree(); + } + + public final K key() + { + return key; + } + + public final int references() + { + return references; + } + + public final int increment() + { + return referencesUpdater.incrementAndGet(this); + } + + public final int decrement() + { + return referencesUpdater.decrementAndGet(this); + } + + final boolean isLocked() + { + return (status & LOCKED_MASK) != 0; + } + + final boolean isLockedHoldingQueue() + { + return (status & LOCKED_MASK) == LOCKED_HOLDING_QUEUE; + } + + final boolean isLoaded() + { + return (status & IS_LOADED) != 0; + } + + public final boolean isModified() + { + return (status & IS_NOT_EVICTED) >= MODIFIED.ordinal(); + } + + final boolean isNested() + { + Invariants.require(isLoaded()); + return (status & IS_NESTED) != 0; + } + + final boolean isShrunk() + { + return (status & SHRUNK) != 0; + } + + public final boolean is(Status status) + { + return (this.status & STATUS_MASK) == status.ordinal(); + } + + final boolean isLoading() + { + return (status & IS_LOADING_OR_WAITING_MASK) == IS_LOADING_OR_WAITING; + } + + final boolean isSavingOrWaiting() + { + return (status & IS_SAVING_OR_WAITING_MASK) == IS_SAVING_OR_WAITING; + } + + public final boolean isComplete() + { + return !is(LOADING) && !is(SAVING); + } + + final int noEvictGeneration() + { + Invariants.require(isNoEvict()); + return (status >>> GENERATION_SHIFT) & GENERATION_MASK; + } + + final int noEvictMaxAge() + { + Invariants.require(isNoEvict()); + return status >>> AGE_SHIFT; + } + + final boolean isNoEvict() + { + return (status & NO_EVICT) != 0; + } + + final int sizeOnHeap() + { + return sizeOnHeap; + } + + final void updateSize(AccordCache.Type parent) + { + // TODO (expected): we aren't weighing the keys + int newSizeOnHeap = Ints.saturatedCast(EMPTY_SIZE + estimateOnHeapSize(parent.adapter())); + parent.updateSize(newSizeOnHeap, newSizeOnHeap - sizeOnHeap, references == 0, true); + sizeOnHeap = newSizeOnHeap; + } + + final void initSize(AccordCache.Type parent) + { + // TODO (expected): we aren't weighing the keys + sizeOnHeap = Ints.saturatedCast(EMPTY_SIZE); + parent.updateSize(sizeOnHeap, sizeOnHeap, false, false); + parent.objectSize.increment(EMPTY_SIZE); + } + + @Override + public final String toString() + { + return "Node{" + status() + + ", key=" + key() + + ", references=" + references + + "}@" + Integer.toHexString(System.identityHashCode(this)); + } + + public final Status status() + { + return Status.VALUES[(status & STATUS_MASK)]; + } + + private void setStatus(Status newStatus) + { + Invariants.require((newStatus.permittedFrom & (1 << (status & STATUS_MASK))) != 0, "%s not permitted from %s", newStatus, status()); + setStatusUnsafe(newStatus); + } + + private void setStatusUnsafe(Status newStatus) + { + status &= ~STATUS_MASK; + status |= newStatus.ordinal(); + } + + public final void initialize(V value) + { + Invariants.require(state == null); + setStatus(LOADED); + state = value; + } + + public final void readyToLoad() + { + Invariants.require(state == null); + setStatus(WAITING_TO_LOAD); + } + + public final void markNoEvict(int generation, int maxAge) + { + Invariants.require((maxAge & ~AGE_MASK) == 0); + Invariants.require((generation & ~GENERATION_MASK) == 0); + status |= NO_EVICT; + status |= generation << GENERATION_SHIFT; + status |= maxAge << AGE_SHIFT; + } + + final void notifyListeners(BiConsumer, AccordCacheEntry> notify) + { + owner.notifyListeners(notify, this); + } + + interface LoadExecutor + { + IOTask load(P1 p1, P2 p2, AccordCacheEntry entry); + } + + // functions as both an identity object, and a register of listeners + public static class UniqueSave + { + @Nullable List onSuccess; + void onSuccess(Runnable onSuccess) + { + if (this.onSuccess == null) + this.onSuccess = new ArrayList<>(); + this.onSuccess.add(onSuccess); + } + + static void notify(List onSuccess) + { + if (onSuccess != null) + { + onSuccess.forEach(run -> { + try { run.run(); } + catch (Throwable t) + { + Thread thread = Thread.currentThread(); + thread.getUncaughtExceptionHandler().uncaughtException(thread, t); + } + }); + } + } + } + + interface SaveExecutor + { + Cancellable save(AccordCacheEntry saving, UniqueSave identity, Runnable save); + } + + final Loading load(LoadExecutor loadExecutor, P1 p1, P2 p2) + { + Invariants.require(is(WAITING_TO_LOAD), "%s", this); + + Loading loading = new Loading(loadExecutor.load(p1, p2, this)); + setStatus(LOADING); + state = loading; + return loading; + } + + public final Loading testLoad() + { + Invariants.require(is(WAITING_TO_LOAD)); + Loading loading = new Loading(null); + setStatus(LOADING); + state = loading; + return loading; + } + + public final Loading loading() + { + Invariants.require(is(LOADING), "%s", this); + return (Loading) state; + } + + // must own the cache's lock when invoked. this is true of most methods in the class, + // but this one is less obvious so named as to draw attention + public final V getExclusive() + { + Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); + Invariants.require(isLoaded(), "%s", this); + if (isShrunk()) + { + AccordCache.Type parent = owner.parent(); + inflate(owner.commandStore, key, parent.adapter()); + updateSize(parent); + } + + return (V) maybeUnwrap(); + } + + public final void releaseExclusive(S safeState, SafeTask task) + { + owner.release(safeState, task); + } + + public final Object getOrShrunkExclusive() + { + Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); + Invariants.require(isLoaded(), "%s", this); + return maybeUnwrap(); + } + + public V tryGetExclusive() + { + Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); + if (!isLoaded() || isShrunk()) + return null; + return (V) maybeUnwrap(); + } + + private Object maybeUnwrap() + { + return isNested() ? ((Nested)state).state : state; + } + + // must own the cache's lock when invoked + void setExclusive(V value) + { + if (value == state) + return; + + Saving cancel = is(SAVING) ? ((Saving)state) : null; + if (is(WAITING_TO_SAVE)) + { + ((WaitingToSave) state).state = value; + if (owner.parent().adapter().canSave(value, null)) + save(); + } + else + { + setStatus(MODIFIED); + state = value; + } + updateSize(owner.parent()); + // TODO (expected): do we want to cancel in-progress saving? + if (cancel != null && cancel.identity.onSuccess == null) + cancel.saving.cancel(); + } + + public void loaded(V value) + { + setStatus(LOADED); + state = value; + updateSize(owner.parent()); + } + + public void testLoaded(V value) + { + setStatus(LOADED); + state = value; + } + + public void failedToLoad() + { + setStatus(FAILED_TO_LOAD); + state = null; + } + + Shrink tryShrink() + { + if (!isLoaded()) + return Shrink.EVICT; + + AccordCache.Type parent = owner.parent(); + Adapter adapter = parent.adapter(); + if (isShrunk() || state == null) + return Shrink.EVICT; + + V cur = (V) maybeUnwrap(); + Shrink shrink = adapter.decideFullShrink(key, cur); + if (shrink == Shrink.PERFORM_WITHOUT_LOCK) + return Shrink.PERFORM_WITHOUT_LOCK; + + Object upd = adapter.fullShrink(key, cur); + if (upd == null || upd == cur) + return Shrink.EVICT; + applyShrink(parent, cur, upd); + return Shrink.DONE; + } + + V tryGetFull() + { + return isShrunk() ? null : (V) maybeUnwrap(); + } + + Object tryGetShrunk() + { + return isShrunk() ? maybeUnwrap() : null; + } + + boolean isNull() + { + return state == null; + } + + boolean saveWhenReady() + { + V full = isShrunk() ? null : (V)state; + Object shrunk = isShrunk() ? state : null; + if (owner.parent().adapter().canSave(full, shrunk)) + return save(); + + setStatus(WAITING_TO_SAVE); + UniqueSave identity = new UniqueSave(); + state = new WaitingToSave<>(identity, state); + return true; + } + + /** + * Submits a save runnable to the specified executor. When the runnable + * has completed, the state save will have either completed or failed. + */ + @VisibleForTesting + boolean save() + { + WaitingToSave waitingToSave = is(WAITING_TO_SAVE) ? (WaitingToSave)state : null; + Object state = waitingToSave == null ? this.state : waitingToSave.state; + V full = isShrunk() ? null : (V)state; + Object shrunk = isShrunk() ? state : null; + Runnable save = owner.parent().adapter().save(owner.commandStore, key, full, shrunk); + + UniqueSave identity = waitingToSave == null ? new UniqueSave() : waitingToSave.identity; + if (null == save) // null mutation -> null Runnable -> no change on disk + { + setStatus(LOADED); + if (waitingToSave != null) + this.state = state; + UniqueSave.notify(identity.onSuccess); + return false; + } + else + { + setStatus(SAVING); + Cancellable saving = owner.parent().parent().saveExecutor.save(this, identity, save); + this.state = new Saving(saving, identity, state); + return true; + } + } + + boolean saved(Object identity, Throwable fail) + { + if (identity instanceof UniqueSave) + UniqueSave.notify(((UniqueSave) identity).onSuccess); + + if (!is(SAVING)) + return false; + + Saving saving = (Saving) state; + if (saving.identity != identity) + return false; + + if (fail != null) + { + setStatus(FAILED_TO_SAVE); + state = new FailedToSave(fail, ((Saving)state).state); + return false; + } + else + { + setStatus(LOADED); + state = saving.state; + return true; + } + } + + protected void saved() + { + Invariants.require(is(MODIFIED)); + setStatus(LOADED); + } + + public SavingOrWaitingToSave savingOrWaitingToSave() + { + return (SavingOrWaitingToSave) state; + } + + public AccordCacheEntry evicted() + { + if (isNoEvict()) + setStatusUnsafe(EVICTED); + else setStatus(EVICTED); + state = null; + return this; + } + + public Throwable failure() + { + return ((FailedToSave)state).cause; + } + + void tryApplyShrink(Object cur, Object upd, IntrusiveLinkedList> queue) + { + if (references() > 0 || !isUnqueued()) + return; + + if (isLoaded() && maybeUnwrap() == cur && upd != cur && upd != null) + applyShrink(owner.parent(), cur, upd); + queue.addLast(this); + } + + private void applyShrink(AccordCache.Type parent, Object cur, Object upd) + { + if (isNested()) ((Nested)this.state).state = upd; + else this.state = upd; + status |= SHRUNK; + updateSize(parent); + } + + private void inflate(AccordCommandStore commandStore, K key, Adapter adapter) + { + Invariants.require(isShrunk()); + if (isNested()) + { + Nested nested = (Nested) state; + nested.state = adapter.inflate(commandStore, key, nested.state); + } + else + { + state = adapter.inflate(commandStore, key, state); + } + status &= ~SHRUNK; + } + + private long estimateOnHeapSize(Adapter adapter) + { + Object current = maybeUnwrap(); + if (current == null) return 0; + else if (isShrunk()) return adapter.estimateShrunkHeapSize(current); + return adapter.estimateHeapSize((V)current); + } + + public static class Loading + { + final IOTask loading; + + Loading(IOTask loading) + { + this.loading = loading; + } + } + + static class Nested + { + Object state; + } + + static class SavingOrWaitingToSave extends Nested + { + final UniqueSave identity; + + SavingOrWaitingToSave(UniqueSave identity, Object state) + { + this.identity = identity; + this.state = state; + } + } + + static class Saving extends SavingOrWaitingToSave + { + final Cancellable saving; + + Saving(Cancellable saving, UniqueSave identity, Object state) + { + super(identity, state); + this.saving = saving; + } + } + + static class WaitingToSave extends SavingOrWaitingToSave + { + WaitingToSave(UniqueSave identity, Object state) + { + super(identity, state); + } + } + + static class FailedToSave extends Nested + { + final Throwable cause; + + FailedToSave(Throwable cause, Object state) + { + this.cause = cause; + this.state = state; + } + + public Throwable failure() + { + return cause; + } + } + + public static & SaferState> AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) + { + AccordCacheEntry node = new AccordCacheEntry<>(key, owner); + node.readyToLoad(); + return node; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java new file mode 100644 index 0000000000..06025e49c7 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java @@ -0,0 +1,510 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.Arrays; + +import accord.utils.Invariants; +import accord.utils.SortedArrays; +import accord.utils.TriConsumer; + +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; + +class AccordCacheEntryQueue +{ + static final AccordCacheEntryQueue EMPTY = new AccordCacheEntryQueue(0); + + private static final int DEFAULT_CAPACITY = 4; + static final int LOCKED_INDEX = 0; + static final int PRIORITY_START_INDEX = LOCKED_INDEX + 1; + /** + * [priorityHead..priorityTail) stores a priority-sorted list of tasks + * (fifoTail...fifoHead] stores a fifo queue that runs ahead of any priority tasks + * (fifoTail-unsequencedCount...fifoTail] stores unsequenced tasks that are waiting for a queued incremental task. + * This only happens for TxnId cache entries, since they may lockAndHoldQueue. Once the lock is released, + * any pending unsequenced tasks are notified and immediately made (irrevocably) runnable for this entry. + */ + SafeTask[] tasks; + // TODO (expected): use bytes/shorts for indexes to keep size down, and have an expanded version of the Queue + // with better algorithmic complexity (e.g. Hash -> IntrusivePriorityHeap) + int priorityHead, priorityTail, fifoHead, fifoTail; + int unsequencedSize; + + public AccordCacheEntryQueue() + { + this(DEFAULT_CAPACITY); + } + + public AccordCacheEntryQueue(int capacity) + { + tasks = new SafeTask[capacity]; + priorityHead = priorityTail = PRIORITY_START_INDEX; + fifoHead = fifoTail = capacity - 1; + } + + AccordCacheEntryQueue(AccordCacheEntryQueue copy) + { + tasks = copy.tasks.clone(); + priorityHead = copy.priorityHead; + priorityTail = copy.priorityTail; + fifoHead = copy.fifoHead; + fifoTail = copy.fifoTail; + } + + // returns true if no fifo tasks already queue (i.e. so we become head) + boolean addFifo(SafeTask task) + { + ensureCapacity(); + boolean isHead = fifoHead == fifoTail; + if (unsequencedSize > 0) // simply displace the unsequence task, as they're an unordered list + tasks[fifoTail - unsequencedSize] = tasks[fifoTail]; + tasks[fifoTail--] = task; + validate(); + return isHead; + } + + boolean addUnsequenced(SafeTask task) + { + Invariants.require(task.isUnsequenced()); + ensureCapacity(); + tasks[fifoTail - unsequencedSize++] = task; + return unsequencedSize == 1 && sequencedSize() == 1; + } + + boolean isLocked(SafeTask task) + { + return tasks[LOCKED_INDEX] == task; + } + + SafeTask lockedBy() + { + return tasks[LOCKED_INDEX]; + } + + boolean removeIfHead(SafeTask task) + { + return removeIfFifoHead(task) || removeIfPriorityHead(task); + } + + boolean removeIfFifoHead(SafeTask task) + { + if (!hasFifo()) + return false; + + if (task != tasks[fifoHead]) + return false; + tasks[fifoHead--] = null; + return true; + } + + boolean removeIfPriorityHead(SafeTask task) + { + if (!hasPriority()) + return false; + + if (task != tasks[priorityHead]) + return false; + tasks[priorityHead++] = null; + return true; + } + + void lock(SafeTask task) + { + tasks[LOCKED_INDEX] = task; + } + + void unlock(SafeTask task) + { + Invariants.require(tasks[LOCKED_INDEX] == task); + tasks[LOCKED_INDEX] = null; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + boolean addPrioritised(SafeTask task) + { + ensureCapacity(); + int insertPos = Arrays.binarySearch(tasks, priorityHead, priorityTail, task, AccordCacheEntryQueue::compare); + if (insertPos < 0) + insertPos = -1 - insertPos; + + if (priorityHead == PRIORITY_START_INDEX || insertPos > (priorityTail + priorityHead) / 2) + { + System.arraycopy(tasks, insertPos, tasks, insertPos + 1, priorityTail - insertPos); + tasks[insertPos] = task; + priorityTail++; + } + else + { + System.arraycopy(tasks, priorityHead, tasks, priorityHead - 1, insertPos - priorityHead); + tasks[insertPos - 1] = task; + priorityHead--; + } + + validate(); + return fifoHead == fifoTail && tasks[priorityHead] == task; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + void addWaitingToLoad(SafeTask task) + { + ensureCapacity(); + tasks[priorityTail++] = task; + } + + private boolean hasTailRoom() + { + if (priorityTail + unsequencedSize <= fifoTail) + return true; + Invariants.require(priorityTail + unsequencedSize == 1 + fifoTail); + return false; + } + + private void ensureCapacity() + { + if (!hasTailRoom()) + { + if (fifoHead == fifoTail && unsequencedSize == 0 && fifoTail < tasks.length - 1) + fifoHead = fifoTail = tasks.length - 1; + else if (priorityHead == priorityTail && priorityHead > PRIORITY_START_INDEX) + priorityHead = priorityTail = PRIORITY_START_INDEX; + else if (totalSize() >= (tasks.length - 1) / 2) compact(new SafeTask[tasks.length * 2]); + else compact(tasks); + Invariants.require(hasTailRoom()); + } + } + + private void compact(SafeTask[] into) + { + if (priorityHead == priorityTail) priorityHead = priorityTail = PRIORITY_START_INDEX; + else + { + int priorityLength = priorityTail - priorityHead; + System.arraycopy(tasks, priorityHead, into, PRIORITY_START_INDEX, priorityLength); + int newTail = PRIORITY_START_INDEX + priorityLength; + Invariants.require(newTail <= priorityTail); + if (into == tasks) + Arrays.fill(into, newTail, priorityTail, null); + priorityHead = PRIORITY_START_INDEX; + priorityTail = newTail; + } + + if (fifoHead == fifoTail && unsequencedSize == 0) fifoHead = fifoTail = into.length - 1; + else + { + int fifoLength = fifoHead - fifoTail; + int copyLength = fifoLength + unsequencedSize; + int copyFrom = (fifoTail - unsequencedSize) + 1; + int copyTo = into.length - copyLength; + Invariants.require(copyTo >= copyFrom); + System.arraycopy(tasks, copyFrom, into, copyTo, copyLength); + if (into == tasks) + Arrays.fill(into, copyFrom, copyTo, null); + fifoHead = into.length - 1; + fifoTail = fifoHead - fifoLength; + } + + if (tasks != into) + { + into[LOCKED_INDEX] = tasks[LOCKED_INDEX]; + tasks = into; + } + validate(); + } + + private void validate() + { + for (int i = PRIORITY_START_INDEX; i < priorityHead; ++i) + Invariants.require(tasks[i] == null); + for (int i = priorityHead; i < priorityTail; ++i) + Invariants.require(tasks[i] != null); + for (int i = priorityTail; i <= fifoTail - unsequencedSize; ++i) + Invariants.require(tasks[i] == null); + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoHead; ++i) + Invariants.require(tasks[i] != null); + for (int i = fifoHead + 1; i < tasks.length; ++i) + Invariants.require(tasks[i] == null); + } + + SafeTask peek() + { + if (hasFifo()) return tasks[fifoHead]; + if (hasPriority()) return tasks[priorityHead]; + return null; + } + + SafeTask peekFifo() + { + return hasFifo() ? tasks[fifoHead] : null; + } + + // second task + SafeTask peekBehind() + { + int fifoSize = fifoSize(); + if (fifoSize > 1) + return tasks[fifoHead - 1]; + int priorityIndex = priorityHead + (1 - fifoSize); + if (priorityIndex < priorityTail) + return tasks[priorityIndex]; + return null; + } + + boolean hasFifo() + { + return fifoHead != fifoTail; + } + + boolean hasPriority() + { + return priorityHead != priorityTail; + } + + boolean hasUnsequenced() + { + return unsequencedSize > 0; + } + + int sequencedSize() + { + return prioritySize() + fifoSize(); + } + + int unsequencedSize() + { + return unsequencedSize; + } + + int totalSize() + { + return sequencedSize() + unsequencedSize; + } + + int prioritySize() + { + return priorityTail - priorityHead; + } + + int fifoSize() + { + return fifoHead - fifoTail; + } + + // true iff was head + boolean removeFifoOrPriority(SafeTask task, boolean permitMissing) + { + int fifoIndex = fifoIndexOf(task); + if (fifoIndex >= 0) + { + if (fifoIndex == fifoHead) + { + tasks[fifoHead--] = null; + validate(); + return true; + } + else + { + if (remove(fifoIndex, fifoTail + 1, fifoHead + 1)) ++fifoTail; + else --fifoHead; + validate(); + return false; + } + } + + int priorityIndex = priorityIndexOf(task); + Invariants.require(priorityIndex >= 0 || permitMissing); + if (priorityIndex >= 0) + { + if (priorityIndex == priorityHead) + { + tasks[priorityHead++] = null; + return !hasFifo(); + } + + if (remove(priorityIndex, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return false; + } + + return false; + } + + boolean removeUnsequenced(SafeTask task) + { + int unsequencedIndex = unsequencedIndexOf(task); + if (unsequencedIndex < 0) + return false; + + --unsequencedSize; + tasks[unsequencedIndex] = tasks[fifoTail - unsequencedSize]; + tasks[fifoTail - unsequencedSize] = null; + return true; + } + + // return true IFF was head + private boolean removePriority(SafeTask task, boolean permitAbsent) + { + int i = priorityIndexOf(task); + if (i < 0) + { + Invariants.require(permitAbsent); + return false; + } + + boolean wasHead = i == priorityHead; + if (remove(i, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return wasHead; + } + + // return true if we move the start forwards, false if we moved the end back + private boolean remove(int i, int start, int end) + { + if (i < (start + end) / 2) + { + System.arraycopy(tasks, start, tasks, start + 1, i - start); + tasks[start] = null; + return true; + } + else + { + System.arraycopy(tasks, i + 1, tasks, i, end - (i + 1)); + tasks[end - 1] = null; + return false; + } + } + + boolean contains(SafeTask task) + { + return indexOf(task) >= 0; + } + + private int indexOf(SafeTask task) + { + if (tasks[priorityHead] == task) + return priorityHead; + + if (tasks[fifoHead] == task) + return fifoHead; + + int i = priorityIndexOf(task); + if (i >= 0) + return i; + + return fifoIndexOf(task); + } + + private int priorityIndexOf(SafeTask task) + { + if (priorityTail - priorityHead > 16) + { + if (tasks[priorityHead] == task) + return priorityHead; + + return Arrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, AccordCacheEntryQueue::compare); + } + + for (int i = priorityHead; i < priorityTail; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int fifoIndexOf(SafeTask task) + { + for (int i = fifoHead; i > fifoTail; --i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int unsequencedIndexOf(SafeTask task) + { + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoTail; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + int drainUnsequenced(TriConsumer, P1, P2> forEach, P1 p1, P2 p2) + { + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoTail; ++i) + { + SafeTask task = tasks[i]; + tasks[i] = null; + // should not be reentrant + forEach.accept(task, p1, p2); + } + int count = unsequencedSize; + unsequencedSize = 0; + return count; + } + + AccordCacheEntry.RunnableStatus ensureHeadFifo(SafeTask task) + { + if (hasFifo()) + { + Invariants.require(tasks[fifoHead] == task); + return NOT_RUNNABLE; + } + + if (tasks[priorityHead] == task) + { + tasks[priorityHead++] = null; + addFifo(task); + return STILL_RUNNABLE; + } + else + { + boolean wasPriorityHead = removePriority(task, false); + boolean isFifoHead = addFifo(task); + if (!isFifoHead) + return NOT_RUNNABLE; + if (wasPriorityHead) + return STILL_RUNNABLE; + if (hasPriority() || hasUnsequenced()) + return NEWLY_BLOCKING_RUNNABLE; + return NEWLY_RUNNABLE; + } + } + + static int compare(SafeTask a, SafeTask b) + { + Invariants.require(a != null && b != null); + int c = Long.compare(a.position, b.position); + if (c == 0) + c = a.executionContext().executionKind().compareTo(b.executionContext().executionKind()); + if (c == 0) + c = Long.compare(a.createdAt, b.createdAt); + return c; + } + + public boolean hasQueued() + { + return hasFifo() || hasPriority(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java new file mode 100644 index 0000000000..37ba493d32 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java @@ -0,0 +1,813 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.stream.Stream; + +import accord.api.Agent; +import accord.api.ExclusiveAsyncExecutor; +import accord.api.RoutingKey; +import accord.impl.AbstractAsyncExecutor; +import accord.local.Command; +import accord.local.cfk.CommandsForKey; +import accord.primitives.TxnId; +import accord.utils.ArrayBuffers; +import accord.utils.Invariants; +import accord.utils.UnhandledEnum; +import accord.utils.async.AsyncCallbacks.CallAndCallback; +import accord.utils.async.AsyncCallbacks.RunOrFail; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.cache.CacheSize; +import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; +import org.apache.cassandra.concurrent.Shutdownable; +import org.apache.cassandra.config.AccordConfig; +import org.apache.cassandra.config.AccordConfig.QueueBalancingModel; +import org.apache.cassandra.config.AccordConfig.QueuePriorityModel; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.metrics.AccordCacheMetrics; +import org.apache.cassandra.metrics.AccordExecutorMetrics; +import org.apache.cassandra.metrics.AccordReplicaMetrics; +import org.apache.cassandra.metrics.AccordSystemMetrics; +import org.apache.cassandra.metrics.LogLinearDecayingHistograms; +import org.apache.cassandra.metrics.LogLinearDecayingHistograms.LogLinearDecayingHistogram; +import org.apache.cassandra.metrics.ShardedDecayingHistograms; +import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistogramsShard; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.SaveExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.UniqueSave; +import org.apache.cassandra.service.accord.execution.ExclusiveExecutor.ExclusiveExecutorTask; +import org.apache.cassandra.service.accord.execution.IOTaskWrapper.WrappableIOTask; +import org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup; +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; +import org.apache.cassandra.service.accord.execution.Task.GlobalGroup; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Condition; +import org.apache.cassandra.utils.concurrent.Future; + +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.AccordCache.CommandAdapter.COMMAND_ADAPTER; +import static org.apache.cassandra.service.accord.execution.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; +import static org.apache.cassandra.service.accord.execution.AccordCache.registerJfrListener; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.EVICTED; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.OTHER; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_SCAN; +import static org.apache.cassandra.service.accord.execution.Task.State.EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED; +import static org.apache.cassandra.service.accord.execution.Task.State.UNREGISTERED; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.COUNTER_MASKS; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.overflowBit; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.selectByOverflowBits; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.setOverflowWhenLessEqual; + +/** + * NOTE: We assume that NO BLOCKING TASKS are submitted to this executor AND WAITED ON by another task executing on this executor. + * (as we do not immediately schedule additional threads for submitted tasks, but schedule new threads only if necessary when the submitting execution completes) + */ +public abstract class AccordExecutor implements CacheSize, LoadExecutor, Boolean>, SaveExecutor, Shutdownable, AbstractAsyncExecutor +{ + static final QueuePriorityModel PRIORITY_MODEL; + static final QueueBalancingModel BALANCING_MODEL; + static final long AGE_TO_FIFO; + // PRIORITY_FAIR blends two strategies (flow: least fairly serviced; age: earliest-queued work) by deficit + // round-robin; weights of BLEND_TOTAL come from a single imbalance ramp (onset..onset+width) trading age->flow. + static final int BLEND_SHIFT = 6, BLEND_TOTAL = 1 << BLEND_SHIFT; + static final int FLOW_ONSET, FLOW_WIDTH_SHIFT; + static final boolean BALANCE_BY_POSITION; + static final long GLOBAL_QUEUE_LIMITS, EXCLUSIVE_QUEUE_LIMITS; + static final int NONSYNC_MIN_BATCH_SIZE, NONSYNC_MAX_BATCH_SIZE, NONSYNC_BLOCKED_LIMIT; + static final boolean NONSYNC_ENABLED; + private static final long LOADING_GROUPS = overflowBit(LOAD) | overflowBit(RANGE_LOAD) | overflowBit(RANGE_SCAN); + + static + { + AccordConfig config = DatabaseDescriptor.getAccord(); + AGE_TO_FIFO = config.queue_priority_age_to_fifo.to(TimeUnit.MICROSECONDS); + PRIORITY_MODEL = config.queue_priority_model != null ? config.queue_priority_model : QueuePriorityModel.HLC_FIFO; + BALANCING_MODEL = config.queue_balancing_model != null ? config.queue_balancing_model : QueueBalancingModel.BLENDED_PRIORITY_PHASE_FAIR; + FLOW_ONSET = config.queue_flow_imbalance_onset == null ? 4 : config.queue_flow_imbalance_onset; + FLOW_WIDTH_SHIFT = config.queue_flow_imbalance_width_shift == null ? 5 : config.queue_flow_imbalance_width_shift; + NONSYNC_MIN_BATCH_SIZE = config.queue_nonsync_min_batch_size == null ? 16 : config.queue_nonsync_min_batch_size; + NONSYNC_MAX_BATCH_SIZE = config.queue_nonsync_max_batch_size == null ? 64 : config.queue_nonsync_max_batch_size; + NONSYNC_ENABLED = config.queue_nonsync_enabled == null || config.queue_nonsync_enabled; + NONSYNC_BLOCKED_LIMIT = config.queue_nonsync_blocked_limit == null ? 8 : config.queue_nonsync_blocked_limit; + Invariants.require(FLOW_ONSET >= 0 && FLOW_WIDTH_SHIFT >= 0); + switch (BALANCING_MODEL) + { + default: throw new UnhandledEnum(BALANCING_MODEL); + case PRIORITY_ONLY: + case BLENDED_PRIORITY_PHASE_FAIR: + BALANCE_BY_POSITION = true; + break; + case PHASE_ONLY: + case PHASE_FAIR: + BALANCE_BY_POSITION = false; + } + + { + // TODO (required): pick default max loads/saves/range loads based on number of threads + long global = COUNTER_MASKS, exclusive = COUNTER_MASKS; + global ^= (0x7fL ^ 1) << (RANGE_SCAN.ordinal() * 8); + if (config.queue_active_limits != null) + { + long[] limits = parseEnumParams(config.queue_active_limits, "queue_active_limits"); + global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), global, limits[0]); + exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), global, limits[1]); + } + GLOBAL_QUEUE_LIMITS = global; + EXCLUSIVE_QUEUE_LIMITS = exclusive; + } + + } + + public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); + + public interface AccordExecutorFactory + { + AccordExecutor get(int executorId, Mode mode, int threads, IntFunction name, Agent agent); + } + + public enum Mode { RUN_WITH_LOCK, RUN_WITHOUT_LOCK } + + // WARNING: this is a shared object, so close is NOT idempotent + public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable + { + final AccordExecutor executor; + + public ExclusiveGlobalCaches(AccordExecutor executor, AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) + { + super(global, commands, commandsForKey); + this.executor = executor; + } + + @Override + public void close() + { + executor.beforeUnlockExternal(); + global.tryShrinkOrEvict(executor.lock); + executor.unlock(TaskRunner.get()); + } + } + + public static class GlobalCaches + { + public final AccordCache global; + public final AccordCache.Type commands; + public final AccordCache.Type commandsForKey; + + public GlobalCaches(AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) + { + this.global = global; + this.commands = commands; + this.commandsForKey = commandsForKey; + } + } + + final LogLinearDecayingHistograms histograms; + final LogLinearDecayingHistogram elapsedPreparingToRun; + final LogLinearDecayingHistogram elapsedWaitingToRun; + final LogLinearDecayingHistogram elapsedRunning; + final LogLinearDecayingHistogram elapsed; + final LogLinearDecayingHistogram keys; + public final AccordReplicaMetrics.Shard replicaMetrics; + + private final Lock lock; + final Agent agent; + public final int executorId; + private final AccordCache cache; + private final ExclusiveGlobalCaches caches; + final AtomicLong uniqueCreatedAt = new AtomicLong(); + final DebugExecutor debug = DebugExecutor.maybeDebug(); + + private long maxWorkingSetSizeInBytes; + private long maxWorkingCapacityInBytes; + + final TaskQueueStandalone> loading = new TaskQueueStandalone<>(ExecutorQueue.LOADING); + final TaskQueueStandalone> waiting = new TaskQueueStandalone<>(ExecutorQueue.WAITING); + final TaskQueueRunnable runnable = new TaskQueueRunnable<>(); + + private final Tranches tranches = new Tranches(this); + + /** + * Newly submitted work must take a position >= minPosition, but this condition does not apply to consequences of + * previously submitted work; this inherits the originating task's position and tranche. + * This is to ensure afterSubmittedAndConsequences functions correctly. + */ + long minPosition = 1; + long nextPosition = 1; + int tasks; + + private boolean hasPausedLoading; + + private List waitingForQuiescence; + + AccordExecutor(Lock lock, int executorId, Agent agent) + { + this.lock = lock; + this.executorId = executorId; + this.cache = new AccordCache(this, 0); + this.agent = agent; + + final AccordCache.Type commands; + final AccordCache.Type commandsForKey; + commands = cache.newType(TxnId.class, COMMAND_ADAPTER, AccordCacheMetrics.CommandsCacheMetrics.newShard(lock)); + registerJfrListener(executorId, commands, "Command"); + + commandsForKey = cache.newType(RoutingKey.class, CFK_ADAPTER, AccordCacheMetrics.CommandsForKeyCacheMetrics.newShard(lock)); + registerJfrListener(executorId, commandsForKey, "CommandsForKey"); + + this.caches = new ExclusiveGlobalCaches(this, cache, commands, commandsForKey); + + DecayingHistogramsShard histogramsShard = HISTOGRAMS.newShard(lock); + this.histograms = histogramsShard.unsafeGetInternal(); + this.elapsedPreparingToRun = AccordExecutorMetrics.INSTANCE.elapsedPreparingToRun.forShard(histogramsShard); + this.elapsedWaitingToRun = AccordExecutorMetrics.INSTANCE.elapsedWaitingToRun.forShard(histogramsShard); + this.elapsedRunning = AccordExecutorMetrics.INSTANCE.elapsedRunning.forShard(histogramsShard); + this.elapsed = AccordExecutorMetrics.INSTANCE.elapsed.forShard(histogramsShard); + this.keys = AccordExecutorMetrics.INSTANCE.keys.forShard(histogramsShard); + this.replicaMetrics = new AccordReplicaMetrics.Shard(histogramsShard); + } + + abstract boolean isInLoop(); + abstract void beforeUnlockExternal(); + abstract void submit(Consumer sync, Function async, P1 p1); + public abstract boolean hasTasks(); + public abstract boolean isOwningThread(); + + public final Lock unsafeLock() + { + return lock; + } + + public final void lock(TaskRunner self) + { + long startAt = DEBUG_EXECUTION ? Clock.Global.nanoTime() : 0; + if (!self.tryEnterAccordLockedExecutor(this)) + throw new UnsupportedOperationException("To ensure system performance, it is not permitted to utilise multiple AccordExecutor simultaneously with the same thread"); + //noinspection LockAcquiredButNotSafelyReleased + lock.lock(); + if (DEBUG_EXECUTION) debug.onEnterLock(startAt); + } + + public final void unlock(TaskRunner self) + { + self.exitAccordLockedExecutor(); + if (DEBUG_EXECUTION) debug.onExitLock(); + lock.unlock(); + } + + public final boolean tryLock(TaskRunner self) + { + AccordExecutor active = self.accordActiveExecutor(); + if (active != null && active != this) + return false; + + return onTryLock(self, lock.tryLock()); + } + + final boolean onTryLock(TaskRunner self, boolean result) + { + if (result && !self.tryEnterAccordLockedExecutor(this)) + { + // shouldn't be possible, we should have checked this already + lock.unlock(); + throw new IllegalStateException(); + } + return result; + } + + public ExclusiveGlobalCaches lockCaches() + { + lock(TaskRunner.get(Thread.currentThread())); + return caches; + } + + public AccordCache cacheExclusive() + { + Invariants.require(isOwningThread()); + return cache; + } + + public AccordCache cacheUnsafe() + { + return cache; + } + + final boolean hasAlreadyWaitingToRun() + { + return runnable.hasWaitingToRun(); + } + + void updateWaitingToRunExclusive() + { + // TODO (expected): this should not be invoked on every update of waiting to run + maybeUnpauseLoading(); + } + + // drain only new work; specifically leave anything that would call completeTask queued. + // this is to maintain invariants in Tranches.complete, where we may have some consequence of some earlier task + // pending but unqueued, so that we have not incremented its tranche count, and in the interim we set the tranche + // count to zero + void drainUnqueuedNewWorkExclusive() + { + } + + final Task pollAlreadyWaitingToRunExclusive() + { + Task next = runnable.poll(); + if (DEBUG_EXECUTION && next != null) DebugTask.get(next).onPolled(); + return next; + } + + public void waitForQuiescence() + { + TaskRunner self = TaskRunner.get(); + Condition condition; + lock(self); + try + { + if (tasks == 0) + return; + + if (waitingForQuiescence == null) + waitingForQuiescence = new ArrayList<>(); + condition = Condition.newOneTimeCondition(); + waitingForQuiescence.add(condition); + } + finally + { + unlock(self); + } + condition.awaitThrowUncheckedOnInterrupt(); + } + + protected void notifyQuiescentExclusive() + { + if (waitingForQuiescence != null) + { + waitingForQuiescence.forEach(Condition::signalAll); + waitingForQuiescence = null; + } + tranches.finishAll(nextPosition); + } + + public void afterSubmittedAndConsequences(Runnable run) + { + TaskRunner self = TaskRunner.get(); + + lock(self); + try + { + drainUnqueuedNewWorkExclusive(); + if (tasks == 0) + { + run.run(); + return; + } + + tranches.registerWait(run); + } + finally + { + unlock(self); + } + } + + private boolean hasNonLoadingWaitingToRun() + { + return runnable.hasWaitingToRunExcluding(LOADING_GROUPS); + } + + final void maybeUnpauseLoading() + { + if (hasPausedLoading && (cache.weightedSize() < maxWorkingCapacityInBytes || !hasNonLoadingWaitingToRun())) + { + hasPausedLoading = false; + runnable.restart(LOADING_GROUPS); + } + } + + final void maybePauseLoading() + { + if (hasPausedLoading) + return; + + if (!hasPausedLoading && cache.weightedSize() >= maxWorkingCapacityInBytes && hasNonLoadingWaitingToRun()) + { + AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); + hasPausedLoading = true; + runnable.stop(LOADING_GROUPS); + } + } + + public ExclusiveExecutor newExclusiveExecutor(int commandStoreId) + { + return new ExclusiveExecutor(this, commandStoreId); + } + + public ExclusiveAsyncExecutor newExclusiveExecutor() + { + return new ExclusiveExecutor(this); + } + + @Override + public IOTaskLoad load(SafeTask parent, Boolean isForRange, AccordCacheEntry entry) + { + IOTaskLoad result = newLoad(entry, isForRange); + result.inherit(parent).submitExclusiveNoExcept(); + return result; + } + + @Override + public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) + { + IOTaskSave task = new IOTaskSave(this, entry, identity, save); + task.submitExclusiveNoExcept(); + return task; + } + + void submitTask(Task task) + { + submit(Task::submitExclusiveNoExcept, i -> i, task); + } + + public Future submit(Runnable run) + { + Task inherit = inherit(); + PlainRunnable task = new PlainRunnable(this, new AsyncPromise<>(), run, OTHER); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); + return task.result; + } + + public void execute(Runnable run) + { + Task inherit = inherit(); + PlainRunnable task = new PlainRunnable(this, null, run, OTHER); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); + } + + @Override + public Cancellable execute(RunOrFail runOrFail) + { + Task inherit = inherit(); + PlainChain task = new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); + return task; + } + + public AsyncChain buildDebuggable(Callable call, Object describe) + { + return new AsyncChains.Head<>() + { + @Override + protected Cancellable start(BiConsumer callback) + { + Task inherit = inherit(); + PlainChainDebuggable task = new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, describe); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); + return task; + } + }; + } + + public void submitExclusive(Runnable runnable) + { + new PlainRunnable(this, null, runnable, OTHER).submitExclusiveNoExcept(); + } + + Cancellable submitExclusive(Task parent, GlobalGroup group, WrappableIOTask wrap) + { + IOTaskWrapper task = new IOTaskWrapper(this, wrap, group); + task.inherit(parent).submitExclusiveNoExcept(); + return task; + } + + Task inherit() + { + TaskRunner self = TaskRunner.get(); + + if (self.accordActiveExecutor() != this) + return null; + + Task task = self.accordActiveSelfTask(); + if (task instanceof ExclusiveExecutorTask) + task = ((ExclusiveExecutorTask)task).queue.task; + return task; + } + + void registerExclusive(Task task) + { + Invariants.require(isOwningThread()); + Invariants.require(task.is(UNREGISTERED)); + ++tasks; + if (task.hasInherited()) + { + tranches.addInherited(task.tranche(), task.position); + } + else + { + long position = task.position; + if (position == 0) + { + task.position = position = nextPosition++; + } + else + { + long delta = nextPosition - position; + if (delta >= AGE_TO_FIFO) + task.position = position = nextPosition++; + else if (delta <= 0) + nextPosition = position + 1; + else if (position < minPosition) + task.position = position = minPosition; + } + task.setTranche(tranches.addNew(position)); + } + } + + final void completedTaskExclusive(Task task) + { + Invariants.require(task.compareTo(EXECUTED) >= 0); + if (DEBUG_EXECUTION) DebugTask.get(task).onCompleted(debug); + unregisterExclusive(task); + runnable.cleanup(task); + cache.tryShrinkOrEvict(lock); + maybeUnpauseLoading(); + } + + final void unregisterExclusive(Task task) + { + int tranch = task.tranche(); + --tasks; + tranches.complete(tranch); + } + + void onScannedRangesExclusive(SafeTask task, Throwable fail) + { + // the task may have already been cancelled, in which case we don't need to fail it + if (task.state().isDone()) + return; + + SafeTask.RangeTxnScanner scanner = task.rangeScanner(); + if (scanner == null && fail == null) fail = new CancellationException(); + if (fail != null) task.tryFailAndCompleteExclusive(fail, FAILED); + else scanner.scannedExclusive(); + } + + void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) + { + cache.saved(state, identity, fail); + } + + void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail) + { + if (loaded.status() == EVICTED) + return; + + try (ArrayBuffers.BufferList> tasks = loaded.drainWaitingToLoad()) + { + if (fail != null) + { + for (SafeTask task : tasks) + task.tryFailAndCompleteExclusive(fail, FAILED); + cache.failedToLoad(loaded); + } + else + { + cache.loaded(loaded, value); + for (SafeTask task : tasks) + task.onLoadOneExclusive(loaded); + } + } + + maybePauseLoading(); + } + + public void executeDirectlyWithLock(Runnable command) + { + TaskRunner self = TaskRunner.get(); + lock(self); + try + { + self.setAccordActiveExecutor(this); + command.run(); + } + finally + { + beforeUnlockExternal(); + self.setAccordActiveExecutor(null); + unlock(self); + } + } + + @Override + public void setCapacity(long bytes) + { + Invariants.require(isOwningThread()); + cache.setCapacity(bytes); + refreshCapacity(); + } + + public void setWorkingSetSize(long bytes) + { + Invariants.require(isOwningThread()); + maxWorkingSetSizeInBytes = bytes; + refreshCapacity(); + } + + private void refreshCapacity() + { + maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; + if (maxWorkingCapacityInBytes < maxWorkingSetSizeInBytes) + maxWorkingCapacityInBytes = Long.MAX_VALUE; + } + + @Override + public long capacity() + { + return cache.capacity(); + } + + @Override + public int size() + { + return cache.size(); + } + + @Override + public long weightedSize() + { + return cache.weightedSize(); + } + + public int unsafePreparingToRunCount() + { + return loading.size(); + } + + public int unsafeWaitingToRunCount() + { + return runnable.waitingCount(); + } + + public int unsafeRunningCount() + { + return runnable.assigned.size(); + } + + public Stream active() + { + return Stream.of(); + } + + public int executorId() + { + return executorId; + } + + public static IntFunction constant(O out) + { + return ignore -> out; + } + + IOTaskLoad newLoad(AccordCacheEntry entry, boolean isForRange) + { + return new IOTaskLoad<>(this, entry, isForRange ? RANGE_LOAD : LOAD); + } + + public List taskSnapshot() + { + List result = new ArrayList<>(); + TaskRunner self = TaskRunner.get(); + lock(self); + try + { + addToSnapshot(result, loading, TaskInfo.Status.LOADING, TaskInfo.Status.LOADING); + for (TaskQueue queue : runnable.queues) + { + if (queue != null) + addToSnapshot(result, queue, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); + } + addToSnapshot(result, runnable.assigned, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); + } + finally + { + unlock(self); + } + result.sort(TaskInfo::compareTo); + return result; + } + + private static void addToSnapshot(List snapshot, TaskQueue queue, TaskInfo.Status ifCurrent, TaskInfo.Status ifQueued) + { + for (int i = 0 ; i < queue.size() ; ++i) + { + Task t = queue.getSingle(i); + if (t instanceof ExclusiveExecutorTask) + { + ExclusiveExecutor q = ((ExclusiveExecutorTask) t).queue; + snapshot.add(new TaskInfo(ifCurrent, q.commandStoreId, q.task)); + for (TaskQueue q0 : q.queues) + { + if (q0 != null) + { + for (int j = 0 ; j < q0.size() ; ++j) + snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q0.getSingle(j))); + } + } + } + else + { + int commmandStoreId = t instanceof SafeTask ? ((SafeTask) t).commandStore.id() : -1; + snapshot.add(new TaskInfo(ifCurrent, commmandStoreId, t)); + } + } + } + + private static long[] parseEnumParams(String input, String describe) + { + String[] specs = input.split(";"); + if (specs.length != 2) + throw new IllegalArgumentException("Invalid specifiers in " + describe + "; expect [GlobalGroup];[ExclusiveGroup] but got: " + input); + long[] result = new long[2]; + result[0] = parseEnumParams(GlobalGroup::valueOf, specs[0], describe + " for GlobalGroup"); + result[1] = parseEnumParams(ExclusiveGroup::valueOf, specs[1], describe + " for ExclusiveGroup"); + return result; + } + + private static long parseEnumParams(Function> get, String input, String describe) + { + long result = 0; + for (String spec : input.split(",")) + { + if (spec.trim().isEmpty()) continue; + + String[] split = spec.split(":"); + if (split.length != 2) + throw new IllegalArgumentException("Invalid specifier " + spec + " in " + describe + ": " + input); + + try + { + Enum queue = get.apply(split[0]); + long value = Long.parseLong(split[1]); + if (value <= 0 || value >= 128) + throw new IllegalArgumentException("Invalid limit " + value + " in queue_active_limits: " + input); + + result |= value << (queue.ordinal() * 8); + } + catch (Throwable t) + { + throw new IllegalArgumentException("Invalid queue identifier " + split[0] + " in " + describe + ": " + input); + } + } + + return result; + } + + protected static void prepareRunComplete(TaskRunner self, Task task) + { + if (task.prepareExclusiveNoExcept()) + { + try { task.runNoExcept(self); } + finally { task.completeExclusiveNoExcept(); } + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorAsyncSubmit.java similarity index 88% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorAsyncSubmit.java index a2c57a4556..55a0086cf1 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorAsyncSubmit.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.function.IntFunction; @@ -26,9 +26,9 @@ import accord.api.Agent; import org.apache.cassandra.utils.concurrent.LockWithAsyncSignal; // WARNING: experimental - needs more testing -public class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSubmit +public class AccordExecutorAsyncSubmit extends AbstractSemiSyncSubmit { - private final AccordExecutorLoops loops; + private final Loops loops; private final LockWithAsyncSignal lock; public AccordExecutorAsyncSubmit(int executorId, Mode mode, int threads, IntFunction name, Agent agent) @@ -40,7 +40,7 @@ public class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSub { super(lock, executorId, agent); this.lock = lock; - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); } @Override @@ -52,7 +52,7 @@ public class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSub } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -76,7 +76,7 @@ public class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSub } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isOwner(Thread.currentThread()); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java similarity index 78% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java index f97e206ef5..ac17576f26 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; @@ -26,12 +26,11 @@ import java.util.function.IntFunction; import accord.api.Agent; // WARNING: experimental - needs more testing -public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSyncSubmit +public class AccordExecutorSemiSyncSubmit extends AbstractSemiSyncSubmit { - private final AccordExecutorLoops loops; + private final Loops loops; private final ReentrantLock lock; private final Condition hasWork; - private int waiting; public AccordExecutorSemiSyncSubmit(int executorId, Mode mode, int threads, IntFunction name, Agent agent) { @@ -43,19 +42,7 @@ public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSync super(lock, executorId, agent); this.lock = lock; this.hasWork = lock.newCondition(); - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); - } - - @Override - void loopYieldExclusive() throws InterruptedException - { - if (waiting > 0 && hasWaitingToRun()) - { - pauseLoop(); - hasWork.signal(); - awaitWork(); - resumeLoop(); - } + this.loops = new Loops(mode, threads, name, this::task); } @Override @@ -67,13 +54,11 @@ public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSync private void awaitWork() throws InterruptedException { - waiting++; - try { hasWork.await(); } - finally { waiting--; } + hasWork.await(); } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -99,7 +84,7 @@ public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSync } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isHeldByCurrentThread(); } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java new file mode 100644 index 0000000000..51275ee4fe --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java @@ -0,0 +1,483 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.IntFunction; + +import javax.annotation.Nullable; + +import accord.api.Agent; +import accord.utils.Invariants; + +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.utils.concurrent.SignalLock; + +import static accord.utils.Invariants.nonNull; +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.Task.State.REGISTERED; + +public class AccordExecutorSignalLoop extends AbstractLoop +{ + private static class ShutdownException extends RuntimeException {} + + private static final int MAX_LOOPS = 1000; // limit the amount of time we hold the lock for + private final SignalLock lock; + private final Loops loops; + private int readyToRunTarget = 1; + private final int readyToRunLimit; + // TODO (desired): intrusive queue using Task.next, but a little challenging because we reuse SequentialQueueTask so have ABA problem + private final ConcurrentLinkedQueue readyToRun = new ConcurrentLinkedQueue<>(); + + private Task pendingExecutedHead, pendingExecutedTail; + private Task pendingNewHead, pendingNewTail; + private int pendingCount; + + private boolean shutdown; + + public AccordExecutorSignalLoop(int executorId, Mode mode, int threads, long spinInterval, long stopCheckInterval, TimeUnit units, IntFunction name, Agent agent) + { + this(new SignalLock(threads, spinInterval, stopCheckInterval, units), executorId, mode, threads, name, agent); + } + + public AccordExecutorSignalLoop(SignalLock lock, int executorId, Mode mode, int threads, IntFunction name, Agent agent) + { + super(lock, executorId, agent); + Invariants.require(threads <= SignalLock.MAX_THREADS); + this.lock = lock; + this.loops = new Loops(mode, threads, name, this::task); + this.readyToRunLimit = Math.min(threads * 4, SignalLock.MAX_SIGNAL_COUNT); + } + + void submit(Consumer sync, Function async, P1 p1) + { + Task next = async.apply(p1); + Task prev = push(next); + if (prev == null) + lock.signalLockWork(); + } + + @Override + final void beforeUnlockExternal() + { + } + + private Task pollReadyToRun() + { + return readyToRun.poll(); + } + + private void addReadyToRun(Task task) + { + readyToRun.add(task); + } + + private boolean hasReadyToRun() + { + return !readyToRun.isEmpty(); + } + + private LoopTask task(int index, String id, Mode mode) + { + return new LoopTask(index, id); + } + + @Override + final void drainUnqueuedNewWorkExclusive() + { + updatePendingUnqueued(); + Task requeue = null, requeueTail = null; + Task submit = null, submitTail = null; + Task cur = pendingNewHead; + while (cur != null) + { + Task next = cur.next; + cur.next = null; + if (cur.isNewWork()) + { + if (submit == null) submit = cur; + else submitTail.next = cur; + submitTail = cur; + --pendingCount; + } + else + { + if (requeue == null) requeue = cur; + else requeueTail.next = cur; + requeueTail = cur; + } + cur = next; + } + + pendingNewHead = requeue; + pendingNewTail = requeueTail; + while (submit != null) + { + Task next = submit.next; + submit.next = null; + submit.submitExclusiveNoExcept(); + submit = next; + } + } + + private boolean enqueueOnePending() + { + if (pendingCount == 0) + return false; + + --pendingCount; + if (pendingExecutedHead != null) + { + Task executed = pendingExecutedHead; + pendingExecutedHead = destructiveNext(executed); + if (pendingExecutedHead == null) + pendingExecutedTail = null; + executed.completeExclusiveNoExcept(); + } + else + { + Task submit = pendingNewHead; + pendingNewHead = destructiveNext(submit); + if (pendingNewHead == null) + pendingNewTail = null; + submit.submitExclusiveNoExcept(); + } + return true; + } + + private void fetchWorkExclusive() + { + boolean hasReadyToRun = hasReadyToRun(); + boolean hadWaitingToRun = hasAlreadyWaitingToRun(); + { + int prevPendingUnqueued = pendingCount; + updateAndEnqueuePendingUntilHasWaitingToRun(prevPendingUnqueued / 2); + } + + if (hadWaitingToRun && hasReadyToRun) + { + lock.addAndGetEnabledThreadCount(1); + } + else if (hadWaitingToRun) + { + readyToRunTarget = Math.min(readyToRunLimit, readyToRunTarget + (1+readyToRunTarget)/2); + } + else if (hasReadyToRun && readyToRunTarget > 1) + { + --readyToRunTarget; + } + + boolean hasDrainedSignal = false; + int loops = 0; + while (true) + { + long state = lock.state(); + int signals = SignalLock.asyncSignalCount(state); + int waiters = SignalLock.waitingEnabledThreadCount(state); + if (signals > 0) + { + if (++loops > MAX_LOOPS) + return; + + if (signals >= readyToRunTarget) + { + if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; + else if (hasDrainedSignal) + lock.signalLockWorkExclusive(); + return; + } + else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) + { + // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary + lock.propagateAsyncWorkSignals(1); + } + } + + Task task = pollAlreadyWaitingToRunExclusive(); + if (task == null) + { + if (updateAndEnqueuePendingUntilHasWaitingToRun(0)) + continue; + + lock.clearLockWork(); + hasDrainedSignal = true; + if (!updateAndEnqueuePendingUntilHasWaitingToRun(0)) + { + if (tasks == 0) + notifyQuiescentExclusive(); + return; + } + } + else if (task.prepareExclusiveNoExcept()) + { + if (DEBUG_EXECUTION) DebugTask.get(task).onPreRun(); + addReadyToRun(task); + boolean incremented = lock.incrementAsyncWork(false); + Invariants.require(incremented); + } + } + } + + private boolean updatePendingUnqueued() + { + if (!hasUnqueued()) + return false; + + int count = 0; + Task addExecutedHead = null, addExecutedTail = null; + Task addNewHead = null, addNewTail = null; + { + Task cur = Task.reverse(acquireUnqueuedExclusive()); + while (cur != null) + { + Task next = cur.next; + if (cur.compareTo(REGISTERED) < 0) + { + if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); + else addNewHead = reverseOne(addNewHead, cur); + } + else + { + if (addExecutedHead == null) addExecutedHead = addExecutedTail = setNextNull(cur); + else addExecutedHead = reverseOne(addExecutedHead, cur); + } + ++count; + cur = next; + } + } + + pendingCount += count; + if (addExecutedHead != null) + { + if (pendingExecutedHead == null) pendingExecutedHead = addExecutedHead; + else pendingExecutedTail.next = addExecutedHead; + pendingExecutedTail = addExecutedTail; + } + if (addNewHead != null) + { + if (pendingNewHead == null) pendingNewHead = addNewHead; + else pendingNewTail.next = addNewHead; + pendingNewTail = addNewTail; + } + return true; + } + + private Task reverseOne(Task prev, Task cur) + { + cur.next = prev; + return cur; + } + + private Task setNextNull(Task cur) + { + cur.next = null; + return cur; + } + + private boolean updateAndEnqueuePendingUntilHasWaitingToRun(int processAtLeast) + { + updatePendingUnqueued(); + int count = 0; + while (enqueueOnePending()) + { + if (++count >= processAtLeast && hasAlreadyWaitingToRun()) + return true; + } + return hasAlreadyWaitingToRun(); + } + + class LoopTask extends Loops.LoopTask + { + final int index; + + LoopTask(int index, String id) + { + super(id); + this.index = index; + } + + private Task awaitWork(TaskRunner self) + { + while (true) + { + if (lock.awaitAsyncOrLock(index)) + { + try + { + if (DEBUG_EXECUTION) debug.onEnterLock(); + fetchWorkExclusive(); + } + catch (Throwable t) + { + unlock(self); + throw t; + } + + if (!unlockAndAcquire(self)) + continue; + } + + if (shutdown) + throw new ShutdownException(); + + return nonNull(pollReadyToRun()); + } + } + + private Task executedAndMaybeGetWork(TaskRunner self, @Nullable Task executed) + { + if (lock.tryAcquireAsyncWork()) + { + if (shutdown) + throw new ShutdownException(); + + return pushExecutedAndReturn(executed, nonNull(pollReadyToRun())); + } + + if (!tryLock(self)) + return pushExecutedAndReturn(executed, null); + + try + { + if (executed != null) + executed.completeExclusiveNoExcept(); + fetchWorkExclusive(); + } + catch (Throwable t) + { + unlock(self); + throw t; + } + + if (unlockAndAcquire(self)) + { + if (shutdown) + throw new ShutdownException(); + + return nonNull(pollReadyToRun()); + } + return null; + } + + private Task pushExecutedAndReturn(Task complete, Task result) + { + if (push(complete) == null) + lock.signalLockWork(); + return result; + } + + final boolean tryLock(TaskRunner self) + { + if (self.accordLockedExecutor() != null) + return false; + + return onTryLock(self, lock.tryLock(index)); + } + + final boolean unlockAndAcquire(TaskRunner self) + { + self.exitAccordLockedExecutor(); + if (DEBUG_EXECUTION) debug.onExitLock(); + return lock.unlockAndAcquireAsyncWork(); + } + + @Override + public void run() + { + Thread thread = Thread.currentThread(); + TaskRunner self = TaskRunner.get(thread); + self.setAccordActiveExecutor(AccordExecutorSignalLoop.this); + setWrapped(self); + + lock.register(index, thread); + Task task = null; + while (true) + { + try + { + if (task != null) + { + try { task = executedAndMaybeGetWork(self, task); } + catch (Throwable t) { task = null; throw t; } + } + if (task == null) + task = awaitWork(self); + + task.runNoExcept(self); + } + catch (ShutdownException ignore) + { + break; + } + catch (Throwable t) + { + agent.onException(t); + } + } + } + } + + @Override + public void shutdown() + { + lock.lock(); + try + { + shutdown = true; + lock.signalAllRegistered(); + } + finally + { + lock.unlock(); + } + } + + @Override + Loops loops() + { + return loops; + } + + @Override + boolean isInLoop() + { + return loops.isInLoop(); + } + + @Override + public boolean isOwningThread() + { + return lock.isOwner(); + } + + @Override + public boolean isTerminated() + { + return loops.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException + { + return loops.awaitTermination(timeout, unit); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java similarity index 83% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java index 4e89f068ab..ee8e79e535 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java @@ -16,22 +16,22 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import accord.api.Agent; import accord.utils.Invariants; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; import org.apache.cassandra.concurrent.ExecutorPlus; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -class AccordExecutorSimple extends AccordExecutor +public class AccordExecutorSimple extends AccordExecutor { final ExecutorPlus executor; final ReentrantLock lock; @@ -82,7 +82,8 @@ class AccordExecutorSimple extends AccordExecutor protected void run() { - Thread self = Thread.currentThread(); + TaskRunner self = TaskRunner.get(); + self.setAccordActiveExecutor(AccordExecutorSimple.this); lock.lock(); try { @@ -95,12 +96,7 @@ class AccordExecutorSimple extends AccordExecutor return; } - try { task.preRunExclusive(); task.runInternal(); } - catch (Throwable t) { task.fail(t); } - finally - { - completeTaskExclusive(task); - } + prepareRunComplete(self, task); } } finally @@ -112,12 +108,12 @@ class AccordExecutorSimple extends AccordExecutor } @Override - void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submit(Consumer sync, Function async, P1 p1) { lock.lock(); try { - sync.accept(this, p1s, p2, p3, p4); + sync.accept(p1); } finally { @@ -128,8 +124,20 @@ class AccordExecutorSimple extends AccordExecutor } } + final boolean hasWaitingToRun() + { + updateWaitingToRunExclusive(); + return hasAlreadyWaitingToRun(); + } + + final Task pollWaitingToRunExclusive() + { + updateWaitingToRunExclusive(); + return pollAlreadyWaitingToRunExclusive(); + } + @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isHeldByCurrentThread(); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSyncSubmit.java similarity index 79% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSyncSubmit.java index 0d852ccde3..e6d47f8910 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSyncSubmit.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import accord.api.Agent; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop +public class AccordExecutorSyncSubmit extends AbstractLockLoop { - private final AccordExecutorLoops loops; + private final Loops loops; private final ReentrantLock lock; private final Condition hasWork; @@ -48,7 +48,7 @@ public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop super(lock, executorId, agent); this.lock = lock; this.hasWork = lock.newCondition(); - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); } @Override @@ -58,7 +58,7 @@ public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -70,7 +70,7 @@ public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isHeldByCurrentThread(); } @@ -89,16 +89,17 @@ public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop hasWork.signal(); } - void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submitExternal(Consumer sync, Function async, P1 p1) { - lock(); + TaskRunner self = TaskRunner.get(); + lock(self); try { - submitExternalExclusive(sync, p1s, p2, p3, p4); + submitExternalExclusive(sync, async, p1); } finally { - unlock(); + unlock(self); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java new file mode 100644 index 0000000000..e93ae0c084 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +final class CancelTask extends Task +{ + final Task cancel; + + CancelTask(Task cancel) + { + super(GlobalGroup.OTHER); + this.cancel = cancel; + } + + @Override + void submitExclusiveMayThrow() + { + cancel.tryCancelExclusive(); + } + + @Override + boolean isNewWork() + { + return false; + } + + @Override + public String description() + { + return "Cancel " + cancel.description(); + } + + @Override + public String briefDescription() + { + return "Cancel " + cancel.briefDescription(); + } + + @Override void unqueueIfQueued() { throw new UnsupportedOperationException(); } + @Override + void maybeCompleteExclusiveMayThrow() { throw new UnsupportedOperationException(); } + @Override void tryCancelExclusive() { throw new UnsupportedOperationException(); } + @Override void reportFailureMayThrow(Throwable fail) { throw new UnsupportedOperationException(); } + @Override boolean runMayThrow() { throw new UnsupportedOperationException(); } + @Override public void cancel() { throw new UnsupportedOperationException(); } + @Override AccordExecutor executor() { throw new UnsupportedOperationException(); } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java new file mode 100644 index 0000000000..5181f11e18 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.Callable; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.concurrent.locks.LockSupport; + +import accord.api.ExclusiveAsyncExecutor; +import accord.local.ExecutionContext; +import accord.utils.IntrusiveHeapNode; +import accord.utils.Invariants; +import accord.utils.async.AsyncCallbacks; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.service.accord.debug.DebugExecution; +import org.apache.cassandra.service.accord.execution.Task.GroupKind; + +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.EXCLUSIVE_QUEUE_LIMITS; +import static org.apache.cassandra.service.accord.execution.Task.ExecutorQueue.RUNNABLE; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.COMMAND_STORE; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; + +public final class ExclusiveExecutor extends TaskQueueMulti implements ExclusiveAsyncExecutor +{ + private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(ExclusiveExecutor.class, Thread.class, "owner"); + + static final class ExclusiveExecutorTask extends Task + { + final ExclusiveExecutor queue; + + ExclusiveExecutorTask(ExclusiveExecutor queue) + { + super(COMMAND_STORE); + this.queue = queue; + } + + @Override AccordExecutor executor() { return queue.executor; } + @Override void submitExclusiveMayThrow() { throw new UnsupportedOperationException(); } + @Override boolean isNewWork() { throw new UnsupportedOperationException(); } + @Override public void cancel() { throw new UnsupportedOperationException(); } + @Override boolean runMayThrow() { throw new UnsupportedOperationException(); } + @Override void unqueueIfQueued() {} + @Override void reportFailureMayThrow(Throwable t) { throw new UnsupportedOperationException(); } + @Override void tryCancelExclusive() { throw new UnsupportedOperationException(); } + + boolean prepareTask() + { + Task task = queue.task; + try + { + task.prepareExclusiveMayThrow(); + task.setStateExclusive(State.PREPARED); + setStateExclusive(State.PREPARED); + return true; + } + catch (Throwable t) + { + task.setStateExclusive(State.FAILED); + task.reportFailureNoExcept(t); + maybeCompleteExclusiveMayThrow(); + return false; + } + } + + @Override + void maybeCompleteExclusiveMayThrow() + { + try + { + unsafeSetStateExclusive(WAITING_TO_RUN); + executor().runnable.cleanup(this); + queue.completeTask(); + } + catch (Throwable t) + { + onException(t); + } + } + + @Override + public String description() + { + return queue.task.description(); + } + + @Override + public String briefDescription() + { + return queue.task.briefDescription(); + } + + protected boolean isInHeap() + { + return super.isInHeap(); + } + } + + private final AccordExecutor executor; + final int commandStoreId; + final ExclusiveExecutorTask selfTask; + Task task; + volatile Thread owner, waiting; + private boolean stopped; + private volatile boolean visibleStopped; + private boolean terminated; + + final DebugExecution.DebugExclusiveExecutor debug; + + ExclusiveExecutor(AccordExecutor executor) + { + this(executor, -1); + } + + ExclusiveExecutor(AccordExecutor executor, int commandStoreId) + { + super(RUNNABLE, commandStoreId < 0 ? GroupKind.NONE : GroupKind.EXCLUSIVE, EXCLUSIVE_QUEUE_LIMITS); + this.executor = executor; + this.commandStoreId = commandStoreId; + this.selfTask = new ExclusiveExecutorTask(this); + this.debug = DebugExecution.DebugExclusiveExecutor.maybeDebug(executor.debug, commandStoreId); + } + + void runTask(TaskRunner self) + { + Thread thread = Thread.currentThread(); + if (!ownerUpdater.compareAndSet(this, null, thread)) + { + if (DEBUG_EXECUTION) debug.onWaiting(); + Invariants.require(waiting == null); + waiting = thread; + outer: + do + { + while (true) + { + Thread owner = this.owner; + if (owner == thread) break outer; + if (owner == null) continue outer; + LockSupport.park(); + } + } + while (!ownerUpdater.compareAndSet(this, null, thread)); + Invariants.require(waiting == thread); + waiting = null; + } + + try + { + if (stopped && reject(task)) + task.rejectAtRuntime(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + task.description())); + else + task.runNoExcept(self); + } + finally + { + // NOTE: we can ONLY safely release owner here due to AccordCacheEntry locking, which remains in place until AccordTask.releaseResourcesExclusive + // this also relies on AccordSafeCommandStore$ExclusiveCaches.acquireIfLoaded returning false when the entry is locked + owner = null; + } + } + + private boolean reject(Task task) + { + if (!(task instanceof SafeTask)) + return true; + + ExecutionContext context = ((SafeTask) task).executionContext(); + return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); + } + + void completeTask() + { + try + { + task.unsetQueue(kind); + task.completeExclusiveNoExcept(); + } + finally + { + active = 0; + task = super.pollMulti(); + if (DEBUG_EXECUTION) debug.onSetTask(task); + if (task != null) + { + selfTask.position = task.position; + selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); + executor.runnable.enqueue(selfTask, false); + } + } + } + + void enqueue(Task newTask, boolean incrementArrivals) + { + + if (task != null) + { + if (incrementArrivals) + executor.runnable.incrementArrivals(selfTask); + // TODO (expected): restore some invariant here +// Invariants.require(selfTask.isInHeap() || selfTask.is(RUNNING)); + super.enqueueMulti(newTask, incrementArrivals); + } + else + { + Invariants.require(isEmptySingle()); + if (incrementArrivals) + incrementArrivals(newTask); + incrementDispatches(newTask); + task = newTask; + task.setQueue(kind); + selfTask.position = newTask.position; + selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); + executor.runnable.enqueue(selfTask, incrementArrivals); + if (DEBUG_EXECUTION) debug.onSetTask(newTask); + } + } + + @Override + void unqueue(Task remove) + { + if (remove == task) removeCurrentTask(remove); + else super.unqueueMulti(remove); + } + + boolean tryUnqueueWaiting(Task remove) + { + if (remove == task) return tryRemoveCurrentTask(remove); + else return super.tryUnqueueWaiting(remove); + } + + private boolean tryRemoveCurrentTask(IntrusiveHeapNode remove) + { + if (executor.runnable.isAssigned(selfTask)) + return false; + + removeCurrentTask(remove); + return true; + } + + private void removeCurrentTask(IntrusiveHeapNode remove) + { + Invariants.require(remove == task); + // cannot overwrite task while it is being executed - this cannot happen for AccordTask + // but can for other tasks that don't track their own state + + decrementDispatches(task); + task.unsetQueue(kind); + task = pollMulti(); + if (DEBUG_EXECUTION) debug.onSetTask(task); + if (executor.runnable.isWaiting(selfTask)) + { + if (task == null) executor.runnable.unqueue(selfTask); + else + { + selfTask.position = task.position; + executor.runnable.requeue(selfTask); + } + } + else + { + Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove); + if (task != null) + { + selfTask.position = task.position; + selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); + executor.runnable.enqueue(selfTask, false); + } + } + Invariants.require(task == null || executor.runnable.isWaiting(selfTask)); + } + + public boolean inExecutor() + { + return owner == Thread.currentThread(); + } + + public boolean stopped() + { + return visibleStopped; + } + + public void stop() + { + Invariants.require(inExecutor()); + this.stopped = true; + this.visibleStopped = true; + } + + public void terminate() + { + Invariants.require(inExecutor()); + this.visibleStopped = this.terminated = this.stopped = true; + } + + @Override + public AsyncChain chain(Runnable run) + { + return AsyncChains.chain(this, run); + } + + @Override + public AsyncChain chain(Callable call) + { + return AsyncChains.chain(this, call); + } + + @Override + public AsyncChain flatChain(Callable> call) + { + return AsyncChains.flatChain(this, call); + } + + @Override + public void execute(Runnable run) + { + Task inherit = executor.inherit(); + PlainRunnable task = new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER); + if (inherit != null) inherit.addConsequence(task); + else executor.submitTask(task); + } + + @Override + public Cancellable execute(AsyncCallbacks.RunOrFail runOrFail) + { + Task inherit = executor.inherit(); + PlainChain task = new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER); + if (inherit != null) inherit.addConsequence(task); + else executor.submitTask(task); + return task; + } + + @Override + public boolean tryExecuteImmediately(Runnable run) + { + Thread thread = Thread.currentThread(); + Thread owner = this.owner; + if (owner != null && owner != thread) + return false; + + TaskRunner self = TaskRunner.get(thread); + AccordExecutor active = self.accordActiveExecutor(); + if (active != null && active != executor) + return false; // prevent cross-executor locking/execution + + if (owner == null && !ownerUpdater.compareAndSet(this, null, thread)) + return false; + + try + { + if (active == null) + self.setAccordActiveExecutor(executor); + + run.run(); + } + catch (Throwable t) + { + selfTask.onException(t); + } + finally + { + if (owner == null) + { + Thread waiting = this.waiting; + Invariants.require(waiting != self); + this.owner = waiting; + if (waiting == null) // recheck, to ensure happens-before relation with a new waiter that expects any non-null owner to notify it + waiting = this.waiting; + if (waiting != null) + LockSupport.unpark(waiting); + } + + if (active == null) + self.setAccordActiveExecutor(null); + } + return true; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTask.java b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java new file mode 100644 index 0000000000..b3607151fe --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.DebuggableTask; + +// a task that may be submitted to this executor or another +public abstract class IOTask extends Plain implements Cancellable, DebuggableTask +{ + IOTask(AccordExecutor executor, GlobalGroup group) + { + super(executor, group); + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return null; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java new file mode 100644 index 0000000000..759ff32ba7 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.Invariants; + +import static org.apache.cassandra.service.accord.execution.IOTaskLoad.FailureHolder.NOT_STARTED; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; + +public class IOTaskLoad extends IOTask +{ + static class FailureHolder + { + static final FailureHolder NOT_STARTED = new FailureHolder(new RuntimeException("Not started")); + + final Throwable fail; + + FailureHolder(Throwable fail) + { + this.fail = fail; + } + } + + final AccordCacheEntry entry; + Object result = NOT_STARTED; + + IOTaskLoad(AccordExecutor executor, AccordCacheEntry entry, GlobalGroup group) + { + super(executor, group); + Invariants.require(group == LOAD || group == RANGE_LOAD); + this.entry = entry; + } + + @Override + void maybeCompleteExclusiveMayThrow() + { + if (!(result instanceof FailureHolder)) + executor.onLoadedExclusive(entry, (V) result, null); + else + executor.onLoadedExclusive(entry, null, ((FailureHolder) result).fail); + super.maybeCompleteExclusiveMayThrow(); + } + + @Override + public boolean runMayThrow() + { + result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); + return true; + } + + @Override + void reportFailureMayThrow(Throwable t) + { + result = new FailureHolder(t); + } + + @Override + public String description() + { + return "Load " + entry.key(); + } + + @Override + public String briefDescription() + { + return description(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java new file mode 100644 index 0000000000..cfe80ee817 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.SAVE; + +class IOTaskSave extends IOTask +{ + private static final Throwable NOT_STARTED = new Throwable(); + + final AccordCacheEntry entry; + final AccordCacheEntry.UniqueSave identity; + final Runnable run; + Throwable failure = NOT_STARTED; + + IOTaskSave(AccordExecutor executor, AccordCacheEntry entry, AccordCacheEntry.UniqueSave identity, Runnable run) + { + super(executor, SAVE); + this.entry = entry; + this.identity = identity; + this.run = run; + } + + @Override + void maybeCompleteExclusiveMayThrow() + { + executor.onSavedExclusive(entry, identity, failure); + super.maybeCompleteExclusiveMayThrow(); + } + + @Override + public boolean runMayThrow() + { + run.run(); + failure = null; + return true; + } + + @Override + void reportFailureMayThrow(Throwable t) + { + failure = t; + } + + @Override + public String description() + { + return "Save " + entry; + } + + @Override + String briefDescription() + { + return description(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java new file mode 100644 index 0000000000..3476dbde2d --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +class IOTaskWrapper extends IOTask +{ + static abstract class WrappableIOTask + { + abstract void runInternal(); + abstract void postRunExclusive(); + abstract void fail(Throwable t); + abstract String description(); + } + + final WrappableIOTask wrapped; + + IOTaskWrapper(AccordExecutor executor, WrappableIOTask wrap, GlobalGroup group) + { + super(executor, group); + this.wrapped = wrap; + } + + @Override + protected boolean runMayThrow() + { + wrapped.runInternal(); + return true; + } + + @Override + void maybeCompleteExclusiveMayThrow() + { + wrapped.postRunExclusive(); + super.maybeCompleteExclusiveMayThrow(); + } + + @Override + public String description() + { + return wrapped.description(); + } + + @Override + public String briefDescription() + { + return description(); + } + + @Override + void reportFailureMayThrow(Throwable fail) + { + wrapped.fail(fail); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorLoops.java b/src/java/org/apache/cassandra/service/accord/execution/Loops.java similarity index 87% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorLoops.java rename to src/java/org/apache/cassandra/service/accord/execution/Loops.java index 3eddaaf9e2..d15afc51c6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorLoops.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Loops.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -29,19 +29,19 @@ import accord.utils.Invariants; import accord.utils.TriFunction; import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; -import org.apache.cassandra.service.accord.AccordExecutor.Mode; -import org.apache.cassandra.service.accord.AccordExecutor.TaskRunner; +import org.apache.cassandra.service.accord.execution.AccordExecutor.Mode; +import org.apache.cassandra.service.accord.execution.TaskRunner.DebuggableWrapper; import org.apache.cassandra.utils.concurrent.Condition; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorThreadTag.INFINITE_LOOP; import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.NON_DAEMON; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -class AccordExecutorLoops +class Loops { - static abstract class LoopTask extends TaskRunner implements Runnable + static abstract class LoopTask extends DebuggableWrapper implements Runnable { final String id; LoopTask(String id) { this.id = id; } @@ -54,7 +54,7 @@ class AccordExecutorLoops private final AtomicInteger running = new AtomicInteger(); private final Condition terminated = Condition.newOneTimeCondition(); - public AccordExecutorLoops(Mode mode, int threads, IntFunction loopName, TriFunction loopFactory) + public Loops(Mode mode, int threads, IntFunction loopName, TriFunction loopFactory) { Invariants.require(mode == RUN_WITH_LOCK ? threads == 1 : threads >= 1); running.addAndGet(threads); diff --git a/src/java/org/apache/cassandra/service/accord/execution/Plain.java b/src/java/org/apache/cassandra/service/accord/execution/Plain.java new file mode 100644 index 0000000000..cc5a3c0ffc --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Plain.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.CancellationException; + +import accord.utils.async.Cancellable; + +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +abstract class Plain extends Task implements Cancellable +{ + final AccordExecutor executor; + + Plain(AccordExecutor executor, GlobalGroup group) + { + super(group); + this.executor = executor; + } + + Plain(AccordExecutor executor, ExclusiveGroup group) + { + super(group); + this.executor = executor; + } + + abstract ExclusiveExecutor exclusiveExecutor(); + + @Override + public final void cancel() + { + executor.submit(Task::tryCancelExclusive, CancelTask::new, this); + } + + @Override + final void tryCancelExclusive() + { + tryFailAndCompleteExclusive(new CancellationException(), CANCELLED); + } + + @Override + final void submitExclusiveMayThrow() + { + executor.registerExclusive(this); + setStateExclusive(WAITING_TO_RUN); + ExclusiveExecutor exclusiveExecutor = exclusiveExecutor(); + if (exclusiveExecutor == null) executor.runnable.enqueue(this, true); + else exclusiveExecutor.enqueue(this, true); + } + + @Override + void maybeCompleteExclusiveMayThrow() + { + if (completeState()) + { + long completeAt = nanoTime(); + executor.elapsedWaitingToRun.increment(runningAt - createdAt, runningAt); + executor.elapsedRunning.increment(completeAt - runningAt, completeAt); + executor.elapsed.increment(completeAt - createdAt, completeAt); + } + } + + @Override + void unqueueIfQueued() + { + if (isQueued()) + { + ExclusiveExecutor exclusiveExecutor = exclusiveExecutor(); + if (exclusiveExecutor != null) exclusiveExecutor.unqueue(this); + else executor.runnable.unqueue(this); + } + } + + @Override + protected boolean isNewWork() + { + return true; + } + + @Override + AccordExecutor executor() + { + return executor; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java new file mode 100644 index 0000000000..dda6a54e2c --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.utils.async.AsyncCallbacks; + +class PlainChain extends Plain +{ + final AsyncCallbacks.RunOrFail runOrFail; + final @Nullable ExclusiveExecutor exclusiveExecutor; + + PlainChain(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group) + { + super(executor, group); + this.runOrFail = runOrFail; + this.exclusiveExecutor = exclusiveExecutor; + } + + @Override + public String description() + { + return runOrFail.toString(); + } + + @Override + public String briefDescription() + { + return description(); + } + + @Override + boolean runMayThrow() + { + runOrFail.run(); + return true; + } + + @Override + void reportFailureMayThrow(Throwable fail) + { + runOrFail.fail(fail); + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return exclusiveExecutor; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java similarity index 54% rename from src/java/org/apache/cassandra/service/accord/AccordSafeState.java rename to src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java index a254c1154d..efcef616fd 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java @@ -15,43 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; -import accord.impl.SafeState; +package org.apache.cassandra.service.accord.execution; -public interface AccordSafeState extends SafeState +import javax.annotation.Nullable; + +import accord.utils.Invariants; +import accord.utils.async.AsyncCallbacks; + +import org.apache.cassandra.concurrent.DebuggableTask; + +class PlainChainDebuggable extends PlainChain implements DebuggableTask { - void set(V update); - V original(); - void markUnsafe(); - boolean isUnsafe(); - void preExecute(); + final Object describe; - AccordCacheEntry global(); - - default boolean hasUpdate() + PlainChainDebuggable(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, @Nullable ExclusiveExecutor exclusiveExecutor, Object describe) { - return original() != current(); + super(executor, runOrFail, exclusiveExecutor, ExclusiveGroup.OTHER); + this.describe = Invariants.nonNull(describe); } - default void revert() + @Override + public String description() { - set(original()); - } - - default K key() - { - return global().key(); - } - - default Throwable failure() - { - return global().failure(); - } - - default void checkNotInvalidated() - { - if (isUnsafe()) - throw new IllegalStateException("Cannot access invalidated " + this); + return describe.toString(); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java new file mode 100644 index 0000000000..e51c83c964 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.utils.async.Cancellable; + +import org.apache.cassandra.utils.concurrent.AsyncPromise; + +class PlainRunnable extends Plain implements Cancellable +{ + final @Nullable AsyncPromise result; + final Runnable run; + final @Nullable ExclusiveExecutor exclusiveExecutor; + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, GlobalGroup group) + { + super(executor, group); + this.result = result; + this.run = run; + this.exclusiveExecutor = null; + } + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group) + { + super(executor, group); + this.result = result; + this.run = run; + this.exclusiveExecutor = exclusiveExecutor; + } + + @Override + protected boolean runMayThrow() + { + run.run(); + if (result != null) + { + try { result.trySuccess(null); } + catch (Throwable t) { onException(t); } + } + return true; + } + + @Override + public String description() + { + // TODO (expected): ensure this is usefully descriptive, or accept a separate description + return run.toString(); + } + + @Override + public String briefDescription() + { + return description(); + } + + @Override + void reportFailureMayThrow(Throwable t) + { + if (result != null) + result.tryFailure(t); + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return exclusiveExecutor; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java new file mode 100644 index 0000000000..3b4274a022 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java @@ -0,0 +1,1567 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.accord.execution; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.CancellationException; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import org.agrona.collections.Object2ObjectHashMap; +import org.agrona.collections.ObjectHashSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.api.Journal; +import accord.api.RoutingKey; +import accord.local.Command; +import accord.local.CommandStore; +import accord.local.CommandSummaries; +import accord.local.CommandSummaries.Summary; +import accord.local.ExecutionContext; +import accord.local.LoadKeys; +import accord.local.SafeCommandStore; +import accord.local.SafeState; +import accord.local.cfk.CommandsForKey; +import accord.primitives.AbstractRanges; +import accord.primitives.AbstractUnseekableKeys; +import accord.primitives.Range; +import accord.primitives.Ranges; +import accord.primitives.Routable; +import accord.primitives.RoutingKeys; +import accord.primitives.Timestamp; +import accord.primitives.TxnId; +import accord.primitives.Unseekables; +import accord.utils.ArrayBuffers.BufferList; +import accord.utils.Invariants; +import accord.utils.UnhandledEnum; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.DebuggableTask; +import org.apache.cassandra.metrics.LogLinearDecayingHistograms; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordCommandStore.Caches; +import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; +import org.apache.cassandra.service.accord.RangeIndex; +import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Loading; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.serializers.CommandSerializers; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.NoSpamLogger; +import org.apache.cassandra.utils.concurrent.Condition; + +import static accord.local.LoadKeys.INCR; +import static accord.local.LoadKeys.NONE; +import static accord.local.LoadKeys.SYNC; +import static accord.local.LoadKeysFor.RECOVERY; +import static accord.local.LoadKeysFor.WRITE; +import static accord.primitives.Routable.Domain.Key; +import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask.SANITY_CHECK; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.HOLD_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.NONSYNC_BLOCKED_LIMIT; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.NONSYNC_ENABLED; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.NONSYNC_MIN_BATCH_SIZE; +import static org.apache.cassandra.service.accord.execution.SaferState.global; +import static org.apache.cassandra.service.accord.execution.SaferState.postExecute; +import static org.apache.cassandra.service.accord.execution.SaferState.preExecute; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_FAILED; +import static org.apache.cassandra.service.accord.execution.Task.RunState.NOT_YET_RUN; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_PERSISTING; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_INCOMPLETE; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED_UNREGISTERED; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED; +import static org.apache.cassandra.service.accord.execution.Task.State.INCOMPLETE; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.PREPARED; +import static org.apache.cassandra.service.accord.execution.Task.State.REGISTERED; +import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_OR_PREPARED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +public final class SafeTask extends Task implements Cancellable, DebuggableTask +{ + private static final Logger logger = LoggerFactory.getLogger(SafeTask.class); + private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES); + + public static SafeTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) + { + return new SafeTask<>((AccordCommandStore) commandStore, context, safeStore -> { + consumer.accept(safeStore); + return null; + }); + } + + public static SafeTask create(CommandStore commandStore, ExecutionContext context, Function function) + { + return new SafeTask<>((AccordCommandStore) commandStore, context, function); + } + + static class NonSyncState extends ExecutionContext.Wrapped implements ExecutionContext + { + RoutingKeys active; + ObjectHashSet blocking, notBlocking; + int loaded, processed; + boolean alwaysReady; + + public NonSyncState(ExecutionContext context) + { + super(context); + } + + @Override + public final Unseekables keys() + { + return active; + } + + final void addLoaded() + { + ++loaded; + } + + final void onNotHead(AccordCacheEntry entry) + { + if ((notBlocking == null || !notBlocking.remove((RoutingKey) entry.key())) && blocking != null) + blocking.remove((RoutingKey) entry.key()); + } + + final void onNewHead(AccordCacheEntry entry) + { + ensureNotBlocking().add((RoutingKey) entry.key()); + } + + final void onNewBlockingHead(AccordCacheEntry entry) + { + ensureBlocking().add((RoutingKey) entry.key()); + } + + final void onStillHeadNewBlocking(AccordCacheEntry entry) + { + notBlocking.remove((RoutingKey) entry.key()); + ensureBlocking().add((RoutingKey) entry.key()); + } + + private ObjectHashSet ensureBlocking() + { + if (blocking == null) + blocking = new ObjectHashSet<>(); + return blocking; + } + + private ObjectHashSet ensureNotBlocking() + { + if (notBlocking == null) + notBlocking = new ObjectHashSet<>(); + return notBlocking; + } + + private int readyCount() + { + return (blocking == null ? 0 : blocking.size()) + (notBlocking == null ? 0 : notBlocking.size()); + } + + final boolean isLoaded(SafeTask owner) + { + return loaded >= Math.min(owner.keys, alwaysReady ? 1 : NONSYNC_MIN_BATCH_SIZE); + } + + final boolean isWaitReady(SafeTask owner) + { + if (readyCount() >= Math.min(owner.keys - processed, alwaysReady ? 1 : NONSYNC_MIN_BATCH_SIZE)) + return true; + + return blocking != null && blocking.size() >= NONSYNC_BLOCKED_LIMIT; + } + + void prepareExclusive(SafeTask owner) + { + try (BufferList keys = new BufferList<>()) + { + if ((blocking == null || !populate(keys, blocking)) && notBlocking != null) + populate(keys, notBlocking); + + keys.forEach(key -> preExecute(owner.refs.get(key), owner, RELEASE_QUEUE)); + keys.sort(RoutingKey::compareTo); + active = RoutingKeys.of(keys); + } + processed += active.size(); + if (processed == owner.keys && owner.isIncremental()) + owner.setIncrementalFinishingExclusive(); + } + + private boolean populate(List keys, ObjectHashSet from) + { + if (keys.size() + from.size() <= AccordExecutor.NONSYNC_MAX_BATCH_SIZE) + { + keys.addAll(from); + from.clear(); + return keys.size() == AccordExecutor.NONSYNC_MAX_BATCH_SIZE; + } + + Iterator iterator = from.iterator(); + while (iterator.hasNext()) + { + if (keys.size() == AccordExecutor.NONSYNC_MAX_BATCH_SIZE) + return true; + + RoutingKey key = iterator.next(); + keys.add(key); + iterator.remove(); + } + + return false; + } + + void postRunExclusive(SafeTask owner) + { + if (active != null) + { + for (RoutingKey key : active) + postExecute(owner.refs.remove(key), owner); + active = null; + } + } + } + + static class IncrementalState extends NonSyncState + { + long waiting, running; + + public IncrementalState(ExecutionContext context) + { + super(context); + } + + void updateMetrics(long waiting, long running) + { + this.running += running; + this.waiting += waiting; + } + } + + final AccordCommandStore commandStore; + private final ExecutionContext context; + private final Function function; + + // TODO (expected): simple custom map that allows (at least): + // - efficient putIfAbsent + // - efficient small collections (2-4 entries) + // - forEach with parameters to avoid boxing lambdas + // - destructive forEach + // - forEach over specific SafeState types + Object2ObjectHashMap> refs = new Object2ObjectHashMap<>(); + + /** + * if is(LOADING), this is the number of cache entries we're waiting to complete loading before we can transition to WAITING_ON_CACHE_QUEUES; + * if is(WAITING_ON_CACHE_QUEUES), it's the number we're waiting to be head of before we can run + *

+ * if isNonSync(), this counts only txnId; otherwise it counts keys and txnId + */ + int waitingForState; + + /** + * Only set when isNonSync() + *

+ * if is(LOADING), this is the cache entries that have finished loading + * otherwise it's the cache entries for which we're at the head of the queue and are ready to run with + */ + @Nullable NonSyncState nonSync; + + int keys; // TODO (expected): not counting keys we add during execution + LogLinearDecayingHistograms.Buffer histogramBuffer; + + @Nullable Object ranges; + long loadedAt; + + private BiConsumer callback; + + public SafeTask(@Nonnull AccordCommandStore commandStore, ExecutionContext context, Function function) + { + super(context, commandStore.executor().uniqueCreatedAt); + this.commandStore = commandStore; + this.context = context; + this.function = function; + if (logger.isTraceEnabled()) + logger.trace("Created {} on {}", this, commandStore); + } + + String loggingId() + { + return executor().executorId + "/" + Long.toHexString(createdAt); + } + + @Override + public String toString() + { + return "@[" + commandStore.id() + ',' + commandStore.node().id() + "] " + context.describe() + ' ' + toBriefString(); + } + + public String toBriefString() + { + return '{' + loggingId() + ',' + currentState() + '}'; + } + + public String summarise() + { + return loggingId() + ' ' + context.executionKind() + + ": primaryTxnId: " + context.primaryTxnId() + + ", state: " + summarise(refs, SaferState::global); + } + + private static String summarise(Map map, Function transform) + { + if (map == null) + return "null"; + + return summarise(map.values(), transform); + } + + private static String summarise(Collection collection) + { + return summarise(collection, Function.identity()); + } + + private static String summarise(Collection collection, Function transform) + { + if (collection == null) + return "null"; + + StringBuilder out = new StringBuilder("["); + int count = 0; + for (V v : collection) + { + if (count++ > 0) + { + out.append(','); + if (count >= 10) + { + out.append("...(*").append(collection.size() - 10).append(')'); + break; + } + } + out.append(transform.apply(v)); + } + out.append(']'); + return out.toString(); + } + + // TODO (expected): try to execute immediately BUT consider ordering requirements + // esp. with deferred actions on e.g. CommandsForKey (not yet supported but also important for performance) + public AsyncChain chain() + { + return new AsyncChains.Head<>() + { + @Override + protected Cancellable start(BiConsumer callback) + { + if (!preSetup(callback)) + executor().submitTask(SafeTask.this); + return SafeTask.this; + } + }; + } + + private boolean preSetup(BiConsumer callback) + { + Invariants.require(this.callback == null); + this.callback = callback; + Task inherit = executor().inherit(); + if (inherit == null) + return false; + + Invariants.require(inherit.is(PREPARED)); + if (!inherit.is(RUNNING)) + return false; + + if (inherit instanceof SafeTask) + { + SafeTask inheritRefs = (SafeTask) inherit; + if (inheritRefs.commandStore == commandStore) + preSetup(inheritRefs); + } + inherit.addConsequence(this); + return true; + } + + // to be invoked only by the CommandStore owning thread, to take references to objects already in use by the current execution + private void preSetup(SafeTask parent) + { + // note we use the caches "unsafely" here deliberately, as we only reference commands we already have references to + // so we do not mutate anything, except the atomic counter of references + LoadKeys loadKeys = loadKeys(context); + if (loadKeys != NONE) + { + Unseekables parentKeysOrRanges = parent.context.keys(); + Unseekables keysOrRanges = context.keys(); + + boolean isKeySubset = parent.isIncremental() ? parent.nonSync.active.containsAll(keysOrRanges) : parentKeysOrRanges.containsAll(keysOrRanges); + if (isKeySubset) + setInheritedRangeScan(); + + setSequencedExclusive(context.executionSequence()); + if (loadKeys != SYNC) + { + setNonSyncExclusive(); + if (loadKeys == INCR) + { + nonSync = new IncrementalState(context); + setIncrementalExclusive(); + // forbid BY_PRIORITY sequencing to avoid priority inversion deadlocks on INCR tasks that lock a TxnId but await some key that has a higher priority task (that is waiting on our locked TxnId) - solvable in future if necessary + Invariants.require(isSequencedByPriorityAtomic() || isUnsequenced(), "INCR tasks may currently only be ATOMIC or UNSEQUENCED"); + } + else + { + nonSync = new NonSyncState(context); + } + } + + if (isSequencedByPriorityAtomic()) + { + if (!isKeySubset) + { + Invariants.require(keysOrRanges.domain() == Key, "ATOMIC tasks over ranges must declare a subset of their parent's task keys() to avoid a range scan across which we would not impose sequencing"); + // to avoid priority inversion deadlocks, if we are not a strict subset of the parent task we permit running with a single key ready + nonSync.alwaysReady = true; + } + + boolean isTxnIdSubset = true; + for (TxnId txnId : context.txnIds()) + isTxnIdSubset &= parent.refs.containsKey(txnId); + Invariants.require(isTxnIdSubset, "ATOMIC tasks must declare a subset of their parent's task txnIds() to avoid a priority inversion deadlock"); + // TODO (required): we're appending to the fifo queue - does this maintain correct order? + setCacheQueuedFifoExclusive(); + } + + if (keysOrRanges.equals(parentKeysOrRanges)) + { + // TODO (desired): custom map we can more cheaply fork/copy + parent.refs.forEach((key, val) -> { + if (val instanceof SaferCommandsForKey) + preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); + }); + } + else + { + switch (keysOrRanges.domain()) + { + case Key: + for (RoutingKey key : (AbstractUnseekableKeys) keysOrRanges) + preSetup(key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); + break; + + case Range: + AbstractRanges ranges = (AbstractRanges) keysOrRanges; + parent.refs.forEach((key, val) -> { + if (val instanceof SaferCommandsForKey && ranges.contains((RoutingKey) key)) + preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); + }); + break; + } + } + } + + for (TxnId txnId : context.txnIds()) + preSetup(txnId, parent.refs, commandStore.cachesUnsafe().commands()); + } + + @Override + void submitExclusiveMayThrow() + { + Caches caches = commandStore.cachesExclusive(); + executor().registerExclusive(this); + setStateExclusive(REGISTERED); + boolean hasPreSetup = hasInherited(); + LoadKeys loadKeys = loadKeys(context); + if (loadKeys != NONE) + { + if (loadKeys != SYNC && !hasPreSetup) + { + setNonSyncExclusive(); + if (loadKeys != INCR) nonSync = new NonSyncState(context); + else + { + nonSync = new IncrementalState(context); + setIncrementalExclusive(); + } + } + + Unseekables keysOrRanges = context.keys(); + switch (keysOrRanges.domain()) + { + case Range: + if (!hasInheritedRangeScan()) setupRangeLoadsExclusive(caches); + else refs.forEach((k, v) -> { + if (v instanceof SaferCommandsForKey) + completePresetupExclusive((SaferCommandsForKey)v); + }); + break; + case Key: + setupKeyLoadsExclusive(hasPreSetup, caches, (AbstractUnseekableKeys) keysOrRanges, hasInheritedRangeScan()); + break; + } + } + + for (TxnId txnId : context.txnIds()) + { + if (hasPreSetup && completePresetupExclusive(txnId)) + continue; + setupExclusive(txnId, caches.commands(), 1); + } + + if (is(SCANNING_RANGES)) + return; + + onSetupOrScannedExclusive(); + } + + private void setupKeyLoadsExclusive(boolean hasPreSetup, Caches caches, Iterable setupKeys, boolean doNotScanRanges) + { + if (context.loadKeys() == NONE) + return; + + if (!doNotScanRanges && context.loadKeysFor() == RECOVERY) + { + Invariants.require(ranges == null); + RangeTxnScanner scanner = new RangeTxnScanner(); + ranges = scanner; + scanner.start(); + } + + int waitsForIncrement = isSync() ? 1 : 0; + for (RoutingKey setupKey : setupKeys) + { + if (hasPreSetup && completePresetupExclusive(setupKey)) + continue; + + setupExclusive(setupKey, caches.commandsForKeys(), waitsForIncrement); + } + } + + private void setupRangeLoadsExclusive(Caches caches) + { + if (context.loadKeysFor() == WRITE) + return; + + RangeTxnAndKeyScanner scanner = new RangeTxnAndKeyScanner(caches.commandsForKeys()); + ranges = scanner; + scanner.start(); + } + + // expects mutual exclusivity only on the command store + private & SaferState> void preSetup(K k, Map> parentMap, AccordCache.Type.Instance cache) + { + S ref = (S) parentMap.get(k); + if (ref == null) + return; + + AccordCacheEntry node = ref.global(); + int refs = node.increment(); + Invariants.require(refs > 1); + S safeState = cache.parent().adapter().safeRef(node); + this.refs.put(k, safeState); + if (cache.isCommandsForKey()) + keys++; + } + + private & SaferState> boolean completePresetupExclusive(K k) + { + S preacquired = (S) refs.get(k); + if (preacquired != null) + { + completePresetupExclusive(preacquired); + return true; + } + return false; + } + + private & SaferState> void completePresetupExclusive(S preacquired) + { + AccordCacheEntry entry = preacquired.global(); + if (entry.isLoaded()) completeSetupOfLoaded(entry); + else completeSetupOfLoading(entry, true); + } + + // expects to hold lock + private & SaferState> void setupExclusive(K k, AccordCache.Type.Instance cache, int waitForIncrement) + { + S safeRef = cache.acquire(k); + AccordCacheEntry entry = safeRef.global(); + Status entryStatus = entry.status(); + boolean submitLoad = false; + boolean isLoaded; + switch (entryStatus) + { + default: throw new UnhandledEnum(entryStatus); + case WAITING_TO_LOAD: + submitLoad = true; + case LOADING: + isLoaded = false; + waitingForState += waitForIncrement; + break; + case WAITING_TO_SAVE: + case SAVING: + case LOADED: + case MODIFIED: + case FAILED_TO_SAVE: + isLoaded = true; + } + + Object prev = refs.putIfAbsent(k, safeRef); + if (prev != null) + { + noSpamLogger.warn("ExecutionContext {} contained key {} more than once", refs, k); + cache.release(safeRef, this); + waitingForState -= waitForIncrement; + } + else + { + if (entry.isCommandsForKey()) + keys++; + + if (isLoaded) completeSetupOfLoaded(entry); + else + { + if (submitLoad) executor().cacheUnsafe().load(executor(), this, is(ExclusiveGroup.RANGE), entry); + completeSetupOfLoading(entry, !submitLoad); + } + } + } + + private void completeSetupOfLoaded(AccordCacheEntry entry) + { + if (isOptional(entry)) + { + nonSync.addLoaded(); + if (isCacheQueuedFifo()) + addQueuedOptionalKey(entry, entry.addFifo(this)); + } + else if (isCacheQueuedFifo()) + { + entry.addFifo(this); + } + } + + private void completeSetupOfLoading(AccordCacheEntry entry, boolean alreadyLoading) + { + if (alreadyLoading) + { + Loading loading = entry.loading(); + if (loading.loading != null && loading.loading.is(RANGE_LOAD) && loading.loading.is(WAITING_TO_RUN) && !is(ExclusiveGroup.RANGE)) + { + // requeue anything setup as a range load that's now needed for a key-based operation, so it can use the correct the queue limits + loading.loading.unqueue(executor().runnable); + loading.loading.override(LOAD); + executor().runnable.enqueue(loading.loading, false); + } + } + + if (isCacheQueuedFifo()) entry.addFifo(this); + else entry.addWaitingToLoad(this); + Invariants.paranoid(entry.waitingCount() == entry.references()); + } + + private void onSetupOrScannedExclusive() + { + if (waitingForState > 0) + { + setStateExclusive(LOADING_REQUIRED); + executor().loading.enqueue(this); + } + else onLoadedRequiredExclusive(); + } + + private void onLoadedRequiredExclusive() + { + if (isSync() || nonSync.isLoaded(this) || isCacheQueuedFifo()) + { + waitOnCacheQueuesExclusive(); + } + else + { + setStateExclusive(LOADING_OPTIONAL); + executor().loading.enqueue(this); + } + } + + boolean isUnsequenced(AccordCacheEntry entry) + { + return isUnsequenced() && (entry.isCommandsForKey() || !isIncremental()); + } + + boolean isOptional(AccordCacheEntry entry) + { + return isNonSync() && entry.isCommandsForKey(); + } + + boolean holdsLocksBetweenRuns() + { // TODO (desired): encode as a state bit + return isIncremental() && context.primaryTxnId() != null; + } + + private void waitOnCacheQueuesExclusive() + { + Invariants.require(waitingForState == 0); + loadedAt = Math.max(createdAt, nanoTime()); + executor().runnable.incrementArrivals(this); + commandStore.exclusiveExecutor().incrementArrivals(this); + + this.refs.forEach((key, safeState) -> { + AccordCacheEntry entry = global(safeState); + boolean optional = isOptional(entry); + if (entry.isLoaded()) + { + RunnableStatus status = addToCacheQueue(entry, false); + if (optional) addQueuedOptionalKey(entry, status); + else if (status == NOT_RUNNABLE) + ++waitingForState; + } + else Invariants.require(optional); + }); + + // TODO (desired): exception-safe rollback for addUnsequenced + setCacheQueuedExclusive(); + if (waitingForState == 0) waitOnOptionalCacheQueuesExclusive(); + else + { + setStateExclusive(WAITING_ON_REQUIRED); + executor().waiting.enqueue(this); + } + } + + private void waitOnOptionalCacheQueuesExclusive() + { + if (isSync() || nonSync.isWaitReady(this)) waitToRunExclusive(); + else + { + setStateExclusive(WAITING_ON_OPTIONAL); + executor().waiting.enqueue(this); + } + } + + RunnableStatus addToCacheQueue(AccordCacheEntry loaded, boolean addIfFifo) + { + if (isCacheQueuedFifo()) return addIfFifo ? loaded.addFifo(this) : loaded.headStatus(this); + else if (isUnsequenced(loaded)) return loaded.addUnsequenced(this); + else return loaded.addPrioritised(this); + } + + void onLoadOneExclusive(AccordCacheEntry loaded) + { + if (isOptional(loaded)) + { + // if we're incremental/async we don't block on keys loading, so we don't need to decrement anything + // however, if we're in fifo mode this loaded key might be ready for us to run with + State state = state(); + switch (state) + { + default: throw new UnhandledEnum(state); + case WAITING_ON_REQUIRED: + case WAITING_ON_OPTIONAL: + case WAITING_TO_RUN: + case PREPARED: + RunnableStatus status = addToCacheQueue(loaded, false); + if (status != NOT_RUNNABLE) + addQueuedOptionalKey(loaded, status); + // fall-through + case LOADING_REQUIRED: + nonSync.addLoaded(); + break; + + case LOADING_OPTIONAL: + addLoadedOptionalKey(); + break; + } + } + else + { + if (--waitingForState == 0) + { + if (is(LOADING_REQUIRED)) + { + unqueue(executor().loading); + onLoadedRequiredExclusive(); + } + else Invariants.require(is(SCANNING_RANGES)); + } + } + } + + void addLoadedOptionalKey() + { + nonSync.addLoaded(); + if (is(LOADING_OPTIONAL) && nonSync.isLoaded(this)) + { + unqueue(executor().loading); + waitOnCacheQueuesExclusive(); + } + } + + // TODO (expected): add vs setup vs onChange; some callers don't need to try + void addQueuedOptionalKey(AccordCacheEntry loaded, RunnableStatus status) + { + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case NOT_RUNNABLE: break; + case STILL_RUNNABLE: + case NEWLY_RUNNABLE: + nonSync.onNewHead(loaded); + break; + case STILL_RUNNABLE_NEWLY_BLOCKING: + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(loaded); + break; + } + + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady(this)) + { + unqueue(executor().waiting); + waitToRunExclusive(); + } + } + + void onChangeHeadStatus(AccordCacheEntry entry, RunnableStatus status) + { + if (isSync() || !entry.isCommandsForKey()) onChangeRequiredHeadStatus(status); + if (isNonSync() && entry.isCommandsForKey()) onChangeOptionalHeadStatus(entry, status); + } + + private void incrementWaitingWhileAlreadyWaiting() + { + Invariants.require(isState(WAITING)); + if (waitingForState == 0) + { + if (is(WAITING_ON_OPTIONAL)) setStateExclusive(WAITING_ON_REQUIRED); + else + { + // TODO (expected): this is potentially costly; maybe we don't want to swap these in and out (but harder to maintain invariants) + unqueue(commandStore.exclusiveExecutor()); + setStateExclusive(WAITING_ON_REQUIRED); + executor().waiting.enqueue(this); + } + } + Invariants.require(waitingForState < refs.size()); + ++waitingForState; + } + + void onChangeRequiredHeadStatus(RunnableStatus newStatus) + { + if (newStatus == NOT_RUNNABLE) + { + incrementWaitingWhileAlreadyWaiting(); + } + else if (newStatus != STILL_RUNNABLE_NEWLY_BLOCKING) + { + Invariants.require(is(WAITING_ON_REQUIRED)); + if (--waitingForState == 0) + { + unqueue(executor().waiting); + waitOnOptionalCacheQueuesExclusive(); + } + } + } + + void onChangeOptionalHeadStatus(AccordCacheEntry entry, RunnableStatus status) + { + Invariants.require(isState(WAITING_OR_PREPARED)); + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case STILL_RUNNABLE: throw UnhandledEnum.invalid(STILL_RUNNABLE); // onChange -> changed (but this means no change) + case NOT_RUNNABLE: + nonSync.onNotHead(entry); + if (is(WAITING_TO_RUN) && !nonSync.isWaitReady(this)) + { + unqueue(commandStore.exclusiveExecutor()); + setStateExclusive(WAITING_ON_OPTIONAL); + executor().waiting.enqueue(this); + } + return; + + case STILL_RUNNABLE_NEWLY_BLOCKING: + nonSync.onStillHeadNewBlocking(entry); + break; + + case NEWLY_RUNNABLE: + nonSync.onNewHead(entry); + break; + + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(entry); + break; + } + + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady(this)) + { + unqueue(executor().waiting); + waitToRunExclusive(); + } + } + + void waitToRunExclusive() + { + setStateExclusive(WAITING_TO_RUN); + commandStore.exclusiveExecutor().enqueue(this, false); + } + + public ExecutionContext executionContext() + { + return context; + } + + @Override + protected void prepareExclusiveMayThrow() + { + try + { + if (ranges instanceof SafeTask.RangeTxnScanner) + ranges = ((SafeTask.RangeTxnScanner)ranges).finish(commandStore.cachesExclusive()); + + if (isSync()) + { + refs.forEach((k, v) -> { + preExecute(v, this, RELEASE_QUEUE); + }); + } + else + { + if (!hasIncrementalStarted()) + { + TxnId primaryTxnId = context.primaryTxnId(); + if (primaryTxnId != null) + { + LockMode lockMode = holdsLocksBetweenRuns() ? HOLD_QUEUE : RELEASE_QUEUE; + preExecute(refs.get(primaryTxnId), this, lockMode); + TxnId additionalTxnId = context.additionalTxnId(); + if (additionalTxnId != null) + preExecute(refs.get(additionalTxnId), this, lockMode); + } + + if (isIncremental() && isSequencedByPriorityAtomic() && !isCacheQueuedFifo()) + { + setCacheQueuedFifoExclusive(); + refs.forEach((key, safeState) -> { + AccordCacheEntry entry = global(safeState); + RunnableStatus status = entry.moveToFifo(this); + if (entry.isLoaded()) + { + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case NOT_RUNNABLE: + case STILL_RUNNABLE: + case STILL_RUNNABLE_NEWLY_BLOCKING: + break; + case NEWLY_RUNNABLE: + nonSync.onNewHead(entry); + break; + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(entry); + break; + } + } + }); + } + + if (isIncremental()) + setIncrementalStartedExclusive(); + } + nonSync.prepareExclusive(this); + } + } + catch (Throwable t) + { + refs.forEach((k, v) -> { v.setAbandoned(); }); + throw t; + } + } + + @Override + public boolean runMayThrow() + { + try + { + SaferCommandStore safeStore = new SaferCommandStore(this, isSync() ? context : nonSync); + commandStore.begin(safeStore); + try + { + if (Tracing.isTracing()) + Tracing.trace(context.describe()); + + R result = function.apply(safeStore); + boolean finished = !isIncremental() || isIncrementalFinishing(); + if (!finished) setRunState(RUN_INCOMPLETE); + else + { + List changes = new ArrayList<>(); + // TODO (expected): save any TxnId we add so that we don't need to iterate all of refs + refs.forEach((key, value) -> { + if (value instanceof SaferCommand) + { + SaferCommand safeCommand = (SaferCommand) value; + Journal.CommandUpdate diff = safeCommand.update(); + if (diff != null) + { + changes.add(diff); + maybeSanityCheck(safeCommand); + } + } + }); + + boolean flush = !changes.isEmpty() || safeStore.fieldUpdates() != null; + if (flush) + { + setRunState(RUN_PERSISTING); + Runnable onFlush = () -> finish(result); + safeStore.persistFieldUpdatesInternal(changes.isEmpty() ? onFlush : null); + if (!changes.isEmpty()) + save(changes, onFlush); + finished = false; + } + } + + safeStore.postExecute(); + if (finished) + finish(result); + return finished; + } + finally + { + commandStore.complete(safeStore); + } + } + catch (Throwable t) + { + refs.forEach((k, v) -> v.setAbandoned()); + throw t; + } + } + + private void save(List diffs, Runnable onFlush) + { + if (SANITY_CHECK && DebugTask.get(this).sanityCheck != null) + { + Condition condition = Condition.newOneTimeCondition(); + this.commandStore.appendCommands(diffs, condition::signal); + condition.awaitUninterruptibly(); + + for (Command check : DebugTask.get(this).sanityCheck) + this.commandStore.sanityCheckCommand(commandStore.unsafeGetRedundantBefore(), check); + + if (onFlush != null) onFlush.run(); + } + else + { + this.commandStore.appendCommands(diffs, onFlush); + } + } + + private void maybeSanityCheck(SaferCommand safeCommand) + { + if (SANITY_CHECK) + { + DebugTask debug = DebugTask.get(this); + if (debug.sanityCheck == null) + debug.sanityCheck = new ArrayList<>(2); + debug.sanityCheck.add(safeCommand.current()); + } + } + + void reportFailureMayThrow(Throwable failure) + { + if (callback == null) executor().agent.onException(failure); + else + { + BiConsumer reportTo = callback; + callback = null; + + if (executor().isInLoop()) reportTo.accept(null, failure); + else executor().submit(() -> reportTo.accept(null, failure)); + } + } + + @Override + void maybeCompleteExclusiveMayThrow() + { + if (isNonSync() && !isEither(NOT_YET_RUN, RUN_FAILED)) + { + if (is(PREPARED)) + { + nonSync.postRunExclusive(this); + if (isIncremental()) + { + long now = nanoTime(); + ((IncrementalState)nonSync).updateMetrics(now - runningAt, runningAt - loadedAt); + if (!isIncrementalFinishing()) + { + setStateExclusive(INCOMPLETE); + waitOnOptionalCacheQueuesExclusive(); + return; + } + } + } + else + { + Invariants.expect(is(FAILED)); + Invariants.expect(isIncremental()); + setRunState(RUN_FAILED); + refs.forEach((k, v) -> v.setAbandoned()); + } + } + + releaseResourcesExclusive(commandStore.cachesExclusive()); + + AccordExecutor executor = executor(); + if (completeState()) + { + long completeAt = nanoTime(); + executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); + if (isIncremental()) + { + IncrementalState incrementalState = (IncrementalState) nonSync; + executor.elapsedWaitingToRun.increment(incrementalState.waiting, runningAt); + executor.elapsedRunning.increment(incrementalState.running, completeAt); + } + else + { + executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); + executor.elapsedRunning.increment(completeAt - runningAt, completeAt); + } + executor.elapsed.increment(completeAt - createdAt, completeAt); + executor.keys.increment(keys, runningAt); + if (histogramBuffer != null) + { + histogramBuffer.flush(runningAt); + histogramBuffer = null; + } + } + else if (histogramBuffer != null) + { + histogramBuffer.clear(); + } + } + + @Override + public void cancel() + { + if (!state().hasStarted()) + executor().submit(Task::tryCancelExclusive, CancelTask::new, this); + } + + void tryCancelExclusive() + { + State state = state(); + switch (state) + { + default: throw new UnhandledEnum(state); + case UNREGISTERED: + failExclusive(new CancellationException(), CANCELLED_UNREGISTERED); + break; + + case REGISTERED: + case SCANNING_RANGES: + case LOADING_REQUIRED: + case LOADING_OPTIONAL: + case WAITING_ON_REQUIRED: + case WAITING_ON_OPTIONAL: + case WAITING_TO_RUN: + if (!hasIncrementalStarted()) + cancelExclusive(); + break; + + case CANCELLED_UNREGISTERED: + case INCOMPLETE: + case PREPARED: + case EXECUTED: + case CANCELLED: + case FAILED: + // cannot safely cancel + } + } + + void cancelExclusive() + { + if (ranges instanceof SafeTask.RangeTxnScanner) + ((SafeTask.RangeTxnScanner)ranges).cancelled = true; // TODO (expected): should we try to cancel this directly? + failAndCompleteExclusive(new CancellationException(), CANCELLED); + } + + void unqueueIfQueued() + { + TaskQueue expected; + switch (queued()) + { + default: throw UnhandledEnum.unknown(queued()); + case NONE: return; + case LOADING: + expected = executor().loading; + break; + case WAITING: + expected = executor().waiting; + break; + case RUNNABLE: + expected = commandStore.exclusiveExecutor(); + break; + } + expected.unqueue(this); + } + + private void finish(R result) + { + if (callback != null) + { + try { callback.accept(result, null); } + catch (Throwable t) { commandStore.agent().onException(t); } + } + } + + void releaseResourcesExclusive(Caches caches) + { + if (refs == null) + return; + + try + { + if (ranges instanceof SafeTask.RangeTxnScanner) + { + ((SafeTask.RangeTxnScanner)ranges).cleanup(caches); + if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedRangeScanner(); + } + ranges = null; + + refs.forEach((key, safeState) -> { + SaferState.postExecute(safeState, this); + }); + if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedState(); + } + catch (Throwable t) + { + releaseResourcesSlowExclusive(t); + commandStore.agent().onException(t); + } + finally + { + refs = null; + } + } + + private void releaseResourcesSlowExclusive(Throwable suppressedBy) + { + if (refs == null) + return; + + refs.forEach((k, safeState) -> { + if (!safeState.isReleased()) + { + try { SaferState.postExecute(safeState, this); } + catch (Throwable t) { suppressedBy.addSuppressed(t); } + } + }); + refs = null; + } + + public class RangeTxnAndKeyScanner extends RangeTxnScanner + { + class KeyWatcher implements AccordCache.Listener + { + @Override + public void onUpdate(AccordCacheEntry state) + { + if (ranges.contains(state.key())) + reference((AccordCacheEntry) state); + } + } + + final Set intersectingKeys = new ObjectHashSet<>(); + final Ranges ranges = ((AbstractRanges) context.keys()).toRanges(); + final AccordCache.Type.Instance commandsForKeyCache; + KeyWatcher keyWatcher = new KeyWatcher(); + + public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) + { + this.commandsForKeyCache = commandsForKeyCache; + } + + protected void runInternal() + { + for (Range range : ranges) + { + CommandsForKeyAccessor.findAllKeysBetween(commandStore.id(), commandStore.tableId(), getPartitioner(), + (TokenKey) range.start(), range.startInclusive(), + (TokenKey) range.end(), range.endInclusive(), + key -> { + if (cancelled) + throw new CancellationException(); + intersectingKeys.add(key); + }); + } + super.runInternal(); + } + + private void reference(AccordCacheEntry entry) + { + switch (entry.status()) + { + default: throw new AssertionError("Unhandled Status: " + entry.status()); + case WAITING_TO_LOAD: + case LOADING: + return; + + case MODIFIED: + case WAITING_TO_SAVE: + case SAVING: + case LOADED: + case FAILED_TO_SAVE: + if (refs.containsKey(entry.key())) + return; + + Object v = entry.getOrShrunkExclusive(); + if (v == null) return; + else if (v instanceof CommandsForKey) + { + if (!loader.isRelevant((CommandsForKey) v)) + return; + } + else + { + TxnId last = CommandSerializers.txnId.deserialize((ByteBuffer) v); + int position = (int)CommandSerializers.txnId.serializedSize(last); + TxnId minUndecided = CommandSerializers.txnId.deserialize((ByteBuffer) v, position); + if (!loader.isRelevant(entry.key(), last, minUndecided)) + return; + } + + refs.put(entry.key(), commandsForKeyCache.acquire(entry)); + ++keys; + + Invariants.require(!isCacheQueuedFifo(), "Unsafe to addFifo in listener as no ordering guarantees"); + if (isOptional(entry)) + addLoadedOptionalKey(); + + if (isCacheQueued()) + { + Invariants.require(isState(WAITING)); + RunnableStatus status = addToCacheQueue(entry, false); + if (isOptional(entry)) addQueuedOptionalKey(entry, status); + else if (status == NOT_RUNNABLE) incrementWaitingWhileAlreadyWaiting(); + } + } + } + + void startInternal(Caches caches) + { + for (Range range : ranges) + { + for (RoutingKey key : caches.commandsForKeys().keysBetween(range.start(), range.startInclusive(), range.end(), range.endInclusive())) + intersectingKeys.add((TokenKey) key); + } + caches.commandsForKeys().register(keyWatcher); + super.startInternal(caches); + } + + void scannedInternal() + { + intersectingKeys.removeAll(refs.keySet()); + setupKeyLoadsExclusive(false, commandStore.cachesExclusive(), intersectingKeys, true); + super.scannedInternal(); + } + + void cleanup(Caches caches) + { + if (keyWatcher != null) + caches.commandsForKeys().tryUnregister(keyWatcher); + super.cleanup(caches); + } + + CommandSummaries finish(Caches caches) + { + caches.commandsForKeys().unregister(keyWatcher); + keyWatcher = null; + return super.finish(caches); + } + } + + public class RangeTxnScanner extends IOTaskWrapper.WrappableIOTask + { + final Map summaries = new HashMap<>(); + final Map guardedSummaries = Collections.synchronizedMap(summaries); + + RangeIndex.Loader loader; + boolean scanned; + Throwable failure; + + volatile boolean cancelled; + + protected void runInternal() + { + loader.load(guardedSummaries, () -> cancelled); + } + + ExecutionContext preLoadContext() + { + return context; + } + + @Override + protected void postRunExclusive() + { + executor().onScannedRangesExclusive(SafeTask.this, failure); + } + + @Override + protected void fail(Throwable t) + { + this.failure = t; + } + + public void start() + { + Caches caches = commandStore.cachesExclusive(); + SafeTask.this.setStateExclusive(SCANNING_RANGES); + executor().loading.enqueue(SafeTask.this); + startInternal(caches); + executor().submitExclusive(SafeTask.this, GlobalGroup.RANGE_SCAN, this); + } + + void startInternal(Caches caches) + { + loader = commandStore.rangeIndex().loader(context.primaryTxnId(), context.executeAt(), context.loadKeysFor(), context.keys()); + loader.loadExclusive(guardedSummaries, caches); + } + + public void scannedExclusive() + { + Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", SafeTask.this, SafeTask::description); + scanned = true; + scannedInternal(); + unqueue(executor().loading); // likely to be requeued to same queue, but simpler invariants if we remove here + onSetupOrScannedExclusive(); + } + + void scannedInternal() + { + } + + void cleanup(Caches caches) + { + if (loader != null) + loader.cleanupExclusive(caches); + } + + CommandSummaries finish(Caches caches) + { + loader.finish(summaries); + loader.cleanupExclusive(caches); + loader = null; + TreeMap byId = new TreeMap<>(summaries); + return (CommandSummaries.ByTxnIdSnapshot) () -> byId; + } + + @Override + public String description() + { + return "Scanning range intersections for " + context.reason() + ' ' + toBriefString(); + } + + @Override + public String toString() + { + return description(); + } + } + + @Override + public String description() + { + return context.describe(); + } + + @Override + public String briefDescription() + { + return context.reason(); + } + + final AccordExecutor executor() + { + return commandStore.executor(); + } + + @Override + protected boolean isNewWork() + { + return true; + } + + public @Nullable SafeTask.RangeTxnScanner rangeScanner() + { + if (ranges instanceof SafeTask.RangeTxnScanner) + return ((SafeTask.RangeTxnScanner) ranges); + return null; + } + + public @Nullable CommandSummaries commandsForRanges() + { + if (ranges instanceof CommandSummaries) + return (CommandSummaries) ranges; + return null; + } + + private static LoadKeys loadKeys(ExecutionContext context) + { + LoadKeys loadKeys = context.loadKeys(); + if (NONSYNC_ENABLED) + return loadKeys; + return loadKeys == NONE ? NONE : SYNC; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/SaferCommand.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommand.java new file mode 100644 index 0000000000..3d066aa425 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommand.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.Objects; + +import accord.api.Journal; +import accord.local.Command; +import accord.local.SafeCommand; +import accord.primitives.TxnId; + +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; + +public class SaferCommand extends SafeCommand implements SaferState +{ + private final AccordCacheEntry global; + private Command original; + + public SaferCommand(AccordCacheEntry global) + { + super(global.key()); + this.global = global; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SaferCommand that = (SaferCommand) o; + return Objects.equals(this.original, that.original) && Objects.equals(this.current(), that.current()); + } + + @Override + public int hashCode() + { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() + { + return "AccordSafeCommand{" + + "status=" + statusString() + + ", global=" + global + + ", original=" + original + + ", current=" + current + + '}'; + } + + public AccordCacheEntry global() + { + return global; + } + + @Override + public void postExecute(SafeTask owner) + { + global.releaseExclusive(this, owner); + } + + public Journal.CommandUpdate update() + { + return new Journal.CommandUpdate(original, current()); + } + + public void preExecute(SafeTask owner, LockMode lockMode) + { + requireUninitialised(); + original = global.lockExclusive(owner, lockMode); + current = original; + if (current == null) + initialise(); + setSafe(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java similarity index 53% rename from src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java rename to src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java index 6a9817bc97..89c3f4b46b 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java @@ -16,15 +16,10 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; -import java.util.Collections; -import java.util.Map; -import java.util.Set; import java.util.function.Predicate; -import javax.annotation.Nullable; - import com.google.common.annotations.VisibleForTesting; import accord.api.Agent; @@ -36,86 +31,76 @@ import accord.impl.AbstractSafeCommandStore; import accord.local.Command; import accord.local.CommandStores; import accord.local.CommandSummaries; +import accord.local.ExecutionContext; import accord.local.NodeCommandStoreService; import accord.local.RedundantBefore; +import accord.local.SafeState; import accord.local.cfk.CommandsForKey; import accord.local.cfk.SafeCommandsForKey; +import accord.primitives.AbstractUnseekableKeys; +import accord.primitives.Routables; import accord.primitives.Timestamp; import accord.primitives.Txn.Kind.Kinds; import accord.primitives.TxnId; import accord.primitives.Unseekables; import accord.utils.Invariants; +import accord.utils.UnhandledEnum; import org.apache.cassandra.metrics.LogLinearDecayingHistograms; +import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches; import org.apache.cassandra.service.accord.AccordCommandStore.SafeRedundantBefore; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; import org.apache.cassandra.service.paxos.PaxosState; import static accord.utils.Invariants.illegalState; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.UNQUEUED; -public class AccordSafeCommandStore extends AbstractSafeCommandStore +public final class SaferCommandStore extends AbstractSafeCommandStore { - final AccordTask task; - private final @Nullable CommandSummaries commandsForRanges; - private final AccordCommandStore commandStore; + final SafeTask task; - private AccordSafeCommandStore(AccordTask task, - @Nullable CommandSummaries commandsForRanges, - AccordCommandStore commandStore) + SaferCommandStore(SafeTask task, ExecutionContext context) { - super(task.preLoadContext(), commandStore); + super(context); this.task = task; - this.commandsForRanges = commandsForRanges; - this.commandStore = commandStore; - } - - @Override - public CommandStores.RangesForEpoch ranges() - { - CommandStores.RangesForEpoch ranges = super.ranges(); - if (ranges != null) - return ranges; - - return commandStore.unsafeGetRangesForEpoch(); - } - - public static AccordSafeCommandStore create(AccordTask task, - @Nullable CommandSummaries commandsForRanges, - AccordCommandStore commandStore) - { - return new AccordSafeCommandStore(task, commandsForRanges, commandStore); } @VisibleForTesting - public Set commandsForKeysKeys() + public Iterable safeCommandsForKeys() { - if (task.commandsForKey() == null) - return Collections.emptySet(); - return task.commandsForKey().keySet(); + return task.refs.values().stream() + .filter(v -> v instanceof SafeCommandsForKey) + .map(v -> (SafeCommandsForKey)v)::iterator; } @Override - protected AccordSafeCommand getInternal(TxnId txnId) + protected SaferCommand getInternal(TxnId txnId) { - Map commands = task.commands(); - if (commands == null) - return null; - return commands.get(txnId); + return (SaferCommand) task.refs.get(txnId); } @Override protected ExclusiveCaches tryGetCaches() { - return commandStore.tryLockCaches(); + if (task.isIncremental()) + { + // We could relax this, but requires careful consideration of semantics when: + // - touching keys we have scheduled to process later + // - touching txnIds we may have to logically lock until the whole processing completes to + // ensure persistent state machine updates are consistent with incremental updates to cache + return null; + } + return commandStore().tryLockCaches(); } - protected AccordSafeCommand add(AccordSafeCommand safeCommand, ExclusiveCaches caches) + protected SaferCommand add(SaferCommand safeCommand, ExclusiveCaches caches) { - Object check = task.ensureCommands().putIfAbsent(safeCommand.txnId(), safeCommand); + Object check = task.refs.putIfAbsent(safeCommand.txnId(), safeCommand); if (check == null) { - safeCommand.preExecute(); + Invariants.require(!task.isIncremental()); // if isIncremental we'll have to supply holdQueue==true, but for now we forbid it + safeCommand.preExecute(task, UNQUEUED); return safeCommand; } else @@ -131,7 +116,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore { - AccordCommandStore.safeRedundantBeforeUpdater.accumulateAndGet(commandStore, update, SafeRedundantBefore::max); - }; + Runnable reportRedundantBefore = SafeRedundantBefore.updater(commandStore(), updates.newRedundantBefore); Runnable prevOnDone = onDone; onDone = prevOnDone == null ? reportRedundantBefore : () -> { try { reportRedundantBefore.run(); } finally { prevOnDone.run(); } }; } - commandStore.persistFieldUpdates(updates, onDone); + commandStore().persistFieldUpdates(updates, onDone); } - protected AccordSafeCommandsForKey add(AccordSafeCommandsForKey safeCfk, ExclusiveCaches caches) + protected SaferCommandsForKey add(SaferCommandsForKey safeCfk, ExclusiveCaches caches) { - Object check = task.ensureCommandsForKey().putIfAbsent(safeCfk.key(), safeCfk); + Object check = task.refs.putIfAbsent(safeCfk.key(), safeCfk); if (check == null) { - safeCfk.preExecute(); + safeCfk.preExecute(task, UNQUEUED); return safeCfk; } else @@ -169,19 +150,19 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore commandsForKey = task.commandsForKey(); - if (commandsForKey == null) + SaferCommandsForKey safeCfk = (SaferCommandsForKey) task.refs.get(key); + if (safeCfk == null || safeCfk.isUninitialised()) return null; - return commandsForKey.get(key); + return safeCfk; } @Override public void setRangesForEpoch(CommandStores.RangesForEpoch rangesForEpoch) { super.setRangesForEpoch(rangesForEpoch); - commandStore.updateMinHlc(PaxosState.ballotTracker().getLowBound().unixMicros() + 1); + commandStore().updateMinHlc(PaxosState.ballotTracker().getLowBound().unixMicros() + 1); } @Override @@ -194,13 +175,13 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore keysOrRanges, Predicate forEach) { - Map commandsForKey = task.commandsForKey; - if (commandsForKey == null) - return true; - - Unseekables skip = context.keys().without(keysOrRanges); - for (SafeCommandsForKey safeCfk : commandsForKey.values()) + Unseekables unseekables = context.keys(); + switch (unseekables.domain()) { - if (skip.contains(safeCfk.key())) - continue; + default: throw new UnhandledEnum(unseekables.domain()); + case Key: + AbstractUnseekableKeys keys = (AbstractUnseekableKeys) context.keys(); + return Routables.foldl(keys, keysOrRanges, (self, f, key, v, index) -> { + SafeCommandsForKey safeCfk = (SafeCommandsForKey) self.task.refs.get(key); + return f.test(safeCfk.current()); + }, this, forEach, Boolean.TRUE, cont -> !cont); - if (!forEach.test(safeCfk.current())) - return false; + case Range: + Unseekables skip = context.keys().without(keysOrRanges); + for (SafeState safeState : task.refs.values()) + { + if (!(safeState instanceof SaferCommandsForKey)) + continue; + + SafeCommandsForKey safeCfk = (SafeCommandsForKey) safeState; + if (skip.contains(safeCfk.key())) + continue; + + if (!forEach.test(safeCfk.current())) + return false; + } + return true; } - return true; } @Override public void visit(Unseekables keysOrRanges, Timestamp startedBefore, Kinds testKind, ActiveCommandVisitor visitor, P1 p1, P2 p2) { visitForKey(keysOrRanges, cfk -> { cfk.visit(startedBefore, testKind, visitor, p1, p2); return true; }); + CommandSummaries commandsForRanges = task.commandsForRanges(); if (commandsForRanges != null) commandsForRanges.visit(keysOrRanges, startedBefore, testKind, visitor, p1, p2); } @@ -268,8 +264,11 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore keysOrRanges, TxnId testTxnId, Kinds testKind, SupersedingCommandVisitor visit) { - return visitForKey(keysOrRanges, cfk -> cfk.visit(testTxnId, testKind, visit)) - && (commandsForRanges == null || commandsForRanges.visit(keysOrRanges, testTxnId, testKind, visit)); + if (!visitForKey(keysOrRanges, cfk -> cfk.visit(testTxnId, testKind, visit))) + return false; + + CommandSummaries commandsForRanges = task.commandsForRanges(); + return commandsForRanges == null || commandsForRanges.visit(keysOrRanges, testTxnId, testKind, visit); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/execution/SaferCommandsForKey.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandsForKey.java new file mode 100644 index 0000000000..ea188159be --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandsForKey.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.Objects; + +import accord.api.RoutingKey; +import accord.local.cfk.CommandsForKey; +import accord.local.cfk.NotifySink; +import accord.local.cfk.SafeCommandsForKey; + +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; + +public class SaferCommandsForKey extends SafeCommandsForKey implements SaferState +{ + public static class CommandsForKeyCacheEntry extends AccordCacheEntry + { + private NotifySink overrideSink; + + CommandsForKeyCacheEntry(RoutingKey key, AccordCache.Type.Instance owner) + { + super(key, owner); + } + } + + private final AccordCacheEntry global; + + public SaferCommandsForKey(AccordCacheEntry global) + { + super(global.key()); + this.global = global; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SaferCommandsForKey that = (SaferCommandsForKey) o; + return Objects.equals(current, that.current); + } + + @Override + public int hashCode() + { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() + { + return "AccordSafeCommandsForKey{" + + "state=" + statusString() + + ", global=" + global + + ", current=" + current + + '}'; + } + + public final AccordCacheEntry global() + { + return global; + } + + @Override + public void postExecute(SafeTask owner) + { + global.releaseExclusive(this, owner); + } + + @Override + public void overrideSink(NotifySink overrideSink) + { + ((CommandsForKeyCacheEntry)global).overrideSink = overrideSink; + } + + @Override + public NotifySink overrideSink() + { + return ((CommandsForKeyCacheEntry)global).overrideSink; + } + + public void preExecute(SafeTask owner, LockMode lockMode) + { + requireUninitialised(); + current = global.lockExclusive(owner, lockMode); + if (current == null) + initialize(); + setSafe(); + } + +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/SaferState.java b/src/java/org/apache/cassandra/service/accord/execution/SaferState.java new file mode 100644 index 0000000000..6708bdaa05 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferState.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.accord.execution; + +import accord.local.SafeState; + +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; + +public interface SaferState & SaferState> +{ + AccordCacheEntry global(); + void postExecute(SafeTask owner); + void preExecute(SafeTask owner, LockMode lockMode); + + static AccordCacheEntry global(SafeState safeState) + { + return safeState.getClass() == SaferCommand.class ? ((SaferCommand) safeState).global() + : ((SaferCommandsForKey) safeState).global(); + } + + static void postExecute(SafeState safeState, SafeTask owner) + { + if (safeState.getClass() == SaferCommand.class) ((SaferCommand) safeState).postExecute(owner); + else ((SaferCommandsForKey) safeState).postExecute(owner); + } + + static void preExecute(SafeState safeState, SafeTask owner, LockMode lockMode) + { + if (safeState.getClass() == SaferCommand.class) ((SaferCommand) safeState).preExecute(owner, lockMode); + else ((SaferCommandsForKey) safeState).preExecute(owner, lockMode); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Task.java b/src/java/org/apache/cassandra/service/accord/execution/Task.java new file mode 100644 index 0000000000..c377ea3346 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Task.java @@ -0,0 +1,939 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.CancellationException; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicLong; + +import accord.local.ExecutionContext; +import accord.messages.Accept; +import accord.messages.Commit; +import accord.messages.MessageType; +import accord.messages.Request; +import accord.primitives.Ballot; +import accord.primitives.SaveStatus; +import accord.primitives.TxnId; +import accord.utils.IntrusiveHeapNode; +import accord.utils.Invariants; +import accord.utils.TinyEnumSet; +import accord.utils.UnhandledEnum; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.DebuggableTask; +import org.apache.cassandra.concurrent.ExecutorLocals; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.service.accord.execution.ExclusiveExecutor.ExclusiveExecutorTask; +import org.apache.cassandra.utils.Closeable; +import org.apache.cassandra.utils.WithResources; + +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY; +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY_ATOMIC; +import static accord.primitives.Routable.Domain.Range; +import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.ORIG_HLC_FIFO; +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.APPLY; +import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.COMMIT; +import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.STABLE; +import static org.apache.cassandra.service.accord.execution.Task.RunState.NOT_YET_RUN; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_FAILED; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_INCOMPLETE; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED_UNREGISTERED; +import static org.apache.cassandra.service.accord.execution.Task.State.EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.PREPARED; +import static org.apache.cassandra.service.accord.execution.Task.State.PREPARED_OR_EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.execution.Task.State.UNREGISTERED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +public abstract class Task extends IntrusiveHeapNode implements Cancellable, DebuggableTask +{ + private static final int WAITING_ON_OPTIONAL_BIT = 1 << 7; + private static final int WAITING_TO_RUN_BIT = 1 << 8; + private static final int INCOMPLETE_BIT = 1 << 10; + + enum State + { + UNREGISTERED(), + CANCELLED_UNREGISTERED(UNREGISTERED), + REGISTERED(UNREGISTERED), + SCANNING_RANGES(REGISTERED), + LOADING_REQUIRED(REGISTERED, SCANNING_RANGES), + LOADING_OPTIONAL(REGISTERED, SCANNING_RANGES, LOADING_REQUIRED), + WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), + WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), + WAITING_TO_RUN(INCOMPLETE_BIT, UNREGISTERED, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), + PREPARED(WAITING_TO_RUN), + INCOMPLETE(PREPARED), + EXECUTED(PREPARED), + FAILED(UNREGISTERED, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, PREPARED, INCOMPLETE), + CANCELLED(UNREGISTERED, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), + ; + + private final int permittedFrom; + public static final int WAITING = TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN); + public static final int WAITING_OR_PREPARED = WAITING | TinyEnumSet.encode(PREPARED); + public static final int PREPARED_OR_EXECUTED = TinyEnumSet.encode(PREPARED, EXECUTED); + static final State[] VALUES = values(); + + static + { + // hack to allow us to create loops in our enum transition declarations + Invariants.require(INCOMPLETE_BIT == 1 << INCOMPLETE.ordinal()); + Invariants.require(WAITING_TO_RUN_BIT == 1 << WAITING_TO_RUN.ordinal()); + Invariants.require(WAITING_ON_OPTIONAL_BIT == 1 << WAITING_ON_OPTIONAL.ordinal()); + } + + State() + { + this.permittedFrom = 0; + } + + State(State... permittedFroms) + { + this(0, permittedFroms); + } + + State(int additional, State... permittedFroms) + { + int permittedFrom = additional; + for (State state : permittedFroms) + permittedFrom |= 1 << state.ordinal(); + this.permittedFrom = permittedFrom; + } + + boolean isPermittedFrom(int prevOrdinal) + { + return (permittedFrom & (1 << prevOrdinal)) != 0; + } + + boolean isDone() + { + return this.compareTo(EXECUTED) >= 0; + } + + boolean hasStarted() + { + return this.compareTo(PREPARED) >= 0; + } + + static State forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + + enum RunState + { + NOT_YET_RUN, RUNNING, RUN_INCOMPLETE, RUN_PERSISTING, RUN_SUCCESS, RUN_FAILED; + + private static final RunState[] VALUES = values(); + + static RunState forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + + enum GlobalGroup + { + COMMAND_STORE, + LOAD, + SAVE, + OTHER, + RANGE_LOAD, + RANGE_SCAN, + } + + enum ExclusiveGroup + { + APPLY, + STABLE, + COMMIT, + ACCEPT, + OTHER, + RECOVER, + PREACCEPT, + RANGE, + } + + public enum GroupKind + { + EXCLUSIVE(ExclusiveGroup.values().length, EXCLUSIVE_GROUP_SHIFT), + GLOBAL(GlobalGroup.values().length, GLOBAL_GROUP_SHIFT), + NONE(0, 0); + + final int count; + final byte shift; + + GroupKind(int count, int shift) + { + this.count = count; + this.shift = (byte) shift; + } + } + + enum ExecutorQueue + { + NONE(), + LOADING(SCANNING_RANGES, LOADING_OPTIONAL, LOADING_REQUIRED), + WAITING(WAITING_ON_OPTIONAL, WAITING_ON_REQUIRED), + RUNNABLE(WAITING_TO_RUN); + + private static final ExecutorQueue[] VALUES = values(); + + final int permittedStates; + + ExecutorQueue(State ... states) + { + this.permittedStates = TinyEnumSet.encode(states); + } + + public static ExecutorQueue forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + + private static final int STATE_MASK = 0xf; + static final int GROUP_MASK = 0x7; + private static final int EXCLUSIVE_GROUP_SHIFT = 4; + private static final int GLOBAL_GROUP_SHIFT = 7; + + private static final int NONSYNC_BIT = 1 << 10; + private static final int CACHE_QUEUED_BIT = 1 << 11; + + private static final int HAS_TRANCHE_BIT = 1 << 12; + private static final int HAS_INHERITED_BIT = 1 << 13; + private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 14; + + private static final int EXECUTOR_QUEUE_SHIFT = 16; + private static final int EXECUTOR_QUEUE_UNSHIFTED_MASK = 0x3; + private static final int EXECUTOR_QUEUE_SHIFTED_MASK = EXECUTOR_QUEUE_UNSHIFTED_MASK << EXECUTOR_QUEUE_SHIFT; + + private static final int INCREMENTAL_SHIFT = 18; + private static final int INCREMENTAL_MASK = 0x3 << INCREMENTAL_SHIFT; + private static final int INCREMENTAL = 0x1 << INCREMENTAL_SHIFT; + private static final int INCREMENTAL_STARTED = 0x2 << INCREMENTAL_SHIFT; + private static final int INCREMENTAL_FINISHING = 0x3 << INCREMENTAL_SHIFT; + + private static final int SEQUENCED_SHIFT = 20; + private static final int SEQUENCED_MASK = 0x3 << SEQUENCED_SHIFT; + private static final int SEQUENCED_PRIORITY = 0x1 << SEQUENCED_SHIFT; + private static final int SEQUENCED_ATOMIC = 0x2 << SEQUENCED_SHIFT; + private static final int SEQUENCED_ATOMIC_AND_QUEUED = 0x3 << SEQUENCED_SHIFT; + + private static final int TRANCHE_SHIFT = 22; + static final int MAX_TRANCHE = 0x3ff; + + static + { + Invariants.require(SEQUENCED_PRIORITY == BY_PRIORITY.ordinal() << SEQUENCED_SHIFT); + Invariants.require(SEQUENCED_ATOMIC == BY_PRIORITY_ATOMIC.ordinal() << SEQUENCED_SHIFT); + Invariants.require(ExecutionContext.ExecutionSequence.values().length <= 3); + } + + // TODO (desired): quite heavy to pass-through tracing session state we mostly don't use + public final WithResources resources; + Task next; + Task consequences; + + long position; + int info; + + public final long createdAt; + long runningAt; + // TODO (desired): expose via executors vtable + private volatile int runState; + + private static final AtomicIntegerFieldUpdater runStateUpdater = AtomicIntegerFieldUpdater.newUpdater(Task.class, "runState"); + + Task(GlobalGroup group) + { + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(group, ExclusiveGroup.OTHER); + createdAt = nanoTime(); + } + + Task(ExclusiveGroup group) + { + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(GlobalGroup.OTHER, group); + createdAt = nanoTime(); + } + + protected Task(ExecutionContext context, AtomicLong lastCreatedAt) + { + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next <= prev ? prev + 1 : next); + ExclusiveGroup group = ExclusiveGroup.OTHER; + TxnId txnId = context.primaryTxnId(); + if (txnId != null) + { + if (txnId.is(Range)) group = ExclusiveGroup.RANGE; + else + { + switch (AccordExecutor.PRIORITY_MODEL) + { + case HLC_FIFO: + case ORIG_HLC_FIFO: + { + // TODO (expected): port to ExecutionKind; also we aren't consistent about using Ballot + if (context instanceof Request) + { + MessageType type = ((Request) context).type(); + if (type instanceof MessageType.StandardMessage) + { + switch ((MessageType.StandardMessage) type) + { + case APPLY_REQ: + { + group = APPLY; + break; + } + case READ_EPHEMERAL_REQ: + case READ_REQ: + case STABLE_THEN_READ_REQ: + { + group = STABLE; + break; + } + case COMMIT_REQ: + { + Commit commit = (Commit) context; + if (AccordExecutor.PRIORITY_MODEL == ORIG_HLC_FIFO && !commit.ballot.equals(Ballot.ZERO)) + txnId = null; + if (commit.kind.saveStatus == SaveStatus.Stable) group = STABLE; + else group = COMMIT; + break; + } + case ACCEPT_REQ: + { + Accept accept = (Accept) context; + if (AccordExecutor.PRIORITY_MODEL == ORIG_HLC_FIFO && !accept.ballot.equals(Ballot.ZERO)) + txnId = null; + group = ExclusiveGroup.ACCEPT; + break; + } + case GET_EPHEMERAL_READ_DEPS_REQ: + case PRE_ACCEPT_REQ: + { + group = ExclusiveGroup.PREACCEPT; + break; + } + default: + { + txnId = null; + } + } + } + } + else + { + txnId = null; + } + break; + } + case FIFO: + { + txnId = null; + break; + } + } + } + } + + this.info = init(GlobalGroup.OTHER, group); + if (txnId != null) + this.position = txnId.hlc(); + } + + public final Task unwrap() + { + if (this instanceof ExclusiveExecutorTask) + return ((ExclusiveExecutorTask) this).queue.task; + return this; + } + + public DebuggableTask debuggable() + { + return this; + } + + public final long creationTimeNanos() + { + return createdAt; + } + + public final long startTimeNanos() + { + if (runState() == NOT_YET_RUN) + return 0; + return runningAt; + } + + abstract void submitExclusiveMayThrow(); + /** Return true if COMPLETED successfully. false indicates more work is being done. Should not throw any exceptions related to reporting success. */ + abstract boolean runMayThrow(); + abstract void maybeCompleteExclusiveMayThrow(); + abstract void tryCancelExclusive(); + abstract void reportFailureMayThrow(Throwable fail); + + abstract AccordExecutor executor(); + abstract void unqueueIfQueued(); + abstract boolean isNewWork(); + abstract String briefDescription(); + + final void submitExclusiveNoExcept() + { + if (is(UNREGISTERED)) + { + try { submitExclusiveMayThrow(); } + catch (Throwable t) + { + tryFailAndCompleteExclusive(t, State.FAILED); + onException(t); + } + } + } + + /** + * Prepare to run while holding the state cache lock. + * If returns false, prepare failed and the task should be discarded with no further action. + */ + final boolean prepareExclusiveNoExcept() + { + if (getClass() == ExclusiveExecutorTask.class) + { + return ((ExclusiveExecutorTask)this).prepareTask(); + } + else + { + try + { + prepareExclusiveMayThrow(); + setStateExclusive(State.PREPARED); + return true; + } + catch (Throwable t) + { + failAndCompleteExclusive(t, State.FAILED); + return false; + } + } + } + + void prepareExclusiveMayThrow() + { + } + + /** + * Run the command; the state cache lock may or may not be held depending on the executor implementation + */ + final void runNoExcept(TaskRunner self) + { + if (getClass() == ExclusiveExecutorTask.class) + { + ((ExclusiveExecutorTask)this).queue.runTask(self); + } + else + { + onRunning(); + self.setAccordActiveTask(this); + try (Closeable close = resources.get()) + { + if (runMayThrow()) + onSuccess(); + else + Invariants.require(compareTo(RUNNING) > 0); + } + catch (Throwable t) + { + setRunState(RunState.RUN_FAILED); + reportFailureNoExcept(t); + } + finally + { + self.setAccordActiveTask(null); + } + } + } + + final void rejectAtRuntime(Throwable reject) + { + setRunState(RunState.RUN_FAILED); + reportFailureNoExcept(reject); + } + + final void completeExclusiveNoExcept() + { + try + { + if (DEBUG_EXECUTION) DebugTask.get(this).onComplete(); + maybeCompleteExclusiveMayThrow(); + } + catch (Throwable t) + { + onException(t); + if (compareTo(EXECUTED) < 0) + failExclusive(t, State.FAILED); + } + finally + { + if (compareTo(EXECUTED) >= 0) + { + try { submitConsequencesExclusive(is(EXECUTED)); } + catch (Throwable t) { onException(t); } + executor().completedTaskExclusive(this); + } + } + } + + final void onException(Throwable t) + { + try { executor().agent.onException(t); } + catch (Throwable t2) { } + } + + final void reportFailureNoExcept(Throwable fail) + { + try { reportFailureMayThrow(fail); } + catch (Throwable t) + { + try { fail.addSuppressed(t); } + catch (Throwable t2) { } + onException(fail); + } + } + + // propagate RunState to State + // true if task was executed successfully + final boolean completeState() + { + RunState runState = runState(); + boolean success; + switch (runState) + { + default: throw UnhandledEnum.unknown(runState); + case RUN_INCOMPLETE: throw UnhandledEnum.invalid(runState); + case NOT_YET_RUN: + Invariants.expect(state().isDone()); + Invariants.expect(consequences == null); + success = false; + break; + + case RUN_FAILED: + if (compareTo(EXECUTED) < 0) + setStateExclusive(State.FAILED); + success = false; + break; + + case RUNNING: + case RUN_PERSISTING: + case RUN_SUCCESS: + setStateExclusive(EXECUTED); + success = true; + break; + } + return success; + } + + final void tryFailAndCompleteExclusive(Throwable fail, State newState) + { + if (is(UNREGISTERED)) + failExclusive(fail, CANCELLED_UNREGISTERED); + else if (compareTo(WAITING_TO_RUN) <= 0) + failAndCompleteExclusive(fail, newState); + } + + final void failExclusive(Throwable fail, State newState) + { + unqueueIfQueued(); + setStateExclusive(newState); + reportFailureNoExcept(fail); + } + + final void failAndCompleteExclusive(Throwable fail, State newState) + { + failExclusive(fail, newState); + completeExclusiveNoExcept(); + } + + final void unqueue(TaskQueue expected) + { + Invariants.require(queuedOrdinal() == expected.kind.ordinal()); + expected.unqueue(this); + info &= ~EXECUTOR_QUEUE_SHIFTED_MASK; + } + + final void unsetQueue(ExecutorQueue expected) + { + Invariants.require(expected.ordinal() == queuedOrdinal()); + info &= ~EXECUTOR_QUEUE_SHIFTED_MASK; + } + + final void onRunning() + { + Invariants.require(is(PREPARED)); + Invariants.require(is(NOT_YET_RUN) || is(RUN_INCOMPLETE)); + runningAt = Math.max(createdAt, nanoTime()); + setRunState(RunState.RUNNING); + if (DEBUG_EXECUTION) ((DebugTask) resources).onRunning(); + } + + final void onSuccess() + { + setRunState(RunState.RUN_SUCCESS); + if (DEBUG_EXECUTION) ((DebugTask) resources).onRunComplete(); + } + + final State state() + { + return State.forOrdinal(stateOrdinal()); + } + + final RunState runState() + { + return RunState.forOrdinal(runState); + } + + final Enum currentState() + { + State state = state(); + if (state == State.PREPARED || state == EXECUTED) + { + RunState runState = runState(); + if (runState == NOT_YET_RUN) + return state; + return runState; + } + return State.forOrdinal(stateOrdinal()); + } + + final int stateOrdinal() + { + return info & STATE_MASK; + } + + final boolean is(State state) + { + return stateOrdinal() == state.ordinal(); + } + + final int compareTo(State state) + { + return stateOrdinal() - state.ordinal(); + } + + final int compareTo(RunState state) + { + return runState - state.ordinal(); + } + + final boolean is(RunState state) + { + return runState == state.ordinal(); + } + + final boolean isEither(RunState state1, RunState state2) + { + int runState = this.runState; + return runState == state1.ordinal() || runState == state2.ordinal(); + } + + final boolean isState(int stateBitSet) + { + return TinyEnumSet.contains(stateBitSet, stateOrdinal()); + } + + final boolean is(GlobalGroup group) + { + return globalGroupOrdinal() == group.ordinal(); + } + + final boolean is(ExclusiveGroup group) + { + return exclusiveGroupOrdinal() == group.ordinal(); + } + + final void override(GlobalGroup group) + { + info = (info & ~(GROUP_MASK << GLOBAL_GROUP_SHIFT)) | (group.ordinal() << GLOBAL_GROUP_SHIFT); + } + + final void setStateExclusive(State state) + { + Invariants.require(state.isPermittedFrom(stateOrdinal()), "%s forbidden from %s", state, this, Task::reportBadStateTransition); + unsafeSetStateExclusive(state); + } + + final void setRunState(RunState state) + { + Invariants.require(isState(PREPARED_OR_EXECUTED) || (state == RUN_FAILED && is(FAILED))); + setRunState(state.ordinal()); + } + + final void setRunState(int newRunState) + { + runStateUpdater.lazySet(this, newRunState); + } + + private static String reportBadStateTransition(Task task) + { + return task.state() + " for " + task.description(); + } + + final void unsafeSetStateExclusive(State state) + { + info = (info & ~STATE_MASK) | state.ordinal(); + } + + final int globalGroupOrdinal() + { + return (info >>> GLOBAL_GROUP_SHIFT) & GROUP_MASK; + } + + final int exclusiveGroupOrdinal() + { + return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; + } + + private boolean isCompatible(ExecutorQueue queue) + { + int self = stateOrdinal(); + return TinyEnumSet.contains(queue.permittedStates, self); + } + + final boolean isSync() + { + return 0 == (info & NONSYNC_BIT); + } + + final boolean isNonSync() + { + return !isSync(); + } + + final void setNonSyncExclusive() + { + info |= NONSYNC_BIT; + } + + final boolean isIncremental() + { + return 0 != (info & INCREMENTAL_MASK); + } + + final void setIncrementalExclusive() + { + info |= INCREMENTAL | NONSYNC_BIT; + } + + final boolean hasIncrementalStarted() + { + return (info & INCREMENTAL_MASK) >= INCREMENTAL_STARTED; + } + + final void setIncrementalStartedExclusive() + { + Invariants.require(isIncremental()); + if (!isIncrementalFinishing()) + info = (info & ~INCREMENTAL_MASK) | INCREMENTAL_STARTED; + } + + final boolean isIncrementalFinishing() + { + return (info & INCREMENTAL_MASK) >= INCREMENTAL_FINISHING; + } + + final void setIncrementalFinishingExclusive() + { + Invariants.require(isIncremental()); + info |= INCREMENTAL_FINISHING; + } + + final void setSequencedExclusive(ExecutionContext.ExecutionSequence sequence) + { + Invariants.require(isUnsequenced()); + info |= sequence.ordinal() << SEQUENCED_SHIFT; + } + + final boolean isUnsequenced() + { + return (info & SEQUENCED_MASK) == 0; + } + + final boolean isSequencedByPriority() + { + return (info & SEQUENCED_MASK) == SEQUENCED_PRIORITY; + } + + final boolean isSequencedByPriorityAtomic() + { + return (info & SEQUENCED_MASK) >= SEQUENCED_ATOMIC; + } + + final boolean isCacheQueuedFifo() + { + return (info & SEQUENCED_MASK) == SEQUENCED_ATOMIC_AND_QUEUED; + } + + final boolean isCacheQueued() + { + return 0 != (info & CACHE_QUEUED_BIT); + } + + final boolean isQueued() + { + return 0 != (info & EXECUTOR_QUEUE_SHIFTED_MASK); + } + + final int queuedOrdinal() + { + return (info >>> EXECUTOR_QUEUE_SHIFT) & EXECUTOR_QUEUE_UNSHIFTED_MASK; + } + + final ExecutorQueue queued() + { + return ExecutorQueue.forOrdinal(queuedOrdinal()); + } + + final void setQueue(ExecutorQueue queue) + { + Invariants.require(isCompatible(queue)); + Invariants.require(!isQueued()); + info |= queue.ordinal() << EXECUTOR_QUEUE_SHIFT; + } + + // supersedes priority, in whichever order they're called + final void setCacheQueuedFifoExclusive() + { + Invariants.require(isSequencedByPriorityAtomic()); + info |= SEQUENCED_ATOMIC_AND_QUEUED | CACHE_QUEUED_BIT; + } + + final void setCacheQueuedExclusive() + { + info |= CACHE_QUEUED_BIT; + } + + final int tranche() + { + Invariants.require((info & HAS_TRANCHE_BIT) != 0); + return info >>> TRANCHE_SHIFT; + } + + final void setTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; + } + + final Task inherit(Task parent) + { + Invariants.require(!hasInherited()); + position = parent.position; + setInheritedWithTranche(parent.tranche()); + return this; + } + + final void setInheritedWithTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; + } + + final boolean hasInherited() + { + return (info & HAS_INHERITED_BIT) != 0; + } + + final void setInheritedRangeScan() + { + info = info | HAS_INHERITED_RANGE_SCAN_BIT; + } + + final boolean hasInheritedRangeScan() + { + return (info & HAS_INHERITED_RANGE_SCAN_BIT) != 0; + } + + void addConsequence(Task task) + { + Task prev = consequences; + Invariants.require(prev == null || prev.is(UNREGISTERED)); + task.next = prev; + consequences = task; + } + + final void submitConsequencesExclusive(boolean success) + { + if (consequences == null) + return; + + Task cur = Task.reverse(consequences); + consequences = null; + + while (cur != null) + { + Task next = cur.next; + cur.next = null; + if (cur.is(UNREGISTERED)) + { + if (success || !(cur instanceof SafeTask || this instanceof SafeTask)) + { + cur.inherit(this); + cur.submitExclusiveNoExcept(); + } + else + { + cur.failExclusive(new CancellationException("Parent task failed"), CANCELLED); + } + } + cur = next; + } + } + + static int init(GlobalGroup global, ExclusiveGroup exclusive) + { + return (global.ordinal() << GLOBAL_GROUP_SHIFT) | (exclusive.ordinal() << EXCLUSIVE_GROUP_SHIFT); + } + + static Task reverse(Task unqueued) + { + Task prev = null; + Task cur = unqueued; + while (cur != null) + { + Task next = cur.next; + cur.next = prev; + prev = cur; + cur = next; + } + return prev; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java new file mode 100644 index 0000000000..7b9f2c352c --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.local.ExecutionContext; + +public class TaskInfo implements Comparable +{ + // sorted in name order for reporting to virtual tables + public enum Status + { + RUNNING, LOADING, WAITING_TO_RUN + } + + final Status status; + final int commandStoreId; + + final Task task; + + public TaskInfo(Status status, int commandStoreId, Task task) + { + this.status = status; + this.commandStoreId = commandStoreId; + this.task = task; + } + + public Status status() + { + return status; + } + + public Integer commandStoreId() + { + return commandStoreId >= 0 ? commandStoreId : null; + } + + public long position() + { + return task.position; + } + + public @Nullable String describe() + { + if (task instanceof SafeTask) + return ((SafeTask) task).executionContext().reason(); + + if (task != null) + return task.description(); + + return null; + } + + public @Nullable ExecutionContext preLoadContext() + { + if (task instanceof SafeTask) + return ((SafeTask) task).executionContext(); + if (task instanceof IOTaskWrapper && ((IOTaskWrapper) task).wrapped instanceof SafeTask.RangeTxnScanner) + return ((SafeTask.RangeTxnScanner) ((IOTaskWrapper) task).wrapped).preLoadContext(); + return null; + } + + @Override + public int compareTo(TaskInfo that) + { + int c = this.status.compareTo(that.status); + if (c == 0) c = Long.compare(this.position(), that.position()); + return c; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java new file mode 100644 index 0000000000..1374742860 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.IntrusivePriorityHeap; + +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; + +// has xSingle methods to distinguish from within MultiTaskQueue whether we're invoking the multi or single variation +class TaskQueue extends IntrusivePriorityHeap +{ + final ExecutorQueue kind; + + public TaskQueue(ExecutorQueue kind) + { + this.kind = kind; + } + + void unqueue(T task) + { + throw new UnsupportedOperationException(); + } + + @Override + public final int compare(T o1, T o2) + { + return Long.compare(o1.position, o2.position); + } + + @Override + protected final void ensureHeapified() + { + super.ensureHeapified(); + } + + protected final boolean requeueSingle(T requeue) + { + int oldIndex = updateNode(requeue); + if (oldIndex < 0) + return false; + + int newIndex = heapIndex(requeue); + return Math.min(oldIndex, newIndex) == 0 && heapifiedSize() > 0; + } + + final T peekSingle() + { + ensureHeapified(); + return peekNode(); + } + + final T pollSingle() + { + ensureHeapified(); + return pollNode(); + } + + final boolean isEmptySingle() + { + return isEmptyInternal(); + } + + final int enqueueSingle(T enqueue) + { + return insertNode(enqueue); + } + + final boolean unqueueSingle(T unqueue) + { + int heapIndex = heapIndex(unqueue); + removeNode(unqueue); + return heapIndex == 0; + } + + final boolean tryUnqueueSingle(T remove) + { + return removeNodeIfContains(remove); + } + + final T getSingle(int index) + { + return super.getNode(index); + } + + final boolean isQueuedSingle(T test) + { + return containsNode(test); + } + + @Override + public String toString() + { + return kind.toString(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java new file mode 100644 index 0000000000..f3ec1f5e85 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java @@ -0,0 +1,627 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.Invariants; +import accord.utils.UnhandledEnum; + +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; +import org.apache.cassandra.service.accord.execution.Task.GroupKind; + +import static org.apache.cassandra.service.accord.execution.Task.ExecutorQueue.RUNNABLE; + +/** + * A {@link TaskQueue} sub-divided into up to eight per-group sub-queues (groups are phases for the exclusive + * executor, or work classes globally) plus a policy for choosing which sub-queue to serve next. The policy balances + * three competing concerns: ordering/priority (run the oldest / highest-priority work), fairness (do not let one + * group monopolise the executor), and throughput (do not let an even split leave a busy group's backlog growing + * without bound). + * + *

Packed counters

+ * Per-group state is held in {@code long}s, one 7-bit lane per group; bit 7 of each byte is a spare "guard" bit used + * to run branch-free min/compare across all eight lanes at once (see {@code minCounters}). The lanes are: + *
    + *
  • {@code positions[]} - the head task's position (HLC, or submission order) per group; drives FIFO/priority.
    • + *
  • {@code recent} - a windowed count of recently served tasks per group (service received).
    • + *
  • {@code arrivals} - a windowed, size-biased count of recent arrivals per group (demand offered).
    • + *
  • {@code current} - tasks currently in flight per group, used to enforce {@code queue_active_limits}.
    • + *
  • {@code hasWork}/{@code dirty} - guard-bit masks: which groups have queued work / need a position refresh.
    • + *
+ * + *

Windowing: a bounded memory of the past

+ * {@code recent} and {@code arrivals} are not running totals. Whenever incrementing {@code recent} tips any lane to + * its maximum (0x80), both {@code recent} and {@code arrivals} are halved (see {@code incrementRecents}), + * and each lane also saturates at 0x7f. They are therefore an exponentially-decaying window keyed on service/time. + * This is what stops a one-off burst of arrivals granting a group a permanent boost: the burst saturates the lane + * briefly and then decays away as other work is served. + * + *

The fair selection counter

+ * The fair models pick the group with the smallest {@code effective} counter, where per lane + * 
effective = min(0x7f, max(0, recent - arrivals) + bias)
+ *
    + *
  • {@code max(0, recent - arrivals)} - a group whose arrivals have outpaced its recent service clamps to 0 and + * is therefore preferred; in steady state each group's service tracks its arrival rate, which bounds backlog. + * The clamp is symmetric, so a group cannot bank "credit" during a lull and later use it to monopolise service.
    • + *
+ * + *

Choosing a group ({@code minGroup} / {@code pollByBlend})

+ *
    + *
  • {@code PRIORITY_ONLY} - always the lowest {@code positions} (oldest / highest priority), ignoring fairness.
    • + *
  • {@code PHASE_OVERRIDE} - strict phase order: the lowest-index group with work, always.
    • + *
  • {@code PHASE_FAIR} - pure fairness: the smallest {@code effective}; ties broken by index, within a group by + * position. It has no priority fallback, so it is deliberately completing-first under contention.
    • + *
  • {@code PRIORITY_FAIR} (default) - a deficit round-robin blend of two strategies, chosen per poll: flow + * (smallest {@code effective}, i.e. least fairly serviced) and age (oldest {@code positions}, i.e. FIFO). + * The flow weight ramps up with the flow imbalance ({@code max-min} of {@code effective}) over + * {@code queue_flow_imbalance_onset..+width}, trading against age zero-sum; when balanced it is pure age. The + * ramp is smooth (not a hard mode switch), so there is no boundary to oscillate around and no hysteresis is + * needed. Age also drains stale/standing backlogs for free: a backlog's items are the oldest work, so age + * clears them without a separate backlog-aware strategy (the {@code effective} flow counter tracks arrival + * rate and is deliberately blind to standing stock).
    • + *
+ * + *

Concurrency limits and eligibility

+ * A group whose in-flight {@code current} count has reached its {@code queue_active_limits} limit is + * {@code saturated} and excluded from selection ({@code disabled}), as is any group with no queued work. + * + *

Note on {@code positions} freshness

+ * {@code positions} is refreshed lazily from the sub-queue heads via the {@code dirty} mask; {@code dirty} must use + * the same guard-bit layout as {@code hasWork} so the refresh loop in {@code minGroupByPriority} actually runs - + * otherwise it silently degenerates to lowest-index-with-work and starves high-index / new-work groups. + * + *

NB it extends {@link TaskQueue} to keep the type hierarchy simple for method dispatch, and for efficiency for + * anonymous ExclusiveExecutors which do not use multiple queues, while letting ExclusiveExecutor share a parent + * class for both use cases. + */ +abstract class TaskQueueMulti extends TaskQueue +{ + private static final TaskQueue[] NO_QUEUES = new TaskQueue[0]; + private static final long[] NO_POSITIONS = new long[0]; + static final long COUNTER_OVERFLOWS = 0x8080808080808080L; + static final long COUNTER_MASKS = 0x7f7f7f7f7f7f7f7fL; + static final long COUNTER_LOWBITS = 0x0101010101010101L; + + final TaskQueue[] queues; + final long[] positions; + final byte groupShift; + final long limits; + + /** + * sets overflow bits for a queue that has been stopped + */ + long stopped; + /** + * sets overflow bits for each counter when it needs its position updated + */ + long dirty; + /** + * sets overflow bits for each counter when there's associated work + */ + long hasWork; + /** + * Stores recent dequeue counts for up to 8 sub queues. + */ + long dispatches; + // TODO (required): increment arrivals based on internal queue for ExclusiveExecutors + // also: experiment with decaying on arrival schedule rather than poll schedule, since this should respond to work growth more accurately + /** + * Stores recent enqueue counts for up to 8 sub queues. + */ + long arrivals; + /** + * Stores currently-active counts for up to 8 sub queues. We can use this to impose limits on specific queues. + */ + long active; + /** + * deficit-round-robin credits for the two PRIORITY_FAIR strategies (flow/age). + */ + int creditFlow, creditAge; + + int waitingCount; + + TaskQueueMulti(ExecutorQueue kind, GroupKind groups, long limits) + { + super(kind); + this.limits = limits; + int queueCount = groups.count; + Invariants.require(queueCount <= 8); + queues = queueCount == 0 ? NO_QUEUES : new TaskQueue[queueCount]; + positions = queueCount > 0 && AccordExecutor.BALANCE_BY_POSITION ? new long[queueCount] : NO_POSITIONS; + groupShift = groups.shift; + } + + final int group(Task task) + { + if (groupShift == 0) + return -1; + + return (task.info >>> groupShift) & Task.GROUP_MASK; + } + + void stop(long groupOverflowBits) + { + stopped |= groupOverflowBits; + } + + void restart(long groupOverflowBits) + { + stopped &= ~groupOverflowBits; + } + + final TaskQueue queue(Task task) + { + int group = group(task); + if (group < 0) + return this; + + return queue(group); + } + + final TaskQueue queue(int group) + { + TaskQueue queue = queues[group]; + if (queue == null) + queues[group] = queue = new TaskQueue<>(RUNNABLE); + + return queue; + } + + private int pollGroup() + { + if (hasWork == 0) + return -1; + + switch (AccordExecutor.BALANCING_MODEL) + { + default: + throw new UnhandledEnum(AccordExecutor.PRIORITY_MODEL); + case PRIORITY_ONLY: + return pollGroupByPriority(); + case PHASE_ONLY: + return pollGroupByIndex(); + case PHASE_FAIR: + return pollGroupByPhaseFair(); + case BLENDED_PRIORITY_PHASE_FAIR: + return pollGroupByBlended(); + } + } + + private int pollGroupByPriority() + { + return pollGroupByPriority(unsaturatedWithWork()); + } + + private int pollGroupByPriority(long enabled) + { + long refresh = dirty & hasWork; + while (refresh != 0) + { + int bitIndex = Long.numberOfTrailingZeros(refresh); + int group = bitIndex / 8; + positions[group] = queues[group].peekSingle().position; + refresh ^= 1L << bitIndex; + } + dirty = 0; + + long minPosition = Long.MAX_VALUE; + int minGroup = -1; + long visit = enabled >>> 7; + while (visit != 0) + { + int bitIndex = Long.numberOfTrailingZeros(visit); + int group = bitIndex / 8; + long position = positions[group]; + if (position < minPosition) + { + minGroup = group; + minPosition = position; + } + visit ^= 1L << bitIndex; + } + + return minGroup; + } + + private int pollGroupByIndex() + { + long visit = (hasWork & unsaturated()) >>> 7; + if (visit == 0) + return -1; + + int bitIndex = Long.numberOfTrailingZeros(visit); + return bitIndex / 8; + } + + private int pollGroupByPhaseFair() + { + return minCounterIndex(recentFlowImbalances()); + } + + // PRIORITY_FAIR selection: a deficit round-robin blend of two strategies, chosen per poll: + // flow -> minCounterIndex(recent - arrivals + bias) (least fairly serviced) + // age -> minGroupByPriority() (earliest-queued work) + // wFlow = ramp(flow imbalance F = max-min of the selection counters); wAge = BLEND_TOTAL - wFlow. As flow gets + // uneven, polls trade from age to flow zero-sum; when balanced it is pure age (FIFO). A ramp (not a hard + // threshold) means there is no mode cliff to oscillate around, so no anti-oscillation penalty is needed. Age is + // also what drains a stale/standing backlog -- its items are the oldest -- so no separate stock strategy is needed. + private int pollGroupByBlended() + { + return pollGroupByBlended(saturatedOrWithoutWork()); + } + + private int pollGroupByBlended(long disabled) + { + long withoutWork = hasWork ^ COUNTER_OVERFLOWS; + long counters = recentFlowImbalances(); + long minMax = minMaxCounterValue(counters, withoutWork); + long min = minMax & 0x7f; + long max = minMax >>> 8; + int flowImbalance = (int) (max - min); + + int flowWeight = flowWeight(flowImbalance); + int priorityWeight = AccordExecutor.BLEND_TOTAL - flowWeight; + + creditFlow += flowWeight; + creditAge += priorityWeight; + + if (creditFlow >= creditAge) + { + creditFlow -= AccordExecutor.BLEND_TOTAL; + if (disabled != withoutWork) + min = minCounterValue(counters, disabled); + + return minCounterIndex(counters, min, disabled); + } + else + { + creditAge -= AccordExecutor.BLEND_TOTAL; + return pollGroupByPriority(disabled ^ COUNTER_OVERFLOWS); + } + } + + private long saturated() + { + return ((active | COUNTER_OVERFLOWS) - limits) & COUNTER_OVERFLOWS; + } + + private long unsaturated() + { + return saturated() ^ COUNTER_OVERFLOWS; + } + + private long unsaturatedWithWork() + { + return hasWork & unsaturated() & ~stopped; + } + + private long saturatedOrWithoutWork() + { + return (hasWork ^ COUNTER_OVERFLOWS) | saturated() | stopped; + } + + // arrivals is a windowed measure of ARRIVAL (incremented on enqueue, size-biased; decays on recent's overflow). + // Combined with recent (service) as effective = max(0, recent - arrivals): a queue whose arrivals outpace its + // service clamps to 0 and is preferred, so service converges to arrival rate (bounding the busy queue's backlog). + private long recentFlowImbalances() + { + return clampedSubtract(dispatches, arrivals); + } + + static long minCounterValue(long counters, long disabled) + { + long mins = counters; + mins |= overflowsToLowMasks(disabled); + mins = minCounters(mins, mins >>> 8); // each slot is min of slots [i..i+1] + mins = minCounters(mins, mins >>> 16); // each slot is min of slots [i..i+3] + mins = minCounters(mins, mins >>> 32); // each slot is min of slots [i..i+7] + return mins & 0x7f; + } + + static long minMaxCounterValue(long counters, long disabled) + { + long mins = counters; + long maxs = counters ^ COUNTER_MASKS; + long overflowMasks = overflowsToLowMasks(disabled); + mins |= overflowMasks; + maxs |= overflowMasks; + mins = minCounters(mins, mins >>> 8) & 0x007f007f007f007fL; // each slot is min of slots [i..i+1] + maxs = (minCounters(maxs, maxs << 8) & 0x7f007f007f007f00L); // each slot is min of slots ~[i..i+1] + long minmaxs = mins | maxs; + minmaxs = minCounters(minmaxs, minmaxs >>> 16); // each slot is min of slots [i..i+3] + minmaxs = minCounters(minmaxs, minmaxs >>> 32); // each slot is min of slots [i..i+7] + return (minmaxs ^ 0x7f00) & 0x7f7f; + } + + /** + * If provided two counters (containing 8 7 bit counters each), + * returns the minimum of each matching counter + */ + private static long minCounters(long a, long b) + { + // set overflow bits where a <= b + long selecta = setOverflowWhenLessEqual(a, b); + return selectByOverflowBits(selecta, a, b); + } + + static long setOverflowWhenLessEqual(long a, long b) + { + return ((b | COUNTER_OVERFLOWS) - a) & COUNTER_OVERFLOWS; + } + + // select a if overflow bit is set; b if it is unset + static long selectByOverflowBits(long selecta, long a, long b) + { + selecta = overflowsToLowMasks(selecta); + a &= selecta; + b &= ~selecta; + return a | b; + } + + static long overflowsToLowMasks(long v) + { + return v - (v >>> 7); + } + + private static int flowWeight(int flowImbalance) + { + if (flowImbalance <= AccordExecutor.FLOW_ONSET) return 0; + return Math.min(AccordExecutor.BLEND_TOTAL, ((flowImbalance - AccordExecutor.FLOW_ONSET) << AccordExecutor.BLEND_SHIFT) >>> AccordExecutor.FLOW_WIDTH_SHIFT); + } + + // per-lane max(0, a - b), carry-free: zero both a and b in lanes where a <= b, then subtract + private static long clampedSubtract(long a, long b) + { + long keep = ~overflowsToLowMasks(setOverflowWhenLessEqual(a, b)); + return (a & keep) - (b & keep); + } + + private int minCounterIndex(long counters) + { + return minCounterIndex(counters, saturatedOrWithoutWork()); + } + + private int minCounterIndex(long counters, long disabled) + { + return minCounterIndex(counters, minCounterValue(counters, disabled), disabled); + } + + private int minCounterIndex(long counters, long minCounterValue, long disabled) + { + long mins = minCounterValue * COUNTER_LOWBITS; + long select = ((mins | COUNTER_OVERFLOWS) - counters) & COUNTER_OVERFLOWS; + // now unset those overflow bits associated with disabled queues + select &= ~disabled; + if (select == 0) + return -1; + return (Long.numberOfTrailingZeros(select) - 7) / 8; + } + + final T pollMulti() + { + int group = pollGroup(); + if (group < 0) + { + // group < 0 can mean EITHER we don't have any nested queues OR those queues are either empty or DISABLED + T result = pollSingle(); + if (result != null) + --waitingCount; + return result; + } + + --waitingCount; + incrementActive(group); + incrementDispatches(group); + + TaskQueue queue = queues[group]; + T head = queue.pollSingle(); + // NOTE: must clear dirty when emptied, symmetrically with unqueue(): the fair selection paths + // never consume the dirty bit, so a group drained during a fairness episode would otherwise + // retain a stale dirty bit and NPE in minGroupByPriority.peekSingle() when balance is restored. + if (queue.isEmptySingle()) + { + unsetHasWork(group); + unsetDirty(group); + } + else setDirty(group); + return head; + } + + final void enqueueMulti(T task, boolean incrementArrivals) + { + task.setQueue(kind); + int group = group(task); + if (group < 0) + { + enqueueSingle(task); + } + else + { + TaskQueue queue = queue(group); + int result = queue.enqueueSingle(task); + if (incrementArrivals) + incrementArrivals(group); + if (result < 0) setHasWork(group); + if (result != 0) setDirty(group); + } + ++waitingCount; + } + + final void requeue(T task) + { + int group = group(task); + if (group < 0) requeueSingle(task); + else + { + TaskQueue queue = queue(group); + Invariants.require(queue != null && queue.isQueuedSingle(task)); + if (queue.requeueSingle(task)) + setDirty(group); + } + } + + final void unqueueMulti(T task) + { + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + Invariants.require(queue.isQueuedSingle(task)); + unqueue(task, group, queue); + } + + // if there is an active collection, we return false and do not remove ourselves from it + boolean tryUnqueueWaiting(T task) + { + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + if (!queue.isQueuedSingle(task)) + return false; + + unqueue(task, group, queue); + return true; + } + + private void unqueue(T task, int group, TaskQueue queue) + { + task.unsetQueue(kind); + boolean dirty = queue.unqueueSingle(task); + --waitingCount; + if (group >= 0) + { + if (queue.isEmptySingle()) + { + unsetHasWork(group); + unsetDirty(group); + } + else if (dirty) setDirty(group); + } + } + + final void incrementActive(int group) + { + active += lowBit(group); + } + + final void decrementActive(int group) + { + active -= lowBit(group); + } + + final void incrementDispatches(Task task) + { + int group = group(task); + if (group >= 0) + incrementDispatches(group); + } + + final void incrementDispatches(int group) + { + dispatches += lowBit(group); + if ((dispatches & COUNTER_OVERFLOWS) != 0) + { + dispatches = (dispatches >>> 1) & COUNTER_MASKS; + arrivals = (arrivals >>> 1) & COUNTER_MASKS; // arrivals (arrival) decays on the service/time clock + } + } + + final void decrementDispatches(Task task) + { + int group = group(task); + if (group >= 0) + decrementDispatches(group); + } + + final void decrementDispatches(int group) + { + long lowBit = lowBit(group); + dispatches -= lowBit; + dispatches += (dispatches >>> 7) & lowBit; + } + + final void incrementArrivals(Task task) + { + int group = group(task); + if (group >= 0) + incrementArrivals(group); + } + + final void incrementArrivals(int group) + { + int shift = group * 8; + long overflowBit = 0x80L << shift; + arrivals += 1L << shift; + // if we overflow, unset the overflow bit and set all other bits for the counter + long overflow = arrivals & overflowBit; + arrivals ^= overflow; + arrivals |= overflow - (overflow >>> 7); + } + + final boolean hasWaitingToRunExcluding(long groupOverflowBits) + { + return (unsaturatedWithWork() & ~groupOverflowBits) != 0; + } + + final void setHasWork(int group) + { + hasWork |= overflowBit(group); + } + + final void unsetHasWork(int group) + { + hasWork &= ~overflowBit(group); + } + + final void setDirty(int group) + { + dirty |= overflowBit(group); + } + + final void unsetDirty(int group) + { + dirty &= ~overflowBit(group); + } + + final boolean hasWaitingToRun() + { + return unsaturatedWithWork() != 0; + } + + final boolean isWaiting(T task) + { + return queue(task).isQueuedSingle(task); + } + + final int waitingCount() + { + return waitingCount; + } + + static long lowBit(int group) + { + return 1L << (group * 8); + } + + static long overflowBit(int group) + { + return 0x80L << (group * 8); + } + + static long overflowBit(Enum group) + { + return overflowBit(group.ordinal()); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java new file mode 100644 index 0000000000..ddd801b8e4 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import static org.apache.cassandra.service.accord.execution.Task.ExecutorQueue.RUNNABLE; + +final class TaskQueueRunnable extends TaskQueueMulti +{ + final TaskQueue assigned; + + TaskQueueRunnable() + { + super(RUNNABLE, Task.GroupKind.GLOBAL, AccordExecutor.GLOBAL_QUEUE_LIMITS); + this.assigned = new TaskQueue<>(RUNNABLE); + } + + T poll() + { + T next = pollMulti(); + if (next == null) + return null; + + assigned.enqueueSingle(next); + return next; + } + + void enqueue(T enqueue, boolean incrementArrivals) + { + enqueueMulti(enqueue, incrementArrivals); + } + + void unqueue(T unqueue) + { + if (assigned.isQueuedSingle(unqueue)) + { + int group = group(unqueue); + if (group >= 0) + decrementActive(group); + + unqueue.unsetQueue(kind); + assigned.unqueueSingle(unqueue); + } + else + { + super.unqueueMulti(unqueue); + } + } + + int waitingOrAssignedCount() + { + return waitingCount + assigned.size(); + } + + boolean hasAssignedOrWaiting() + { + return waitingCount > 0 || !assigned.isEmptySingle(); + } + + boolean hasAssigned() + { + return !assigned.isEmptySingle(); + } + + boolean isAssigned(T task) + { + return assigned.isQueuedSingle(task); + } + + void cleanup(T task) + { + if (assigned.tryUnqueueSingle(task)) + { + int group = group(task); + if (group >= 0) + decrementActive(group); + task.unsetQueue(kind); + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java new file mode 100644 index 0000000000..4f7757dc25 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; + +final class TaskQueueStandalone extends TaskQueue +{ + TaskQueueStandalone(ExecutorQueue kind) + { + super(kind); + } + + void enqueue(T enqueue) + { + enqueue.setQueue(kind); + enqueueSingle(enqueue); + } + + boolean tryUnqueue(T unqueue) + { + if (!containsNode(unqueue)) + return false; + + unqueue(unqueue); + return true; + } + + void unqueue(T unqueue) + { + unqueue.unsetQueue(kind); + removeNode(unqueue); + } + + T peek() + { + return peekSingle(); + } + + T poll() + { + return pollSingle(); + } + + boolean isEmpty() + { + return isEmptySingle(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java new file mode 100644 index 0000000000..0d9705a186 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.apache.cassandra.concurrent.CassandraThread; +import org.apache.cassandra.concurrent.DebuggableTask; + +public interface TaskRunner +{ + AccordExecutor accordActiveExecutor(); + void setAccordActiveExecutor(AccordExecutor newExecutor); + + AccordExecutor accordLockedExecutor(); + boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor); + void exitAccordLockedExecutor(); + + // to be called only by the thread itself, so can (eventually) avoid any memory barriers + Task accordActiveSelfTask(); + Task accordActiveTask(); + void setAccordActiveTask(Task newActiveTask); + + static TaskRunner get() + { + return get(Thread.currentThread()); + } + + static TaskRunner get(Thread thread) + { + return thread instanceof CassandraThread ? (CassandraThread) thread : ThreadLocalTaskRunner.threadLocal.get(); + } + + final class ThreadLocalTaskRunner implements TaskRunner + { + private AccordExecutor lockedExecutor; + private int lockedExecutorDepth; + private AccordExecutor activeExecutor; + volatile Task activeTask; + + private static final ThreadLocal threadLocal = ThreadLocal.withInitial(ThreadLocalTaskRunner::new); + + @Override + public AccordExecutor accordActiveExecutor() + { + return activeExecutor; + } + + @Override + public void setAccordActiveExecutor(AccordExecutor newExecutor) + { + activeExecutor = newExecutor; + } + + @Override + public AccordExecutor accordLockedExecutor() + { + return lockedExecutor; + } + + @Override + public boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) + { + if (lockedExecutor == null) lockedExecutor = newLockedExecutor; + else if (lockedExecutor != newLockedExecutor) return false; + ++lockedExecutorDepth; + return true; + } + + @Override + public void exitAccordLockedExecutor() + { + if (--lockedExecutorDepth == 0) + lockedExecutor = null; + } + + @Override + public void setAccordActiveTask(Task newActiveTask) + { + activeTask = newActiveTask; + } + + @Override + public Task accordActiveTask() + { + return activeTask; + } + + @Override + public Task accordActiveSelfTask() + { + return activeTask; + } + } + + abstract class DebuggableWrapper implements DebuggableTask.DebuggableTaskRunner + { + private volatile TaskRunner wrapped; + + protected void setWrapped(TaskRunner wrapped) + { + this.wrapped = wrapped; + } + + @Override + public DebuggableTask running() + { + Task running = wrapped.accordActiveTask(); + return running == null ? null : running.debuggable(); + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Tranches.java b/src/java/org/apache/cassandra/service/accord/execution/Tranches.java new file mode 100644 index 0000000000..5b5b3f2668 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Tranches.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayDeque; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.utils.Invariants; + +import static org.apache.cassandra.service.accord.execution.Task.MAX_TRANCHE; + +/** + * Tasks are separated into tranches based on their position. + *

+ * When we want to wait for all submitted tasks and their consequences to complete, we create a new tranche + * and track the number of tasks still extant for all prior tranches - once these reach zero we can signal + * the completion of the work. + */ +final class Tranches +{ + private static final Logger logger = LoggerFactory.getLogger(Tranches.class); + + class WithDeferred implements Runnable + { + final Runnable run; + final ArrayDeque deferred; + + WithDeferred(Runnable run, Runnable deferred) + { + this.run = run; + this.deferred = new ArrayDeque<>(1); + this.deferred.add(deferred); + } + + WithDeferred(Runnable run, ArrayDeque deferred) + { + this.run = run; + this.deferred = deferred; + } + + @Override + public void run() + { + Runnable register = deferred.poll(); + if (!deferred.isEmpty()) + { + Invariants.require(runs[firstIndex] != null); + Invariants.require(!(runs[firstIndex] instanceof WithDeferred)); + runs[firstIndex] = new WithDeferred(runs[firstIndex], deferred); + } + registerWait(register); + try + { + run.run(); + } + catch (Throwable t) + { + owner.agent.onException(t); + } + } + } + + final AccordExecutor owner; + + int firstTranche; + int firstIndex; + long[] mins = new long[8]; + int[] counts = new int[8]; + Runnable[] runs = new Runnable[8]; + + // the next tranche, that all new work is being collected against + // (and will move to the array accounting once a new wait is registered) + long nextMin; + int nextTranche; + int nextCount; + + Tranches(AccordExecutor owner) + { + this.owner = owner; + } + + private int trancheToIndex(int tranche) + { + return firstIndex + trancheToIndexOffset(tranche); + } + + private int trancheToIndexOffset(int tranche) + { + int offset = tranche - firstTranche; + if (offset < 0) + offset += MAX_TRANCHE + 1; + return offset; + } + + int size() + { + return trancheToIndexOffset(nextTranche); + } + + int capacity() + { + return counts.length; + } + + int addNew(long position) + { + Invariants.require(position >= nextMin); + ++nextCount; + return nextTranche; + } + + void addInherited(int tranche, long position) + { + if (tranche == nextTranche) + { + Invariants.require(position >= nextMin); + ++nextCount; + } + else + { + int index = trancheToIndex(tranche); + Invariants.require(counts[index] > 0); + Invariants.require(mins[index] <= position); + ++counts[index]; + } + } + + void complete(int tranche) + { + if (tranche == nextTranche) + { + --nextCount; + } + else + { + + if (counts[trancheToIndex(tranche)] == 1) + owner.drainUnqueuedNewWorkExclusive(); // make sure we don't have any pending + + if (--counts[trancheToIndex(tranche)] == 0 && tranche == firstTranche) + { + do + { + advance(); + } + while (firstTranche != nextTranche && counts[firstIndex] == 0); + } + + if (firstIndex >= counts.length / 2) + compact(); + } + } + + public void finishAll(long nextPosition) + { + while (firstTranche != nextTranche) + { + logger.warn("{} processed all pending tasks (<{}) but found {} waiting for {}", this, + nextPosition, counts[firstIndex], size() == 1 ? nextMin : mins[firstIndex + 1]); + advance(); + } + } + + private void advance() + { + Runnable run = runs[firstIndex]; + runs[firstIndex] = null; + ++firstIndex; + if (firstTranche == MAX_TRANCHE) firstTranche = 0; + else ++firstTranche; + try + { + run.run(); + } + catch (Throwable t) + { + owner.agent.onException(t); + } + } + + public void registerWait(Runnable run) + { + int newNextTranche = (nextTranche + 1) % (MAX_TRANCHE + 1); + if (newNextTranche == firstTranche) + { + Runnable cur = runs[firstIndex]; + if (cur instanceof WithDeferred) ((WithDeferred) cur).deferred.add(run); + else runs[firstIndex] = new WithDeferred(runs[firstIndex], run); + return; + } + + if ((firstIndex + size()) == capacity()) + growOrCompact(); + + int index = firstIndex + size(); + mins[index] = nextMin; + counts[index] = nextCount; + runs[index] = run; + nextMin = owner.minPosition = owner.nextPosition; + nextCount = 0; + nextTranche = newNextTranche; + } + + private void compact() + { + if (size() <= capacity() / 4 && capacity() > 8) resize(capacity() / 2); + else compact(mins, counts, runs); + } + + private void growOrCompact() + { + if (size() > capacity() / 2) resize(capacity() * 2); + else compact(); + } + + private void resize(int newSize) + { + Invariants.require(newSize > 0); + long[] newMins = new long[newSize]; + int[] newCounts = new int[newSize]; + Runnable[] newRuns = new Runnable[newSize]; + compact(newMins, newCounts, newRuns); + mins = newMins; + counts = newCounts; + runs = newRuns; + } + + private void compact(long[] newMins, int[] newCounts, Runnable[] newRuns) + { + int size = size(); + System.arraycopy(mins, firstIndex, newMins, 0, size); + System.arraycopy(counts, firstIndex, newCounts, 0, size); + System.arraycopy(runs, firstIndex, newRuns, 0, size); + firstIndex = 0; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java b/src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java new file mode 100644 index 0000000000..d4851481a6 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.local.ExecutionContext; + +// run the task even on a stopped commandStore +public interface Unstoppable extends ExecutionContext.Empty +{ +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java b/src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java new file mode 100644 index 0000000000..20badc4eda --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +// run the task even on a terminated commandStore +public interface Unterminatable extends Unstoppable +{ +} diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java index 61de28c26c..adaf531a5b 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java @@ -23,6 +23,7 @@ import java.util.function.BiConsumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Result; import accord.api.Update; import accord.coordinate.CoordinationAdapter; @@ -30,7 +31,6 @@ import accord.coordinate.CoordinationAdapter.Adapters.TxnAdapter; import accord.coordinate.ExecuteFlag.CoordinationFlags; import accord.coordinate.ExecutePath; import accord.local.Node; -import accord.local.SequentialAsyncExecutor; import accord.messages.Apply; import accord.primitives.Ballot; import accord.primitives.Deps; @@ -47,6 +47,7 @@ import org.apache.cassandra.service.accord.topology.AccordEndpointMapper; import org.apache.cassandra.service.accord.txn.AccordUpdate; import org.apache.cassandra.service.accord.txn.TxnRead; +import static accord.api.ProtocolModifiers.sendMinimal; import static accord.messages.Apply.Kind.Maximal; import static accord.messages.Apply.Kind.Minimal; @@ -55,12 +56,13 @@ public class AccordInteropAdapter extends TxnAdapter private static final Logger logger = LoggerFactory.getLogger(AccordInteropAdapter.class); public static final class AccordInteropFactory extends DefaultFactory { - final AccordInteropAdapter standard, recovery; + final AccordInteropAdapter minimal, maximal, recovery; public AccordInteropFactory(AccordEndpointMapper endpointMapper) { - standard = new AccordInteropAdapter(endpointMapper, Minimal); - recovery = new AccordInteropAdapter(endpointMapper, Maximal); + minimal = new AccordInteropAdapter(endpointMapper, Minimal, true); + maximal = new AccordInteropAdapter(endpointMapper, Maximal, true); + recovery = new AccordInteropAdapter(endpointMapper, Maximal, false); } @Override @@ -68,37 +70,37 @@ public class AccordInteropAdapter extends TxnAdapter { if (txnId.isSyncPoint()) return super.get(txnId, step); - return (CoordinationAdapter) (step == Kind.Recovery ? recovery : standard); + return (CoordinationAdapter) (step == Kind.Recovery ? recovery : sendMinimal() ? minimal : maximal); } }; private final AccordEndpointMapper endpointMapper; - private final Apply.Kind applyKind; + private final boolean isUserFacing; - private AccordInteropAdapter(AccordEndpointMapper endpointMapper, Apply.Kind applyKind) + private AccordInteropAdapter(AccordEndpointMapper endpointMapper, Apply.Kind applyKind, boolean isUserFacing) { - super(Minimal); + super(applyKind); this.endpointMapper = endpointMapper; - this.applyKind = applyKind; + this.isUserFacing = isUserFacing; } @Override - public void execute(Node node, SequentialAsyncExecutor executor, Topologies any, FullRoute route, Ballot ballot, ExecutePath path, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer callback) + public void execute(Node node, ExclusiveAsyncExecutor executor, Topologies any, FullRoute route, Ballot ballot, ExecutePath path, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer callback) { if (!doInteropExecute(node, executor, route, ballot, txnId, txn, executeAt, stableDeps, callback)) super.execute(node, executor, any, route, ballot, path, flags, txnId, txn, executeAt, stableDeps, sendDeps, callback); } @Override - public void persist(Node node, SequentialAsyncExecutor executor, Topologies any, Route require, Route sendTo, FullRoute route, Ballot ballot, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, boolean informDurableOnDone, BiConsumer callback) + public void persist(Node node, ExclusiveAsyncExecutor executor, Topologies any, Route require, Route sendTo, FullRoute route, Ballot ballot, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, boolean informDurableOnDone, BiConsumer callback) { - if (applyKind == Minimal && doInteropPersist(node, executor, any, require, sendTo, ballot, txnId, txn, executeAt, deps, writes, result, route, informDurableOnDone, callback)) + if (isUserFacing && doInteropPersist(node, executor, any, require, sendTo, ballot, txnId, txn, executeAt, deps, writes, result, route, informDurableOnDone, callback)) return; super.persist(node, executor, any, require, sendTo, route, ballot, flags, txnId, txn, executeAt, deps, writes, result, informDurableOnDone, callback); } - private boolean doInteropExecute(Node node, SequentialAsyncExecutor executor, FullRoute route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer callback) + private boolean doInteropExecute(Node node, ExclusiveAsyncExecutor executor, FullRoute route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer callback) { // Unrecoverable repair always needs to be run by AccordInteropExecution AccordUpdate.Kind updateKind = AccordUpdate.kind(txn.update()); @@ -121,7 +123,7 @@ public class AccordInteropAdapter extends TxnAdapter return true; } - private boolean doInteropPersist(Node node, SequentialAsyncExecutor executor, Topologies any, Route require, Route sendTo, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, boolean informDurableOnDone, BiConsumer callback) + private boolean doInteropPersist(Node node, ExclusiveAsyncExecutor executor, Topologies any, Route require, Route sendTo, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, boolean informDurableOnDone, BiConsumer callback) { Update update = txn.update(); ConsistencyLevel consistencyLevel = update instanceof AccordUpdate ? ((AccordUpdate) update).cassandraCommitCL() : null; diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java index 49b467cc7f..874c2e305a 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java @@ -28,12 +28,12 @@ import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.function.BiConsumer; import accord.api.Data; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Result; import accord.coordinate.CoordinationAdapter; import accord.coordinate.ExecuteFlag.CoordinationFlags; import accord.local.Node; import accord.local.Node.Id; -import accord.local.SequentialAsyncExecutor; import accord.messages.Commit; import accord.messages.Commit.Kind; import accord.primitives.AbstractRanges; @@ -94,6 +94,7 @@ import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.transport.Dispatcher; +import static accord.api.ProtocolModifiers.sendMinimal; import static accord.coordinate.CoordinationAdapter.Factory.Kind.Standard; import static accord.primitives.Txn.Kind.Write; import static accord.topology.SelectShards.LIVE; @@ -125,7 +126,7 @@ public class AccordInteropExecution implements ReadCoordinator private final Timestamp executeAt; private final Deps deps; private final BiConsumer callback; - private final SequentialAsyncExecutor executor; + private final ExclusiveAsyncExecutor executor; private final ConsistencyLevel consistencyLevel; private final AccordEndpointMapper endpointMapper; @@ -141,9 +142,10 @@ public class AccordInteropExecution implements ReadCoordinator private volatile long uniqueHlc; public AccordInteropExecution(Node node, TxnId txnId, Txn txn, AccordUpdate.Kind updateKind, FullRoute route, Ballot ballot, Timestamp executeAt, Deps deps, BiConsumer callback, - SequentialAsyncExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper) throws TopologyException + ExclusiveAsyncExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper) throws TopologyException { requireArgument(!txn.read().keys().isEmpty() || updateKind == AccordUpdate.Kind.UNRECOVERABLE_REPAIR); + // TODO (required): this does not support privileged coordinator optimisation, as must apply local stable before any other message is sent this.node = node; this.txnId = txnId; this.txn = txn; @@ -217,8 +219,7 @@ public class AccordInteropExecution implements ReadCoordinator // TODO (desired): It would be better to use the re-use the command from the transaction but it's fragile // to try and figure out exactly what changed for things like read repair and short read protection // Also this read scope doesn't reflect the contents of this particular read and is larger than it needs to be - // TODO (required): understand interop and whether StableFastPath is appropriate - AccordInteropStableThenRead commit = new AccordInteropStableThenRead(id, allTopologies, txnId, Kind.StableFastPath, executeAt, txn, deps, route, message.payload); + AccordInteropStableThenRead commit = new AccordInteropStableThenRead(id, allTopologies, txnId, commitKind(), executeAt, txn, deps, route, message.payload); node.send(id, commit, executor, new AccordInteropRead.ReadCallback(id, to, message, callback, this), null); } @@ -376,7 +377,7 @@ public class AccordInteropExecution implements ReadCoordinator { InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(to); if (endpoint != null && !contacted.contains(endpoint)) - node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); + node.send(to, new Commit(commitKind(), to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); } } @@ -387,7 +388,7 @@ public class AccordInteropExecution implements ReadCoordinator for (Node.Id to : allTopologies.nodes()) { if (!executeTopology.contains(to)) - node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); + node.send(to, new Commit(commitKind(), to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); } } AsyncChain result; @@ -415,6 +416,11 @@ public class AccordInteropExecution implements ReadCoordinator }); } + private static Commit.Kind commitKind() + { + return sendMinimal() ? Kind.StableFastPath : Kind.StableWithTxnAndDeps; + } + private AsyncChain executeUnrecoverableRepairUpdate() { return AsyncChains.chain(Stage.ACCORD_MIGRATION.executor(), () -> { @@ -423,7 +429,7 @@ public class AccordInteropExecution implements ReadCoordinator // and can be extended similar to MessageType which allows additional types not from Accord to be added // This commit won't necessarily execute before the interop read repair message so there could be an insufficient which is fine for (Node.Id to : executeTopology.nodes()) - node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); + node.send(to, new Commit(commitKind(), to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); repairUpdate.runBRR(AccordInteropExecution.this); return new TxnData(); }); diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java index 7cdb7032a8..ac17755bb8 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java @@ -20,13 +20,13 @@ package org.apache.cassandra.service.accord.interop; import java.util.function.BiConsumer; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Result; import accord.coordinate.ExecuteFlag; import accord.coordinate.Persist; import accord.coordinate.tracking.AllTracker; import accord.coordinate.tracking.QuorumTracker; import accord.local.Node; -import accord.local.SequentialAsyncExecutor; import accord.messages.Apply; import accord.primitives.Ballot; import accord.primitives.Deps; @@ -53,7 +53,7 @@ import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; */ public class AccordInteropPersist extends Persist { - public AccordInteropPersist(Node node, SequentialAsyncExecutor executor, Topologies topologies, TxnId txnId, Route sendTo, Ballot ballot, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, ConsistencyLevel consistencyLevel, ExecuteFlag.CoordinationFlags flags, boolean informDurableOnDone, Apply.Kind applyKind, BiConsumer callback) + public AccordInteropPersist(Node node, ExclusiveAsyncExecutor executor, Topologies topologies, TxnId txnId, Route sendTo, Ballot ballot, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, ConsistencyLevel consistencyLevel, ExecuteFlag.CoordinationFlags flags, boolean informDurableOnDone, Apply.Kind applyKind, BiConsumer callback) { super(node, executor, topologies, txnId, ballot, sendTo, txn, executeAt, deps, writes, result.toPersistable(), fullRoute, flags, informDurableOnDone, AccordInteropApply.FACTORY, applyKind, consistencyLevel == ALL ? AllTracker::new : QuorumTracker::new, (ignore, fail) -> { if (fail != null) callback.accept(null, fail); diff --git a/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java b/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java index f2caf35537..32f17625fd 100644 --- a/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java +++ b/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java @@ -93,6 +93,7 @@ public abstract class AbstractSegmentCompactor implements SegmentCompactor> compact(Collection> segments) @@ -129,6 +130,9 @@ public abstract class AbstractSegmentCompactor implements SegmentCompactor reader; while ((reader = readers.poll()) != null) { + if (stopped) + throw new Stopped(); + if (key == null || !reader.key().equals(key)) { maybeWritePartition(key, merger, serializer, firstDescriptor, firstOffset); @@ -232,6 +236,12 @@ public abstract class AbstractSegmentCompactor implements SegmentCompactor implements } @Override - public void onUpdate(AccordCacheEntry state) + public void onUpdate(AccordCacheEntry state) { Summary summary = loader.ifRelevant(state); if (summary != null) @@ -162,7 +162,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements if (isMaybeRelevant(i)) { TxnId txnId = i.txnId; - AccordCacheEntry entry = c.getUnsafe(txnId); + AccordCacheEntry entry = c.getUnsafe(txnId); Invariants.expect(entry != null, "%s found interval %s but no matching transaction in cache", owner.commandStore, i); if (entry != null) { @@ -199,7 +199,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements } @Override - protected void finish(Map into) + public void finish(Map into) { } @@ -233,7 +233,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements } @Override - protected void cleanupExclusive(Caches caches) + public void cleanupExclusive(Caches caches) { if (commandWatcher != null) { @@ -267,7 +267,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements } @Override - public void onUpdate(AccordCacheEntry state) + public void onUpdate(AccordCacheEntry state) { TxnId txnId = state.key(); if (txnId.is(Routable.Domain.Range)) @@ -334,7 +334,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree implements } @Override - public void onEvict(AccordCacheEntry state) + public void onEvict(AccordCacheEntry state) { TxnId txnId = state.key(); if (txnId.is(Routable.Domain.Range)) diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java index 0b3a75f90c..f394b68df3 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java @@ -39,6 +39,7 @@ import accord.local.DurableBefore; import accord.local.MaxConflicts; import accord.local.MaxDecidedRX; import accord.local.RedundantBefore; +import accord.local.RedundantStatus; import accord.primitives.Range; import accord.primitives.Ranges; import accord.primitives.SaveStatus; @@ -272,11 +273,13 @@ public class CommandStoreSerializers } } - private short cast(long v) + private int cast(long v) { - if ((v & ~0xFFFF) != 0) + if (v < 0 || v >= (1 << RedundantStatus.Property.WAS_OWNED.ordinal())) + throw new IllegalStateException("Should not be serializing WAS_OWNED or NOT_OWNED statuses"); + if ((v & ~0xFFFF) != 0) // retain this throw new IllegalStateException("Cannot serialize RedundantStatus larger than 0xFFFF. Requires serialization changes."); - return (short)v; + return (int) v; } @Override @@ -298,7 +301,7 @@ public class CommandStoreSerializers bounds[i] = CommandSerializers.txnId.deserialize(in); int[] statuses = new int[count * 2]; for (int i = 0 ; i < statuses.length ; ++i) - statuses[i] = in.readShort(); + statuses[i] = in.readShort() & 0xFFFF; return new RedundantBefore.Bounds(range, startEpoch, endEpoch, bounds, statuses, staleUntilAtLeast); } diff --git a/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java index 2acd8e2464..c413780546 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java @@ -25,7 +25,11 @@ import accord.impl.AbstractFetchCoordinator.FetchRequest; import accord.impl.AbstractFetchCoordinator.FetchResponse; import accord.messages.ReadData.CommitOrReadNack; import accord.messages.ReadData.ReadReply; +import accord.primitives.PartialDeps; +import accord.primitives.PartialTxn; import accord.primitives.Ranges; +import accord.primitives.TxnId; +import accord.utils.Invariants; import accord.utils.VIntCoding; import org.apache.cassandra.db.TypeSizes; @@ -51,18 +55,20 @@ public class FetchSerializers out.writeUnsignedVInt(request.executeAtEpoch); CommandSerializers.txnId.serialize(request.txnId, out); KeySerializers.ranges.serialize((Ranges) request.scope, out); - DepsSerializers.partialDeps.serialize(request.partialDeps, out); + DepsSerializers.partialDeps.serialize(PartialDeps.NONE, out); StreamingTxn.serializer.serialize(request.read, out, version); } @Override public FetchRequest deserialize(DataInputPlus in, Version version) throws IOException { - return new AccordFetchRequest(in.readUnsignedVInt(), - CommandSerializers.txnId.deserialize(in), - KeySerializers.ranges.deserialize(in), - DepsSerializers.partialDeps.deserialize(in), - StreamingTxn.serializer.deserialize(in, version)); + long epoch = in.readUnsignedVInt(); + TxnId txnId = CommandSerializers.txnId.deserialize(in); + Ranges ranges = KeySerializers.ranges.deserialize(in); + PartialDeps deps = DepsSerializers.partialDeps.deserialize(in); + Invariants.require(deps.isEmpty()); + PartialTxn txn = StreamingTxn.serializer.deserialize(in, version); + return new AccordFetchRequest(epoch, txnId, ranges, txn); } @Override @@ -71,7 +77,7 @@ public class FetchSerializers return TypeSizes.sizeofUnsignedVInt(request.executeAtEpoch) + CommandSerializers.txnId.serializedSize(request.txnId) + KeySerializers.ranges.serializedSize((Ranges) request.scope) - + DepsSerializers.partialDeps.serializedSize(request.partialDeps) + + DepsSerializers.partialDeps.serializedSize(PartialDeps.NONE) + StreamingTxn.serializer.serializedSize(request.read, version); } }; diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java b/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java index 29f0220790..87ce533638 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java @@ -58,10 +58,10 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.TokenRange; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; import org.apache.cassandra.service.accord.serializers.Version; diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java index bddb2b6fba..3c72c34ba1 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java @@ -55,8 +55,8 @@ import org.apache.cassandra.io.ParameterisedVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; import org.apache.cassandra.service.accord.serializers.Version; diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java index 719eb0f590..90cd9898c9 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java @@ -64,8 +64,8 @@ import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; import org.apache.cassandra.service.accord.serializers.Version; diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java index 07147fac17..1629b94fa3 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java +++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java @@ -74,6 +74,7 @@ import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.vint.VIntCoding; +import static accord.local.BootstrapReason.GAIN_OWNERSHIP; import static com.google.common.collect.ImmutableList.of; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.cassandra.tcm.MultiStepOperation.Kind.JOIN; @@ -396,7 +397,7 @@ public class BootstrapAndJoin extends MultiStepOperation Node node = service.node(); node.commandStores().forAllUnsafe(commandStore -> { - AsyncResult ready = commandStore.resumeBootstrap(node); + AsyncResult ready = commandStore.resumeBootstrap(node, GAIN_OWNERSHIP); bootstraps.add(AccordService.toFuture(ready.flatMap(e -> e.reads))); }); } diff --git a/src/java/org/apache/cassandra/utils/NoSpamLogger.java b/src/java/org/apache/cassandra/utils/NoSpamLogger.java index a2bf815b30..b14088913c 100644 --- a/src/java/org/apache/cassandra/utils/NoSpamLogger.java +++ b/src/java/org/apache/cassandra/utils/NoSpamLogger.java @@ -17,6 +17,11 @@ */ package org.apache.cassandra.utils; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; @@ -64,16 +69,18 @@ public class NoSpamLogger CLOCK = clock; } - public class NoSpamLogStatement extends AtomicLong + public static class NoSpamLogStatement extends AtomicLong { private static final long serialVersionUID = 1L; + private final Logger wrapped; private final String statement; private final long minIntervalNanos; - public NoSpamLogStatement(String statement, long minIntervalNanos) + public NoSpamLogStatement(Logger wrapped, String statement, long minIntervalNanos) { super(Long.MIN_VALUE); + this.wrapped = wrapped; this.statement = statement; this.minIntervalNanos = minIntervalNanos; } @@ -159,6 +166,154 @@ public class NoSpamLogger } } + public static class NoDuplicateSpamLogStatement + { + private static final long serialVersionUID = 1L; + private static final int PRUNE_SIZE = 32; + + private final Logger wrapped; + private final String statement; + private final long minIntervalNanos; + private final ConcurrentHashMap lastLogged = new ConcurrentHashMap<>(); + private final AtomicLong nextPruneAt = new AtomicLong(); + + public NoDuplicateSpamLogStatement(Logger wrapped, String statement, long minInterval, TimeUnit units) + { + this(wrapped, statement, units.toNanos(minInterval)); + } + + public NoDuplicateSpamLogStatement(Logger wrapped, String statement, long minIntervalNanos) + { + this.wrapped = wrapped; + this.statement = statement; + this.minIntervalNanos = minIntervalNanos; + } + + private boolean shouldLog(long id, long nowNanos) + { + Long expected = lastLogged.getOrDefault(id, Long.MIN_VALUE); + if (nowNanos < expected || !lastLogged.replace(id, expected, nowNanos + minIntervalNanos)) + return false; + + if (lastLogged.size() >= PRUNE_SIZE) + { + long pruneAt = nextPruneAt.get(); + if (nowNanos >= pruneAt && nextPruneAt.compareAndSet(pruneAt, nowNanos + minIntervalNanos)) + { + for (Map.Entry e : lastLogged.entrySet()) + { + if (nowNanos < e.getValue()) + lastLogged.remove(e.getKey(), e.getValue()); + } + } + } + return true; + } + + public boolean log(Level l, long id, long nowNanos, Object... objects) + { + if (!shouldLog(id, nowNanos)) return false; + return logNoCheck(l, objects); + } + + private boolean logNoCheck(Level l, Object... objects) + { + switch (l) + { + case DEBUG: + wrapped.debug(statement, objects); + break; + case INFO: + wrapped.info(statement, objects); + break; + case WARN: + wrapped.warn(statement, objects); + break; + case ERROR: + wrapped.error(statement, objects); + break; + default: + throw new AssertionError(); + } + return true; + } + + public boolean debug(long id, long nowNanos, Object... objects) + { + return log(Level.DEBUG, id, nowNanos, objects); + } + + public boolean debug(long id, Object... objects) + { + return debug(id, CLOCK.nanoTime(), objects); + } + + public boolean info(long id, long nowNanos, Object... objects) + { + return log(Level.INFO, id, nowNanos, objects); + } + + public boolean info(long id, Object... objects) + { + return info(id, CLOCK.nanoTime(), objects); + } + + public boolean warn(long id, long nowNanos, Object... objects) + { + return log(Level.WARN, id, nowNanos, objects); + } + + public boolean warn(long id, Object... objects) + { + return warn(id, CLOCK.nanoTime(), objects); + } + + public boolean error(long id, long nowNanos, Object... objects) + { + return log(Level.ERROR, id, nowNanos, objects); + } + + public boolean error(long id, Object... objects) + { + return error(id, CLOCK.nanoTime(), objects); + } + + public static long exceptionId(Throwable throwable) + { + return exceptionId(throwable, Collections.newSetFromMap(new IdentityHashMap<>())); + } + + private static long exceptionId(Throwable throwable, Set visited) + { + long id = throwable.getClass().hashCode(); + for (StackTraceElement ste : throwable.getStackTrace()) + { + id *= 31; + id += ste.getClassName().hashCode(); + id *= 31; + id += ste.getLineNumber(); + } + + for (Throwable suppressed : throwable.getSuppressed()) + { + if (!visited.add(suppressed)) + continue; + + id *= 31; + id += exceptionId(suppressed, visited); + } + for (Throwable cause = throwable.getCause() ; cause != null ; cause = cause.getCause()) + { + if (!visited.add(cause)) + break; + + id *= 31; + id += exceptionId(cause, visited); + } + return id; + } + } + private static final NonBlockingHashMap wrappedLoggers = new NonBlockingHashMap<>(); @VisibleForTesting @@ -305,7 +460,7 @@ public class NoSpamLogger NoSpamLogStatement statement = lastMessage.get(key); if (statement == null) { - statement = new NoSpamLogStatement(s, minIntervalNanos); + statement = new NoSpamLogStatement(wrapped, s, minIntervalNanos); NoSpamLogStatement temp = lastMessage.putIfAbsent(key, statement); if (temp != null) statement = temp; diff --git a/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java b/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java index 9542d396bd..9a95f38a03 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java +++ b/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java @@ -428,7 +428,7 @@ public final class SignalLock implements Lock { long cur = state; int count = asyncSignalCount(cur); - if (count == MAX_THREADS) + if (count == MAX_SIGNAL_COUNT) return false; if (signalIfWaiting) diff --git a/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java b/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java index 88a785d9a9..d82bd4ab83 100644 --- a/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java +++ b/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java @@ -24,6 +24,7 @@ import java.util.function.Function; import org.junit.Test; import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.utils.concurrent.Semaphore; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; diff --git a/test/conf/logback-info.xml b/test/conf/logback-info.xml new file mode 100644 index 0000000000..bc121ca649 --- /dev/null +++ b/test/conf/logback-info.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + ./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log + + %-5level [%thread] ${instance_id} %date{"yyyy-MM-dd'T'HH:mm:ss,SSS", UTC} %F:%L - %msg%n + + + INFO + + true + + + + + %-5level %date{"HH:mm:ss,SSS"} %msg%n + + + INFO + + + + + + + + + + diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java index 7e91f5a7fb..4b573b2551 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java +++ b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java @@ -83,7 +83,7 @@ import org.apache.cassandra.distributed.shared.DistributedTestBase; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.Verb; -import org.apache.cassandra.service.accord.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCache; import static java.lang.System.currentTimeMillis; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java index 506407cb13..107488f0d5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java @@ -46,7 +46,7 @@ import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.streaming.StreamEvent; import org.apache.cassandra.streaming.StreamEventHandler; @@ -262,7 +262,7 @@ public class AccordBootstrapTest extends AccordBootstrapTestBase getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { if (safeStore.ranges().currentRanges().contains(partitionKey)) { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + SaferCommandStore ss = (SaferCommandStore) safeStore; Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java index f38b84ede2..03c16edd2a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java @@ -81,6 +81,7 @@ import org.apache.cassandra.distributed.api.ICoordinator; import org.apache.cassandra.distributed.api.QueryResults; import org.apache.cassandra.distributed.api.SimpleQueryResult; import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.distributed.shared.FutureUtils; import org.apache.cassandra.distributed.test.sai.SAIUtil; import org.apache.cassandra.distributed.util.QueryResultUtil; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -93,6 +94,7 @@ import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FailingConsumer; import org.apache.cassandra.utils.Pair; @@ -116,8 +118,10 @@ import static org.junit.Assert.fail; public abstract class AccordCQLTestBase extends AccordTestBase { private static final Logger logger = LoggerFactory.getLogger(AccordCQLTestBase.class); + private static final int CAS_SIMULATOR_LITE_CONCURRENCY = 20; - protected AccordCQLTestBase(TransactionalMode transactionalMode) { + protected AccordCQLTestBase(TransactionalMode transactionalMode) + { super(transactionalMode); } @@ -131,7 +135,9 @@ public abstract class AccordCQLTestBase extends AccordTestBase public static void setupClass() throws IOException { AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL) - .set("paxos_variant", PaxosVariant.v2.name())), 2); + .set("paxos_variant", PaxosVariant.v2.name()) + .set("accord.migration_concurrency", "" + (1 + CAS_SIMULATOR_LITE_CONCURRENCY)) + ), 2); SHARED_CLUSTER.schemaChange("CREATE TYPE " + KEYSPACE + ".person (height int, age int)"); } @@ -3332,15 +3338,18 @@ public abstract class AccordCQLTestBase extends AccordTestBase coordinator.execute("INSERT INTO " + qualifiedAccordTableName + " (pk, count, seq1, seq2) VALUES (1, 0, '', []) USING TIMESTAMP 0", ConsistencyLevel.ALL); ListType LIST_TYPE = ListType.getInstance(Int32Type.instance, true); - ExecutorService es = Executors.newCachedThreadPool(); - List> futures = new ArrayList<>(); - for (int ii = 0; ii < 10; ii++) + Future[] futures = new Future[CAS_SIMULATOR_LITE_CONCURRENCY]; + long[] starts = new long[CAS_SIMULATOR_LITE_CONCURRENCY]; + for (int id = 0; id < CAS_SIMULATOR_LITE_CONCURRENCY; id++) { - int id = ii; - futures.add(es.submit(() -> coordinator.execute("UPDATE " + qualifiedAccordTableName + " SET count = count + 1, seq1 = seq1 + ?, seq2 = seq2 + ? WHERE pk = ? IF EXISTS", ConsistencyLevel.ALL, id + ",", ByteBufferUtil.getArray(LIST_TYPE.decompose(singletonList(id))), 1))); + starts[id] = Clock.Global.nanoTime(); + futures[id] = FutureUtils.map(coordinator.asyncExecuteWithResult("UPDATE " + qualifiedAccordTableName + " SET count = count + 1, seq1 = seq1 + ?, seq2 = seq2 + ? WHERE pk = ? IF EXISTS", ConsistencyLevel.ALL, id + ",", ByteBufferUtil.getArray(LIST_TYPE.decompose(singletonList(id))), 1), SimpleQueryResult::toObjectArrays); + } + for (int id = 0; id < CAS_SIMULATOR_LITE_CONCURRENCY; id++) + { + futures[id].get(); + System.out.println(String.format("Completed in %.3fs", (Clock.Global.nanoTime() - starts[id]) * 0.000000001)); } - for (Future f : futures) - f.get(); Object[][] result = coordinator.execute("SELECT pk, count, seq1, seq2 FROM " + qualifiedAccordTableName + " WHERE pk = 1", ConsistencyLevel.SERIAL); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java index 8b96f0d1da..97f98f319b 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java @@ -22,12 +22,11 @@ import com.google.common.base.Throwables; import org.assertj.core.api.Assertions; -import accord.api.RoutingKey; import accord.local.CommandStores; -import accord.local.LoadKeys; +import accord.local.ExecutionContext; import accord.local.Node; -import accord.local.PreLoadContext; import accord.local.cfk.CommandsForKey; +import accord.local.cfk.SafeCommandsForKey; import accord.primitives.Ranges; import accord.primitives.Routable; import accord.primitives.Txn; @@ -42,12 +41,10 @@ import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandsForKey; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.TokenRange; -import static accord.local.LoadKeysFor.READ_WRITE; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; import static org.apache.cassandra.service.accord.AccordService.getBlocking; @@ -135,19 +132,16 @@ public class AccordDropTableBase extends TestBaseImpl TableId tableId = TableId.fromString(s); AccordService accord = (AccordService) AccordService.instance(); TxnId syntheticTxnId = new TxnId(TxnId.MAX_EPOCH, 0, Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range, new Node.Id(1)); - PreLoadContext ctx = PreLoadContext.contextFor(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), LoadKeys.SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = ExecutionContext.unsequencedReadWrite(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), "Test"); CommandStores stores = accord.node().commandStores(); for (int storeId : stores.ids()) { AccordCommandStore store = (AccordCommandStore) stores.forId(storeId); getBlocking(store.chain(ctx, input -> { - AccordSafeCommandStore safe = (AccordSafeCommandStore) input; - for (RoutingKey key : safe.commandsForKeysKeys()) + SaferCommandStore safe = (SaferCommandStore) input; + for (SafeCommandsForKey safeCfk : safe.safeCommandsForKeys()) { - AccordSafeCommandsForKey safeCFK = (AccordSafeCommandsForKey) safe.ifLoadedAndInitialised(key); - if (safeCFK == null) // we read and found a key, but its null at load time... so ignore it - continue; - CommandsForKey cfk = safeCFK.current(); + CommandsForKey cfk = safeCfk.current(); CommandsForKey.TxnInfo minUndecided = cfk.minUndecidedManaged(); if (minUndecided != null) throw new AssertionError("Undecided txn: " + minUndecided); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java index 95d79b6fa7..1802efa82a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java @@ -62,7 +62,7 @@ import org.apache.cassandra.harry.checker.ModelChecker; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.IAccordService.DelegatingAccordService; @@ -89,7 +89,7 @@ public class AccordIncrementalRepairTest extends TestBaseImpl } @Override - public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) + public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) { return delegate.sync(requestedBy, onOrAfter, ranges, include, syncLocal, syncRemote, 10L, TimeUnit.MINUTES).map(v -> { executedBarriers = true; @@ -98,7 +98,7 @@ public class AccordIncrementalRepairTest extends TestBaseImpl } @Override - public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) + public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) { return delegate.sync(onOrAfter, keys, syncLocal, syncRemote).map(v -> { executedBarriers = true; @@ -209,7 +209,7 @@ public class AccordIncrementalRepairTest extends TestBaseImpl Node node = accordService().node(); AtomicReference waitFor = new AtomicReference<>(null); getBlocking(node.commandStores().forEach("Test", RoutingKeys.of(key), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { - AccordSafeCommandStore store = (AccordSafeCommandStore) safeStore; + SaferCommandStore store = (SaferCommandStore) safeStore; SafeCommandsForKey safeCfk = store.ifLoadedAndInitialised(key); if (safeCfk == null) return; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java index f8e2f82d2d..c1bd8d6b48 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java @@ -47,7 +47,7 @@ public class AccordJournalIntegrationTest extends TestBaseImpl { try (WithProperties wp = new WithProperties().set(CassandraRelevantProperties.DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED, "true"); Cluster cluster = init(Cluster.build(1) - .withConfig(config -> config.set("accord.catchup_on_start", "DISABLED")) + .withConfig(config -> config.set("accord.catchup_on_start", "false")) .withoutVNodes() .start())) { @@ -96,7 +96,7 @@ public class AccordJournalIntegrationTest extends TestBaseImpl { try (Cluster cluster = Cluster.build(1) .withoutVNodes() - .withConfig(config -> config.set("accord.catchup_on_start", "DISABLED")) + .withConfig(config -> config.set("accord.catchup_on_start", "false")) .start()) { cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};"); @@ -128,7 +128,7 @@ public class AccordJournalIntegrationTest extends TestBaseImpl .withoutVNodes() .withConfig(c -> { c.with(GOSSIP).with(NETWORK); - c.set("accord.catchup_on_start", "DISABLED"); + c.set("accord.catchup_on_start", "false"); }) .start()) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java index cee395977d..f7e52f108a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java @@ -56,7 +56,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.local.CommandStore; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.primitives.PartialDeps; import accord.primitives.TxnId; @@ -355,6 +355,12 @@ public class AccordLoadTest extends AccordTestBase return builder.setArtificialLatencies(LATENCIES); } + private static SettingsBuilder populate(SettingsBuilder builder, int keyCount) + { + return builder.setKeySelector(roundrobin(keyCount)) + .setReadRatio(0f); + } + private static SettingsBuilder ycsbA(SettingsBuilder builder, int keyCount) { return builder.setKeySelector(ycsbZipfian(keyCount)) @@ -832,7 +838,7 @@ public class AccordLoadTest extends AccordTestBase if (storeId.get() >= 0) { CommandStore commandStore = service.node().commandStores().forId(storeId.get()); - List> result = AccordService.getBlocking(commandStore.submit(PreLoadContext.contextFor(candidate, "LoadTest"), safeStore -> { + List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.unsequenced(candidate, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(candidate); PartialDeps deps = safeCommand.current().partialDeps(); if (deps == null) @@ -855,7 +861,7 @@ public class AccordLoadTest extends AccordTestBase for (List info : result) { TxnId txnId = TxnId.parse(info.get(0)); - AccordService.getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, "LoadTest"), safeStore -> { + AccordService.getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(txnId); if (safeCommand.current().executeAt != null) info.add(safeCommand.current().executeAt.toString()); @@ -915,7 +921,7 @@ public class AccordLoadTest extends AccordTestBase cluster.forEach(() -> { refresh(AccordExecutorMetrics.INSTANCE.elapsedRunning); refresh(AccordExecutorMetrics.INSTANCE.elapsed); - System.out.printf("%tT.%tL (%d %d %d %d %d %d)ms (%d %d %d %d %d %d)ms (%d %d %d %d %.0f, %d %d %d)us %d %d %d\n", nowMillis, nowMillis, + System.out.printf("%tT.%tL (%d %d %d %d %d %d)ms (%d %d %d %d %d %d)ms (%d %d %d)us %d %.0f (%d %d %d)us %d %d %d\n", nowMillis, nowMillis, getLatency(AccordCoordinatorMetrics.readMetrics.preacceptLatency, 0.5), getLatency(AccordCoordinatorMetrics.readMetrics.executeLatency, 0.5), getLatency(AccordCoordinatorMetrics.readMetrics.applyLatency, 0.5), @@ -1092,13 +1098,13 @@ public class AccordLoadTest extends AccordTestBase try { test.setup(); - test.testLoad(withArtificialLatencies(ycsbA(new SettingsBuilder(), 100_000) + test.testLoad(populate(new SettingsBuilder(), 1_000_000) // .setRatePerSecond(400).setMinRatePerSecond(200) // .setRatePerSecond(800).setMinRatePerSecond(200) .setRatePerSecond(1600).setMinRatePerSecond(200) .setIncreaseRatePerSecondInterval(5000) // .setTraceLast(5000) - ).build()); + .build()); } finally { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java index dff776cbf2..6e6bb30ab7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java @@ -54,7 +54,7 @@ import org.apache.cassandra.metrics.DefaultNameFactory; import org.apache.cassandra.metrics.RatioGaugeSet; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.exceptions.AccordReadPreemptedException; import org.apache.cassandra.service.accord.exceptions.AccordWritePreemptedException; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java index 4d600f0c71..881728882f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java @@ -38,7 +38,7 @@ import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.api.PartitionKey; import static com.google.common.collect.Iterables.getOnlyElement; @@ -112,7 +112,7 @@ public class AccordMoveTest extends AccordBootstrapTestBase getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), moveMax, moveMax, safeStore -> { if (!safeStore.ranges().allAt(preMove).contains(partitionKey)) { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + SaferCommandStore ss = (SaferCommandStore) safeStore; Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java index 79cf938c7e..4f4b7942f0 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java @@ -47,7 +47,7 @@ import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordOperations; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; @@ -187,7 +187,7 @@ public class AccordRecoverFromAvailabilityLossTest extends AccordBootstrapTestBa getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { if (safeStore.ranges().currentRanges().contains(partitionKey)) { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + SaferCommandStore ss = (SaferCommandStore) safeStore; Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java index 990c875024..eeb57dd204 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java @@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory; import accord.local.CommandStore; import accord.local.CommandStores.PreviouslyOwned; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.primitives.AbstractRanges; import accord.primitives.Ranges; import accord.topology.ActiveEpochs; @@ -121,7 +121,7 @@ public class AccordRegainRangesTest extends AccordTestBase Ranges range = Ranges.EMPTY; for (CommandStore commandStore : AccordService.instance().node().commandStores().all()) { - Ranges safeToReadRanges = getBlocking(commandStore.submit((PreLoadContext.Empty) () -> "No overlapping safeToReadRanges", safeCommandStore -> { + Ranges safeToReadRanges = getBlocking(commandStore.submit((ExecutionContext.Empty) () -> "No overlapping safeToReadRanges", safeCommandStore -> { Ranges mergedRanges = Ranges.EMPTY; for (Ranges r : safeCommandStore.safeToReadAt().values()) mergedRanges = mergedRanges.union(AbstractRanges.UnionMode.MERGE_ADJACENT, r); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java index 763cbd4d5a..c3ec06b6a7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java @@ -170,7 +170,12 @@ public abstract class AccordTestBase extends TestBaseImpl { SHARED_CLUSTER.filters().reset(); for (IInvokableInstance instance : SHARED_CLUSTER) - instance.runOnInstance(() -> AccordService.instance().node().commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start())); + { + instance.runOnInstance(() -> { + if (AccordService.isStarted()) + AccordService.instance().node().commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start()); + }); + } truncateSystemTables(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java index 8770df01c9..67493f5a2c 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java @@ -36,8 +36,9 @@ import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.accord.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.utils.ByteBufferUtil; @@ -86,7 +87,7 @@ public class AccordJournalConsistentExpungeTest extends TestBaseImpl Node node = service.node(); AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable()); - Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); + Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); TxnId txnId = TxnId.NONE; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java index 44bcc0f306..08779458fc 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java @@ -98,7 +98,7 @@ public class AccordJournalReplayTest extends TestBaseImpl .set("accord.retry_syncpoint", "1s*attempts") .set("accord.retry_durability", "1s*attempts") .set("accord.journal.replay_save_point", "NO") - .set("accord.catchup_on_start", "DISABLED") + .set("accord.catchup_on_start", "false") .with(NETWORK, GOSSIP)) .start()) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java index 04f53770c5..6ab7388879 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java @@ -66,7 +66,7 @@ public class JournalAccessRouteIndexOnStartupRaceTest extends TestBaseImpl public void test() throws IOException { try (Cluster cluster = Cluster.build(1) - .withConfig(config -> config.set("accord.catchup_on_start", "DISABLED")) + .withConfig(config -> config.set("accord.catchup_on_start", "false")) .withInstanceInitializer(BBHelper::install).start()) { IInvokableInstance node = cluster.get(1); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java new file mode 100644 index 0000000000..fc1763a273 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java @@ -0,0 +1,976 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import java.io.IOException; +import java.security.SecureRandom; +import java.time.Duration; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Snapshot; +import com.codahale.metrics.Timer; +import com.google.common.util.concurrent.RateLimiter; + +import org.agrona.collections.IntArrayList; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.local.Catchup; +import accord.local.CommandStore; +import accord.local.Node; +import accord.local.ExecutionContext; +import accord.local.SafeCommand; +import accord.primitives.PartialDeps; +import accord.primitives.TxnId; +import accord.utils.Functions; +import accord.utils.Invariants; +import accord.utils.UnhandledEnum; + +import org.apache.cassandra.concurrent.NamedThreadFactory; +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.distributed.api.IInstance; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.distributed.api.IMessage; +import org.apache.cassandra.distributed.api.IMessageFilters; +import org.apache.cassandra.distributed.test.accord.AccordTestBase; +import org.apache.cassandra.distributed.test.accord.load.LoadSettings.ClusterChaos; +import org.apache.cassandra.metrics.AccordCoordinatorMetrics; +import org.apache.cassandra.metrics.AccordExecutorMetrics; +import org.apache.cassandra.metrics.ShardedDecayingHistograms.ShardedDecayingHistogram; +import org.apache.cassandra.metrics.ShardedHistogram; +import org.apache.cassandra.metrics.SnapshottingTimer; +import org.apache.cassandra.net.ArtificialLatency; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.service.accord.AccordKeyspace; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.service.accord.debug.AccordTracing; +import org.apache.cassandra.service.accord.debug.AccordTracing.Message; +import org.apache.cassandra.service.accord.debug.CoordinationKinds; +import org.apache.cassandra.service.accord.debug.TxnKindsAndDomains; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import org.apache.cassandra.utils.concurrent.WaitQueue; + +import static accord.coordinate.Coordination.CoordinationKind.Client; +import static accord.coordinate.Coordination.CoordinationKind.Execute; +import static accord.local.BootstrapReason.LOG_CORRUPTED; +import static accord.local.BootstrapReason.LOG_INCOMPLETE; +import static java.lang.System.currentTimeMillis; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS; +import static org.apache.cassandra.distributed.test.accord.load.LoadSettings.ClusterChaos.REBOOTSTRAP_RESET; +import static org.apache.cassandra.service.accord.debug.AccordTracing.BucketMode.LEAKY; +import static org.apache.cassandra.service.accord.debug.AccordTracing.BucketMode.RING; +import static org.apache.cassandra.service.accord.debug.AccordTracing.BucketMode.SLOWEST; + +public class AccordLoadTestBase extends AccordTestBase +{ + private static long CHAOS_TIMEOUT_NANOS = TimeUnit.MINUTES.toNanos(10L); + private static final Logger logger = LoggerFactory.getLogger(AccordLoadTestBase.class); + + @Before + public void setup() + { + setupCluster(); + super.setup(); + } + + public void setupCluster() + { + setupCluster(5); + } + + public void setupCluster(int nodeCount) + { + setupCluster(nodeCount, config -> { + config.with(Feature.NETWORK, Feature.GOSSIP) + .set("accord.shard_durability_target_splits", "8") + .set("accord.shard_durability_max_splits", "16") + .set("accord.shard_durability_cycle", "1m") + .set("accord.queue_submission_model", "SIGNAL") + .set("accord.command_store_shard_count", "8") + .set("accord.queue_thread_count", "4") + .set("accord.queue_shard_count", "1") + .set("accord.replica_execution", "ALL") + .set("accord.send_stable", "TO_ALL_REPLICA_EXECUTABLE_ELSE_FOR_READS") + .set("accord.send_minimal", "false") + .set("accord.catchup_on_start_fail_latency", "2m"); + }); + } + + public void setupCluster(int nodeCount, Consumer configure) + { + Invariants.require(SHARED_CLUSTER == null); + try { SHARED_CLUSTER = createCluster(nodeCount, builder -> builder.withDCs(nodeCount).withConfig(configure)); } + catch (IOException e) { throw new RuntimeException(e); } + CassandraRelevantProperties.SIMULATOR_STARTED.setString(Long.toString(MILLISECONDS.toSeconds(currentTimeMillis()))); + } + + public void testLoad(final LoadSettings settings) throws Exception + { + Cluster cluster = SHARED_CLUSTER; + cluster.schemaChange("CREATE TABLE " + qualifiedAccordTableName + " (k int, v int, PRIMARY KEY(k)) WITH transactional_mode = 'full'"); +// long seed = new SecureRandom().nextLong(); + long seed = 3705626102508196273L; + try + { + final ConcurrentHashMap verbs = new ConcurrentHashMap<>(); + cluster.filters().outbound().messagesMatching(new IMessageFilters.Matcher() + { + @Override + public boolean matches(int i, int i1, IMessage iMessage) + { + verbs.computeIfAbsent(Verb.fromId(iMessage.verb()), ignore -> new AtomicInteger()).incrementAndGet(); + return false; + } + }).drop(); + + boolean waitForTransactions = settings.totalTransactions < Long.MAX_VALUE; + boolean waitForClusterChaos = settings.totalClusterChaos < Long.MAX_VALUE; + + int clientCount = settings.clients < 0 ? cluster.size() : settings.clients; + long nextRepairAt = settings.repairInterval; + long nextCompactionAt = settings.compactionInterval; + long nextJournalFlushAt = settings.journalFlushInterval; + long nextDataFlushAt = settings.dataFlushInterval; + long nextCfkFlushAt = settings.cfkFlushInterval; + long nextChaosAt = settings.clusterChaosInterval; + final ExecutorService chaosExecutor = Executors.newFixedThreadPool(settings.clusterChaosConcurrency, new NamedThreadFactory("ClusterChaos")); + final ExecutorService clientExecutor = Executors.newFixedThreadPool(clientCount, new NamedThreadFactory("Client")); + final BitSet initialised = new BitSet(); + final Supplier clusterChaos = settings.clusterChaos.apply(seed); + + List chaosActive = new ArrayList<>(); + cluster.get(1).nodetoolResult("cms", "reconfigure", "datacenter1:1", "datacenter2:1", "datacenter3:1").asserts().success(); + if (settings.cfkCompactionPeriodSeconds < Integer.MAX_VALUE && settings.cfkCompactionPeriodSeconds > 0) + { + cluster.forEach(i -> i.acceptOnInstance(period -> { + ((AccordService) AccordService.instance()).journal().compactor().updateCompactionPeriod(period, SECONDS); + }, settings.cfkCompactionPeriodSeconds)); + } + + if (settings.artificialLatencies != null) + { + for (int i = 0 ; i < cluster.size() ; ++i) + { + StringBuilder str = new StringBuilder(); + for (int j = 0 ; j < settings.artificialLatencies[i].length ; ++j) + { + if (j > 0) + str.append(","); + str.append("datacenter") + .append(j + 1) + .append(':') + .append(settings.artificialLatencies[i][j]) + .append("ms"); + } + cluster.get(i + 1).acceptOnInstance(latencies -> { + ArtificialLatency.setArtificialLatencies(latencies); + ArtificialLatency.setArtificialLatencyOnlyPermittedConsistencyLevels(false); + ArtificialLatency.setArtificialLatencyVerbs(ArtificialLatency.recommendedVerbs()); + ArtificialLatency.setEnabled(true); + }, str.toString()); + } + } + + if (settings.traceSlowest > 0f) + { + float traceSlowest = settings.traceSlowest; + for (int i = 0 ; i < cluster.size() ; ++i) + { + cluster.get(i + 1).runOnInstance(() -> { + AccordTracing tracing = ((AccordAgent) AccordService.unsafeInstance().agent()).tracing(); + tracing.setPattern(1, pattern -> pattern.withChance(traceSlowest) + .withKinds(TxnKindsAndDomains.parse("{K*}")) + .withTraceNew(CoordinationKinds.ALL), + SLOWEST, -1, 2, LEAKY, 10, 1, CoordinationKinds.ALL); + }); + } + } + + if (settings.traceLast > 0) + { + int traceLast = settings.traceLast; + for (int i = 0 ; i < cluster.size() ; ++i) + { + cluster.get(i + 1).runOnInstance(() -> { + AccordTracing tracing = ((AccordAgent) AccordService.unsafeInstance().agent()).tracing(); + tracing.setPattern(2, pattern -> pattern.withKinds(TxnKindsAndDomains.parse("{KW}")) + .withTraceNew(CoordinationKinds.ALL), + RING, -1, traceLast, LEAKY, 10, 1, CoordinationKinds.ALL); + }); + } + } + + final AtomicBoolean stop = new AtomicBoolean(); + final AtomicBoolean pauseOrStop = new AtomicBoolean(); + final WaitQueue waitQueue = WaitQueue.newWaitQueue(); + final Random random = new Random(); + final Semaphore completed = new Semaphore(0); + final AtomicIntegerArray coordinatorIndexes = new AtomicIntegerArray(clientCount); + final Set chaosCandidates = new ConcurrentSkipListSet<>(); + for (int i = 1; i <= cluster.size() ; ++i) + chaosCandidates.add(i); + final List> clients = new ArrayList<>(); + final AtomicReferenceArray rateLimiters = new AtomicReferenceArray<>(clientCount); + final AtomicReference readHistogram = new AtomicReference<>(new EstimatedHistogram(200)); + final AtomicReference writeHistogram = new AtomicReference<>(new EstimatedHistogram(200)); + final List chaosHistory = new ArrayList<>(); + if (settings.clients >= cluster.size()) + throw new IllegalArgumentException("Cannot have more clients than nodes"); + if (settings.clusterChaosInterval < Integer.MAX_VALUE && settings.clients + 1 >= cluster.size()) + throw new IllegalArgumentException("If restarting, cannot have as many clients as nodes, as must reroute client requests during restart"); + + int clientRatePerSecond = Math.min(settings.ratePerSecond, settings.minRatePerSecond) / clientCount; + for (int client = 0 ; client < clientCount ; ++client) + { + rateLimiters.set(client, RateLimiter.create(clientRatePerSecond)); + final int clientIndex = client; + coordinatorIndexes.set(client, client + 1); + clients.add(clientExecutor.submit(() -> { + final Semaphore inFlight = new Semaphore(settings.clientConcurrency); + while (true) + { + while (pauseOrStop.get()) + { + if (stop.get()) + break; + + WaitQueue.Signal signal = waitQueue.register(); + if (pauseOrStop.get()) signal.awaitThrowUncheckedOnInterrupt(); + else signal.cancel(); + } + + int coordinatorIdx = coordinatorIndexes.get(clientIndex); + ICoordinator coordinator = cluster.coordinator(coordinatorIdx); + try + { + rateLimiters.get(clientIndex).acquire(); + inFlight.acquire(); + long commandStart = System.nanoTime(); + IntArrayList keys = new IntArrayList(settings.keysPerOperation, -1); + for (int i = 0 ; i < settings.keysPerOperation ; ++i) + { + int k = settings.keySelector.getAsInt(); + if (!keys.containsInt(k)) + keys.add(k); + } + if (!keys.intStream().allMatch(initialised::get)) + { + coordinator.executeWithResult((success, fail) -> { + inFlight.release(); + completed.release(); + if (fail == null) + { + long elapsed = System.nanoTime() - commandStart; + writeHistogram.get().add(NANOSECONDS.toMicros(elapsed)); + synchronized (initialised) + { + keys.forEachInt(initialised::set); + } + } + else + { + logger.error("{}", fail.toString()); + } + }, "UPDATE " + qualifiedAccordTableName + " SET v = 0 WHERE k IN ?", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, keys); + } + else if (random.nextFloat() < settings.readRatio) + { + coordinator.executeWithResult((success, fail) -> { + inFlight.release(); + completed.release(); + if (fail == null) + readHistogram.get().add(NANOSECONDS.toMicros(System.nanoTime() - commandStart)); + }, "BEGIN TRANSACTION\n" + + "SELECT * FROM " + qualifiedAccordTableName + " WHERE k IN ?;\n" + + "COMMIT TRANSACTION;", ConsistencyLevel.SERIAL, keys + ); + } + else + { + coordinator.executeWithResult((success, fail) -> { + inFlight.release(); + completed.release(); + if (fail == null) + { + long elapsed = System.nanoTime() - commandStart; + writeHistogram.get().add(NANOSECONDS.toMicros(elapsed)); + } + else + logger.error("{}", fail.toString()); + }, "BEGIN TRANSACTION\n" + + // "UPDATE " + qualifiedAccordTableName + " SET v = ? WHERE k = ?;\n" + + "UPDATE " + qualifiedAccordTableName + " SET v += ? WHERE k IN ?;\n" + + "COMMIT TRANSACTION;", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, random.nextInt(100), keys); + } + } + catch (RejectedExecutionException e) + { + inFlight.release(); + } + catch (InterruptedException e) + { + throw new UncheckedInterruptedException(e); + } + } + })); + } + + int targetClientRatePerSecond = settings.ratePerSecond / clientCount; + int nextRateLimitIncrease = settings.increaseRatePerSecondInterval; + long remainingTransactions = settings.totalTransactions; + int remainingClusterChaos = settings.totalClusterChaos; + while (true) + { + long batchStart = System.nanoTime(); + int batchSize = 0; + + if (completed.tryAcquire(settings.batchSize, settings.batchPeriodNanos, NANOSECONDS)) + batchSize = settings.batchSize; + batchSize += completed.drainPermits(); + + if (clientRatePerSecond < targetClientRatePerSecond) + { + if ((nextRateLimitIncrease -= batchSize) <= 0) + { + clientRatePerSecond = Math.min(clientRatePerSecond * 2, targetClientRatePerSecond); + for (int i = 0 ; i < clientCount ; ++i) + rateLimiters.set(i, RateLimiter.create(clientRatePerSecond)); + nextRateLimitIncrease = settings.increaseRatePerSecondInterval; + } + } + + if ((nextRepairAt -= batchSize) <= 0) + { + nextRepairAt += settings.repairInterval; + System.out.println("repairing..."); + cluster.coordinator(1).instance().nodetool("repair", qualifiedAccordTableName); + } + + if ((nextCompactionAt -= batchSize) <= 0) + { + nextCompactionAt += settings.compactionInterval; + compactJournalCfs(cluster); + } + + if ((nextJournalFlushAt -= batchSize) <= 0) + { + nextJournalFlushAt += settings.journalFlushInterval; + flushJournal(cluster); + } + + if ((nextDataFlushAt -= batchSize) <= 0) + { + nextDataFlushAt += settings.dataFlushInterval; + flushData(cluster); + } + + if ((nextCfkFlushAt -= batchSize) <= 0) + { + nextCfkFlushAt += settings.cfkFlushInterval; + flushCfk(cluster); + } + + if ((nextChaosAt -= batchSize) <= 0) + { + Iterator iter = chaosActive.iterator(); + while (iter.hasNext()) + { + ChaosActive chaos = iter.next(); + if (chaos.future.isDone()) + { + chaos.future.get(); + iter.remove(); + Invariants.require(cluster.size() <= chaosActive.size() + chaosCandidates.size()); + } + else + { + long elapsedNanos = Clock.Global.nanoTime() - chaos.startedAt; + if (elapsedNanos >= CHAOS_TIMEOUT_NANOS) + throw new AssertionError("Chaos " + chaos.kind + " has been running for " + NANOSECONDS.toSeconds(elapsedNanos) + "s with seed " + seed); + } + } + + if (chaosActive.size() < settings.clusterChaosConcurrency) + { + if (remainingClusterChaos > 0) + { + --remainingClusterChaos; + nextChaosAt += settings.clusterChaosInterval; + ClusterChaos chaos = clusterChaos.get(); + chaosActive.add(new ChaosActive(chaos, chaos(cluster, coordinatorIndexes, chaosCandidates, chaosExecutor, random, chaos, chaosHistory))); + } + else if (chaosActive.isEmpty() && (remainingTransactions <= 0 || !waitForTransactions)) + { + break; + } + } + } + + if ((remainingTransactions -= batchSize) <= 0 && (remainingClusterChaos <= 0 || !waitForClusterChaos)) + break; + + long nowMillis = System.currentTimeMillis(); + EstimatedHistogram reads = readHistogram.getAndSet(new EstimatedHistogram(200)); + EstimatedHistogram writes = writeHistogram.getAndSet(new EstimatedHistogram(200)); + + maybePrintSlowestTraces(cluster, settings); + maybePrintLastTraces(cluster, pauseOrStop, settings, waitQueue); + printInstanceMetrics(nowMillis, cluster); + printRates(nowMillis, batchStart, batchSize, reads, writes); + printVerbs(nowMillis, verbs); + } + } + catch (Throwable t) + { + t.printStackTrace(); + System.exit(1); + } + + logger.info("Workload completed successfully"); + } + + static void safeForEach(Cluster cluster, IIsolatedExecutor.SerializableRunnable run) + { + safeForEach(cluster, ignore -> run.run(), null); + } + + static

void safeForEach(Cluster cluster, IIsolatedExecutor.SerializableConsumer

consumer, P param) + { + for (IInvokableInstance i : cluster) + { + try + { + if (!i.isShutdown()) + { + i.acceptOnInstance(consumer, param); + } + } + catch (Throwable t) + { + logger.error("", t); + } + } + } + + private void compactJournalCfs(Cluster cluster) + { + System.out.println("compacting journal cfs..."); + for (IInvokableInstance i : cluster) + { + try { i.nodetool("compact", "system_accord.journal"); } + catch (Throwable t) { logger.error("", t); } + } + } + + private void flushJournal(Cluster cluster) + { + System.out.println("flushing journal..."); + safeForEach(cluster, () -> { + if (AccordService.started()) + ((AccordService) AccordService.instance()).journal().closeCurrentSegmentForTestingIfNonEmpty(); + }); + } + + private void flushData(Cluster cluster) + { + System.out.println("flushing data..."); + safeForEach(cluster, name -> { + Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(KEYSPACE, name).id).forceFlush(UNIT_TESTS); + }, accordTableName); + } + + private void flushCfk(Cluster cluster) + { + System.out.println("flushing cfk..."); + safeForEach(cluster, () -> { + if (CommitLog.instance.isStarted()) + AccordKeyspace.AccordColumnFamilyStores.commandsForKey.forceFlush(UNIT_TESTS); + }); + } + + private static class ChaosActive + { + final ClusterChaos kind; + final Future future; + final long startedAt; + + private ChaosActive(ClusterChaos kind, Future future) + { + this.kind = kind; + this.future = future; + this.startedAt = Clock.Global.nanoTime(); + } + } + + private static Future chaos(Cluster cluster, AtomicIntegerArray coordinatorIndexes, Set candidates, ExecutorService chaosExecutor, Random random, ClusterChaos chaos, List history) + { + List snapshot = new ArrayList<>(candidates); + int nodeIdx; + { + int i = random.nextInt(snapshot.size()); + Integer remove = snapshot.get(i); + candidates.remove(remove); + snapshot = new ArrayList<>(candidates); + Invariants.require(snapshot.size() > 0); + nodeIdx = remove; + } + + for (int i = 0; i < coordinatorIndexes.length(); ++i) + { + if (nodeIdx == coordinatorIndexes.get(i)) + { + int j = random.nextInt(snapshot.size()); + int replaceIdx = snapshot.get(j); + coordinatorIndexes.set(j, replaceIdx); + } + } + + String describe = String.format("%s node %d...", chaos, nodeIdx); + history.add(describe); + System.out.println("========= BEGIN CHAOS =========="); + System.out.println(describe); + System.out.println(candidates); + System.out.println("========= BEGIN CHAOS =========="); + switch (chaos) + { + default: throw UnhandledEnum.unknown(chaos); + case REBOOTSTRAP_INCOMPLETE: + case REBOOTSTRAP_RESET: + { + IInvokableInstance node = cluster.get(nodeIdx); + return node.asyncAcceptsOnInstance((Set cnds) -> { + try + { + Node accordNode = AccordService.instance().node(); + AccordService.getBlocking(accordNode.commandStores().rebootstrap(accordNode, chaos == REBOOTSTRAP_RESET ? LOG_CORRUPTED : LOG_INCOMPLETE)); + } + finally + { + Invariants.require(cnds.add(nodeIdx)); + System.out.println("========== END CHAOS ==========="); + System.out.println(describe); + System.out.println("========== END CHAOS ==========="); + } + }).apply(candidates); + } + case REBOOTSTRAP_IF_BEHIND: + { + IInvokableInstance node = cluster.get(nodeIdx); + return node.asyncAcceptsOnInstance((Set cmds) -> { + try + { + AccordService.getBlocking(Catchup.rebootstrapIfBehind(AccordService.instance().node())); + } + finally + { + Invariants.require(cmds.add(nodeIdx)); + System.out.println("========== END CHAOS ==========="); + System.out.println(String.format("%s node %d...", chaos, nodeIdx)); + System.out.println("========== END CHAOS ==========="); + } + }).apply(candidates); + } + case RESTART: + case RESTART_AND_REBOOTSTRAP_INCOMPLETE: + case RESTART_AND_REBOOTSTRAP_RESET: + case RESTART_AND_REBOOTSTRAP_AFTER_TIMEOUT: + { + return chaosExecutor.submit(() -> { + IInvokableInstance node = cluster.get(nodeIdx); + try + { + node.shutdown().get(); + switch (chaos) + { + case RESTART_AND_REBOOTSTRAP_AFTER_TIMEOUT: + node.config().set("accord.catchup_on_start_on_timeout", "REBOOTSTRAP"); + node.config().set("accord.catchup_on_start_success_latency", "0s"); + node.config().set("accord.catchup_on_start_fail_latency", "0s"); + break; + case RESTART_AND_REBOOTSTRAP_INCOMPLETE: + node.config().set("accord.journal.replay", "REBOOTSTRAP_INCOMPLETE"); + break; + case RESTART_AND_REBOOTSTRAP_RESET: + node.config().set("accord.journal.replay", "REBOOTSTRAP_RESET"); + break; + } + node.startup(); + return null; + } + catch (InterruptedException | ExecutionException e) + { + throw new RuntimeException(e); + } + finally + { + Invariants.require(candidates.add(nodeIdx)); + node.config().set("accord.catchup_on_start_success_latency", "60s"); + node.config().set("accord.catchup_on_start_fail_latency", "120s"); + node.config().set("accord.catchup_on_start_on_timeout", "IGNORE"); + node.config().set("accord.journal.replay", "PART_NON_DURABLE"); + System.out.println("========== END CHAOS ==========="); + System.out.println(String.format("%s node %d...", chaos, nodeIdx)); + System.out.println("========== END CHAOS ==========="); + } + }); + } + } + } + + + private static void printRates(long nowMillis, long batchStartNanos, long batchSize, EstimatedHistogram reads, EstimatedHistogram writes) + { + System.out.println(String.format("%tT.%tL rate: %.2f/s (%d total)", nowMillis, nowMillis, (((float)batchSize * 1000) / NANOSECONDS.toMillis(System.nanoTime() - batchStartNanos)), batchSize)); + System.out.println(String.format("%tT.%tL reads : %d %d %d %d %d %d", nowMillis, nowMillis, reads.percentile(.25)/1000, reads.percentile(.5)/1000, reads.percentile(.95)/1000, reads.percentile(.99)/1000, reads.percentile(.999)/1000, reads.percentile(1)/1000)); + System.out.println(String.format("%tT.%tL writes: %d %d %d %d %d %d", nowMillis, nowMillis, writes.percentile(.25)/1000, writes.percentile(.5)/1000, writes.percentile(.95)/1000, writes.percentile(.99)/1000, writes.percentile(.999)/1000, writes.percentile(1)/1000)); + } + + private static void printInstanceMetrics(long nowMillis, Cluster cluster) + { + safeForEach(cluster, () -> { + refresh(AccordExecutorMetrics.INSTANCE.elapsedRunning); + refresh(AccordExecutorMetrics.INSTANCE.elapsed); + System.out.println(String.format("%tT.%tL (%d %d %d %d %d %d)ms (%d %d %d %d %d %d)ms (%d %d %d %d %.0f, %d %d %d)us %d %d %d", nowMillis, nowMillis, + getLatency(AccordCoordinatorMetrics.readMetrics.preacceptLatency, 0.5), + getLatency(AccordCoordinatorMetrics.readMetrics.executeLatency, 0.5), + getLatency(AccordCoordinatorMetrics.readMetrics.applyLatency, 0.5), + getLatency(AccordCoordinatorMetrics.readMetrics.preacceptLatency, 0.999), + getLatency(AccordCoordinatorMetrics.readMetrics.executeLatency, 0.999), + getLatency(AccordCoordinatorMetrics.readMetrics.applyLatency, 0.999), + getLatency(AccordCoordinatorMetrics.writeMetrics.preacceptLatency, 0.95), + getLatency(AccordCoordinatorMetrics.writeMetrics.executeLatency, 0.5), + getLatency(AccordCoordinatorMetrics.writeMetrics.applyLatency, 0.5), + getLatency(AccordCoordinatorMetrics.writeMetrics.preacceptLatency, 0.999), + getLatency(AccordCoordinatorMetrics.writeMetrics.executeLatency, 0.999), + getLatency(AccordCoordinatorMetrics.writeMetrics.applyLatency, 0.999), + getLatency(AccordExecutorMetrics.INSTANCE.elapsedRunning, 0.5), + getLatency(AccordExecutorMetrics.INSTANCE.elapsedRunning, 0.9), + getLatency(AccordExecutorMetrics.INSTANCE.elapsedRunning, 1.0), + getCount(AccordExecutorMetrics.INSTANCE.elapsedRunning), + getTotal(AccordExecutorMetrics.INSTANCE.elapsedRunning), + getLatency(AccordExecutorMetrics.INSTANCE.elapsed, 0.5), + getLatency(AccordExecutorMetrics.INSTANCE.elapsed, 0.9), + getLatency(AccordExecutorMetrics.INSTANCE.elapsed, 0.999), + AccordExecutorMetrics.INSTANCE.running.getValue(), + AccordExecutorMetrics.INSTANCE.waitingToRun.getValue(), + AccordExecutorMetrics.INSTANCE.preparingToRun.getValue() + )); + clear(AccordExecutorMetrics.INSTANCE.elapsedRunning); + clear(AccordExecutorMetrics.INSTANCE.elapsed); + }); + } + + private static void printVerbs(long nowMillis, Map verbs) + { + class VerbCount + { + final Verb verb; + final int count; + + VerbCount(Verb verb, int count) + { + this.verb = verb; + this.count = count; + } + } + List verbCounts = new ArrayList<>(); + for (Map.Entry e : verbs.entrySet()) + { + int count = e.getValue().getAndSet(0); + if (count != 0) verbCounts.add(new VerbCount(e.getKey(), count)); + } + verbCounts.sort(Comparator.comparing(v -> -v.count)); + + StringBuilder verbSummary = new StringBuilder(); + for (VerbCount vs : verbCounts) + { + { + if (verbSummary.length() > 0) + verbSummary.append(", "); + verbSummary.append(vs.verb); + verbSummary.append(": "); + verbSummary.append(vs.count); + } + } + System.out.println(String.format("%tT.%tL verbs: %s", nowMillis, nowMillis, verbSummary)); + } + + private static void maybePrintSlowestTraces(Cluster cluster, LoadSettings settings) + { + if (settings.traceSlowest > 0f) + { + safeForEach(cluster, () -> { + AccordTracing tracing = ((AccordAgent)AccordService.instance().agent()).tracing(); + + tracing.forEach(Functions.alwaysTrue(), (txnId, state) -> { + state.forEach(event -> { + if (event.elapsedNanos() < MILLISECONDS.toNanos(100)) + return; + + for (Message message : event.messages()) + { + long multiplier = message.atNanos < event.doneAtNanos() ? 1 : -1; + System.out.println(String.format("%s %s %s %s %s %s", txnId, event.kind, multiplier * (message.atNanos - event.atNanos)/1000000, message.nodeId, message.commandStoreId, message.message)); + } + }); + }); + tracing.eraseAll(); + }); + } + } + + private static void maybePrintLastTraces(Cluster cluster, AtomicBoolean pauseOrStop, LoadSettings settings, WaitQueue waitQueue) + { + if (settings.traceLast > 0) + { + pauseOrStop.set(true); + Map>> print = new HashMap<>(); + for (int i = 1 ; i <= cluster.size() ; ++i) + { + cluster.get(i).acceptOnInstance(out -> { + AccordService service = (AccordService)AccordService.instance(); + AccordTracing tracing = ((AccordAgent)AccordService.instance().agent()).tracing(); + PriorityQueue candidates = new PriorityQueue<>(Comparator.comparingLong(c -> -c.elapsedMicros)); + tracing.forEach(Functions.alwaysTrue(), (txnId, events) -> { + events.forEach(event -> { + if (event.kind == Client) + { + long doneAtMicros = event.doneAtMicros(); + long elapsedMicros = doneAtMicros - event.txnId().hlc(); + if (elapsedMicros > 350000 && elapsedMicros < 390000) + candidates.add(new SortedByElapsed(txnId, elapsedMicros)); + } + }); + }); + + AtomicInteger storeId = new AtomicInteger(); + while (!candidates.isEmpty()) + { + SortedByElapsed sortedCandidate = candidates.poll(); + if (sortedCandidate.elapsedMicros < 300000) + return; + + TxnId candidate = sortedCandidate.txnId; + storeId.lazySet(-1); + tracing.forEach(candidate, events -> { + events.forEach(event -> { + if (storeId.get() >= 0) + return; + for (Message message : event.messages()) + { + if (message.nodeId < 0 && message.commandStoreId >= 0) + { + storeId.set(message.commandStoreId); + break; + } + } + }); + }); + + if (storeId.get() >= 0) + { + CommandStore commandStore = service.node().commandStores().forId(storeId.get()); + List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.unsequenced(candidate, "LoadTest"), safeStore -> { + SafeCommand safeCommand = safeStore.unsafeGet(candidate); + PartialDeps deps = safeCommand.current().partialDeps(); + if (deps == null) + return null; + List> infos = new ArrayList<>(); + for (TxnId txnId : deps.txnIds()) + { + List info = new ArrayList<>(); + info.add(txnId.toString()); + infos.add(info); + } + List info = new ArrayList<>(); + info.add(candidate.toString()); + infos.add(info); + return infos; + })); + + if (result != null) + { + for (List info : result) + { + TxnId txnId = TxnId.parse(info.get(0)); + AccordService.getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "LoadTest"), safeStore -> { + SafeCommand safeCommand = safeStore.unsafeGet(txnId); + if (safeCommand.current().executeAt != null) + info.add(safeCommand.current().executeAt.toString()); + })); + } + + out.put(candidate.toString(), result); + return; + } + } + } + }, print); + } + + for (int i = 1 ; i <= cluster.size() ; ++i) + { + cluster.get(i).acceptOnInstance(out -> { + AccordTracing tracing = ((AccordAgent)AccordService.instance().agent()).tracing(); + for (Map.Entry>> e : out.entrySet()) + { + TxnId parentId = TxnId.parse(e.getKey()); + for (List infos : e.getValue()) + { + TxnId depId = TxnId.parse(infos.get(0)); + tracing.forEach(depId, events -> { + events.forEach(event -> { + infos.add(event.kind + ": [" + (event.idMicros - parentId.hlc()) + "..." + (event.doneAtMicros() - parentId.hlc()) + "][" + (event.idMicros - depId.hlc()) + "..." + (event.doneAtMicros() - depId.hlc()) + "]"); + if (event.kind == Execute) + { + for (Message message : event.messages()) + { + if (message.nodeId == parentId.node.id) + { + long atMicros = (event.idMicros + (message.atNanos - event.atNanos)/1000) - parentId.hlc(); + infos.add(atMicros + ": " + message.message); + } + } + } + }); + }); + } + } + }, print); + } + + for (Map.Entry>> e : print.entrySet()) + { + System.out.println("======" + e.getKey() + "======"); + for (List infos : e.getValue()) + System.out.println(infos); + } + if (!print.isEmpty()) + System.out.println(); + pauseOrStop.set(false); + waitQueue.signalAll(); + } + } + + private static void refresh(Histogram histogram) + { + if (histogram instanceof ShardedHistogram) + ((ShardedHistogram) histogram).refresh(); + if (histogram instanceof ShardedDecayingHistogram) + ((ShardedDecayingHistogram) histogram).refresh(); + } + + private static long getLatency(Histogram histogram, double percentile) + { + return (long)(histogram.getSnapshot().getValue(percentile) / 1000); + } + + private static long getCount(Histogram histogram) + { + return histogram.getSnapshot().size(); + } + + private static double getTotal(Histogram histogram) + { + Snapshot snapshot = histogram.getSnapshot(); + return (snapshot.getMean() * 0.0001d * snapshot.size()); + } + + private static void clear(Histogram histogram) + { + if (histogram instanceof ShardedHistogram) + ((ShardedHistogram) histogram).clear(); + if (histogram instanceof ShardedDecayingHistogram) + ((ShardedDecayingHistogram) histogram).clear(); + } + + private static long getLatency(Timer timer, double percentile) + { + if (timer instanceof SnapshottingTimer) + return (long) (((SnapshottingTimer) timer).getPercentileSnapshot().getValue(percentile) / 1000); + return (long)(timer.getSnapshot().getValue(0.999) / 1000); + } + + private static long getSize(Timer timer) + { + if (timer instanceof SnapshottingTimer) + return ((SnapshottingTimer) timer).getPercentileSnapshot().size(); + return timer.getSnapshot().size(); + } + + @Override + protected Logger logger() + { + return logger; + } + + static class SortedByElapsed + { + final TxnId txnId; + final long elapsedMicros; + + SortedByElapsed(TxnId txnId, long elapsedMicros) + { + this.txnId = txnId; + this.elapsedMicros = elapsedMicros; + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java new file mode 100644 index 0000000000..4ddbf72fe0 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.distributed.api.Feature; + +import static org.apache.cassandra.distributed.test.accord.load.LoadSettings.ycsbZipfian; + +public class AccordRebootstrapLoadTest extends AccordLoadTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(AccordRebootstrapLoadTest.class); + + @Override + protected Logger logger() + { + return logger; + } + + public void setupCluster(int nodeCount) + { + setupCluster(nodeCount, config -> { + config.with(Feature.NETWORK, Feature.GOSSIP) + .set("accord.shard_durability_target_splits", "8") + .set("accord.shard_durability_max_splits", "16") + .set("accord.shard_durability_cycle", "1m") + .set("accord.queue_submission_model", "SIGNAL") + .set("accord.command_store_shard_count", "8") + .set("accord.queue_thread_count", "4") + .set("accord.queue_shard_count", "1") + .set("accord.send_minimal", "false") // TODO (expected): only required because of misordering of preaccept, accept, commit if queued together + .set("accord.catchup_on_start_fail_latency", "2m"); + }); + } + + @Test + public void testLoad() throws Exception + { + testLoad(new LoadSettings.Builder() + .setKeySelector(ycsbZipfian(100_000)) + .setRatePerSecond(200) + .setClusterChaosInterval(10000) + .setClusterChaosConcurrency(2) + .setTotalClusterChaos(10) + .build()); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java new file mode 100644 index 0000000000..e72cd30aad --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import java.util.Arrays; + +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.distributed.shared.DistributedTestBase; + +import static org.apache.cassandra.distributed.test.accord.load.LoadSettings.ycsbZipfian; + +public class AccordYcsbLoadTest extends AccordLoadTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(AccordYcsbLoadTest.class); + + private static final int[][] LATENCIES = new int[][] { + new int[] { 0, 44, 64, 43, 84 }, + new int[] { 44, 0, 30, 3, 45 }, + new int[] { 64, 30, 0, 28, 37 }, + new int[] { 43, 3, 28, 0, 49 }, + new int[] { 84, 45, 37, 49, 0 } + }; + + private static LoadSettings.Builder withArtificialLatencies(LoadSettings.Builder builder) + { + return builder.setArtificialLatencies(LATENCIES); + } + + private static LoadSettings.Builder ycsbA(LoadSettings.Builder builder, int keyCount) + { + return builder.setKeySelector(ycsbZipfian(keyCount)) + .setReadRatio(0.5f); + } + + private static LoadSettings.Builder ycsbB(LoadSettings.Builder builder, int keyCount) + { + return builder.setKeySelector(ycsbZipfian(keyCount)) + .setReadRatio(0.95f); + } + + private static LoadSettings.Builder ycsbC(LoadSettings.Builder builder, int keyCount) + { + return builder.setKeySelector(ycsbZipfian(keyCount)) + .setReadRatio(1.0f); + + } + + @Override + protected Logger logger() + { + return logger; + } + + private static void computeWorstLatencies() + { + int[] qs = new int[LATENCIES.length]; + for (int i = 0 ; i < qs.length ; ++i) + { + int[] copy = LATENCIES[i].clone(); + Arrays.sort(copy); + qs[i] = copy[copy.length/2]; + } + int[] ws = new int[qs.length]; + for (int i = 0 ; i < qs.length ; ++i) + { + int iw = Integer.MIN_VALUE; + for (int j = 0; j < qs.length ; ++j) + iw = Math.max(iw, qs[i] + 3*qs[j] + LATENCIES[i][j]); + ws[i] = iw; + } + System.out.println(Arrays.toString(ws)); + Arrays.fill(ws, 0); + for (int i = 0 ; i < qs.length ; ++i) + { + for (int j = 0 ; j < qs.length ; ++j) + { + if (j == i) continue; + if (qs[j] > 2*qs[i]) continue; + int w = qs[i] + 4*qs[j] + LATENCIES[i][j]; + if (w > ws[i]) + ws[i] = w; + } + } + System.out.println(Arrays.toString(ws)); + } + + @Ignore + @Test + public void testLoad() throws Exception + { + testLoad(ycsbA(new LoadSettings.Builder(), 100_000) + .setRatePerSecond(1600).setMinRatePerSecond(200) + .setIncreaseRatePerSecondInterval(5000) + .build()); + } + + public static void main(String[] args) throws Throwable + { + computeWorstLatencies(); + + DistributedTestBase.beforeClass(); + AccordYcsbLoadTest test = new AccordYcsbLoadTest(); + try + { + test.setupCluster(); + test.setup(); + test.testLoad(withArtificialLatencies(ycsbA(new LoadSettings.Builder(), 100_000) + .setRatePerSecond(1600).setMinRatePerSecond(200) + .setIncreaseRatePerSecondInterval(5000) + ).build()); + } + finally + { + test.tearDown(); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java new file mode 100644 index 0000000000..004df82941 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntSupplier; +import java.util.function.LongFunction; +import java.util.function.Supplier; + +import org.apache.commons.math3.distribution.ZipfDistribution; +import org.apache.commons.math3.random.JDKRandomGenerator; + +import accord.utils.DefaultRandom; + +import static java.util.concurrent.TimeUnit.SECONDS; + +class LoadSettings +{ + enum ClusterChaos { + RESTART, + RESTART_AND_REBOOTSTRAP_INCOMPLETE, RESTART_AND_REBOOTSTRAP_RESET, + RESTART_AND_REBOOTSTRAP_AFTER_TIMEOUT, + + REBOOTSTRAP_INCOMPLETE, REBOOTSTRAP_RESET, + REBOOTSTRAP_IF_BEHIND + } + + final int repairInterval; + final int compactionInterval; + final int journalFlushInterval; + final int cfkFlushInterval; + final int cfkCompactionPeriodSeconds; + final int dataFlushInterval; + final int clusterChaosInterval; + final int clusterChaosDecay; + final int clusterChaosConcurrency; + final LongFunction> clusterChaos; + final int batchSize; + final long batchPeriodNanos; + final int clientConcurrency; + final int clients; + final int ratePerSecond; + final int minRatePerSecond; + final int increaseRatePerSecondInterval; + final int keysPerOperation; + final float readRatio; + final IntSupplier keySelector; + final boolean readBeforeWrite; + final float traceSlowest; + final int traceLast; + final long totalTransactions; + final int totalClusterChaos; + final int[][] artificialLatencies; + + LoadSettings(Builder builder) + { + this.repairInterval = builder.repairInterval; + this.compactionInterval = builder.compactionInterval; + this.journalFlushInterval = builder.journalFlushInterval; + this.cfkFlushInterval = builder.cfkFlushInterval; + this.cfkCompactionPeriodSeconds = builder.cfkCompactionPeriodSeconds; + this.dataFlushInterval = builder.dataFlushInterval; + this.clusterChaosInterval = builder.clusterChaosInterval; + this.clusterChaosDecay = builder.clusterChaosDecay; + this.clusterChaosConcurrency = builder.clusterChaosConcurrency; + this.clusterChaos = builder.clusterChaos; + this.batchSize = builder.batchSize; + this.batchPeriodNanos = builder.batchPeriodNanos; + this.clientConcurrency = builder.clientConcurrency; + this.clients = builder.clients; + this.ratePerSecond = builder.ratePerSecond; + this.minRatePerSecond = builder.minRatePerSecond; + this.increaseRatePerSecondInterval = builder.increaseRatePerSecondInterval; + this.keysPerOperation = builder.keysPerOperation; + this.readRatio = builder.readRatio; + this.keySelector = builder.keySelector; + this.readBeforeWrite = builder.readBeforeWrite; + this.artificialLatencies = builder.artificialLatencies; + this.traceSlowest = builder.traceSlowest; + this.traceLast = builder.traceLast; + this.totalTransactions = builder.totalTransactions; + this.totalClusterChaos = builder.totalClusterChaos; + } + + // interval is measured in terms of *operations* unless otherwise specified + public static class Builder + { + int repairInterval = Integer.MAX_VALUE; + int compactionInterval = Integer.MAX_VALUE; + int journalFlushInterval = Integer.MAX_VALUE; + int cfkFlushInterval = Integer.MAX_VALUE; + int cfkCompactionPeriodSeconds = 0; + int dataFlushInterval = Integer.MAX_VALUE; + int clusterChaosInterval = Integer.MAX_VALUE; + int clusterChaosDecay = 1; + int clusterChaosConcurrency = 1; + LongFunction> clusterChaos = seed -> new DefaultRandom(seed).randomWeightedPicker(LoadSettings.ClusterChaos.values()); + int batchSize = 1000; + long batchPeriodNanos = SECONDS.toNanos(10); + int clientConcurrency = 50; + int clients = -1; + int ratePerSecond = 1000; + int minRatePerSecond = 50; + int increaseRatePerSecondInterval = 1000; + int keysPerOperation = 1; + float readRatio = 0.5f; + IntSupplier keySelector; + boolean readBeforeWrite; + float traceSlowest; + int traceLast; + int[][] artificialLatencies; + long totalTransactions = Long.MAX_VALUE; + int totalClusterChaos = Integer.MAX_VALUE; + + public Builder setRepairInterval(int repairInterval) + { + this.repairInterval = repairInterval; + return this; + } + + public Builder setCompactionInterval(int compactionInterval) + { + this.compactionInterval = compactionInterval; + return this; + } + + public Builder setJournalFlushInterval(int journalFlushInterval) + { + this.journalFlushInterval = journalFlushInterval; + return this; + } + + public Builder setCfkFlushInterval(int cfkFlushInterval) + { + this.cfkFlushInterval = cfkFlushInterval; + return this; + } + + public Builder setCfkCompactionPeriodSeconds(int cfkCompactionPeriodSeconds) + { + this.cfkCompactionPeriodSeconds = cfkCompactionPeriodSeconds; + return this; + } + + public Builder setDataFlushInterval(int dataFlushInterval) + { + this.dataFlushInterval = dataFlushInterval; + return this; + } + + public Builder setClusterChaosInterval(int clusterChaosInterval) + { + this.clusterChaosInterval = clusterChaosInterval; + return this; + } + + public Builder setClusterChaosDecay(int clusterChaosDecay) + { + this.clusterChaosDecay = clusterChaosDecay; + return this; + } + + public Builder setClusterChaosConcurrency(int clusterChaosConcurrency) + { + this.clusterChaosConcurrency = clusterChaosConcurrency; + return this; + } + + public Builder setClusterChaos(LongFunction> clusterChaos) + { + this.clusterChaos = clusterChaos; + return this; + } + + public Builder setTotalTransactions(long totalTransactions) + { + this.totalTransactions = totalTransactions; + return this; + } + + public Builder setTotalClusterChaos(int totalClusterChaos) + { + this.totalClusterChaos = totalClusterChaos; + return this; + } + + public Builder setBatchSize(int batchSize) + { + this.batchSize = batchSize; + return this; + } + + public Builder setBatchPeriodNanos(long batchPeriodNanos) + { + this.batchPeriodNanos = batchPeriodNanos; + return this; + } + + public Builder setClientConcurrency(int clientConcurrency) + { + this.clientConcurrency = clientConcurrency; + return this; + } + + public Builder setClients(int clients) + { + this.clients = clients; + return this; + } + + public Builder setRatePerSecond(int ratePerSecond) + { + this.ratePerSecond = ratePerSecond; + return this; + } + + public Builder setMinRatePerSecond(int minRatePerSecond) + { + this.minRatePerSecond = minRatePerSecond; + return this; + } + + public Builder setIncreaseRatePerSecondInterval(int increaseRatePerSecondInterval) + { + this.increaseRatePerSecondInterval = increaseRatePerSecondInterval; + return this; + } + + public Builder setKeysPerOperation(int keysPerOperation) + { + this.keysPerOperation = keysPerOperation; + return this; + } + + public Builder setReadRatio(float readRatio) + { + this.readRatio = readRatio; + return this; + } + + public Builder setReadBeforeWrite(boolean readBeforeWrite) + { + this.readBeforeWrite = readBeforeWrite; + return this; + } + + public Builder setTraceSlowest(float traceSlowest) + { + this.traceSlowest = traceSlowest; + return this; + } + + public Builder setTraceLast(int traceLast) + { + this.traceLast = traceLast; + return this; + } + + public Builder setKeySelector(IntSupplier keySelector) + { + this.keySelector = keySelector; + return this; + } + + public Builder setArtificialLatencies(int[][] artificialLatencies) + { + this.artificialLatencies = artificialLatencies; + return this; + } + + public LoadSettings build() + { + return new LoadSettings(this); + } + } + + static IntSupplier ycsbZipfian(int keyCount) + { + ZipfDistribution distribution = new ZipfDistribution(new JDKRandomGenerator(), keyCount, 0.99); + int count = distribution.inverseCumulativeProbability(0.65f); + float[] probs = new float[count]; + for (int i = 0 ; i < probs.length ; ++i) + probs[i] = (float) distribution.cumulativeProbability(i); + // zipf is slow to compute, so we cache the first 65% of the distribution then use uniform probability; this is good enough for our purposes + float max = probs[probs.length - 1]; + float inv_incr = probs.length >= keyCount ? 0f : 1f / ((1f-max)/(keyCount - probs.length)); + Random random = new Random(); + return () -> { + float v = random.nextFloat(); + if (v < max) + { + int i = Arrays.binarySearch(probs, v); + if (i < 0) i = -1 - i; + return i; + } + else + { + return (int)((v - max)*inv_incr); + } + }; + } + + static IntSupplier roundrobin(int keyCount) + { + AtomicInteger next = new AtomicInteger(); + return () -> { + int v = next.incrementAndGet(); + if (v < keyCount) + return v; + return next.updateAndGet(i -> i > keyCount ? 0 : i + 1); + }; + } + + static IntSupplier uniform(int keyCount) + { + Random random = new Random(); + return () -> random.nextInt(keyCount); + } +} diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordCatchupTest.java similarity index 99% rename from test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java rename to test/distributed/org/apache/cassandra/fuzz/topology/AccordCatchupTest.java index 746d3b5bb7..6f2135b794 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordCatchupTest.java @@ -46,7 +46,7 @@ import org.apache.cassandra.service.consensus.TransactionalMode; import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce; import static org.apache.cassandra.harry.checker.TestHelper.withRandom; -public class AccordHardCatchupTest extends FuzzTestBase +public class AccordCatchupTest extends FuzzTestBase { private static final int WRITES = 10; private static final int POPULATION = 1000; @@ -117,7 +117,7 @@ public class AccordHardCatchupTest extends FuzzTestBase writeAndValidate.run(); history.customThrowing(() -> { - cluster.get(2).config().set("accord.catchup_on_start", "HARD"); + cluster.get(2).config().set("accord.catchup_on_start", "true"); cluster.get(2).startup(); cluster.get(2).logs().watchFor(".*Catchup.*"); cluster.get(1).logs().watchFor("/127.0.0.2:.* is now UP"); diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java index d5a8f79c63..04121c55d4 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java @@ -65,7 +65,7 @@ public class AccordRebootstrapTest extends FuzzTestBase try (Cluster cluster = builder.withNodes(3) .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(100)) .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(100, "dc0", "rack0")) - .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP)) + .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP).set("accord.catchup_on_start_fail_latency", "60s")) .start()) { IInvokableInstance cmsInstance = cluster.get(1); @@ -74,7 +74,7 @@ public class AccordRebootstrapTest extends FuzzTestBase HashSet downInstances = new HashSet<>(); AtomicInteger nextId = new AtomicInteger(); withRandom(rng -> { - Generator schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bootstrap_fuzz" + (nextId.incrementAndGet()), POPULATION, + Generator schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bsfuzz" + (nextId.incrementAndGet()), POPULATION, SchemaSpec.optionsBuilder() .addWriteTimestamps(false) .withTransactionalMode(TransactionalMode.full) diff --git a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java index de181ef688..e1a50740ef 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java +++ b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java @@ -401,10 +401,13 @@ public class ActionSchedule implements CloseableIterator, LongConsumer advance(scheduled.poll()); } - Action perform = runnable.poll(); + Action perform = runnableByDeadline.peek(); if (perform == null) throw new NoSuchElementException(); + if (perform.deadline() < now - currentJitter) runnable.remove(perform); + else perform = runnable.poll(); + if (!runnableByDeadline.remove(perform) && perform.deadline() > 0) throw new IllegalStateException(); time.tick(perform.deadline()); diff --git a/test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java b/test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java new file mode 100644 index 0000000000..2d1a68a0d2 --- /dev/null +++ b/test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.LockSupport; +import java.util.function.LongSupplier; +import java.util.function.ToLongFunction; + +import org.junit.Test; + +import accord.api.AsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; +import accord.api.ProgressLog; +import accord.api.Result; +import accord.api.RoutingKey; +import accord.api.Scheduler; +import accord.coordinate.Coordinations; +import accord.impl.DefaultLocalListeners; +import accord.impl.TestAgent; +import accord.impl.DefaultLocalListeners.NotifySink; +import accord.impl.DefaultRemoteListeners; +import accord.impl.basic.InMemoryJournal; +import accord.local.CommandStores.RangesForEpoch; +import accord.local.DurableBefore; +import accord.local.TimeService; +import accord.local.durability.DurabilityService; +import accord.local.ExecutionContext; +import accord.local.LoadKeys; +import accord.local.LoadKeysFor; +import accord.local.Node.Id; +import accord.local.NodeCommandStoreService; +import accord.local.SafeCommandStore; +import accord.local.SafeState; +import accord.primitives.Ballot; +import accord.primitives.Range; +import accord.primitives.Ranges; +import accord.primitives.Route; +import accord.primitives.RoutingKeys; +import accord.primitives.Timestamp; +import accord.primitives.TxnId; +import accord.primitives.Writes; +import accord.topology.TopologyManager; +import accord.utils.DefaultRandom; +import accord.utils.Invariants; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.TokenRange; +import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.simulator.test.SimulationTestBase; +import org.apache.cassandra.utils.concurrent.CountDownLatch; +import org.apache.cassandra.utils.concurrent.Future; + +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; + +/** + * Simulator driven test of {@link SafeTask} execution on a real {@link AccordCommandStore}, i.e. of the parts of the + * executor {@link AccordExecutorTest} cannot reach: cache references, the per-key/per-txnId queues, and the ordering + * guarantees they provide. + * + *

The command store is real, but everything below it is synthetic: an in-memory journal, and cache load functions + * that return {@code null} (an uninitialised value) rather than reading from {@code system_accord}. So no schema, no + * cluster metadata, no commit log - only the executor and task machinery is under test. + * + *

Scope (phase 1)

+ * A single executor and command store; {@code SYNC} key-domain contexts; no nesting, no cancellation, no failures. + * Verifies: + *
    + *
  • liveness: every submission is notified exactly once, and successfully;
    • + *
  • declared access: while running, a task holds a reference for every key and txnId it declared;
    • + *
  • mutual exclusion: two tasks that declare the same key or txnId never execute concurrently - the + * central guarantee of the cache-entry queues;
    • + *
  • no leaks: once everything has been notified, no cache entry is still referenced and no task retains + * its references.
    • + *
+ * + *

Notes

+ *
    + *
  • only the {@code SIGNAL} and {@code ASYNC} submission models can be simulated; {@code SYNC}/{@code SEMI_SYNC} + * (the production default) need the simulator to intercept {@link java.util.concurrent.locks.ReentrantLock}, + * see the TODO in {@link AccordExecutorTest};
    • + *
  • only one test method may be run per JVM (pre-existing limitation of {@link AccordExecutorTest}); ant forks + * per test method.
    • + *
+ */ +public class AccordCommandStoreExecutorTest extends SimulationTestBase +{ + private static final int KEYS = 8; + private static final int TXN_IDS = 8; + private static final int SUBMIT_THREADS = 8; + private static final int OUTER_LOOP = 5; + private static final int INNER_LOOP = 10; + private static final int MAX_KEYS_PER_TASK = 3; + private static final int MAX_TXN_IDS_PER_TASK = 2; + private static final int MAX_TASKS = SUBMIT_THREADS * OUTER_LOOP * INNER_LOOP; + + @Test + public void signalLoopTest() + { + test(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, 4, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new RecordingAgent())); + } + + @Test + public void asyncSubmitTest() + { + test(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, 4, i -> "Loop" + i, new RecordingAgent())); + } + + private void test(SerializableSupplier executorSupplier) + { + simulate(arr(() -> { + try + { + DatabaseDescriptor.daemonInitialization(); + AccordExecutor executor = executorSupplier.get(); + Env env = new Env(executor); + + for (float loadDelayChance : new float[]{ 0f, 0.1f }) + { + for (float sleepChance : new float[]{ 0f, 0.1f }) + { + System.out.printf("loadDelayChance %.2f, sleepChance %.2f%n", loadDelayChance, sleepChance); + env.loadDelayChance = loadDelayChance; + env.round(sleepChance); + } + } + } + catch (Throwable t) + { + throw new RuntimeException(t); + } + }), + () -> {}, 1L); + } + + /** + * Records everything reported to the agent: on these paths any report indicates a broken internal invariant, + * not a failed operation. We do not use {@link AccordAgent}, as reporting an exception there touches + * {@code AccordSystemMetrics}, which requires a started {@code AccordService}. + */ + public static class RecordingAgent extends TestAgent + { + static final List exceptions = new CopyOnWriteArrayList<>(); + + @Override + public void onException(Throwable t) + { + exceptions.add(t); + System.out.println("### agent.onException: " + t); + t.printStackTrace(System.out); + } + + @Override + public void onException(Throwable t, String context) + { + onException(t); + } + } + + static class Env + { + final AccordExecutor executor; + final AccordCommandStore store; + final RoutingKey[] keys = new RoutingKey[KEYS]; + final TxnId[] txnIds = new TxnId[TXN_IDS]; + + /** key/txnId ordinal -> id of the task currently executing with it, or -1 */ + final AtomicIntegerArray keyOwner = new AtomicIntegerArray(KEYS); + final AtomicIntegerArray txnIdOwner = new AtomicIntegerArray(TXN_IDS); + + /** reset for every round: task ids are round-local */ + AtomicInteger nextTaskId = new AtomicInteger(); + /** task id -> number of times its callback has been invoked; must be exactly one for every submission */ + AtomicIntegerArray notifications = new AtomicIntegerArray(MAX_TASKS); + final List failures = new CopyOnWriteArrayList<>(); + SafeTask[] tasks = new SafeTask[MAX_TASKS]; + + volatile float loadDelayChance; + + Env(AccordExecutor executor) + { + this.executor = executor; + TableId tableId = TableId.fromUUID(new java.util.UUID(0, 1)); + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + this.store = newCommandStore(tableId, partitioner, executor); + for (int i = 0 ; i < KEYS ; ++i) + keys[i] = new TokenKey(tableId, partitioner.getToken(Int32Type.instance.decompose(i))); + for (int i = 0 ; i < TXN_IDS ; ++i) + txnIds[i] = TxnId.fromValues(1, 1 + i, 0, new Id(1)); + for (int i = 0 ; i < KEYS ; ++i) + keyOwner.set(i, -1); + for (int i = 0 ; i < TXN_IDS ; ++i) + txnIdOwner.set(i, -1); + + // synthetic loads: an uninitialised value, optionally after some simulated latency + executor.cacheUnsafe().types().forEach(type -> type.unsafeSetLoadFunction((ignoreStore, ignoreKey) -> { + maybePark(loadDelayChance); + return null; + })); + } + + void round(float sleepChance) throws ExecutionException, InterruptedException + { + nextTaskId = new AtomicInteger(); + notifications = new AtomicIntegerArray(MAX_TASKS); + tasks = new SafeTask[MAX_TASKS]; + ExecutorPlus submit = executorFactory().pooled("submit", SUBMIT_THREADS); + try + { + List> submitting = new ArrayList<>(); + for (int i = 0 ; i < SUBMIT_THREADS ; ++i) + { + int id = i; + submitting.add(submit.submit(() -> { + for (int outer = 0 ; outer < OUTER_LOOP ; ++outer) + { + CountDownLatch inner = CountDownLatch.newCountDownLatch(INNER_LOOP); + for (int j = 0 ; j < INNER_LOOP ; ++j) + submitOne(sleepChance, inner); + inner.awaitUninterruptibly(); + System.out.println("Loop " + id + '.' + outer); + } + })); + } + for (Future f : submitting) + f.get(); + } + finally + { + submit.shutdown(); + } + + verifyRoundComplete(); + } + + private void submitOne(float sleepChance, CountDownLatch done) + { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + int taskId = nextTaskId.getAndIncrement(); + int[] keyOrdinals = distinct(rnd, KEYS, 1 + rnd.nextInt(MAX_KEYS_PER_TASK)); + int[] txnIdOrdinals = distinct(rnd, TXN_IDS, rnd.nextInt(1 + MAX_TXN_IDS_PER_TASK)); + + RoutingKeys declaredKeys = keys(keyOrdinals); + TxnId primary = txnIdOrdinals.length > 0 ? txnIds[txnIdOrdinals[0]] : null; + TxnId additional = txnIdOrdinals.length > 1 ? txnIds[txnIdOrdinals[1]] : null; + ExecutionContext context = ExecutionContext.contextFor(primary, additional, declaredKeys, LoadKeys.SYNC, LoadKeysFor.READ_WRITE, "task" + taskId); + + Cancellable submitted = + store.execute(context, (java.util.function.Consumer) safeStore -> body(taskId, keyOrdinals, txnIdOrdinals, safeStore, sleepChance), + (success, fail) -> { + notifications.incrementAndGet(taskId); + if (fail != null) + failures.add(fail); + done.decrement(); + }); + tasks[taskId] = (SafeTask) submitted; + } + + private void body(int taskId, int[] keyOrdinals, int[] txnIdOrdinals, SafeCommandStore safeStore, float sleepChance) + { + SafeTask task = ((SaferCommandStore) safeStore).task; + + // declared access: we must hold a reference for everything we declared, and be able to look it up + for (int k : keyOrdinals) + { + SafeState ref = task.refs.get(keys[k]); + Invariants.require(ref != null, "task %d declared key %d but holds no reference for it", taskId, k); + Invariants.require(SaferState.global(ref).references() > 0, "task %d holds an unreferenced entry for key %d", taskId, k); + safeStore.ifLoadedAndInitialised(keys[k]); + } + for (int t : txnIdOrdinals) + { + SafeState ref = task.refs.get(txnIds[t]); + Invariants.require(ref != null, "task %d declared txnId %d but holds no reference for it", taskId, t); + Invariants.require(SaferState.global(ref).references() > 0, "task %d holds an unreferenced entry for txnId %d", taskId, t); + safeStore.ifInitialised(txnIds[t]); + } + + // mutual exclusion: nobody else may be executing with any key or txnId we declared + int keysTaken = 0, txnIdsTaken = 0; + try + { + while (keysTaken < keyOrdinals.length) + { + int k = keyOrdinals[keysTaken]; + Invariants.require(keyOwner.compareAndSet(k, -1, taskId), + "task %d ran concurrently with task %d, which also declared key %d", taskId, keyOwner.get(k), k); + ++keysTaken; + } + while (txnIdsTaken < txnIdOrdinals.length) + { + int t = txnIdOrdinals[txnIdsTaken]; + Invariants.require(txnIdOwner.compareAndSet(t, -1, taskId), + "task %d ran concurrently with task %d, which also declared txnId %d", taskId, txnIdOwner.get(t), t); + ++txnIdsTaken; + } + + maybePark(sleepChance); + } + finally + { + while (keysTaken > 0) + keyOwner.set(keyOrdinals[--keysTaken], -1); + while (txnIdsTaken > 0) + txnIdOwner.set(txnIdOrdinals[--txnIdsTaken], -1); + } + } + + private void verifyRoundComplete() + { + Invariants.require(failures.isEmpty(), "%s", failures); + Invariants.require(RecordingAgent.exceptions.isEmpty(), "%s", RecordingAgent.exceptions); + + int lastTaskId = nextTaskId.get(); + Invariants.require(lastTaskId == MAX_TASKS, "submitted %d tasks, expected %d", lastTaskId, MAX_TASKS); + for (int taskId = 0 ; taskId < lastTaskId ; ++taskId) + { + Invariants.require(notifications.get(taskId) == 1, "task %d was notified %d times", taskId, notifications.get(taskId)); + Invariants.require(tasks[taskId].refs == null, "task %d did not release its references", taskId); + } + for (int i = 0 ; i < KEYS ; ++i) + Invariants.require(keyOwner.get(i) == -1, "key %d is still owned by task %d", i, keyOwner.get(i)); + for (int i = 0 ; i < TXN_IDS ; ++i) + Invariants.require(txnIdOwner.get(i) == -1, "txnId %d is still owned by task %d", i, txnIdOwner.get(i)); + + try (AccordCommandStore.ExclusiveCaches caches = store.lockCaches()) + { + for (AccordCacheEntry entry : caches.commands()) + Invariants.require(entry.references() == 0, "%s is still referenced", entry); + for (AccordCacheEntry entry : caches.commandsForKeys()) + Invariants.require(entry.references() == 0, "%s is still referenced", entry); + } + } + + private RoutingKeys keys(int[] ordinals) + { + RoutingKey[] result = new RoutingKey[ordinals.length]; + for (int i = 0 ; i < ordinals.length ; ++i) + result[i] = keys[ordinals[i]]; + return RoutingKeys.of(result); + } + } + + private static void maybePark(float chance) + { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + if (chance > 0 && rnd.nextFloat() < chance) + LockSupport.parkNanos(rnd.nextInt(10000, 100000)); + } + + /** {@code count} distinct ordinals in [0..limit), in ascending order (so a task never declares a key twice) */ + private static int[] distinct(ThreadLocalRandom rnd, int limit, int count) + { + int mask = 0; + for (int i = 0 ; i < count ; ++i) + mask |= 1 << rnd.nextInt(limit); + int[] result = new int[Integer.bitCount(mask)]; + for (int i = 0, ordinal = 0 ; mask != 0 ; ++ordinal, mask >>>= 1) + { + if ((mask & 1) != 0) + result[i++] = ordinal; + } + return result; + } + + private static AccordCommandStore newCommandStore(TableId tableId, IPartitioner partitioner, AccordExecutor executor) + { + AtomicLong clock = new AtomicLong(); + LongSupplier now = clock::incrementAndGet; + Id nodeId = new Id(1); + NodeCommandStoreService node = new NodeCommandStoreService() + { + private final ToLongFunction elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MICROSECONDS, this::now); + private long stamp = 0; + + @Override public AsyncExecutor someExecutor() { return null; } + @Override public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } + @Override public accord.api.Timeouts timeouts() { return null; } + @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; } + @Override public DurabilityService durability() { return null; } + @Override public Id id() { return nodeId; } + @Override public long epoch() { return 1; } + @Override public long now() { return now.getAsLong(); } + @Override public long uniqueNow(long atLeast) { return now.getAsLong(); } + @Override public long elapsed(TimeUnit units) { return elapsed.applyAsLong(units); } + @Override public TopologyManager topology() { throw new UnsupportedOperationException(); } + @Override public Coordinations coordinations() { return new Coordinations(); } + @Override public Scheduler scheduler() { return null; } + @Override public long currentStamp() { return stamp; } + @Override public void updateStamp() { ++stamp; } + @Override public boolean isReplaying() { return false; } + @Override public void reportLocalExecution(TxnId txnId, Route route, Ballot ballot, Timestamp applyAt, Writes writes, Result result) {} + }; + + Range range = TokenRange.fullRange(tableId, partitioner); + RangesForEpoch rangesForEpoch = new RangesForEpoch(1, Ranges.of(range)); + AccordCommandStore store = new AccordCommandStore(0, node, new RecordingAgent(), null, + cs -> new ProgressLog.NoOpProgressLog(), + cs -> new DefaultLocalListeners(null, new DefaultRemoteListeners.NoOpRemoteListeners(), new NotifySink.NoOpNotifySink()), + rangesForEpoch, + new InMemoryJournal(nodeId, new DefaultRandom(1)), + executor); + executor.executeDirectlyWithLock(() -> { + executor.setCapacity(8 << 20); + executor.setWorkingSetSize(4 << 20); + }); + return store; + } +} diff --git a/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java b/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java index fcdc7deb41..f6262a7db2 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java @@ -45,8 +45,8 @@ public class AbstractPairOfSequencesPaxosSimulationTest "\tat accord.impl.RequestCallbacks$CallbackStripe$RegisteredCallback.lambda$safeInvoke$0(RequestCallbacks.java:140)\n" + "\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n" + "\tat accord.utils.async.AsyncChains.lambda$encapsulate$0(AsyncChains.java:498)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + "\tat org.apache.cassandra.service.accord.AccordExecutorSimple.run(AccordExecutorSimple.java:95)\n" + "\tat org.apache.cassandra.concurrent.FutureTask$2.call(FutureTask.java:124)\n" + "\tat org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)\n" + @@ -77,8 +77,8 @@ public class AbstractPairOfSequencesPaxosSimulationTest "\tat accord.impl.RequestCallbacks$CallbackStripe$RegisteredCallback.lambda$safeInvoke$0(RequestCallbacks.java:140)\n" + "\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n" + "\tat accord.utils.async.AsyncChains.lambda$encapsulate$0(AsyncChains.java:498)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + "\tat org.apache.cassandra.service.accord.AccordExecutorSimple.run(AccordExecutorSimple.java:95)\n" + "\tat org.apache.cassandra.concurrent.FutureTask$2.call(FutureTask.java:124)\n" + "\tat org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)\n" + diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index 44888830cc..e075a0da7c 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -18,7 +18,11 @@ package org.apache.cassandra.simulator.test; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -26,64 +30,277 @@ import java.util.concurrent.Future; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.LockSupport; import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +import javax.annotation.Nullable; import org.junit.Test; import accord.api.AsyncExecutor; -import accord.local.SequentialAsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; +import accord.utils.async.Cancellable; -import org.apache.cassandra.concurrent.ExecutorFactory; +import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; -import org.apache.cassandra.service.accord.AccordExecutor; -import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; -import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutorAsyncSubmit; +import org.apache.cassandra.service.accord.execution.AccordExecutorSignalLoop; import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.simulator.test.AccordExecutorTest.Submitted.Consequences; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.SignalLock; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; -import static org.apache.cassandra.service.accord.AccordService.toFuture; +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; // TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit public class AccordExecutorTest extends SimulationTestBase { - static final int THREAD_COUNT = 8; - @Test public void signalLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, THREAD_COUNT, -1, -1, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent())); + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new AccordAgent()), + 16); } @Test public void signalSpinLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, THREAD_COUNT, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent())); + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), + 16); } @Test public void ayncSubmitTest() { - executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, THREAD_COUNT, i -> "Loop" + i, new AccordAgent())); + int threads = 32; + executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, threads, i -> "Loop" + i, new AccordAgent()), + 16); } - public void executorTest(SerializableSupplier supplier) + static class Submitted + { + final AtomicInteger nextId = new AtomicInteger(); + final AtomicInteger doneBefore = new AtomicInteger(); + final ConcurrentHashMap consequences = new ConcurrentHashMap<>(); + + boolean isDone() + { + return isDoneBetween(0, nextId.get()); + } + + boolean isDoneBefore(int before) + { + if (!isDoneBetween(doneBefore.get(), before)) + return false; + doneBefore.accumulateAndGet(before, Integer::max); + return true; + } + + boolean isDoneBetween(int from, int before) + { + for (int id = from ; id < before ; ++id) + { + if (!consequences.get(id).isDone()) + return false; + } + return true; + } + + class Consequences extends ConcurrentLinkedQueue> + { + volatile boolean started; + + /** + * The number of submissions in this group that have not yet invoked their callback. + *

+ * We cannot decide this by iterating the queue: each level of the recursion appends the next level's + * future while it runs, i.e. strictly before its own future completes, so the queue is still growing + * while it is being consumed - and {@link ConcurrentLinkedQueue}'s iterator prefetches, so an iterator + * that has reached the tail terminates and never sees the later additions. A counter is exact, because + * for the same reason it can only reach zero once the whole group has finished: a consequence is + * registered before its parent's future completes. + */ + final AtomicInteger outstanding = new AtomicInteger(); + + void submitted(Future future) + { + outstanding.incrementAndGet(); + add(future); + } + + void completed() + { + outstanding.decrementAndGet(); + } + + boolean isDone() + { + return outstanding.get() == 0; + } + + void ensureStarted() + { + if (started) + return; + + synchronized (this) + { + if (started) + return; + + while (true) + { + int id = nextId.get(); + Object prev = consequences.putIfAbsent(id, this); + nextId.compareAndSet(id, id + 1); + if (prev == null) + break; + } + + started = true; + } + } + } + + Consequences allocate() + { + return new Consequences(); + } + } + + static class Control extends ConcurrentLinkedQueue + { + final Submitted submitted; + final AtomicInteger count = new AtomicInteger(); + final float cancelChance; + float processChance; + + Control(float cancelChance, Submitted submitted) + { + this(submitted, cancelChance, ThreadLocalRandom.current().nextFloat() * 0.5f); + } + + Control(Submitted submitted, float cancelChance, float processChance) + { + this.submitted = submitted; + this.cancelChance = cancelChance; + this.processChance = processChance; + } + + void submit(AsyncExecutor executor, Consequences consequences, Consumer run) + { + AsyncPromise future = new AsyncPromise<>(); + consequences.submitted(future); + Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { + try + { + if (fail == null) future.trySuccess(null); + else + { + future.tryFailure(fail); + // replace the cancelled work with an equivalent submission, so that cancelling does not + // simply reduce the amount of work we perform. + // + // NOTE: this callback may be invoked from inside the executor (e.g. while cancelling, which + // happens with the executor's lock held), so we must not run the body here: it re-enters the + // executor (afterSubmittedAndConsequences, and the lock itself), which is forbidden for a + // thread that already holds it. Submit it as ordinary work instead, so it runs in a legal + // context - this also widens the set of interleavings we explore. + if (fail instanceof CancellationException) + submit(executor, submitted.allocate(), run); + } + } + finally + { + consequences.completed(); + } + }); + consequences.ensureStarted(); + + if (cancel != null && ThreadLocalRandom.current().nextFloat() <= cancelChance) + { + add(cancel); + count.incrementAndGet(); + } + + if (count.get() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance) + { + int cancelCount = 0; + do + { + ++cancelCount; + + float delta = ThreadLocalRandom.current().nextFloat() - 0.5f; + if (delta < 0) processChance /= delta; + else processChance *= -delta; + if (processChance < 0.001f || processChance > 0.999f) + processChance = cancelChance; + } while (count.decrementAndGet() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance); + + // do outside of loop to avoid reentry + while (cancelCount-- > 0) + remove().cancel(); + } + } + } + + public void executorTest(SerializableSupplier supplier, int submissionThreads) { simulate(arr(() -> { try { DatabaseDescriptor.daemonInitialization(); + ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); AccordExecutor executor = supplier.get(); Lock lock = executor.unsafeLock(); - SequentialAsyncExecutor sequentialExecutor = executor.newSequentialExecutor(); - Executor lockExecutor = ExecutorFactory.Global.executorFactory().sequential("lock"); - ConcurrentLinkedQueue> await = new ConcurrentLinkedQueue<>(); + ExclusiveAsyncExecutor sequentialExecutor = executor.newExclusiveExecutor(); + Executor lockExecutor = executorFactory().sequential("lock"); for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) + { for (float lockChance : new float[] { 0f, 0.01f, 0.1f }) - submitLoop(lock, executor, sequentialExecutor, lockExecutor, 200, 10, await, sleepChance, lockChance); + { + for (float cancelChance : new float[] { 0f, 0.01f, 0.1f }) + { + System.out.println(String.format("sleepChance %.2f, lockChance %.2f, cancelChance %.2f", sleepChance, lockChance, cancelChance)); + List> done = new ArrayList<>(); + Submitted submitted = new Submitted(); + for (int i = 0; i < submissionThreads; ++i) + { + int id = i; + done.add(submit.submit(() -> { + try + { + submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance, new Control(cancelChance, submitted), submitted); + } + catch (ExecutionException | InterruptedException e) + { + throw new RuntimeException(e); + } + })); + } + for (Future f : done) + f.get(); + + // the awaits in submitLoop only wait for the submissions they can see, and the + // deeper levels of each group are only registered as they run, so work may still + // be in flight here; drain the executor before verifying that everything we + // recorded has in fact run + executor.waitForQuiescence(); + if (!submitted.isDone()) + throw new AssertionError(); + // nothing is running, so we can now safely inspect every future we recorded + for (int id = 0 ; id < submitted.nextId.get() ; ++id) + await(submitted.consequences.get(id), CancellationException.class); + } + } + } } catch (Throwable t) { @@ -93,26 +310,34 @@ public class AccordExecutorTest extends SimulationTestBase () -> {}, 1L); } - private static void submitLoop(Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, ConcurrentLinkedQueue> await, float sleepChance, float lockChance) throws ExecutionException, InterruptedException + private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance, Control control, Submitted submitted) throws ExecutionException, InterruptedException { + ConcurrentLinkedQueue> awaitConsequences = new ConcurrentLinkedQueue<>(); while (outerLoop-- > 0) { + List>> allAwaitSubmitted = new ArrayList<>(); for (int i = 0; i < innerLoop; ++i) - submitRecursive(lock, executor, sequentialExecutor, 1 + i, await, sleepChance, lockChance); + { + Consequences awaitSubmitted = submitted.allocate(); + allAwaitSubmitted.add(awaitSubmitted); + submitRecursive(lock, executor, sequentialExecutor, 1 + i, awaitSubmitted, awaitConsequences, submitted, sleepChance, lockChance, control); + } AtomicBoolean done = new AtomicBoolean(); submitUntil(lock, lockExecutor, sleepChance, done::get); - while (!await.isEmpty()) - await.poll().get(); + for (Collection> awaitSubmitted : allAwaitSubmitted) + await(awaitSubmitted, CancellationException.class); + await(awaitConsequences, null); done.set(true); - System.out.println("Loop " + (1 + outerLoop)); + System.out.println("Loop " + id + '.' + (1 + outerLoop)); } } - private static void submitRecursive(Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, int count, Collection> await, float sleepChance, float lockChance) + private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Consequences consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) { AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; - await.add(toFuture(submitTo.chain(() -> { + + control.submit(submitTo, consequences, nextConsequences -> { ThreadLocalRandom rnd = ThreadLocalRandom.current(); boolean locked = false; if (rnd.nextFloat() < lockChance) @@ -120,10 +345,21 @@ public class AccordExecutorTest extends SimulationTestBase if (rnd.nextBoolean()) locked = lock.tryLock(); else { locked = true; lock.lock(); } } + if (ThreadLocalRandom.current().nextFloat() < 0.01f) + { + int expectDoneBefore = submitted.nextId.get(); + AsyncPromise afterConsequences = new AsyncPromise<>(); + executor.afterSubmittedAndConsequences(() -> { + if (!submitted.isDoneBefore(expectDoneBefore)) + throw new AssertionError(); + afterConsequences.setSuccess(null); + }); + awaitConsequences.add(afterConsequences); + } try { if (count > 1) - submitRecursive(lock, executor, sequentialExecutor, count -1, await, sleepChance, lockChance); + submitRecursive(lock, executor, sequentialExecutor, count -1, nextConsequences, awaitConsequences, submitted, sleepChance, lockChance, control); if (rnd.nextFloat() < sleepChance) LockSupport.parkNanos(rnd.nextInt(10000, 100000)); } @@ -132,7 +368,7 @@ public class AccordExecutorTest extends SimulationTestBase if (locked) lock.unlock(); } - }).beginAsResult())); + }); } private static void submitUntil(Lock lock, Executor executor, float sleepChance, BooleanSupplier done) @@ -161,4 +397,22 @@ public class AccordExecutorTest extends SimulationTestBase } }); } + + /** + * Waits for those submissions that are visible in {@code await}; note that this deliberately does not wait for + * the whole tree of consequences (which is still being appended to as each level runs), so that submission + * threads continue to overlap their outer loops - {@link Submitted#isDone} is verified once the executor drains. + */ + private static void await(Collection> await, @Nullable Class ignore) throws InterruptedException, ExecutionException + { + for (Future future : await) + { + try { future.get(); } + catch (ExecutionException e) + { + if (ignore == null || !(ignore.isInstance(e.getCause()))) + throw e; + } + } + } } diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index 6f836c0e86..cca5292f81 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -188,7 +188,7 @@ import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.accord.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCache; import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.transport.Event; diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index 4901566aae..f0c32490b9 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -83,16 +83,14 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTestUtils; import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.utils.FBUtilities; -import static accord.local.LoadKeys.SYNC; -import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.PreLoadContext.contextFor; +import static accord.local.ExecutionContext.unsequencedReadWrite; import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE; import static accord.primitives.Routable.Domain.Range; import static accord.primitives.Timestamp.Flag.HLC_BOUND; @@ -287,11 +285,11 @@ public class CompactionAccordIteratorsTest try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { cacheSize = cache.global.capacity(); - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { - cache.global.setCapacity(cacheSize); + commandStore.executor().setCapacity(cacheSize); } commandsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS); while (commandStore.executor().hasTasks()) @@ -322,28 +320,28 @@ public class CompactionAccordIteratorsTest PartialDeps partialDeps = Deps.NONE.intersecting(AccordTestUtils.fullRange(txn)); PartialTxn partialTxn = txn.slice(commandStore.unsafeGetRangesForEpoch().currentRanges(), true); Route partialRoute = route.overlapping(commandStore.unsafeGetRangesForEpoch().currentRanges()); - getBlocking(commandStore.execute(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.execute(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.preaccept(safe, txnId, partialTxn, route, (a, b) -> {}); })); flush(commandStore); - getBlocking(commandStore.execute(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.execute(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, txnId, partialDeps, (a, b) -> {}); })); flush(commandStore); - getBlocking(commandStore.execute(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.execute(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, partialTxn, txnId, partialDeps, (a, b) -> {}); })); flush(commandStore); - getBlocking(commandStore.chain(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.chain(unsequencedReadWrite(txnId, route, "Test"), safe -> { return AccordTestUtils.processTxnResultDirect(safe, txnId, partialTxn, txnId); - }).flatMap(i -> i).flatMap(result -> commandStore.chain(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + }).flatMap(i -> i).flatMap(result -> commandStore.chain(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.apply(safe, txnId, route, txnId, partialDeps, partialTxn, result.left, result.right, (a, b) -> {}); }))); flush(commandStore); // The apply chain is asychronous, so it is easiest to just spin until it is applied // in order to have the updated state in the system table spinAssertEquals(true, 5, () -> { - return getBlocking(commandStore.submit(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + return getBlocking(commandStore.submit(unsequencedReadWrite(txnId, route, "Test"), safe -> { StoreParticipants participants = StoreParticipants.all(route); Command command = safe.get(txnId, participants).current(); return command.hasBeen(Status.Applied); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java index f3d94e7827..92081f8918 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java @@ -18,8 +18,6 @@ package org.apache.cassandra.service.accord; -import java.util.NavigableMap; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; import org.junit.Assert; @@ -32,7 +30,7 @@ import org.slf4j.LoggerFactory; import accord.api.Key; import accord.api.Result.PersistableResult; import accord.local.Command; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.StoreParticipants; import accord.local.cfk.CommandsForKey; import accord.primitives.Ballot; @@ -64,6 +62,9 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SaferCommand; +import org.apache.cassandra.service.accord.execution.SaferCommandsForKey; +import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializerTest.TestSafeCommandStore; import org.apache.cassandra.service.accord.txn.TxnDataResult; import org.apache.cassandra.service.accord.txn.TxnUpdate; @@ -73,14 +74,15 @@ import org.apache.cassandra.utils.Pair; import static accord.primitives.Status.Durability.AllQuorums; import static com.google.common.collect.Iterables.getOnlyElement; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; import static org.apache.cassandra.service.accord.AccordService.getBlocking; import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.preaccepted; import static org.apache.cassandra.service.accord.AccordTestUtils.ballot; import static org.apache.cassandra.service.accord.AccordTestUtils.createAccordCommandStore; import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn; -import static org.apache.cassandra.service.accord.AccordTestUtils.loaded; import static org.apache.cassandra.service.accord.AccordTestUtils.timestamp; import static org.apache.cassandra.service.accord.AccordTestUtils.txnId; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; public class AccordCommandStoreTest { @@ -139,7 +141,8 @@ public class AccordCommandStoreTest Command expected = Command.Executed.executed(txnId, SaveStatus.Applied, AllQuorums, StoreParticipants.all(route), promised, executeAt, txn, dependencies, accepted, waitingOn, result.left, TxnDataResult.PERSISTABLE); - AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null)); + SaferCommand safeCommand = new SaferCommand(loaded(txnId, null)); + safeCommand.preExecute(new SafeTask<>(null, ExecutionContext.unsequenced(txnId, "Test"), null), RELEASE_QUEUE); safeCommand.set(expected); // In practice we should never need to save it with the condition boolean set // Not sure why this test does that @@ -167,24 +170,17 @@ public class AccordCommandStoreTest Command command1 = preaccepted(txnId1, txn, timestamp(1, clock.incrementAndGet(), 1)); Command command2 = preaccepted(txnId2, txn, timestamp(1, clock.incrementAndGet(), 1)); - AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null)); - cfk.initialize(); + SaferCommandsForKey cfk = new SaferCommandsForKey(loaded(key, null)); + cfk.preExecute(new SafeTask<>(null, ExecutionContext.unsequenced(txnId1, "Test"), null), RELEASE_QUEUE); - cfk.set(cfk.current().update(new TestSafeCommandStore(PreLoadContext.contextFor(command1.txnId(), "Test")), command1).cfk()); - cfk.set(cfk.current().update(new TestSafeCommandStore(PreLoadContext.contextFor(command1.txnId(), "Test")), command2).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command1).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command2).cfk()); - CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run(); + CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null).run(); logger.info("E: {}", cfk); CommandsForKey actual = CommandsForKeyAccessor.load(commandStore.id(), key); logger.info("A: {}", actual); Assert.assertEquals(cfk.current(), actual); } - - private static > NavigableMap toNavigableMap(V safeState) - { - TreeMap map = new TreeMap<>(); - map.put(safeState.key(), safeState); - return map; - } } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index 214e17e6d1..539d58b268 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -27,9 +27,9 @@ import org.junit.Test; import accord.api.Key; import accord.api.RoutingKey; import accord.local.Command; +import accord.local.ExecutionContext; import accord.local.LoadKeys; import accord.local.Node; -import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; @@ -98,7 +98,7 @@ public class AccordCommandTest public void basicCycleTest() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((PreLoadContext.Empty)() -> "Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", unused -> commandStore.executor().setCapacity(0))); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(1); @@ -174,7 +174,7 @@ public class AccordCommandTest Commit commit = Commit.SerializerSupport.create(txnId, route, 1, 1, Commit.Kind.StableWithTxnAndDeps, Ballot.ZERO, executeAt, partialTxn, deps, fullRoute); getBlocking(commandStore.execute(commit, commit::apply)); - getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, Keys.of(key).toParticipants(), LoadKeys.SYNC, READ_WRITE, "Test"), safeStore -> { + getBlocking(commandStore.execute(ExecutionContext.unsequencedReadWrite(txnId, Keys.of(key).toParticipants(), "Test"), safeStore -> { Command before = safeStore.ifInitialised(txnId).current(); Assert.assertEquals(commit.executeAt, before.executeAt()); Assert.assertTrue(before.hasBeen(Status.Committed)); @@ -191,7 +191,7 @@ public class AccordCommandTest public void computeDeps() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((PreLoadContext.Empty)()->"Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)()->"Test", unused -> commandStore.executor().setCapacity(0))); TxnId txnId1 = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(2); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java b/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java index 66514033a5..2353fd5ac2 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java @@ -48,6 +48,7 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.journal.TestParams; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.journal.CommandChanges; import org.apache.cassandra.utils.AccordGenerators; @@ -70,7 +71,7 @@ import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.p * *

Mechanism under test: when the FULL-input cleanup path decides EXPUNGE, * {@link accord.impl.CommandChange.Builder#construct} returns {@code null}. - * Downstream, {@link AccordSafeCommand#preExecute} maps {@code null} to a + * Downstream, {@link SaferCommand#preExecute} maps {@code null} to a * {@code Command.NotDefined.uninitialised(...)} — which is exactly the bogus * NotDefined the user observed. The journal load API itself should never collapse * "this txnId has been erased" into the same answer as "we have never heard of this diff --git a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java index b467edce53..d73fdaa62b 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java @@ -70,6 +70,7 @@ import org.apache.cassandra.schema.SchemaProvider; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.utils.CassandraGenerators; import static accord.local.Command.Committed.committed; @@ -82,6 +83,7 @@ import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME; import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor.findAllKeysBetween; import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor.makeSystemTableKeyBytes; import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport; import static org.apache.cassandra.utils.AccordGenerators.fromQT; @@ -120,7 +122,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory Command.Committed committed = committed(id, SaveStatus.Committed, Status.Durability.NotDurable, participants, Ballot.ZERO, id, partialTxn, deps.intersecting(scope), Ballot.ZERO, waitingOn); - AccordSafeCommand safeCommand = new AccordSafeCommand(AccordTestUtils.loaded(id, null)); + SaferCommand safeCommand = new SaferCommand(loaded(id, null)); safeCommand.set(committed); AccordTestUtils.appendCommandsBlocking(store, null, committed); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java index b4af888878..8ab7e56bff 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java @@ -84,7 +84,7 @@ public class AccordMessageSinkTest TxnId id = nextTxnId(epoch, txn); Ranges ranges = Ranges.of(IntKey.range(40, 50)); PartialTxn partialTxn = txn.slice(ranges, true); - Request request = new AccordFetchRequest(epoch, id, ranges, PartialDeps.NONE, partialTxn); + Request request = new AccordFetchRequest(epoch, id, ranges, partialTxn); checkRequestReplies(request, new AbstractFetchCoordinator.FetchResponse(null, null, id), diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index fd2301046a..c5689e813a 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -37,12 +37,14 @@ import org.junit.Assert; import accord.api.AsyncExecutor; import accord.api.Data; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.ProgressLog.NoOpProgressLog; import accord.api.RemoteListeners.NoOpRemoteListeners; import accord.api.Result; import accord.api.Result.PersistableResult; import accord.api.RoutingKey; +import accord.api.Scheduler; import accord.api.Timeouts; import accord.coordinate.Coordinations; import accord.impl.DefaultLocalListeners; @@ -51,12 +53,11 @@ import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.Node; import accord.local.Node.Id; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; import accord.local.SafeCommandStore; -import accord.local.SequentialAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.durability.DurabilityService; @@ -82,11 +83,8 @@ import accord.topology.TopologyManager; import accord.utils.SortedArrays.SortedArrayList; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; -import accord.utils.async.Cancellable; import org.apache.cassandra.ServerTestUtils; -import org.apache.cassandra.concurrent.ExecutorPlus; -import org.apache.cassandra.concurrent.ManualExecutor; import org.apache.cassandra.config.AccordConfig; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; @@ -103,9 +101,12 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; -import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutorSyncSubmit; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; @@ -114,7 +115,6 @@ import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Condition; -import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static accord.primitives.Routable.Domain.Key; @@ -123,8 +123,9 @@ import static accord.primitives.SaveStatus.PreAccepted; import static accord.primitives.Status.Durability.NotDurable; import static accord.primitives.Txn.Kind.Write; import static java.lang.String.format; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.service.accord.AccordService.getBlocking; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; public class AccordTestUtils { @@ -132,7 +133,7 @@ public class AccordTestUtils public static class Commands { - public static Command notDefined(TxnId txnId, PartialTxn txn) + public static Command notDefined(TxnId txnId) { return Command.NotDefined.notDefined(txnId, NotDefined, NotDurable, StoreParticipants.empty(txnId), Ballot.ZERO); } @@ -162,17 +163,10 @@ public class AccordTestUtils } } - public static AccordCacheEntry loaded(K key, V value) + public static SaferCommand safeCommand(Command command) { - AccordCacheEntry global = new AccordCacheEntry<>(key, null); - global.initialize(value); - return global; - } - - public static AccordSafeCommand safeCommand(Command command) - { - AccordCacheEntry global = loaded(command.txnId(), command); - return new AccordSafeCommand(global); + AccordCacheEntry global = loaded(command.txnId(), command); + return new SaferCommand(global); } public static Function testableLoad(K key, V val) @@ -183,39 +177,6 @@ public class AccordTestUtils }; } - private static LoadExecutor loadExecutor(ExecutorPlus executor) - { - return new LoadExecutor<>() - { - @Override - public Cancellable load(P1 p1, P2 p2, AccordCacheEntry entry) - { - Future future = executor.submit(() -> { - V v; - try { v = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); } - catch (Throwable t) - { - entry.failedToLoad(); - throw t; - } - entry.loaded(v); - }); - return () -> future.cancel(true); - } - }; - } - - public static void testLoad(ManualExecutor executor, AccordSafeState safeState, V val) - { - Assert.assertEquals(AccordCacheEntry.Status.WAITING_TO_LOAD, safeState.global().status()); - safeState.global().load(loadExecutor(executor), null, null); - Assert.assertEquals(AccordCacheEntry.Status.LOADING, safeState.global().status()); - executor.runOne(); - Assert.assertEquals(AccordCacheEntry.Status.LOADED, safeState.global().status()); - safeState.preExecute(); - Assert.assertEquals(val, safeState.current()); - } - public static TxnId txnId(long epoch, long hlc, int node) { return txnId(epoch, hlc, node, Write); @@ -244,7 +205,7 @@ public class AccordTestUtils public static AsyncChain> processTxnResult(AccordCommandStore commandStore, TxnId txnId, PartialTxn txn, Timestamp executeAt) throws Throwable { AtomicReference>> result = new AtomicReference<>(); - getBlocking(commandStore.execute((PreLoadContext.Empty)() -> "Test", + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", safeStore -> result.set(processTxnResultDirect(safeStore, txnId, txn, executeAt)))); return result.get(); } @@ -364,21 +325,11 @@ public class AccordTestUtils { NodeCommandStoreService time = new NodeCommandStoreService() { - @Override - public AsyncExecutor someExecutor() - { - return null; - } - - @Override - public SequentialAsyncExecutor someSequentialExecutor() - { - return null; - } - private ToLongFunction elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MICROSECONDS, this::now); private long stamp = 0; + @Override public AsyncExecutor someExecutor() { return null; } + @Override public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } @Override public Timeouts timeouts() { return null; } @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; } @Override public DurabilityService durability() { return null; } @@ -389,6 +340,7 @@ public class AccordTestUtils @Override public long elapsed(TimeUnit timeUnit) { return elapsed.applyAsLong(timeUnit); } @Override public TopologyManager topology() { throw new UnsupportedOperationException(); } @Override public Coordinations coordinations() { return new Coordinations(); } + @Override public Scheduler scheduler() { return null; } @Override public long currentStamp() { return stamp; } @Override public void updateStamp() {++stamp;} @Override public boolean isReplaying() { return false; } @@ -419,7 +371,7 @@ public class AccordTestUtils Node.Id node = new Id(1); Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node))); AccordCommandStore store = createAccordCommandStore(node, now, topology); - store.execute((PreLoadContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20)); + store.execute((ExecutionContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().setCapacity(1 << 20)); return store; } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java similarity index 90% rename from test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java rename to test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java index c5fafc9103..9f88b78353 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java @@ -46,7 +46,7 @@ import org.slf4j.LoggerFactory; import accord.api.RoutingKey; import accord.local.CheckedCommands; import accord.local.Command; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; @@ -77,29 +77,33 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches; -import org.apache.cassandra.service.accord.AccordExecutor.ExclusiveGlobalCaches; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor.ExclusiveGlobalCaches; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Condition; +import static accord.local.ExecutionContext.contextFor; +import static accord.local.ExecutionContext.unsequencedReadWrite; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.PreLoadContext.contextFor; import static accord.utils.Property.qt; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; import static org.apache.cassandra.service.accord.AccordService.getBlocking; import static org.apache.cassandra.service.accord.AccordTestUtils.createAccordCommandStore; import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn; import static org.apache.cassandra.service.accord.AccordTestUtils.keys; -import static org.apache.cassandra.service.accord.AccordTestUtils.loaded; import static org.apache.cassandra.service.accord.AccordTestUtils.txnId; -public class AccordTaskTest +public class SafeTaskTest { - private static final Logger logger = LoggerFactory.getLogger(AccordTaskTest.class); + private static final Logger logger = LoggerFactory.getLogger(SafeTaskTest.class); private static final AtomicLong clock = new AtomicLong(0); @BeforeClass @@ -127,7 +131,7 @@ public class AccordTaskTest AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, "Test"), instance -> { + getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "Test"), instance -> { // TODO review: This change to `ifInitialized` was done in a lot of places and it doesn't preserve this property // I fixed this reference to point to `ifLoadedAndInitialised` and but didn't update other places Assert.assertNull(instance.ifInitialised(txnId)); @@ -141,7 +145,7 @@ public class AccordTaskTest AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, "Test"), safe -> { + getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "Test"), safe -> { StoreParticipants participants = StoreParticipants.empty(txnId); SafeCommand command = safe.get(txnId, participants); Assert.assertNotNull(command); @@ -155,7 +159,7 @@ public class AccordTaskTest Txn txn = AccordTestUtils.createWriteTxn((int)clock.incrementAndGet()); TokenKey key = ((PartitionKey) Iterables.getOnlyElement(txn.keys())).toUnseekable(); - getBlocking(commandStore.execute((PreLoadContext.Empty)() -> "Test", instance -> { + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", instance -> { SafeCommandsForKey cfk = instance.ifLoadedAndInitialised(key); Assert.assertNull(cfk); })); @@ -172,9 +176,6 @@ public class AccordTaskTest private static Command createStableAndPersist(AccordCommandStore commandStore, TxnId txnId, Timestamp executeAt) { Command command = AccordTestUtils.Commands.stable(txnId, createPartialTxn(0), executeAt); - AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null)); - safeCommand.set(command); - appendDiffToLog(commandStore).accept(null, command); return command; } @@ -198,7 +199,7 @@ public class AccordTaskTest route.overlapping(ranges); PartialDeps deps = PartialDeps.builder(ranges, true).build(); - Command command = getBlocking(commandStore.submit(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + Command command = getBlocking(commandStore.submit(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.preaccept(safe, txnId, partialTxn, route, appendDiffToLog(commandStore)); CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, partialTxn, executeAt, deps, appendDiffToLog(commandStore)); return safe.ifInitialised(txnId).current(); @@ -209,12 +210,12 @@ public class AccordTaskTest try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches()) { cacheSize = cache.global.capacity(); - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches()) { - cache.global.setCapacity(cacheSize); + commandStore.executor().setCapacity(cacheSize); } while (commandStore.executor().hasTasks()) @@ -246,7 +247,7 @@ public class AccordTaskTest Route partialRoute = route.overlapping(ranges); PartialDeps deps = PartialDeps.builder(ranges, true).build(); - Command command = getBlocking(commandStore.submit(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + Command command = getBlocking(commandStore.submit(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.preaccept(safe, txnId, partialTxn, route, appendDiffToLog(commandStore)); CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, executeAt, deps, appendDiffToLog(commandStore)); CheckedCommands.commit(safe, SaveStatus.Committed, Ballot.ZERO, txnId, route, partialTxn, executeAt, deps, appendDiffToLog(commandStore)); @@ -259,11 +260,11 @@ public class AccordTaskTest try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { cacheSize = cache.global.capacity(); - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { - cache.global.setCapacity(cacheSize); + commandStore.executor().setCapacity(cacheSize); } while (commandStore.executor().hasTasks()) @@ -281,7 +282,7 @@ public class AccordTaskTest AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } Gen txnIdGen = rs -> txnId(1, clock.incrementAndGet(), 1); @@ -298,7 +299,7 @@ public class AccordTaskTest awaitDone(commandStore, ids, participants); assertNoReferences(commandStore, ids, participants); - PreLoadContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); Consumer consumer = Mockito.mock(Consumer.class); Map failed = selectFailedTxn(rs, ids); @@ -312,7 +313,7 @@ public class AccordTaskTest throw new NullPointerException("txn_id " + txnId); }); } - AccordTask o1 = AccordTask.create(commandStore, ctx, consumer); + SafeTask o1 = SafeTask.create(commandStore, ctx, consumer); AssertionUtils.assertThatThrownBy(() -> getBlocking(o1.chain())) .hasRootCause() .isInstanceOf(NullPointerException.class) @@ -333,7 +334,7 @@ public class AccordTaskTest return cmd; }); } - AccordTask o2 = AccordTask.create(commandStore, ctx, store -> { + SafeTask o2 = SafeTask.create(commandStore, ctx, store -> { ids.forEach(id -> { store.ifInitialised(id).readyToExecute(store); }); @@ -363,13 +364,13 @@ public class AccordTaskTest assertNoReferences(commandStore, ids, participants); createCommand(commandStore, rs, ids); - PreLoadContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); Consumer consumer = Mockito.mock(Consumer.class); String errorMsg = "txn_ids " + ids; Mockito.doThrow(new NullPointerException(errorMsg)).when(consumer).accept(Mockito.any()); - AccordTask operation = AccordTask.create(commandStore, ctx, consumer); + SafeTask operation = SafeTask.create(commandStore, ctx, consumer); AssertionUtils.assertThatThrownBy(() -> getBlocking(operation.chain())) .hasRootCause() @@ -441,7 +442,7 @@ public class AccordTaskTest AssertionError error = null; for (T key : keys) { - AccordCacheEntry node = cache.getUnsafe(key); + AccordCacheEntry node = cache.getUnsafe(key); if (node == null) continue; try { @@ -476,7 +477,7 @@ public class AccordTaskTest { for (T key : keys) { - AccordCacheEntry node = cache.getUnsafe(key); + AccordCacheEntry node = cache.getUnsafe(key); if (node == null) continue; Awaitility.await("For node " + node.key() + " to complete") .atMost(Duration.ofMinutes(1)) diff --git a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java index 0d2b8ccb93..f62a784f1e 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java @@ -21,7 +21,7 @@ package org.apache.cassandra.service.accord; import org.assertj.core.api.Assertions; import org.junit.Test; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.StoreParticipants; import accord.primitives.SaveStatus; import accord.primitives.TxnId; @@ -41,7 +41,7 @@ public class SimpleSimulatedAccordCommandStoreTest extends SimulatedAccordComman for (int i = 0, examples = 100; i < examples; i++) { TxnId id = AccordGens.txnIds().next(rs); - instance.process(PreLoadContext.contextFor(id, "Test"), (safe) -> { + instance.process(ExecutionContext.unsequenced(id, "Test"), (safe) -> { var safeCommand = safe.get(id, StoreParticipants.empty(id)); var command = safeCommand.current(); Assertions.assertThat(command.saveStatus()).isEqualTo(SaveStatus.Uninitialised); diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 0fabb67352..ae5b6468e3 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -35,12 +35,14 @@ import javax.annotation.Nullable; import org.assertj.core.api.Assertions; import accord.api.AsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.LocalListeners; import accord.api.ProgressLog; import accord.api.RemoteListeners; import accord.api.Result; import accord.api.RoutingKey; +import accord.api.Scheduler; import accord.api.Timeouts; import accord.coordinate.Coordinations; import accord.impl.DefaultLocalListeners; @@ -54,12 +56,11 @@ import accord.local.CommandStore; import accord.local.CommandStores; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.Node; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.SequentialAsyncExecutor; import accord.local.TimeService; import accord.local.durability.DurabilityService; import accord.messages.BeginRecovery; @@ -100,6 +101,10 @@ import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutorSimple; import org.apache.cassandra.service.accord.journal.RangeSearcher; import org.apache.cassandra.service.accord.journal.SegmentRangeSearcher; import org.apache.cassandra.service.accord.topology.AccordTopology; @@ -182,7 +187,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable } @Override - public SequentialAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } @@ -264,6 +269,12 @@ public class SimulatedAccordCommandStore implements AutoCloseable } @Override public Coordinations coordinations() { return new Coordinations(); } + + @Override + public Scheduler scheduler() + { + return null; + } }; TestAgent.RethrowAgent agent = new TestAgent.RethrowAgent() @@ -448,7 +459,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable return process(request, request); } - public T process(PreLoadContext loadCtx, Function function) throws ExecutionException, InterruptedException + public T process(ExecutionContext loadCtx, Function function) throws ExecutionException, InterruptedException { var result = processAsync(loadCtx, function); processAll(); @@ -460,7 +471,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable return processAsync(request, request); } - public AsyncResult processAsync(PreLoadContext loadCtx, Function function) + public AsyncResult processAsync(ExecutionContext loadCtx, Function function) { return commandStore.submit(loadCtx, function); } diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java index 805ba040b8..5da64a83c2 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java @@ -23,6 +23,7 @@ import java.util.concurrent.locks.LockSupport; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.BooleanSupplier; +import java.util.function.Function; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -32,7 +33,7 @@ import org.junit.Test; import accord.api.RoutingKey; import accord.impl.basic.SimulatedFault; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommandStore; import accord.messages.PreAccept; import accord.primitives.FullRoute; @@ -52,6 +53,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.accord.SimulatedAccordCommandStore.FunctionWrapper; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.utils.Pair; import static accord.utils.Property.qt; @@ -107,7 +109,7 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase { case Task: { - PreLoadContext ctx = (PreLoadContext.Empty)()->"Test"; + ExecutionContext ctx = (ExecutionContext.Empty)()->"Test"; instance.maybeCacheEvict(ctx.keys()); operation(instance, ctx, actionGen.next(rs), rs::nextBoolean).chain().begin(counter); } @@ -178,9 +180,12 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase private enum Action { SUCCESS, FAILURE, LOAD_FAILURE } - private static AccordTask operation(SimulatedAccordCommandStore instance, PreLoadContext ctx, Action action, BooleanSupplier delay) + private static SafeTask operation(SimulatedAccordCommandStore instance, ExecutionContext ctx, Action action, BooleanSupplier delay) { - return new SimulatedOperation(instance.commandStore, ctx, action == Action.FAILURE ? SimulatedOperation.Action.FAILURE : SimulatedOperation.Action.SUCCESS); + + Function function = action == Action.FAILURE ? safeStore -> { throw new SimulatedFault("Operation failed for keys " + ctx.keys()); } + : safeStore -> null; + return SafeTask.create(instance.commandStore, ctx, function); } private static class Counter implements BiConsumer @@ -196,26 +201,6 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase } } - private static class SimulatedOperation extends AccordTask - { - enum Action { SUCCESS, FAILURE} - private final Action action; - - public SimulatedOperation(AccordCommandStore commandStore, PreLoadContext preLoadContext, Action action) - { - super(commandStore, preLoadContext); - this.action = action; - } - - @Override - public Void apply(SafeCommandStore safe) - { - if (action == Action.FAILURE) - throw new SimulatedFault("Operation failed for keys " + keys()); - return null; - } - } - private static class SimulatedLoadFunctionWrapper implements FunctionWrapper { final Supplier actions; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheEntryTest.java similarity index 81% rename from test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java rename to test/unit/org/apache/cassandra/service/accord/execution/AccordCacheEntryTest.java index a91c082039..4f61205a65 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java +++ b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheEntryTest.java @@ -15,19 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import org.junit.Assert; import org.junit.Test; -import org.apache.cassandra.service.accord.AccordCache.Type; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import accord.local.SafeState; + +import org.apache.cassandra.service.accord.execution.AccordCache.Type; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; public class AccordCacheEntryTest { - static class CacheEntry extends AccordCacheEntry + static class TestSafeState extends SafeState implements SaferState { - public CacheEntry(String key, Type.Instance instance) + @Override public AccordCacheEntry global() { return null; } + @Override public void preExecute(SafeTask owner, LockMode lockMode) {} + @Override public void postExecute(SafeTask owner) {} + } + + static class CacheEntry extends AccordCacheEntry + { + public CacheEntry(String key, Type.Instance instance) { super(key, instance); } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheTest.java similarity index 91% rename from test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java rename to test/unit/org/apache/cassandra/service/accord/execution/AccordCacheTest.java index 5cb919a66e..669d25fe30 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java +++ b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.function.Function; @@ -23,14 +23,19 @@ import org.agrona.concurrent.NoOpLock; import org.junit.Assert; import org.junit.Test; +import accord.local.ExecutionContext; +import accord.local.SafeState; + import org.apache.cassandra.cache.CacheSize; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.ManualExecutor; import org.apache.cassandra.metrics.AccordCacheMetrics; -import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.SaveExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; -import static org.apache.cassandra.service.accord.AccordTestUtils.testLoad; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.testLoad; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.mockito.Mockito.mock; @@ -40,85 +45,56 @@ public class AccordCacheTest { private static final long DEFAULT_NODE_SIZE = nodeSize(0); - private static abstract class TestSafeState implements AccordSafeState + private static abstract class TestSafeState & SaferState> extends SafeState implements SaferState { - protected boolean invalidated = false; - protected final AccordCacheEntry global; - private T original = null; + protected final AccordCacheEntry global; - public TestSafeState(AccordCacheEntry global) + public TestSafeState(AccordCacheEntry global) { this.global = global; } - public AccordCacheEntry global() + public AccordCacheEntry global() { return global; } - @Override - public T key() - { - return global.key(); - } + public final T key() { return global.key(); } - @Override - public T current() + public void preExecute(SafeTask owner, LockMode lockMode) { - return global.getExclusive(); - } - - @Override - public void set(T update) - { - global.setExclusive(update); - } - - @Override - public T original() - { - return original; - } - - @Override - public void preExecute() - { - original = global.getExclusive(); - } - - @Override - public Throwable failure() - { - return global.failure(); - } - - @Override - public void markUnsafe() - { - invalidated = true; - } - - @Override - public boolean isUnsafe() - { - return invalidated; + requireUninitialised(); + current = global.lockExclusive(owner, lockMode); + setSafe(); } } - private static class SafeString extends TestSafeState + private static class SafeString extends TestSafeState { - public SafeString(AccordCacheEntry global) + public SafeString(AccordCacheEntry global) { super(global); } + + @Override + public void postExecute(SafeTask owner) + { + global.releaseExclusive(this, owner); + } } - private static class SafeInt extends TestSafeState + private static class SafeInt extends TestSafeState { - public SafeInt(AccordCacheEntry global) + public SafeInt(AccordCacheEntry global) { super(global); } + + @Override + public void postExecute(SafeTask owner) + { + global.releaseExclusive(this, owner); + } } private static long emptyNodeSize() @@ -264,6 +240,7 @@ public class AccordCacheTest assertCacheMetrics(cacheMetrics, 0, 3, 3, 3); SafeString safeString = instance.acquire("1"); + safeString.preExecute(new SafeTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals(Status.LOADED, safeString.global.status()); assertCacheState(cache, 1, 3, nodeSize(1) * 3); @@ -392,6 +369,7 @@ public class AccordCacheTest assertCacheState(cache, 1, 1, nodeSize(1)); SafeString safeString2 = instance.acquire("0"); + safeString2.preExecute(new SafeTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals("0", safeString2.current()); Assert.assertEquals(Status.LOADED, safeString1.global.status()); Assert.assertEquals(2, instance.references("0", SafeString.class)); diff --git a/test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java b/test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java new file mode 100644 index 0000000000..1a92a205d9 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.junit.Assert; + +import accord.local.ExecutionContext; +import accord.local.SafeState; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.concurrent.ManualExecutor; + +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; + +public class AccordExecutionTestUtils +{ + public static & SaferState> AccordCacheEntry loaded(K key, V value) + { + AccordCacheEntry global = new AccordCacheEntry<>(key, null); + global.initialize(value); + return global; + } + + private static AccordCacheEntry.LoadExecutor loadExecutor(ExecutorPlus executor) + { + return new AccordCacheEntry.LoadExecutor<>() + { + @Override + public IOTask load(P1 p1, P2 p2, AccordCacheEntry entry) + { + executor.submit(() -> { + V v; + try { v = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); } + catch (Throwable t) + { + entry.failedToLoad(); + throw t; + } + entry.loaded(v); + }); + return null; + } + }; + } + + public static & SaferState> void testLoad(ManualExecutor executor, S safeState, V val) + { + Assert.assertEquals(AccordCacheEntry.Status.WAITING_TO_LOAD, safeState.global().status()); + safeState.global().load(loadExecutor(executor), null, null); + Assert.assertEquals(AccordCacheEntry.Status.LOADING, safeState.global().status()); + executor.runOne(); + Assert.assertEquals(AccordCacheEntry.Status.LOADED, safeState.global().status()); + safeState.preExecute(new SafeTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); + Assert.assertEquals(val, safeState.current()); + } +} diff --git a/test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java b/test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java new file mode 100644 index 0000000000..a1a737317d --- /dev/null +++ b/test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java @@ -0,0 +1,529 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import com.google.common.collect.Sets; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; + +import accord.api.RoutingKey; +import accord.local.ExecutionContext; +import accord.local.Node; +import accord.local.Node.Id; +import accord.local.SafeCommandStore; +import accord.primitives.RoutingKeys; +import accord.primitives.TxnId; +import accord.topology.Shard; +import accord.topology.Topology; +import accord.utils.SortedArrays.SortedArrayList; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordTestUtils; +import org.apache.cassandra.service.accord.TokenRange; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.service.accord.execution.AccordExecutor.Mode; +import org.apache.cassandra.utils.concurrent.CountDownLatch; + +import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Lifecycle tests for {@link Task} and its interaction with {@link AccordExecutor}, covering the invariants the + * executor's callers rely on: + *

    + *
  • every task is notified exactly once, and every {@code chain()} eventually completes (success, failure or + * cancellation) - a task that is dropped silently hangs its caller forever;
    • + *
  • a task is completed exactly once, so {@code tasks}/{@link Tranches} accounting returns to zero and + * {@link AccordExecutor#hasTasks()} becomes false (otherwise {@code waitForQuiescence} and + * {@code afterSubmittedAndConsequences} never fire again);
    • + *
  • a task always releases its cache references, even when it fails;
    • + *
  • no failure is reported to the {@link accord.api.Agent} on any of these paths - an agent exception here means + * an internal invariant was broken, not that the user's operation failed.
    • + *
+ * + * Each test creates a private executor + command store so that failures cannot leak between tests. + */ +public class TaskLifecycleTest +{ + private static final long TIMEOUT_SECONDS = 30; + private static final AtomicLong clock = new AtomicLong(0); + + private final List executors = new CopyOnWriteArrayList<>(); + private final List stores = new CopyOnWriteArrayList<>(); + + @BeforeClass + public static void beforeClass() throws Throwable + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); + } + + @After + public void after() + { + stores.forEach(AccordCommandStore::shutdown); + stores.clear(); + executors.forEach(AccordExecutor::shutdown); + executors.clear(); + } + + /** + * An executor plus one command store, and a record of everything reported to the agent. + */ + private class Env + { + final List agentExceptions = new CopyOnWriteArrayList<>(); + final AccordExecutor executor; + final AccordCommandStore store; + + Env(Mode mode) + { + AccordAgent agent = new AccordAgent() + { + @Override + public void onException(Throwable t) + { + agentExceptions.add(t); + } + + @Override + public void onException(Throwable t, String context) + { + agentExceptions.add(t); + } + }; + agent.setup(Id.NONE); + this.executor = new AccordExecutorSyncSubmit(0, mode, "TaskLifecycleTest", agent); + executors.add(executor); + this.store = newStore(executor); + stores.add(store); + } + + void assertQuiescent() + { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (executor.hasTasks() && System.nanoTime() < deadline) + Thread.yield(); + assertThat(executor.hasTasks()).describedAs("executor still has registered tasks").isFalse(); + assertThat(executor.unsafeRunningCount()).describedAs("tasks still assigned to a runner").isZero(); + } + + void assertNoAgentExceptions() + { + assertThat(agentExceptions).describedAs("internal failures reported to the agent").isEmpty(); + } + } + + private static AccordCommandStore newStore(AccordExecutor executor) + { + TableMetadata metadata = Schema.instance.getTableMetadata("ks", "tbl"); + TokenRange range = TokenRange.fullRange(metadata.id, Murmur3Partitioner.instance); + Node.Id node = new Id(1); + Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[]{ node }), Sets.newHashSet(node))); + AccordCommandStore store = AccordTestUtils.createAccordCommandStore(node, clock::incrementAndGet, topology, executor); + // NOTE: capacity must be set via the executor, so that its derived maxWorkingCapacityInBytes is refreshed + executor.executeDirectlyWithLock(() -> { + executor.setCapacity(1 << 20); + executor.setWorkingSetSize(1 << 20); + }); + return store; + } + + private TxnId nextTxnId() + { + return AccordTestUtils.txnId(1, clock.incrementAndGet(), 1); + } + + private static Consumer noop() + { + return ignore -> {}; + } + + private static void await(CountDownLatch latch, String what) throws InterruptedException + { + assertThat(latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)).describedAs(what).isTrue(); + } + + /** + * A task that fails must cancel *every* consequence it accumulated, and must still be unregistered. + */ + @Test + public void failedTaskCancelsAllConsequences() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + int count = 4; + List> children = new ArrayList<>(); + CountDownLatch childrenDone = CountDownLatch.newCountDownLatch(count); + for (int i = 0 ; i < count ; ++i) + children.add(new AtomicReference<>()); + + AtomicReference parentFailure = new AtomicReference<>(); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + RuntimeException failure = new RuntimeException("deliberate failure"); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + for (int i = 0 ; i < count ; ++i) + { + AtomicReference child = children.get(i); + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "child" + i), noop(), + (r, f) -> { child.set(f == null ? new AssertionError("child should not have run") : f); childrenDone.decrement(); }); + } + throw failure; + }, (r, f) -> { parentFailure.set(f); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + assertThat(parentFailure.get()).isSameAs(failure); + await(childrenDone, "every consequence was notified"); + for (int i = 0 ; i < count ; ++i) + assertThat(children.get(i).get()).describedAs("child" + i).isInstanceOf(CancellationException.class); + + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * The happy path counterpart: consequences of a successful task all run. + */ + @Test + public void successfulTaskSubmitsAllConsequences() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + int count = 4; + AtomicInteger ran = new AtomicInteger(); + CountDownLatch childrenDone = CountDownLatch.newCountDownLatch(count); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + for (int i = 0 ; i < count ; ++i) + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "child" + i), + (Consumer) s -> ran.incrementAndGet(), + (r, f) -> { assertThat(f).isNull(); childrenDone.decrement(); }); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + await(childrenDone, "every consequence was notified"); + assertThat(ran.get()).isEqualTo(count); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * Cancelling one consequence must not disturb its siblings. Here the cancellation is submitted from within the + * parent, so it is applied after the parent has completed (i.e. once the consequence has been submitted). + */ + @Test + public void cancellingOneConsequenceDoesNotAffectSiblings() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + AtomicReference cancelled = new AtomicReference<>(); + CountDownLatch cancelledDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch siblingDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + Cancellable cancel = env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "cancelled"), noop(), + (r, f) -> { cancelled.set(f); cancelledDone.decrement(); }); + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "sibling"), noop(), + (r, f) -> { assertThat(f).isNull(); siblingDone.decrement(); }); + cancel.cancel(); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + await(siblingDone, "sibling of the cancelled consequence ran"); + await(cancelledDone, "cancelled consequence was notified"); + assertThat(cancelled.get()).isInstanceOf(CancellationException.class); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * As above, but the cancellation arrives from a foreign thread while the parent is still running, so the + * consequence is terminated before it has ever been submitted. + */ + @Test + public void externallyCancellingOneConsequenceDoesNotAffectSiblings() throws Throwable + { + Env env = new Env(RUN_WITHOUT_LOCK); // the parent blocks, so it must not hold the executor lock + AtomicReference toCancel = new AtomicReference<>(); + AtomicReference cancelled = new AtomicReference<>(); + CountDownLatch cancelledDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch siblingDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch consequencesAdded = CountDownLatch.newCountDownLatch(1); + CountDownLatch releaseParent = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + toCancel.set(env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "cancelled"), noop(), + (r, f) -> { cancelled.set(f); cancelledDone.decrement(); })); + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "sibling"), noop(), + (r, f) -> { assertThat(f).isNull(); siblingDone.decrement(); }); + consequencesAdded.decrement(); + releaseParent.awaitUninterruptibly(); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(consequencesAdded, "consequences were registered"); + toCancel.get().cancel(); + releaseParent.decrement(); + + await(parentDone, "parent was notified"); + await(siblingDone, "sibling of the cancelled consequence ran"); + await(cancelledDone, "cancelled consequence was notified"); + assertThat(cancelled.get()).isInstanceOf(CancellationException.class); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * A task whose prepare fails must be failed and completed, and the {@link ExclusiveExecutor} it was dispatched + * from must go on to dispatch the next task. + */ + @Test + public void prepareFailureCompletesTaskAndDispatchesNext() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + ExclusiveExecutor exclusive = env.executor.newExclusiveExecutor(0); + + CountDownLatch failed = CountDownLatch.newCountDownLatch(1); + CountDownLatch ran = CountDownLatch.newCountDownLatch(1); + RuntimeException failure = new RuntimeException("deliberate prepare failure"); + TestTask first = new TestTask(env.executor, exclusive, failure, failed); + TestTask second = new TestTask(env.executor, exclusive, null, ran); + + env.executor.executeDirectlyWithLock(() -> { + first.submitExclusiveNoExcept(); + second.submitExclusiveNoExcept(); + }); + + await(failed, "the task that failed to prepare was notified"); + assertThat(first.failure).isSameAs(failure); + await(ran, "the next queued task was dispatched"); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * A chain submitted to a command store hosted by a *different* executor must be submitted to that executor + * independently, not attached as a consequence of the running task (whose executor's lock we hold). + */ + @Test + public void consequenceOnAnotherExecutorIsSubmittedIndependently() throws Throwable + { + Env a = new Env(RUN_WITHOUT_LOCK); + Env b = new Env(RUN_WITHOUT_LOCK); + + CountDownLatch childDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + + a.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parentOnA"), (Consumer) safe -> { + b.store.execute(ExecutionContext.unsequenced(nextTxnId(), "childOnB"), noop(), + (r, f) -> { assertThat(f).isNull(); childDone.decrement(); }); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + await(childDone, "child on the other executor ran"); + a.assertNoAgentExceptions(); + b.assertNoAgentExceptions(); + a.assertQuiescent(); + b.assertQuiescent(); + } + + /** + * An incremental task with more keys than a single batch can hold must run several batches and then complete once. + */ + @Test + public void incrementalTaskRunsMultipleBatches() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + AtomicInteger batches = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + CountDownLatch done = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequencedIncrementalWrite(keys(0, 200), "incremental"), + (Consumer) safe -> batches.incrementAndGet(), + (r, f) -> { failure.set(f); done.decrement(); }); + + await(done, "incremental task completed"); + assertThat(failure.get()).isNull(); + assertThat(batches.get()).describedAs("expected several batches").isGreaterThan(1); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * An incremental task can be failed while it is parked between batches - this is what a late failure of one of + * its (optional) key loads does, see {@link AccordExecutor#onLoadedExclusive}. It must release its cache + * references, exactly as it would if it were failed before it first ran. + */ + @Test + public void incrementalTaskFailedBetweenBatchesReleasesResources() throws Throwable + { + Env env = new Env(RUN_WITHOUT_LOCK); + AtomicInteger batches = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + CountDownLatch done = CountDownLatch.newCountDownLatch(1); + RuntimeException loadFailure = new RuntimeException("simulated load failure"); + + Cancellable submitted = + env.store.execute(ExecutionContext.unsequencedIncrementalWrite(keys(1000, 1200), "incremental"), + (Consumer) safe -> { + batches.incrementAndGet(); + try { Thread.sleep(5); } catch (InterruptedException e) { throw new RuntimeException(e); } + }, + (r, f) -> { failure.set(f); done.decrement(); }); + + SafeTask task = (SafeTask) submitted; + AtomicReference failedIn = new AtomicReference<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (failedIn.get() == null && failure.get() == null && System.nanoTime() < deadline) + { + env.executor.executeDirectlyWithLock(() -> { + Task.State state = task.state(); + if (batches.get() > 0 && task.isState(Task.State.WAITING)) + { + failedIn.set(state); + task.tryFailAndCompleteExclusive(loadFailure, Task.State.FAILED); + } + }); + } + + assertThat(failedIn.get()).describedAs("did not observe the task parked between batches").isNotNull(); + await(done, "task was notified of the failure"); + assertThat(failure.get()).isSameAs(loadFailure); + int stillHeld = task.refs == null ? 0 : task.refs.size(); + assertThat(stillHeld).describedAs("cache references were not released").isZero(); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * The executor throttles loading while the cache is over its working-set budget, but must always be able to make + * *some* progress: if the only work that can run is loading, it must run regardless of the budget. + */ + @Test + public void progressesWithZeroCacheCapacity() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + env.executor.executeDirectlyWithLock(() -> { + env.executor.setCapacity(0); + env.executor.setWorkingSetSize(0); + }); + + for (int i = 0 ; i < 2 ; ++i) + { + AtomicReference failure = new AtomicReference<>(); + CountDownLatch done = CountDownLatch.newCountDownLatch(1); + env.store.execute(ExecutionContext.unsequencedIncrementalWrite(keys(2000 + i * 200, 2200 + i * 200), "zeroCapacity" + i), + noop(), (r, f) -> { failure.set(f); done.decrement(); }); + await(done, "task completed with a zero capacity cache"); + assertThat(failure.get()).isNull(); + } + + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + private static RoutingKeys keys(int from, int to) + { + TableMetadata metadata = Schema.instance.getTableMetadata("ks", "tbl"); + List keys = new ArrayList<>(to - from); + for (int i = from ; i < to ; ++i) + keys.add(AccordTestUtils.key(metadata, i).toUnseekable()); + return RoutingKeys.of(keys); + } + + /** + * A minimal {@link Plain} task: optionally fails during prepare, otherwise records that it ran. + */ + private static class TestTask extends Plain + { + final ExclusiveExecutor exclusiveExecutor; + final RuntimeException failPrepareWith; + final CountDownLatch notified; + volatile Throwable failure; + + TestTask(AccordExecutor executor, ExclusiveExecutor exclusiveExecutor, RuntimeException failPrepareWith, CountDownLatch notified) + { + super(executor, ExclusiveGroup.OTHER); + this.exclusiveExecutor = exclusiveExecutor; + this.failPrepareWith = failPrepareWith; + this.notified = notified; + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return exclusiveExecutor; + } + + @Override + void prepareExclusiveMayThrow() + { + if (failPrepareWith != null) + throw failPrepareWith; + } + + @Override + boolean runMayThrow() + { + if (failPrepareWith != null) + throw new AssertionError("should not have run"); + notified.decrement(); + return true; + } + + @Override + void reportFailureMayThrow(Throwable fail) + { + failure = fail; + notified.decrement(); + } + + @Override + public String description() + { + return "TestTask[failPrepare=" + (failPrepareWith != null) + ']'; + } + + @Override + String briefDescription() + { + return description(); + } + } +} diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 138dbb4f12..3b975207b7 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -49,12 +49,14 @@ import org.junit.Test; import accord.api.Agent; import accord.api.AsyncExecutor; import accord.api.DataStore; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.Key; import accord.api.OwnershipEventListener; import accord.api.ProgressLog; import accord.api.Result; import accord.api.RoutingKey; +import accord.api.Scheduler; import accord.api.Timeouts; import accord.coordinate.Coordinations; import accord.impl.AbstractReplayer; @@ -66,13 +68,12 @@ import accord.local.CommandBuilder; import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.Node; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; import accord.local.RedundantBefore; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.SequentialAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.cfk.CommandsForKey; @@ -505,8 +506,8 @@ public class CommandsForKeySerializerTest { int next = source.nextInt(commands.size()); Command command = commands.get(next); - if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(PreLoadContext.contextFor(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); - else cfk = cfk.update(new TestSafeCommandStore(PreLoadContext.contextFor(command.txnId(), "Test")), command).cfk(); + if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(ExecutionContext.unsequenced(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); + else cfk = cfk.update(new TestSafeCommandStore(ExecutionContext.unsequenced(command.txnId(), "Test")), command).cfk(); commands.set(next, commands.get(commands.size() - 1)); commands.remove(commands.size() - 1); } @@ -547,33 +548,10 @@ public class CommandsForKeySerializerTest static class TestSafeCommand extends SafeCommand { - final Command command; TestSafeCommand(Command command) { super(command.txnId); - this.command = command; - } - - @Override - public Command current() - { - return command; - } - - @Override - public void markUnsafe() - { - } - - @Override - public boolean isUnsafe() - { - return false; - } - - @Override - protected void set(Command command) - { + current = command; } } @@ -667,8 +645,8 @@ public class CommandsForKeySerializerTest } @Override public boolean inStore() { return true; } - @Override public AsyncChain chain(PreLoadContext context, Consumer consumer) { throw new UnsupportedOperationException();} - @Override public AsyncChain chain(PreLoadContext context, Function apply) { throw new UnsupportedOperationException(); } + @Override public AsyncChain chain(ExecutionContext context, Consumer consumer) { throw new UnsupportedOperationException();} + @Override public AsyncChain chain(ExecutionContext context, Function apply) { throw new UnsupportedOperationException(); } @Override public Journal.Replayer replayer(AbstractReplayer.Mode mode) { throw new UnsupportedOperationException(); } @@ -702,9 +680,9 @@ public class CommandsForKeySerializerTest public static class TestSafeCommandStore extends AbstractSafeCommandStore { - public TestSafeCommandStore(PreLoadContext context) + public TestSafeCommandStore(ExecutionContext context) { - super(context, TestCommandStore.INSTANCE); + super(context); } @Override protected CommandStoreCaches tryGetCaches() { return null; } @@ -712,13 +690,14 @@ public class CommandsForKeySerializerTest @Override protected SafeCommandsForKey add(SafeCommandsForKey safeCfk, CommandStoreCaches caches) { return null; } @Override protected SafeCommand getInternal(TxnId txnId) { return null; } @Override protected SafeCommandsForKey getInternal(RoutingKey key) { return null; } + @Override public CommandStore commandStore() { return TestCommandStore.INSTANCE; } @Override public DataStore dataStore() { return null; } @Override public Agent agent() { return null; } @Override public ProgressLog progressLog() { return null; } @Override public NodeCommandStoreService node() { return new NodeCommandStoreService() { @Override public AsyncExecutor someExecutor() { throw new UnsupportedOperationException(); } - @Override public SequentialAsyncExecutor someSequentialExecutor() { throw new UnsupportedOperationException(); } + @Override public ExclusiveAsyncExecutor someExclusiveExecutor() { throw new UnsupportedOperationException(); } @Override public long epoch() { return 0;} @Override public Node.Id id() { return Node.Id.NONE; } @Override public Timeouts timeouts() { return null; } @@ -733,6 +712,7 @@ public class CommandsForKeySerializerTest @Override public long now() { return 0; } @Override public long elapsed(TimeUnit unit) { return 0; } @Override public Coordinations coordinations() { return new Coordinations(); } + @Override public Scheduler scheduler() { return null; } }; } @Override public boolean visit(Unseekables keysOrRanges, TxnId testTxnId, Kind.Kinds testKind, SupersedingCommandVisitor visit) { return false; } @Override public void visit(Unseekables keysOrRanges, Timestamp startedBefore, Kind.Kinds testKind, ActiveCommandVisitor visit, P1 p1, P2 p2) { } diff --git a/test/unit/org/apache/cassandra/utils/AccordGenerators.java b/test/unit/org/apache/cassandra/utils/AccordGenerators.java index 0e7d7a3685..a05b644b53 100644 --- a/test/unit/org/apache/cassandra/utils/AccordGenerators.java +++ b/test/unit/org/apache/cassandra/utils/AccordGenerators.java @@ -175,7 +175,7 @@ public class AccordGenerators switch (targetType) { case notDefined: - return AccordTestUtils.Commands.notDefined(id, txn); + return AccordTestUtils.Commands.notDefined(id); case preaccepted: return AccordTestUtils.Commands.preaccepted(id, txn, executeAt); case committed: