diff --git a/modules/accord b/modules/accord index 2b9e54004f..f9dc83e404 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 2b9e54004f702c7b626e94391af21fa080292975 +Subproject commit f9dc83e404253645db02d54c913174b796fae123 diff --git a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java index ff7f2d2cb3..4c19d1c010 100644 --- a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java +++ b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java @@ -75,9 +75,7 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.WriteResponseHandler; -import org.apache.cassandra.service.accord.AccordService; -import org.apache.cassandra.service.accord.IAccordService; -import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult; +import org.apache.cassandra.service.accord.IAccordService.IAccordResult; import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitMutations; @@ -400,7 +398,7 @@ public class BatchlogManager implements BatchlogManagerMBean private final ClusterMetadata cm; private List> replayHandlers = ImmutableList.of(); - private AsyncTxnResult accordResult; + private IAccordResult accordResult; @Nullable private Dispatcher.RequestTime accordTxnStart; @@ -452,8 +450,7 @@ public class BatchlogManager implements BatchlogManagerMBean { if (accordResult != null) { - IAccordService accord = AccordService.instance(); - TxnResult.Kind kind = accord.getTxnResult(accordResult).kind(); + TxnResult.Kind kind = accordResult.awaitAndGet().kind(); if (kind == retry_new_protocol) throw new RetryOnDifferentSystemException(); } diff --git a/src/java/org/apache/cassandra/config/AccordSpec.java b/src/java/org/apache/cassandra/config/AccordSpec.java index 566a0d16ef..d968f3ada0 100644 --- a/src/java/org/apache/cassandra/config/AccordSpec.java +++ b/src/java/org/apache/cassandra/config/AccordSpec.java @@ -20,13 +20,11 @@ package org.apache.cassandra.config; import java.util.concurrent.TimeUnit; -import accord.primitives.TxnId; import accord.utils.Invariants; import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.cassandra.journal.Params; import org.apache.cassandra.service.consensus.TransactionalMode; -import static accord.primitives.Routable.Domain.Range; import static org.apache.cassandra.config.AccordSpec.QueueShardModel.THREAD_POOL_PER_SHARD; import static org.apache.cassandra.config.AccordSpec.QueueSubmissionModel.SYNC; @@ -136,43 +134,32 @@ public class AccordSpec public DataStorageSpec.LongMebibytesBound working_set_size = null; public boolean shrink_cache_entries_before_eviction = true; - // TODO (expected): we should be able to support lower recover delays, at least for txns - public volatile DurationSpec.IntMillisecondsBound recover_delay = new DurationSpec.IntMillisecondsBound(5000); - public volatile DurationSpec.IntMillisecondsBound range_syncpoint_recover_delay = new DurationSpec.IntMillisecondsBound("5m"); - public String slowPreAccept = "30ms <= p50*2 <= 100ms"; - public String slowRead = "30ms <= p50*2 <= 100ms"; + public DurationSpec.IntMillisecondsBound range_syncpoint_timeout = new DurationSpec.IntMillisecondsBound("3m"); + public DurationSpec.IntMillisecondsBound repair_timeout = new DurationSpec.IntMillisecondsBound("10m"); + public String recover_txn = "5s*attempts <= 60s"; + public String recover_syncpoint = "60s <= 30s*attempts...60s*attempts <= 600s"; + public String fetch_txn = "1s*attempts"; + public String fetch_syncpoint = "5s*attempts"; + public String expire_txn = "5s*attempts"; + public String expire_syncpoint = "60s*attempts<=300s"; + public String expire_epoch_wait = "10s"; + // we don't want to wait ages for durability as it blocks other durability progress; even this might be too long, as we can always retry + public String expire_durability = "10s*attempts <= 30s"; + public String slow_syncpoint_preaccept = "10s"; + public String slow_txn_preaccept = "30ms <= p50*2 <= 100ms"; + public String slow_read = "30ms <= p50*2 <= 100ms"; + public String retry_syncpoint = "10s*attempts <= 600s"; + public String retry_durability = "10s*attempts <= 600s"; + public String retry_fetch_min_epoch = "200ms...1s*attempts <= 1s,retries=3"; + public String retry_fetch_topology = "200ms...1s*attempts <= 1s,retries=100"; - public long recoveryDelayFor(TxnId txnId, TimeUnit unit) - { - if (txnId.isSyncPoint() && txnId.is(Range)) - return range_syncpoint_recover_delay.to(unit); - return recover_delay.to(unit); - } - - /** - * When a barrier transaction is requested how many times to repeat attempting the barrier before giving up - */ - public int barrier_retry_attempts = 5; - - /** - * When a barrier transaction fails how long the initial backoff should be before being increased - * as part of exponential backoff on each attempt - */ - public DurationSpec.IntMillisecondsBound barrier_retry_inital_backoff_millis = new DurationSpec.IntMillisecondsBound("1s"); - - public DurationSpec.IntMillisecondsBound barrier_max_backoff = new DurationSpec.IntMillisecondsBound("10m"); - - public DurationSpec.IntMillisecondsBound range_syncpoint_timeout = new DurationSpec.IntMillisecondsBound("2m"); - - public volatile DurationSpec.IntSecondsBound fast_path_update_delay = new DurationSpec.IntSecondsBound("60m"); + public volatile DurationSpec.IntSecondsBound fast_path_update_delay = null; public volatile DurationSpec.IntSecondsBound gc_delay = new DurationSpec.IntSecondsBound("5m"); public volatile int shard_durability_target_splits = 128; public volatile DurationSpec.IntSecondsBound durability_txnid_lag = new DurationSpec.IntSecondsBound(5); public volatile DurationSpec.IntSecondsBound shard_durability_cycle = new DurationSpec.IntSecondsBound(15, TimeUnit.MINUTES); public volatile DurationSpec.IntSecondsBound global_durability_cycle = new DurationSpec.IntSecondsBound(10, TimeUnit.MINUTES); - public volatile DurationSpec.IntSecondsBound default_durability_retry_delay = new DurationSpec.IntSecondsBound(10, TimeUnit.SECONDS); - public volatile DurationSpec.IntSecondsBound max_durability_retry_delay = new DurationSpec.IntSecondsBound(10, TimeUnit.MINUTES); public enum TransactionalRangeMigration { @@ -194,24 +181,6 @@ public class AccordSpec public boolean ephemeralReadEnabled = true; public boolean state_cache_listener_jfr_enabled = true; public final JournalSpec journal = new JournalSpec(); - public final RetrySpec minEpochSyncRetry = new MinEpochRetrySpec(); - public final RetrySpec fetchRetry = new FetchRetrySpec(); - - public static class MinEpochRetrySpec extends RetrySpec - { - public MinEpochRetrySpec() - { - maxAttempts = new MaxAttempt(3); - } - } - - public static class FetchRetrySpec extends RetrySpec - { - public FetchRetrySpec() - { - maxAttempts = new MaxAttempt(100); - } - } public static class JournalSpec implements Params { diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 62ae907ead..7a48a67a5a 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -159,6 +159,8 @@ public class Config @Replaces(oldName = "cas_contention_timeout_in_ms", converter = Converters.MILLIS_DURATION_LONG, deprecated = true) public volatile DurationSpec.LongMillisecondsBound cas_contention_timeout = new DurationSpec.LongMillisecondsBound("1800ms"); + public volatile DurationSpec.LongMillisecondsBound accord_preaccept_timeout = new DurationSpec.LongMillisecondsBound("1s"); + @Replaces(oldName = "truncate_request_timeout_in_ms", converter = Converters.MILLIS_DURATION_LONG, deprecated = true) public volatile DurationSpec.LongMillisecondsBound truncate_request_timeout = new DurationSpec.LongMillisecondsBound("60000ms"); @@ -177,13 +179,15 @@ public class Config public volatile DurationSpec.LongMillisecondsBound stream_transfer_task_timeout = new DurationSpec.LongMillisecondsBound("12h"); - public volatile DurationSpec.LongMillisecondsBound transaction_timeout = new DurationSpec.LongMillisecondsBound("10s"); - public volatile DurationSpec.LongMillisecondsBound cms_await_timeout = new DurationSpec.LongMillisecondsBound("120000ms"); public volatile int cms_default_max_retries = 10; - public volatile DurationSpec.IntMillisecondsBound cms_default_retry_backoff = new DurationSpec.IntMillisecondsBound("50ms"); - public volatile DurationSpec.IntMillisecondsBound cms_default_max_retry_backoff = new DurationSpec.IntMillisecondsBound("1s"); + @Deprecated(since="5.1") + public volatile DurationSpec.IntMillisecondsBound cms_default_retry_backoff = null; + @Deprecated(since="5.1") + public volatile DurationSpec.IntMillisecondsBound cms_default_max_retry_backoff = null; + public String cms_retry_delay = "0 <= 50ms*1*attempts <= 10s,retries=10"; public volatile int epoch_aware_debounce_inflight_tracker_max_size = 100; + /** * How often we should snapshot the cluster metadata. */ diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 51add020cd..0804117a8b 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -119,6 +119,7 @@ import org.apache.cassandra.security.JREProvider; import org.apache.cassandra.security.SSLFactory; import org.apache.cassandra.service.CacheService.CacheType; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.api.AccordWaitStrategies; import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.paxos.Paxos; import org.apache.cassandra.tcm.RegistrationStatus; @@ -1564,10 +1565,10 @@ public class DatabaseDescriptor conf.truncate_request_timeout = LOWEST_ACCEPTED_TIMEOUT; } - if (conf.transaction_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds()) + if (conf.accord_preaccept_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds()) { - logInfo("transaction_timeout", conf.transaction_timeout, LOWEST_ACCEPTED_TIMEOUT); - conf.transaction_timeout = LOWEST_ACCEPTED_TIMEOUT; + logInfo("accord_preaccept_timeout", conf.accord_preaccept_timeout, LOWEST_ACCEPTED_TIMEOUT); + conf.accord_preaccept_timeout = LOWEST_ACCEPTED_TIMEOUT; } } @@ -2532,16 +2533,6 @@ public class DatabaseDescriptor return conf.cas_contention_timeout.to(unit); } - public static long getTransactionTimeout(TimeUnit unit) - { - return conf.transaction_timeout.to(unit); - } - - public static void setTransactionTimeout(long timeOutInMillis) - { - conf.transaction_timeout = new DurationSpec.LongMillisecondsBound(timeOutInMillis); - } - public static void setCasContentionTimeout(long timeOutInMillis) { conf.cas_contention_timeout = new DurationSpec.LongMillisecondsBound(timeOutInMillis); @@ -5330,26 +5321,16 @@ public class DatabaseDescriptor conf.accord.range_migration = Preconditions.checkNotNull(val); } - public static int getAccordBarrierRetryAttempts() - { - return conf.accord.barrier_retry_attempts; - } - - public static long getAccordBarrierRetryInitialBackoffMillis() - { - return conf.accord.barrier_retry_inital_backoff_millis.toMilliseconds(); - } - - public static long getAccordBarrierRetryMaxBackoffMillis() - { - return conf.accord.barrier_max_backoff.toMilliseconds(); - } - public static long getAccordRangeSyncPointTimeoutNanos() { return conf.accord.range_syncpoint_timeout.to(TimeUnit.NANOSECONDS); } + public static long getAccordRepairTimeoutNanos() + { + return conf.accord.repair_timeout.to(TimeUnit.NANOSECONDS); + } + public static boolean getAccordTransactionsEnabled() { return conf.accord.enabled; @@ -5405,29 +5386,43 @@ public class DatabaseDescriptor return conf.accord.shrink_cache_entries_before_eviction; } - public static long getAccordRecoverDelay(TimeUnit units) + public static String getAccordRecoverTxnDelay() { - return conf.accord.recover_delay.to(units); + return conf.accord.recover_txn; } - public static void setAccordRecoverDelay(long time, TimeUnit units) + public static void setAccordRecoverTxnDelay(String recoverTxnDelay) { - conf.accord.recover_delay = new IntMillisecondsBound(time, units); + AccordWaitStrategies.setRecoverTxn(recoverTxnDelay); + conf.accord.recover_txn = recoverTxnDelay; } - public static long getAccordRangeSyncPointRecoverDelay(TimeUnit units) + public static String getAccordExpireTxnDelay() { - return conf.accord.range_syncpoint_recover_delay.to(units); + return conf.accord.expire_txn; } - public static void setAccordRangeSyncPointRecoverDelay(long time, TimeUnit units) + public static void setAccordExpireTxnDelay(String expireTxnDelay) { - conf.accord.range_syncpoint_recover_delay = new IntMillisecondsBound(time, units); + AccordWaitStrategies.setExpireTxn(expireTxnDelay); + conf.accord.expire_txn = expireTxnDelay; + } + + public static String getAccordRecoverSyncPointDelay() + { + return conf.accord.recover_syncpoint; + } + + public static void setAccordRecoverSyncPointDelay(String recoverSyncPointDelay) + { + AccordWaitStrategies.setRecoverSyncPoint(recoverSyncPointDelay); + conf.accord.recover_syncpoint = recoverSyncPointDelay; } public static long getAccordFastPathUpdateDelayMillis() { - return conf.accord.fast_path_update_delay.to(TimeUnit.MILLISECONDS); + DurationSpec.IntSecondsBound bound = conf.accord.fast_path_update_delay; + return bound == null ? -1 : bound.to(TimeUnit.MILLISECONDS); } public static void setAccordFastPathUpdateDelaySeconds(long seconds) @@ -5470,26 +5465,6 @@ public class DatabaseDescriptor conf.accord.global_durability_cycle = new DurationSpec.IntSecondsBound(seconds); } - public static long getAccordDefaultDurabilityRetryDelay(TimeUnit unit) - { - return conf.accord.default_durability_retry_delay.to(unit); - } - - public static void setAccordDefaultDurabilityRetryDelaySeconds(long seconds) - { - conf.accord.default_durability_retry_delay = new DurationSpec.IntSecondsBound(seconds); - } - - public static long getAccordMaxDurabilityRetryDelay(TimeUnit unit) - { - return conf.accord.max_durability_retry_delay.to(unit); - } - - public static void setAccordMaxDurabilityRetryDelaySeconds(long seconds) - { - conf.accord.max_durability_retry_delay = new DurationSpec.IntSecondsBound(seconds); - } - public static long getAccordShardDurabilityCycle(TimeUnit unit) { return conf.accord.shard_durability_cycle.to(unit); @@ -5837,6 +5812,11 @@ public class DatabaseDescriptor return conf.cms_default_max_retry_backoff; } + public static String getCMSRetryDelay() + { + return conf.cms_retry_delay; + } + public static DurationSpec getCmsAwaitTimeout() { return conf.cms_await_timeout; diff --git a/src/java/org/apache/cassandra/config/RetrySpec.java b/src/java/org/apache/cassandra/config/RetrySpec.java index 4f113af962..ff9b58f827 100644 --- a/src/java/org/apache/cassandra/config/RetrySpec.java +++ b/src/java/org/apache/cassandra/config/RetrySpec.java @@ -23,6 +23,10 @@ import java.util.Objects; import javax.annotation.Nullable; import org.apache.cassandra.config.DurationSpec.LongMillisecondsBound; +import org.apache.cassandra.repair.SharedContext; +import org.apache.cassandra.service.RetryStrategy; +import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory; +import org.apache.cassandra.service.WaitStrategy; public class RetrySpec { @@ -153,6 +157,13 @@ public class RetrySpec return !isEnabled() ? null : maxSleepTime; } + public static WaitStrategy toStrategy(SharedContext ctx, RetrySpec spec) + { + if (!spec.isEnabled()) + return WaitStrategy.None.INSTANCE; + return RetryStrategy.parse(spec.baseSleepTime.toMilliseconds() + "ms * 2^attempts <= " + spec.maxSleepTime.toMilliseconds() + "ms,retries=" + (spec.maxAttempts.value - 1), LatencySourceFactory.none()); + } + @Override public String toString() { diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 409a25c7bb..36baaacef5 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -28,6 +28,7 @@ import com.google.common.collect.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import accord.primitives.Ranges; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -48,7 +49,9 @@ import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.AccordTopology; import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping; import org.apache.cassandra.streaming.IncomingStream; import org.apache.cassandra.streaming.StreamReceiver; import org.apache.cassandra.streaming.StreamSession; @@ -58,8 +61,11 @@ import org.apache.cassandra.utils.CloseableIterator; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.Refs; +import static accord.local.durability.DurabilityService.SyncLocal.Self; +import static accord.local.durability.DurabilityService.SyncRemote.NoRemote; import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_MUTATION_REPAIR_ROWS_PER_BATCH; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; public class CassandraStreamReceiver implements StreamReceiver { @@ -248,7 +254,15 @@ public class CassandraStreamReceiver implements StreamReceiver if (session.streamOperation().requiresBarrierTransaction() && cfs.metadata().requiresAccordSupport() && CassandraVersion.CASSANDRA_5_0.compareTo(minVersion) >= 0) - accordService.postStreamReceivingBarrier(cfs, ranges); + { + Ranges accordRanges = AccordTopology.toAccordRanges(cfs.getTableId(), ranges); + long startedAtNanos = nanoTime(); + long deadlineNanos = startedAtNanos + DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(); + // TODO (expected): use the source bounds for the streams to avoid waiting unnecessarily long + AccordService.getBlocking(accordService.maxConflict(accordRanges) + .flatMap(min -> accordService.sync("[Stream #" + session.planId() + ']', min, accordRanges, null, Self, NoRemote)) + , accordRanges, new LatencyRequestBookkeeping(cfs.metric.accordPostStreamRepair), startedAtNanos, deadlineNanos); + } boolean requiresWritePath = requiresWritePath(cfs); Collection readers = sstables; diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 0c66a40d4c..1655f061a4 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -33,7 +33,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.RoutingKey; -import accord.impl.DurabilityScheduling; import accord.impl.progresslog.DefaultProgressLog; import accord.impl.progresslog.TxnStateKind; import accord.local.CommandStore; @@ -41,6 +40,7 @@ import accord.local.CommandStores; import accord.local.DurableBefore; import accord.local.MaxConflicts; import accord.local.RejectBefore; +import accord.local.durability.ShardDurability; import accord.primitives.Status; import accord.primitives.TxnId; import accord.utils.Invariants; @@ -85,22 +85,22 @@ import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; public class AccordDebugKeyspace extends VirtualKeyspace { - public static final String DURABILITY_SCHEDULING = "durability_scheduling"; - public static final String DURABLE_BEFORE = "durable_before"; - public static final String EXECUTOR_CACHE = "executor_cache"; - public static final String MAX_CONFLICTS = "max_conflicts"; - public static final String MIGRATION_STATE = "migration_state"; - public static final String PROGRESS_LOG = "progress_log"; - public static final String REDUNDANT_BEFORE = "redundant_before"; - public static final String REJECT_BEFORE = "reject_before"; - public static final String TXN_BLOCKED_BY = "txn_blocked_by"; + public static final String DURABILITY_SERVICE = "durability_service"; + public static final String DURABLE_BEFORE = "durable_before"; + public static final String EXECUTOR_CACHE = "executor_cache"; + public static final String MAX_CONFLICTS = "max_conflicts"; + public static final String MIGRATION_STATE = "migration_state"; + public static final String PROGRESS_LOG = "progress_log"; + public static final String REDUNDANT_BEFORE = "redundant_before"; + public static final String REJECT_BEFORE = "reject_before"; + public static final String TXN_BLOCKED_BY = "txn_blocked_by"; public static final AccordDebugKeyspace instance = new AccordDebugKeyspace(); private AccordDebugKeyspace() { super(VIRTUAL_ACCORD_DEBUG, List.of( - new DurabilitySchedulingTable(), + new DurabilityServiceTable(), new DurableBeforeTable(), new ExecutorCacheTable(), new MaxConflictsTable(), @@ -113,25 +113,33 @@ public class AccordDebugKeyspace extends VirtualKeyspace } // TODO (consider): use a different type for the three timestamps in micros - public static final class DurabilitySchedulingTable extends AbstractVirtualTable + public static final class DurabilityServiceTable extends AbstractVirtualTable { - private DurabilitySchedulingTable() + private DurabilityServiceTable() { - super(parse(VIRTUAL_ACCORD_DEBUG, DURABILITY_SCHEDULING, - "Accord per-Range Durability Scheduling State", + super(parse(VIRTUAL_ACCORD_DEBUG, DURABILITY_SERVICE, + "Accord per-Range Durability Service State", "CREATE TABLE %s (\n" + " keyspace_name text,\n" + " table_name text,\n" + " token_sort blob,\n" + " token_start text,\n" + " token_end text,\n" + + " last_started_at_micros bigint,\n" + + " cycle_started_at_micros bigint,\n" + + " retries int,\n" + + " min text,\n" + + " active text,\n" + + " waiting text,\n" + " node_offset int,\n" + - " \"index\" int,\n" + - " number_of_splits int,\n" + - " range_started_at bigint,\n" + - " cycle_started_at bigint,\n" + - " retry_delay_micros bigint,\n" + - " is_defunct boolean,\n" + + " cycle_offset int,\n" + + " activeIndex int,\n" + + " nextIndex int,\n" + + " nextToIndex int,\n" + + " endIndex int,\n" + + " current_splits int,\n" + + " stopping boolean,\n" + + " stopped boolean,\n" + " PRIMARY KEY (keyspace_name, table_name, token_start)" + ')', UTF8Type.instance)); } @@ -139,23 +147,30 @@ public class AccordDebugKeyspace extends VirtualKeyspace @Override public DataSet data() { - DurabilityScheduling.ImmutableView view = ((AccordService) AccordService.instance()).durabilityScheduling(); + ShardDurability.ImmutableView view = ((AccordService) AccordService.instance()).shardDurability(); SimpleDataSet ds = new SimpleDataSet(metadata()); while (view.advance()) { - TableId tableId = (TableId) view.range().start().prefix(); + TableId tableId = (TableId) view.shard().range.start().prefix(); TableMetadata tableMetadata = tableMetadata(tableId); - ds.row(keyspace(tableMetadata), table(tableId, tableMetadata), sortToken(view.range().start())) - .column("start_token", printToken(view.range().start())) - .column("end_token", printToken(view.range().end())) + ds.row(keyspace(tableMetadata), table(tableId, tableMetadata), sortToken(view.shard().range.start())) + .column("start_token", printToken(view.shard().range.start())) + .column("end_token", printToken(view.shard().range.end())) + .column("last_started_at", approxTime.translate().toMillisSinceEpoch(view.lastStartedAtMicros() * 1000)) + .column("cycle_started_at", approxTime.translate().toMillisSinceEpoch(view.cycleStartedAtMicros() * 1000)) + .column("active", Objects.toString(view.active())) + .column("waiting", Objects.toString(view.waiting())) .column("node_offset", view.nodeOffset()) - .column("index", view.index()) - .column("number_of_splits", view.numberOfSplits()) - .column("range_started_at", view.rangeStartedAtMicros()) - .column("cycle_started_at", view.cycleStartedAtMicros()) - .column("retry_delay_micros", view.retryDelayMicros()) - .column("is_defunct", view.isDefunct()); + .column("cycle_offset", view.cycleOffset()) + .column("activeIndex", view.activeIndex()) + .column("nextIndex", view.nextIndex()) + .column("nextToIndex", view.toIndex()) + .column("endIndex", view.cycleLength()) + .column("current_splits", view.currentSplits()) + .column("stopping", view.stopping()) + .column("stopping", view.stopping()) + ; } return ds; } diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java index faeefcf973..350b7445e1 100644 --- a/src/java/org/apache/cassandra/hints/HintsDispatcher.java +++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Queue; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import java.util.function.BooleanSupplier; import java.util.function.Function; import javax.annotation.Nonnull; @@ -37,14 +38,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask; -import org.apache.cassandra.concurrent.ImmediateExecutor; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.exceptions.RetryOnDifferentSystemException; -import org.apache.cassandra.exceptions.WriteFailureException; -import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.io.util.File; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; @@ -52,9 +51,7 @@ import org.apache.cassandra.metrics.HintsServiceMetrics; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.RequestCallback; -import org.apache.cassandra.service.accord.AccordService; -import org.apache.cassandra.service.accord.IAccordService; -import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult; +import org.apache.cassandra.service.accord.IAccordService.IAccordResult; import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitMutation; @@ -353,8 +350,8 @@ final class HintsDispatcher implements AutoCloseable ClusterMetadata cm = ClusterMetadata.current(); SplitHint splitHint = splitHintIntoAccordAndNormal(cm, hint); Mutation accordHintMutation = splitHint.accordMutation; - Dispatcher.RequestTime requestTime = null; - AsyncTxnResult accordTxnResult = null; + Dispatcher.RequestTime requestTime; + IAccordResult accordTxnResult = null; if (accordHintMutation != null) { requestTime = Dispatcher.RequestTime.forImmediateExecution(); @@ -362,7 +359,7 @@ final class HintsDispatcher implements AutoCloseable } Hint normalHint = splitHint.normalHint; - Callback callback = new Callback(address, hint.creationTime, requestTime, accordTxnResult); + Callback callback = new Callback(address, hint.creationTime, accordTxnResult); if (normalHint != null) { // We had a hint that was supposed to be done on Accord for the batch log (otherwise address would be non-null), @@ -438,7 +435,7 @@ final class HintsDispatcher implements AutoCloseable return callback; } - static final class Callback implements RequestCallback, Runnable + static final class Callback implements RequestCallback, BiConsumer { enum Outcome { SUCCESS, TIMEOUT, FAILURE, INTERRUPTED, RETRY_DIFFERENT_SYSTEM } @@ -449,23 +446,18 @@ final class HintsDispatcher implements AutoCloseable @Nullable private final InetAddressAndPort to; private final long hintCreationNanoTime; - @Nullable - private final Dispatcher.RequestTime requestTime; - private final AsyncTxnResult accordTxnResult; private Callback(@Nonnull InetAddressAndPort to, long hintCreationTimeMillisSinceEpoch) { - this(to, hintCreationTimeMillisSinceEpoch, null, null); + this(to, hintCreationTimeMillisSinceEpoch, null); } - private Callback(@Nullable InetAddressAndPort to, long hintCreationTimeMillisSinceEpoch, Dispatcher.RequestTime requestTime, @Nullable AsyncTxnResult accordTxnResult) + private Callback(@Nullable InetAddressAndPort to, long hintCreationTimeMillisSinceEpoch, @Nullable IAccordResult accordTxnResult) { this.to = to != null ? to : ACCORD_HINT_ENDPOINT; this.hintCreationNanoTime = approxTime.translate().fromMillisSinceEpoch(hintCreationTimeMillisSinceEpoch); - this.requestTime = requestTime; - this.accordTxnResult = accordTxnResult; if (accordTxnResult != null) - accordTxnResult.addListener(this, ImmediateExecutor.INSTANCE); + accordTxnResult.addCallback(this); else accordOutcome = SUCCESS; } @@ -533,34 +525,35 @@ final class HintsDispatcher implements AutoCloseable } @Override - public void run() + public void accept(TxnResult success, Throwable fail) { - try + if (fail != null) { - IAccordService accord = AccordService.instance(); - TxnResult.Kind kind = accord.getTxnResult(accordTxnResult).kind(); - if (kind == retry_new_protocol) + if (fail instanceof RequestExecutionException || fail instanceof RetryOnDifferentSystemException) + { + if (fail instanceof RetryOnDifferentSystemException) + accordOutcome = RETRY_DIFFERENT_SYSTEM; + else + accordOutcome = TIMEOUT; + String msg = "Accord hint delivery transaction failed retriably"; + if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime())) + logger.error(msg, fail); + } + else + { + accordOutcome = FAILURE; + String msg = "Accord hint delivery transaction failed permanently"; + if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime())) + logger.error(msg, fail); + } + } + else + { + if (success.kind() == retry_new_protocol) accordOutcome = RETRY_DIFFERENT_SYSTEM; else accordOutcome = SUCCESS; } - catch (WriteTimeoutException | WriteFailureException | RetryOnDifferentSystemException e) - { - if (e instanceof RetryOnDifferentSystemException) - accordOutcome = RETRY_DIFFERENT_SYSTEM; - else - accordOutcome = TIMEOUT; - String msg = "Accord hint delivery transaction failed retriably"; - if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime())) - logger.error(msg, e); - } - catch (Exception e) - { - accordOutcome = FAILURE; - String msg = "Accord hint delivery transaction failed permanently"; - if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime())) - logger.error(msg, e); - } maybeSignal(); } } diff --git a/src/java/org/apache/cassandra/index/accord/RangeMemoryIndex.java b/src/java/org/apache/cassandra/index/accord/RangeMemoryIndex.java index 43eba0f806..faefc0a51b 100644 --- a/src/java/org/apache/cassandra/index/accord/RangeMemoryIndex.java +++ b/src/java/org/apache/cassandra/index/accord/RangeMemoryIndex.java @@ -36,8 +36,8 @@ import java.util.TreeSet; import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; +import accord.primitives.Participants; import accord.primitives.Routable; -import accord.primitives.Route; import accord.primitives.Unseekable; import org.apache.cassandra.cache.IMeasurableMemory; import org.apache.cassandra.db.Clustering; @@ -52,7 +52,7 @@ import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.RTree; import org.apache.cassandra.utils.RangeTree; -import static org.apache.cassandra.index.accord.RouteIndexFormat.deserializeParticipants; +import static org.apache.cassandra.index.accord.RouteIndexFormat.deserializeTouches; public class RangeMemoryIndex { @@ -107,10 +107,10 @@ public class RangeMemoryIndex public synchronized long add(DecoratedKey key, Clustering clustering, ByteBuffer value) { - Route route; + Participants route; try { - route = deserializeParticipants(value); + route = deserializeTouches(value); } catch (IOException e) { @@ -120,7 +120,7 @@ public class RangeMemoryIndex return add(key, route); } - public synchronized long add(DecoratedKey key, Route route) + public synchronized long add(DecoratedKey key, Participants route) { if (route == null || route.domain() != Routable.Domain.Range) return 0; diff --git a/src/java/org/apache/cassandra/index/accord/RouteIndexFormat.java b/src/java/org/apache/cassandra/index/accord/RouteIndexFormat.java index 1821e2c162..33c17b84c2 100644 --- a/src/java/org/apache/cassandra/index/accord/RouteIndexFormat.java +++ b/src/java/org/apache/cassandra/index/accord/RouteIndexFormat.java @@ -35,7 +35,6 @@ import com.google.common.collect.Maps; import accord.local.StoreParticipants; import accord.primitives.Participants; -import accord.primitives.Route; import accord.primitives.TxnId; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.ByteBufferAccessor; @@ -72,7 +71,7 @@ public class RouteIndexFormat { public static final Supplier CHECKSUM_SUPPLIER = CRC32C::new; - static final LocalVersionedSerializer> participants = localSerializer(KeySerializers.participants); + static final LocalVersionedSerializer> touches = localSerializer(KeySerializers.participants); private static LocalVersionedSerializer localSerializer(IVersionedSerializer serializer) { return new LocalVersionedSerializer<>(AccordSerializerVersion.CURRENT, AccordSerializerVersion.serializer, serializer); @@ -80,23 +79,23 @@ public class RouteIndexFormat public static ByteBuffer serialize(Participants value) throws IOException { - int size = Math.toIntExact(participants.serializedSize(value)); + int size = Math.toIntExact(touches.serializedSize(value)); try (DataOutputBuffer buffer = new DataOutputBuffer(size)) { - participants.serialize(value, buffer); + touches.serialize(value, buffer); return buffer.buffer(true); } } - static Route deserializeParticipants(ByteBuffer bytes) throws IOException + static Participants deserializeTouches(ByteBuffer bytes) throws IOException { if (bytes == null || ByteBufferAccessor.instance.isEmpty(bytes)) return null; try (DataInputBuffer in = new DataInputBuffer(bytes, true)) { - MessageVersionProvider versionProvider = participants.deserializeVersion(in); - return KeySerializers.route.deserialize(in, versionProvider.messageVersion()); + MessageVersionProvider versionProvider = touches.deserializeVersion(in); + return KeySerializers.participants.deserialize(in, versionProvider.messageVersion()); } } diff --git a/src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java b/src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java index dec6a5d916..3dc3509dff 100644 --- a/src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java @@ -18,6 +18,8 @@ package org.apache.cassandra.metrics; +import javax.annotation.Nullable; + import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; @@ -25,6 +27,8 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; public class AccordClientRequestMetrics extends ClientRequestMetrics { + public final @Nullable ClientRequestMetrics shared; + public final Histogram keySize; // During migration back to Paxos it's possible a transaction runs @@ -41,10 +45,13 @@ public class AccordClientRequestMetrics extends ClientRequestMetrics public final Meter accordMigrationRejects; public final Meter preempted; public final Meter topologyMismatches; + public final boolean isWrite; - public AccordClientRequestMetrics(String scope) + public AccordClientRequestMetrics(String scope, ClientRequestMetrics shared, boolean isWrite) { super(scope); + this.shared = shared; + this.isWrite = isWrite; keySize = Metrics.histogram(factory.createMetricName("KeySizeHistogram"), false); migrationSkippedReads = Metrics.meter(factory.createMetricName("MigrationSkippedReads")); @@ -64,6 +71,6 @@ public class AccordClientRequestMetrics extends ClientRequestMetrics Metrics.remove(factory.createMetricName("AccordMigrationRejects")); Metrics.remove(factory.createMetricName("Preempted")); Metrics.remove(factory.createMetricName("TopologyMismatches")); - } + } diff --git a/src/java/org/apache/cassandra/metrics/AccordMetrics.java b/src/java/org/apache/cassandra/metrics/AccordMetrics.java index 8195d46f45..6367fe7873 100644 --- a/src/java/org/apache/cassandra/metrics/AccordMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordMetrics.java @@ -21,7 +21,7 @@ package org.apache.cassandra.metrics; import java.lang.reflect.Field; import java.util.concurrent.TimeUnit; -import accord.api.EventsListener; +import accord.api.EventListener; import accord.local.Command; import accord.primitives.Deps; import accord.primitives.PartialDeps; @@ -181,7 +181,7 @@ public class AccordMetrics return builder.toString(); } - public static class Listener implements EventsListener + public static class Listener implements EventListener { public final static Listener instance = new Listener(AccordMetrics.readMetrics, AccordMetrics.writeMetrics); @@ -198,7 +198,7 @@ public class AccordMetrics { if (txnId.isWrite()) return writeMetrics; - else if (txnId.isRead()) + else if (txnId.isSomeRead()) return readMetrics; else return null; @@ -289,6 +289,8 @@ public class AccordMetrics @Override public void onTimeout(TxnId txnId) { + // TODO (required): we appear to be marking this twice, once in AccordResult and once here. + // why does AccordMetricsTest only see this one? remove duplication. AccordMetrics metrics = forTransaction(txnId); if (metrics != null) metrics.timeouts.mark(); diff --git a/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java b/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java index 05d17a0fda..d996e0140b 100644 --- a/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java +++ b/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java @@ -21,6 +21,7 @@ import java.util.EnumMap; import java.util.Map; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.service.accord.ClientRequestBookkeeping; public final class ClientRequestsMetricsHolder { @@ -29,8 +30,10 @@ public final class ClientRequestsMetricsHolder public static final CASClientWriteRequestMetrics casWriteMetrics = new CASClientWriteRequestMetrics("CASWrite"); public static final CASClientRequestMetrics casReadMetrics = new CASClientRequestMetrics("CASRead"); public static final ViewWriteMetrics viewWriteMetrics = new ViewWriteMetrics("ViewWrite"); - public static final AccordClientRequestMetrics accordReadMetrics = new AccordClientRequestMetrics("AccordRead"); - public static final AccordClientRequestMetrics accordWriteMetrics = new AccordClientRequestMetrics("AccordWrite"); + public static final AccordClientRequestMetrics accordReadMetrics = new AccordClientRequestMetrics("AccordRead", readMetrics, false); + public static final AccordClientRequestMetrics accordWriteMetrics = new AccordClientRequestMetrics("AccordWrite", writeMetrics, true); + public static final ClientRequestBookkeeping accordReadBookkeeping = new ClientRequestBookkeeping(false, accordReadMetrics); + public static final ClientRequestBookkeeping accordWriteBookkeeping = new ClientRequestBookkeeping(true, accordWriteMetrics); public static final Map readMetricsMap = new EnumMap<>(ConsistencyLevel.class); public static final Map writeMetricsMap = new EnumMap<>(ConsistencyLevel.class); diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index b15fc8fbd7..209c5a7a8e 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -104,9 +104,9 @@ public class KeyspaceMetrics /** Latency for locally run key migrations **/ public final LatencyMetrics keyMigration; /** Latency for range migrations run by locally coordinated Accord repairs **/ - public final LatencyMetrics rangeMigration; + public final LatencyMetrics accordRepair; + public final LatencyMetrics accordPostStreamRepair; public final Meter rangeMigrationUnexpectedFailures; - public final Meter rangeMigrationDependencyLimitFailures; public final Meter mutationsRejectedOnWrongSystem; public final Meter readsRejectedOnWrongSystem; /** Writes failed ideal consistency **/ @@ -256,9 +256,9 @@ public class KeyspaceMetrics casPropose = createLatencyMetrics("CasPropose"); casCommit = createLatencyMetrics("CasCommit"); keyMigration = createLatencyMetrics("KeyMigration"); - rangeMigration = createLatencyMetrics("RangeMigration"); + accordRepair = createLatencyMetrics("AccordRepair"); + accordPostStreamRepair = createLatencyMetrics("AccordPostStreamRepair"); rangeMigrationUnexpectedFailures = createKeyspaceMeter("RangeMigrationUnexpectedFailures"); - rangeMigrationDependencyLimitFailures = createKeyspaceMeter("RangeMigratingDependencyLimitFailures"); mutationsRejectedOnWrongSystem = createKeyspaceMeter("MutationsRejectedOnWrongSystem"); readsRejectedOnWrongSystem = createKeyspaceMeter("ReadsRejectedOnWrongSystem"); writeFailedIdealCL = createKeyspaceCounter("WriteFailedIdealCL"); diff --git a/src/java/org/apache/cassandra/metrics/RepairMetrics.java b/src/java/org/apache/cassandra/metrics/RepairMetrics.java index 27dbbd3118..bcdb1b44f9 100644 --- a/src/java/org/apache/cassandra/metrics/RepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/RepairMetrics.java @@ -87,8 +87,8 @@ public class RepairMetrics public static void retry(Verb verb, int attempt) { - retries.update(attempt); - retriesByVerb.get(verb).update(attempt); + retries.update(attempt - 1); + retriesByVerb.get(verb).update(attempt - 1); } public static void retryTimeout(Verb verb) diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index d65bdebcdd..719c8af811 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -194,8 +194,8 @@ public class TableMetrics public final LatencyMetrics keyMigration; /** Latency for range migrations run by locally coordinated Accord repairs **/ public final LatencyMetrics accordRepair; + public final LatencyMetrics accordPostStreamRepair; public final TableMeter accordRepairUnexpectedFailures; - public final TableMeter accordRepairDependencyLimitFailures; public final TableMeter mutationsRejectedOnWrongSystem; public final TableMeter readsRejectedOnWrongSystem; /** percent of the data that is repaired */ @@ -816,9 +816,9 @@ public class TableMetrics casPropose = createLatencyMetrics("CasPropose", cfs.keyspace.metric.casPropose); casCommit = createLatencyMetrics("CasCommit", cfs.keyspace.metric.casCommit); keyMigration = createLatencyMetrics("KeyMigration", cfs.keyspace.metric.keyMigration, GLOBAL_KEY_MIGRATION_LATENCY); - accordRepair = createLatencyMetrics("AccordRepair", cfs.keyspace.metric.rangeMigration, GLOBAL_RANGE_MIGRATION_LATENCY); + accordRepair = createLatencyMetrics("AccordRepair", cfs.keyspace.metric.accordRepair, GLOBAL_RANGE_MIGRATION_LATENCY); + accordPostStreamRepair = createLatencyMetrics("AccordPostStreamRepair", cfs.keyspace.metric.accordPostStreamRepair); accordRepairUnexpectedFailures = createTableMeter("AccordRepairUnexpectedFailures", cfs.keyspace.metric.rangeMigrationUnexpectedFailures); - accordRepairDependencyLimitFailures = createTableMeter("AccordRepairDependencyLimitFaiures", cfs.keyspace.metric.rangeMigrationDependencyLimitFailures); mutationsRejectedOnWrongSystem = createTableMeter("MutationsRejectedOnWrongSystem", cfs.keyspace.metric.mutationsRejectedOnWrongSystem); readsRejectedOnWrongSystem = createTableMeter("ReadsRejectedOnWrongSystem", cfs.keyspace.metric.readsRejectedOnWrongSystem); diff --git a/src/java/org/apache/cassandra/net/MessageDelivery.java b/src/java/org/apache/cassandra/net/MessageDelivery.java index c57366d450..ccface2253 100644 --- a/src/java/org/apache/cassandra/net/MessageDelivery.java +++ b/src/java/org/apache/cassandra/net/MessageDelivery.java @@ -22,16 +22,18 @@ import java.util.Collection; import java.util.Iterator; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import accord.utils.Invariants; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.utils.Backoff; +import org.apache.cassandra.service.WaitStrategy; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Accumulator; import org.apache.cassandra.utils.concurrent.AsyncPromise; @@ -39,6 +41,7 @@ import org.apache.cassandra.utils.concurrent.CountDownLatch; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.Promise; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.net.MessageFlag.CALL_BACK_ON_FAILURE; public interface MessageDelivery @@ -83,7 +86,7 @@ public interface MessageDelivery public void sendWithCallback(Message message, InetAddressAndPort to, RequestCallback cb, ConnectionType specifyConnection); public Future> sendWithResult(Message message, InetAddressAndPort to); - public default Future> sendWithRetries(Backoff backoff, RetryScheduler retryThreads, + public default Future> sendWithRetries(WaitStrategy backoff, RetryScheduler retryThreads, Verb verb, REQ request, Iterator candidates, RetryPredicate shouldRetry, @@ -99,14 +102,14 @@ public interface MessageDelivery return promise; } - public default void sendWithRetries(Backoff backoff, RetryScheduler retryThreads, + public default void sendWithRetries(WaitStrategy backoff, RetryScheduler retryThreads, Verb verb, REQ request, Iterator candidates, OnResult onResult, RetryPredicate shouldRetry, RetryErrorMessage errorMessage) { - sendWithRetries(this, backoff, retryThreads, verb, request, candidates, onResult, shouldRetry, errorMessage, 0); + sendWithRetries(this, backoff, retryThreads, verb, request, candidates, onResult, shouldRetry, errorMessage, 1); } public void respond(V response, Message message); public default void respondWithFailure(RequestFailureReason reason, Message message) @@ -139,7 +142,7 @@ public interface MessageDelivery } private static void sendWithRetries(MessageDelivery messaging, - Backoff backoff, + WaitStrategy backoff, RetryScheduler retryThreads, Verb verb, REQ request, Iterator candidates, @@ -148,6 +151,7 @@ public interface MessageDelivery RetryErrorMessage errorMessage, int attempt) { + Invariants.require(backoff != null); if (Thread.currentThread().isInterrupted()) { onResult.result(attempt, null, new InterruptedException(errorMessage.apply(attempt, ResponseFailureReason.Interrupted, null, null))); @@ -169,10 +173,11 @@ public interface MessageDelivery @Override public void onFailure(InetAddressAndPort from, RequestFailure failure) { - // TODO (required): we already have a separate retry predicate, backoff should not be taken into consideration when retrying - if (!backoff.mayRetry(attempt)) + long retryDelay = backoff.computeWait(attempt + 1, NANOSECONDS); + // TODO (required): we already have a separate retry predicate, retries should not be taken into consideration when retrying + if (retryDelay < 0) { - onResult.result(attempt, null, new MaxRetriesException(attempt, errorMessage.apply(attempt, ResponseFailureReason.MaxRetries, from, failure))); + onResult.result(attempt, null, new GivingUpException(attempt, errorMessage.apply(attempt, ResponseFailureReason.GiveUp, from, failure))); return; } if (!shouldRetry.test(attempt, from, failure)) @@ -183,7 +188,7 @@ public interface MessageDelivery try { retryThreads.schedule(() -> sendWithRetries(messaging, backoff, retryThreads, verb, request, candidates, onResult, shouldRetry, errorMessage, attempt + 1), - backoff.computeWaitTime(attempt), backoff.unit()); + retryDelay, NANOSECONDS); } catch (Throwable t) { @@ -194,7 +199,7 @@ public interface MessageDelivery messaging.sendWithCallback(Message.outWithFlag(verb, request, CALL_BACK_ON_FAILURE), candidates.next(), new Request()); } - enum ResponseFailureReason { MaxRetries, Rejected, NoMoreCandidates, Interrupted, FailedSchedule } + enum ResponseFailureReason {GiveUp, Rejected, NoMoreCandidates, Interrupted, FailedSchedule } interface RetryScheduler { @@ -212,7 +217,7 @@ public interface MessageDelivery } } - class NoMoreCandidatesException extends IllegalStateException + class NoMoreCandidatesException extends TimeoutException { public NoMoreCandidatesException(String s) { @@ -220,7 +225,7 @@ public interface MessageDelivery } } - class FailedResponseException extends IllegalStateException + class FailedResponseException extends RuntimeException { public final InetAddressAndPort from; public final RequestFailure failure; @@ -233,17 +238,17 @@ public interface MessageDelivery } } - class MaxRetriesException extends IllegalStateException + class GivingUpException extends TimeoutException { public final int attempts; - public MaxRetriesException(int attempts, String message) + public GivingUpException(int attempts, String message) { super(message); this.attempts = attempts; } } - class FailedScheduleException extends IllegalStateException + class FailedScheduleException extends RuntimeException { public FailedScheduleException(String message, Throwable cause) { diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java index 77a980d2cb..20173ad443 100644 --- a/src/java/org/apache/cassandra/net/Verb.java +++ b/src/java/org/apache/cassandra/net/Verb.java @@ -95,10 +95,11 @@ import org.apache.cassandra.service.accord.serializers.CommitSerializers; import org.apache.cassandra.service.accord.serializers.EnumSerializer; import org.apache.cassandra.service.accord.serializers.FetchSerializers; import org.apache.cassandra.service.accord.serializers.GetEphmrlReadDepsSerializers; +import org.apache.cassandra.service.accord.serializers.GetMaxConflictSerializers; import org.apache.cassandra.service.accord.serializers.InformDurableSerializers; import org.apache.cassandra.service.accord.serializers.LatestDepsSerializers; import org.apache.cassandra.service.accord.serializers.PreacceptSerializers; -import org.apache.cassandra.service.accord.serializers.QueryDurableBeforeSerializers; +import org.apache.cassandra.service.accord.serializers.GetDurableBeforeSerializers; import org.apache.cassandra.service.accord.serializers.ReadDataSerializers; import org.apache.cassandra.service.accord.serializers.RecoverySerializers; import org.apache.cassandra.service.accord.serializers.SetDurableSerializers; @@ -318,51 +319,52 @@ public enum Verb ACCORD_NOT_ACCEPT_REQ (124, P2, writeTimeout, IMMEDIATE, () -> AcceptSerializers.notAccept, AccordService::requestHandlerOrNoop, ACCORD_ACCEPT_RSP ), ACCORD_READ_RSP (125, P2, readTimeout, IMMEDIATE, () -> ReadDataSerializers.reply, AccordService::responseHandlerOrNoop ), ACCORD_READ_REQ (126, P2, readTimeout, IMMEDIATE, () -> ReadDataSerializers.readData, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ), - ACCORD_COMMIT_REQ (127, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ), - ACCORD_COMMIT_INVALIDATE_REQ (128, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.invalidate, AccordService::requestHandlerOrNoop ), - ACCORD_APPLY_RSP (129, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.reply, AccordService::responseHandlerOrNoop ), - ACCORD_APPLY_REQ (130, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.request, AccordService::requestHandlerOrNoop, ACCORD_APPLY_RSP ), - ACCORD_BEGIN_RECOVER_RSP (131, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.reply, AccordService::responseHandlerOrNoop ), - ACCORD_BEGIN_RECOVER_REQ (132, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.request, AccordService::requestHandlerOrNoop, ACCORD_BEGIN_RECOVER_RSP ), - ACCORD_BEGIN_INVALIDATE_RSP (133, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.reply, AccordService::responseHandlerOrNoop ), - ACCORD_BEGIN_INVALIDATE_REQ (134, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_BEGIN_INVALIDATE_RSP ), - ACCORD_AWAIT_RSP (136, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.syncReply, AccordService::responseHandlerOrNoop ), - ACCORD_AWAIT_REQ (135, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_AWAIT_RSP ), - ACCORD_AWAIT_ASYNC_RSP_REQ (137, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.asyncReply, AccordService::requestHandlerOrNoop ), - ACCORD_WAIT_UNTIL_APPLIED_REQ (138, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.waitUntilApplied, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ), - ACCORD_STABLE_THEN_READ_REQ (139, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.stableThenRead, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ), - ACCORD_INFORM_DURABLE_REQ (140, P2, writeTimeout, IMMEDIATE, () -> InformDurableSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ), - ACCORD_CHECK_STATUS_RSP (141, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.reply, AccordService::responseHandlerOrNoop ), - ACCORD_CHECK_STATUS_REQ (142, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_CHECK_STATUS_RSP ), - ACCORD_RECOVER_AWAIT_RSP (143, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.recoverReply, AccordService::responseHandlerOrNoop ), - ACCORD_RECOVER_AWAIT_REQ (144, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.recoverRequest, AccordService::requestHandlerOrNoop, ACCORD_RECOVER_AWAIT_RSP), - ACCORD_GET_LATEST_DEPS_RSP (167, P2, writeTimeout, IMMEDIATE, () -> LatestDepsSerializers.reply, AccordService::responseHandlerOrNoop ), - ACCORD_GET_LATEST_DEPS_REQ (168, P2, writeTimeout, IMMEDIATE, () -> LatestDepsSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_LATEST_DEPS_RSP), - ACCORD_GET_EPHMRL_READ_DEPS_RSP (161, P2, writeTimeout, IMMEDIATE, () -> GetEphmrlReadDepsSerializers.reply, AccordService::responseHandlerOrNoop ), - ACCORD_GET_EPHMRL_READ_DEPS_REQ (162, P2, writeTimeout, IMMEDIATE, () -> GetEphmrlReadDepsSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_EPHMRL_READ_DEPS_RSP), - ACCORD_FETCH_DATA_RSP (145, P2, writeTimeout, IMMEDIATE, () -> FetchSerializers.reply, AccordService::responseHandlerOrNoop ), - ACCORD_FETCH_DATA_REQ (146, P2, writeTimeout, IMMEDIATE, () -> FetchSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_FETCH_DATA_RSP ), - ACCORD_SET_SHARD_DURABLE_REQ (147, P2, writeTimeout, MISC, () -> SetDurableSerializers.shardDurable, AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ), - ACCORD_SET_GLOBALLY_DURABLE_REQ (148, P2, writeTimeout, MISC, () -> SetDurableSerializers.globallyDurable,AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ), - ACCORD_QUERY_DURABLE_BEFORE_RSP (149, P2, writeTimeout, IMMEDIATE, () -> QueryDurableBeforeSerializers.reply, AccordService::responseHandlerOrNoop ), - ACCORD_QUERY_DURABLE_BEFORE_REQ (150, P2, writeTimeout, IMMEDIATE, () -> QueryDurableBeforeSerializers.request,AccordService::requestHandlerOrNoop, ACCORD_QUERY_DURABLE_BEFORE_RSP ), + ACCORD_STABLE_THEN_READ_REQ (127, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.stableThenRead, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ), + ACCORD_COMMIT_REQ (128, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ), + ACCORD_COMMIT_INVALIDATE_REQ (129, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.invalidate, AccordService::requestHandlerOrNoop ), + ACCORD_APPLY_RSP (130, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.reply, AccordService::responseHandlerOrNoop ), + ACCORD_APPLY_REQ (131, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.request, AccordService::requestHandlerOrNoop, ACCORD_APPLY_RSP ), + ACCORD_APPLY_AND_WAIT_REQ (132, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.readData, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP), + ACCORD_BEGIN_RECOVER_RSP (133, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.reply, AccordService::responseHandlerOrNoop ), + ACCORD_BEGIN_RECOVER_REQ (134, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.request, AccordService::requestHandlerOrNoop, ACCORD_BEGIN_RECOVER_RSP ), + ACCORD_BEGIN_INVALIDATE_RSP (135, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.reply, AccordService::responseHandlerOrNoop ), + ACCORD_BEGIN_INVALIDATE_REQ (136, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_BEGIN_INVALIDATE_RSP ), + ACCORD_AWAIT_RSP (137, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.syncReply, AccordService::responseHandlerOrNoop ), + ACCORD_AWAIT_REQ (138, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_AWAIT_RSP ), + ACCORD_AWAIT_ASYNC_RSP_REQ (139, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.asyncReply, AccordService::requestHandlerOrNoop ), + ACCORD_WAIT_UNTIL_APPLIED_REQ (140, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.waitUntilApplied, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ), + ACCORD_RECOVER_AWAIT_RSP (141, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.recoverReply, AccordService::responseHandlerOrNoop ), + ACCORD_RECOVER_AWAIT_REQ (142, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.recoverRequest, AccordService::requestHandlerOrNoop, ACCORD_RECOVER_AWAIT_RSP), + ACCORD_INFORM_DURABLE_REQ (143, P2, writeTimeout, IMMEDIATE, () -> InformDurableSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ), + ACCORD_CHECK_STATUS_RSP (144, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.reply, AccordService::responseHandlerOrNoop ), + ACCORD_CHECK_STATUS_REQ (145, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_CHECK_STATUS_RSP ), + ACCORD_FETCH_DATA_RSP (146, P2, writeTimeout, IMMEDIATE, () -> FetchSerializers.reply, AccordService::responseHandlerOrNoop ), + ACCORD_FETCH_DATA_REQ (147, P2, writeTimeout, IMMEDIATE, () -> FetchSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_FETCH_DATA_RSP ), + ACCORD_GET_EPHMRL_READ_DEPS_RSP (148, P2, readTimeout, IMMEDIATE, () -> GetEphmrlReadDepsSerializers.reply, AccordService::responseHandlerOrNoop ), + ACCORD_GET_EPHMRL_READ_DEPS_REQ (149, P2, readTimeout, IMMEDIATE, () -> GetEphmrlReadDepsSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_EPHMRL_READ_DEPS_RSP), + ACCORD_GET_LATEST_DEPS_RSP (150, P2, readTimeout, IMMEDIATE, () -> LatestDepsSerializers.reply, AccordService::responseHandlerOrNoop ), + ACCORD_GET_LATEST_DEPS_REQ (151, P2, readTimeout, IMMEDIATE, () -> LatestDepsSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_LATEST_DEPS_RSP), + ACCORD_GET_MAX_CONFLICT_RSP (152, P2, readTimeout, IMMEDIATE, () -> GetMaxConflictSerializers.reply, AccordService::responseHandlerOrNoop ), + ACCORD_GET_MAX_CONFLICT_REQ (153, P2, readTimeout, IMMEDIATE, () -> GetMaxConflictSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_MAX_CONFLICT_RSP), + ACCORD_GET_DURABLE_BEFORE_RSP (154, P2, readTimeout, IMMEDIATE, () -> GetDurableBeforeSerializers.reply, AccordService::responseHandlerOrNoop ), + ACCORD_GET_DURABLE_BEFORE_REQ (155, P2, readTimeout, IMMEDIATE, () -> GetDurableBeforeSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_DURABLE_BEFORE_RSP ), + ACCORD_SET_SHARD_DURABLE_REQ (156, P2, rpcTimeout, MISC, () -> SetDurableSerializers.shardDurable, AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ), + ACCORD_SET_GLOBALLY_DURABLE_REQ (157, P2, rpcTimeout, MISC, () -> SetDurableSerializers.globallyDurable,AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ), - ACCORD_SYNC_NOTIFY_RSP (151, P2, writeTimeout, MISC, () -> EnumSerializer.simpleReply, RESPONSE_HANDLER), - ACCORD_SYNC_NOTIFY_REQ (152, P2, writeTimeout, MISC, () -> Notification.listSerializer, () -> AccordSyncPropagator.verbHandler, ACCORD_SYNC_NOTIFY_RSP ), + ACCORD_SYNC_NOTIFY_RSP (158, P2, writeTimeout, MISC, () -> EnumSerializer.simpleReply, RESPONSE_HANDLER), + ACCORD_SYNC_NOTIFY_REQ (159, P2, writeTimeout, MISC, () -> Notification.listSerializer, () -> AccordSyncPropagator.verbHandler, ACCORD_SYNC_NOTIFY_RSP ), - ACCORD_APPLY_AND_WAIT_REQ (153, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.readData, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP), - CONSENSUS_KEY_MIGRATION (154, P1, writeTimeout, MISC, () -> ConsensusKeyMigrationFinished.serializer,() -> ConsensusKeyMigrationState.consensusKeyMigrationFinishedHandler), + CONSENSUS_KEY_MIGRATION (160, P1, writeTimeout, MISC, () -> ConsensusKeyMigrationFinished.serializer,() -> ConsensusKeyMigrationState.consensusKeyMigrationFinishedHandler), - ACCORD_INTEROP_READ_RSP (155, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.replySerializer, AccordService::responseHandlerOrNoop), - ACCORD_INTEROP_READ_REQ (156, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_RSP), - ACCORD_INTEROP_STABLE_THEN_READ_REQ(157, P2, writeTimeout, IMMEDIATE, () -> AccordInteropStableThenRead.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_RSP), - ACCORD_INTEROP_READ_REPAIR_RSP (158, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.replySerializer, AccordService::responseHandlerOrNoop), - ACCORD_INTEROP_READ_REPAIR_REQ (159, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_REPAIR_RSP), - ACCORD_INTEROP_APPLY_REQ (160, P2, writeTimeout, IMMEDIATE, () -> AccordInteropApply.serializer, AccordService::requestHandlerOrNoop, ACCORD_APPLY_RSP), - // TODO (desired): swap verb order to make IDS sequential? - ACCORD_FETCH_MIN_EPOCH_RSP (166, P0, shortTimeout, FETCH_METADATA, () -> FetchMinEpoch.Response.serializer, RESPONSE_HANDLER), - ACCORD_FETCH_MIN_EPOCH_REQ (165, P0, shortTimeout, FETCH_METADATA, () -> FetchMinEpoch.serializer, () -> FetchMinEpoch.handler, ACCORD_FETCH_MIN_EPOCH_RSP), + ACCORD_INTEROP_READ_RSP (161, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.replySerializer, AccordService::responseHandlerOrNoop), + ACCORD_INTEROP_READ_REQ (162, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_RSP), + ACCORD_INTEROP_STABLE_THEN_READ_REQ(163, P2, writeTimeout, IMMEDIATE, () -> AccordInteropStableThenRead.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_RSP), + ACCORD_INTEROP_READ_REPAIR_RSP (164, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.replySerializer, AccordService::responseHandlerOrNoop), + ACCORD_INTEROP_READ_REPAIR_REQ (165, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_REPAIR_RSP), + ACCORD_INTEROP_APPLY_REQ (166, P2, writeTimeout, IMMEDIATE, () -> AccordInteropApply.serializer, AccordService::requestHandlerOrNoop, ACCORD_APPLY_RSP), + ACCORD_FETCH_MIN_EPOCH_RSP (167, P0, shortTimeout, FETCH_METADATA, () -> FetchMinEpoch.Response.serializer, RESPONSE_HANDLER), + ACCORD_FETCH_MIN_EPOCH_REQ (168, P0, shortTimeout, FETCH_METADATA, () -> FetchMinEpoch.serializer, () -> FetchMinEpoch.handler, ACCORD_FETCH_MIN_EPOCH_RSP), ACCORD_FETCH_TOPOLOGY_RSP (169, P0, shortTimeout, FETCH_METADATA, () -> FetchTopology.Response.serializer, RESPONSE_HANDLER), ACCORD_FETCH_TOPOLOGY_REQ (170, P0, shortTimeout, FETCH_METADATA, () -> FetchTopology.serializer, () -> FetchTopology.handler, ACCORD_FETCH_TOPOLOGY_RSP), diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index 800f745153..f20fe189f2 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -43,7 +43,6 @@ import accord.primitives.Ranges; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; @@ -191,8 +190,7 @@ public class RepairJob extends AsyncFuture implements Runnable requireAllEndpoints = true; } logger.info("{} {}.{} starting accord repair, require all endpoints {}", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily, requireAllEndpoints); - IPartitioner partitioner = metadata.partitioner; - AccordRepair repair = new AccordRepair(ctx, cfs, partitioner, desc.keyspace, desc.ranges, requireAllEndpoints, allEndpoints); + AccordRepair repair = new AccordRepair(ctx, cfs, desc.sessionId, desc.keyspace, desc.ranges, requireAllEndpoints, allEndpoints); return repair.repair(taskExecutor); }, taskExecutor); } diff --git a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java index 2eab88ef91..c615bf9f52 100644 --- a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java +++ b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java @@ -45,8 +45,8 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.service.WaitStrategy; import org.apache.cassandra.streaming.PreviewKind; -import org.apache.cassandra.utils.Backoff; import org.apache.cassandra.utils.CassandraVersion; import org.apache.cassandra.utils.NoSpamLogger; import org.apache.cassandra.utils.TimeUUID; @@ -134,11 +134,11 @@ public abstract class RepairMessage void onFailure(Exception e); } - private static Backoff backoff(SharedContext ctx, Verb verb) + private static WaitStrategy backoff(SharedContext ctx, Verb verb) { RepairRetrySpec retrySpec = DatabaseDescriptor.getRepairRetrySpec(); RetrySpec spec = verb == Verb.VALIDATION_RSP ? retrySpec.getMerkleTreeResponseSpec() : retrySpec; - return Backoff.fromConfig(ctx, spec); + return RetrySpec.toStrategy(ctx, spec); } public static Supplier notDone(Future f) @@ -172,12 +172,12 @@ public abstract class RepairMessage } @VisibleForTesting - static void sendMessageWithRetries(SharedContext ctx, Backoff backoff, Supplier allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback finalCallback) + static void sendMessageWithRetries(SharedContext ctx, WaitStrategy backoff, Supplier allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback finalCallback) { if (!ALLOWS_RETRY.contains(verb)) throw new AssertionError("Repair verb " + verb + " does not support retry, but a request to send with retry was given!"); BiConsumer maybeRecordRetry = (attempt, reason) -> { - if (attempt <= 0) + if (attempt <= 1) return; // we don't know what the prefix kind is... so use NONE... this impacts logPrefix as it will cause us to use "repair" rather than "preview repair" which may not be correct... but close enough... String prefix = PreviewKind.NONE.logPrefix(request.parentRepairSession()); @@ -229,7 +229,7 @@ public abstract class RepairMessage (attempt, retryReason, from, failure) -> { switch (retryReason) { - case MaxRetries: + case GiveUp: maybeRecordRetry.accept(attempt, failure.reason); finalCallback.onFailure(from, failure); return null; diff --git a/src/java/org/apache/cassandra/service/FailureRecordingCallback.java b/src/java/org/apache/cassandra/service/FailureRecordingCallback.java index b672c4bf3f..aebeea49df 100644 --- a/src/java/org/apache/cassandra/service/FailureRecordingCallback.java +++ b/src/java/org/apache/cassandra/service/FailureRecordingCallback.java @@ -66,7 +66,7 @@ public abstract class FailureRecordingCallback implements RequestCallbackWith public static void push(AtomicReferenceFieldUpdater headUpdater, O owner, InetAddressAndPort from, RequestFailureReason reason) { - push(headUpdater, owner, new FailureResponses(from, reason)); + getAndPush(headUpdater, owner, new FailureResponses(from, reason)); } public static void pushExclusive(AtomicReferenceFieldUpdater headUpdater, O owner, InetAddressAndPort from, RequestFailureReason reason) diff --git a/src/java/org/apache/cassandra/service/RetryStrategy.java b/src/java/org/apache/cassandra/service/RetryStrategy.java index 7f48612a8f..a89f93363d 100644 --- a/src/java/org/apache/cassandra/service/RetryStrategy.java +++ b/src/java/org/apache/cassandra/service/RetryStrategy.java @@ -20,32 +20,38 @@ package org.apache.cassandra.service; import com.google.common.annotations.VisibleForTesting; -import org.apache.cassandra.config.DatabaseDescriptor; +import accord.utils.Invariants; +import accord.utils.RandomSource; import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory; import org.apache.cassandra.service.TimeoutStrategy.Wait; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.function.DoubleSupplier; import java.util.function.LongBinaryOperator; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static java.lang.Math.*; -import static java.util.Arrays.stream; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import static java.util.concurrent.TimeUnit.*; -import static org.apache.cassandra.service.TimeoutStrategy.parseWait; +import static org.apache.cassandra.service.TimeoutStrategy.parseInMicros; import static org.apache.cassandra.utils.Clock.Global.nanoTime; /** *

A strategy for making retry timing decisions for operations. - * The strategy is defined by four factors:

    + * The strategy is defined by six factors:
      + *
    • {@link #minMinMicros} + *
    • {@link #maxMaxMicros} *
    • {@link #min} *
    • {@link #max} - *
    • {@link #spread} *
    • {@link #waitRandomizer} + *
    • {@link #maxAttempts} *
    * - *

    The first three represent time periods, and may be defined dynamically based on a simple calculation over:

      + *

      The first two represent the absolute upper and lower bound times we are permitted to produce as constants

      + *

      The next two represent time periods, and may be defined dynamically based on a simple calculation over:

        *
      • {@code pX()} recent experienced latency distribution for successful operations, * e.g. {@code p50(rw)} the maximum of read and write median latencies, * {@code p999(r)} the 99.9th percentile of read latencies @@ -59,16 +65,13 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime; *
      • dynamic linear {@code pX() * constant * attempts} *
      • dynamic exponential {@code pX() * constant ^ attempts} * - *

        Furthermore, the dynamic calculations can be bounded with a min/max, like so: - * {@code min[mu]s <= dynamic expr <= max[mu]s} - * * e.g. - *

      • {@code 10ms <= p50(rw)*0.66} + *
      • {@code 10ms <= p50(rw)*0.66...p99(rw)} *
      • {@code 10ms <= p95(rw)*1.8^attempts <= 100ms} *
      • {@code 5ms <= p50(rw)*0.5} * *

        These calculations are put together to construct a range from which we draw a random number. - * The period we wait for {@code X} will be drawn so that {@code min <= X < max}. + * The period we wait for {@code X} will be drawn so that {@code minMin <= min <= max <= maxMax}. * *

        With the constraint that {@code max} must be {@code spread} greater than {@code min}, * but no greater than its expression-defined maximum. {@code max} will be increased up until @@ -87,19 +90,34 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime; * The quantized exponential specifier partitions the range into {@code attempts} buckets, then applies the pure * exponential approach to draw values from [0..attempts), before drawing a uniform value from the corresponding bucket */ -public class RetryStrategy +public class RetryStrategy implements WaitStrategy { private static final Pattern RANDOMIZER = Pattern.compile( "uniform|exp(onential)?[(](?[0-9.]+)[)]|q(uantized)?exp(onential)?[(](?[0-9.]+)[)]"); - final static WaitRandomizerFactory randomizers = new WaitRandomizerFactory(){}; + static final Pattern PARSE = Pattern.compile( + "(\\s*(?0|[0-9]+[mu]?s)\\s*<=)?" + + "(\\s*(?[^=]+)\\s*[.]{3})?" + + "(\\s*(?[^=]+))" + + "(\\s*<=\\s*(?0|[0-9]+[mu]?s))?"); - protected interface WaitRandomizer + public static final WaitRandomizerFactory randomizers = new WaitRandomizerFactory(){}; + + public interface WaitRandomizer { long wait(long min, long max, int attempts); } - interface WaitRandomizerFactory + public static WaitRandomizerFactory randomizers(RandomSource random) + { + return new WaitRandomizerFactory() + { + @Override public LongBinaryOperator uniformLongSupplier() { return random::nextLong; } + @Override public DoubleSupplier uniformDoubleSupplier() { return random::nextDouble; } + }; + } + + public interface WaitRandomizerFactory { default LongBinaryOperator uniformLongSupplier() { return (min, max) -> ThreadLocalRandom.current().nextLong(min, max); } // DO NOT USE METHOD HANDLES (want to fetch afresh each time) default DoubleSupplier uniformDoubleSupplier() { return () -> ThreadLocalRandom.current().nextDouble(); } @@ -108,7 +126,7 @@ public class RetryStrategy default WaitRandomizer exponential(double power) { return new Exponential(uniformLongSupplier(), uniformDoubleSupplier(), power); } default WaitRandomizer quantizedExponential(double power) { return new QuantizedExponential(uniformLongSupplier(), uniformDoubleSupplier(), power); } - static class Uniform implements WaitRandomizer + class Uniform implements WaitRandomizer { final LongBinaryOperator uniformLong; @@ -124,7 +142,7 @@ public class RetryStrategy } } - static abstract class AbstractExponential implements WaitRandomizer + abstract class AbstractExponential implements WaitRandomizer { final LongBinaryOperator uniformLong; final DoubleSupplier uniformDouble; @@ -138,7 +156,7 @@ public class RetryStrategy } } - static class Exponential extends AbstractExponential + class Exponential extends AbstractExponential { public Exponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power) { @@ -158,7 +176,7 @@ public class RetryStrategy } } - static class QuantizedExponential extends AbstractExponential + class QuantizedExponential extends AbstractExponential { public QuantizedExponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power) { @@ -180,108 +198,111 @@ public class RetryStrategy } public final WaitRandomizer waitRandomizer; - public final Wait min, max, spread; + public final long minMinMicros, maxMaxMicros; + public final @Nullable Wait min; + public final @Nonnull Wait max; + public final int maxAttempts; - public RetryStrategy(String waitRandomizer, String min, String max, String spread, LatencySourceFactory latencies) - { - this.waitRandomizer = parseWaitRandomizer(waitRandomizer); - this.min = parseBound(min, true, latencies); - this.max = parseBound(max, false, latencies); - this.spread = parseBound(spread, true, latencies); - } - - protected RetryStrategy(WaitRandomizer waitRandomizer, Wait min, Wait max, Wait spread) + protected RetryStrategy(WaitRandomizer waitRandomizer, long minMinMicros, Wait min, Wait max, long maxMaxMicros, int retries) { this.waitRandomizer = waitRandomizer; + this.minMinMicros = minMinMicros; this.min = min; this.max = max; - this.spread = spread; + this.maxMaxMicros = maxMaxMicros; + this.maxAttempts = retries == Integer.MAX_VALUE ? Integer.MAX_VALUE : retries + 1; + Invariants.require(maxAttempts >= 1); } - protected Wait parseBound(String spec, boolean isMin, LatencySourceFactory latencies) + public long computeWaitUntil(int attempts) { - long defaultMaxMicros = DatabaseDescriptor.getRpcTimeout(MICROSECONDS); - return parseWait(spec, 0, defaultMaxMicros, isMin ? 0 : defaultMaxMicros, latencies); + long wait = computeWait(attempts, NANOSECONDS); + if (wait < 0) + return -1; + return nanoTime() + wait; } - protected long computeWaitUntil(int attempts) + public long computeWait(int attempt, TimeUnit units) { - long wait = computeWait(attempts); - return nanoTime() + MICROSECONDS.toNanos(wait); - } + if (attempt > maxAttempts) + return -1; - protected long computeWait(int attempts) - { - long minWaitMicros = min.get(attempts); - long maxWaitMicros = max.get(attempts); - long spreadMicros = spread.get(attempts); - - if (minWaitMicros + spreadMicros > maxWaitMicros) + long result; + if (min == null) { - maxWaitMicros = minWaitMicros + spreadMicros; - if (maxWaitMicros > this.max.max) + result = max.getMicros(attempt); + } + else + { + long min = this.min.getMicros(attempt); + long max = this.max.getMicros(attempt); + result = min >= max ? min : waitRandomizer.wait(min, max, attempt); + } + + if (result > maxMaxMicros) result = maxMaxMicros; + if (result < minMinMicros) result = minMinMicros; + return units.convert(result, MICROSECONDS); + } + + public static RetryStrategy parse(String spec, LatencySourceFactory latencies) + { + return parse(spec, latencies, null); + } + + public static RetryStrategy parse(String spec, LatencySourceFactory latencies, WaitRandomizer randomizer) + { + String original = spec; + int retries = Integer.MAX_VALUE; + int end = spec.length(); + { + int next; + while ((next = spec.lastIndexOf(',', end - 1)) >= 0) { - maxWaitMicros = this.max.max; - minWaitMicros = max(this.min.min, min(this.min.max, maxWaitMicros - spreadMicros)); + int mid = spec.indexOf('=', next + 1); + if (mid <= next || mid >= end) + throw new IllegalArgumentException("Invalid modifier specification: '" + spec.substring(next, end) + "'; expecting '=' for value assignment"); + String key = spec.substring(next + 1, mid).trim(); + String value = spec.substring(mid + 1, end).trim(); + switch (key) + { + default: throw new IllegalArgumentException("Invalid modifier specification: unrecognised property '" + key + '\''); + case "retries": + retries = Integer.parseInt(value); + break; + case "rnd": + if (randomizer != null) + throw new IllegalArgumentException("Randomizer already specified, cannot re-specify: " + value); + randomizer = parseWaitRandomizer(value); + break; + } + end = next; } + if (end != spec.length()) + spec = spec.substring(0, end); } - return waitRandomizer.wait(minWaitMicros, maxWaitMicros, attempts); - } + Matcher m = PARSE.matcher(spec); + if (!m.matches()) + throw new IllegalArgumentException("Invalid specification: '" + spec + "'; does not match " + PARSE); - public static class ParsedStrategy - { - public final String waitRandomizer, min, max, spread; - public final RetryStrategy strategy; - - protected ParsedStrategy(String waitRandomizer, String min, String max, String spread, RetryStrategy strategy) - { - this.waitRandomizer = waitRandomizer; - this.min = min; - this.max = max; - this.spread = spread; - this.strategy = strategy; - } - - public String toString() - { - return "min=" + min + ",max=" + max + ",spread=" + spread + ",random=" + waitRandomizer; - } + long minMin = parseInMicros(m.group("minmin"), 0); + long maxMax = parseInMicros(m.group("maxmax"), Long.MAX_VALUE); + Wait max = TimeoutStrategy.parseWait(m.group("max"), latencies); + String minSpec = m.group("min"); + Wait min = minSpec == null ? null : TimeoutStrategy.parseWait(minSpec, latencies); + if (min == null && randomizer != null) + throw new IllegalArgumentException("Invalid to specify randomiser when no range specified: '" + original + '\''); + if (min instanceof Wait.Constant && minMin != 0) + throw new IllegalArgumentException("Invalid to specify an absolute minimum constant when the min bound is itself a constant: '" + original + '\''); + if (max instanceof Wait.Constant && maxMax != Long.MAX_VALUE) + throw new IllegalArgumentException("Invalid to specify an absolute maximum constant when the max bound is itself a constant: '" + original + '\''); + if (randomizer == null) + randomizer = randomizers.uniform(); + return new RetryStrategy(randomizer, minMin, min, max, maxMax, retries); } @VisibleForTesting - public static ParsedStrategy parseStrategy(String spec, LatencySourceFactory latencies, ParsedStrategy defaultStrategy) - { - String[] args = spec.split(","); - String waitRandomizer = find(args, "random"); - String min = find(args, "min"); - String max = find(args, "max"); - String spread = find(args, "spread"); - if (spread == null) - spread = find(args, "delta"); - - if (waitRandomizer == null) waitRandomizer = defaultStrategy.waitRandomizer; - if (min == null) min = defaultStrategy.min; - if (max == null) max = defaultStrategy.max; - if (spread == null) spread = defaultStrategy.spread; - - RetryStrategy strategy = new RetryStrategy(waitRandomizer, min, max, spread, latencies); - return new ParsedStrategy(waitRandomizer, min, max, spread, strategy); - } - - protected static String find(String[] args, String param) - { - return stream(args).filter(s -> s.startsWith(param + '=')) - .map(s -> s.substring(param.length() + 1)) - .findFirst().orElse(null); - } - - static WaitRandomizer parseWaitRandomizer(String input) - { - return parseWaitRandomizer(input, randomizers); - } - - static WaitRandomizer parseWaitRandomizer(String input, WaitRandomizerFactory randomizers) + protected static WaitRandomizer parseWaitRandomizer(String input) { Matcher m = RANDOMIZER.matcher(input); if (!m.matches()) diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index c640263454..e560732229 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -137,7 +137,7 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableParams; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.IAccordService; -import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult; +import org.apache.cassandra.service.accord.IAccordService.IAccordResult; import org.apache.cassandra.service.accord.txn.TxnData; import org.apache.cassandra.service.accord.txn.TxnDataKeyValue; import org.apache.cassandra.service.accord.txn.TxnDataValue; @@ -1258,7 +1258,7 @@ public class StorageProxy implements StorageProxyMBean { SplitMutations splitMutations = splitMutationsIntoAccordAndNormal(cm, (List)mutations); List accordMutations = splitMutations.accordMutations(); - AsyncTxnResult accordResult = accordMutations != null ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null; + IAccordResult accordResult = accordMutations != null ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null; List normalMutations = splitMutations.normalMutations(); Tracing.trace("Split mutations into Accord {} and normal {}", accordMutations, normalMutations); @@ -1298,8 +1298,7 @@ public class StorageProxy implements StorageProxyMBean { if (accordResult != null) { - IAccordService accord = AccordService.instance(); - TxnResult.Kind kind = accord.getTxnResult(accordResult).kind(); + TxnResult.Kind kind = accordResult.awaitAndGet().kind(); if (kind == retry_new_protocol && failure == null) { Tracing.trace("Accord returned retry new protocol"); @@ -1464,7 +1463,7 @@ public class StorageProxy implements StorageProxyMBean } // Start Accord executing so it executes while the mutations are synchronously applied - AsyncTxnResult accordResult = !accordMutations.isEmpty() ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null; + IAccordResult accordResult = !accordMutations.isEmpty() ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null; Throwable failure = null; try @@ -1512,8 +1511,7 @@ public class StorageProxy implements StorageProxyMBean // the batch log. if (accordResult != null) { - IAccordService accord = AccordService.instance(); - TxnResult.Kind kind = accord.getTxnResult(accordResult).kind(); + TxnResult.Kind kind = accordResult.awaitAndGet().kind(); if (kind == retry_new_protocol && failure == null) continue; Tracing.trace("Successfully wrote Accord mutations"); @@ -2205,7 +2203,7 @@ public class StorageProxy implements StorageProxyMBean return null; } - public static AsyncTxnResult readWithAccord(ClusterMetadata cm, PartitionRangeReadCommand command, AbstractBounds range, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + public static IAccordResult readWithAccord(ClusterMetadata cm, PartitionRangeReadCommand command, AbstractBounds range, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { if (consistencyLevel != null && !IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS.contains(consistencyLevel)) throw new InvalidRequestException(consistencyLevel + " is not supported by Accord"); @@ -2220,7 +2218,7 @@ public class StorageProxy implements StorageProxyMBean return accordService.coordinateAsync(tableMetadata.epoch.getEpoch(), txn, consistencyLevel, requestTime); } - private static AsyncTxnResult readWithAccordAsync(ClusterMetadata cm, SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + private static IAccordResult readWithAccordAsync(ClusterMetadata cm, SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { if (consistencyLevel != null && !IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS.contains(consistencyLevel)) throw new InvalidRequestException(consistencyLevel + " is not supported by Accord"); @@ -2238,16 +2236,16 @@ public class StorageProxy implements StorageProxyMBean private static ConsensusAttemptResult readWithAccord(ClusterMetadata cm, SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { - AsyncTxnResult asyncTxnResult = readWithAccordAsync(cm, group, consistencyLevel, requestTime); - return getConsensusAttemptResultFromAsyncTxnResult(asyncTxnResult, group.queries.size(), index -> group.queries.get(index).isReversed()); + IAccordResult accordResult = readWithAccordAsync(cm, group, consistencyLevel, requestTime); + return getConsensusAttemptResultFromAsyncTxnResult(accordResult, group.queries.size(), index -> group.queries.get(index).isReversed()); } /* * Used for both the SERIAL and non-SERIAL read path into Accord */ - public static ConsensusAttemptResult getConsensusAttemptResultFromAsyncTxnResult(AsyncTxnResult asyncTxnResult, int numQueries, IntPredicate isQueryReversed) + public static ConsensusAttemptResult getConsensusAttemptResultFromAsyncTxnResult(IAccordResult accordResult, int numQueries, IntPredicate isQueryReversed) { - TxnResult txnResult = AccordService.instance().getTxnResult(asyncTxnResult); + TxnResult txnResult = accordResult.awaitAndGet(); // TODO (required): Converge on a single approach to RETRY_NEW_PROTOCOL, this works for now because reads don't support it anyways if (txnResult.kind() == retry_new_protocol) return RETRY_NEW_PROTOCOL; @@ -2385,7 +2383,7 @@ public class StorageProxy implements StorageProxyMBean { SplitReads splitReads = splitReadsIntoAccordAndNormal(cm, group, coordinator, requestTime); SinglePartitionReadCommand.Group accordReads = splitReads.accordReads; - AsyncTxnResult accordResult = accordReads != null ? readWithAccordAsync(cm, accordReads, consistencyLevel, requestTime) : null; + IAccordResult accordResult = accordReads != null ? readWithAccordAsync(cm, accordReads, consistencyLevel, requestTime) : null; SinglePartitionReadCommand.Group normalReads = splitReads.normalReads; Tracing.trace("Split reads into Accord {} and normal {}", accordReads, normalReads); @@ -3347,9 +3345,6 @@ public class StorageProxy implements StorageProxyMBean public Long getTruncateRpcTimeout() { return DatabaseDescriptor.getTruncateRpcTimeout(MILLISECONDS); } public void setTruncateRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setTruncateRpcTimeout(timeoutInMillis); } - public Long getTransactionTimeout() { return DatabaseDescriptor.getTransactionTimeout(MILLISECONDS); } - public void setTransactionTimeout(Long value) { DatabaseDescriptor.setTransactionTimeout(value); } - public Long getNativeTransportMaxConcurrentConnections() { return DatabaseDescriptor.getNativeTransportMaxConcurrentConnections(); } public void setNativeTransportMaxConcurrentConnections(Long nativeTransportMaxConcurrentConnections) { DatabaseDescriptor.setNativeTransportMaxConcurrentConnections(nativeTransportMaxConcurrentConnections); } diff --git a/src/java/org/apache/cassandra/service/StorageProxyMBean.java b/src/java/org/apache/cassandra/service/StorageProxyMBean.java index f89cdad95e..5c2d5ec22c 100644 --- a/src/java/org/apache/cassandra/service/StorageProxyMBean.java +++ b/src/java/org/apache/cassandra/service/StorageProxyMBean.java @@ -51,8 +51,6 @@ public interface StorageProxyMBean public void setRangeRpcTimeout(Long timeoutInMillis); public Long getTruncateRpcTimeout(); public void setTruncateRpcTimeout(Long timeoutInMillis); - public Long getTransactionTimeout(); - public void setTransactionTimeout(Long timeoutInMillis); public void setNativeTransportMaxConcurrentConnections(Long nativeTransportMaxConcurrentConnections); public Long getNativeTransportMaxConcurrentConnections(); diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index be6699f292..90e2f598d2 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -1270,17 +1270,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return DatabaseDescriptor.getTruncateRpcTimeout(MILLISECONDS); } - public void setTransactionTimeout(long value) - { - DatabaseDescriptor.setTransactionTimeout(value); - logger.info("set transaction timeout to {} ms", value); - } - - public long getTransactionTimeout() - { - return DatabaseDescriptor.getTransactionTimeout(MILLISECONDS); - } - /** @deprecated See CASSANDRA-15234 */ @Deprecated(since = "4.1") public void setStreamThroughputMbPerSec(int value) diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 73e9fcd680..61c378a4bd 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -791,9 +791,6 @@ public interface StorageServiceMBean extends NotificationEmitter public void setTruncateRpcTimeout(long value); public long getTruncateRpcTimeout(); - public void setTransactionTimeout(long value); - public long getTransactionTimeout(); - public void setStreamThroughputMbitPerSec(int value); /** * @return stream_throughput_outbound in megabits diff --git a/src/java/org/apache/cassandra/service/TimeoutStrategy.java b/src/java/org/apache/cassandra/service/TimeoutStrategy.java index 8cbc67698f..7ca39db222 100644 --- a/src/java/org/apache/cassandra/service/TimeoutStrategy.java +++ b/src/java/org/apache/cassandra/service/TimeoutStrategy.java @@ -24,25 +24,26 @@ import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; +import com.google.common.annotations.VisibleForTesting; + +import accord.utils.Invariants; import com.codahale.metrics.Snapshot; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.metrics.ClientRequestMetrics; -import org.apache.cassandra.utils.NoSpamLogger; +import org.apache.cassandra.service.TimeoutStrategy.LatencySupplier.Constant; +import org.apache.cassandra.service.TimeoutStrategy.LatencySupplier.Percentile; import static java.lang.Double.parseDouble; import static java.lang.Integer.parseInt; import static java.lang.Math.max; -import static java.lang.Math.min; import static java.lang.Math.pow; import static java.util.concurrent.TimeUnit.MICROSECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.config.DatabaseDescriptor.getCasContentionTimeout; +import static org.apache.cassandra.config.DatabaseDescriptor.getReadRpcTimeout; +import static org.apache.cassandra.config.DatabaseDescriptor.getWriteRpcTimeout; import static org.apache.cassandra.utils.Clock.Global.nanoTime; /** @@ -74,21 +75,21 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime; * TODO (expected): permit simple constant addition (e.g. p50+5ms) * TODO (required): track separate stats per-DC as inputs to these decisions */ -public class TimeoutStrategy +public class TimeoutStrategy implements WaitStrategy { - private static final Logger logger = LoggerFactory.getLogger(TimeoutStrategy.class); + static final Pattern PARSE = Pattern.compile( + "(\\s*(?0|[0-9]+[mu]?s)\\s*<=)?" + + "(\\s*(?[^=]+))" + + "(\\s*<=\\s*(?0|[0-9]+[mu]?s))?"); - static final Pattern BOUND = Pattern.compile( - "(?0|[0-9]+[mu]s)" + - "|((?0|[0-9]+[mu]s) *<= *)?" + - "(p(?[0-9]+)(\\((?r|w|rw|wr)\\))?|(?0|[0-9]+[mu]s))" + - "\\s*([*]\\s*(?[0-9.]+)?\\s*(?[*^]\\s*attempts)?)?" + - "( *<= *(?0|[0-9]+[mu]s))?"); + static final Pattern WAIT = Pattern.compile( + "\\s*(?0|[0-9]+[mu]?s)" + + "|\\s*((p(?[0-9]+)(\\((?r|w|rw|wr)\\))?)?|(?0|[0-9]+[mu]?s))" + + "\\s*(([*]\\s*(?[0-9.]+))?\\s*(?[*^]\\s*attempts)?)?\\s*"); static final Pattern TIME = Pattern.compile( - "0|([0-9]+)ms|([0-9]+)us"); + "0|[0-9]+[mu]?s"); // Factories can be useful for testing purposes, to supply custom implementations of selectors and modifiers. - final static LatencySupplierFactory selectors = new LatencySupplierFactory(){}; final static LatencyModifierFactory modifiers = new LatencyModifierFactory(){}; interface LatencyModifierFactory @@ -99,9 +100,64 @@ public class TimeoutStrategy default LatencyModifier multiplyByAttemptsExp(double base) { return (l, a) -> saturatedCast(l * pow(base, a)); } } - interface LatencySupplier + public interface Wait { - long get(); + long getMicros(int attempts); + + class Constant implements Wait + { + final long micros; + public Constant(long micros) { this.micros = micros; } + @Override public long getMicros(int attempts) { return micros; } + } + + class Modifying implements Wait + { + final LatencySupplier supplier; + final LatencyModifier modifier; + + Modifying(LatencySupplier supplier, LatencyModifier modifier) + { + this.supplier = supplier; + this.modifier = modifier; + } + + @Override + public long getMicros(int attempts) + { + return modifier.modify(supplier.getMicros(), attempts); + } + } + } + + public interface LatencySupplier + { + long getMicros(); + + class Constant implements LatencySupplier + { + final long micros; + public Constant(long micros) {this.micros = micros; } + @Override public long getMicros() { return micros; } + } + + class Percentile implements LatencySupplier + { + final LatencySource latencies; + final double percentile; + + public Percentile(LatencySource latencies, double percentile) + { + this.latencies = latencies; + this.percentile = percentile; + } + + @Override + public long getMicros() + { + return latencies.get(percentile); + } + } } public interface LatencySource @@ -109,12 +165,6 @@ public class TimeoutStrategy long get(double percentile); } - interface LatencySupplierFactory - { - default LatencySupplier constant(long latency) { return () -> latency; } - default LatencySupplier percentile(double percentile, LatencySource latencies) { return () -> latencies.get(percentile); } - } - public interface LatencySourceFactory { LatencySource source(String params); @@ -129,6 +179,11 @@ public class TimeoutStrategy LatencySource source = new TimeLimitedLatencySupplier(latencies.latency::getSnapshot, 10, SECONDS); return ignore -> source; } + + static LatencySourceFactory none() + { + return ignore -> ignore2 -> { throw new UnsupportedOperationException(); }; + } } public static class ReadWriteLatencySourceFactory implements LatencySourceFactory @@ -209,58 +264,22 @@ public class TimeoutStrategy } } - public static class Wait - { - final long min, max, onFailure; - final LatencyModifier modifier; - final LatencySupplier supplier; - - Wait(long min, long max, long onFailure, LatencyModifier modifier, LatencySupplier supplier) - { - Preconditions.checkArgument(min<=max, "min (%s) must be less than or equal to max (%s)", min, max); - this.min = min; - this.max = max; - this.onFailure = onFailure; - this.modifier = modifier; - this.supplier = supplier; - } - - long get(int attempts) - { - try - { - long base = supplier.get(); - return max(min, min(max, modifier.modify(base, attempts))); - } - catch (Throwable t) - { - NoSpamLogger.getLogger(logger, 1L, MINUTES).info("", t); - return onFailure; - } - } - - public String toString() - { - return "Bound{" + - "min=" + min + - ", max=" + max + - ", onFailure=" + onFailure + - ", modifier=" + modifier + - ", supplier=" + supplier + - '}'; - } - } - final Wait wait; + final long minMicros, maxMicros; - public TimeoutStrategy(String spec, LatencySourceFactory latencies) + public TimeoutStrategy(Wait wait, long minMicros, long maxMicros) { - this.wait = parseWait(spec, latencies); + this.minMicros = minMicros; + this.maxMicros = maxMicros; + this.wait = wait; } public long computeWait(int attempts, TimeUnit units) { - return units.convert(wait.get(attempts), MICROSECONDS); + long wait = this.wait.getMicros(attempts); + if (wait < minMicros) wait = minMicros; + else if (wait > maxMicros) wait = maxMicros; + return units.convert(wait, MICROSECONDS); } public long computeWaitUntil(int attempts) @@ -269,32 +288,29 @@ public class TimeoutStrategy return nanoTime() + nanos; } - protected Wait parseWait(String spec, LatencySourceFactory latencies) - { - long defaultMicros = DatabaseDescriptor.getRpcTimeout(MICROSECONDS); - return parseWait(spec, 0, defaultMicros, defaultMicros, latencies); - } - - private static LatencySupplier parseLatencySupplier(Matcher m, LatencySupplierFactory selectors, LatencySourceFactory latenciesFactory) + private static LatencySupplier parseLatencySupplier(Matcher m, LatencySourceFactory latenciesFactory) { String perc = m.group("perc"); if (perc == null) - return selectors.constant(parseInMicros(m.group("constbase"))); + return new Constant(parseInMicros(m.group("constbase"))); - LatencySource latencies = latenciesFactory.source(m.group("rw")); + String rw = m.group("rw"); + if (rw == null) rw = "rw"; + LatencySource latencies = latenciesFactory.source(rw); double percentile = parseDouble("0." + perc); - return selectors.percentile(percentile, latencies); + return new Percentile(latencies, percentile); } - private static LatencyModifier parseLatencyModifier(Matcher m, LatencyModifierFactory modifiers) + private static @Nullable LatencyModifier parseLatencyModifier(String spec, Matcher m, LatencyModifierFactory modifiers) { String mod = m.group("mod"); - if (mod == null) - return modifiers.identity(); - - double modifier = parseDouble(mod); - String modkind = m.group("modkind"); + double modifier = 1.0; + if (mod != null) modifier = Double.parseDouble(mod); + else if (modkind == null) return null; + else if (!modkind.startsWith("*")) + throw new IllegalArgumentException("Invalid latency modifier specification: " + spec + ". Expect constant factor as base for exponent."); + if (modkind == null) return modifiers.multiply(modifier); @@ -313,31 +329,46 @@ public class TimeoutStrategy return (long) v; } - public static Wait parseWait(String input, long defaultMin, long defaultMax, long onFailure, LatencySourceFactory latencies) + public static TimeoutStrategy parse(String input, LatencySourceFactory latencies) { - return parseWait(input, defaultMin, defaultMax, onFailure, latencies, selectors, modifiers); + Matcher m = PARSE.matcher(input); + if (!m.matches()) + throw new IllegalArgumentException("Invalid specification: '" + input + "'; does not match " + PARSE); + long min = parseInMicros(m.group("min"), 0); + long max = parseInMicros(m.group("max"), Long.MAX_VALUE); + Wait wait = TimeoutStrategy.parseWait(m.group("wait"), latencies); + return new TimeoutStrategy(wait, min, max); + } + + public static Wait parseWait(String input, LatencySourceFactory latencies) + { + return parseWait(input, latencies, modifiers); } @VisibleForTesting - public static Wait parseWait(String input, long defaultMinMicros, long defaultMaxMicros, long onFailure, LatencySourceFactory latencies, LatencySupplierFactory selectors, LatencyModifierFactory modifiers) + static Wait parseWait(String input, LatencySourceFactory latencies, LatencyModifierFactory modifiers) { - Matcher m = BOUND.matcher(input); + Matcher m = WAIT.matcher(input); if (!m.matches()) - throw new IllegalArgumentException(input + " does not match " + BOUND); + throw new IllegalArgumentException(input + " does not match " + WAIT); String maybeConst = m.group("const"); if (maybeConst != null) { long v = parseInMicros(maybeConst); - return new Wait(v, v, v, modifiers.identity(), selectors.constant(v)); + return new Wait.Constant(v); } - long min = parseInMicros(m.group("min"), defaultMinMicros); - long max = parseInMicros(m.group("max"), defaultMaxMicros); - return new Wait(min, max, onFailure, parseLatencyModifier(m, modifiers), parseLatencySupplier(m, selectors, latencies)); + LatencySupplier supplier = parseLatencySupplier(m, latencies); + LatencyModifier modifier = parseLatencyModifier(input, m, modifiers); + if (modifier == null && supplier instanceof LatencySupplier.Constant) + return new Wait.Constant(((Constant) supplier).micros); + if (modifier == null) + modifier = modifiers.identity(); + return new Wait.Modifying(supplier, modifier); } - private static long parseInMicros(String input, long orElse) + public static long parseInMicros(String input, long orElse) { if (input == null) return orElse; @@ -345,19 +376,25 @@ public class TimeoutStrategy return parseInMicros(input); } - private static long parseInMicros(String input) + public static long parseInMicros(String text) { - Matcher m = TIME.matcher(input); + Matcher m = TIME.matcher(text); if (!m.matches()) - throw new IllegalArgumentException(input + " does not match " + TIME); + throw new IllegalArgumentException(text + " does not match " + TIME); - String text; - if (null != (text = m.group(1))) - return parseInt(text) * 1000; - else if (null != (text = m.group(2))) - return parseInt(text); - else + if (text.length() == 1) + { + Invariants.require(text.charAt(0) == '0'); return 0; + } + + char penultimate = text.charAt(text.length() - 2); + switch (penultimate) + { + default: return parseInt(text.substring(0, text.length() - 1)) * 1000_000L; + case 'm': return parseInt(text.substring(0, text.length() - 2)) * 1000L; + case 'u': return parseInt(text.substring(0, text.length() - 2)); + } } private static String orElse(Supplier get, String orElse) @@ -365,4 +402,10 @@ public class TimeoutStrategy String result = get.get(); return result != null ? result : orElse; } + + @VisibleForTesting + public static long maxQueryTimeoutMicros() + { + return max(max(getCasContentionTimeout(MICROSECONDS), getWriteRpcTimeout(MICROSECONDS)), getReadRpcTimeout(MICROSECONDS)); + } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordConfiguration.java b/src/java/org/apache/cassandra/service/WaitStrategy.java similarity index 59% rename from src/java/org/apache/cassandra/service/accord/AccordConfiguration.java rename to src/java/org/apache/cassandra/service/WaitStrategy.java index a0da2d99af..cba13ca987 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordConfiguration.java +++ b/src/java/org/apache/cassandra/service/WaitStrategy.java @@ -16,18 +16,32 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service; -import accord.api.LocalConfig; -import org.apache.cassandra.config.Config; +import java.util.concurrent.TimeUnit; -// TODO (expected): should this be merged with AccordSpec? -public class AccordConfiguration implements LocalConfig +public interface WaitStrategy { - private final Config config; + // a value of below 0 means give up + long computeWaitUntil(int attempts); + // a value of below 0 means give up + long computeWait(int attempts, TimeUnit units); - public AccordConfiguration(Config config) + enum None implements WaitStrategy { - this.config = config; + INSTANCE; + + @Override + public long computeWait(int attempts, TimeUnit units) + { + return -1; + } + + @Override + public long computeWaitUntil(int attempts) + { + return -1; + } } + } diff --git a/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java b/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java index 271fc3b219..f01def7639 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java @@ -65,7 +65,7 @@ import org.apache.cassandra.utils.concurrent.Future; import static org.apache.cassandra.service.accord.AccordTopology.tcmIdToAccord; import static org.apache.cassandra.utils.Simulate.With.MONITORS; -// TODO: listen to FailureDetector and rearrange fast path accordingly +// TODO (desired): listen to FailureDetector and rearrange fast path accordingly @Simulate(with=MONITORS) public class AccordConfigurationService extends AbstractConfigurationService implements AccordEndpointMapper, AccordSyncPropagator.Listener, Shutdownable { @@ -389,9 +389,6 @@ public class AccordConfigurationService extends AbstractConfigurationService l.onRemoveNode(epoch, removed)); - - if (shareShard(current, removed, localId)) - AccordService.instance().tryMarkRemoved(current, removed); } private long[] nonCompletedEpochsBefore(long max) @@ -467,7 +464,9 @@ public class AccordConfigurationService extends AbstractConfigurationService { if (topology != null) + { future.setSuccess(topology); + } else { Invariants.require(future == pendingTopologies.remove(epoch_)); @@ -607,12 +606,12 @@ public class AccordConfigurationService extends AbstractConfigurationService e : tables.entrySet()) { + // TODO (required): is it safe to ignore null table metadata / cfs? TableMetadata tableMetadata = metadata.schema.getTableMetadata(e.getKey()); - if (!tableMetadata.isAccordEnabled()) + if (tableMetadata == null || !tableMetadata.isAccordEnabled()) continue; ColumnFamilyStore cfs = Keyspace.openAndGetStoreIfExists(tableMetadata); + if (cfs == null) + continue; + // TODO (required): when we can safely map TxnId.hlc() -> local timestamp, consult Memtable timestamps Memtable memtable = cfs.getCurrentMemtable(); e.getValue().id = memtable.getMemtableId(); @@ -102,6 +106,10 @@ public class AccordDataStore implements DataStore TableMetadata tableMetadata = metadata.schema.getTableMetadata(e.getKey()); SnapshotBounds bounds = e.getValue(); ColumnFamilyStore cfs = Keyspace.openAndGetStoreIfExists(tableMetadata); + + // TODO (required): is it safe to ignore null cfs? + if (cfs == null) continue; + View view = cfs.getTracker().getView(); for (Memtable memtable : view.getAllMemtables()) { diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java index 626b08fe34..9d5c97b463 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java @@ -18,6 +18,7 @@ package org.apache.cassandra.service.accord; +import java.util.concurrent.Callable; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; @@ -28,6 +29,7 @@ import java.util.function.IntFunction; import accord.api.Agent; import accord.api.RoutingKey; +import accord.local.AgentExecutor; import accord.local.Command; import accord.local.cfk.CommandsForKey; import accord.primitives.TxnId; @@ -40,6 +42,8 @@ import accord.utils.QuintConsumer; import accord.utils.TriConsumer; import accord.utils.TriFunction; import accord.utils.UnhandledEnum; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; import accord.utils.async.Cancellable; import org.agrona.collections.Object2ObjectHashMap; import org.apache.cassandra.cache.CacheSize; @@ -60,7 +64,7 @@ import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LO import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLoaded, AccordCacheEntry.OnSaved, Shutdownable +public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLoaded, AccordCacheEntry.OnSaved, Shutdownable, AgentExecutor { public interface AccordExecutorFactory { @@ -314,6 +318,18 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo return true; } + @Override + public Agent agent() + { + return agent; + } + + @Override + public AsyncChain build(Callable task) + { + return AsyncChains.ofCallable(this, task); + } + private void parkRangeLoad(AccordTask task) { if (task.queued() != waitingToLoadRangeTxns) diff --git a/src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java index b95e5cbde8..4c878ce8a3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java @@ -163,6 +163,9 @@ public abstract class AccordFastPathCoordinator implements ChangeListener, Confi AccordFastPath fastPath = cm.accordFastPath; long updateDelayMillis = getAccordFastPathUpdateDelayMillis(); + if (updateDelayMillis < 0) + return; + if (isShutdown(fastPath)) { updateFastPath(localId, Status.NORMAL, Clock.Global.currentTimeMillis(), updateDelayMillis); diff --git a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java index bdb9f03584..cfa9db8101 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java +++ b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java @@ -18,18 +18,13 @@ package org.apache.cassandra.service.accord; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.util.Collections; import java.util.EnumSet; -import java.util.List; import java.util.Map; import java.util.Set; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,13 +34,10 @@ import accord.local.AgentExecutor; import accord.local.Node; import accord.messages.Callback; import accord.messages.MessageType; -import accord.messages.ReadData; import accord.messages.Reply; import accord.messages.ReplyContext; import accord.messages.Request; -import accord.messages.TxnRequest; import accord.primitives.TxnId; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; @@ -54,157 +46,107 @@ import org.apache.cassandra.net.MessageFlag; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.ResponseContext; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.service.TimeoutStrategy; import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.utils.Clock; -import static accord.messages.MessageType.Kind.REMOTE; -import static accord.primitives.Routable.Domain.Range; +import static accord.messages.MessageType.StandardMessage.*; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.expire; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowPreaccept; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowRead; public class AccordMessageSink implements MessageSink { private static final Logger logger = LoggerFactory.getLogger(AccordMessageSink.class); - public static final class AccordMessageType extends MessageType + public enum AccordMessageType implements MessageType { - public static final AccordMessageType INTEROP_READ_REQ = remote("INTEROP_READ_REQ"); - public static final AccordMessageType INTEROP_READ_RSP = remote("INTEROP_READ_RSP"); - public static final AccordMessageType INTEROP_STABLE_THEN_READ_REQ = remote("INTEROP_STABLE_THEN_READ_REQ"); - public static final AccordMessageType INTEROP_READ_REPAIR_REQ = remote("INTEROP_READ_REPAIR_REQ"); - public static final AccordMessageType INTEROP_READ_REPAIR_RSP = remote("INTEROP_READ_REPAIR_RSP"); - public static final AccordMessageType INTEROP_APPLY_MINIMAL_REQ = remote("INTEROP_APPLY_MINIMAL_REQ"); - public static final AccordMessageType INTEROP_APPLY_MAXIMAL_REQ = remote("INTEROP_APPLY_MAXIMAL_REQ"); + INTEROP_READ_REQ(Verb.ACCORD_INTEROP_READ_REQ), + INTEROP_STABLE_THEN_READ_REQ(Verb.ACCORD_INTEROP_STABLE_THEN_READ_REQ), + INTEROP_READ_RSP(Verb.ACCORD_INTEROP_READ_RSP), + INTEROP_READ_REPAIR_REQ(Verb.ACCORD_INTEROP_READ_REPAIR_REQ), + INTEROP_READ_REPAIR_RSP(Verb.ACCORD_INTEROP_READ_REPAIR_RSP), + INTEROP_APPLY_REQ(Verb.ACCORD_INTEROP_APPLY_REQ); + final Verb verb; - public static final List values; - - static + AccordMessageType(Verb verb) { - ImmutableList.Builder builder = ImmutableList.builder(); - for (Field f : AccordMessageType.class.getDeclaredFields()) - { - if (f.getType().equals(AccordMessageType.class) && Modifier.isStatic(f.getModifiers())) - { - try - { - builder.add((MessageType) f.get(null)); - } - catch (IllegalAccessException e) - { - throw new RuntimeException(e); - } - } - } - values = builder.build(); - } - - protected static AccordMessageType remote(String name) - { - return new AccordMessageType(name, REMOTE); - } - - private AccordMessageType(String name, MessageType.Kind kind) - { - super(name, kind); + this.verb = verb; } } private static class VerbMapping { - private static final VerbMapping instance = new VerbMapping(); + private static final Map> overrideReplyVerbs = ImmutableMap.>builder() + // read takes Result | Nack + .put(Verb.ACCORD_FETCH_DATA_REQ, EnumSet.of(Verb.ACCORD_FETCH_DATA_RSP, Verb.ACCORD_READ_RSP /* nack */)) + .put(Verb.ACCORD_INTEROP_STABLE_THEN_READ_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_RSP, Verb.ACCORD_READ_RSP)) + .put(Verb.ACCORD_INTEROP_READ_REPAIR_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_REPAIR_RSP, Verb.ACCORD_READ_RSP)) + .build(); - private final Map mapping; - private final Map> overrideReplyVerbs = ImmutableMap.>builder() - // read takes Result | Nack - .put(Verb.ACCORD_FETCH_DATA_REQ, EnumSet.of(Verb.ACCORD_FETCH_DATA_RSP, Verb.ACCORD_READ_RSP /* nack */)) - .put(Verb.ACCORD_INTEROP_STABLE_THEN_READ_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_RSP, Verb.ACCORD_READ_RSP)) - .put(Verb.ACCORD_INTEROP_READ_REPAIR_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_REPAIR_RSP, Verb.ACCORD_READ_RSP)) - .build(); - - private VerbMapping() + static { - ImmutableMap.Builder builder = ImmutableMap.builder(); - builder.put(MessageType.SIMPLE_RSP, Verb.ACCORD_SIMPLE_RSP); - builder.put(MessageType.PRE_ACCEPT_REQ, Verb.ACCORD_PRE_ACCEPT_REQ); - builder.put(MessageType.PRE_ACCEPT_RSP, Verb.ACCORD_PRE_ACCEPT_RSP); - builder.put(MessageType.ACCEPT_REQ, Verb.ACCORD_ACCEPT_REQ); - builder.put(MessageType.ACCEPT_RSP, Verb.ACCORD_ACCEPT_RSP); - builder.put(MessageType.NOT_ACCEPT_REQ, Verb.ACCORD_NOT_ACCEPT_REQ); - builder.put(MessageType.RECOVER_AWAIT_REQ, Verb.ACCORD_RECOVER_AWAIT_REQ); - builder.put(MessageType.RECOVER_AWAIT_RSP, Verb.ACCORD_RECOVER_AWAIT_RSP); - builder.put(MessageType.GET_LATEST_DEPS_REQ, Verb.ACCORD_GET_LATEST_DEPS_REQ); - builder.put(MessageType.GET_LATEST_DEPS_RSP, Verb.ACCORD_GET_LATEST_DEPS_RSP); - builder.put(MessageType.GET_EPHEMERAL_READ_DEPS_REQ, Verb.ACCORD_GET_EPHMRL_READ_DEPS_REQ); - builder.put(MessageType.GET_EPHEMERAL_READ_DEPS_RSP, Verb.ACCORD_GET_EPHMRL_READ_DEPS_RSP); - builder.put(MessageType.COMMIT_SLOW_PATH_REQ, Verb.ACCORD_COMMIT_REQ); - builder.put(MessageType.COMMIT_MAXIMAL_REQ, Verb.ACCORD_COMMIT_REQ); - builder.put(MessageType.STABLE_FAST_PATH_REQ, Verb.ACCORD_COMMIT_REQ); - builder.put(MessageType.STABLE_SLOW_PATH_REQ, Verb.ACCORD_COMMIT_REQ); - builder.put(MessageType.STABLE_MAXIMAL_REQ, Verb.ACCORD_COMMIT_REQ); - builder.put(MessageType.COMMIT_INVALIDATE_REQ, Verb.ACCORD_COMMIT_INVALIDATE_REQ); - builder.put(MessageType.APPLY_MINIMAL_REQ, Verb.ACCORD_APPLY_REQ); - builder.put(MessageType.APPLY_MAXIMAL_REQ, Verb.ACCORD_APPLY_REQ); - builder.put(MessageType.APPLY_RSP, Verb.ACCORD_APPLY_RSP); - builder.put(MessageType.READ_REQ, Verb.ACCORD_READ_REQ); - builder.put(MessageType.STABLE_THEN_READ_REQ, Verb.ACCORD_STABLE_THEN_READ_REQ); - builder.put(MessageType.READ_EPHEMERAL_REQ, Verb.ACCORD_READ_REQ); - builder.put(MessageType.READ_RSP, Verb.ACCORD_READ_RSP); - builder.put(MessageType.BEGIN_RECOVER_REQ, Verb.ACCORD_BEGIN_RECOVER_REQ); - builder.put(MessageType.BEGIN_RECOVER_RSP, Verb.ACCORD_BEGIN_RECOVER_RSP); - builder.put(MessageType.BEGIN_INVALIDATE_REQ, Verb.ACCORD_BEGIN_INVALIDATE_REQ); - builder.put(MessageType.BEGIN_INVALIDATE_RSP, Verb.ACCORD_BEGIN_INVALIDATE_RSP); - builder.put(MessageType.AWAIT_REQ, Verb.ACCORD_AWAIT_REQ); - builder.put(MessageType.AWAIT_RSP, Verb.ACCORD_AWAIT_RSP); - builder.put(MessageType.ASYNC_AWAIT_COMPLETE_REQ, Verb.ACCORD_AWAIT_ASYNC_RSP_REQ); - builder.put(MessageType.WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_WAIT_UNTIL_APPLIED_REQ); - builder.put(MessageType.APPLY_THEN_WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_APPLY_AND_WAIT_REQ); - builder.put(MessageType.INFORM_DURABLE_REQ, Verb.ACCORD_INFORM_DURABLE_REQ); - builder.put(MessageType.CHECK_STATUS_REQ, Verb.ACCORD_CHECK_STATUS_REQ); - builder.put(MessageType.CHECK_STATUS_RSP, Verb.ACCORD_CHECK_STATUS_RSP); - builder.put(MessageType.FETCH_DATA_REQ, Verb.ACCORD_FETCH_DATA_REQ); - builder.put(MessageType.FETCH_DATA_RSP, Verb.ACCORD_FETCH_DATA_RSP); - builder.put(MessageType.SET_SHARD_DURABLE_REQ, Verb.ACCORD_SET_SHARD_DURABLE_REQ); - builder.put(MessageType.SET_GLOBALLY_DURABLE_REQ, Verb.ACCORD_SET_GLOBALLY_DURABLE_REQ); - builder.put(MessageType.QUERY_DURABLE_BEFORE_REQ, Verb.ACCORD_QUERY_DURABLE_BEFORE_REQ); - builder.put(MessageType.QUERY_DURABLE_BEFORE_RSP, Verb.ACCORD_QUERY_DURABLE_BEFORE_RSP); - builder.put(AccordMessageType.INTEROP_READ_REQ, Verb.ACCORD_INTEROP_READ_REQ); - builder.put(AccordMessageType.INTEROP_READ_RSP, Verb.ACCORD_INTEROP_READ_RSP); - builder.put(AccordMessageType.INTEROP_STABLE_THEN_READ_REQ, Verb.ACCORD_INTEROP_STABLE_THEN_READ_REQ); - builder.put(AccordMessageType.INTEROP_READ_REPAIR_REQ, Verb.ACCORD_INTEROP_READ_REPAIR_REQ); - builder.put(AccordMessageType.INTEROP_READ_REPAIR_RSP, Verb.ACCORD_INTEROP_READ_REPAIR_RSP); - builder.put(AccordMessageType.INTEROP_APPLY_MINIMAL_REQ, Verb.ACCORD_INTEROP_APPLY_REQ); - builder.put(AccordMessageType.INTEROP_APPLY_MAXIMAL_REQ, Verb.ACCORD_INTEROP_APPLY_REQ); - mapping = builder.build(); - - for (MessageType type : Iterables.concat(AccordMessageType.values, MessageType.values)) - { - // Any request can receive a generic failure response - if (type == MessageType.FAILURE_RSP || type.isLocal()) - continue; - - if (mapping.containsKey(type)) - { - if (type.isLocal()) throw new AssertionError("Extraneous mapping for LOCAL Accord MessageType " + type); - } - else - { - if (type.isRemote()) throw new AssertionError("Missing mapping for REMOTE Accord MessageType " + type); - } - } + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.put(SIMPLE_RSP, Verb.ACCORD_SIMPLE_RSP); + builder.put(PRE_ACCEPT_REQ, Verb.ACCORD_PRE_ACCEPT_REQ); + builder.put(PRE_ACCEPT_RSP, Verb.ACCORD_PRE_ACCEPT_RSP); + builder.put(ACCEPT_REQ, Verb.ACCORD_ACCEPT_REQ); + builder.put(ACCEPT_RSP, Verb.ACCORD_ACCEPT_RSP); + builder.put(NOT_ACCEPT_REQ, Verb.ACCORD_NOT_ACCEPT_REQ); + builder.put(RECOVER_AWAIT_REQ, Verb.ACCORD_RECOVER_AWAIT_REQ); + builder.put(RECOVER_AWAIT_RSP, Verb.ACCORD_RECOVER_AWAIT_RSP); + builder.put(GET_LATEST_DEPS_REQ, Verb.ACCORD_GET_LATEST_DEPS_REQ); + builder.put(GET_LATEST_DEPS_RSP, Verb.ACCORD_GET_LATEST_DEPS_RSP); + builder.put(GET_EPHEMERAL_READ_DEPS_REQ, Verb.ACCORD_GET_EPHMRL_READ_DEPS_REQ); + builder.put(GET_EPHEMERAL_READ_DEPS_RSP, Verb.ACCORD_GET_EPHMRL_READ_DEPS_RSP); + builder.put(GET_MAX_CONFLICT_REQ, Verb.ACCORD_GET_MAX_CONFLICT_REQ); + builder.put(GET_MAX_CONFLICT_RSP, Verb.ACCORD_GET_MAX_CONFLICT_RSP); + builder.put(COMMIT_REQ, Verb.ACCORD_COMMIT_REQ); + builder.put(COMMIT_INVALIDATE_REQ, Verb.ACCORD_COMMIT_INVALIDATE_REQ); + builder.put(APPLY_REQ, Verb.ACCORD_APPLY_REQ); + builder.put(APPLY_RSP, Verb.ACCORD_APPLY_RSP); + builder.put(READ_REQ, Verb.ACCORD_READ_REQ); + builder.put(STABLE_THEN_READ_REQ, Verb.ACCORD_STABLE_THEN_READ_REQ); + builder.put(READ_EPHEMERAL_REQ, Verb.ACCORD_READ_REQ); + builder.put(READ_RSP, Verb.ACCORD_READ_RSP); + builder.put(BEGIN_RECOVER_REQ, Verb.ACCORD_BEGIN_RECOVER_REQ); + builder.put(BEGIN_RECOVER_RSP, Verb.ACCORD_BEGIN_RECOVER_RSP); + builder.put(BEGIN_INVALIDATE_REQ, Verb.ACCORD_BEGIN_INVALIDATE_REQ); + builder.put(BEGIN_INVALIDATE_RSP, Verb.ACCORD_BEGIN_INVALIDATE_RSP); + builder.put(AWAIT_REQ, Verb.ACCORD_AWAIT_REQ); + builder.put(AWAIT_RSP, Verb.ACCORD_AWAIT_RSP); + builder.put(ASYNC_AWAIT_COMPLETE_REQ, Verb.ACCORD_AWAIT_ASYNC_RSP_REQ); + builder.put(WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_WAIT_UNTIL_APPLIED_REQ); + builder.put(APPLY_THEN_WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_APPLY_AND_WAIT_REQ); + builder.put(INFORM_DURABLE_REQ, Verb.ACCORD_INFORM_DURABLE_REQ); + builder.put(CHECK_STATUS_REQ, Verb.ACCORD_CHECK_STATUS_REQ); + builder.put(CHECK_STATUS_RSP, Verb.ACCORD_CHECK_STATUS_RSP); + builder.put(FETCH_DATA_REQ, Verb.ACCORD_FETCH_DATA_REQ); + builder.put(FETCH_DATA_RSP, Verb.ACCORD_FETCH_DATA_RSP); + builder.put(SET_SHARD_DURABLE_REQ, Verb.ACCORD_SET_SHARD_DURABLE_REQ); + builder.put(SET_GLOBALLY_DURABLE_REQ, Verb.ACCORD_SET_GLOBALLY_DURABLE_REQ); + builder.put(GET_DURABLE_BEFORE_REQ, Verb.ACCORD_GET_DURABLE_BEFORE_REQ); + builder.put(GET_DURABLE_BEFORE_RSP, Verb.ACCORD_GET_DURABLE_BEFORE_RSP); + builder.put(FAILURE_RSP, Verb.FAILURE_RSP); + Map mapping = builder.build(); + StandardMessage.initialise(mapping); } - } - private static Verb getVerb(MessageType type) - { - return VerbMapping.instance.mapping.get(type); - } + private static Verb getVerb(MessageType type) + { + if (type.getClass() == StandardMessage.class) + return (Verb) ((StandardMessage) type).mapToImplementation(); + return ((AccordMessageType)type).verb; + } - private static Verb getVerb(Request request) - { - MessageType type = request.type(); - if (type != null) - return getVerb(type); - - return null; + private static Verb getVerb(Request request) + { + MessageType type = request.type(); + if (type != null) + return getVerb(type); + return null; + } } private final AccordAgent agent; @@ -228,7 +170,7 @@ public class AccordMessageSink implements MessageSink @Override public void send(Node.Id to, Request request) { - Verb verb = getVerb(request); + Verb verb = VerbMapping.getVerb(request); Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type()); Message message = Message.out(verb, request); InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to); @@ -236,59 +178,43 @@ public class AccordMessageSink implements MessageSink messaging.send(message, endpoint); } - private static boolean isRangeBarrier(Request request) - { - TxnId txnId = null; - if (request instanceof TxnRequest) - { - TxnRequest txnRequest = (TxnRequest) request; - txnId = txnRequest.txnId; - } - else if (request instanceof ReadData) - { - ReadData epochRequest = (ReadData)request; - txnId = epochRequest.txnId; - } - - return txnId != null && txnId.isSyncPoint() && txnId.is(Range); - } - // TODO (expected): permit bulk send to save esp. on callback registration (and combine records) @Override - public void send(Node.Id to, Request request, AgentExecutor executor, Callback callback) + public void send(Node.Id to, Request request, int attempt, AgentExecutor executor, Callback callback) { - Verb verb = getVerb(request); + Verb verb = VerbMapping.getVerb(request); Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type()); long nowNanos = Clock.Global.nanoTime(); - long delayedAtNanos = Long.MAX_VALUE; - long expiresAtNanos; - if (isRangeBarrier(request)) - expiresAtNanos = nowNanos + DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(); - else - expiresAtNanos = nowNanos + verb.expiresAfterNanos(); + TxnId txnId = request.primaryTxnId(); + long slowAtNanos = Long.MAX_VALUE; + long expiresAtNanos = nowNanos + expire(txnId, verb).computeWait(attempt, NANOSECONDS); switch (verb) { case ACCORD_READ_REQ: case ACCORD_STABLE_THEN_READ_REQ: - if (agent.slowRead() == null || isRangeBarrier(request)) - break; - case ACCORD_CHECK_STATUS_REQ: - delayedAtNanos = nowNanos + agent.slowRead().computeWait(1, NANOSECONDS); + { + TimeoutStrategy slow = slowRead(txnId); + if (slow != null) + slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS); break; + } case ACCORD_PRE_ACCEPT_REQ: - if (agent.slowPreaccept() == null || isRangeBarrier(request)) - break; - delayedAtNanos = nowNanos + agent.slowPreaccept().computeWait(1, NANOSECONDS); + { + TimeoutStrategy slow = slowPreaccept(txnId); + if (slow != null) + slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS); + break; + } } Message message = Message.out(verb, request, expiresAtNanos); InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to); logger.trace("Sending {} {} to {}", verb, message.payload, endpoint); - callbacks.registerAt(message.id(), executor, callback, to, nowNanos, delayedAtNanos, expiresAtNanos, NANOSECONDS); + callbacks.registerAt(message.id(), executor, callback, to, nowNanos, slowAtNanos, expiresAtNanos, NANOSECONDS); messaging.send(message, endpoint); } @@ -317,7 +243,7 @@ public class AccordMessageSink implements MessageSink private static void checkReplyType(Reply reply, ResponseContext respondTo) { - Verb verb = getVerb(reply.type()); + Verb verb = VerbMapping.getVerb(reply.type()); Preconditions.checkNotNull(verb, "Verb is null for type %s", reply.type()); Set allowedVerbs = expectedReplyTypes(respondTo.verb()); Preconditions.checkArgument(allowedVerbs.contains(verb), "Expected reply message with verbs %s but got %s; reply type was %s, request verb was %s", allowedVerbs, verb, reply.type(), respondTo.verb()); @@ -325,7 +251,7 @@ public class AccordMessageSink implements MessageSink private static Set expectedReplyTypes(Verb verb) { - Set extra = VerbMapping.instance.overrideReplyVerbs.get(verb); + Set extra = VerbMapping.overrideReplyVerbs.get(verb); if (extra != null) return extra; Verb v = verb.responseVerb; return v == null ? Collections.emptySet() : Collections.singleton(v); diff --git a/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java b/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java index d1e507fd2c..aa7f9cb554 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java +++ b/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java @@ -25,7 +25,6 @@ import accord.api.Key; import accord.api.Result; import accord.api.RoutingKey; import accord.local.Command; -import accord.local.Command.WaitingOn; import accord.local.ICommand; import accord.local.Node; import accord.local.StoreParticipants; @@ -79,7 +78,6 @@ import static accord.local.Command.NotAcceptedWithoutDefinition.notAccepted; import static accord.local.Command.NotDefined.notDefined; import static accord.local.Command.PreAccepted.preaccepted; import static accord.local.Command.Truncated.*; -import static accord.local.StoreParticipants.empty; import static accord.local.cfk.CommandsForKey.InternalStatus.ACCEPTED; import static accord.primitives.Status.Durability.NotDurable; import static accord.primitives.TxnId.NO_TXNIDS; @@ -400,7 +398,7 @@ public class AccordObjectSizes return size; } - private static long EMPTY_WAITING_ON_SIZE = measure(new WaitingOn(null, null, null, null, null)); + private static long EMPTY_WAITING_ON_SIZE = measure(new WaitingOn(null, null, null, null)); private static long EMPTY_BIT_SET_SIZE = measure(new ImmutableBitSet(0)); private static long waitingOn(WaitingOn waitingOn) { diff --git a/src/java/org/apache/cassandra/service/accord/AccordResult.java b/src/java/org/apache/cassandra/service/accord/AccordResult.java new file mode 100644 index 0000000000..86315b6d60 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/AccordResult.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.function.BiConsumer; +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.coordinate.CoordinationFailed; +import accord.coordinate.Exhausted; +import accord.coordinate.Preempted; +import accord.coordinate.Timeout; +import accord.coordinate.TopologyMismatch; +import accord.primitives.Seekable; +import accord.primitives.Seekables; +import accord.primitives.TxnId; +import accord.utils.UnhandledEnum; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.txn.RetryWithNewProtocolResult; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.concurrent.AsyncFuture; + +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +public class AccordResult extends AsyncFuture implements BiConsumer, IAccordService.IAccordResult +{ + private static final Logger logger = LoggerFactory.getLogger(AccordResult.class); + + final @Nullable TxnId txnId; + final Seekables keysOrRanges; + final RequestBookkeeping bookkeeping; + final long startedAtNanos, deadlineAtNanos; + final boolean isTxnRequest; + + public AccordResult(@Nullable TxnId txnId, Seekables keysOrRanges, RequestBookkeeping bookkeeping, long startedAtNanos, long deadlineAtNanos, boolean isTxnRequest) + { + this.txnId = txnId; + this.keysOrRanges = keysOrRanges; + this.bookkeeping = bookkeeping; + this.startedAtNanos = startedAtNanos; + this.deadlineAtNanos = deadlineAtNanos; + this.isTxnRequest = isTxnRequest; + } + + @Override + public V awaitAndGet() throws RequestExecutionException + { + try + { + if (!awaitUntil(deadlineAtNanos)) + accept(null, new Timeout(txnId, null)); + } + catch (InterruptedException e) + { + accept(null, e); + } + + Throwable fail = fail(); + if (fail != null) + throw (RequestExecutionException) fail; + return success(); + } + + @Override + public void accept(V success, Throwable fail) + { + if (fail != null) + { + RequestExecutionException report; + CoordinationFailed coordinationFailed = findCoordinationFailed(fail); + TxnId txnId = this.txnId; + if (coordinationFailed != null) + { + if (txnId == null && coordinationFailed.txnId() != null) + txnId = coordinationFailed.txnId(); + + if (coordinationFailed instanceof Timeout) + { + report = bookkeeping.newTimeout(txnId, keysOrRanges); + } + else if (coordinationFailed instanceof Preempted) + { + report = bookkeeping.newPreempted(txnId, keysOrRanges); + } + else if (coordinationFailed instanceof Exhausted) + { + report = bookkeeping.newExhausted(txnId, keysOrRanges); + } + else if (isTxnRequest && coordinationFailed instanceof TopologyMismatch) + { + // Excluding bugs topology mismatch can occur because a table was dropped in between creating the txn + // and executing it. + // It could also race with the table stopping/starting being managed by Accord. + // The caller can retry if the table indeed exists and is managed by Accord. + Set txnDroppedTables = txnDroppedTables(keysOrRanges); + Tracing.trace("Accord returned topology mismatch: " + coordinationFailed.getMessage()); + logger.debug("Accord returned topology mismatch", coordinationFailed); + bookkeeping.markTopologyMismatch(); + // Throw IRE in case the caller fails to check if the table still exists + if (!txnDroppedTables.isEmpty()) + { + Tracing.trace("Accord txn uses dropped tables {}", txnDroppedTables); + logger.debug("Accord txn uses dropped tables {}", txnDroppedTables); + throw new InvalidRequestException("Accord transaction uses dropped tables"); + } + trySuccess((V) RetryWithNewProtocolResult.instance); + return; + } + else + { + report = bookkeeping.newFailed(txnId, keysOrRanges); + } + // this case happens when a non-timeout exception is seen, and we are unable to move forward + if (txnId != null && txnId.isSyncPoint()) + AccordAgent.onFailedBarrier(txnId, fail); + } + else + { + report = bookkeeping.newFailed(txnId, keysOrRanges); + } + report.addSuppressed(fail); + tryFailure(report); + } + else + { + if (success == RetryWithNewProtocolResult.instance) + { + bookkeeping.markRetryDifferentSystem(); + Tracing.trace("Got retry different system error from Accord, will retry"); + } + trySuccess(success); + } + bookkeeping.markElapsedNanos(nanoTime() - startedAtNanos); + } + + @Override + public boolean awaitUntil(long nanoTimeDeadline) throws InterruptedException + { + if (super.awaitUntil(nanoTimeDeadline)) + return true; + + accept(null, new Timeout(null, null)); + return false; + } + + public Throwable fail() + { + return cause(); + } + + public V success() + { + return getNow(); + } + + @Override + public AccordResult addCallback(BiConsumer callback) + { + super.addCallback(callback); + return this; + } + + private static CoordinationFailed findCoordinationFailed(Throwable fail) + { + while (fail != null) + { + if (fail instanceof CoordinationFailed) + return (CoordinationFailed) fail; + Throwable next = fail.getCause(); + if (next == fail) + return null; + fail = next; + } + return null; + } + + private static Set txnDroppedTables(Seekables keys) + { + Set tables = new HashSet<>(); + for (Seekable seekable : keys) + { + switch (seekable.domain()) + { + default: UnhandledEnum.unknown(seekable.domain()); + case Key: + tables.add(((PartitionKey) seekable).table()); + break; + case Range: + tables.add(((TokenRange) seekable).table()); + break; + } + } + + Iterator tablesIterator = tables.iterator(); + while (tablesIterator.hasNext()) + { + TableId table = tablesIterator.next(); + if (Schema.instance.getTableMetadata(table) != null) + tablesIterator.remove(); + } + return tables; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 034a798ea4..912617fbe3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -18,10 +18,9 @@ package org.apache.cassandra.service.accord; -import java.math.BigInteger; -import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; @@ -32,43 +31,25 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Stopwatch; -import com.google.common.base.Throwables; import com.google.common.collect.Streams; import com.google.common.primitives.Ints; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import accord.api.BarrierType; import accord.api.Journal; -import accord.api.LocalConfig; -import accord.api.Result; -import accord.api.RoutingKey; -import accord.coordinate.Barrier; -import accord.coordinate.Barrier.AsyncSyncPoint; -import accord.coordinate.CoordinateSyncPoint; -import accord.coordinate.CoordinationAdapter.Adapters.SyncPointAdapter; -import accord.coordinate.CoordinationFailed; -import accord.coordinate.ExecuteSyncPoint; -import accord.coordinate.Exhausted; -import accord.coordinate.FailureAccumulator; -import accord.coordinate.Invalidated; -import accord.coordinate.Preempted; -import accord.coordinate.Timeout; -import accord.coordinate.TopologyMismatch; -import accord.coordinate.tracking.AllTracker; -import accord.coordinate.tracking.RequestStatus; +import accord.coordinate.CoordinateMaxConflict; +import accord.coordinate.CoordinateTransaction; +import accord.coordinate.KeyBarriers; import accord.impl.AbstractConfigurationService; import accord.impl.DefaultLocalListeners; import accord.impl.DefaultRemoteListeners; -import accord.impl.DurabilityScheduling; +import accord.local.UniqueTimeService.AtomicUniqueTimeWithStaleReservation; +import accord.local.durability.DurabilityService; import accord.impl.RequestCallbacks; import accord.impl.SizeOfIntersectionSorter; import accord.impl.progresslog.DefaultProgressLogs; @@ -83,11 +64,9 @@ import accord.local.SafeCommand; import accord.local.ShardDistributor.EvenSplit; import accord.local.cfk.CommandsForKey; import accord.local.cfk.SafeCommandsForKey; -import accord.messages.Callback; -import accord.messages.ReadData; +import accord.local.durability.ShardDurability; import accord.messages.Reply; import accord.messages.Request; -import accord.messages.WaitUntilApplied; import accord.primitives.FullRoute; import accord.primitives.Keys; import accord.primitives.Ranges; @@ -96,12 +75,9 @@ import accord.primitives.SaveStatus; import accord.primitives.Seekable; import accord.primitives.Seekables; import accord.primitives.Status; -import accord.primitives.SyncPoint; import accord.primitives.Timestamp; import accord.primitives.Txn; -import accord.primitives.Txn.Kind; import accord.primitives.TxnId; -import accord.topology.Topologies; import accord.topology.Topology; import accord.topology.TopologyManager; import accord.utils.DefaultRandom; @@ -109,79 +85,64 @@ import accord.utils.Invariants; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; import accord.utils.async.AsyncResult; -import accord.utils.async.AsyncResults; import org.apache.cassandra.concurrent.Shutdownable; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.db.WriteType; -import org.apache.cassandra.dht.AccordSplitter; -import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.exceptions.RequestTimeoutException; -import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.journal.Params; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.metrics.AccordClientRequestMetrics; -import org.apache.cassandra.metrics.ClientRequestMetrics; -import org.apache.cassandra.metrics.ClientRequestsMetricsHolder; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessageDelivery; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.SharedContext; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordSyncPropagator.Notification; +import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping; import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.service.accord.api.AccordRoutableKey; import org.apache.cassandra.service.accord.api.TokenKey.KeyspaceSplitter; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.api.AccordScheduler; import org.apache.cassandra.service.accord.api.AccordTimeService; import org.apache.cassandra.service.accord.api.AccordTopologySorter; import org.apache.cassandra.service.accord.api.CompositeTopologySorter; -import org.apache.cassandra.service.accord.api.PartitionKey; -import org.apache.cassandra.service.accord.exceptions.ReadExhaustedException; -import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException; -import org.apache.cassandra.service.accord.exceptions.WritePreemptedException; import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory; -import org.apache.cassandra.service.accord.repair.RepairSyncPointAdapter; -import org.apache.cassandra.service.accord.txn.RetryWithNewProtocolResult; +import org.apache.cassandra.service.accord.txn.TxnQuery; +import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.service.accord.txn.TxnResult; +import org.apache.cassandra.service.accord.txn.TxnUpdate; import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.consensus.migration.TableMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; -import org.apache.cassandra.utils.Blocking; -import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.ExecutorUtils; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import static accord.local.durability.DurabilityService.SyncLocal.Self; +import static accord.local.durability.DurabilityService.SyncRemote.All; import static accord.messages.SimpleReply.Ok; import static accord.primitives.Routable.Domain.Key; -import static accord.primitives.Routable.Domain.Range; +import static accord.primitives.Txn.Kind.Write; import static accord.primitives.TxnId.Cardinality.cardinality; -import static accord.topology.Topologies.SelectNodeOwnership.SHARE; -import static accord.utils.Invariants.require; -import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.config.DatabaseDescriptor.getAccordCommandStoreShardCount; +import static org.apache.cassandra.config.DatabaseDescriptor.getAccordGlobalDurabilityCycle; +import static org.apache.cassandra.config.DatabaseDescriptor.getAccordShardDurabilityCycle; +import static org.apache.cassandra.config.DatabaseDescriptor.getAccordShardDurabilityTargetSplits; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; -import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics; -import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics; -import static org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.maybeSaveAccordKeyMigrationLocally; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadBookkeeping; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteBookkeeping; import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.getTableMetadata; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -201,7 +162,6 @@ public class AccordService implements IAccordService, Shutdownable private final AccordJournal journal; private final AccordVerbHandler requestHandler; private final AccordResponseVerbHandler responseHandler; - private final LocalConfig configuration; @GuardedBy("this") private State state = State.INIT; @@ -257,8 +217,8 @@ public class AccordService implements IAccordService, Shutdownable // when replacing another node but using the same ip the hostId will also match, this causes no TCM transactions // to be committed... // In order to bootup correctly, need to pull in the current epoch - ClusterMetadata metadata = ClusterMetadata.current(); - as.configurationService().listener.notifyPostCommit(metadata, metadata, false); + ClusterMetadata current = ClusterMetadata.current(); + as.configService().listener.notifyPostCommit(current, current, false); } instance = as; @@ -328,7 +288,6 @@ public class AccordService implements IAccordService, Shutdownable final RequestCallbacks callbacks = new RequestCallbacks(time); this.scheduler = new AccordScheduler(); this.dataStore = new AccordDataStore(); - this.configuration = new AccordConfiguration(DatabaseDescriptor.getRawConfig()); this.journal = new AccordJournal(DatabaseDescriptor.getAccord().journal, agent); this.configService = new AccordConfigurationService(localId); this.fastPathCoordinator = AccordFastPathCoordinator.create(localId, configService); @@ -336,7 +295,7 @@ public class AccordService implements IAccordService, Shutdownable this.node = new Node(localId, messageSink, configService, - time, + time, new AtomicUniqueTimeWithStaleReservation(time), () -> dataStore, new KeyspaceSplitter(new EvenSplit<>(getAccordCommandStoreShardCount(), getPartitioner().accordSplitter())), agent, @@ -351,7 +310,6 @@ public class AccordService implements IAccordService, Shutdownable AccordCommandStores.factory(), new AccordInteropFactory(agent, configService), journal.durableBeforePersister(), - configuration, journal); this.nodeShutdown = toShutdownable(node); this.requestHandler = new AccordVerbHandler<>(node, configService); @@ -440,13 +398,10 @@ public class AccordService implements IAccordService, Shutdownable configService.start(); fastPathCoordinator.start(); ClusterMetadataService.instance().log().addListener(fastPathCoordinator); - node.durabilityScheduling().setDefaultRetryDelay(Ints.checkedCast(DatabaseDescriptor.getAccordDefaultDurabilityRetryDelay(SECONDS)), SECONDS); - node.durabilityScheduling().setMaxRetryDelay(Ints.checkedCast(DatabaseDescriptor.getAccordMaxDurabilityRetryDelay(SECONDS)), SECONDS); - node.durabilityScheduling().setTargetShardSplits(Ints.checkedCast(DatabaseDescriptor.getAccordShardDurabilityTargetSplits())); - node.durabilityScheduling().setGlobalCycleTime(Ints.checkedCast(DatabaseDescriptor.getAccordGlobalDurabilityCycle(SECONDS)), SECONDS); - node.durabilityScheduling().setShardCycleTime(Ints.checkedCast(DatabaseDescriptor.getAccordShardDurabilityCycle(SECONDS)), SECONDS); - node.durabilityScheduling().setTxnIdLag(Ints.checkedCast(DatabaseDescriptor.getAccordScheduleDurabilityTxnIdLag(SECONDS)), TimeUnit.SECONDS); - node.durabilityScheduling().start(); + node.durability().shards().setTargetShardSplits(Ints.checkedCast(getAccordShardDurabilityTargetSplits())); + node.durability().shards().setShardCycleTime(Ints.checkedCast(getAccordShardDurabilityCycle(SECONDS)), SECONDS); + node.durability().global().setGlobalCycleTime(Ints.checkedCast(getAccordGlobalDurabilityCycle(SECONDS)), SECONDS); + node.durability().start(); state = State.STARTED; } @@ -540,133 +495,84 @@ public class AccordService implements IAccordService, Shutdownable return responseHandler; } - public DurabilityScheduling.ImmutableView durabilityScheduling() + public ShardDurability.ImmutableView shardDurability() { - return node.durabilityScheduling().immutableView(); - } - - private Seekables barrier(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, BiFunction, AsyncSyncPoint> syncPoint) - { - Stopwatch sw = Stopwatch.createStarted(); - keysOrRanges = intersectionWithAccordManagedRanges(keysOrRanges); - // It's possible none of them were Accord managed and we aren't going to treat that as an error - if (keysOrRanges.isEmpty()) - { - logger.info("Skipping barrier because there are no ranges managed by Accord"); - return keysOrRanges; - } - - FullRoute route = node.computeRoute(epoch, keysOrRanges); - AccordClientRequestMetrics metrics = isForWrite ? accordWriteMetrics : accordReadMetrics; - try - { - logger.debug("Starting barrier key: {} epoch: {} barrierType: {} isForWrite {}", keysOrRanges, epoch, barrierType, isForWrite); - AsyncResult asyncResult = syncPoint == null - ? Barrier.barrier(node, keysOrRanges, route, epoch, barrierType) - : Barrier.barrier(node, keysOrRanges, route, epoch, barrierType, syncPoint); - long deadlineNanos = requestTime.startedAtNanos() + timeoutNanos; - TxnId txnId = AsyncChains.getBlocking(asyncResult, deadlineNanos - nanoTime(), NANOSECONDS); - if (keysOrRanges.domain() == Key) - { - PartitionKey key = (PartitionKey)keysOrRanges.get(0); - maybeSaveAccordKeyMigrationLocally(key, Epoch.create(txnId.epoch())); - } - logger.debug("Completed barrier attempt in {}ms, {}ms since attempts start, barrier key: {} epoch: {} barrierType: {} isForWrite {}", - sw.elapsed(MILLISECONDS), - NANOSECONDS.toMillis(nanoTime() - requestTime.startedAtNanos()), - keysOrRanges, epoch, barrierType, isForWrite); - return keysOrRanges; - } - catch (ExecutionException e) - { - Throwable cause = Throwables.getRootCause(e); - if (cause instanceof Timeout) - { - TxnId txnId = ((Timeout) cause).txnId(); - ((AccordAgent) node.agent()).onFailedBarrier(txnId, keysOrRanges, cause); - metrics.timeouts.mark(); - throw newBarrierTimeout(((CoordinationFailed)cause).txnId(), barrierType, isForWrite, keysOrRanges); - } - if (cause instanceof Preempted) - { - TxnId txnId = ((Preempted) cause).txnId(); - ((AccordAgent) node.agent()).onFailedBarrier(txnId, keysOrRanges, cause); - //TODO need to improve - // Coordinator "could" query the accord state to see whats going on but that doesn't exist yet. - // Protocol also doesn't have a way to denote "unknown" outcome, so using a timeout as the closest match - throw newBarrierPreempted(((CoordinationFailed)cause).txnId(), barrierType, isForWrite, keysOrRanges); - } - if (cause instanceof Exhausted) - { - TxnId txnId = ((Exhausted) cause).txnId(); - ((AccordAgent) node.agent()).onFailedBarrier(txnId, keysOrRanges, cause); - // this case happens when a non-timeout exception is seen, and we are unable to move forward - metrics.failures.mark(); - throw newBarrierExhausted(((CoordinationFailed)cause).txnId(), barrierType, isForWrite, keysOrRanges); - } - // unknown error - metrics.failures.mark(); - throw new RuntimeException(cause); - } - catch (InterruptedException e) - { - metrics.failures.mark(); - throw new UncheckedInterruptedException(e); - } - catch (TimeoutException e) - { - metrics.timeouts.mark(); - throw newBarrierTimeout(null, barrierType, isForWrite, keysOrRanges); - } - finally - { - // TODO Should barriers have a dedicated latency metric? Should it be a read/write metric? - // What about counts for timeouts/failures/preempts? - metrics.addNano(nanoTime() - requestTime.startedAtNanos()); - } + return node.durability().shards().immutableView(); } @Override - public Seekables barrier(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite) + public AsyncChain sync(Object requestedBy, Timestamp minBound, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) { - return barrier(keysOrRanges, epoch, requestTime, timeoutNanos, barrierType, isForWrite, null); - } - - public static BiFunction, AsyncSyncPoint> repairSyncPoint(Set allNodes) - { - return (node, route) -> { - TxnId txnId = node.nextTxnId(Kind.SyncPoint, route.domain()); - AsyncResult> async = CoordinateSyncPoint.coordinate(node, Kind.SyncPoint, route, (SyncPointAdapter)RepairSyncPointAdapter.create(allNodes)); - return new AsyncSyncPoint(txnId, async); - }; + return node.durability().sync(requestedBy, minBound, ranges, include, syncLocal, syncRemote); } @Override - public Seekables repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List allEndpoints) + public AsyncChain sync(Timestamp minBound, Keys keys, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) { - Set allNodes = allEndpoints.stream().map(configService::mappedId).collect(Collectors.toUnmodifiableSet()); - return barrier(keysOrRanges, epoch, requestTime, timeoutNanos, barrierType, isForWrite, repairSyncPoint(allNodes)); + if (keys.size() != 1) + return syncInternal(minBound, keys, syncLocal, syncRemote); + + return KeyBarriers.find(node, minBound, keys.get(0).toUnseekable(), syncLocal, syncRemote) + .flatMap(found -> KeyBarriers.await(node, found, syncLocal, syncRemote)) + .flatMap(success -> { + if (success) + return null; + return syncInternal(minBound, keys, syncLocal, syncRemote); + }); } - private static Seekables intersectionWithAccordManagedRanges(Seekables keysOrRanges) + private AsyncChain syncInternal(Timestamp minBound, Keys keys, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) { - TableId tableId = null; - for (Seekable keyOrRange : keysOrRanges) - { - TableId newTableId; - if (keysOrRanges.domain() == Key) - newTableId = ((PartitionKey)keyOrRange).table(); - else if (keysOrRanges.domain() == Range) - newTableId = ((TokenRange) keyOrRange).table(); - else - throw new IllegalStateException("Unexpected domain " + keysOrRanges.domain()); + TxnId txnId = node.nextTxnId(minBound, Write, Key, cardinality(keys)); + FullRoute route = node.computeRoute(txnId, keys); + Txn txn = new Txn.InMemory(Write, keys, TxnRead.createNoOpRead(keys), TxnQuery.NONE, TxnUpdate.empty()); + return CoordinateTransaction.coordinate(node, route, txnId, txn) + .map(ignore -> (Void)null).beginAsResult(); + } - if (tableId == null) - tableId = newTableId; - else if (!tableId.equals(newTableId)) - throw new IllegalArgumentException("Currently only one table is handled here."); - } + @Override + public AsyncChain maxConflict(Ranges ranges) + { + return node.commandStores().any().build(() -> CoordinateMaxConflict.maxConflict(node, ranges)).flatMap(i -> i); + } + public static V getBlocking(AsyncChain async, Seekables keysOrRanges, RequestBookkeeping bookkeeping, long startedAt, long deadline, boolean isTxnRequest) + { + return getBlocking(async, null, keysOrRanges, bookkeeping, startedAt, deadline, isTxnRequest); + } + + public static V getBlocking(AsyncChain async, @Nullable TxnId txnId, Seekables keysOrRanges, RequestBookkeeping bookkeeping, long startedAt, long deadline, boolean isTxnRequest) + { + AccordResult result = new AccordResult<>(txnId, keysOrRanges, bookkeeping, startedAt, deadline, isTxnRequest); + async.begin(result); + return result.awaitAndGet(); + } + + public static void getBlocking(AsyncChain async, Seekables keysOrRanges, RequestBookkeeping bookkeeping, long startedAt, long deadline) + { + getBlocking(async, keysOrRanges, bookkeeping, startedAt, deadline, false); + } + + public static Keys intersecting(Keys keys) + { + if (keys.isEmpty()) + return keys; + + TableId tableId = tableId(keys, r -> ((AccordRoutableKey)r).table()); + return sliceToAccord(tableId, keys, Keys::slice); + } + + public static Ranges intersecting(Ranges ranges) + { + if (ranges.isEmpty()) + return ranges; + + TableId tableId = tableId(ranges, r -> ((TokenRange)r).table()); + return sliceToAccord(tableId, ranges, Ranges::slice); + } + + private static > C sliceToAccord(TableId tableId, C collection, BiFunction slice) + { ClusterMetadata cm = ClusterMetadata.current(); TableMetadata tm = getTableMetadata(cm, tableId); @@ -678,119 +584,26 @@ public class AccordService implements IAccordService, Shutdownable TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tm.id); // null is fine could be completely migrated or was always an Accord table on creation if (tms == null) - return keysOrRanges; + return collection; // Use migratingAndMigratedRanges (not accordSafeToReadRanges) because barriers are allowed even if Accord can't perform // a read because they are only finishing/recovering existing Accord transactions Ranges migratingAndMigratedRanges = AccordTopology.toAccordRanges(tms.tableId, tms.migratingAndMigratedRanges); - return keysOrRanges.slice(migratingAndMigratedRanges); + return slice.apply(collection, migratingAndMigratedRanges); } - switch (keysOrRanges.domain()) + return slice.apply(collection, Ranges.EMPTY); + } + + + private static > TableId tableId(C collection, Function getTableId) + { + TableId tableId = getTableId.apply(collection.get(0)); + for (int i = 1, maxi = collection.size() ; i < maxi ; ++i) { - case Key: - return Keys.EMPTY; - case Range: - return Ranges.EMPTY; - default: - throw new IllegalStateException("Only keys and ranges are supported"); + TableId check = getTableId.apply(collection.get(i)); + Invariants.require(tableId.equals(check), "Currently only one table is handled here."); } - } - - @VisibleForTesting - static ReadTimeoutException newBarrierTimeout(@Nonnull TxnId txnId, BarrierType barrierType, boolean isForWrite, Seekables keysOrRanges) - { - return new ReadTimeoutException(barrierType.global ? ConsistencyLevel.ANY : ConsistencyLevel.QUORUM, 0, 0, false, String.format("Timeout waiting on barrier %s / %s / %s; impacted ranges %s", txnId, barrierType, isForWrite ? "write" : "not write", keysOrRanges)); - } - - @VisibleForTesting - static ReadTimeoutException newBarrierPreempted(@Nullable TxnId txnId, BarrierType barrierType, boolean isForWrite, Seekables keysOrRanges) - { - return new ReadPreemptedException(barrierType.global ? ConsistencyLevel.ANY : ConsistencyLevel.QUORUM, 0, 0, false, String.format("Preempted waiting on barrier %s / %s / %s; impacted ranges %s", txnId, barrierType, isForWrite ? "write" : "not write", keysOrRanges)); - } - - @VisibleForTesting - static ReadExhaustedException newBarrierExhausted(@Nullable TxnId txnId, BarrierType barrierType, boolean isForWrite, Seekables keysOrRanges) - { - return new ReadExhaustedException(barrierType.global ? ConsistencyLevel.ANY : ConsistencyLevel.QUORUM, 0, 0, false, String.format("Exhausted (too many failures from peers) waiting on barrier %s / %s / %s; impacted ranges %s", txnId, barrierType, isForWrite ? "write" : "not write", keysOrRanges)); - } - - @VisibleForTesting - static boolean isTimeout(Throwable t) - { - return t instanceof Timeout || t instanceof ReadTimeoutException || t instanceof Preempted || t instanceof ReadPreemptedException; - } - - @VisibleForTesting - static Seekables doWithRetries(Blocking blocking, Supplier action, int retryAttempts, long initialBackoffMillis, long maxBackoffMillis) throws InterruptedException - { - // Since we could end up having the barrier transaction or the transaction it listens to invalidated - Throwable existingFailures = null; - Seekables success = null; - long backoffMillis = initialBackoffMillis; - for (int attempt = 0; attempt < retryAttempts; attempt++) - { - try - { - success = action.get(); - break; - } - catch (TopologyMismatch topologyMismatch) - { - // Retry topology mismatch immediately because we should be able calculate the correct ranges immediately - backoffMillis = 0; - } - catch (RequestExecutionException | CoordinationFailed newFailures) - { - logger.error("Had failure on barrier", newFailures); - existingFailures = FailureAccumulator.append(existingFailures, newFailures, AccordService::isTimeout); - - try - { - blocking.sleep(backoffMillis); - } - catch (InterruptedException e) - { - if (existingFailures != null) - e.addSuppressed(existingFailures); - throw e; - } - backoffMillis = Math.min(backoffMillis * 2, maxBackoffMillis); - } - catch (Throwable t) - { - // if an unknown/unexpected error happens retry stops right away - if (existingFailures != null) - t.addSuppressed(existingFailures); - existingFailures = t; - break; - } - } - if (success == null) - { - logger.error("Ran out of retries for barrier"); - require(existingFailures != null, "Didn't have success, but also didn't have failures"); - Throwables.throwIfUnchecked(existingFailures); - throw new RuntimeException(existingFailures); - } - return success; - } - - @Override - public Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException - { - return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().barrier(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(), barrierType, isForWrite), - DatabaseDescriptor.getAccordBarrierRetryAttempts(), - DatabaseDescriptor.getAccordBarrierRetryInitialBackoffMillis(), - DatabaseDescriptor.getAccordBarrierRetryMaxBackoffMillis()); - } - - @Override - public Seekables repairWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List allEndpoints) throws InterruptedException - { - return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().repair(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(), barrierType, isForWrite, allEndpoints), - DatabaseDescriptor.getAccordBarrierRetryAttempts(), - DatabaseDescriptor.getAccordBarrierRetryInitialBackoffMillis(), - DatabaseDescriptor.getAccordBarrierRetryMaxBackoffMillis()); + return tableId; } @Override @@ -805,195 +618,27 @@ public class AccordService implements IAccordService, Shutdownable return node.topology(); } - private static Set txnDroppedTables(Seekables keys) - { - Set tables = new HashSet<>(); - for (Seekable seekable : keys) - { - switch (seekable.domain()) - { - default: - throw new IllegalStateException("Unhandled domain " + seekable.domain()); - case Key: - tables.add(((PartitionKey) seekable).table()); - break; - case Range: - tables.add(((TokenRange) seekable).table()); - break; - } - } - - Iterator tablesIterator = tables.iterator(); - while (tablesIterator.hasNext()) - { - TableId table = tablesIterator.next(); - if (Schema.instance.getTableMetadata(table) != null) - tablesIterator.remove(); - } - return tables; - } - /** * Consistency level is just echoed back in timeouts, in the future it may be used for interoperability * with non-Accord operations. */ @Override - public @Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull Dispatcher.RequestTime requestTime) + public @Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull Dispatcher.RequestTime requestTime) throws RequestExecutionException { - AsyncTxnResult asyncTxnResult = coordinateAsync(minEpoch, txn, consistencyLevel, requestTime); - return getTxnResult(asyncTxnResult); + return coordinateAsync(minEpoch, txn, consistencyLevel, requestTime).awaitAndGet(); } @Override - public @Nonnull AsyncTxnResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull Dispatcher.RequestTime requestTime) + public @Nonnull IAccordResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull Dispatcher.RequestTime requestTime) { TxnId txnId = node.nextTxnId(txn.kind(), txn.keys().domain(), cardinality(txn.keys())); - ClientRequestMetrics sharedMetrics; - AccordClientRequestMetrics metrics; - if (txn.isWrite()) - { - sharedMetrics = ClientRequestsMetricsHolder.writeMetrics; - metrics = accordWriteMetrics; - } - else - { - sharedMetrics = ClientRequestsMetricsHolder.readMetrics; - metrics = accordReadMetrics; - } - metrics.keySize.update(txn.keys().size()); - long deadlineNanos = requestTime.computeDeadline(DatabaseDescriptor.getTransactionTimeout(NANOSECONDS)); - AsyncResult asyncResult = node.coordinate(txnId, txn, minEpoch, deadlineNanos); - Seekables keys = txn.keys(); - AsyncTxnResult asyncTxnResult = new AsyncTxnResult(txnId, minEpoch, consistencyLevel, txn.isWrite(), requestTime); - asyncResult.addCallback((success, failure) -> { - long durationNanos = nanoTime() - requestTime.startedAtNanos(); - sharedMetrics.addNano(durationNanos); - metrics.addNano(durationNanos); - Throwable cause = failure != null ? Throwables.getRootCause(failure) : null; - if (success != null) - { - if (((TxnResult) success).kind() == TxnResult.Kind.retry_new_protocol) - { - metrics.retryDifferentSystem.mark(); - Tracing.trace("Got retry different system error from Accord, will retry"); - } - asyncTxnResult.trySuccess((TxnResult) success); - return; - } - - sharedMetrics.failures.mark(); - metrics.failures.mark(); - - if (cause instanceof Timeout) - { - // Don't mark the metric here, should be done in getTxnResult to ensure it only happens once - // since both Accord and the thread blocked on the result can trigger a timeout - asyncTxnResult.tryFailure(newTimeout(txnId, txn.isWrite(), consistencyLevel)); - return; - } - if (cause instanceof Preempted || cause instanceof Invalidated) - { - metrics.preempted.mark(); - //TODO need to improve - // Coordinator "could" query the accord state to see whats going on but that doesn't exist yet. - // Protocol also doesn't have a way to denote "unknown" outcome, so using a timeout as the closest match - asyncTxnResult.tryFailure(newPreempted(txnId, txn.isWrite(), consistencyLevel)); - return; - } - // TODO (desired): It would be better to check if the topology mismatch is due to Accord ownership changes - // or the txn accessing tables that don't exist or something. - if (cause instanceof TopologyMismatch) - { - // Excluding bugs topology mismatch can occur because a table was dropped in between creating the txn - // and executing it. - // It could also race with the table stopping/starting being managed by Accord. - // The caller can retry if the table indeed exists and is managed by Accord. - Set txnDroppedTables = txnDroppedTables(keys); - Tracing.trace("Accord returned topology mismatch: " + cause.getMessage()); - logger.debug("Accord returned topology mismatch", cause); - metrics.topologyMismatches.mark(); - // Throw IRE in case the caller fails to check if the table still exists - if (!txnDroppedTables.isEmpty()) - { - Tracing.trace("Accord txn uses dropped tables {}", txnDroppedTables); - logger.debug("Accord txn uses dropped tables {}", txnDroppedTables); - asyncTxnResult.setFailure(new InvalidRequestException("Accord transaction uses dropped tables")); - } - asyncTxnResult.setSuccess(RetryWithNewProtocolResult.instance); - return; - } - asyncTxnResult.tryFailure(new RuntimeException(cause)); - }); - return asyncTxnResult; - } - - @Override - public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult) - { - ClientRequestMetrics sharedMetrics; - AccordClientRequestMetrics metrics; - if (asyncTxnResult.isWrite) - { - sharedMetrics = ClientRequestsMetricsHolder.writeMetrics; - metrics = accordWriteMetrics; - } - else - { - sharedMetrics = ClientRequestsMetricsHolder.readMetrics; - metrics = accordReadMetrics; - } - try - { - long deadlineNanos = asyncTxnResult.requestTime.computeDeadline(DatabaseDescriptor.getTransactionTimeout(NANOSECONDS)); - TxnResult result = asyncTxnResult.get(deadlineNanos - nanoTime(), NANOSECONDS); - return result; - } - catch (ExecutionException e) - { - // Metrics except timeout have already been handled - Throwable cause = e.getCause(); - if (cause instanceof RequestTimeoutException) - { - // Mark here instead of in coordinate async since this is where the request timeout actually occurs - metrics.timeouts.mark(); - sharedMetrics.timeouts.mark(); - cause.addSuppressed(e); - throw (RequestTimeoutException) cause; - } - else if (cause instanceof RuntimeException) - throw (RuntimeException) cause; - else - throw new RuntimeException(cause); - } - catch (InterruptedException e) - { - metrics.failures.mark(); - sharedMetrics.failures.mark(); - throw new UncheckedInterruptedException(e); - } - catch (TimeoutException e) - { - metrics.timeouts.mark(); - sharedMetrics.timeouts.mark(); - throw newTimeout(asyncTxnResult.txnId, asyncTxnResult.isWrite, asyncTxnResult.consistencyLevel); - } - } - - private static RequestTimeoutException newTimeout(TxnId txnId, boolean isWrite, ConsistencyLevel consistencyLevel) - { - // Client protocol doesn't handle null consistency level so use ANY - if (consistencyLevel == null) - consistencyLevel = ConsistencyLevel.ANY; - return isWrite ? new WriteTimeoutException(WriteType.CAS, consistencyLevel, 0, 0, txnId.toString()) - : new ReadTimeoutException(consistencyLevel, 0, 0, false, txnId.toString()); - } - - private static RuntimeException newPreempted(TxnId txnId, boolean isWrite, ConsistencyLevel consistencyLevel) - { - if (consistencyLevel == null) - consistencyLevel = ConsistencyLevel.ANY; - return isWrite ? new WritePreemptedException(WriteType.CAS, consistencyLevel, 0, 0, txnId.toString()) - : new ReadPreemptedException(consistencyLevel, 0, 0, false, txnId.toString()); + long timeout = txnId.isWrite() ? DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS) : DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS); + ClientRequestBookkeeping bookkeeping = txn.isWrite() ? accordWriteBookkeeping : accordReadBookkeeping; + bookkeeping.metrics.keySize.update(txn.keys().size()); + long deadlineNanos = requestTime.computeDeadline(timeout); + AccordResult result = new AccordResult<>(txnId, txn.keys(), bookkeeping, requestTime.startedAtNanos(), deadlineNanos, true); + ((AsyncResult)node.coordinate(txnId, txn, minEpoch, deadlineNanos)).begin(result); + return result; } @Override @@ -1217,35 +862,6 @@ public class AccordService implements IAccordService, Shutdownable return node.topology().minEpoch(); } - @Override - public void tryMarkRemoved(Topology topology, Id target) - { - if (node.commandStores().count() == 0) return; // when starting up stores can be empty, so ignore - Ranges ranges = topology.rangesForNode(target); - if (ranges.isEmpty()) return; - long startNanos = Clock.Global.nanoTime(); - exclusiveSyncPointWithRetries(ranges, 0) - .begin((s, f) -> { - if (f != null) - { - logger.warn("Unable to mark the ranges for {} as durable after node left; took {}", target, Duration.ofNanos(Clock.Global.nanoTime() - startNanos), f); - node.agent().onUncaughtException(f); - } - else - { - logger.info("Marked {} ranges as durable after node left; took {}", target, Duration.ofNanos(Clock.Global.nanoTime() - startNanos)); - } - }); - } - - private AsyncChain> exclusiveSyncPointWithRetries(Ranges ranges, int attempt) - { - return CoordinateSyncPoint.exclusiveSyncPoint(node, ranges) - .recover(t -> - //TODO (operability): make this configurable / monitorable? - attempt <= 3 && t instanceof Invalidated || t instanceof Preempted || t instanceof Timeout ? exclusiveSyncPointWithRetries(ranges, attempt + 1) : null); - } - public Node node() { return node; @@ -1326,7 +942,7 @@ public class AccordService implements IAccordService, Shutdownable } @VisibleForTesting - public AccordConfigurationService configurationService() + public AccordConfigurationService configService() { return configService; } @@ -1348,225 +964,27 @@ public class AccordService implements IAccordService, Shutdownable } @Override - public void awaitTableDrop(TableId id) + public void awaitDone(TableId id, long epoch) { // Need to make sure no existing txn are still being processed for this table... this is only used by DROP TABLE so NEW txn are expected to be blocked, so just need to "wait" for existing ones to complete Topology topology = node.topology().current(); - List ranges = topology.reduce(new ArrayList<>(), + List rangeList = topology.reduce(new ArrayList<>(), s -> ((TokenRange) s.range).table().equals(id), (accum, s) -> { accum.add((TokenRange) s.range); return accum; }); - if (ranges.isEmpty()) return; // nothing to see here + if (rangeList.isEmpty()) return; // nothing to see here - ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(id); - Invariants.require(cfs != null, "Unable to find table %s", id); - BigInteger targetSplitSize = BigInteger.valueOf(Math.max(1, cfs.estimateKeys() / 1_000_000)); - - List> syncs = new ArrayList<>(ranges.size()); - for (TokenRange range : ranges) - syncs.add(awaitTableDrop(cfs, range, targetSplitSize)); - AsyncChain all = AsyncChains.allOf(syncs); - try - { - AsyncChains.getBlocking(all); - } - catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - throw new UncheckedInterruptedException(e); - } - catch (ExecutionException e) - { - throw new RuntimeException(e); - } + Ranges ranges = Ranges.of(rangeList.toArray(accord.primitives.Range[]::new)); + long startedAt = nanoTime(); + long deadline = startedAt + DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(); + // TODO (required): relax this requirement - too expensive + getBlocking(node.durability().sync("Drop Keyspace/Table (Epoch " + epoch + ')', TxnId.minForEpoch(epoch), ranges, Self, All), ranges, new LatencyRequestBookkeeping(null), startedAt, deadline, false); } public Params journalConfiguration() { return journal.configuration(); } - - private AsyncChain awaitTableDrop(ColumnFamilyStore cfs, TokenRange range, BigInteger targetSplitSize) - { - List splits = split(cfs, range, targetSplitSize); - List> syncs = new ArrayList<>(splits.size()); - for (TokenRange tr : splits) - syncs.add(awaitTableDropSubRange(tr)); - return AsyncChains.allOf(syncs); - } - - private List split(ColumnFamilyStore cfs, TokenRange range, BigInteger targetSplitSize) - { - if (targetSplitSize.equals(BigInteger.ONE)) return Collections.singletonList(range); - - AccordSplitter splitter = cfs.getPartitioner().accordSplitter().apply(Ranges.single(range)); - RoutingKey remainingStart = range.start(); - - BigInteger rangeSize = splitter.sizeOf(range); - BigInteger divide = splitter.divide(rangeSize, targetSplitSize); - BigInteger rangeStep = divide.equals(BigInteger.ZERO) ? rangeSize : BigInteger.ONE.max(divide); - BigInteger offset = BigInteger.ZERO; - List result = new ArrayList<>(); - - while (splitter.compare(offset, rangeSize) < 0) - { - BigInteger remaining = rangeSize.subtract(offset); - BigInteger length = remaining.min(rangeStep); - - TokenRange next = splitter.subRange(range, offset, splitter.add(offset, length)); - result.add(next); - remainingStart = next.end(); - offset = offset.add(length); - } - - if (!remainingStart.equals(range.end())) - result.add(range.newRange(remainingStart, range.end())); - assert result.get(0).start().equals(range.start()) : String.format("Starting range %s does not have the same start as %s", result.get(0), range); - assert result.get(result.size() - 1).end().equals(range.end()) : String.format("Ending range %s does not have the same end as %s", result.get(result.size() - 1), range); - return result; - } - - private AsyncChain awaitTableDropSubRange(TokenRange range) - { - return awaitTableDropSubRange(Ranges.single(range), 0); - } - - private AsyncChain awaitTableDropSubRange(Ranges ranges, int attempt) - { - return exclusiveSyncPoint(ranges, attempt) - .flatMap(s -> s == null ? AsyncChains.success(null) : Await.coordinate(node, s)); - } - - private AsyncChain> exclusiveSyncPoint(Ranges ranges, int attempt) - { - //TODO (on merge): CASSANDRA-19769 has the same logic... should this be refactored? Would make it nice so we could split the range on retries? - return CoordinateSyncPoint.exclusiveSyncPoint(node, ranges) - .recover(t -> { - //TODO (operability): make this configurable / monitorable? - if (attempt > 3) return null; - switch (shouldRetry(t)) - { - case SUCCESS: - return AsyncChains.success(null); - case RETRY: - return exclusiveSyncPoint(ranges, attempt + 1); - case FAIL: - return null; - default: - throw new UnsupportedOperationException(); - } - }); - } - - private enum RetryDecission { SUCCESS, RETRY, FAIL } - private static RetryDecission shouldRetry(Throwable t) - { - if (t.getClass() == ExecuteSyncPoint.SyncPointErased.class) - return RetryDecission.SUCCESS; - if (t instanceof Invalidated || t instanceof Preempted || t instanceof Timeout) - return RetryDecission.RETRY; - return RetryDecission.FAIL; - } - - // TODO (required): this should use ExecuteSyncPoint.ExecuteExclusive, or perhaps should not exist at all (should have some mechanism to request and await durability) - private static class Await extends AsyncResults.SettableResult> implements Callback - { - private final Node node; - private final AllTracker tracker; - private final SyncPoint exclusiveSyncPoint; - - private Await(Node node, SyncPoint exclusiveSyncPoint) - { - Topologies topologies = node.topology().forEpoch(exclusiveSyncPoint.route, exclusiveSyncPoint.syncId.epoch(), SHARE); - this.node = node; - this.tracker = new AllTracker(topologies); - this.exclusiveSyncPoint = exclusiveSyncPoint; - } - - public static AsyncChain coordinate(Node node, SyncPoint sp) - { - return node.withEpoch(sp.syncId.epoch(), () -> { - Await coordinate = new Await(node, sp); - coordinate.start(); - AsyncChain chain = coordinate.map(i -> null); - return chain.recover(t -> { - switch (shouldRetry(t)) - { - case SUCCESS: return AsyncChains.success(null); - case RETRY: return coordinate(node, sp); - case FAIL: return null; - default: throw new UnsupportedOperationException(); - } - }); - }); - } - - private void start() - { - node.send(tracker.nodes(), to -> new WaitUntilApplied(to, tracker.topologies(), exclusiveSyncPoint.syncId, exclusiveSyncPoint.route, exclusiveSyncPoint.syncId.epoch()), this); - } - @Override - public void onSuccess(Node.Id from, ReadData.ReadReply reply) - { - if (!reply.isOk()) - { - ReadData.CommitOrReadNack nack = (ReadData.CommitOrReadNack) reply; - switch (nack) - { - default: throw new AssertionError("Unhandled: " + reply); - - case Insufficient: - CoordinateSyncPoint.sendApply(node, from, exclusiveSyncPoint); - return; - case Rejected: - tryFailure(new RuntimeException(nack.name())); - case Redundant: - tryFailure(new ExecuteSyncPoint.SyncPointErased()); - return; - } - } - else - { - if (tracker.recordSuccess(from) == RequestStatus.Success) - { - node.configService().reportEpochRedundant(exclusiveSyncPoint.route.toRanges(), exclusiveSyncPoint.syncId.epoch() - 1); - trySuccess(exclusiveSyncPoint); - } - } - } - - private Throwable cause; - - @Override - public void onFailure(Node.Id from, Throwable failure) - { - synchronized (this) - { - if (cause == null) cause = failure; - else - { - try - { - cause.addSuppressed(failure); - } - catch (Throwable t) - { - // can not always add suppress - node.agent().onUncaughtException(failure); - } - } - failure = cause; - } - if (tracker.recordFailure(from) == RequestStatus.Failed) - tryFailure(failure); - } - - @Override - public boolean onCallbackFailure(Node.Id from, Throwable failure) - { - return tryFailure(failure); - } - } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java b/src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java index 5e2e0b8a85..9e9134158c 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java @@ -88,7 +88,7 @@ public class AccordSyncPropagator { final long epoch; ImmutableSet syncComplete = ImmutableSet.of(); // TODO (desired): propagate ack's for other nodes - Ranges closed = Ranges.EMPTY, redundant = Ranges.EMPTY; + Ranges closed = Ranges.EMPTY, retired = Ranges.EMPTY; PendingEpoch(long epoch) { @@ -118,19 +118,19 @@ public class AccordSyncPropagator return new Notification(epoch, Collections.emptySet(), addClosed, Ranges.EMPTY); } - Notification redundant(Ranges addRedundant) + Notification retired(Ranges addRetired) { - if (redundant.containsAll(addRedundant)) + if (retired.containsAll(addRetired)) return null; - addRedundant = addRedundant.without(redundant); - redundant = redundant.with(addRedundant); - return new Notification(epoch, Collections.emptySet(), Ranges.EMPTY, addRedundant); + addRetired = addRetired.without(retired); + retired = retired.with(addRetired); + return new Notification(epoch, Collections.emptySet(), Ranges.EMPTY, addRetired); } boolean isEmpty() { - return syncComplete.isEmpty() && closed.isEmpty() && redundant.isEmpty(); + return syncComplete.isEmpty() && closed.isEmpty() && retired.isEmpty(); } boolean ack(Notification notification) @@ -141,8 +141,8 @@ public class AccordSyncPropagator else syncComplete = ImmutableSet.copyOf(Iterables.filter(syncComplete, v -> !notification.syncComplete.contains(v))); } closed = closed.without(notification.closed); - redundant = redundant.without(notification.redundant); - return syncComplete.isEmpty() && closed.isEmpty() && redundant.isEmpty(); + retired = retired.without(notification.redundant); + return syncComplete.isEmpty() && closed.isEmpty() && retired.isEmpty(); } @Override @@ -152,7 +152,7 @@ public class AccordSyncPropagator "epoch=" + epoch + ", syncComplete=" + syncComplete + ", closed=" + closed + - ", redundant=" + redundant + + ", redundant=" + retired + '}'; } } @@ -277,9 +277,9 @@ public class AccordSyncPropagator report(epoch, notify, PendingEpoch::closed, closed); } - public void reportRedundant(long epoch, Collection notify, Ranges redundant) + public void reportRetired(long epoch, Collection notify, Ranges redundant) { - report(epoch, notify, PendingEpoch::redundant, redundant); + report(epoch, notify, PendingEpoch::retired, redundant); } private synchronized void report(long epoch, Collection notify, ReportPending report, T param) diff --git a/src/java/org/apache/cassandra/service/accord/AccordTopology.java b/src/java/org/apache/cassandra/service/accord/AccordTopology.java index c78a787d99..ebfd99fd8a 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTopology.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTopology.java @@ -42,7 +42,6 @@ import accord.topology.Topology; import accord.utils.Invariants; import accord.utils.SortedArrays.SortedArrayList; import accord.utils.TinyEnumSet; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; @@ -393,7 +392,8 @@ public class AccordTopology try { ClusterMetadataService.instance().fetchLogFromCMS(epoch); - AccordService.instance().epochReady(epoch).get(DatabaseDescriptor.getTransactionTimeout(MILLISECONDS), MILLISECONDS); + IAccordService service = AccordService.instance(); + service.epochReady(epoch).get(service.agent().expireEpochWait(MILLISECONDS), MILLISECONDS); } catch (InterruptedException e) { diff --git a/src/java/org/apache/cassandra/service/accord/ClientRequestBookkeeping.java b/src/java/org/apache/cassandra/service/accord/ClientRequestBookkeeping.java new file mode 100644 index 0000000000..0d960362ef --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/ClientRequestBookkeeping.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord; + +import java.util.function.Function; + +import com.codahale.metrics.Meter; +import org.apache.cassandra.metrics.AccordClientRequestMetrics; +import org.apache.cassandra.metrics.ClientRequestMetrics; + +import static org.apache.cassandra.service.accord.RequestBookkeeping.ThrowsExceptionType.READ; +import static org.apache.cassandra.service.accord.RequestBookkeeping.ThrowsExceptionType.WRITE; + +// TODO (expected): merge with AccordClientRequestMetrics instead +public class ClientRequestBookkeeping extends RequestBookkeeping +{ + final AccordClientRequestMetrics metrics; + + public ClientRequestBookkeeping(boolean isWrite, AccordClientRequestMetrics metrics) + { + super(isWrite ? WRITE : READ); + this.metrics = metrics; + } + + @Override + final void markTimeout() + { + mark(metrics -> metrics.timeouts); + } + + @Override + final void markPreempted() + { + metrics.preempted.mark(); + } + + final void markFailure() + { + mark(metrics -> metrics.failures); + } + + @Override + final void markRetryDifferentSystem() + { + metrics.retryDifferentSystem.mark(); + } + + @Override + final void markTopologyMismatch() + { + metrics.topologyMismatches.mark(); + } + + private void mark(Function get) + { + get.apply(metrics).mark(); + if (metrics.shared != null) + get.apply(metrics.shared).mark(); + } + + public final void markElapsedNanos(long nanos) + { + metrics.addNano(nanos); + if (metrics.shared != null) + metrics.shared.addNano(nanos); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/FetchMinEpoch.java b/src/java/org/apache/cassandra/service/accord/FetchMinEpoch.java index 3c6e18c3d6..495ea7e0db 100644 --- a/src/java/org/apache/cassandra/service/accord/FetchMinEpoch.java +++ b/src/java/org/apache/cassandra/service/accord/FetchMinEpoch.java @@ -28,7 +28,6 @@ import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterators; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.io.IVersionedSerializer; @@ -39,13 +38,14 @@ import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.SharedContext; -import org.apache.cassandra.utils.Backoff; +import org.apache.cassandra.service.WaitStrategy; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.FutureCombiner; import static org.apache.cassandra.net.MessageDelivery.RetryErrorMessage; import static org.apache.cassandra.net.MessageDelivery.RetryPredicate; import static org.apache.cassandra.net.MessageDelivery.logger; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryFetchMinEpoch; // TODO (required, efficiency): this can be simplified: we seem to always use "entire range" public class FetchMinEpoch @@ -80,7 +80,7 @@ public class FetchMinEpoch } else { - logger.error("Accord service is not started, resopnding with error to {}", message); + logger.error("Accord service is not started, responding with error to {}", message); MessagingService.instance().respondWithFailure(RequestFailure.BOOTING, message); } }; @@ -110,7 +110,7 @@ public class FetchMinEpoch @VisibleForTesting static Future fetch(SharedContext context, InetAddressAndPort to) { - Backoff backoff = Backoff.fromConfig(context, DatabaseDescriptor.getAccord().minEpochSyncRetry); + WaitStrategy backoff = retryFetchMinEpoch(); return context.messaging().sendWithRetries(backoff, context.optionalTasks()::schedule, Verb.ACCORD_FETCH_MIN_EPOCH_REQ, diff --git a/src/java/org/apache/cassandra/service/accord/FetchTopology.java b/src/java/org/apache/cassandra/service/accord/FetchTopology.java index dc8df4f454..b58358e9aa 100644 --- a/src/java/org/apache/cassandra/service/accord/FetchTopology.java +++ b/src/java/org/apache/cassandra/service/accord/FetchTopology.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.Collection; import accord.topology.Topology; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.io.IVersionedSerializer; @@ -35,10 +34,12 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.MessagingUtils; import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.SharedContext; +import org.apache.cassandra.service.WaitStrategy; import org.apache.cassandra.service.accord.serializers.TopologySerializers; -import org.apache.cassandra.utils.Backoff; import org.apache.cassandra.utils.concurrent.Future; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryFetchTopology; + public class FetchTopology { public String toString() @@ -127,7 +128,7 @@ public class FetchTopology public static Future fetch(SharedContext context, Collection peers, long epoch) { FetchTopology request = new FetchTopology(epoch); - Backoff backoff = Backoff.fromConfig(context, DatabaseDescriptor.getAccord().fetchRetry); + WaitStrategy backoff = retryFetchTopology(); return context.messaging().sendWithRetries(backoff, context.optionalTasks()::schedule, Verb.ACCORD_FETCH_TOPOLOGY_REQ, diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java index 6155b78cc6..01dddf520a 100644 --- a/src/java/org/apache/cassandra/service/accord/IAccordService.java +++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java @@ -18,11 +18,13 @@ package org.apache.cassandra.service.accord; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -30,7 +32,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.Agent; -import accord.api.BarrierType; +import accord.local.durability.DurabilityService.SyncLocal; +import accord.local.durability.DurabilityService.SyncRemote; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; import accord.local.Node; @@ -38,21 +41,19 @@ import accord.local.Node.Id; import accord.local.RedundantBefore; import accord.messages.Reply; import accord.messages.Request; +import accord.primitives.Keys; import accord.primitives.Ranges; -import accord.primitives.Seekables; +import accord.primitives.Timestamp; import accord.primitives.Txn; import accord.primitives.TxnId; -import accord.topology.Topology; import accord.topology.TopologyManager; import accord.utils.Invariants; +import accord.utils.async.AsyncChain; import org.agrona.collections.Int2ObjectHashMap; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.journal.Params; -import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.schema.TableId; @@ -61,15 +62,10 @@ import org.apache.cassandra.service.accord.api.AccordScheduler; import org.apache.cassandra.service.accord.api.AccordTopologySorter; import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.transport.Dispatcher.RequestTime; -import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.ImmediateFuture; -import static com.google.common.base.Preconditions.checkNotNull; - - // Avoid default methods that aren't just providing wrappers around other methods // so it will be a compile error if DelegatingAccordService doesn't implement them public interface IAccordService @@ -82,55 +78,21 @@ public interface IAccordService IVerbHandler requestHandler(); IVerbHandler responseHandler(); - Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException; + AsyncChain sync(Object requestedBy, @Nullable Timestamp minBound, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote); + AsyncChain sync(@Nullable Timestamp minBound, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote); + AsyncChain maxConflict(Ranges ranges); - Seekables barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite); + @Nonnull IAccordResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime); + @Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime) throws RequestExecutionException; - Seekables repairWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List allEndpoints) throws InterruptedException; - - Seekables repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List allEndpoints); - - default void postStreamReceivingBarrier(ColumnFamilyStore cfs, List> ranges) + interface IAccordResult { - String ks = cfs.keyspace.getName(); - Ranges accordRanges = AccordTopology.toAccordRanges(ks, ranges); - try - { - barrierWithRetries(accordRanges, Epoch.FIRST.getEpoch(), BarrierType.global_async, true); - } - catch (InterruptedException e) - { - throw new RuntimeException(e); - } + V success(); + Throwable fail(); + V awaitAndGet() throws RequestExecutionException; + IAccordResult addCallback(BiConsumer callback); } - @Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime); - - class AsyncTxnResult extends AsyncPromise - { - public final @Nonnull TxnId txnId; - public final long minEpoch; - @Nullable - public final ConsistencyLevel consistencyLevel; - public final boolean isWrite; - public final Dispatcher.RequestTime requestTime; - - public AsyncTxnResult(@Nonnull TxnId txnId, long minEpoch, @Nullable ConsistencyLevel consistencyLevel, boolean isWrite, @Nonnull Dispatcher.RequestTime requestTime) - { - checkNotNull(txnId); - checkNotNull(requestTime); - this.txnId = txnId; - this.minEpoch = minEpoch; - this.consistencyLevel = consistencyLevel; - this.isWrite = isWrite; - this.requestTime = requestTime; - } - } - - @Nonnull - AsyncTxnResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime); - TxnResult getTxnResult(AsyncTxnResult asyncTxnResult); - long currentEpoch(); void setCacheSize(long kb); @@ -197,8 +159,9 @@ public interface IAccordService @Nullable Long minEpoch(); - void tryMarkRemoved(Topology topology, Node.Id node); - void awaitTableDrop(TableId id); + void awaitDone(TableId id, long epoch); + + AccordConfigurationService configService(); Params journalConfiguration(); @@ -225,43 +188,31 @@ public interface IAccordService } @Override - public Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException - { - throw new UnsupportedOperationException(); - } - - @Override - public Seekables barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite) - { - throw new UnsupportedOperationException("No accord barriers should be executed when accord.enabled = false in cassandra.yaml"); - } - - @Override - public Seekables repairWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List allEndpoints) throws InterruptedException - { - return null; - } - - @Override - public Seekables repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List allEndpoints) - { - throw new UnsupportedOperationException("No accord repairs should be executed when accord.enabled = false in cassandra.yaml"); - } - - @Override - public @Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull Dispatcher.RequestTime requestTime) + public AsyncChain sync(Object requestedBy, @Nullable Timestamp onOrAfter, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote) { throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml"); } @Override - public @Nonnull AsyncTxnResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + public AsyncChain sync(@Nullable Timestamp onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote) { throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml"); } @Override - public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult) + public AsyncChain maxConflict(Ranges ranges) + { + throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml"); + } + + @Override + public @Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull RequestTime requestTime) + { + throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml"); + } + + @Override + public @Nonnull IAccordResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime) { throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml"); } @@ -347,15 +298,15 @@ public interface IAccordService } @Override - public void tryMarkRemoved(Topology topology, Id node) + public void awaitDone(TableId id, long epoch) { } @Override - public void awaitTableDrop(TableId id) + public AccordConfigurationService configService() { - + return null; } @Override @@ -399,33 +350,27 @@ public interface IAccordService } @Override - public Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException + public AsyncChain sync(Object requestedBy, @Nullable Timestamp onOrAfter, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote) { - return delegate.barrierWithRetries(keysOrRanges, minEpoch, barrierType, isForWrite); + return delegate.sync(requestedBy, onOrAfter, ranges, include, syncLocal, syncRemote); } @Override - public Seekables barrier(@Nonnull Seekables keysOrRanges, long minEpoch, RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite) + public AsyncChain sync(@Nullable Timestamp onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote) { - return delegate.barrier(keysOrRanges, minEpoch, requestTime, timeoutNanos, barrierType, isForWrite); + return delegate.sync(onOrAfter, keys, syncLocal, syncRemote); } @Override - public Seekables repairWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List allEndpoints) throws InterruptedException + public AsyncChain maxConflict(Ranges ranges) { - return delegate.repairWithRetries(keysOrRanges, minEpoch, barrierType, isForWrite, allEndpoints); + return delegate.maxConflict(ranges); } @Override - public Seekables repair(@Nonnull Seekables keysOrRanges, long epoch, RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List allEndpoints) + public AccordConfigurationService configService() { - return delegate.repair(keysOrRanges, epoch, requestTime, timeoutNanos, barrierType, isForWrite, allEndpoints); - } - - @Override - public void postStreamReceivingBarrier(ColumnFamilyStore cfs, List> ranges) - { - IAccordService.super.postStreamReceivingBarrier(cfs, ranges); + return delegate.configService(); } @Nonnull @@ -437,17 +382,11 @@ public interface IAccordService @Nonnull @Override - public AsyncTxnResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime) + public IAccordResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime) { return delegate.coordinateAsync(minEpoch, txn, consistencyLevel, requestTime); } - @Override - public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult) - { - return delegate.getTxnResult(asyncTxnResult); - } - @Override public long currentEpoch() { @@ -534,15 +473,9 @@ public interface IAccordService } @Override - public void tryMarkRemoved(Topology topology, Id node) + public void awaitDone(TableId id, long epoch) { - delegate.tryMarkRemoved(topology, node); - } - - @Override - public void awaitTableDrop(TableId id) - { - delegate.awaitTableDrop(id); + delegate.awaitDone(id, epoch); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/RequestBookkeeping.java b/src/java/org/apache/cassandra/service/accord/RequestBookkeeping.java new file mode 100644 index 0000000000..a4876e15a5 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/RequestBookkeeping.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord; + +import java.util.Collections; +import javax.annotation.Nullable; + +import accord.primitives.Seekables; +import accord.primitives.TxnId; +import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.exceptions.ReadFailureException; +import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.exceptions.RequestFailureException; +import org.apache.cassandra.exceptions.RequestTimeoutException; +import org.apache.cassandra.exceptions.WriteFailureException; +import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.service.accord.exceptions.AccordReadExhaustedException; +import org.apache.cassandra.service.accord.exceptions.AccordReadPreemptedException; +import org.apache.cassandra.service.accord.exceptions.AccordWriteExhaustedException; +import org.apache.cassandra.service.accord.exceptions.AccordWritePreemptedException; + +import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; +import static org.apache.cassandra.service.accord.RequestBookkeeping.ThrowsExceptionType.WRITE; + +public abstract class RequestBookkeeping +{ + public enum ThrowsExceptionType { READ, WRITE } + + final ThrowsExceptionType exceptionType; + + protected RequestBookkeeping(ThrowsExceptionType exceptionType) + { + this.exceptionType = exceptionType; + } + + abstract void markFailure(); + abstract void markTimeout(); + abstract void markPreempted(); + abstract void markRetryDifferentSystem(); + abstract void markTopologyMismatch(); + abstract void markElapsedNanos(long nanos); + + public RequestTimeoutException newTimeout(@Nullable TxnId txnId, Seekables keysOrRanges) + { + markTimeout(); + return newTimeout(txnId, exceptionType, keysOrRanges); + } + + public RequestTimeoutException newPreempted(@Nullable TxnId txnId, Seekables keysOrRanges) + { + markPreempted(); + return newPreempted(txnId, exceptionType, keysOrRanges); + } + + public RequestTimeoutException newExhausted(@Nullable TxnId txnId, Seekables keysOrRanges) + { + markFailure(); + return newExhausted(txnId, exceptionType, keysOrRanges); + } + + public RequestFailureException newFailed(@Nullable TxnId txnId, Seekables keysOrRanges) + { + markFailure(); + return newFailed(txnId, exceptionType, keysOrRanges); + } + + private static RequestTimeoutException newTimeout(TxnId txnId, ThrowsExceptionType type, Seekables keysOrRanges) + { + // Client protocol doesn't handle null consistency level so use ANY + return type == WRITE ? new WriteTimeoutException(WriteType.CAS, SERIAL, 0, 0, describe(txnId, keysOrRanges)) + : new ReadTimeoutException(SERIAL, 0, 0, false, describe(txnId, keysOrRanges)); + } + + private static RequestTimeoutException newPreempted(TxnId txnId, ThrowsExceptionType type, Seekables keysOrRanges) + { + return type == WRITE ? new AccordWritePreemptedException(0, 0, describe(txnId, keysOrRanges)) + : new AccordReadPreemptedException(0, 0, false, describe(txnId, keysOrRanges)); + } + + private static RequestTimeoutException newExhausted(TxnId txnId, ThrowsExceptionType type, Seekables keysOrRanges) + { + return type == WRITE ? new AccordWriteExhaustedException(0, 0, describe(txnId, keysOrRanges)) + : new AccordReadExhaustedException(0, 0, false, describe(txnId, keysOrRanges)); + } + + private static RequestFailureException newFailed(TxnId txnId, ThrowsExceptionType type, Seekables keysOrRanges) + { + // TODO (required): plumb in per-peer failure responses from Accord, and describe the txnId/keys + return type == WRITE ? new WriteFailureException(SERIAL, 0, 0, WriteType.CAS, Collections.emptyMap()) + : new ReadFailureException(SERIAL, 0, 0, false, Collections.emptyMap()); + } + + private static String describe(TxnId txnId, Seekables keysOrRanges) + { + return txnId + ": " + keysOrRanges; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/TimeOnlyRequestBookkeeping.java b/src/java/org/apache/cassandra/service/accord/TimeOnlyRequestBookkeeping.java new file mode 100644 index 0000000000..1a6e258703 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/TimeOnlyRequestBookkeeping.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord; + +import org.apache.cassandra.metrics.LatencyMetrics; +import static org.apache.cassandra.service.accord.RequestBookkeeping.ThrowsExceptionType.READ; + +public abstract class TimeOnlyRequestBookkeeping extends RequestBookkeeping +{ + public static class LatencyRequestBookkeeping extends TimeOnlyRequestBookkeeping + { + final LatencyMetrics latency; + + public LatencyRequestBookkeeping(LatencyMetrics latency) + { + this.latency = latency; + } + + public final void markElapsedNanos(long nanos) + { + if (latency != null) + latency.addNano(nanos); + } + } + + TimeOnlyRequestBookkeeping() + { + super(READ); + } + + @Override + final void markTimeout() + { + } + + @Override + final void markPreempted() + { + } + + final void markFailure() + { + } + + @Override + final void markRetryDifferentSystem() + { + throw new UnsupportedOperationException(); + } + + @Override + void markTopologyMismatch() + { + throw new UnsupportedOperationException(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index b640f47c37..7c2cdf972c 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -19,6 +19,7 @@ package org.apache.cassandra.service.accord.api; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import javax.annotation.Nullable; @@ -27,7 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.Agent; -import accord.api.EventsListener; +import accord.api.EventListener; import accord.api.ProgressLog.BlockedUntil; import accord.api.Result; import accord.api.RoutingKey; @@ -35,29 +36,31 @@ import accord.local.Command; import accord.local.Node; import accord.local.SafeCommand; import accord.local.SafeCommandStore; +import accord.local.TimeService; import accord.messages.ReplyContext; import accord.primitives.Keys; import accord.primitives.Participants; import accord.primitives.Ranges; import accord.primitives.Routable; -import accord.primitives.Seekables; import accord.primitives.Status; import accord.primitives.Timestamp; import accord.primitives.Txn; import accord.primitives.Txn.Kind; import accord.primitives.TxnId; import accord.topology.Shard; +import accord.topology.Topologies; import accord.utils.DefaultRandom; import accord.utils.Invariants; import accord.utils.RandomSource; import accord.utils.SortedList; import accord.utils.UnhandledEnum; -import org.apache.cassandra.config.AccordSpec; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import accord.utils.async.AsyncResult; +import accord.utils.async.AsyncResults; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.metrics.AccordMetrics; -import org.apache.cassandra.metrics.ClientRequestsMetricsHolder; import org.apache.cassandra.net.ResponseContext; -import org.apache.cassandra.service.TimeoutStrategy; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; @@ -65,24 +68,38 @@ import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.JVMStabilityInspector; import static accord.primitives.Routable.Domain.Key; -import static accord.primitives.Txn.Kind.Write; +import static accord.utils.SortedArrays.SortedArrayList.ofSorted; import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.config.DatabaseDescriptor.getAccordScheduleDurabilityTxnIdLag; import static org.apache.cassandra.config.DatabaseDescriptor.getReadRpcTimeout; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.expireEpochWait; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.fetch; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.recover; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryBootstrap; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryDurability; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retrySyncPoint; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowTxnPreaccept; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowRead; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; // TODO (expected): merge with AccordService public class AccordAgent implements Agent { private static final Logger logger = LoggerFactory.getLogger(AccordAgent.class); - // TODO (required): this should be configurable and have exponential back-off, escaping to operator input past a certain number of retries - private long retryBootstrapDelayMicros = SECONDS.toMicros(1L); - private final RandomSource random = new DefaultRandom(); + private static BiConsumer onFailedBarrier; + public static void setOnFailedBarrier(BiConsumer newOnFailedBarrier) { onFailedBarrier = newOnFailedBarrier; } + public static void onFailedBarrier(TxnId txnId, Throwable cause) + { + BiConsumer invoke = onFailedBarrier; + if (invoke != null) invoke.accept(txnId, cause); + } - // TODO (required): make hot property - private TimeoutStrategy slowPreaccept, slowRead; + + private final RandomSource random = new DefaultRandom(); protected Node.Id self; public AccordAgent() @@ -94,11 +111,6 @@ public class AccordAgent implements Agent self = id; } - public void setRetryBootstrapDelay(long delay, TimeUnit units) - { - retryBootstrapDelayMicros = units.toMicros(delay); - } - @Override public void onRecover(Node node, Result success, Throwable fail) { @@ -114,22 +126,17 @@ public class AccordAgent implements Agent throw error; } - public void onFailedBarrier(TxnId id, Seekables keysOrRanges, Throwable cause) - { - - } - @Override - public void onFailedBootstrap(String phase, Ranges ranges, Runnable retry, Throwable failure) + public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Throwable failure) { logger.error("Failed bootstrap at {} for {}", phase, ranges, failure); - AccordService.instance().scheduler().once(retry, retryBootstrapDelayMicros, MICROSECONDS); + AccordService.instance().scheduler().once(retry, retryBootstrap.computeWait(attempts, MICROSECONDS), MICROSECONDS); } @Override public void onStale(Timestamp staleSince, Ranges ranges) { - // TODO (required): decide how to handle this - maybe do nothing besides log? Maybe configurably try some number of repair attempts to catch up. + logger.error("This replica has become stale for {} as of {}", ranges, staleSince); } @Override @@ -147,10 +154,18 @@ public class AccordAgent implements Agent } @Override - public long preAcceptTimeout() + public Topologies selectPreferred(Node.Id from, Topologies to) { - // TODO: should distinguish between reads and writes (Aleksey: why? and why read rpc timeout is being used?) - return getReadRpcTimeout(MICROSECONDS); + SortedList nodes = to.nodes(); + int i = nodes.indexOf(from); + Node.Id node = i <= 0 ? nodes.get(nodes.size() - 1) : to.nodes().get(i - 1); + return to.select(ofSorted(node)); + } + + @Override + public boolean rejectPreAccept(TimeService time, TxnId txnId) + { + return time.now() - getReadRpcTimeout(MICROSECONDS) > txnId.hlc(); } // TODO (expected): we probably want additional configuration here so we can prune on shorter time horizons when we have a lot of transactions on a single key @@ -190,13 +205,13 @@ public class AccordAgent implements Agent } @Override - public EventsListener metricsEventsListener() + public EventListener eventListener() { return AccordMetrics.Listener.instance; } @Override - public long attemptCoordinationDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int retryCount) + public long slowCoordinatorDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int retryCount) { SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId); Invariants.nonNull(safeCommand); @@ -210,8 +225,7 @@ public class AccordAgent implements Agent // TODO (expected): make this a configurable calculation on normal request latencies (like ContentionStrategy) long oneSecond = SECONDS.toMicros(1L); - long startTime = mostRecentAttempt.hlc() + DatabaseDescriptor.getAccord().recoveryDelayFor(txnId, MICROSECONDS) - + (retryCount == 0 ? 0 : random.nextLong(oneSecond << Math.min(retryCount, 4))); + long startTime = mostRecentAttempt.hlc() + recover(txnId).computeWait(retryCount, MICROSECONDS); startTime = nonClashingStartTime(startTime, shard == null ? null : shard.nodes, node.id(), oneSecond, random); long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()); @@ -244,36 +258,34 @@ public class AccordAgent implements Agent } @Override - public long seekProgressDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int retryCount, BlockedUntil blockedUntil, TimeUnit units) + public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil blockedUntil, TimeUnit units) { - // TODO (required): make this configurable and dependent upon normal request latencies, and perhaps offset from txnId.hlc() - return units.convert((1L << Math.min(retryCount, 4)), SECONDS); + return fetch(txnId).computeWait(attempt, units); } @Override - public long retryAwaitTimeout(Node node, SafeCommandStore safeStore, TxnId txnId, int retryCount, BlockedUntil retrying, TimeUnit units) + public long slowAwaitDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil retrying, TimeUnit units) { - // TODO (expected): integrate with contention backoff - return units.convert((1L << Math.min(retryCount, 4)), SECONDS); + // TODO (desired): separate config? + return fetch(txnId).computeWait(attempt, units); } @Override - public long localSlowAt(TxnId txnId, Status.Phase phase, TimeUnit unit) + public long retrySyncPointDelay(Node node, int attempt, TimeUnit units) { - switch (phase) - { - default: throw new UnhandledEnum(phase); - case PreAccept: return unit.convert(slowPreaccept().computeWaitUntil(1), NANOSECONDS); - case Execute: return unit.convert(slowRead().computeWaitUntil(1), NANOSECONDS); - } + return retrySyncPoint.computeWait(attempt, units); } @Override - public long localExpiresAt(TxnId txnId, Status.Phase phase, TimeUnit unit) + public long retryDurabilityDelay(Node node, int attempt, TimeUnit units) { - // TODO (expected): make this configurable - return txnId.is(Write) ? DatabaseDescriptor.getWriteRpcTimeout(unit) - : DatabaseDescriptor.getReadRpcTimeout(unit); + return retryDurability.computeWait(attempt, units); + } + + @Override + public long expireEpochWait(TimeUnit units) + { + return expireEpochWait.computeWait(1, units); } @Override @@ -282,35 +294,59 @@ public class AccordAgent implements Agent return unit.convert(((ResponseContext)replyContext).expiresAtNanos(), NANOSECONDS); } + @Override + public long selfSlowAt(TxnId txnId, Status.Phase phase, TimeUnit unit) + { + switch (phase) + { + default: throw new UnhandledEnum(phase); + case PreAccept: return unit.convert(slowTxnPreaccept.computeWaitUntil(1), NANOSECONDS); + case Execute: return unit.convert(slowRead.computeWaitUntil(1), NANOSECONDS); + } + } + + @Override + public long selfExpiresAt(TxnId txnId, Status.Phase phase, TimeUnit unit) + { + long delayNanos; + switch (txnId.kind()) + { + default: throw new UnhandledEnum(txnId.kind()); + case Write: + delayNanos = DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS); + break; + case EphemeralRead: + case Read: + delayNanos = DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS); + break; + case ExclusiveSyncPoint: + delayNanos = DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(); + } + return unit.convert(nanoTime() + delayNanos, NANOSECONDS); + } + + @Override + public AsyncChain awaitStaleId(Node node, TxnId staleId, boolean isRequested) + { + long waitMicros = (staleId.hlc() + getAccordScheduleDurabilityTxnIdLag(MICROSECONDS)) - node.now(); + if (waitMicros <= 0) + return AsyncChains.success(staleId); + + logger.debug("Waiting {} micros for {} to be stale", waitMicros, staleId); + AsyncResult.Settable result = AsyncResults.settable(); + node.scheduler().selfRecurring(() -> result.setSuccess(staleId), waitMicros, MICROSECONDS); + return result; + } + + @Override + public long minStaleHlc(Node node, boolean requested) + { + return node.now() - (100 + getAccordScheduleDurabilityTxnIdLag(MICROSECONDS)); + } + @Override public void onViolation(String message, Participants participants, @Nullable TxnId notWitnessed, @Nullable Timestamp notWitnessedExecuteAt, @Nullable TxnId by, @Nullable Timestamp byEexecuteAt) { logger.error(message); } - - public TimeoutStrategy slowRead() - { - if (slowRead == null) - { - synchronized (this) - { - AccordSpec config = DatabaseDescriptor.getAccord(); - slowRead = new TimeoutStrategy(config.slowRead, TimeoutStrategy.LatencySourceFactory.of(ClientRequestsMetricsHolder.accordReadMetrics)); - } - } - return slowRead; - } - - public TimeoutStrategy slowPreaccept() - { - if (slowPreaccept == null) - { - synchronized (this) - { - AccordSpec config = DatabaseDescriptor.getAccord(); - slowPreaccept = new TimeoutStrategy(config.slowPreAccept, TimeoutStrategy.LatencySourceFactory.of(ClientRequestsMetricsHolder.accordReadMetrics)); - } - } - return slowPreaccept; - } } diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java new file mode 100644 index 0000000000..b5d990c091 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.api; + +import javax.annotation.Nullable; + +import accord.primitives.TxnId; +import org.apache.cassandra.config.AccordSpec; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.service.RetryStrategy; +import org.apache.cassandra.service.TimeoutStrategy; + +import static accord.primitives.Txn.Kind.ExclusiveSyncPoint; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics; +import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory.none; +import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory.of; +import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory.rw; + +public class AccordWaitStrategies +{ + static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead; + static TimeoutStrategy expireTxn, expireSyncPoint, expireDurability, expireEpochWait; + static TimeoutStrategy fetchTxn, fetchSyncPoint; + static RetryStrategy recoverTxn, recoverSyncPoint, retrySyncPoint, retryDurability, retryBootstrap; + static RetryStrategy retryFetchMinEpoch, retryFetchTopology; + + public static @Nullable TimeoutStrategy slowRead(@Nullable TxnId txnId) + { + if (txnId != null && txnId.isSyncPoint()) + return null; + return slowRead; + } + + public static TimeoutStrategy expire(@Nullable TxnId txnId, Verb verb) + { + if (txnId == null || !txnId.isSyncPoint()) + return expireTxn; + return verb == Verb.ACCORD_WAIT_UNTIL_APPLIED_REQ ? expireDurability : expireSyncPoint; + } + + public static TimeoutStrategy fetch(@Nullable TxnId txnId) + { + if (txnId == null || !txnId.isSyncPoint()) + return fetchTxn; + return fetchSyncPoint; + } + + public static TimeoutStrategy slowPreaccept(TxnId txnId) + { + if (txnId.isSyncPoint()) + return expireSyncPoint; + return slowTxnPreaccept; + } + + public static RetryStrategy retryFetchMinEpoch() + { + return retryFetchMinEpoch; + } + + public static RetryStrategy retryFetchTopology() + { + return retryFetchTopology; + } + + static + { + AccordSpec config = DatabaseDescriptor.getAccord(); + setSlowRead(config.slow_read); + setSlowTxnPreaccept(config.slow_txn_preaccept); + setSlowSyncPointPreaccept(config.slow_syncpoint_preaccept); + setExpireTxn(config.expire_txn); + setExpireSyncPoint(config.expire_syncpoint); + setExpireDurability(config.expire_durability); + setExpireEpochWait(config.expire_epoch_wait); + setFetchTxn(config.fetch_txn); + setFetchSyncPoint(config.fetch_syncpoint); + setRecoverTxn(config.recover_txn); + setRecoverSyncPoint(config.recover_syncpoint); + setRetrySyncPoint(config.retry_syncpoint); + setRetryDurability(config.retry_durability); + setRetryFetchMinEpoch(config.retry_fetch_min_epoch); + setRetryFetchTopology(config.retry_fetch_topology); + } + + public static void setSlowRead(String spec) + { + slowRead = TimeoutStrategy.parse(spec, of(accordReadMetrics)); + } + + public static void setSlowTxnPreaccept(String spec) + { + slowTxnPreaccept = TimeoutStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics)); + } + + public static void setSlowSyncPointPreaccept(String spec) + { + slowSyncPointPreaccept = TimeoutStrategy.parse(spec, none()); + } + + public static void setExpireTxn(String spec) + { + expireTxn = TimeoutStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics)); + } + + public static void setExpireSyncPoint(String spec) + { + expireSyncPoint = TimeoutStrategy.parse(spec, none()); + } + + public static void setExpireDurability(String spec) + { + expireDurability = TimeoutStrategy.parse(spec, none()); + } + + public static void setExpireEpochWait(String spec) + { + expireEpochWait = TimeoutStrategy.parse(spec, none()); + } + + public static void setFetchTxn(String spec) + { + fetchTxn = TimeoutStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics)); + } + + public static void setFetchSyncPoint(String spec) + { + fetchSyncPoint = TimeoutStrategy.parse(spec, none()); + } + + public static void setRecoverTxn(String spec) + { + recoverTxn = RetryStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics)); + } + + public static void setRecoverSyncPoint(String spec) + { + recoverSyncPoint = RetryStrategy.parse(spec, none()); + } + + public static void setRetrySyncPoint(String spec) + { + retrySyncPoint = RetryStrategy.parse(spec, none()); + } + + public static void setRetryDurability(String spec) + { + retryDurability = RetryStrategy.parse(spec, none()); + } + + public static void setRetryFetchMinEpoch(String spec) + { + retryFetchMinEpoch = RetryStrategy.parse(spec, none()); + } + + public static void setRetryFetchTopology(String spec) + { + retryFetchTopology = RetryStrategy.parse(spec, none()); + } + + static RetryStrategy recover(TxnId txnId) + { + if (txnId.is(ExclusiveSyncPoint)) + return recoverSyncPoint; + return recoverTxn; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/api/PartitionKey.java b/src/java/org/apache/cassandra/service/accord/api/PartitionKey.java index 84ba0f40fc..f42166f890 100644 --- a/src/java/org/apache/cassandra/service/accord/api/PartitionKey.java +++ b/src/java/org/apache/cassandra/service/accord/api/PartitionKey.java @@ -25,7 +25,7 @@ import com.google.common.base.Preconditions; import accord.api.Key; import accord.primitives.Routable; -import org.apache.cassandra.config.DatabaseDescriptor; +import accord.utils.Invariants; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.TypeSizes; @@ -33,16 +33,15 @@ import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.db.marshal.ValueAccessor; import org.apache.cassandra.db.partitions.Partition; import org.apache.cassandra.db.partitions.PartitionUpdate; -import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.ObjectSizes; +import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; + // final in part because we refer to its class directly in AccordRoutableKey.compareTo public final class PartitionKey extends AccordRoutableKey implements Key { @@ -50,7 +49,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key static { - DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER); + DecoratedKey key = getPartitioner().decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER); EMPTY_SIZE = ObjectSizes.measureDeep(new PartitionKey(null, key)); } @@ -140,6 +139,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key int position = offset; position += key.table().serializeCompact(dst, accessor, position); ByteBuffer bytes = key.partitionKey().getKey(); + Invariants.require(key.partitionKey().getPartitioner() == getPartitioner()); int numBytes = ByteBufferAccessor.instance.size(bytes); Preconditions.checkState(numBytes <= Short.MAX_VALUE); position += accessor.putShort(dst, position, (short) numBytes); @@ -159,8 +159,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key public PartitionKey deserialize(DataInputPlus in, int version) throws IOException { TableId tableId = TableId.deserializeCompact(in).intern(); - IPartitioner partitioner = Schema.instance.getExistingTablePartitioner(tableId); - DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in)); + DecoratedKey key = getPartitioner().decorateKey(ByteBufferUtil.readWithShortLength(in)); return new PartitionKey(tableId, key); } @@ -168,12 +167,11 @@ public final class PartitionKey extends AccordRoutableKey implements Key { TableId tableId = TableId.deserializeCompact(src, accessor, offset).intern(); offset += tableId.serializedCompactSize(); - TableMetadata metadata = Schema.instance.getTableMetadata(tableId); int numBytes = accessor.getShort(src, offset); offset += TypeSizes.SHORT_SIZE; ByteBuffer bytes = ByteBuffer.allocate(numBytes); accessor.copyTo(src, offset, bytes, ByteBufferAccessor.instance, 0, numBytes); - DecoratedKey key = metadata.partitioner.decorateKey(bytes); + DecoratedKey key = getPartitioner().decorateKey(bytes); return new PartitionKey(tableId, key); } diff --git a/src/java/org/apache/cassandra/service/accord/api/TokenKey.java b/src/java/org/apache/cassandra/service/accord/api/TokenKey.java index d29c306869..9ff6959ae7 100644 --- a/src/java/org/apache/cassandra/service/accord/api/TokenKey.java +++ b/src/java/org/apache/cassandra/service/accord/api/TokenKey.java @@ -296,6 +296,7 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran return result; } + // WARNING: consumes buffer! public TokenKey deserialize(ByteBuffer buffer) { return deserialize(buffer, getPartitioner()); @@ -303,8 +304,16 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran public TokenKey deserialize(ByteBuffer buffer, IPartitioner partitioner) { + TableId tableId = TableId.deserializeCompactComparable(buffer, ByteBufferAccessor.instance, 0); + int offset = tableId.serializedCompactComparableSize(); + return deserializeWithPrefix(tableId, buffer.remaining() - offset, buffer, ByteBufferAccessor.instance, offset, partitioner); + } + + // WARNING: consumes buffer! + public TokenKey deserializeAndConsume(ByteBuffer buffer, IPartitioner partitioner) + { + TableId tableId = TableId.deserializeCompactComparable(buffer, ByteBufferAccessor.instance, 0); int offset = buffer.position(); - TableId tableId = TableId.deserializeCompactComparable(buffer, ByteBufferAccessor.instance, offset); buffer.position(offset + tableId.serializedCompactComparableSize()); return deserializeWithPrefix(tableId, buffer.remaining(), buffer, partitioner); } @@ -403,21 +412,13 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran return new TokenKey((TableId) tableId, sentinel, token); } - public TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, ByteBuffer buffer) - { - return deserializeWithPrefixAndImpliedLength(tableId, buffer, getPartitioner()); - } - - public TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, ByteBuffer buffer, IPartitioner partitioner) + // WARNING: consumes buffer! + public TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, ByteBuffer buffer, IPartitioner partitioner) { return deserializeWithPrefix(tableId, buffer.remaining(), buffer, partitioner); } - public TokenKey deserializeWithPrefix(Object tableId, int length, ByteBuffer buffer) - { - return deserializeWithPrefix(tableId, length, buffer, getPartitioner()); - } - + // WARNING: consumes buffer! public TokenKey deserializeWithPrefix(Object tableId, int length, ByteBuffer buffer, IPartitioner partitioner) { byte sentinel = buffer.get(); @@ -590,6 +591,12 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran return subSplitter.splitRange(range, from, to, numSplits); } + @Override + public Ranges selectFirstSubRanges(Range range, Ranges subRanges, int totalSplits) + { + return subSplitter.selectFirstSubRanges(range, subRanges, totalSplits); + } + @Override public int numberOfSplitsPossible(Range range) { diff --git a/src/java/org/apache/cassandra/service/accord/exceptions/ReadExhaustedException.java b/src/java/org/apache/cassandra/service/accord/exceptions/AccordReadExhaustedException.java similarity index 63% rename from src/java/org/apache/cassandra/service/accord/exceptions/ReadExhaustedException.java rename to src/java/org/apache/cassandra/service/accord/exceptions/AccordReadExhaustedException.java index c9fc1bd14b..697b31d6e3 100644 --- a/src/java/org/apache/cassandra/service/accord/exceptions/ReadExhaustedException.java +++ b/src/java/org/apache/cassandra/service/accord/exceptions/AccordReadExhaustedException.java @@ -18,15 +18,19 @@ package org.apache.cassandra.service.accord.exceptions; -import com.google.common.collect.ImmutableMap; +import org.apache.cassandra.exceptions.ReadTimeoutException; -import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.exceptions.ReadFailureException; +import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; -public class ReadExhaustedException extends ReadFailureException +public class AccordReadExhaustedException extends ReadTimeoutException { - public ReadExhaustedException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent, String msg) + public AccordReadExhaustedException(int received, int blockFor, boolean dataPresent) { - super(msg, consistency, received, blockFor, dataPresent, ImmutableMap.of()); + super(SERIAL, received, blockFor, dataPresent); + } + + public AccordReadExhaustedException(int received, int blockFor, boolean dataPresent, String msg) + { + super(SERIAL, received, blockFor, dataPresent, msg); } } diff --git a/src/java/org/apache/cassandra/service/accord/exceptions/ReadPreemptedException.java b/src/java/org/apache/cassandra/service/accord/exceptions/AccordReadPreemptedException.java similarity index 68% rename from src/java/org/apache/cassandra/service/accord/exceptions/ReadPreemptedException.java rename to src/java/org/apache/cassandra/service/accord/exceptions/AccordReadPreemptedException.java index c67256c157..53b37b7c89 100644 --- a/src/java/org/apache/cassandra/service/accord/exceptions/ReadPreemptedException.java +++ b/src/java/org/apache/cassandra/service/accord/exceptions/AccordReadPreemptedException.java @@ -18,19 +18,20 @@ package org.apache.cassandra.service.accord.exceptions; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.exceptions.ReadTimeoutException; +import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; + // shim to allow tests to tell the difference between preemption and other protocol timeouts -public class ReadPreemptedException extends ReadTimeoutException +public class AccordReadPreemptedException extends ReadTimeoutException { - public ReadPreemptedException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent) + public AccordReadPreemptedException(int received, int blockFor, boolean dataPresent) { - super(consistency, received, blockFor, dataPresent); + super(SERIAL, received, blockFor, dataPresent); } - public ReadPreemptedException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent, String msg) + public AccordReadPreemptedException(int received, int blockFor, boolean dataPresent, String msg) { - super(consistency, received, blockFor, dataPresent, msg); + super(SERIAL, received, blockFor, dataPresent, msg); } } diff --git a/src/java/org/apache/cassandra/service/accord/exceptions/AccordWriteExhaustedException.java b/src/java/org/apache/cassandra/service/accord/exceptions/AccordWriteExhaustedException.java new file mode 100644 index 0000000000..6aca227240 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/exceptions/AccordWriteExhaustedException.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.exceptions; + +import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.exceptions.WriteTimeoutException; + +import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; + +public class AccordWriteExhaustedException extends WriteTimeoutException +{ + public AccordWriteExhaustedException(int received, int blockFor) + { + super(WriteType.CAS, SERIAL, received, blockFor); + } + + public AccordWriteExhaustedException(int received, int blockFor, String msg) + { + super(WriteType.CAS, SERIAL, received, blockFor, msg); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/exceptions/WritePreemptedException.java b/src/java/org/apache/cassandra/service/accord/exceptions/AccordWritePreemptedException.java similarity index 69% rename from src/java/org/apache/cassandra/service/accord/exceptions/WritePreemptedException.java rename to src/java/org/apache/cassandra/service/accord/exceptions/AccordWritePreemptedException.java index f2f28ef67d..ee02e6dfad 100644 --- a/src/java/org/apache/cassandra/service/accord/exceptions/WritePreemptedException.java +++ b/src/java/org/apache/cassandra/service/accord/exceptions/AccordWritePreemptedException.java @@ -18,20 +18,21 @@ package org.apache.cassandra.service.accord.exceptions; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.WriteType; import org.apache.cassandra.exceptions.WriteTimeoutException; +import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; + // quick hack to allow tests to tell the difference between preemption and other protocol timeouts -public class WritePreemptedException extends WriteTimeoutException +public class AccordWritePreemptedException extends WriteTimeoutException { - public WritePreemptedException(WriteType writeType, ConsistencyLevel consistency, int received, int blockFor) + public AccordWritePreemptedException(int received, int blockFor) { - super(writeType, consistency, received, blockFor); + super(WriteType.CAS, SERIAL, received, blockFor); } - public WritePreemptedException(WriteType writeType, ConsistencyLevel consistency, int received, int blockFor, String msg) + public AccordWritePreemptedException(int received, int blockFor, String msg) { - super(writeType, consistency, received, blockFor, msg); + super(WriteType.CAS, SERIAL, received, blockFor, msg); } } diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java index 23be986736..91acc361d1 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java @@ -53,7 +53,7 @@ import static accord.messages.Apply.Kind.Minimal; public class AccordInteropAdapter extends TxnAdapter { private static final Logger logger = LoggerFactory.getLogger(AccordInteropAdapter.class); - public static final class AccordInteropFactory implements CoordinationAdapter.Factory + public static final class AccordInteropFactory extends DefaultFactory { final AccordInteropAdapter standard, recovery; @@ -67,6 +67,8 @@ public class AccordInteropAdapter extends TxnAdapter @Override public CoordinationAdapter get(TxnId txnId, Kind step) { + if (txnId.isSyncPoint()) + return super.get(txnId, step); return (CoordinationAdapter) (step == Kind.Recovery ? recovery : standard); } }; diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java index 241c8acd01..fe3a1c74b3 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java @@ -221,12 +221,7 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL @Override public MessageType type() { - switch (kind) - { - case Minimal: return AccordMessageType.INTEROP_APPLY_MINIMAL_REQ; - case Maximal: return AccordMessageType.INTEROP_APPLY_MAXIMAL_REQ; - default: throw new IllegalStateException(); - } + return AccordMessageType.INTEROP_APPLY_REQ; } @Override diff --git a/src/java/org/apache/cassandra/service/accord/repair/AccordRepair.java b/src/java/org/apache/cassandra/service/accord/repair/AccordRepair.java index 02780219d2..d3a68eb20a 100644 --- a/src/java/org/apache/cassandra/service/accord/repair/AccordRepair.java +++ b/src/java/org/apache/cassandra/service/accord/repair/AccordRepair.java @@ -18,37 +18,43 @@ package org.apache.cassandra.service.accord.repair; -import java.math.BigInteger; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.Executor; +import java.util.stream.Collectors; import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import accord.api.BarrierType; -import accord.api.RoutingKey; +import accord.local.durability.DurabilityService; +import accord.local.Node; import accord.primitives.Ranges; -import accord.primitives.Seekables; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.dht.AccordSplitter; -import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.metrics.LatencyMetrics; import org.apache.cassandra.repair.SharedContext; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTopology; +import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.service.accord.RequestBookkeeping; +import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping; import org.apache.cassandra.service.accord.TokenRange; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Future; -import static com.google.common.base.Preconditions.checkState; -import static org.apache.cassandra.config.CassandraRelevantProperties.ACCORD_REPAIR_RANGE_STEP_UPDATE_INTERVAL; +import static accord.local.durability.DurabilityService.SyncLocal.NoLocal; +import static accord.local.durability.DurabilityService.SyncRemote.All; +import static accord.local.durability.DurabilityService.SyncRemote.Quorum; +import static accord.primitives.Timestamp.mergeMax; +import static accord.primitives.Timestamp.minForEpoch; +import static org.apache.cassandra.config.DatabaseDescriptor.getAccordRepairTimeoutNanos; /* * Accord repair consists of creating a barrier transaction for all the ranges which ensure that all Accord transactions @@ -56,33 +62,28 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.ACCORD_REP */ public class AccordRepair { - private static final Logger logger = LoggerFactory.getLogger(AccordRepair.class); - - public static final BigInteger TWO = BigInteger.valueOf(2); - private final SharedContext ctx; private final ColumnFamilyStore cfs; + private final TimeUUID repairId; private final Ranges ranges; - private final AccordSplitter splitter; private final boolean requireAllEndpoints; private final List endpoints; - private BigInteger rangeStep; - private final Epoch minEpoch = ClusterMetadata.current().epoch; private volatile Throwable shouldAbort = null; + private volatile Thread waiting; - public AccordRepair(SharedContext ctx, ColumnFamilyStore cfs, IPartitioner partitioner, String keyspace, Collection> ranges, boolean requireAllEndpoints, List endpoints) + public AccordRepair(SharedContext ctx, ColumnFamilyStore cfs, TimeUUID repairId, String keyspace, Collection> ranges, boolean requireAllEndpoints, List endpoints) { this.ctx = ctx; this.cfs = cfs; + this.repairId = repairId; this.requireAllEndpoints = requireAllEndpoints; this.endpoints = endpoints; this.ranges = AccordTopology.toAccordRanges(keyspace, ranges); - this.splitter = partitioner.accordSplitter().apply(this.ranges); } public Epoch minEpoch() @@ -117,100 +118,65 @@ public class AccordRepair protected void abort(@Nullable Throwable reason) { shouldAbort = reason == null ? new RuntimeException("Abort") : reason; + Thread thread = waiting; + if (thread != null) + thread.interrupt(); } private List repairRange(TokenRange range) throws Throwable { List repairedRanges = new ArrayList<>(); - int rangeStepUpdateInterval = ACCORD_REPAIR_RANGE_STEP_UPDATE_INTERVAL.getInt(); - RoutingKey remainingStart = range.start(); - // TODO (expected): repair ranges should have a configurable lower limit of split size so already small repairs aren't broken up into excessively tiny ones - // TODO (expected): we should support lower range divisions for accord repair - BigInteger rangeSize = splitter.sizeOf(range); - if (rangeStep == null) + List ids = endpoints == null ? null : endpoints.stream().map(AccordService.instance().configService()::mappedId).collect(Collectors.toList()); + DurabilityService.SyncRemote syncRemote = requireAllEndpoints ? All : Quorum; + + if (shouldAbort != null) + throw shouldAbort; + + LatencyMetrics latency = null; { - BigInteger divide = splitter.divide(rangeSize, 10000); - rangeStep = divide.equals(BigInteger.ZERO) ? rangeSize : BigInteger.ONE.max(divide); + TableMetadata metadata = Schema.instance.getTableMetadata(range.table()); + if (metadata != null) + { + ColumnFamilyStore cfs = Keyspace.openAndGetStore(metadata); + if (cfs != null) + latency = cfs.metric.accordRepair; + } } - - BigInteger offset = BigInteger.ZERO; - - TokenRange lastRepaired = null; - int iteration = 0; - while (true) + long start = ctx.clock().nanoTime(); + try { + IAccordService service = AccordService.instance(); + Ranges ranges = AccordService.intersecting(Ranges.of(range)); + waiting = Thread.currentThread(); + RequestBookkeeping bookkeeping = new LatencyRequestBookkeeping(latency); + AccordService.getBlocking(service.maxConflict(ranges).flatMap(conflict -> { + conflict = mergeMax(conflict, minForEpoch(this.minEpoch.getEpoch())); + return service.sync("[repairId #" + repairId + ']', conflict, Ranges.of(range), ids, NoLocal, syncRemote); + }), ranges, bookkeeping, start, start + getAccordRepairTimeoutNanos()); + waiting = null; + if (shouldAbort != null) throw shouldAbort; - iteration++; - if (iteration % rangeStepUpdateInterval == 0) - rangeStep = rangeStep.multiply(TWO); - BigInteger remaining = rangeSize.subtract(offset); - BigInteger length = remaining.min(rangeStep); - - long start = ctx.clock().nanoTime(); - boolean dependencyOverflow = false; - try - { - // Splitter is approximate so it can't work right up to the end - TokenRange toRepair; - if (splitter.compare(offset, rangeSize) >= 0) - { - if (remainingStart.equals(range.end())) - { - logger.info("Completed barriers for {} in {} iterations", range, iteration - 1); - return repairedRanges; - } - - // Final repair is whatever remains - toRepair = range.newRange(remainingStart, range.end()); - } - else - { - toRepair = splitter.subRange(range, offset, splitter.add(offset, length)); - checkState(iteration > 1 || toRepair.start().equals(range.start())); - } - checkState(!toRepair.equals(lastRepaired), "Shouldn't repair the same range twice"); - checkState(lastRepaired == null || toRepair.start().equals(lastRepaired.end()), "Next range should directly follow previous range"); - lastRepaired = toRepair; - - Ranges barrieredRanges; - if (requireAllEndpoints) - barrieredRanges = (Ranges)AccordService.instance().repairWithRetries(Seekables.of(toRepair), minEpoch.getEpoch(), BarrierType.global_sync, false, endpoints); - else - barrieredRanges = (Ranges)AccordService.instance().barrierWithRetries(Seekables.of(toRepair), minEpoch.getEpoch(), BarrierType.global_sync, false); - for (accord.primitives.Range barrieredRange : barrieredRanges) - repairedRanges.add(barrieredRange); - - remainingStart = toRepair.end(); - } - catch (RuntimeException e) - { - cfs.metric.accordRepairUnexpectedFailures.mark(); - throw e; - } - catch (Throwable t) - { - cfs.metric.accordRepairUnexpectedFailures.mark(); - throw new RuntimeException(t); - } - finally - { - long end = ctx.clock().nanoTime(); - cfs.metric.accordRepair.addNano(end - start); - } - - // TODO when dependency limits are added to Accord need to test repair overflow - if (dependencyOverflow) - { - offset = offset.subtract(rangeStep); - if (rangeStep.equals(BigInteger.ONE)) - throw new IllegalStateException("Unable to repair without overflowing with range step of 1"); - rangeStep = BigInteger.ONE.max(rangeStep.divide(TWO)); - continue; - } - - offset = offset.add(length); + for (accord.primitives.Range r : ranges) + repairedRanges.add(r); } + catch (Throwable t) + { + cfs.metric.accordRepairUnexpectedFailures.mark(); + if (shouldAbort != null) + { + shouldAbort.addSuppressed(t); + throw shouldAbort; + } + throw t; + } + finally + { + long end = ctx.clock().nanoTime(); + cfs.metric.accordRepair.addNano(end - start); + } + + return repairedRanges; } } diff --git a/src/java/org/apache/cassandra/service/accord/repair/RepairSyncPointAdapter.java b/src/java/org/apache/cassandra/service/accord/repair/RepairSyncPointAdapter.java deleted file mode 100644 index 9691cdbc43..0000000000 --- a/src/java/org/apache/cassandra/service/accord/repair/RepairSyncPointAdapter.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.service.accord.repair; - -import java.util.Collection; -import java.util.function.BiConsumer; - -import com.google.common.collect.ImmutableSet; - -import accord.api.Result; -import accord.coordinate.CoordinationAdapter; -import accord.coordinate.ExecuteFlag; -import accord.coordinate.ExecutePath; -import accord.coordinate.ExecuteSyncPoint; -import accord.local.Node; -import accord.primitives.Deps; -import accord.primitives.FullRoute; -import accord.primitives.SyncPoint; -import accord.primitives.Timestamp; -import accord.primitives.Txn; -import accord.primitives.TxnId; -import accord.primitives.Unseekable; -import accord.primitives.Writes; -import accord.topology.Topologies; - -/** - * Sync point adapter used for accord-only repairs. - * - * Repair has the requirement that all client writes begun before the repair will be fully replicated once repair - * has completed. In the case of accord, repairs that compare data on disk satisfy this requirement by running - * a sync point as part of streaming if differences are found. For accord-only repairs, the barrier used by normal - * repairs is not sufficient since it only requires a quorum of nodes to respond before completing. This sync point - * adapter requires responses from all of the supplied endpoints before completing. Note that shards only block on the - * intersection of the provided replicas and their own endpoints. - */ -public class RepairSyncPointAdapter extends CoordinationAdapter.Adapters.AsyncInclusiveSyncPointAdapter -{ - private final ImmutableSet requiredResponses; - - public RepairSyncPointAdapter(Collection requiredResponses) - { - this.requiredResponses = ImmutableSet.copyOf(requiredResponses); - } - - @Override - public void execute(Node node, Topologies all, FullRoute route, ExecutePath path, ExecuteFlag.ExecuteFlags executeFlags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer, Throwable> callback) - { - RequiredResponseTracker tracker = new RequiredResponseTracker(requiredResponses, all); - ExecuteSyncPoint.ExecuteInclusive execute = new ExecuteSyncPoint.ExecuteInclusive<>(node, new SyncPoint<>(txnId, executeAt, stableDeps, (FullRoute) route), tracker, executeAt); - execute.addCallback(callback); - execute.start(); - } - - @Override - public void persist(Node node, Topologies all, FullRoute route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer, Throwable> callback) - { - throw new UnsupportedOperationException(); - } - - public static CoordinationAdapter> create(Collection requiredResponses) - { - return new RepairSyncPointAdapter<>(requiredResponses); - } -} diff --git a/src/java/org/apache/cassandra/service/accord/serializers/DepsSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/DepsSerializers.java index df2cecfbc6..9e56320646 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/DepsSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/DepsSerializers.java @@ -72,7 +72,7 @@ public class DepsSerializers this.tokenRange = tokenRange; } - abstract D deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version) throws IOException; + abstract D deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, DataInputPlus in, int version) throws IOException; @Override public void serialize(D deps, DataOutputPlus out, int version) throws IOException @@ -144,16 +144,6 @@ public class DepsSerializers for (int i = 0; i < rangesToTxnIdsCount; i++) out.writeUnsignedVInt32(rangesToTxnIds(rangeDeps, i)); } - - { - RoutingKeys keys = deps.directKeyDeps.keys(); - boolean isSubset = isSubset(keys, deps.keyDeps.keys()); - out.writeBoolean(isSubset); - if (isSubset) serializeSubset(keys, deps.keyDeps.keys(), out); - else KeySerializers.routingKeys.serialize(keys, out, version); - - serializeKeyDepsWithoutKeys(deps.directKeyDeps, out, version); - } } private void serializeKeyDepsWithoutKeys(KeyDeps keyDeps, DataOutputPlus out, int version) throws IOException @@ -193,14 +183,7 @@ public class DepsSerializers rangeDeps = RangeDeps.SerializerSupport.create(ranges, txnIds, rangesToTxnIds); } - KeyDeps directKeyDeps; - { - boolean isSubset = in.readBoolean(); - RoutingKeys directKeys = isSubset ? (RoutingKeys) deserializeSubset(keys, in) : KeySerializers.routingKeys.deserialize(in, version); - directKeyDeps = deserializeKeyDeps(directKeys, in, version); - } - - return deserialize(keyDeps, rangeDeps, directKeyDeps, in, version); + return deserialize(keyDeps, rangeDeps, in, version); } private long serializedSizeWithoutKeys(D deps, int version) @@ -224,13 +207,6 @@ public class DepsSerializers for (int i = 0; i < rangesToTxnIdsCount; i++) size += sizeofUnsignedVInt(rangesToTxnIds(rangeDeps, i)); } - - { - boolean isSubset = isSubset(deps.directKeyDeps.keys(), deps.keyDeps.keys()); - size += 1; - size += isSubset ? serializedSubsetSize(deps.directKeyDeps.keys(), deps.keyDeps.keys()) : KeySerializers.routingKeys.serializedSize(deps.directKeyDeps.keys(), version); - size += serializedSizeOfKeyDepsWithoutKeys(deps.directKeyDeps, version); - } return size; } } @@ -250,19 +226,19 @@ public class DepsSerializers this.deps = new DepsSerializer<>(tokenRange) { @Override - Deps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version) + Deps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, DataInputPlus in, int version) { - return new Deps(keyDeps, rangeDeps, directKeyDeps); + return new Deps(keyDeps, rangeDeps); } }; this.nullableDeps = NullableSerializer.wrap(deps); this.partialDeps = new DepsSerializer<>(tokenRange) { @Override - PartialDeps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version) throws IOException + PartialDeps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, DataInputPlus in, int version) throws IOException { Participants covering = KeySerializers.participants.deserialize(in, version); - return new PartialDeps(covering, keyDeps, rangeDeps, directKeyDeps); + return new PartialDeps(covering, keyDeps, rangeDeps); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/serializers/QueryDurableBeforeSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/GetDurableBeforeSerializers.java similarity index 73% rename from src/java/org/apache/cassandra/service/accord/serializers/QueryDurableBeforeSerializers.java rename to src/java/org/apache/cassandra/service/accord/serializers/GetDurableBeforeSerializers.java index 833c161ed0..f79039ae81 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/QueryDurableBeforeSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/GetDurableBeforeSerializers.java @@ -19,33 +19,31 @@ package org.apache.cassandra.service.accord.serializers; import java.io.IOException; -import accord.messages.QueryDurableBefore; -import accord.messages.QueryDurableBefore.DurableBeforeReply; -import org.apache.cassandra.db.TypeSizes; +import accord.messages.GetDurableBefore; +import accord.messages.GetDurableBefore.DurableBeforeReply; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -public class QueryDurableBeforeSerializers +public class GetDurableBeforeSerializers { - public static final IVersionedSerializer request = new IVersionedSerializer() + public static final IVersionedSerializer request = new IVersionedSerializer() { @Override - public void serialize(QueryDurableBefore msg, DataOutputPlus out, int version) throws IOException + public void serialize(GetDurableBefore msg, DataOutputPlus out, int version) throws IOException { - out.writeLong(msg.waitForEpoch()); } @Override - public QueryDurableBefore deserialize(DataInputPlus in, int version) throws IOException + public GetDurableBefore deserialize(DataInputPlus in, int version) throws IOException { - return new QueryDurableBefore(in.readLong()); + return new GetDurableBefore(); } @Override - public long serializedSize(QueryDurableBefore msg, int version) + public long serializedSize(GetDurableBefore msg, int version) { - return TypeSizes.LONG_SIZE; + return 0; } }; diff --git a/src/java/org/apache/cassandra/service/accord/serializers/GetMaxConflictSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/GetMaxConflictSerializers.java new file mode 100644 index 0000000000..5be2af043a --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/serializers/GetMaxConflictSerializers.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.serializers; + +import java.io.IOException; + +import accord.messages.GetMaxConflict; +import accord.messages.GetMaxConflict.GetMaxConflictOk; +import accord.primitives.Route; +import accord.primitives.Timestamp; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +public class GetMaxConflictSerializers +{ + public static final IVersionedSerializer request = new IVersionedSerializer<>() + { + @Override + public void serialize(GetMaxConflict msg, DataOutputPlus out, int version) throws IOException + { + KeySerializers.route.serialize(msg.scope, out, version); + out.writeUnsignedVInt(msg.waitForEpoch); + out.writeUnsignedVInt(msg.minEpoch); + out.writeUnsignedVInt(msg.executionEpoch); + } + + @Override + public GetMaxConflict deserialize(DataInputPlus in, int version) throws IOException + { + Route scope = KeySerializers.route.deserialize(in, version); + long waitForEpoch = in.readUnsignedVInt(); + long minEpoch = in.readUnsignedVInt(); + long executionEpoch = in.readUnsignedVInt(); + return GetMaxConflict.SerializationSupport.create(scope, waitForEpoch, minEpoch, executionEpoch); + } + + @Override + public long serializedSize(GetMaxConflict msg, int version) + { + return KeySerializers.route.serializedSize(msg.scope(), version) + + TypeSizes.sizeofUnsignedVInt(msg.waitForEpoch) + + TypeSizes.sizeofUnsignedVInt(msg.minEpoch) + + TypeSizes.sizeofUnsignedVInt(msg.executionEpoch); + } + }; + + public static final IVersionedSerializer reply = new IVersionedSerializer<>() + { + @Override + public void serialize(GetMaxConflictOk reply, DataOutputPlus out, int version) throws IOException + { + CommandSerializers.timestamp.serialize(reply.maxConflict, out, version); + out.writeUnsignedVInt(reply.latestEpoch); + } + + @Override + public GetMaxConflictOk deserialize(DataInputPlus in, int version) throws IOException + { + Timestamp maxConflict = CommandSerializers.timestamp.deserialize(in, version); + long latestEpoch = in.readUnsignedVInt(); + return new GetMaxConflictOk(maxConflict, latestEpoch); + } + + @Override + public long serializedSize(GetMaxConflictOk reply, int version) + { + return CommandSerializers.timestamp.serializedSize(reply.maxConflict, version) + + TypeSizes.sizeofUnsignedVInt(reply.latestEpoch); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java index 2e3a4e827f..64e9369611 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java @@ -52,6 +52,7 @@ import accord.primitives.Seekable; import accord.primitives.Seekables; import accord.primitives.Unseekables; import accord.primitives.Unseekables.UnseekablesKind; +import accord.utils.Invariants; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; @@ -382,8 +383,7 @@ public class KeySerializers case 5: kind = UnseekablesKind.PartialRangeRoute; result = (RS)partialRangeRoute.deserialize(in, version); break; case 6: kind = UnseekablesKind.FullRangeRoute; result = (RS)fullRangeRoute.deserialize(in, version); break; } - if (!permitted.contains(kind)) - throw new IllegalStateException(); + Invariants.require(permitted.contains(kind)); return result; } diff --git a/src/java/org/apache/cassandra/service/accord/serializers/WaitingOnSerializer.java b/src/java/org/apache/cassandra/service/accord/serializers/WaitingOnSerializer.java index e0abec30b5..4236e7fbc6 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/WaitingOnSerializer.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/WaitingOnSerializer.java @@ -23,7 +23,6 @@ import java.io.IOException; import accord.impl.CommandChange.WaitingOnProvider; import accord.local.Command; import accord.local.Command.WaitingOn; -import accord.primitives.KeyDeps; import accord.primitives.PartialDeps; import accord.primitives.RangeDeps; import accord.primitives.RoutingKeys; @@ -76,12 +75,11 @@ public class WaitingOnSerializer { RoutingKeys keys = deps.keyDeps.keys(); RangeDeps directRangeDeps = deps.rangeDeps; - KeyDeps directKeyDeps = deps.directKeyDeps; - int txnIdCount = directRangeDeps.txnIdCount() + directKeyDeps.txnIdCount(); + int txnIdCount = directRangeDeps.txnIdCount(); Invariants.require(waitingOn.size()/64 == (txnIdCount + keys.size() + 63) / 64); Invariants.require(appliedOrInvalidated == null || (appliedOrInvalidated.size()/64 == (txnIdCount + 63)/64)); - WaitingOn result = new WaitingOn(keys, directRangeDeps, directKeyDeps, waitingOn, appliedOrInvalidated); + WaitingOn result = new WaitingOn(keys, directRangeDeps, waitingOn, appliedOrInvalidated); if (executeAtLeast != null) return new Command.WaitingOnWithExecuteAt(result, executeAtLeast); else if (uniqueHlc != 0) return new Command.WaitingOnWithMinUniqueHlc(result, uniqueHlc); return result; diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnDataKeyValue.java b/src/java/org/apache/cassandra/service/accord/txn/TxnDataKeyValue.java index c56b2c49ba..a68c796be2 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnDataKeyValue.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnDataKeyValue.java @@ -20,6 +20,7 @@ package org.apache.cassandra.service.accord.txn; import java.io.IOException; +import accord.utils.Invariants; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.db.rows.DeserializationHelper; @@ -48,10 +49,9 @@ public class TxnDataKeyValue extends FilteredPartition implements TxnDataValue } @Override - public TxnDataValue merge(TxnDataValue other) + public TxnDataValue merge(TxnDataValue that) { - // TODO (review): In Accord, for keys, these should always be identical and really there shouldn't be duplicates - // so we could return either + Invariants.require(this.equals(that)); return this; } diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java b/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java index 638781f85f..543fb49b33 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java @@ -49,6 +49,7 @@ import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.utils.ObjectSizes; +import static accord.api.Data.NOOP_DATA; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.CAS_READ; @@ -210,7 +211,7 @@ public abstract class TxnQuery implements Query ClusterMetadata clusterMetadata = ClusterMetadata.current(); checkState(clusterMetadata.epoch.getEpoch() >= executeAt.epoch(), "TCM epoch %d is < executeAt epoch %d", clusterMetadata.epoch.getEpoch(), executeAt.epoch()); boolean reads = read != null && !read.keys().isEmpty(); - if (transactionShouldBeBlocked(clusterMetadata, reads, keys)) + if (transactionShouldBeBlocked(clusterMetadata, reads, keys, data, update)) { if (txnId.isWrite()) ClientRequestsMetricsHolder.accordWriteMetrics.accordMigrationRejects.mark(); @@ -258,8 +259,11 @@ public abstract class TxnQuery implements Query } }; - private static boolean transactionShouldBeBlocked(ClusterMetadata clusterMetadata, boolean reads, Seekables keys) + private static boolean transactionShouldBeBlocked(ClusterMetadata clusterMetadata, boolean reads, Seekables keys, Data data, Update update) { + if (data == NOOP_DATA && (update == null || update.keys().isEmpty())) + return false; + // TxnQuery needs to be smart enough to allow blind writes through for the non-transactional use cases during migration // This also allows blind write TransactionStatement to run before TransactionStatemetns with reads can run, // but this is harmless since we only promise that TransactionStatement works when migrated to Accord. diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java b/src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java index b7930c84f7..ffdb99e21e 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java @@ -110,6 +110,11 @@ public class TxnUpdate extends AccordUpdate this.preserveTimestamps = preserveTimestamps; } + public static TxnUpdate empty() + { + return new TxnUpdate(Collections.emptyList(), TxnCondition.none(), null, false); + } + @Override public long estimatedSizeOnHeap() { @@ -221,6 +226,9 @@ public class TxnUpdate extends AccordUpdate if (!checkCondition(data)) return TxnWrite.EMPTY_CONDITION_FAILED; + if (keys.isEmpty()) + return new TxnWrite(Collections.emptyList(), true); + List fragments = deserialize(this.fragments, TxnWrite.Fragment.serializer); List updates = new ArrayList<>(fragments.size()); QueryOptions options = QueryOptions.forProtocolVersion(ProtocolVersion.CURRENT); @@ -365,8 +373,9 @@ public class TxnUpdate extends AccordUpdate if (conditionResult != null) return conditionResult; TxnCondition condition = AccordSerializers.deserialize(this.condition, TxnCondition.serializer); - conditionResult = condition.applies((TxnData) data); - return conditionResult; + if (condition == TxnCondition.none()) + return conditionResult = true; + return conditionResult = condition.applies((TxnData) data); } @Override diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java index c6dc28c4a8..5f3df247bd 100644 --- a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java @@ -21,7 +21,6 @@ package org.apache.cassandra.service.consensus.migration; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import java.util.SortedSet; import java.util.UUID; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; @@ -33,9 +32,8 @@ import com.google.common.primitives.Ints; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import accord.api.BarrierType; import accord.primitives.Keys; -import accord.primitives.Seekables; +import accord.primitives.Timestamp; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; @@ -62,6 +60,9 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.service.accord.RequestBookkeeping; +import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.paxos.AbstractPaxosRepair.Failure; import org.apache.cassandra.service.paxos.AbstractPaxosRepair.Result; @@ -70,11 +71,15 @@ import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.Collectors3; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.UUIDSerializer; +import static accord.local.durability.DurabilityService.SyncLocal.Self; +import static accord.local.durability.DurabilityService.SyncRemote.NoRemote; +import static accord.local.durability.DurabilityService.SyncRemote.Quorum; +import static org.apache.cassandra.config.DatabaseDescriptor.getReadRpcTimeout; +import static org.apache.cassandra.config.DatabaseDescriptor.getWriteRpcTimeout; import static org.apache.cassandra.net.Verb.CONSENSUS_KEY_MIGRATION; import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget.paxos; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -290,27 +295,23 @@ public abstract class ConsensusKeyMigrationState boolean isForWrite) { ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId); - if (isForWrite) - ClientRequestsMetricsHolder.casWriteMetrics.accordKeyMigrations.mark(); - else - ClientRequestsMetricsHolder.casReadMetrics.accordKeyMigrations.mark(); + if (isForWrite) ClientRequestsMetricsHolder.casWriteMetrics.accordKeyMigrations.mark(); + else ClientRequestsMetricsHolder.casReadMetrics.accordKeyMigrations.mark(); + // Global will always create a transaction to effect the barrier so all replicas + // will soon be ready to execute, but only waits for the local replica to be ready + // Local will only create a transaction if it can't find an existing one to wait on + Keys partitionKeys = AccordService.intersecting(Keys.of(keys, k -> new PartitionKey(tableId, k))); + if (partitionKeys.isEmpty()) + throw new RetryOnDifferentSystemException(); + + IAccordService accord = AccordService.instance(); + long start = nanoTime(); - try - { - // Global will always create a transaction to effect the barrier so all replicas - // will soon be ready to execute, but only waits for the local replica to be ready - // Local will only create a transaction if it can't find an existing one to wait on - BarrierType barrierType = global ? BarrierType.global_async : BarrierType.local; - SortedSet partitionKeys = keys.stream().map(key -> new PartitionKey(tableId, key)).collect(Collectors3.toSortedSet()); - Seekables keysOrRanges = AccordService.instance().barrier(new Keys(partitionKeys), minEpoch, requestTime, DatabaseDescriptor.getTransactionTimeout(TimeUnit.NANOSECONDS), barrierType, isForWrite); - if (keysOrRanges.isEmpty()) - throw new RetryOnDifferentSystemException(); - // We don't save the state to the cache here. Accord will notify the agent every time a barrier happens. - } - finally - { - cfs.metric.keyMigration.addNano(nanoTime() - start); - } + long deadline = requestTime.computeDeadline(isForWrite ? getWriteRpcTimeout(TimeUnit.NANOSECONDS) : getReadRpcTimeout(TimeUnit.NANOSECONDS)); + RequestBookkeeping bookkeeping = new LatencyRequestBookkeeping(cfs == null ? null : cfs.metric.keyMigration); + AccordService.getBlocking(accord.sync(Timestamp.minForEpoch(minEpoch), partitionKeys, Self, global ? Quorum : NoRemote), + partitionKeys, bookkeeping, start, deadline); + maybeSaveAccordKeyMigrationLocally((PartitionKey) partitionKeys.get(0), Epoch.create(minEpoch)); } static void repairKeyPaxos(EndpointsForToken naturalReplicas, diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationMutationHelper.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationMutationHelper.java index 81c29b7d09..362bf917d3 100644 --- a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationMutationHelper.java +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationMutationHelper.java @@ -32,7 +32,6 @@ import com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import accord.coordinate.CoordinationFailed; import accord.primitives.Keys; import accord.primitives.Routable.Domain; import accord.primitives.Txn; @@ -52,12 +51,13 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableParams; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.IAccordService; -import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult; +import org.apache.cassandra.service.accord.IAccordService.IAccordResult; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.txn.TxnCondition; import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.service.accord.txn.TxnReferenceOperations; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.accord.txn.TxnUpdate; import org.apache.cassandra.service.accord.txn.TxnWrite; import org.apache.cassandra.service.consensus.TransactionalMode; @@ -183,7 +183,7 @@ public class ConsensusMigrationMutationHelper void consume(@Nullable T accordMutation, @Nullable T normalMutation, List mutations, int mutationIndex); } - public static SplitMutations splitMutationsIntoAccordAndNormal(ClusterMetadata cm, List mutations) + public static SplitMutations splitMutationsIntoAccordAndNormal(ClusterMetadata cm, List mutations) { SplitMutations splitMutations = new SplitMutations<>(); splitMutationsIntoAccordAndNormal(cm, mutations, splitMutations); @@ -233,12 +233,12 @@ public class ConsensusMigrationMutationHelper return new SplitMutation(accordMutation, normalMutation); } - public AsyncTxnResult mutateWithAccordAsync(ClusterMetadata cm, Mutation mutation, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + public IAccordResult mutateWithAccordAsync(ClusterMetadata cm, Mutation mutation, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { return mutateWithAccordAsync(cm, ImmutableList.of(mutation), consistencyLevel, requestTime); } - public static AsyncTxnResult mutateWithAccordAsync(ClusterMetadata cm, Collection mutations, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + public static IAccordResult mutateWithAccordAsync(ClusterMetadata cm, Collection mutations, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { if (consistencyLevel != null && !IAccordService.SUPPORTED_COMMIT_CONSISTENCY_LEVELS.contains(consistencyLevel)) throw new InvalidRequestException(consistencyLevel + " is not supported by Accord"); @@ -260,17 +260,7 @@ public class ConsensusMigrationMutationHelper ConsistencyLevel clForCommit = consistencyLevelForCommit(cm, mutations, consistencyLevel); TxnUpdate update = new TxnUpdate(fragments, TxnCondition.none(), clForCommit, true); Txn.InMemory txn = new Txn.InMemory(Keys.of(partitionKeys), TxnRead.empty(Domain.Key), TxnQuery.NONE, update); - IAccordService accordService = AccordService.instance(); - try - { - return accordService.coordinateAsync(minEpoch, txn, clForCommit, requestTime); - } - catch (CoordinationFailed coordinationFailed) - { - AsyncTxnResult failure = new AsyncTxnResult(coordinationFailed.txnId(), minEpoch, clForCommit, true, requestTime); - failure.setFailure(coordinationFailed.wrap()); - return failure; - } + return AccordService.instance().coordinateAsync(minEpoch, txn, clForCommit, requestTime); } public static void validateSafeToExecuteNonTransactionally(IMutation mutation) throws RetryOnDifferentSystemException diff --git a/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java b/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java index e513bd1a17..3f983faa8c 100644 --- a/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java +++ b/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java @@ -19,6 +19,7 @@ package org.apache.cassandra.service.paxos; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import org.apache.cassandra.config.DatabaseDescriptor; @@ -32,70 +33,38 @@ import org.apache.cassandra.service.TimeoutStrategy.ReadWriteLatencySourceFactor import org.apache.cassandra.service.TimeoutStrategy.Wait; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.NoSpamLogger; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static java.lang.Math.*; -import static java.util.concurrent.TimeUnit.*; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.util.Arrays.stream; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; import static org.apache.cassandra.config.DatabaseDescriptor.*; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics; +import static org.apache.cassandra.service.TimeoutStrategy.parseInMicros; import static org.apache.cassandra.utils.Clock.waitUntil; import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized; /** * See {@link RetryStrategy} + * TODO (expected): deprecate in favour of pure RetryStrategy */ public class ContentionStrategy extends RetryStrategy { private static final Logger logger = LoggerFactory.getLogger(RetryStrategy.class); - private static final String DEFAULT_WAIT_RANDOMIZER = "uniform"; - private static final String DEFAULT_MIN = "0"; - private static final String DEFAULT_MAX = "100ms"; - private static final String DEFAULT_SPREAD = "100ms"; - private static final LatencySourceFactory LATENCIES = new ReadWriteLatencySourceFactory(casReadMetrics, casWriteMetrics); - - private static volatile ContentionStrategy current; - private static volatile ParsedStrategy currentParsed; - private static final RetryStrategy.ParsedStrategy defaultStrategy; - static - { - defaultStrategy = new ParsedStrategy(DEFAULT_WAIT_RANDOMIZER, DEFAULT_MIN, DEFAULT_MAX, DEFAULT_SPREAD, Integer.MAX_VALUE, - new ContentionStrategy(DEFAULT_WAIT_RANDOMIZER, DEFAULT_MIN, DEFAULT_MAX, DEFAULT_SPREAD, Integer.MAX_VALUE)); - - String waitRandomizer = orElse(DatabaseDescriptor::getPaxosContentionWaitRandomizer, DEFAULT_WAIT_RANDOMIZER); - String min = orElse(DatabaseDescriptor::getPaxosContentionMinWait, DEFAULT_MIN); - String max = orElse(DatabaseDescriptor::getPaxosContentionMaxWait, DEFAULT_MAX); - String spread = orElse(DatabaseDescriptor::getPaxosContentionMinDelta, DEFAULT_SPREAD); - - current = new ContentionStrategy(waitRandomizer, min, max, spread, Integer.MAX_VALUE); - currentParsed = new ParsedStrategy(waitRandomizer, min, max, spread, Integer.MAX_VALUE, current); - } - - final int traceAfterAttempts; - - public ContentionStrategy(String waitRandomizer, String min, String max, String spread, int traceAfterAttempts) - { - super(waitRandomizer, min, max, spread, LATENCIES); - this.traceAfterAttempts = traceAfterAttempts; - } - - public ContentionStrategy(WaitRandomizer waitRandomizer, Wait min, Wait max, Wait spread, int traceAfterAttempts) - { - super(waitRandomizer, min, max, spread); - this.traceAfterAttempts = traceAfterAttempts; - } - - @Override - protected Wait parseBound(String spec, boolean isMin, LatencySourceFactory latencies) - { - return TimeoutStrategy.parseWait(spec, 0, maxQueryTimeoutMicros(), isMin ? 0 : maxQueryTimeoutMicros(), latencies); - } - public enum Type { READ("Contended Paxos Read"), WRITE("Contended Paxos Write"), REPAIR("Contended Paxos Repair"); @@ -110,6 +79,71 @@ public class ContentionStrategy extends RetryStrategy } } + static final Pattern LEGACY = Pattern.compile( + "(?0|[0-9]+[mu]?s)" + + "|((?0|[0-9]+[mu]?s) *<= *)?" + + "(?[^=]+)?" + + "( *<= *(?0|[0-9]+[mu]?s))?"); + + private static final String DEFAULT_WAIT_RANDOMIZER = "uniform"; + private static final String DEFAULT_MIN = "0"; + private static final String DEFAULT_MAX = "100ms"; + private static final String DEFAULT_SPREAD = "100ms"; + private static final String MAX_INT = "" + Integer.MAX_VALUE; + private static final LatencySourceFactory LATENCIES = new ReadWriteLatencySourceFactory(casReadMetrics, casWriteMetrics); + + private static volatile ContentionStrategy current; + private static volatile ParsedStrategy currentParsed; + static + { + String waitRandomizer = orElse(DatabaseDescriptor::getPaxosContentionWaitRandomizer, DEFAULT_WAIT_RANDOMIZER); + String min = orElse(DatabaseDescriptor::getPaxosContentionMinWait, DEFAULT_MIN); + String max = orElse(DatabaseDescriptor::getPaxosContentionMaxWait, DEFAULT_MAX); + String spread = orElse(DatabaseDescriptor::getPaxosContentionMinDelta, DEFAULT_SPREAD); + current = parse(waitRandomizer, min, max, spread, MAX_INT, MAX_INT); + currentParsed = new ParsedStrategy(waitRandomizer, min, max, spread, MAX_INT, MAX_INT, current); + } + + final @Nullable LegacyWait spread; + final int traceAfterAttempts; + + public ContentionStrategy(WaitRandomizer waitRandomizer, LegacyWait min, LegacyWait max, LegacyWait spread, int retries, int traceAfterAttempts) + { + super(waitRandomizer, min.min, min, max, max.max, retries); + this.traceAfterAttempts = traceAfterAttempts; + this.spread = spread; + } + + public long computeWait(int attempt, TimeUnit units) + { + if (attempt > maxAttempts) + return -1; + + long minWaitMicros = min.getMicros(attempt); + long maxWaitMicros = max.getMicros(attempt); + long spreadMicros = spread == null ? 0 : spread.getMicros(attempt); + + if (minWaitMicros + spreadMicros >= maxWaitMicros) + { + if (spreadMicros == 0) + return units.convert(maxWaitMicros, MICROSECONDS); + + if (maxWaitMicros < minWaitMicros) + maxWaitMicros = minWaitMicros; + long newMaxWaitMicros = minWaitMicros + spreadMicros; + if (newMaxWaitMicros > maxMaxMicros) + { + newMaxWaitMicros = maxMaxMicros; + minWaitMicros = max(this.minMinMicros, maxWaitMicros - spreadMicros); + } + maxWaitMicros = newMaxWaitMicros; + if (minWaitMicros >= maxWaitMicros) + return minWaitMicros; + } + + return units.convert(waitRandomizer.wait(minWaitMicros, maxWaitMicros, attempt), MICROSECONDS); + } + long computeWaitUntilForContention(int attempts, TableMetadata table, DecoratedKey partitionKey, ConsistencyLevel consistency, Type type) { if (attempts >= traceAfterAttempts && !Tracing.isTracing()) @@ -162,41 +196,71 @@ public class ContentionStrategy extends RetryStrategy return current.computeWaitUntilForContention(attempts, table, partitionKey, consistency, type); } - public static class ParsedStrategy extends RetryStrategy.ParsedStrategy + public static class ParsedStrategy { - public final int trace; + public final String waitRandomizer, min, max, spread, retries, trace; public final ContentionStrategy strategy; - ParsedStrategy(String waitRandomizer, String min, String max, String minDelta, int trace, ContentionStrategy strategy) + protected ParsedStrategy(String waitRandomizer, String min, String max, String spread, String retries, String trace, ContentionStrategy strategy) { - super(waitRandomizer, min, max, minDelta, strategy); + this.waitRandomizer = waitRandomizer; + this.min = min; + this.max = max; + this.spread = spread; + this.retries = retries; this.trace = trace; this.strategy = strategy; } - @Override public String toString() { - return super.toString() + (trace == Integer.MAX_VALUE ? "" : ",trace=" + current.traceAfterAttempts); + return "min=" + min + ",max=" + max + ",spread=" + spread + ",retries=" + retries + + ",random=" + waitRandomizer + ",trace=" + current.traceAfterAttempts; } } @VisibleForTesting - public static ParsedStrategy parseStrategy(String spec) + public static ParsedStrategy parseStrategy(String spec, ParsedStrategy defaultStrategy) { - RetryStrategy.ParsedStrategy parsed = RetryStrategy.parseStrategy(spec, LATENCIES, defaultStrategy); String[] args = spec.split(","); - + String waitRandomizer = find(args, "random"); + String min = find(args, "min"); + String max = find(args, "max"); + String spread = find(args, "spread"); String trace = find(args, "trace"); - int traceAfterAttempts = trace == null ? current.traceAfterAttempts: Integer.parseInt(trace); + if (spread == null) + spread = find(args, "delta"); + String retries = find(args, "retries"); - ContentionStrategy strategy = new ContentionStrategy(parsed.strategy.waitRandomizer, parsed.strategy.min, parsed.strategy.max, parsed.strategy.spread, traceAfterAttempts); - return new ParsedStrategy(parsed.waitRandomizer, parsed.min, parsed.max, parsed.spread, traceAfterAttempts, strategy); + if (waitRandomizer == null) waitRandomizer = defaultStrategy.waitRandomizer; + if (min == null) min = defaultStrategy.min; + if (max == null) max = defaultStrategy.max; + if (spread == null) spread = defaultStrategy.spread; + if (retries == null) retries = defaultStrategy.retries; + if (trace == null) trace = defaultStrategy.trace; + + ContentionStrategy strategy = parse(waitRandomizer, min, max, spread, retries, trace); + return new ParsedStrategy(waitRandomizer, min, max, spread, retries, trace, strategy); } + private static ContentionStrategy parse(String waitRandomizerString, String minString, String maxString, String spreadString, String retriesString, String traceString) + { + return new ContentionStrategy(parseWaitRandomizer(waitRandomizerString), + parseLegacy(minString, true), parseLegacy(maxString, false), parseLegacy(spreadString, false), + Integer.parseInt(retriesString), Integer.parseInt(traceString)); + } + + private static String find(String[] args, String param) + { + return stream(args).filter(s -> s.startsWith(param + '=')) + .map(s -> s.substring(param.length() + 1)) + .findFirst().orElse(null); + } + + public static synchronized void setStrategy(String spec) { - ParsedStrategy parsed = parseStrategy(spec); + ParsedStrategy parsed = parseStrategy(spec, currentParsed); currentParsed = parsed; current = parsed.strategy; setPaxosContentionWaitRandomizer(parsed.waitRandomizer); @@ -210,15 +274,74 @@ public class ContentionStrategy extends RetryStrategy return currentParsed.toString(); } - @VisibleForTesting - static long maxQueryTimeoutMicros() - { - return max(max(getCasContentionTimeout(MICROSECONDS), getWriteRpcTimeout(MICROSECONDS)), getReadRpcTimeout(MICROSECONDS)); - } - private static String orElse(Supplier get, String orElse) { String result = get.get(); return result != null ? result : orElse; } + + @VisibleForTesting + static LegacyWait parseLegacy(String spec, boolean isMin) + { + long defaultMaxMicros = getRpcTimeout(MICROSECONDS); + return parseLegacy(spec, 0, defaultMaxMicros, isMin ? 0 : defaultMaxMicros, LATENCIES); + } + + public static LegacyWait parseLegacy(String input, long defaultMinMicros, long defaultMaxMicros, long onFailure, LatencySourceFactory latencies) + { + Matcher m = LEGACY.matcher(input); + if (!m.matches()) + throw new IllegalArgumentException(input + " does not match " + LEGACY); + + String maybeConst = m.group("const"); + if (maybeConst != null) + { + long v = parseInMicros(maybeConst); + return new LegacyWait(v, v, v, new Wait.Constant(v)); + } + + long min = parseInMicros(m.group("min"), defaultMinMicros); + long max = parseInMicros(m.group("max"), defaultMaxMicros); + return new LegacyWait(min, max, onFailure, TimeoutStrategy.parseWait(m.group("delegate"), latencies)); + } + + + private static class LegacyWait implements Wait + { + final long min, max, onFailure; + final Wait delegate; + + LegacyWait(long min, long max, long onFailure, Wait delegate) + { + Preconditions.checkArgument(min <= max, "min (%s) must be less than or equal to max (%s)", min, max); + this.min = min; + this.max = max; + this.onFailure = onFailure; + this.delegate = delegate; + } + + public long getMicros(int attempts) + { + try + { + return max(min, min(max, delegate.getMicros(attempts))); + } + catch (Throwable t) + { + NoSpamLogger.getLogger(logger, 1L, MINUTES).info("", t); + return onFailure; + } + } + + public String toString() + { + return "Bound{" + + "min=" + min + + ", max=" + max + + ", onFailure=" + onFailure + + ", delegate=" + delegate + + '}'; + } + } + } diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosRepairState.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosRepairState.java index 5636d5cb73..b4522719a8 100644 --- a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosRepairState.java +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosRepairState.java @@ -130,7 +130,7 @@ public class PaxosRepairState private static void add(SharedContext ctx, AtomicReference pendingCleanup, Message message) { PendingCleanup next = new PendingCleanup(message); - PendingCleanup prev = IntrusiveStack.push(AtomicReference::get, AtomicReference::compareAndSet, pendingCleanup, next); + PendingCleanup prev = IntrusiveStack.getAndPush(AtomicReference::get, AtomicReference::compareAndSet, pendingCleanup, next); if (prev == null) Stage.MISC.execute(() -> cleanup(ctx, pendingCleanup)); } diff --git a/src/java/org/apache/cassandra/service/reads/range/AccordRangeResponse.java b/src/java/org/apache/cassandra/service/reads/range/AccordRangeResponse.java index 91a4ffc2c5..3ec8e6b008 100644 --- a/src/java/org/apache/cassandra/service/reads/range/AccordRangeResponse.java +++ b/src/java/org/apache/cassandra/service/reads/range/AccordRangeResponse.java @@ -20,30 +20,25 @@ package org.apache.cassandra.service.reads.range; import java.util.function.IntPredicate; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.exceptions.RetryOnDifferentSystemException; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult; -import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult; -import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.service.accord.IAccordService.IAccordResult; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.utils.AbstractIterator; public class AccordRangeResponse extends AbstractIterator implements PartitionIterator { - private final AsyncTxnResult asyncTxnResult; + private final IAccordResult accordResult; // Range queries don't support reverse, but dutifully threading it through anyways private final boolean reversed; - private final ConsistencyLevel cl; - private final Dispatcher.RequestTime requestTime; private PartitionIterator result; - public AccordRangeResponse(AsyncTxnResult asyncTxnResult, boolean reversed, ConsistencyLevel cl, Dispatcher.RequestTime requestTime) + public AccordRangeResponse(IAccordResult accordResult, boolean reversed) { - this.asyncTxnResult = asyncTxnResult; - this.cl = cl; - this.requestTime = requestTime; + this.accordResult = accordResult; this.reversed = reversed; } @@ -54,7 +49,7 @@ public class AccordRangeResponse extends AbstractIterator implement IntPredicate alwaysTrue = ignored -> true; IntPredicate alwaysFalse = ignored -> false; // TODO (required): Handle retry on different system - ConsensusAttemptResult consensusAttemptResult = StorageProxy.getConsensusAttemptResultFromAsyncTxnResult(asyncTxnResult, 1, reversed ? alwaysTrue : alwaysFalse); + ConsensusAttemptResult consensusAttemptResult = StorageProxy.getConsensusAttemptResultFromAsyncTxnResult(accordResult, 1, reversed ? alwaysTrue : alwaysFalse); if (consensusAttemptResult.shouldRetryOnNewConsensusProtocol) throw new RetryOnDifferentSystemException(); result = consensusAttemptResult.serialReadResult; diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java index 11d9539777..1d0c01e674 100644 --- a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java +++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java @@ -51,7 +51,8 @@ import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.metrics.ClientRangeRequestMetrics; import org.apache.cassandra.net.Message; import org.apache.cassandra.service.StorageProxy; -import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult; +import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.RangeReadTarget; import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.RangeReadWithTarget; @@ -195,9 +196,9 @@ public class RangeCommandIterator extends AbstractIterator implemen private PartitionIterator executeAccord(ClusterMetadata cm, PartitionRangeReadCommand rangeCommand, ConsistencyLevel cl) { - //TODO (nicetohave): https://issues.apache.org/jira/browse/CASSANDRA-20210 More efficient reads across command stores - AsyncTxnResult result = StorageProxy.readWithAccord(cm, rangeCommand, rangeCommand.dataRange().keyRange(), cl, requestTime); - return new AccordRangeResponse(result, rangeCommand.isReversed(), cl, requestTime); + //TODO (expected): https://issues.apache.org/jira/browse/CASSANDRA-20210 More efficient reads across command stores + IAccordService.IAccordResult result = StorageProxy.readWithAccord(cm, rangeCommand, rangeCommand.dataRange().keyRange(), cl, requestTime); + return new AccordRangeResponse(result, rangeCommand.isReversed()); } private SingleRangeResponse executeNormal(ReplicaPlan.ForRangeRead replicaPlan, PartitionRangeReadCommand rangeCommand, ReadCoordinator readCoordinator) diff --git a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java index e5c58ef3ad..72d46a5af5 100644 --- a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java +++ b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java @@ -18,7 +18,6 @@ package org.apache.cassandra.tcm; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.slf4j.Logger; @@ -50,9 +49,9 @@ public abstract class AbstractLocalProcessor implements Processor * the time when this method returns. */ @Override - public final Commit.Result commit(Entry.Id entryId, Transformation transform, final Epoch lastKnown, Retry.Deadline retryPolicy) + public final Commit.Result commit(Entry.Id entryId, Transformation transform, final Epoch lastKnown, Retry retryPolicy) { - while (!retryPolicy.reachedMax()) + while (!retryPolicy.hasExpired()) { ClusterMetadata previous = log.waitForHighestConsecutive(); if (!previous.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort())) @@ -112,7 +111,8 @@ public abstract class AbstractLocalProcessor implements Processor } else { - retryPolicy.maybeSleep(); + if (!retryPolicy.maybeSleep()) + break; // TODO: could also add epoch from mis-application from [applied]. fetchLogAndWait(null, retryPolicy); } @@ -121,13 +121,12 @@ public abstract class AbstractLocalProcessor implements Processor { logger.error("Caught error while trying to perform a local commit", e); JVMStabilityInspector.inspectThrowable(e); - retryPolicy.maybeSleep(); + if (!retryPolicy.maybeSleep()) + break; } } return Commit.Result.failed(SERVER_ERROR, - String.format("Could not perform commit after %d/%d tries. Time remaining: %dms", - retryPolicy.tries, retryPolicy.maxTries, - TimeUnit.NANOSECONDS.toMillis(retryPolicy.remainingNanos()))); + String.format("Could not perform commit; policy %s gave up", retryPolicy)); } public Commit.Result maybeFailure(Entry.Id entryId, Epoch lastKnown, Supplier orElse) @@ -196,6 +195,6 @@ public abstract class AbstractLocalProcessor implements Processor return logState; } - public abstract ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy); + public abstract ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy); protected abstract boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch); } diff --git a/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java b/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java index 475c1d0853..d124707300 100644 --- a/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java +++ b/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java @@ -76,7 +76,7 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor } @Override - public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retry) + public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retry) { return log.waitForHighestConsecutive(); } @@ -109,7 +109,7 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor } @Override - public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry.Deadline retryPolicy) + public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry retryPolicy) { return getLocalState(lowEpoch, highEpoch, includeSnapshot); } diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java index a7498df1ea..a9fff2dcde 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java @@ -78,6 +78,7 @@ import org.apache.cassandra.utils.concurrent.ImmediateFuture; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.stream.Collectors.toSet; import static org.apache.cassandra.config.CassandraRelevantProperties.TCM_SKIP_CMS_RECONFIGURATION_AFTER_TOPOLOGY_CHANGE; +import static org.apache.cassandra.config.DatabaseDescriptor.getCmsAwaitTimeout; import static org.apache.cassandra.tcm.ClusterMetadataService.State.GOSSIP; import static org.apache.cassandra.tcm.ClusterMetadataService.State.LOCAL; import static org.apache.cassandra.tcm.ClusterMetadataService.State.REMOTE; @@ -667,8 +668,7 @@ public class ClusterMetadataService if (ourEpoch.isEqualOrAfter(awaitAtLeast)) return metadata; - Retry.Deadline deadline = Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), - new Retry.Jitter(TCMMetrics.instance.fetchLogRetries)); + Retry deadline = Retry.untilElapsed(getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.fetchLogRetries); // responses for ALL withhout knowing we have pending metadata = processor.fetchLogAndWait(awaitAtLeast, deadline); if (metadata.epoch.isBefore(awaitAtLeast)) @@ -876,9 +876,9 @@ public class ClusterMetadataService } @Override - public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy) + public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy) { - while (!retryPolicy.reachedMax()) + while (true) { try { @@ -891,14 +891,15 @@ public class ClusterMetadataService } catch (NotCMSException e) { - retryPolicy.maybeSleep(); + if (!retryPolicy.maybeSleep()) + break; } } return Commit.Result.failed(ExceptionCode.SERVER_ERROR, "Could not commit " + transform.kind() + " at epoch " + lastKnown); } @Override - public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy) + public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy) { return delegate().fetchLogAndWait(waitFor, retryPolicy); } @@ -910,7 +911,7 @@ public class ClusterMetadataService } @Override - public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy) + public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy) { return delegate().getLogState(start, end, includeSnapshot, retryPolicy); } diff --git a/src/java/org/apache/cassandra/tcm/Commit.java b/src/java/org/apache/cassandra/tcm/Commit.java index f6008f92f7..6767eeaf48 100644 --- a/src/java/org/apache/cassandra/tcm/Commit.java +++ b/src/java/org/apache/cassandra/tcm/Commit.java @@ -364,7 +364,7 @@ public class Commit { checkCMSState(); logger.info("Received commit request {} from {}", message.payload, message.from()); - Retry.Deadline retryPolicy = Retry.Deadline.at(message.expiresAtNanos(), new Retry.Jitter(TCMMetrics.instance.commitRetries)); + Retry retryPolicy = Retry.until(message.expiresAtNanos(), TCMMetrics.instance.commitRetries); Result result = processor.commit(message.payload.entryId, message.payload.transform, message.payload.lastKnown, retryPolicy); if (result.isSuccess()) { diff --git a/src/java/org/apache/cassandra/tcm/FetchCMSLog.java b/src/java/org/apache/cassandra/tcm/FetchCMSLog.java index ae1d431abb..71d7996469 100644 --- a/src/java/org/apache/cassandra/tcm/FetchCMSLog.java +++ b/src/java/org/apache/cassandra/tcm/FetchCMSLog.java @@ -25,7 +25,6 @@ import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; @@ -37,6 +36,8 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.tcm.log.LogState; import org.apache.cassandra.utils.FBUtilities; +import static org.apache.cassandra.config.DatabaseDescriptor.getCmsAwaitTimeout; + public class FetchCMSLog { public static final Serializer serializer = new Serializer(); @@ -115,8 +116,7 @@ public class FetchCMSLog // If both we and the other node believe it should be caught up with a linearizable read boolean consistentFetch = request.consistentFetch && !ClusterMetadataService.instance().isCurrentMember(message.from()); - Retry.Deadline retry = Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), - new Retry.Jitter(TCMMetrics.instance.fetchLogRetries)); + Retry retry = Retry.untilElapsed(getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.fetchLogRetries); LogState delta; if (consistentFetch) delta = processor.get().getLogState(message.payload.lowerBound, Epoch.MAX, false, retry); diff --git a/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java b/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java index e7ad50d9de..b3a6e318f5 100644 --- a/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java +++ b/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java @@ -70,7 +70,7 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor } @Override - public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy) + public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy) { ClusterMetadata metadata = log.waitForHighestConsecutive(); @@ -106,7 +106,7 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor for (Replica peer : replicas) requests.add(new FetchLogRequest(peer, MessagingService.instance(), metadata.epoch)); - while (!retryPolicy.reachedMax()) + while (!retryPolicy.hasExpired()) { Iterator iter = requests.iterator(); boolean hasRequestToSelf = false; @@ -153,7 +153,8 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor if (collected.size() < blockFor) { - retryPolicy.maybeSleep(); + if (!retryPolicy.maybeSleep()) + break; continue; } @@ -174,9 +175,9 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor } @Override - public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy) + public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy) { - while (!retryPolicy.reachedMax()) + while (true) { if (Thread.currentThread().isInterrupted()) throw new RuntimeException("Can not reconstruct during shutdown", new InterruptedException()); @@ -186,10 +187,10 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor } catch (RuntimeException e) // honestly best to only retry timeouts, but everything gets wrapped in a RuntimeException... { - retryPolicy.maybeSleep(); + if (!retryPolicy.maybeSleep()) + throw new RuntimeException(String.format("Could not reconstruct range %d, %d", start.getEpoch(), end.getEpoch()), new TimeoutException()); } } - throw new RuntimeException(String.format("Could not reconstruct range %d, %d", start.getEpoch(), end.getEpoch()), new TimeoutException()); } private static T unwrap(Promise promise) diff --git a/src/java/org/apache/cassandra/tcm/PeerLogFetcher.java b/src/java/org/apache/cassandra/tcm/PeerLogFetcher.java index 7192551c68..0565520bbd 100644 --- a/src/java/org/apache/cassandra/tcm/PeerLogFetcher.java +++ b/src/java/org/apache/cassandra/tcm/PeerLogFetcher.java @@ -95,8 +95,7 @@ public class PeerLogFetcher Verb.TCM_FETCH_PEER_LOG_REQ, new FetchPeerLog(before), new RemoteProcessor.CandidateIterator(Collections.singletonList(remote), false), - Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), - new Retry.Jitter(TCMMetrics.instance.fetchLogRetries))); + Retry.untilElapsed(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.fetchLogRetries)); return fetchRes.map((logState) -> { log.append(logState); diff --git a/src/java/org/apache/cassandra/tcm/Processor.java b/src/java/org/apache/cassandra/tcm/Processor.java index 2d8795ed91..9a27ac9e66 100644 --- a/src/java/org/apache/cassandra/tcm/Processor.java +++ b/src/java/org/apache/cassandra/tcm/Processor.java @@ -26,11 +26,14 @@ import java.util.concurrent.TimeUnit; import com.codahale.metrics.Meter; import accord.utils.Invariants; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.metrics.TCMMetrics; +import org.apache.cassandra.service.WaitStrategy; import org.apache.cassandra.tcm.log.Entry; import org.apache.cassandra.tcm.log.LogState; -import org.apache.cassandra.utils.Clock; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.cassandra.config.DatabaseDescriptor.getCmsAwaitTimeout; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; public interface Processor { @@ -49,8 +52,7 @@ public interface Processor } return commit(entryId, transform, lastKnown, - Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), - new Retry.Jitter(TCMMetrics.instance.commitRetries))); + Retry.untilElapsed(getCmsAwaitTimeout().to(NANOSECONDS), TCMMetrics.instance.commitRetries)); } /** @@ -58,33 +60,27 @@ public interface Processor * to overflow the long, since messaging is using only 32 bits for deadlines. To achieve that, we are * giving `timeoutNanos` every time we retry, but will retry indefinitely. */ - private static Retry.Deadline unsafeRetryIndefinitely() + private static Retry unsafeRetryIndefinitely() { - long timeoutNanos = DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS); + long timeoutNanos = getCmsAwaitTimeout().to(NANOSECONDS); Meter retryMeter = TCMMetrics.instance.commitRetries; - return new Retry.Deadline(Clock.Global.nanoTime() + timeoutNanos, - new Retry.Jitter(retryMeter)) + return Retry.withNoTimeLimit(retryMeter, new WaitStrategy() { @Override - public boolean reachedMax() + public long computeWaitUntil(int attempts) { - return false; + return nanoTime() + timeoutNanos; } @Override - public long remainingNanos() + public long computeWait(int attempts, TimeUnit units) { - return timeoutNanos; + return units.convert(timeoutNanos, NANOSECONDS); } - - public String toString() - { - return String.format("RetryIndefinitely{tries=%d}", currentTries()); - } - }; + }); } - Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy); + Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy); /** * Fetches log from CMS up to the highest currently known epoch. @@ -103,11 +99,10 @@ public interface Processor default ClusterMetadata fetchLogAndWait(Epoch waitFor) { return fetchLogAndWait(waitFor, - Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), - new Retry.Jitter(TCMMetrics.instance.fetchLogRetries))); + Retry.untilElapsed(getCmsAwaitTimeout().to(NANOSECONDS), TCMMetrics.instance.fetchLogRetries)); } - ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy); + ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy); /** * Queries node's _local_ state. It is not guaranteed to be contiguous, but can be used for restoring CMS state/ @@ -117,12 +112,12 @@ public interface Processor /** * Queries global log state. */ - LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy); + LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy); /** * Reconstructs */ - default List reconstruct(Epoch lowEpoch, Epoch highEpoch, Retry.Deadline retryPolicy) + default List reconstruct(Epoch lowEpoch, Epoch highEpoch, Retry retryPolicy) { LogState logState = getLogState(lowEpoch, highEpoch, true, retryPolicy); if (logState.isEmpty()) return Collections.emptyList(); diff --git a/src/java/org/apache/cassandra/tcm/ReconstructLogState.java b/src/java/org/apache/cassandra/tcm/ReconstructLogState.java index dcc0b395e7..4dea1efeb8 100644 --- a/src/java/org/apache/cassandra/tcm/ReconstructLogState.java +++ b/src/java/org/apache/cassandra/tcm/ReconstructLogState.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; @@ -34,6 +33,8 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.tcm.log.LogState; import org.apache.cassandra.utils.FBUtilities; +import static org.apache.cassandra.config.DatabaseDescriptor.getCmsAwaitTimeout; + public class ReconstructLogState { public static final Serializer serializer = new Serializer(); @@ -97,8 +98,7 @@ public class ReconstructLogState throw new NotCMSException("This node is not in the CMS, can't generate a consistent log fetch response to " + message.from()); LogState result = processor.get().getLogState(request.lowerBound, request.higherBound, request.includeSnapshot, - Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), - new Retry.Jitter(TCMMetrics.instance.fetchLogRetries))); + Retry.untilElapsed(getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.fetchLogRetries)); MessagingService.instance().send(message.responseWith(result), message.from()); } diff --git a/src/java/org/apache/cassandra/tcm/RemoteProcessor.java b/src/java/org/apache/cassandra/tcm/RemoteProcessor.java index b0ee46b2c4..fe87719593 100644 --- a/src/java/org/apache/cassandra/tcm/RemoteProcessor.java +++ b/src/java/org/apache/cassandra/tcm/RemoteProcessor.java @@ -44,12 +44,12 @@ import org.apache.cassandra.net.MessageDelivery; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.RequestCallbackWithFailure; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.service.WaitStrategy; import org.apache.cassandra.tcm.Discovery.DiscoveredNodes; import org.apache.cassandra.tcm.log.Entry; import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.log.LogState; import org.apache.cassandra.utils.AbstractIterator; -import org.apache.cassandra.utils.Backoff; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Future; @@ -73,7 +73,7 @@ public final class RemoteProcessor implements Processor @Override @SuppressWarnings("resource") - public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy) + public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy) { try { @@ -126,7 +126,7 @@ public final class RemoteProcessor implements Processor } @Override - public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy) + public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy) { // Synchonous, non-debounced call if we are waiting for the highest epoch (without knowing/caring what it is). // Should be used sparingly. @@ -158,7 +158,7 @@ public final class RemoteProcessor implements Processor } @Override - public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry.Deadline retryPolicy) + public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry retryPolicy) { try { @@ -203,7 +203,7 @@ public final class RemoteProcessor implements Processor Verb.TCM_FETCH_CMS_LOG_REQ, new FetchCMSLog(currentEpoch, ClusterMetadataService.state() == REMOTE), candidates, - new Retry.Backoff(TCMMetrics.instance.fetchLogRetries)); + Retry.withNoTimeLimit(TCMMetrics.instance.fetchLogRetries)); return remoteRequest.map((replay) -> { if (!replay.isEmpty()) { @@ -217,12 +217,12 @@ public final class RemoteProcessor implements Processor } // todo rename to send with retries or something - public static RSP sendWithCallback(Verb verb, REQ request, CandidateIterator candidates, Retry retryPolicy) + public static RSP sendWithCallback(Verb verb, REQ request, CandidateIterator candidates, WaitStrategy backoff) { try { Promise promise = new AsyncPromise<>(); - sendWithCallbackAsync(promise, verb, request, candidates, retryPolicy); + sendWithCallbackAsync(promise, verb, request, candidates, backoff); return promise.await().get(); } catch (InterruptedException | ExecutionException e) @@ -231,10 +231,10 @@ public final class RemoteProcessor implements Processor } } - public static void sendWithCallbackAsync(Promise promise, Verb verb, REQ request, CandidateIterator candidates, Retry retryPolicy) + public static void sendWithCallbackAsync(Promise promise, Verb verb, REQ request, CandidateIterator candidates, WaitStrategy backoff) { //TODO (now): the retry defines how long to wait for a retry, but the old behavior scheduled the message right away... should this be delayed as well? - MessagingService.instance().sendWithRetries(Backoff.fromRetry(retryPolicy), MessageDelivery.ImmediateRetryScheduler.instance, + MessagingService.instance().sendWithRetries(backoff, MessageDelivery.ImmediateRetryScheduler.instance, verb, request, candidates, (attempt, success, failure) -> { if (failure != null) promise.tryFailure(failure); @@ -263,8 +263,8 @@ public final class RemoteProcessor implements Processor { case NoMoreCandidates: return String.format("Ran out of candidates while sending %s: %s", verb, candidates); - case MaxRetries: - return String.format("Could not succeed sending %s to %s after %d tries", verb, candidates, retryPolicy.tries); + case GiveUp: + return String.format("Could not succeed sending %s to %s; policy %s gave up", verb, candidates, backoff); case Interrupted: case FailedSchedule: return null; diff --git a/src/java/org/apache/cassandra/tcm/Retry.java b/src/java/org/apache/cassandra/tcm/Retry.java index bb5f3dedac..a66f64e113 100644 --- a/src/java/org/apache/cassandra/tcm/Retry.java +++ b/src/java/org/apache/cassandra/tcm/Retry.java @@ -18,235 +18,143 @@ package org.apache.cassandra.tcm; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; -import java.util.function.DoubleSupplier; import com.codahale.metrics.Meter; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.service.RetryStrategy; +import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory; +import org.apache.cassandra.service.WaitStrategy; import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; -public abstract class Retry +// TODO (expected): unwrap this, use RetryStrategy directly +public class Retry implements WaitStrategy { - protected static final int MAX_TRIES = DatabaseDescriptor.getCmsDefaultRetryMaxTries(); - private static final int DEFAULT_BACKOFF_MS = DatabaseDescriptor.getDefaultRetryBackoff().toMilliseconds(); - private static final int DEFAULT_MAX_BACKOFF_MS = DatabaseDescriptor.getDefaultMaxRetryBackoff().toMilliseconds(); + private static final WaitStrategy DEFAULT_STRATEGY; + static + { + DurationSpec.IntMillisecondsBound defaultBackoff = DatabaseDescriptor.getDefaultRetryBackoff(); + DurationSpec.IntMillisecondsBound defaultMaxBackoff = DatabaseDescriptor.getDefaultMaxRetryBackoff(); + String defaultSpec = DatabaseDescriptor.getCMSRetryDelay(); + if (defaultSpec == null || (defaultBackoff != null || defaultMaxBackoff != null)) + { + defaultSpec = (defaultBackoff == null ? "50ms" : defaultBackoff.toMilliseconds() + "ms") + + "*attempts <=" + (defaultMaxBackoff == null ? "10s" : defaultMaxBackoff.toMilliseconds() + "ms") + + ",retries=" + DatabaseDescriptor.getCmsDefaultRetryMaxTries(); + } + DEFAULT_STRATEGY = RetryStrategy.parse(defaultSpec, LatencySourceFactory.none()); + } - protected final int maxTries; - protected int tries; + public final long deadlineNanos; protected Meter retryMeter; + private final WaitStrategy delegate; + int attempts = 1; - public Retry(Meter retryMeter) + public Retry(long deadlineNanos, Meter retryMeter, WaitStrategy delegate) { - this(MAX_TRIES, retryMeter); - } - - public Retry(int maxTries, Meter retryMeter) - { - this.maxTries = maxTries; + this.deadlineNanos = deadlineNanos; this.retryMeter = retryMeter; + this.delegate = delegate; } - public int currentTries() + public Retry(long deadlineNanos, Meter retryMeter) { - return tries; + this(deadlineNanos, retryMeter, DEFAULT_STRATEGY); } - public boolean reachedMax() + public int attempts() { - return tries >= maxTries; + return attempts; } - public void maybeSleep() + public boolean hasExpired() { - sleepUninterruptibly(computeSleepFor(), TimeUnit.MILLISECONDS); + return nanoTime() >= deadlineNanos; } - public long computeSleepFor() + public boolean maybeSleep() + { + long wait = computeWait(attempts, TimeUnit.MILLISECONDS); + if (wait < 0) + return false; + sleepUninterruptibly(wait, TimeUnit.MILLISECONDS); + return true; + } + + @Override + public long computeWaitUntil(int attempts) + { + long wait = computeWaitInternal(attempts, TimeUnit.NANOSECONDS); + if (wait < 0) + return -1; + long now = nanoTime(); + if (now >= deadlineNanos) + return -1; + return Math.min(deadlineNanos, wait + now); + } + + @Override + public long computeWait(int attempts, TimeUnit units) + { + long wait = computeWaitInternal(attempts, TimeUnit.NANOSECONDS); + if (wait < 0) + return -1; + + if (deadlineNanos == Long.MAX_VALUE) + return wait; + + long now = nanoTime(); + wait = Math.min(deadlineNanos - now, wait); + if (wait <= 0) + return -1; + return units.convert(wait, TimeUnit.NANOSECONDS); + } + + private long computeWaitInternal(int attempts, TimeUnit units) { - tries++; retryMeter.mark(); - return sleepFor(); + attempts = Math.max(attempts, ++this.attempts); + return delegate.computeWait(attempts, units); } - protected abstract long sleepFor(); - - protected abstract long maxWait(); - - public static class Jitter extends Retry + // imposes attempt limit + public static Retry withNoTimeLimit(Meter retryMeter) { - private final Random random; - private final int maxJitterMs; - - public Jitter(Meter retryMeter) - { - this(MAX_TRIES, DEFAULT_BACKOFF_MS, new Random(), retryMeter); - } - - private Jitter(int maxTries, int maxJitterMs, Random random, Meter retryMeter) - { - super(maxTries, retryMeter); - this.random = random; - this.maxJitterMs = maxJitterMs; - } - - public long sleepFor() - { - int actualBackoff = ThreadLocalRandom.current().nextInt(maxJitterMs / 2, maxJitterMs); - return random.nextInt(actualBackoff); - } - - @Override - protected long maxWait() - { - return maxJitterMs; - } - - @Override - public String toString() - { - return "Jitter{" + - ", maxTries=" + maxTries + - ", tries=" + tries + - ", maxJitterMs=" + maxJitterMs + - '}'; - } + return new Retry(Long.MAX_VALUE, retryMeter, DEFAULT_STRATEGY); } - public static class Backoff extends Retry + public static Retry withNoTimeLimit(Meter retryMeter, WaitStrategy delegate) { - protected final int backoffMs; - - public Backoff(Meter retryMeter) - { - this(MAX_TRIES, DEFAULT_BACKOFF_MS, retryMeter); - } - - public Backoff(int maxTries, int backoffMs, Meter retryMeter) - { - super(maxTries, retryMeter); - this.backoffMs = backoffMs; - } - - public long sleepFor() - { - return (long) tries * backoffMs; - } - - @Override - protected long maxWait() - { - return backoffMs; - } - - @Override - public String toString() - { - return "Backoff{" + - "backoffMs=" + backoffMs + - ", maxTries=" + maxTries + - ", tries=" + tries + - '}'; - } + return new Retry(Long.MAX_VALUE, retryMeter, delegate); } - public static class ExponentialBackoff extends Retry + public static Retry until(long deadlineNanos, Meter retryMeter) { - private final long baseSleepTimeMillis; - private final long maxSleepMillis; - private final DoubleSupplier randomSource; - - public ExponentialBackoff(int maxAttempts, long baseSleepTimeMillis, long maxSleepMillis, DoubleSupplier randomSource, Meter retryMeter) - { - super(maxAttempts, retryMeter); - this.baseSleepTimeMillis = baseSleepTimeMillis; - this.maxSleepMillis = maxSleepMillis; - this.randomSource = randomSource; - } - - public ExponentialBackoff(Meter retryMeter) - { - this(MAX_TRIES, DEFAULT_BACKOFF_MS, DEFAULT_MAX_BACKOFF_MS, ThreadLocalRandom.current()::nextDouble, retryMeter); - } - - @Override - protected long sleepFor() - { - return org.apache.cassandra.utils.Backoff.ExponentialBackoff.computeWaitTime(tries, baseSleepTimeMillis, maxSleepMillis, randomSource); - } - - @Override - protected long maxWait() - { - return maxSleepMillis; - } + return new Retry(deadlineNanos, retryMeter, DEFAULT_STRATEGY); } - public static class Deadline extends Retry + public static Retry untilElapsed(long timeoutNanos, Meter retryMeter) { - public final long deadlineNanos; - protected final Retry delegate; - - public Deadline(long deadlineNanos, Retry delegate) - { - super(delegate.maxTries, delegate.retryMeter); - assert deadlineNanos > 0 : String.format("Deadline should be strictly positive but was %d.", deadlineNanos); - this.deadlineNanos = deadlineNanos; - this.delegate = delegate; - } - - public static Deadline at(long deadlineNanos, Retry delegate) - { - return new Deadline(deadlineNanos, delegate); - } - - public static Deadline after(long timeoutNanos, Retry delegate) - { - return new Deadline(Clock.Global.nanoTime() + timeoutNanos, delegate); - } - - public static Deadline wrap(Retry delegate) - { - long deadlineMillis = delegate.maxTries * delegate.maxWait(); - return new Deadline(Clock.Global.nanoTime() + TimeUnit.MILLISECONDS.toNanos(deadlineMillis), delegate); - } - - @Override - public boolean reachedMax() - { - return delegate.reachedMax() || Clock.Global.nanoTime() > deadlineNanos; - } - - public long remainingNanos() - { - return Math.max(0, deadlineNanos - Clock.Global.nanoTime()); - } - - @Override - public int currentTries() - { - return delegate.currentTries(); - } - - @Override - public long sleepFor() - { - return delegate.sleepFor(); - } - - @Override - protected long maxWait() - { - return deadlineNanos; - } - - public String toString() - { - return String.format("Deadline{remainingMs=%d, tries=%d/%d}", TimeUnit.NANOSECONDS.toMillis(remainingNanos()), currentTries(), delegate.maxTries); - } + return new Retry(nanoTime() + timeoutNanos, retryMeter, DEFAULT_STRATEGY); } + public static Retry untilElapsed(long timeoutNanos, Meter retryMeter, WaitStrategy waitStrategy) + { + return new Retry(nanoTime() + timeoutNanos, retryMeter, waitStrategy); + } + + public String toString() + { + if (deadlineNanos == Long.MAX_VALUE) + return "RetryIndefinitely{attempts=" + attempts + '}'; + return String.format("Retry{remainingMs=%d, attempts=%d}", TimeUnit.NANOSECONDS.toMillis(remainingNanos()), attempts()); + } + + public long remainingNanos() + { + return Math.max(0, deadlineNanos - nanoTime()); + } } diff --git a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java index ffacb78627..d062d202e6 100644 --- a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java @@ -142,13 +142,13 @@ public class StubClusterMetadataService extends ClusterMetadataService private StubProcessor() {} @Override - public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy) + public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy) { throw new UnsupportedOperationException(); } @Override - public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy) + public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy) { throw new UnsupportedOperationException(); } @@ -160,7 +160,7 @@ public class StubClusterMetadataService extends ClusterMetadataService } @Override - public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy) + public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy) { throw new UnsupportedOperationException(); } diff --git a/src/java/org/apache/cassandra/tcm/migration/GossipProcessor.java b/src/java/org/apache/cassandra/tcm/migration/GossipProcessor.java index be853d89e7..602f52a3a7 100644 --- a/src/java/org/apache/cassandra/tcm/migration/GossipProcessor.java +++ b/src/java/org/apache/cassandra/tcm/migration/GossipProcessor.java @@ -30,13 +30,13 @@ import org.apache.cassandra.tcm.log.LogState; public class GossipProcessor implements Processor { @Override - public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy) + public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy) { throw new IllegalStateException("Can't commit transformations when running in gossip mode. Enable the ClusterMetadataService with `nodetool cms initialize`."); } @Override - public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy) + public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy) { return ClusterMetadata.current(); } @@ -48,7 +48,7 @@ public class GossipProcessor implements Processor } @Override - public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy) + public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy) { throw new IllegalStateException("Can't reconstruct log state when running in gossip mode. Enable the ClusterMetadataService with `nodetool addtocms`."); } diff --git a/src/java/org/apache/cassandra/tcm/sequences/DropAccordTable.java b/src/java/org/apache/cassandra/tcm/sequences/DropAccordTable.java index 46047ef494..68fc2471ac 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/DropAccordTable.java +++ b/src/java/org/apache/cassandra/tcm/sequences/DropAccordTable.java @@ -175,7 +175,7 @@ public class DropAccordTable extends MultiStepOperation // There are retries baked into this call, but trying to handle timeouts more broadly is put on hold as there is active work to define a EpochSyncPoint that should be far cheaper // which would avoid the timeout issues // NOTE: ExclusiveSyncPoint must find all keys in the range, then make sure nothing is blocking them... this causes a lot of IO. EpochSyncPoint just needs to validate that the last txn processed is in the newer epoch, this can work with in-memory state. - AccordService.instance().awaitTableDrop(table.id); + AccordService.instance().awaitDone(table.id, metadata.epoch.getEpoch()); long awaitEndNanos = nanoTime(); logger.info("Wait for Accord to see the drop table was success. " + "Took {} to wait for Accord to learn about the change, then {} to process everything", diff --git a/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java index 8212a57d9d..56eda18283 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java +++ b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java @@ -50,6 +50,8 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.RequestCallbackWithFailure; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.service.RetryStrategy; +import org.apache.cassandra.service.WaitStrategy; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Retry; @@ -58,6 +60,13 @@ import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.concurrent.AsyncPromise; +import static org.apache.cassandra.config.DatabaseDescriptor.getCmsDefaultRetryMaxTries; +import static org.apache.cassandra.config.DatabaseDescriptor.getProgressBarrierBackoff; +import static org.apache.cassandra.config.DatabaseDescriptor.getProgressBarrierDefaultConsistencyLevel; +import static org.apache.cassandra.config.DatabaseDescriptor.getProgressBarrierMinConsistencyLevel; +import static org.apache.cassandra.config.DatabaseDescriptor.getProgressBarrierTimeout; +import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory.none; + /** * ProgressBarrier is responsible for ensuring that epoch visibility plays together with quorum consistency. * @@ -73,10 +82,16 @@ import org.apache.cassandra.utils.concurrent.AsyncPromise; public class ProgressBarrier { private static final Logger logger = LoggerFactory.getLogger(ProgressBarrier.class); - private static final ConsistencyLevel MIN_CL = DatabaseDescriptor.getProgressBarrierMinConsistencyLevel(); - private static final ConsistencyLevel DEFAULT_CL = DatabaseDescriptor.getProgressBarrierDefaultConsistencyLevel(); - private static final long TIMEOUT_MILLIS = DatabaseDescriptor.getProgressBarrierTimeout(TimeUnit.MILLISECONDS); - private static final long BACKOFF_MILLIS = DatabaseDescriptor.getProgressBarrierBackoff(TimeUnit.MILLISECONDS); + private static final ConsistencyLevel MIN_CL = getProgressBarrierMinConsistencyLevel(); + private static final ConsistencyLevel DEFAULT_CL = getProgressBarrierDefaultConsistencyLevel(); + private static final long TIMEOUT_MILLIS = getProgressBarrierTimeout(TimeUnit.MILLISECONDS); + private static final long BACKOFF_MILLIS = getProgressBarrierBackoff(TimeUnit.MILLISECONDS); + private static final WaitStrategy WAIT_STRATEGY; + static + { + WAIT_STRATEGY = RetryStrategy.parse(BACKOFF_MILLIS + "ms" + "*attempts <=" + TIMEOUT_MILLIS + "ms,retries=" + + getCmsDefaultRetryMaxTries(), none()); + } public final Epoch waitFor; // Location of the affected node; used for LOCAL_QUORUM @@ -195,11 +210,8 @@ public class ProgressBarrier requests.add(new WatermarkRequest(peer, messagingService, waitFor)); long start = Clock.Global.nanoTime(); - Retry.Deadline deadline = Retry.Deadline.after(TimeUnit.MILLISECONDS.toNanos(TIMEOUT_MILLIS), - new Retry.Backoff(DatabaseDescriptor.getCmsDefaultRetryMaxTries(), - (int) BACKOFF_MILLIS, - TCMMetrics.instance.progressBarrierRetries)); - while (!deadline.reachedMax()) + Retry deadline = Retry.untilElapsed(TimeUnit.MILLISECONDS.toNanos(TIMEOUT_MILLIS), TCMMetrics.instance.progressBarrierRetries, WAIT_STRATEGY); + while (!deadline.hasExpired()) { for (WatermarkRequest request : requests) request.retry(); @@ -219,7 +231,8 @@ public class ProgressBarrier // No need to try processing until we collect enough nodes to pass all conditions if (collected.size() < maxWaitFor) { - deadline.maybeSleep(); + if (!deadline.maybeSleep()) + break; continue; } diff --git a/src/java/org/apache/cassandra/tcm/sequences/ReconfigureCMS.java b/src/java/org/apache/cassandra/tcm/sequences/ReconfigureCMS.java index 38566812bf..fe1ec17f3d 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/ReconfigureCMS.java +++ b/src/java/org/apache/cassandra/tcm/sequences/ReconfigureCMS.java @@ -343,13 +343,13 @@ public class ReconfigureCMS extends MultiStepOperation>> remaining = ActiveRepairService.instance() .repairPaxosForTopologyChangeAsync(SchemaConstants.METADATA_KEYSPACE_NAME, Collections.singletonList(entireRange), "CMS reconfiguration"); - while (!retry.reachedMax()) + while (true) { Map>, Future> tasks = new HashMap<>(); for (Supplier> supplier : remaining) @@ -377,7 +377,8 @@ public class ReconfigureCMS extends MultiStepOperation extends AbstractFuture { - @SuppressWarnings({ "rawtypes" }) - private static final AtomicReferenceFieldUpdater waitingUpdater = AtomicReferenceFieldUpdater.newUpdater(AsyncFuture.class, WaitQueue.class, "waiting"); - @SuppressWarnings({ "unused" }) - private volatile WaitQueue waiting; - public AsyncFuture() { super(); @@ -100,10 +99,7 @@ public class AsyncFuture extends AbstractFuture if (resultUpdater.compareAndSet(this, current, v)) { if (v != UNCANCELLABLE) - { ListenerList.notify(listenersUpdater, this); - AsyncAwaitable.signalAll(waitingUpdater, this); - } return true; } } @@ -122,6 +118,17 @@ public class AsyncFuture extends AbstractFuture ListenerList.notify(listenersUpdater, this); } + /** + * Logically append {@code newListener} to {@link #listeners} + * (at this stage it is a stack, so we actually prepend) + * + * @param newListener must be either a {@link ListenerList} or {@link GenericFutureListener} + */ + boolean appendListenerIfNotNotifying(ListenerList newListener) + { + return ListenerList.pushIfNotNotifying(listenersUpdater, this, newListener); + } + /** * Support {@link com.google.common.util.concurrent.Futures#transform} natively * @@ -161,14 +168,64 @@ public class AsyncFuture extends AbstractFuture @Override public AsyncFuture await() throws InterruptedException { - //noinspection unchecked - return AsyncAwaitable.await(waitingUpdater, Future::isDone, this); + if (isDone()) + return this; + + Waiting waiting = new Waiting<>(); + if (!appendListenerIfNotNotifying(waiting)) + { + Invariants.require(isDone()); + return this; + } + + while (true) + { + if (isDone()) + return this; + + LockSupport.park(); + + if (Thread.interrupted()) + { + waiting.cancel(); + throw new InterruptedException(); + } + } } @Override public boolean awaitUntil(long nanoTimeDeadline) throws InterruptedException { - return AsyncAwaitable.awaitUntil(waitingUpdater, Future::isDone, this, nanoTimeDeadline); + if (isDone()) + return true; + + Waiting waiting = new Waiting<>(); + if (!appendListenerIfNotNotifying(waiting)) + { + Invariants.require(isDone()); + return true; + } + + while (true) + { + if (isDone()) + return true; + + long wait = nanoTimeDeadline - nanoTime(); + if (wait <= 0) + { + waiting.cancel(); + return false; + } + + LockSupport.parkNanos(wait); + + if (Thread.interrupted()) + { + waiting.cancel(); + throw new InterruptedException(); + } + } } } diff --git a/src/java/org/apache/cassandra/utils/concurrent/ConcurrentLinkedStack.java b/src/java/org/apache/cassandra/utils/concurrent/ConcurrentLinkedStack.java index 5632a10770..bebe1c7434 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/ConcurrentLinkedStack.java +++ b/src/java/org/apache/cassandra/utils/concurrent/ConcurrentLinkedStack.java @@ -38,7 +38,7 @@ public class ConcurrentLinkedStack public void push(T value) { - IntrusiveStack.push(headUpdater, this, (Node)new Node<>(value)); + IntrusiveStack.getAndPush(headUpdater, this, (Node)new Node<>(value)); } public boolean isEmpty() diff --git a/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java b/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java index 5e59de3b17..d27ed598f0 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java +++ b/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java @@ -65,39 +65,51 @@ public class IntrusiveStack> implements Iterable T next; @Inline - protected static > T push(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend) + protected static > T getAndPush(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend) { - return push(headUpdater, owner, prepend, (prev, next) -> { + return getAndPush(headUpdater, owner, prepend, (prev, next) -> { next.next = prev; return next; }); } - protected static > T push(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend, BiFunction combine) + protected static > T getAndPush(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend, BiFunction combine) { while (true) { T head = headUpdater.get(owner); - if (headUpdater.compareAndSet(owner, head, combine.apply(head, prepend))) + T newHead = combine.apply(head, prepend); + if (headUpdater.compareAndSet(owner, head, newHead)) return head; } } + protected static > T pushAndGet(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend, BiFunction combine) + { + while (true) + { + T head = headUpdater.get(owner); + T newHead = combine.apply(head, prepend); + if (head == newHead || headUpdater.compareAndSet(owner, head, newHead)) + return newHead; + } + } + protected interface Setter { public boolean compareAndSet(O owner, T expect, T update); } @Inline - protected static > T push(Function getter, Setter setter, O owner, T prepend) + protected static > T getAndPush(Function getter, Setter setter, O owner, T prepend) { - return push(getter, setter, owner, prepend, (prev, next) -> { + return getAndPush(getter, setter, owner, prepend, (prev, next) -> { next.next = prev; return next; }); } - protected static > T push(Function getter, Setter setter, O owner, T prepend, BiFunction combine) + protected static > T getAndPush(Function getter, Setter setter, O owner, T prepend, BiFunction combine) { while (true) { diff --git a/src/java/org/apache/cassandra/utils/concurrent/ListenerList.java b/src/java/org/apache/cassandra/utils/concurrent/ListenerList.java index 40b908b4e1..1ba3f83d50 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/ListenerList.java +++ b/src/java/org/apache/cassandra/utils/concurrent/ListenerList.java @@ -20,6 +20,7 @@ package org.apache.cassandra.utils.concurrent; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.concurrent.locks.LockSupport; import java.util.function.BiConsumer; import java.util.function.Consumer; import javax.annotation.Nullable; @@ -50,13 +51,22 @@ abstract class ListenerList extends IntrusiveStack> { Notifying result = new Notifying(); result.next = next; - next.next = prev == NOTIFYING ? null : prev; + next.next = prev.next; return result; } next.next = prev; return next; } + static ListenerList pushHeadIfNotNotifying(ListenerList prev, ListenerList next) + { + if (prev instanceof Notifying) + return prev; + + next.next = prev; + return next; + } + /** * Logically append {@code newListener} to {@link #listeners} * (at this stage it is a stack, so we actually prepend) @@ -66,7 +76,12 @@ abstract class ListenerList extends IntrusiveStack> @Inline static void push(AtomicReferenceFieldUpdater updater, T in, ListenerList newListener) { - IntrusiveStack.push(updater, in, newListener, ListenerList::pushHead); + IntrusiveStack.getAndPush(updater, in, newListener, ListenerList::pushHead); + } + + static boolean pushIfNotNotifying(AtomicReferenceFieldUpdater updater, T in, ListenerList newListener) + { + return newListener == IntrusiveStack.pushAndGet(updater, in, newListener, ListenerList::pushHeadIfNotNotifying); } /** @@ -91,13 +106,22 @@ abstract class ListenerList extends IntrusiveStack> if (updater.compareAndSet(in, listeners, NOTIFYING)) { - while (true) + try { - notifyExclusive(listeners, in); - if (updater.compareAndSet(in, NOTIFYING, null)) - return; + while (true) + { + notifyExclusive(listeners, in); + if (updater.compareAndSet(in, NOTIFYING, null)) + return; - listeners = updater.getAndSet(in, NOTIFYING); + listeners = updater.getAndSet(in, NOTIFYING); + } + } + catch (Throwable t) + { + Thread thread = Thread.currentThread(); + try { thread.getUncaughtExceptionHandler().uncaughtException(thread, t); } + catch (Throwable t2) { t.addSuppressed(t2); t.printStackTrace(); } } } } @@ -354,6 +378,7 @@ abstract class ListenerList extends IntrusiveStack> static class Notifying extends ListenerList { static final Notifying NOTIFYING = new Notifying(); + static final Notifying DONE = new Notifying(); @Override void notifySelf(Executor notifyExecutor, Future future) @@ -361,6 +386,27 @@ abstract class ListenerList extends IntrusiveStack> } } + static class Waiting extends ListenerList + { + volatile Thread waiting = Thread.currentThread(); + + @Override + void notifySelf(Executor notifyExecutor, Future future) + { + Thread thread = waiting; + if (thread != null) + { + LockSupport.unpark(thread); + waiting = null; + } + } + + void cancel() + { + waiting = null; + } + } + /** * @return true iff the invoking thread is executing {@code executor} */ diff --git a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java index 8134931a34..45ef68cb81 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java @@ -101,7 +101,7 @@ public class InstanceConfig implements IInstanceConfig .set("accord.journal_directory", accord.journal_directory) .set("accord.queue_shard_count", accord.queue_shard_count.toString()) .set("accord.command_store_shard_count", accord.command_store_shard_count.toString()) - .set("accord.recover_delay", accord.recover_delay.toString()) + .set("accord.expire_txn", accord.expire_txn) .set("accord.enable_virtual_debug_only_keyspace", "true") .set("partitioner", "org.apache.cassandra.dht.Murmur3Partitioner") .set("start_native_transport", true) diff --git a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java index 654196a597..694729f8bd 100644 --- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java +++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java @@ -490,12 +490,12 @@ public class ClusterUtils public static Callable pauseBeforeEnacting(IInvokableInstance instance, long epoch) { - return pauseBeforeEnacting(instance, Epoch.create(epoch), 10, TimeUnit.SECONDS); + return pauseBeforeEnacting(instance, Epoch.create(epoch), 30, TimeUnit.SECONDS); } public static Callable pauseBeforeEnacting(IInvokableInstance instance, Epoch epoch) { - return pauseBeforeEnacting(instance, epoch, 10, TimeUnit.SECONDS); + return pauseBeforeEnacting(instance, epoch, 30, TimeUnit.SECONDS); } protected static Callable pauseBeforeEnacting(IInvokableInstance instance, diff --git a/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java b/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java index 65a1ef8237..e0805620a7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java @@ -66,8 +66,7 @@ public class QueriesTableTest extends TestBaseImpl { SHARED_CLUSTER = init(Cluster.build(1).withInstanceInitializer(QueryDelayHelper::install) .withConfig(c -> c.with(Feature.NATIVE_PROTOCOL, Feature.GOSSIP) - .set("write_request_timeout", "10s") - .set("transaction_timeout", "15s")).start()); + .set("write_request_timeout", "10s")).start()); DRIVER_CLUSTER = JavaDriverUtils.create(SHARED_CLUSTER); SESSION = DRIVER_CLUSTER.connect(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java b/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java index 98b92aa06b..2586531a4f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java @@ -106,7 +106,10 @@ public class ShortReadProtectionTest extends TestBaseImpl // but maybe that is out of scope and is covered by the dedicated BRR tests? cluster = init(Cluster.build() .withNodes(NUM_NODES) - .withConfig(config -> config.set("hinted_handoff_enabled", false)) + .withConfig(config -> + config.set("hinted_handoff_enabled", false) + .set("accord.shard_durability_target_splits", 4) + ) .start()); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java index 29e152a149..ea7dcd8814 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java @@ -109,7 +109,7 @@ public class AccordBootstrapTest extends TestBaseImpl Assertions.assertThat(completed) .describedAs("Epoch %s did not become ready within timeout on %s -> %s", epoch, FBUtilities.getBroadcastAddressAndPort(), - service().configurationService().getEpochSnapshot(epoch)) + service().configService().getEpochSnapshot(epoch)) .isTrue(); } catch (InterruptedException e) @@ -122,7 +122,7 @@ public class AccordBootstrapTest extends TestBaseImpl { try { - AccordConfigurationService configService = service().configurationService(); + AccordConfigurationService configService = service().configService(); boolean completed = configService.unsafeLocalSyncNotified(epoch).await(30, TimeUnit.SECONDS); Assert.assertTrue(String.format("Local sync notification for epoch %s did not become ready within timeout on %s", epoch, FBUtilities.getBroadcastAddressAndPort()), completed); @@ -195,7 +195,7 @@ public class AccordBootstrapTest extends TestBaseImpl Assert.assertEquals(initialMax, ClusterMetadata.current().epoch.getEpoch()); System.out.println("Awaiting " + initialMax); awaitEpoch(initialMax); - AccordConfigurationService configService = service().configurationService(); + AccordConfigurationService configService = service().configService(); long minEpoch = configService.minEpoch(); Assert.assertEquals(initialMax, configService.maxEpoch()); @@ -222,7 +222,7 @@ public class AccordBootstrapTest extends TestBaseImpl node.runOnInstance(() -> { ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(schemaChangeMax)); awaitEpoch(schemaChangeMax); - AccordConfigurationService configService = service().configurationService(); + AccordConfigurationService configService = service().configService(); for (long epoch = initialMax + 1; epoch <= schemaChangeMax; epoch++) { @@ -260,7 +260,7 @@ public class AccordBootstrapTest extends TestBaseImpl Assert.assertEquals(bootstrapMax, ClusterMetadata.current().epoch.getEpoch()); AccordService service = (AccordService) AccordService.instance(); awaitEpoch(bootstrapMax); - AccordConfigurationService configService = service.configurationService(); + AccordConfigurationService configService = service.configService(); awaitLocalSyncNotification(bootstrapMax); Assert.assertEquals(EpochSnapshot.completed(bootstrapMax), configService.getEpochSnapshot(bootstrapMax)); @@ -381,7 +381,7 @@ public class AccordBootstrapTest extends TestBaseImpl node.runOnInstance(() -> { Assert.assertEquals(initialMax, ClusterMetadata.current().epoch.getEpoch()); awaitEpoch(initialMax); - AccordConfigurationService configService = service().configurationService(); + AccordConfigurationService configService = service().configService(); long minEpoch = configService.minEpoch(); Assert.assertEquals(initialMax, configService.maxEpoch()); @@ -404,7 +404,7 @@ public class AccordBootstrapTest extends TestBaseImpl Assert.assertEquals(schemaChangeMax, ClusterMetadata.current().epoch.getEpoch()); AccordService service = (AccordService) AccordService.instance(); awaitEpoch(schemaChangeMax); - AccordConfigurationService configService = service.configurationService(); + AccordConfigurationService configService = service.configService(); for (long epoch = initialMax + 1; epoch <= schemaChangeMax; epoch++) { @@ -439,7 +439,7 @@ public class AccordBootstrapTest extends TestBaseImpl Assert.assertEquals(moveMax, ClusterMetadata.current().epoch.getEpoch()); AccordService service = (AccordService) AccordService.instance(); awaitEpoch(moveMax); - AccordConfigurationService configService = service.configurationService(); + AccordConfigurationService configService = service.configService(); awaitLocalSyncNotification(moveMax); Assert.assertEquals(EpochSnapshot.completed(moveMax), configService.getEpochSnapshot(moveMax)); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropKeyspaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropKeyspaceTest.java index fd9b653a84..26392586c0 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropKeyspaceTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropKeyspaceTest.java @@ -39,8 +39,9 @@ public class AccordDropKeyspaceTest extends AccordDropTableBase try (Cluster cluster = Cluster.build(3) .withoutVNodes() .withConfig(c -> c.with(GOSSIP, NETWORK, NATIVE_PROTOCOL) - .set("auto_snapshot", false)) - .start()) + .set("auto_snapshot", false) + .set("accord.shard_durability_target_splits", 4) + ).start()) { fixDistributedSchemas(cluster); for (int i = 0; i < examples; i++) diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableTest.java index da2669dd5f..15455978b4 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableTest.java @@ -39,6 +39,7 @@ public class AccordDropTableTest extends AccordDropTableBase try (Cluster cluster = Cluster.build(3) .withoutVNodes() .withConfig(c -> c.with(GOSSIP, NETWORK, NATIVE_PROTOCOL) + .set("accord.shard_durability_target_splits", 4) .set("auto_snapshot", false)) .start()) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java index f96c57b649..f164679085 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java @@ -18,13 +18,13 @@ package org.apache.cassandra.distributed.test.accord; -import java.util.List; +import java.util.Collection; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import javax.annotation.Nonnull; +import javax.annotation.Nullable; import com.google.common.collect.Iterables; import org.junit.After; @@ -35,7 +35,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import accord.api.BarrierType; import accord.impl.progresslog.DefaultProgressLogs; import accord.local.Node; import accord.local.PreLoadContext; @@ -43,10 +42,14 @@ import accord.local.SafeCommand; import accord.local.StoreParticipants; import accord.local.cfk.CommandsForKey; import accord.local.cfk.SafeCommandsForKey; -import accord.primitives.Seekables; +import accord.local.durability.DurabilityService; +import accord.primitives.Keys; +import accord.primitives.Ranges; import accord.primitives.Status; +import accord.primitives.Timestamp; import accord.primitives.TxnId; import accord.utils.async.AsyncChains; +import accord.utils.async.AsyncResult; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; @@ -64,7 +67,6 @@ import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.IAccordService.DelegatingAccordService; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.consensus.TransactionalMode; -import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.concurrent.Future; @@ -87,35 +89,21 @@ public class AccordIncrementalRepairTest extends AccordTestBase } @Override - public Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException + public AsyncResult sync(Object requestedBy, @Nullable Timestamp onOrAfter, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) { - Seekables retval = delegate.barrierWithRetries(keysOrRanges, minEpoch, barrierType, isForWrite); - executedBarriers = true; - return retval; + return delegate.sync(requestedBy, onOrAfter, ranges, include, syncLocal, syncRemote).map(v -> { + executedBarriers = true; + return v; + }).beginAsResult(); } @Override - public Seekables barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite) + public AsyncResult sync(@Nullable Timestamp onOrAfter, Keys keys, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) { - Seekables retval = delegate.barrier(keysOrRanges, minEpoch, requestTime, timeoutNanos, barrierType, isForWrite); - executedBarriers = true; - return retval; - } - - @Override - public Seekables repairWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List allEndpoints) throws InterruptedException - { - Seekables retval = delegate.repairWithRetries(keysOrRanges, minEpoch, barrierType, isForWrite, allEndpoints); - executedBarriers = true; - return retval; - } - - @Override - public Seekables repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List allEndpoints) - { - Seekables retval = delegate.repair(keysOrRanges, epoch, requestTime, timeoutNanos, barrierType, isForWrite, allEndpoints); - executedBarriers = true; - return retval; + return delegate.sync(onOrAfter, keys, syncLocal, syncRemote).map(v -> { + executedBarriers = true; + return v; + }).beginAsResult(); } public void reset() @@ -143,7 +131,10 @@ public class AccordIncrementalRepairTest extends AccordTestBase @BeforeClass public static void setupClass() throws Throwable { - setupCluster(opt -> opt.withConfig(conf -> conf.with(Feature.NETWORK, Feature.GOSSIP).set("accord.recover_delay", "1s")), 3); + setupCluster(opt -> opt.withConfig(conf -> conf.with(Feature.NETWORK, Feature.GOSSIP) + .set("accord.recover_txn", "1s") + .set("accord.shard_durability_target_splits", 16) + ), 3); for (IInvokableInstance instance : SHARED_CLUSTER) instance.runOnInstance(() -> AccordService.unsafeSetNewAccordService(new BarrierRecordingService(AccordService.instance()))); // setupCluster(opt -> opt, 3); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java index 0b90db1355..f0f83cb036 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java @@ -46,8 +46,8 @@ import org.apache.cassandra.metrics.RatioGaugeSet; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.accord.AccordService; -import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException; -import org.apache.cassandra.service.accord.exceptions.WritePreemptedException; +import org.apache.cassandra.service.accord.exceptions.AccordReadPreemptedException; +import org.apache.cassandra.service.accord.exceptions.AccordWritePreemptedException; import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.utils.AssertionUtils; import org.assertj.core.api.Assertions; @@ -130,19 +130,23 @@ public class AccordMetricsTest extends AccordTestBase { ScheduledExecutorService exec = Executors.newScheduledThreadPool(1); IMessageFilters.Matcher delay = (from, to, m) -> { - exec.schedule(() -> SHARED_CLUSTER.get(to).receiveMessageWithInvokingThread(m), 10L, TimeUnit.SECONDS); + exec.schedule(() -> { + System.err.println("Receiving..."); + SHARED_CLUSTER.get(to).receiveMessageWithInvokingThread(m); + }, 10L, TimeUnit.SECONDS); return true; }; IMessageFilters.Filter preacceptDelay = SHARED_CLUSTER.filters().outbound().verbs(Verb.ACCORD_PRE_ACCEPT_REQ.id).from(1).to(1) .messagesMatching(delay) .drop(); - long originalAccordRecoverDelay = SHARED_CLUSTER.get(1).callOnInstance(() -> DatabaseDescriptor.getAccordRecoverDelay(TimeUnit.MILLISECONDS)); - SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setAccordRecoverDelay(100L, TimeUnit.MILLISECONDS)); - long originalTransactionTimeoutMillis = SHARED_CLUSTER.get(1).callOnInstance(() -> DatabaseDescriptor.getTransactionTimeout(TimeUnit.MILLISECONDS)); - SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setTransactionTimeout(12_000)); + String originalAccordRetryTxnDelay = SHARED_CLUSTER.get(1).callOnInstance(DatabaseDescriptor::getAccordRecoverTxnDelay); + SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setAccordRecoverTxnDelay("100ms")); + String originalAccordExpireTxnDelay = SHARED_CLUSTER.get(1).callOnInstance(DatabaseDescriptor::getAccordExpireTxnDelay); + SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setAccordExpireTxnDelay("12s")); long originalWriteRpcTimeoutMillis = SHARED_CLUSTER.get(1).callOnInstance(() -> DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.MILLISECONDS)); SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setWriteRpcTimeout(12_000)); + SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setReadRpcTimeout(12_000)); try { @@ -154,7 +158,7 @@ public class AccordMetricsTest extends AccordTestBase } catch (RuntimeException ex) { - Assertions.assertThat(ex).is(AssertionUtils.rootCauseIs(WritePreemptedException.class)); + Assertions.assertThat(ex).is(AssertionUtils.rootCauseIs(AccordWritePreemptedException.class)); } assertCoordinatorMetrics(0, "rw", 0, 0, 1, 0, 0); @@ -172,7 +176,7 @@ public class AccordMetricsTest extends AccordTestBase } catch (RuntimeException ex) { - Assertions.assertThat(ex).is(AssertionUtils.rootCauseIs(ReadPreemptedException.class)); + Assertions.assertThat(ex).is(AssertionUtils.rootCauseIs(AccordReadPreemptedException.class)); } assertCoordinatorMetrics(0, "ro", 0, 0, 1, 0, 0); @@ -184,9 +188,9 @@ public class AccordMetricsTest extends AccordTestBase } finally { - SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setAccordRecoverDelay(originalAccordRecoverDelay, TimeUnit.SECONDS)); + SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setAccordExpireTxnDelay(originalAccordExpireTxnDelay)); + SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setAccordRecoverTxnDelay(originalAccordRetryTxnDelay)); SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setWriteRpcTimeout(originalWriteRpcTimeoutMillis)); - SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setTransactionTimeout(originalTransactionTimeoutMillis)); preacceptDelay.off(); exec.shutdown(); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java index ffb1c4546b..fee7e6af63 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java @@ -647,8 +647,8 @@ public class AccordMigrationTest extends AccordTestBase Integer nextMigratingKey = paxosMigratingKeys.next(); addExpectedMigratedKey(expectedKeyMigrations, nextMigratingKey, tableUUID); - // Paxos migrating keys should be key migrated which means a local barrier is run by Paxos during read at each replica, the key migration barrier is also counted as a write - assertTargetPaxosWrite(runCasNoApply, 1, nextMigratingKey, expectedKeyMigrations, 1, 1, 1, 0, 0); + // Paxos migrating keys should be key migrated which means a local barrier is run by Paxos during read at each replica + assertTargetPaxosWrite(runCasNoApply, 1, nextMigratingKey, expectedKeyMigrations, 0, 1, 1, 0, 0); // A key from a range migrated to Accord is now not migrating/migrated and should be accessed through Accord assertTargetPaxosWrite(runCasNoApply, 1, accordKeys.next(), expectedKeyMigrations, 1, 0, 0, 0, 0); @@ -657,7 +657,7 @@ public class AccordMigrationTest extends AccordTestBase cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new RoutesToAccordOnce())); nextMigratingKey = paxosMigratingKeys.next(); addExpectedMigratedKey(expectedKeyMigrations, nextMigratingKey, tableUUID); - assertTargetPaxosWrite(runCasNoApply, 1, nextMigratingKey, expectedKeyMigrations, 2, 1, 1, 1, 1); + assertTargetPaxosWrite(runCasNoApply, 1, nextMigratingKey, expectedKeyMigrations, 1, 1, 1, 1, 1); // Repair the currently migrating range from when targets were switched, but it's not an Accord repair, this is to make sure the wrong repair type doesn't trigger progress nodetool(coordinator, "repair", "-st", upperMidToken.toString(), "-et", maxAlignedWithLocalRanges.toString(), "--skip-accord"); @@ -666,7 +666,7 @@ public class AccordMigrationTest extends AccordTestBase // Paxos migrating keys should still need key migration after non-Accord repair nextMigratingKey = paxosMigratingKeys.next(); addExpectedMigratedKey(expectedKeyMigrations, nextMigratingKey, tableUUID); - assertTargetPaxosWrite(runCasNoApply, 1, nextMigratingKey, expectedKeyMigrations, 1, 1, 1, 0, 0); + assertTargetPaxosWrite(runCasNoApply, 1, nextMigratingKey, expectedKeyMigrations, 0, 1, 1, 0, 0); // Now do it with an Accord repair so key migration shouldn't be necessary nodetool(coordinator, "consensus_admin", "finish-migration", "-st", upperMidToken.toString(), "-et", maxAlignedWithLocalRanges.toString()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordProgressLogTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordProgressLogTest.java index 545892f696..6bbbce6a52 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordProgressLogTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordProgressLogTest.java @@ -46,7 +46,7 @@ public class AccordProgressLogTest extends TestBaseImpl .withoutVNodes() .withConfig(c -> c.with(Feature.NETWORK) .set("accord.enabled", "true") - .set("accord.recover_delay", "1s")) + .set("accord.recover_txn", "1s")) .start())) { cluster.schemaChange("CREATE KEYSPACE ks WITH replication={'class':'SimpleStrategy', 'replication_factor': 3}"); @@ -85,7 +85,8 @@ public class AccordProgressLogTest extends TestBaseImpl { try (Cluster cluster = init(Cluster.build(2) .withoutVNodes() - .withConfig(c -> c.with(Feature.NETWORK).set("accord.enabled", "true")) + .withConfig(c -> c.with(Feature.NETWORK) + .set("accord.enabled", "true")) .start())) { cluster.schemaChange("CREATE KEYSPACE ks WITH replication={'class':'SimpleStrategy', 'replication_factor': 3}"); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordSimpleFastPathTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordSimpleFastPathTest.java index d074d8affe..cf024f43f8 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordSimpleFastPathTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordSimpleFastPathTest.java @@ -109,7 +109,7 @@ public class AccordSimpleFastPathTest extends TestBaseImpl Assert.assertEquals(idSet(), accordFastPath.unavailableIds()); long epoch = cm.epoch.getEpoch(); - AccordConfigurationService configService = ((AccordService) AccordService.instance()).configurationService(); + AccordConfigurationService configService = ((AccordService) AccordService.instance()).configService(); Topology topology = configService.getTopologyForEpoch(epoch); Assert.assertFalse(topology.shards().isEmpty()); topology.shards().forEach(shard -> Assert.assertEquals(idSet(1, 2, 3), shard.nodes.without(shard.notInFastPath))); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java index 1203470239..b9e7ba46b1 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java @@ -96,8 +96,8 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.api.TokenKey; -import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException; -import org.apache.cassandra.service.accord.exceptions.WritePreemptedException; +import org.apache.cassandra.service.accord.exceptions.AccordReadPreemptedException; +import org.apache.cassandra.service.accord.exceptions.AccordWritePreemptedException; import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; @@ -171,9 +171,9 @@ public abstract class AccordTestBase extends TestBaseImpl ClusterUtils.waitForCMSToQuiesce(SHARED_CLUSTER, 1); SHARED_CLUSTER.forEach(() -> Util.spinUntilTrue(() -> ClusterMetadata.current().epoch.getEpoch() == - ((AccordService) AccordService.instance()).configurationService().currentEpoch() && + ((AccordService) AccordService.instance()).configService().currentEpoch() && AccordService.instance().topology().current().epoch() == - ((AccordService) AccordService.instance()).configurationService().currentEpoch(), + ((AccordService) AccordService.instance()).configService().currentEpoch(), 60)); } @@ -380,13 +380,11 @@ public abstract class AccordTestBase extends TestBaseImpl .withConfig(c -> c.with(Feature.GOSSIP) .set("sasi_indexes_enabled", "true") .set("write_request_timeout", "10s") - .set("transaction_timeout", "15s") .set("native_transport_timeout", "30s") .set("cms_await_timeout", "1s") .set("cms_default_max_retries", 10_000) .set("accord.ephemeral_read_enabled", "false") - .set("accord.shard_durability_target_splits", "1") - .set("accord.shard_durability_cycle", "60s") + .set("accord.shard_durability_target_splits", "4") .set("accord.command_store_shard_count", "2") .set("accord.queue_shard_count", "2")) .withInstanceInitializer(EnforceUpdateDoesNotPerformRead::install); @@ -469,7 +467,7 @@ public abstract class AccordTestBase extends TestBaseImpl } catch (RuntimeException ex) { - if (count <= MAX_RETRIES && (hasRootCause(ex, ReadPreemptedException.class) || hasRootCause(ex, WritePreemptedException.class) || hasRootCause(ex, Invalidated.class))) + if (count <= MAX_RETRIES && (hasRootCause(ex, AccordReadPreemptedException.class) || hasRootCause(ex, AccordWritePreemptedException.class) || hasRootCause(ex, Invalidated.class))) { logger.warn("[Retry attempt={}] Preempted failure for\n{}", count, check); return executeWithRetry0(count + 1, cluster, inst, check, boundValues); @@ -489,19 +487,12 @@ public abstract class AccordTestBase extends TestBaseImpl private static TxnId maybeExtractId(Throwable ex) { - if (hasRootCause(ex, ReadPreemptedException.class) - || hasRootCause(ex, WritePreemptedException.class) + if (hasRootCause(ex, AccordReadPreemptedException.class) + || hasRootCause(ex, AccordWritePreemptedException.class) || hasRootCause(ex, ReadTimeoutException.class) || hasRootCause(ex, WriteTimeoutException.class)) { - try - { - return TxnId.parse(ex.getMessage()); - } - catch (Throwable t) - { - // ignore - } + return TxnId.tryParse(ex.getMessage()); } return null; } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java index 9553377847..45a62d3a97 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java @@ -735,7 +735,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase new Processor() { @Override - public Commit.Result commit(Entry.Id entryId, Transformation event, Epoch lastKnown, Retry.Deadline retryPolicy) + public Commit.Result commit(Entry.Id entryId, Transformation event, Epoch lastKnown, Retry retryPolicy) { if (lastKnown == null) lastKnown = log.waitForHighestConsecutive().epoch; @@ -749,7 +749,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase } @Override - public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy) + public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy) { Epoch since = log.waitForHighestConsecutive().epoch; LogState logState = driver.requestResponse(new FetchCMSLog(since, true)); @@ -764,7 +764,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase } @Override - public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy) + public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy) { return getLocalState(start, end, includeSnapshot); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ReconstructEpochTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/ReconstructEpochTest.java index a38050a895..b1b1411c58 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ReconstructEpochTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ReconstructEpochTest.java @@ -24,8 +24,6 @@ import java.util.concurrent.TimeUnit; import org.junit.Assert; import org.junit.Test; -import com.codahale.metrics.Meter; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.metrics.TCMMetrics; @@ -35,7 +33,8 @@ import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Retry; import org.apache.cassandra.tcm.log.Entry; import org.apache.cassandra.tcm.log.LogState; -import org.apache.cassandra.utils.Clock; + +import static org.apache.cassandra.config.DatabaseDescriptor.getCmsAwaitTimeout; public class ReconstructEpochTest extends TestBaseImpl { @@ -82,7 +81,7 @@ public class ReconstructEpochTest extends TestBaseImpl .getLogState(Epoch.create(start), Epoch.create(end), true, - unsafeRetryIndefinitely()); + Retry.untilElapsed(getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.commitRetries)); Assert.assertEquals(start, logState.baseState.epoch.getEpoch()); Iterator iter = logState.entries.iterator(); @@ -92,31 +91,4 @@ public class ReconstructEpochTest extends TestBaseImpl }); } } - - private static Retry.Deadline unsafeRetryIndefinitely() - { - long timeoutNanos = DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS); - Meter retryMeter = TCMMetrics.instance.commitRetries; - return new Retry.Deadline(Clock.Global.nanoTime() + timeoutNanos, - new Retry.Jitter(retryMeter)) - { - @Override - public boolean reachedMax() - { - return false; - } - - @Override - public long remainingNanos() - { - return timeoutNanos; - } - - public String toString() - { - return String.format("RetryIndefinitely{tries=%d}", currentTries()); - } - }; - } - } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/TestProcessor.java b/test/distributed/org/apache/cassandra/distributed/test/log/TestProcessor.java index 2b2906bc07..27ddbd18be 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/TestProcessor.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/TestProcessor.java @@ -55,7 +55,7 @@ public class TestProcessor implements Processor } @Override - public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy) + public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy) { maybePause(transform); waitIfPaused(); @@ -65,7 +65,7 @@ public class TestProcessor implements Processor } @Override - public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy) + public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy) { return delegate.fetchLogAndWait(waitFor, retryPolicy); } @@ -77,7 +77,7 @@ public class TestProcessor implements Processor } @Override - public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy) + public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy) { return delegate.getLogState(start, end, includeSnapshot, retryPolicy); } diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java index 395f20b0d5..5fbb93efca 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java @@ -29,7 +29,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; -import org.junit.Ignore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +37,6 @@ import accord.coordinate.Preempted; import accord.coordinate.Timeout; import accord.local.Node; import accord.primitives.Ranges; -import accord.primitives.Seekables; import accord.primitives.TxnId; import accord.utils.Gen; import accord.utils.Gens; @@ -85,7 +83,6 @@ import static org.apache.cassandra.utils.AbstractTypeGenerators.overridePrimitiv import static org.apache.cassandra.utils.AbstractTypeGenerators.stringComparator; import static org.apache.cassandra.utils.AccordGenerators.fromQT; -@Ignore public class AccordTopologyMixupTest extends TopologyMixupTestBase { private static final Logger logger = LoggerFactory.getLogger(AccordTopologyMixupTest.class); @@ -272,6 +269,7 @@ public class AccordTopologyMixupTest extends TopologyMixupTestBase keysOrRanges, Throwable cause) + public InterceptAgent() { - if (cause instanceof Timeout || cause instanceof Preempted) - { - SharedState.debugTxn(null, "Repair Barrier", id.toString()); - } + super(); } + { + AccordAgent.setOnFailedBarrier((id, cause) -> { + if (cause instanceof Timeout || cause instanceof Preempted) + { + SharedState.debugTxn(null, "Repair Barrier", id.toString()); + } + }); + } @Override - public void onFailedBootstrap(String phase, Ranges ranges, Runnable retry, Throwable failure) + public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Throwable failure) { if (failure instanceof Exhausted) { Exhausted e = (Exhausted) failure; SharedState.debugTxn(self.id, "Bootstrap#" + phase, e.txnId().toString()); } - super.onFailedBootstrap(phase, ranges, retry, failure); + super.onFailedBootstrap(attempts, phase, ranges, retry, failure); } } } diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/HarryOnAccordTopologyMixupTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/HarryOnAccordTopologyMixupTest.java index cadb631490..f133540146 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/HarryOnAccordTopologyMixupTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/HarryOnAccordTopologyMixupTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.fuzz.topology; -import org.junit.Ignore; import accord.utils.Gen; import accord.utils.Invariants; @@ -30,7 +29,6 @@ import org.apache.cassandra.distributed.shared.ClusterUtils; import org.apache.cassandra.fuzz.topology.AccordTopologyMixupTest.ListenerHolder; import org.apache.cassandra.service.consensus.TransactionalMode; -@Ignore public class HarryOnAccordTopologyMixupTest extends HarryTopologyMixupTest { static @@ -77,6 +75,7 @@ public class HarryOnAccordTopologyMixupTest extends HarryTopologyMixupTest super.onConfigure(config); config.set("accord.command_store_shard_count", 1) .set("accord.queue_shard_count", 1) + .set("accord.shard_durability_target_splits", 4) .set("concurrent_accord_operations", 1); } diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java b/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java index 1ea20e4ca3..f38ad8108c 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java @@ -537,7 +537,6 @@ public abstract class TopologyMixupTestBase depsGen() { Gen keyDepsGen = AccordGenerators.keyDepsGen(DatabaseDescriptor.getPartitioner()); - return AccordGens.deps(keyDepsGen::next, - (rs) -> Deps.NONE.rangeDeps, - (rs) -> Deps.NONE.directKeyDeps); + return AccordGens.deps(keyDepsGen::next, (rs) -> Deps.NONE.rangeDeps); } } \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index 06e25aa509..4447cdb93c 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -304,6 +304,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.security.AbstractCryptoProvider", "org.apache.cassandra.tcm.RegistrationStateCallbacks", "org.apache.cassandra.service.consensus.TransactionalMode", + "org.apache.cassandra.service.WaitStrategy", "org.apache.cassandra.transport.ProtocolException", "org.apache.cassandra.utils.Closeable", "org.apache.cassandra.utils.CloseableIterator", diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java index c52e82921f..dcd617adcf 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java @@ -383,7 +383,6 @@ public class DatabaseDescriptorTest testConfig.cas_contention_timeout = lowerThanLowestTimeout; testConfig.counter_write_request_timeout = lowerThanLowestTimeout; testConfig.request_timeout = lowerThanLowestTimeout; - testConfig.transaction_timeout = lowerThanLowestTimeout; DatabaseDescriptor.checkForLowestAcceptedTimeouts(testConfig); @@ -394,7 +393,6 @@ public class DatabaseDescriptorTest assertEquals(testConfig.cas_contention_timeout, DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT); assertEquals(testConfig.counter_write_request_timeout, DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT); assertEquals(testConfig.request_timeout, DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT); - assertEquals(testConfig.transaction_timeout, DatabaseDescriptor.LOWEST_ACCEPTED_TIMEOUT); } @Test diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java index 85f5d94832..2c7a260269 100644 --- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java +++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java @@ -509,15 +509,11 @@ public class YamlConfigurationLoaderTest public void testAccordConfig() { Map accordSpec = ImmutableMap.of("fast_path_update_delay", "60s", - "default_durability_retry_delay", "60s", - "max_durability_retry_delay", "60s", "durability_txnid_lag", "60s", "shard_durability_cycle", "60s", "global_durability_cycle", "60s"); AccordSpec spec = from("accord", accordSpec).accord; assertThat(spec.fast_path_update_delay.to(TimeUnit.NANOSECONDS)).isEqualTo(60000000000L); - assertThat(spec.default_durability_retry_delay.to(TimeUnit.NANOSECONDS)).isEqualTo(60000000000L); - assertThat(spec.max_durability_retry_delay.to(TimeUnit.NANOSECONDS)).isEqualTo(60000000000L); assertThat(spec.durability_txnid_lag.to(TimeUnit.NANOSECONDS)).isEqualTo(60000000000L); assertThat(spec.shard_durability_cycle.to(TimeUnit.NANOSECONDS)).isEqualTo(60000000000L); assertThat(spec.global_durability_cycle.to(TimeUnit.NANOSECONDS)).isEqualTo(60000000000L); diff --git a/test/unit/org/apache/cassandra/db/virtual/AccordVirtualTablesTest.java b/test/unit/org/apache/cassandra/db/virtual/AccordVirtualTablesTest.java index 0239b34ae2..dc01f9f37f 100644 --- a/test/unit/org/apache/cassandra/db/virtual/AccordVirtualTablesTest.java +++ b/test/unit/org/apache/cassandra/db/virtual/AccordVirtualTablesTest.java @@ -167,7 +167,7 @@ public class AccordVirtualTablesTest extends CQLTester return sorter; } }; - TopologyManager tm = new TopologyManager(supplier, null, N1, null, null, null); + TopologyManager tm = new TopologyManager(supplier, null, N1, null, null); var mock = Mockito.mock(IAccordService.class); Mockito.when(mock.topology()).thenReturn(tm); diff --git a/test/unit/org/apache/cassandra/hints/HintsServiceTest.java b/test/unit/org/apache/cassandra/hints/HintsServiceTest.java index 45658705c2..78aa85c59e 100644 --- a/test/unit/org/apache/cassandra/hints/HintsServiceTest.java +++ b/test/unit/org/apache/cassandra/hints/HintsServiceTest.java @@ -31,6 +31,8 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import accord.primitives.Keys; +import accord.primitives.TxnId; import com.datastax.driver.core.utils.MoreFutures; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.DatabaseDescriptor; @@ -47,9 +49,12 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.AccordResult; import org.apache.cassandra.service.accord.AccordTestUtils; -import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult; +import org.apache.cassandra.service.accord.IAccordService.IAccordResult; +import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping; import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.transport.Dispatcher; @@ -229,12 +234,13 @@ public class HintsServiceTest } @Override - public AsyncTxnResult mutateWithAccordAsync(ClusterMetadata cm, Mutation mutation, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + public IAccordResult mutateWithAccordAsync(ClusterMetadata cm, Mutation mutation, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { accordTxnCount.incrementAndGet(); - AsyncTxnResult asyncTxnResult = new AsyncTxnResult(AccordTestUtils.txnId(42, 43, 44), 42, consistencyLevel, true, requestTime); - asyncTxnResult.setSuccess(new TxnData()); - return asyncTxnResult; + TxnId txnId = AccordTestUtils.txnId(42, 43, 44); + AccordResult result = new AccordResult<>(txnId, Keys.EMPTY, new LatencyRequestBookkeeping(null), requestTime.startedAtNanos(), requestTime.startedAtNanos(), true); + result.accept(new TxnData(), null); + return result; } }); sendHintsWithRetryDifferentSystemUUID(metadata); diff --git a/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java b/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java index 2af230b466..8451ba609d 100644 --- a/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java +++ b/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java @@ -482,7 +482,7 @@ public class RouteIndexTest extends CQLTester.InMemory storeRangesForEpochs.put(i, new RangesForEpoch(1, Ranges.of(TokenRange.fullRange(tableId, getPartitioner())))); accordService = startAccord(); - accordService.configurationService().listener.notifyPostCommit(null, ClusterMetadata.current(), false); + accordService.configService().listener.notifyPostCommit(null, ClusterMetadata.current(), false); accordService.epochReady(ClusterMetadata.current().epoch).awaitUninterruptibly(); } diff --git a/test/unit/org/apache/cassandra/net/MessageDeliveryTest.java b/test/unit/org/apache/cassandra/net/MessageDeliveryTest.java index cd90f9b977..73b773e92d 100644 --- a/test/unit/org/apache/cassandra/net/MessageDeliveryTest.java +++ b/test/unit/org/apache/cassandra/net/MessageDeliveryTest.java @@ -40,12 +40,13 @@ import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessageDelivery.FailedResponseException; -import org.apache.cassandra.net.MessageDelivery.MaxRetriesException; +import org.apache.cassandra.net.MessageDelivery.GivingUpException; import org.apache.cassandra.net.SimulatedMessageDelivery.Action; import org.apache.cassandra.net.SimulatedMessageDelivery.SimulatedMessageReceiver; +import org.apache.cassandra.service.RetryStrategy; +import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.StubClusterMetadataService; -import org.apache.cassandra.utils.Backoff; import org.mockito.Mockito; import static accord.utils.Property.qt; @@ -74,7 +75,7 @@ public class MessageDeliveryTest MessageDelivery messaging = simulatedMessages(rs, scheduler, failures, (i1, i2, i3) -> Action.DROP); int expectedRetries = 3; - Backoff backoff = new Backoff.ExponentialBackoff(expectedRetries, 200, 1000, rs.fork()::nextDouble); + RetryStrategy backoff = RetryStrategy.parse("200ms*2^attempts <= 1000ms,retries=" + expectedRetries, LatencySourceFactory.none()); Future> result = messaging.sendWithRetries(backoff, scheduler::schedule, @@ -86,7 +87,7 @@ public class MessageDeliveryTest factory.processAll(); assertThat(result).isDone(); - assertThat(getMaxRetriesException(result).attempts).isEqualTo(expectedRetries); + assertThat(getMaxRetriesException(result).attempts).isEqualTo(expectedRetries + 1); }); } @@ -99,7 +100,7 @@ public class MessageDeliveryTest ScheduledExecutorPlus scheduler = factory.scheduled("ignored"); MessageDelivery messaging = simulatedMessages(rs, scheduler, failures, (i1, i2, i3) -> Action.DELIVER); - Backoff backoff = Mockito.mock(Backoff.class); + RetryStrategy backoff = Mockito.mock(RetryStrategy.class); Future> result = messaging.sendWithRetries(backoff, scheduler::schedule, @@ -111,9 +112,7 @@ public class MessageDeliveryTest factory.processAll(); assertThat(result).isDone(); assertThat(result.get().header.verb).isEqualTo(Verb.ECHO_RSP); - Mockito.verify(backoff, Mockito.never()).mayRetry(Mockito.anyInt()); - Mockito.verify(backoff, Mockito.never()).computeWaitTime(Mockito.anyInt()); - Mockito.verify(backoff, Mockito.never()).unit(); + Mockito.verify(backoff, Mockito.never()).computeWait(Mockito.anyInt(), Mockito.any()); }); } @@ -130,7 +129,8 @@ public class MessageDeliveryTest AtomicInteger attempts = new AtomicInteger(0); MessageDelivery messaging = simulatedMessages(rs, scheduler, failures, (i1, i2, i3) -> attempts.incrementAndGet() >= (expectedAttempts + 1) ? Action.DELIVER : Action.DROP); - Backoff backoff = Mockito.spy(new Backoff.ExponentialBackoff(maxAttempts, 200, 1000, rs.fork()::nextDouble)); + RetryStrategy backoff = RetryStrategy.parse("200ms*2^attempts <= 1000ms,retries=" + (maxAttempts-1), LatencySourceFactory.none()); + backoff = Mockito.spy(backoff); Future> result = messaging.sendWithRetries(backoff, scheduler::schedule, @@ -142,9 +142,7 @@ public class MessageDeliveryTest factory.processAll(); assertThat(result).isDone(); assertThat(result.get().header.verb).isEqualTo(Verb.ECHO_RSP); - Mockito.verify(backoff, Mockito.times(expectedAttempts)).mayRetry(Mockito.anyInt()); - Mockito.verify(backoff, Mockito.times(expectedAttempts)).computeWaitTime(Mockito.anyInt()); - Mockito.verify(backoff, Mockito.times(expectedAttempts)).unit(); + Mockito.verify(backoff, Mockito.times(expectedAttempts)).computeWait(Mockito.anyInt(), Mockito.any()); }); } @@ -158,7 +156,8 @@ public class MessageDeliveryTest MessageDelivery messaging = simulatedMessages(rs, scheduler, failures, (i1, i2, i3) -> Action.DROP); - Backoff backoff = Mockito.spy(new Backoff.ExponentialBackoff(3, 200, 1000, rs.fork()::nextDouble)); + RetryStrategy backoff = RetryStrategy.parse("0 <= 200ms*2^attempts <= 1000ms,retries=3", LatencySourceFactory.none()); + backoff = Mockito.spy(backoff); Future> result = messaging.sendWithRetries(backoff, scheduler::schedule, @@ -172,9 +171,7 @@ public class MessageDeliveryTest FailedResponseException e = getFailedResponseException(result); assertThat(e.from).isEqualTo(ID1); assertThat(e.failure).isEqualTo(RequestFailure.TIMEOUT); - Mockito.verify(backoff, Mockito.times(1)).mayRetry(Mockito.anyInt()); - Mockito.verify(backoff, Mockito.never()).computeWaitTime(Mockito.anyInt()); - Mockito.verify(backoff, Mockito.never()).unit(); + Mockito.verify(backoff, Mockito.atMostOnce()).computeWait(Mockito.anyInt(), Mockito.any()); }); } @@ -212,9 +209,9 @@ public class MessageDeliveryTest return messaging; } - private static MaxRetriesException getMaxRetriesException(Future> result) throws InterruptedException + private static GivingUpException getMaxRetriesException(Future> result) throws InterruptedException { - MaxRetriesException ex; + GivingUpException ex; try { result.get(); @@ -223,7 +220,7 @@ public class MessageDeliveryTest } catch (ExecutionException e) { - ex = (MaxRetriesException) e.getCause(); + ex = (GivingUpException) e.getCause(); } return ex; } diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairMessageTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairMessageTest.java index 7bbea89f12..b7fa71c959 100644 --- a/test/unit/org/apache/cassandra/repair/messages/RepairMessageTest.java +++ b/test/unit/org/apache/cassandra/repair/messages/RepairMessageTest.java @@ -32,9 +32,11 @@ import org.apache.cassandra.net.MessageDelivery; import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.SharedContext; +import org.apache.cassandra.service.RetryStrategy; +import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory; +import org.apache.cassandra.service.WaitStrategy; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.StubClusterMetadataService; -import org.apache.cassandra.utils.Backoff; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; import org.assertj.core.api.Assertions; @@ -53,7 +55,7 @@ public class RepairMessageTest private static final Answer REJECT_ALL = ignore -> { throw new UnsupportedOperationException(); }; - private static final int[] attempts = {1, 2, 10}; + private static final int[] retries = { 1, 2, 10 }; // Tests may use verb / message pairs that do not make sense... that is due to the fact that the message sending logic does not validate this and delegates such validation to messaging, which is mocked within the class... // By using messages with simpler state it makes the test easier to read, even though the verb -> message mapping is incorrect. private static final Verb VERB = Verb.PREPARE_MSG; @@ -104,18 +106,18 @@ public class RepairMessageTest @Test public void retryWithTimeout() { - test((maxAttempts, callback) -> { + test((maxRetries, callback) -> { callback.onFailure(ADDRESS, RequestFailure.TIMEOUT); - assertMetrics(maxAttempts, true, false); + assertMetrics(maxRetries, true, false); }); } @Test public void retryWithFailure() { - test((maxAttempts, callback) -> { + test((maxRetries, callback) -> { callback.onFailure(ADDRESS, RequestFailure.UNKNOWN); - assertMetrics(maxAttempts, false, true); + assertMetrics(maxRetries, false, true); }); } @@ -124,9 +126,9 @@ public class RepairMessageTest assertMetrics(0, false, false); } - private void assertMetrics(long attempts, boolean timeout, boolean failure) + private void assertMetrics(long retries, boolean timeout, boolean failure) { - if (attempts == 0) + if (retries == 0) { assertThat(RepairMetrics.retries).isEmpty(); assertThat(RepairMetrics.retriesByVerb.get(VERB)).isEmpty(); @@ -137,8 +139,8 @@ public class RepairMessageTest } else { - assertThat(RepairMetrics.retries).hasCount(1).hasMax(attempts); - assertThat(RepairMetrics.retriesByVerb.get(VERB)).hasCount(1).hasMax(attempts); + assertThat(RepairMetrics.retries).hasCount(1).hasMax(retries); + assertThat(RepairMetrics.retriesByVerb.get(VERB)).hasCount(1).hasMax(retries); assertThat(RepairMetrics.retryTimeout).hasCount(timeout ? 1 : 0); assertThat(RepairMetrics.retryTimeoutByVerb.get(VERB)).hasCount(timeout ? 1 : 0); assertThat(RepairMetrics.retryFailure).hasCount(failure ? 1 : 0); @@ -146,9 +148,9 @@ public class RepairMessageTest } } - private static Backoff backoff(int maxAttempts) + private static WaitStrategy backoff(int maxRetries) { - return new Backoff.ExponentialBackoff(maxAttempts, 100, 1000, () -> .5); + return RetryStrategy.parse("0 <= 100ms * 2^attempts <= 1000ms,retries=" + maxRetries, LatencySourceFactory.none()); } private static SharedContext ctx() @@ -194,22 +196,22 @@ public class RepairMessageTest private void test(TestCase fn) { - test(attempts, fn); + test(retries, fn); } - private void test(int[] attempts, TestCase fn) + private void test(int[] retries, TestCase fn) { SharedContext ctx = ctx(); MessageDelivery messaging = ctx.messaging(); - for (int maxAttempts : attempts) + for (int maxRetries : retries) { before(); - sendMessageWithRetries(ctx, backoff(maxAttempts), always(), PAYLOAD, VERB, ADDRESS, RepairMessage.NOOP_CALLBACK); - for (int i = 0; i < maxAttempts; i++) + sendMessageWithRetries(ctx, backoff(maxRetries), always(), PAYLOAD, VERB, ADDRESS, RepairMessage.NOOP_CALLBACK); + for (int i = 0; i < maxRetries; i++) callback(messaging).onFailure(ADDRESS, RequestFailure.TIMEOUT); - fn.test(maxAttempts, callback(messaging)); + fn.test(maxRetries, callback(messaging)); Mockito.verifyNoInteractions(messaging); } } diff --git a/test/unit/org/apache/cassandra/service/RetryStrategyTest.java b/test/unit/org/apache/cassandra/service/RetryStrategyTest.java new file mode 100644 index 0000000000..26c3633a5b --- /dev/null +++ b/test/unit/org/apache/cassandra/service/RetryStrategyTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service; + +import java.util.function.IntFunction; + +import org.junit.Test; + +import accord.utils.Gen; +import accord.utils.RandomTestRunner; + +public class RetryStrategyTest +{ + @Test + public void fuzzParser() + { + Gen> expressionGen = random -> i -> { + switch (i) + { + case 0: return String.format("p%d * %d", random.nextInt(100), random.nextInt(0, Integer.MAX_VALUE)); + case 1: return String.format("p%d * %d * attempts", random.nextInt(100), random.nextInt(0, Integer.MAX_VALUE)); + case 2: return String.format("p%d * attempts", random.nextInt(100)); + case 3: return String.format("p%d * %d ^ attempts", random.nextInt(100), random.nextInt(0, Integer.MAX_VALUE)); + case 4: return String.format("%dms", random.nextInt(0, Integer.MAX_VALUE)); + default: + throw new IllegalArgumentException(Integer.toString(i)); + } + }; + + RandomTestRunner.test().check(rs -> { + IntFunction expression = expressionGen.next(rs); + for (int i = 0; i < 10_000; i++) + { + RetryStrategy.parse(String.format("%dms <= %s", rs.nextInt(0, Integer.MAX_VALUE), expression.apply(rs.nextInt(5))), + new TestLatencySourceFactory()); + RetryStrategy.parse(String.format("%dms <= %s <= %dms", + rs.nextInt(0, Integer.MAX_VALUE), + expression.apply(rs.nextInt(4)), + rs.nextInt(0, Integer.MAX_VALUE)), + new TestLatencySourceFactory()); + RetryStrategy.parse(String.format("%dms <= %s ... %s <= %dms", + rs.nextInt(0, Integer.MAX_VALUE), + expression.apply(rs.nextInt(4)), + expression.apply(rs.nextInt(4)), + rs.nextInt(0, Integer.MAX_VALUE)), + new TestLatencySourceFactory()); + } + }); + } + + private static class TestLatencySourceFactory implements TimeoutStrategy.LatencySourceFactory + { + + @Override + public TimeoutStrategy.LatencySource source(String params) + { + return percentile -> 10; + } + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java index 98506a9056..6faca72d34 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java @@ -129,7 +129,7 @@ public class AccordCommandStoreTest Timestamp executeAt = timestamp(1, clock.incrementAndGet(), 1); SimpleBitSet waitingOnApply = new SimpleBitSet(3); waitingOnApply.set(1); - Command.WaitingOn waitingOn = new Command.WaitingOn(dependencies.keyDeps.keys(), dependencies.rangeDeps, dependencies.directKeyDeps, new ImmutableBitSet(waitingOnApply), new ImmutableBitSet(2)); + Command.WaitingOn waitingOn = new Command.WaitingOn(dependencies.keyDeps.keys(), dependencies.rangeDeps, new ImmutableBitSet(waitingOnApply), new ImmutableBitSet(2)); Pair result = AccordTestUtils.processTxnResult(commandStore, txnId, txn, executeAt); Command expected = Command.Executed.executed(txnId, SaveStatus.Applied, Majority, StoreParticipants.all(route), diff --git a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java index e15a4e502a..f609b2ad73 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java @@ -111,7 +111,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory RoutingKey routingKey = partialTxn.keys().get(0).asKey().toUnseekable(); FullRoute route = partialTxn.keys().toRoute(routingKey); StoreParticipants participants = StoreParticipants.all(route); - Deps deps = new Deps(KeyDeps.none(((Keys) txn.keys()).toParticipants()), RangeDeps.NONE, KeyDeps.NONE); + Deps deps = new Deps(KeyDeps.none(((Keys) txn.keys()).toParticipants()), RangeDeps.NONE); Command.WaitingOn waitingOn = null; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordServiceTest.java b/test/unit/org/apache/cassandra/service/accord/AccordServiceTest.java deleted file mode 100644 index afdea31883..0000000000 --- a/test/unit/org/apache/cassandra/service/accord/AccordServiceTest.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.service.accord; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.function.Supplier; - -import org.junit.Test; - -import accord.api.BarrierType; -import accord.coordinate.Exhausted; -import accord.coordinate.Preempted; -import accord.coordinate.Timeout; -import accord.impl.IntKey; -import accord.primitives.Ranges; -import accord.primitives.Seekables; -import accord.primitives.TxnId; -import org.apache.cassandra.utils.Blocking; -import org.assertj.core.api.Condition; -import org.mockito.Mockito; - -import static accord.utils.Property.qt; -import static org.apache.cassandra.service.accord.AccordService.doWithRetries; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -public class AccordServiceTest -{ - @Test - public void retryExpectedFailures() throws InterruptedException - { - Blocking blocking = Mockito.mock(Blocking.class); - class Task implements Supplier - { - private int attempts = 0; - - @Override - public Seekables get() - { - switch (attempts) - { - case 0: - attempts++; - throw new Timeout(null, null); - case 1: - attempts++; - throw AccordService.newBarrierTimeout(TxnId.NONE, BarrierType.local, true, Ranges.EMPTY); - case 2: - attempts++; - throw new Preempted(null, null); - case 3: - attempts++; - throw AccordService.newBarrierPreempted(TxnId.NONE, BarrierType.local, true, Ranges.EMPTY); - case 4: - attempts++; - throw new Exhausted(null, null, null); - case 5: - attempts++; - throw AccordService.newBarrierExhausted(TxnId.NONE, BarrierType.local, true, Ranges.EMPTY); - default: - return Ranges.of(IntKey.range(1, 2)); - } - } - } - Task failing = new Task(); - assertThat(doWithRetries(blocking, failing, Integer.MAX_VALUE, 100, 1000)).isEqualTo(Ranges.of(IntKey.range(1,2))); - verify(blocking).sleep(100); - verify(blocking).sleep(200); - verify(blocking).sleep(400); - verify(blocking).sleep(800); - verify(blocking, times(2)).sleep(1000); // hit max backoff, so stays at 1k - } - - @Test - public void retryThrowsTimeout() - { - Blocking blocking = Mockito.mock(Blocking.class); - qt().check(rs -> { - List timeoutFailures = new ArrayList<>(4); - timeoutFailures.add(() -> {throw new Timeout(null, null);}); - timeoutFailures.add(() -> {throw AccordService.newBarrierTimeout(TxnId.NONE, BarrierType.local, true, Ranges.EMPTY);}); - timeoutFailures.add(() -> {throw new Preempted(null, null);}); - timeoutFailures.add(() -> {throw AccordService.newBarrierPreempted(TxnId.NONE, BarrierType.local, true, Ranges.EMPTY);}); - Collections.shuffle(timeoutFailures, rs.asJdkRandom()); - Iterator it = timeoutFailures.iterator(); - Supplier failing = () -> { - if (!it.hasNext()) throw new IllegalStateException("Called too many times"); - it.next().run(); // this throws... - return Ranges.EMPTY; - }; - assertThatThrownBy(() -> doWithRetries(blocking, failing, timeoutFailures.size(), 100, 1000)).is(new Condition<>(AccordService::isTimeout, "timeout")); - assertThat(it).isExhausted(); - }); - } - - @Test - public void retryThrowsNonTimeout() - { - Blocking blocking = Mockito.mock(Blocking.class); - qt().check(rs -> { - List timeoutFailures = new ArrayList<>(5); - timeoutFailures.add(() -> {throw new Timeout(null, null);}); - timeoutFailures.add(() -> {throw AccordService.newBarrierTimeout(TxnId.NONE, BarrierType.local, true, Ranges.EMPTY);}); - timeoutFailures.add(() -> {throw new Preempted(null, null);}); - timeoutFailures.add(() -> {throw AccordService.newBarrierPreempted(TxnId.NONE, BarrierType.local, true, Ranges.EMPTY);}); - timeoutFailures.add(() -> {throw new Exhausted(null, null, null);}); - Collections.shuffle(timeoutFailures, rs.asJdkRandom()); - Iterator it = timeoutFailures.iterator(); - Supplier failing = () -> { - if (!it.hasNext()) throw new IllegalStateException("Called too many times"); - it.next().run(); // this throws... - return Ranges.EMPTY; - }; - assertThatThrownBy(() -> doWithRetries(blocking, failing, timeoutFailures.size(), 100, 1000)).isInstanceOf(Exhausted.class); - assertThat(it).isExhausted(); - }); - } - - @Test - public void retryShortCircuitError() - { - class Unexpected implements Runnable - { - final boolean isError; - - Unexpected(boolean isError) - { - this.isError = isError; - } - - @Override - public void run() - { - if (isError) throw new AssertionError(); - throw new NullPointerException(); - } - } - qt().check(rs -> { - List failures = new ArrayList<>(6); - failures.add(() -> {throw new Timeout(null, null);}); - failures.add(() -> {throw AccordService.newBarrierTimeout(TxnId.NONE, BarrierType.local, true, Ranges.EMPTY);}); - failures.add(() -> {throw new Preempted(null, null);}); - failures.add(() -> {throw AccordService.newBarrierPreempted(TxnId.NONE, BarrierType.local, true, Ranges.EMPTY);}); - failures.add(() -> {throw new Exhausted(null, null, null);}); - boolean isError = rs.nextBoolean(); - failures.add(new Unexpected(isError)); - Collections.shuffle(failures, rs.asJdkRandom()); - int unexpectedIndex = -1; - for (int i = 0; i < failures.size(); i++) - { - if (failures.get(i) instanceof Unexpected) - { - unexpectedIndex = i; - break; - } - } - Iterator it = failures.iterator(); - Supplier failing = () -> { - if (!it.hasNext()) throw new IllegalStateException("Called too many times"); - it.next().run(); // this throws... - return Ranges.EMPTY; - }; - Blocking blocking = Mockito.mock(Blocking.class); - assertThatThrownBy(() -> doWithRetries(blocking, failing, failures.size(), 100, 1000)).isInstanceOf(isError ? AssertionError.class : NullPointerException.class); - verify(blocking, times(unexpectedIndex)).sleep(Mockito.anyLong()); - }); - } -} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/service/accord/AccordSyncPropagatorTest.java b/test/unit/org/apache/cassandra/service/accord/AccordSyncPropagatorTest.java index 1385edc1e7..08f5b8401f 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordSyncPropagatorTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordSyncPropagatorTest.java @@ -132,7 +132,7 @@ public class AccordSyncPropagatorTest scheduler.schedule(() -> { Ranges subrange = Ranges.of(range); inst.propagator.reportClosed(epoch, nodes, subrange); - scheduler.schedule(() -> inst.propagator.reportRedundant(epoch, nodes, subrange), 1, TimeUnit.MINUTES); + scheduler.schedule(() -> inst.propagator.reportRetired(epoch, nodes, subrange), 1, TimeUnit.MINUTES); }, rs.nextInt(30, 300), TimeUnit.SECONDS); } } @@ -449,10 +449,10 @@ public class AccordSyncPropagatorTest } @Override - public void reportEpochRedundant(Ranges ranges, long epoch) + public void reportEpochRetired(Ranges ranges, long epoch) { Topology topology = getTopologyForEpoch(epoch); - instances.get(localId).propagator.reportRedundant(epoch, topology.nodes(), ranges); + instances.get(localId).propagator.reportRetired(epoch, topology.nodes(), ranges); } @Override diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index f4cecfab04..0cb57ad9c1 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -54,6 +54,7 @@ import accord.local.PreLoadContext; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; import accord.local.TimeService; +import accord.local.durability.DurabilityService; import accord.primitives.Ballot; import accord.primitives.FullKeyRoute; import accord.primitives.FullRoute; @@ -116,7 +117,6 @@ import static org.apache.cassandra.service.accord.AccordExecutor.wrap; public class AccordTestUtils { - private static final AccordAgent AGENT = new AccordAgent(); public static final TableId TABLE_ID1 = TableId.fromString("00000000-0000-0000-0000-000000000001"); public static class Commands @@ -326,13 +326,12 @@ public class AccordTestUtils public static AccordCommandStore createAccordCommandStore( Node.Id node, LongSupplier now, Topology topology, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor) { - AccordAgent agent = new AccordAgent(); - AccordExecutor executor = new AccordExecutorSyncSubmit(0, RUN_WITH_LOCK, CommandStore.class.getSimpleName() + '[' + 0 + ']', new AccordCacheMetrics("test"), loadExecutor, saveExecutor, loadExecutor, agent); - return createAccordCommandStore(node, now, topology, agent, executor); + AccordExecutor executor = new AccordExecutorSyncSubmit(0, RUN_WITH_LOCK, CommandStore.class.getSimpleName() + '[' + 0 + ']', new AccordCacheMetrics("test"), loadExecutor, saveExecutor, loadExecutor, new AccordAgent()); + return createAccordCommandStore(node, now, topology, executor); } public static AccordCommandStore createAccordCommandStore( - Node.Id node, LongSupplier now, Topology topology, AccordAgent agent, AccordExecutor executor) + Node.Id node, LongSupplier now, Topology topology, AccordExecutor executor) { NodeCommandStoreService time = new NodeCommandStoreService() { @@ -340,16 +339,16 @@ public class AccordTestUtils @Override public Timeouts timeouts() { return null; } @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; } + @Override public DurabilityService durability() { return null; } @Override public Id id() { return node;} @Override public long epoch() {return 1; } @Override public long now() {return now.getAsLong(); } - @Override public Timestamp uniqueNow() { return uniqueNow(Timestamp.NONE); } - @Override public Timestamp uniqueNow(Timestamp atLeast) { return Timestamp.fromValues(1, now.getAsLong(), node); } + @Override public long uniqueNow(long atLeast) { return now.getAsLong(); } @Override public long elapsed(TimeUnit timeUnit) { return elapsed.applyAsLong(timeUnit); } @Override public TopologyManager topology() { throw new UnsupportedOperationException(); } }; - + AccordAgent agent = new AccordAgent(); if (new File(DatabaseDescriptor.getAccordJournalDirectory()).exists()) ServerTestUtils.cleanupDirectory(DatabaseDescriptor.getAccordJournalDirectory()); AccordSpec.JournalSpec spec = new AccordSpec.JournalSpec(); diff --git a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java index d9b70a7068..6a03368717 100644 --- a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java +++ b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java @@ -50,8 +50,7 @@ import org.slf4j.LoggerFactory; import accord.api.ConfigurationService; import accord.api.ConfigurationService.EpochReady; import accord.api.Journal; -import accord.api.LocalConfig; -import accord.api.Scheduler; +import accord.impl.DefaultTimeouts; import accord.impl.SizeOfIntersectionSorter; import accord.impl.TestAgent; import accord.local.Node; @@ -669,7 +668,8 @@ public class EpochSyncTest this.token = token; this.epoch = epoch; // TODO (review): Should there be a real scheduler here? Is it possible to adapt the Scheduler interface to scheduler used in this test? - this.topology = new TopologyManager(SizeOfIntersectionSorter.SUPPLIER, new TestAgent.RethrowAgent(), id, Scheduler.NEVER_RUN_SCHEDULED, TimeService.ofNonMonotonic(globalExecutor::currentTimeMillis, TimeUnit.MILLISECONDS), LocalConfig.DEFAULT); + TimeService time = TimeService.ofNonMonotonic(globalExecutor::currentTimeMillis, TimeUnit.MILLISECONDS); + this.topology = new TopologyManager(SizeOfIntersectionSorter.SUPPLIER, new TestAgent.RethrowAgent(), id, time, new DefaultTimeouts(time)); AccordConfigurationService.DiskStateManager instance = MockDiskStateManager.instance; Journal journal = null; // TODO config = new AccordConfigurationService(node, messagingService, failureDetector, instance, scheduler); @@ -699,12 +699,6 @@ public class EpochSyncTest topology.onEpochSyncComplete(node, epoch); } - @Override - public void onRemoveNode(long epoch, Node.Id removed) - { - topology.onRemoveNode(epoch, removed); - } - @Override public void truncateTopologyUntil(long epoch) { diff --git a/test/unit/org/apache/cassandra/service/accord/FetchMinEpochTest.java b/test/unit/org/apache/cassandra/service/accord/FetchMinEpochTest.java index f06df7e5e3..5c1e2c5a81 100644 --- a/test/unit/org/apache/cassandra/service/accord/FetchMinEpochTest.java +++ b/test/unit/org/apache/cassandra/service/accord/FetchMinEpochTest.java @@ -35,7 +35,6 @@ import accord.utils.Gen; import accord.utils.Gens; import accord.utils.RandomSource; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.RetrySpec; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.IVersionedSerializers; import org.apache.cassandra.io.util.DataOutputBuffer; @@ -43,6 +42,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessageDelivery; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.SimulatedMessageDelivery.Action; +import org.apache.cassandra.service.accord.api.AccordWaitStrategies; import org.apache.cassandra.utils.SimulatedMiniCluster; import org.apache.cassandra.utils.SimulatedMiniCluster.Node; import org.apache.cassandra.utils.concurrent.Future; @@ -65,7 +65,7 @@ public class FetchMinEpochTest private static void boundedRetries(int retries) { - DatabaseDescriptor.getAccord().minEpochSyncRetry.maxAttempts = new RetrySpec.MaxAttempt(retries); + AccordWaitStrategies.setRetryFetchMinEpoch("0<=200ms*2^attempts<=60s,retries=" + retries); } @Test @@ -85,7 +85,8 @@ public class FetchMinEpochTest public void fetchOneNodeAlwaysFails() { int expectedMaxAttempts = 3; - boundedRetries(expectedMaxAttempts); + int expectedMaxRetries = expectedMaxAttempts - 1; + boundedRetries(expectedMaxRetries); qt().check(rs -> { SimulatedMiniCluster cluster = new SimulatedMiniCluster.Builder(rs, node -> msg -> {throw new IllegalStateException();}).build(); Node from = cluster.createNodeAndJoin(); @@ -95,7 +96,7 @@ public class FetchMinEpochTest assertThat(f).isNotDone(); cluster.processAll(); assertThat(f).isDone(); - MessageDelivery.MaxRetriesException maxRetries = getMaxRetriesException(f); + MessageDelivery.GivingUpException maxRetries = getMaxRetriesException(f); Assertions.assertThat(maxRetries.attempts).isEqualTo(expectedMaxAttempts); }); } @@ -212,9 +213,9 @@ public class FetchMinEpochTest } - private static MessageDelivery.MaxRetriesException getMaxRetriesException(Future f) throws InterruptedException, ExecutionException + private static MessageDelivery.GivingUpException getMaxRetriesException(Future f) throws InterruptedException, ExecutionException { - MessageDelivery.MaxRetriesException maxRetries; + MessageDelivery.GivingUpException maxRetries; try { f.get(); @@ -223,9 +224,9 @@ public class FetchMinEpochTest } catch (ExecutionException e) { - if (e.getCause() instanceof MessageDelivery.MaxRetriesException) + if (e.getCause() instanceof MessageDelivery.GivingUpException) { - maxRetries = (MessageDelivery.MaxRetriesException) e.getCause(); + maxRetries = (MessageDelivery.GivingUpException) e.getCause(); } else { diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 489751ef93..9fb2c7173b 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -55,6 +55,7 @@ import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.TimeService; +import accord.local.durability.DurabilityService; import accord.messages.BeginRecovery; import accord.messages.PreAccept; import accord.messages.Reply; @@ -67,7 +68,6 @@ import accord.primitives.Routable; import accord.primitives.RoutableKey; import accord.primitives.Route; import accord.primitives.RoutingKeys; -import accord.primitives.Timestamp; import accord.primitives.Txn; import accord.primitives.TxnId; import accord.primitives.Unseekables; @@ -173,9 +173,9 @@ public class SimulatedAccordCommandStore implements AutoCloseable @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; } @Override - public Timestamp uniqueNow() + public DurabilityService durability() { - return uniqueNow(Timestamp.NONE); + return null; } @Override @@ -203,10 +203,10 @@ public class SimulatedAccordCommandStore implements AutoCloseable } @Override - public Timestamp uniqueNow(Timestamp atLeast) + public long uniqueNow(long atLeast) { - var now = Timestamp.fromValues(epoch(), now(), nodeId); - if (now.compareTo(atLeast) < 0) + long now = now(); + if (now <= atLeast) throw new UnsupportedOperationException(); return now; } @@ -221,9 +221,9 @@ public class SimulatedAccordCommandStore implements AutoCloseable TestAgent.RethrowAgent agent = new TestAgent.RethrowAgent() { @Override - public long preAcceptTimeout() + public boolean rejectPreAccept(TimeService time, TxnId txnId) { - return Long.MAX_VALUE; + return false; } @Override diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index b27b58c03c..01299f4e58 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -66,12 +66,14 @@ import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; +import accord.local.TimeService; import accord.local.cfk.CommandsForKey; import accord.local.cfk.CommandsForKey.InternalStatus; import accord.local.cfk.CommandsForKey.TxnInfo; import accord.local.cfk.CommandsForKey.Unmanaged; import accord.local.cfk.SafeCommandsForKey; import accord.local.cfk.Serialize; +import accord.local.durability.DurabilityService; import accord.messages.ReplyContext; import accord.primitives.Ballot; import accord.primitives.KeyDeps; @@ -185,7 +187,7 @@ public class CommandsForKeySerializerTest { for (TxnId id : deps) keyBuilder.add(((Key)txn.keys().get(0)).toUnseekable(), id); - builder.partialDeps(new PartialDeps(AccordTestUtils.fullRange(txn), keyBuilder.build(), RangeDeps.NONE, KeyDeps.NONE)); + builder.partialDeps(new PartialDeps(AccordTestUtils.fullRange(txn), keyBuilder.build(), RangeDeps.NONE)); } } @@ -455,8 +457,7 @@ public class CommandsForKeySerializerTest final Supplier kindSupplier = () -> { float v = source.nextFloat(); if (v < 0.5) return Kind.Read; - if (v < 0.95) return Kind.Write; - if (v < 0.97) return Kind.SyncPoint; + if (v < 0.97) return Kind.Write; return Kind.ExclusiveSyncPoint; }; @@ -657,22 +658,27 @@ public class CommandsForKeySerializerTest @Override public AsyncChain build(Callable task) { throw new UnsupportedOperationException(); } @Override public void onRecover(Node node, Result success, Throwable fail) { throw new UnsupportedOperationException(); } @Override public void onInconsistentTimestamp(Command command, Timestamp prev, Timestamp next) { throw new UnsupportedOperationException(); } - @Override public void onFailedBootstrap(String phase, Ranges ranges, Runnable retry, Throwable failure) { throw new UnsupportedOperationException(); } + @Override public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Throwable failure) { throw new UnsupportedOperationException(); } @Override public void onStale(Timestamp staleSince, Ranges ranges) { throw new UnsupportedOperationException(); } @Override public void onUncaughtException(Throwable t) { throw new UnsupportedOperationException(); } @Override public void onCaughtException(Throwable t, String context) { throw new UnsupportedOperationException(); } - @Override public long preAcceptTimeout() { throw new UnsupportedOperationException(); } + @Override public boolean rejectPreAccept(TimeService time, TxnId txnId) { throw new UnsupportedOperationException(); } @Override public long cfkHlcPruneDelta() { return 0; } @Override public int cfkPruneInterval() { return 0; } @Override public long maxConflictsHlcPruneDelta() { return 0; } @Override public long maxConflictsPruneInterval() { return 0; } @Override public Txn emptySystemTxn(Kind kind, Routable.Domain domain) { throw new UnsupportedOperationException(); } - @Override public long attemptCoordinationDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int retryCount) { return 0; } - @Override public long seekProgressDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int retryCount, ProgressLog.BlockedUntil blockedUntil, TimeUnit units) { return 0; } - @Override public long retryAwaitTimeout(Node node, SafeCommandStore safeStore, TxnId txnId, int retryCount, ProgressLog.BlockedUntil retrying, TimeUnit units) { return 0; } - @Override public long localSlowAt(TxnId txnId, Status.Phase phase, TimeUnit unit) { return 0; } - @Override public long localExpiresAt(TxnId txnId, Status.Phase phase, TimeUnit unit) { return 0; } + @Override public long slowCoordinatorDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int retryCount) { return 0; } + @Override public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int retryCount, ProgressLog.BlockedUntil blockedUntil, TimeUnit units) { return 0; } + @Override public long slowAwaitDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int retryCount, ProgressLog.BlockedUntil retrying, TimeUnit units) { return 0; } + @Override public long retrySyncPointDelay(Node node, int attempt, TimeUnit units) { return 0; } + @Override public long retryDurabilityDelay(Node node, int attempt, TimeUnit units) { return 0; } + @Override public long expireEpochWait(TimeUnit units) { return 0; } @Override public long expiresAt(ReplyContext replyContext, TimeUnit unit) { return 0; } + @Override public long selfSlowAt(TxnId txnId, Status.Phase phase, TimeUnit unit) { return 0; } + @Override public long selfExpiresAt(TxnId txnId, Status.Phase phase, TimeUnit unit) { return 0; } + @Override public AsyncChain awaitStaleId(Node node, TxnId staleId, boolean isRequested) { return null; } + @Override public long minStaleHlc(Node node, boolean requested) { return 0; } } public static class TestSafeCommandStore extends AbstractSafeCommandStore @@ -696,8 +702,8 @@ public class CommandsForKeySerializerTest @Override public Node.Id id() { return Node.Id.NONE; } @Override public Timeouts timeouts() { return null; } @Override public DurableBefore durableBefore() { return null;} - @Override public Timestamp uniqueNow() { return null; } - @Override public Timestamp uniqueNow(Timestamp atLeast) { return null; } + @Override public DurabilityService durability() { return null; } + @Override public long uniqueNow(long atLeast) { return 0; } @Override public TopologyManager topology() { return null; } @Override public long now() { return 0; } @Override public long elapsed(TimeUnit unit) { return 0; } diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/TokenKeyTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/TokenKeyTest.java index 9bdb85c268..9cecb33186 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/TokenKeyTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/TokenKeyTest.java @@ -71,7 +71,9 @@ public class TokenKeyTest { ByteBuffer buffer = serializer.serialize(key); TokenKey roundTrip = serializer.deserialize(buffer, partitioner); + TokenKey roundTrip2 = serializer.deserializeAndConsume(buffer, partitioner); Assertions.assertThat(roundTrip).isEqualTo(key); + Assertions.assertThat(roundTrip2).isEqualTo(key); } { TokenKey roundTrip = serializer.deserializeWithPrefixAndImpliedLength(key.prefix(), serializer.serializeWithoutPrefixOrLength(key), partitioner); diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/WaitingOnSerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/WaitingOnSerializerTest.java index e481e83940..aa97ee534b 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/WaitingOnSerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/WaitingOnSerializerTest.java @@ -67,7 +67,7 @@ public class WaitingOnSerializerTest } try (DataInputBuffer buf = new DataInputBuffer(bb, true)) { - PartialDeps deps = new PartialDeps(RoutingKeys.EMPTY, KeyDeps.none(waitingOn.keys), waitingOn.directRangeDeps, waitingOn.directKeyDeps); + PartialDeps deps = new PartialDeps(RoutingKeys.EMPTY, KeyDeps.none(waitingOn.keys), waitingOn.directRangeDeps); Command.WaitingOn read = WaitingOnSerializer.deserializeProvider(txnId, buf).provide(txnId, deps, null, 0); Assertions.assertThat(read).isEqualTo(waitingOn); Assertions.assertThat(buf.available()).isEqualTo(0); @@ -90,7 +90,7 @@ public class WaitingOnSerializerTest return rs -> { Deps deps = depsGen.next(rs); if (deps.isEmpty()) return Command.WaitingOn.empty(Routable.Domain.Key); - int txnIdCount = deps.rangeDeps.txnIdCount() + deps.directKeyDeps.txnIdCount(); + int txnIdCount = deps.rangeDeps.txnIdCount(); int keyCount = deps.keyDeps.keys().size(); int[] selected = Gens.arrays(Gens.ints().between(0, txnIdCount + keyCount - 1)).unique().ofSizeBetween(0, txnIdCount + keyCount).next(rs); SimpleBitSet waitingOn = new SimpleBitSet(txnIdCount + keyCount, false); @@ -111,7 +111,7 @@ public class WaitingOnSerializerTest } } - return new Command.WaitingOn(deps.keyDeps.keys(), deps.rangeDeps, deps.directKeyDeps, Utils.ensureImmutable(waitingOn), Utils.ensureImmutable(appliedOrInvalidated)); + return new Command.WaitingOn(deps.keyDeps.keys(), deps.rangeDeps, Utils.ensureImmutable(waitingOn), Utils.ensureImmutable(appliedOrInvalidated)); }; } } \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/tcm/ValidatingClusterMetadataService.java b/test/unit/org/apache/cassandra/tcm/ValidatingClusterMetadataService.java index 0f9ddf0f06..8c2dc5a2ec 100644 --- a/test/unit/org/apache/cassandra/tcm/ValidatingClusterMetadataService.java +++ b/test/unit/org/apache/cassandra/tcm/ValidatingClusterMetadataService.java @@ -120,13 +120,13 @@ public class ValidatingClusterMetadataService extends StubClusterMetadataService return new Processor() { @Override - public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy) + public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy) { return delegate.commit(entryId, transform, lastKnown, retryPolicy); } @Override - public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy) + public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy) { return delegate.fetchLogAndWait(waitFor, retryPolicy); } @@ -145,7 +145,7 @@ public class ValidatingClusterMetadataService extends StubClusterMetadataService } @Override - public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry.Deadline retryPolicy) + public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry retryPolicy) { return getLocalState(lowEpoch, highEpoch, includeSnapshot); } diff --git a/test/unit/org/apache/cassandra/test/asserts/ExtendedAssertions.java b/test/unit/org/apache/cassandra/test/asserts/ExtendedAssertions.java index fc6040e1b9..f5bdd7ff72 100644 --- a/test/unit/org/apache/cassandra/test/asserts/ExtendedAssertions.java +++ b/test/unit/org/apache/cassandra/test/asserts/ExtendedAssertions.java @@ -77,7 +77,7 @@ public class ExtendedAssertions isNotNull(); Snapshot snapshot = actual.getSnapshot(); if (snapshot.getMax() != expected) - throw failure("Expected max %d but given %d", expected, actual.getCount()); + throw failure("Expected max %d but given %d", expected, snapshot.getMax()); return this; } } diff --git a/test/unit/org/apache/cassandra/utils/AccordGenerators.java b/test/unit/org/apache/cassandra/utils/AccordGenerators.java index f97cdcf5d9..1b6079fdfe 100644 --- a/test/unit/org/apache/cassandra/utils/AccordGenerators.java +++ b/test/unit/org/apache/cassandra/utils/AccordGenerators.java @@ -448,7 +448,7 @@ public class AccordGenerators public static Gen depsGen(IPartitioner partitioner) { - return AccordGens.deps(keyDepsGen(partitioner), rangeDepsGen(partitioner), directKeyDepsGen(partitioner)); + return AccordGens.deps(keyDepsGen(partitioner), rangeDepsGen(partitioner)); } public static Gen redundantBeforeEntry(IPartitioner partitioner) diff --git a/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestPromise.java b/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestPromise.java index 982d42df5d..35a43ca2ac 100644 --- a/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestPromise.java +++ b/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestPromise.java @@ -50,7 +50,7 @@ public abstract class AbstractTestPromise } catch (Throwable t) { - throw new AssertionError("" + i, t); + throw new AssertionError(i + ": " + waitingOn.get(i), t); } } }