natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Integrate RX with Cassandra Repair, so that repair safely flushes pending topology and other durability requirements. 

Also improve:
 - Introduce DurabilityService
 - Retire SyncPoint, replace Barrier with Write and RX
 - MessageType -> enum, restore GetMaxConflict
 - Standardise backoff logic with WaitStrategy
 - improve TimeoutStrategy/RetryStrategy specification strings
 - Forbid KX, remove directKeyDeps
 - Introduce UniqueTimeService, permitting hlc reservations for sync points avoid delay when min TxnId is sufficiently in the past
 - Remove ListStore custom purge logic
Also fix:
 - RejectBefore should reject on both epoch and hlc
 - Do not record sync success for removed nodes
 - Support GlobalDurability detecting no command store to run on
 - Incorrect ballot constructor
 - Serializing 15-bit ballot flags incorrectly
 - TopologyManager.hasEpoch deadlock
 - Computing withOpenEpochs incorrectly, sometimes stopping one epoch short
 - PartitionKey serializer should not depend on schema information that can be erased

patch by Benedict; reviewed by Alex Petrov for CASSANDRA-20395
This commit is contained in:
Benedict Elliott Smith 2025-02-03 21:08:42 +00:00 committed by David Capwell
parent a12f0ac9db
commit 430c55dbc4
137 changed files with 2855 additions and 2942 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/accord

@ -1 +1 @@
Subproject commit 2b9e54004f702c7b626e94391af21fa080292975
Subproject commit f9dc83e404253645db02d54c913174b796fae123

9
src/java/org/apache/cassandra/batchlog/BatchlogManager.java
View File

@ -75,9 +75,7 @@ import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.WriteResponseHandler;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.service.accord.IAccordService.IAccordResult;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitMutations;
@ -400,7 +398,7 @@ public class BatchlogManager implements BatchlogManagerMBean
private final ClusterMetadata cm;
private List<ReplayWriteResponseHandler<Mutation>> replayHandlers = ImmutableList.of();
private AsyncTxnResult accordResult;
private IAccordResult<TxnResult> accordResult;
@Nullable
private Dispatcher.RequestTime accordTxnStart;
@ -452,8 +450,7 @@ public class BatchlogManager implements BatchlogManagerMBean
{
if (accordResult != null)
{
IAccordService accord = AccordService.instance();
TxnResult.Kind kind = accord.getTxnResult(accordResult).kind();
TxnResult.Kind kind = accordResult.awaitAndGet().kind();
if (kind == retry_new_protocol)
throw new RetryOnDifferentSystemException();
}

69
src/java/org/apache/cassandra/config/AccordSpec.java
View File

@ -20,13 +20,11 @@ package org.apache.cassandra.config;
import java.util.concurrent.TimeUnit;
import accord.primitives.TxnId;
import accord.utils.Invariants;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.cassandra.journal.Params;
import org.apache.cassandra.service.consensus.TransactionalMode;
import static accord.primitives.Routable.Domain.Range;
import static org.apache.cassandra.config.AccordSpec.QueueShardModel.THREAD_POOL_PER_SHARD;
import static org.apache.cassandra.config.AccordSpec.QueueSubmissionModel.SYNC;
@ -136,43 +134,32 @@ public class AccordSpec
public DataStorageSpec.LongMebibytesBound working_set_size = null;
public boolean shrink_cache_entries_before_eviction = true;
// TODO (expected): we should be able to support lower recover delays, at least for txns
public volatile DurationSpec.IntMillisecondsBound recover_delay = new DurationSpec.IntMillisecondsBound(5000);
public volatile DurationSpec.IntMillisecondsBound range_syncpoint_recover_delay = new DurationSpec.IntMillisecondsBound("5m");
public String slowPreAccept = "30ms <= p50*2 <= 100ms";
public String slowRead = "30ms <= p50*2 <= 100ms";
public DurationSpec.IntMillisecondsBound range_syncpoint_timeout = new DurationSpec.IntMillisecondsBound("3m");
public DurationSpec.IntMillisecondsBound repair_timeout = new DurationSpec.IntMillisecondsBound("10m");
public String recover_txn = "5s*attempts <= 60s";
public String recover_syncpoint = "60s <= 30s*attempts...60s*attempts <= 600s";
public String fetch_txn = "1s*attempts";
public String fetch_syncpoint = "5s*attempts";
public String expire_txn = "5s*attempts";
public String expire_syncpoint = "60s*attempts<=300s";
public String expire_epoch_wait = "10s";
// we don't want to wait ages for durability as it blocks other durability progress; even this might be too long, as we can always retry
public String expire_durability = "10s*attempts <= 30s";
public String slow_syncpoint_preaccept = "10s";
public String slow_txn_preaccept = "30ms <= p50*2 <= 100ms";
public String slow_read = "30ms <= p50*2 <= 100ms";
public String retry_syncpoint = "10s*attempts <= 600s";
public String retry_durability = "10s*attempts <= 600s";
public String retry_fetch_min_epoch = "200ms...1s*attempts <= 1s,retries=3";
public String retry_fetch_topology = "200ms...1s*attempts <= 1s,retries=100";
public long recoveryDelayFor(TxnId txnId, TimeUnit unit)
{
if (txnId.isSyncPoint() && txnId.is(Range))
return range_syncpoint_recover_delay.to(unit);
return recover_delay.to(unit);
}
/**
* When a barrier transaction is requested how many times to repeat attempting the barrier before giving up
*/
public int barrier_retry_attempts = 5;
/**
* When a barrier transaction fails how long the initial backoff should be before being increased
* as part of exponential backoff on each attempt
*/
public DurationSpec.IntMillisecondsBound barrier_retry_inital_backoff_millis = new DurationSpec.IntMillisecondsBound("1s");
public DurationSpec.IntMillisecondsBound barrier_max_backoff = new DurationSpec.IntMillisecondsBound("10m");
public DurationSpec.IntMillisecondsBound range_syncpoint_timeout = new DurationSpec.IntMillisecondsBound("2m");
public volatile DurationSpec.IntSecondsBound fast_path_update_delay = new DurationSpec.IntSecondsBound("60m");
public volatile DurationSpec.IntSecondsBound fast_path_update_delay = null;
public volatile DurationSpec.IntSecondsBound gc_delay = new DurationSpec.IntSecondsBound("5m");
public volatile int shard_durability_target_splits = 128;
public volatile DurationSpec.IntSecondsBound durability_txnid_lag = new DurationSpec.IntSecondsBound(5);
public volatile DurationSpec.IntSecondsBound shard_durability_cycle = new DurationSpec.IntSecondsBound(15, TimeUnit.MINUTES);
public volatile DurationSpec.IntSecondsBound global_durability_cycle = new DurationSpec.IntSecondsBound(10, TimeUnit.MINUTES);
public volatile DurationSpec.IntSecondsBound default_durability_retry_delay = new DurationSpec.IntSecondsBound(10, TimeUnit.SECONDS);
public volatile DurationSpec.IntSecondsBound max_durability_retry_delay = new DurationSpec.IntSecondsBound(10, TimeUnit.MINUTES);
public enum TransactionalRangeMigration
{
@ -194,24 +181,6 @@ public class AccordSpec
public boolean ephemeralReadEnabled = true;
public boolean state_cache_listener_jfr_enabled = true;
public final JournalSpec journal = new JournalSpec();
public final RetrySpec minEpochSyncRetry = new MinEpochRetrySpec();
public final RetrySpec fetchRetry = new FetchRetrySpec();
public static class MinEpochRetrySpec extends RetrySpec
{
public MinEpochRetrySpec()
{
maxAttempts = new MaxAttempt(3);
}
}
public static class FetchRetrySpec extends RetrySpec
{
public FetchRetrySpec()
{
maxAttempts = new MaxAttempt(100);
}
}
public static class JournalSpec implements Params
{

12
src/java/org/apache/cassandra/config/Config.java
View File

@ -159,6 +159,8 @@ public class Config
@Replaces(oldName = "cas_contention_timeout_in_ms", converter = Converters.MILLIS_DURATION_LONG, deprecated = true)
public volatile DurationSpec.LongMillisecondsBound cas_contention_timeout = new DurationSpec.LongMillisecondsBound("1800ms");
public volatile DurationSpec.LongMillisecondsBound accord_preaccept_timeout = new DurationSpec.LongMillisecondsBound("1s");
@Replaces(oldName = "truncate_request_timeout_in_ms", converter = Converters.MILLIS_DURATION_LONG, deprecated = true)
public volatile DurationSpec.LongMillisecondsBound truncate_request_timeout = new DurationSpec.LongMillisecondsBound("60000ms");
@ -177,13 +179,15 @@ public class Config
public volatile DurationSpec.LongMillisecondsBound stream_transfer_task_timeout = new DurationSpec.LongMillisecondsBound("12h");
public volatile DurationSpec.LongMillisecondsBound transaction_timeout = new DurationSpec.LongMillisecondsBound("10s");
public volatile DurationSpec.LongMillisecondsBound cms_await_timeout = new DurationSpec.LongMillisecondsBound("120000ms");
public volatile int cms_default_max_retries = 10;
public volatile DurationSpec.IntMillisecondsBound cms_default_retry_backoff = new DurationSpec.IntMillisecondsBound("50ms");
public volatile DurationSpec.IntMillisecondsBound cms_default_max_retry_backoff = new DurationSpec.IntMillisecondsBound("1s");
@Deprecated(since="5.1")
public volatile DurationSpec.IntMillisecondsBound cms_default_retry_backoff = null;
@Deprecated(since="5.1")
public volatile DurationSpec.IntMillisecondsBound cms_default_max_retry_backoff = null;
public String cms_retry_delay = "0 <= 50ms*1*attempts <= 10s,retries=10";
public volatile int epoch_aware_debounce_inflight_tracker_max_size = 100;
/**
* How often we should snapshot the cluster metadata.
*/

94
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -119,6 +119,7 @@ import org.apache.cassandra.security.JREProvider;
import org.apache.cassandra.security.SSLFactory;
import org.apache.cassandra.service.CacheService.CacheType;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.api.AccordWaitStrategies;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.paxos.Paxos;
import org.apache.cassandra.tcm.RegistrationStatus;
@ -1564,10 +1565,10 @@ public class DatabaseDescriptor
conf.truncate_request_timeout = LOWEST_ACCEPTED_TIMEOUT;
}
if (conf.transaction_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
if (conf.accord_preaccept_timeout.toMilliseconds() < LOWEST_ACCEPTED_TIMEOUT.toMilliseconds())
{
logInfo("transaction_timeout", conf.transaction_timeout, LOWEST_ACCEPTED_TIMEOUT);
conf.transaction_timeout = LOWEST_ACCEPTED_TIMEOUT;
logInfo("accord_preaccept_timeout", conf.accord_preaccept_timeout, LOWEST_ACCEPTED_TIMEOUT);
conf.accord_preaccept_timeout = LOWEST_ACCEPTED_TIMEOUT;
}
}
@ -2532,16 +2533,6 @@ public class DatabaseDescriptor
return conf.cas_contention_timeout.to(unit);
}
public static long getTransactionTimeout(TimeUnit unit)
{
return conf.transaction_timeout.to(unit);
}
public static void setTransactionTimeout(long timeOutInMillis)
{
conf.transaction_timeout = new DurationSpec.LongMillisecondsBound(timeOutInMillis);
}
public static void setCasContentionTimeout(long timeOutInMillis)
{
conf.cas_contention_timeout = new DurationSpec.LongMillisecondsBound(timeOutInMillis);
@ -5330,26 +5321,16 @@ public class DatabaseDescriptor
conf.accord.range_migration = Preconditions.checkNotNull(val);
}
public static int getAccordBarrierRetryAttempts()
{
return conf.accord.barrier_retry_attempts;
}
public static long getAccordBarrierRetryInitialBackoffMillis()
{
return conf.accord.barrier_retry_inital_backoff_millis.toMilliseconds();
}
public static long getAccordBarrierRetryMaxBackoffMillis()
{
return conf.accord.barrier_max_backoff.toMilliseconds();
}
public static long getAccordRangeSyncPointTimeoutNanos()
{
return conf.accord.range_syncpoint_timeout.to(TimeUnit.NANOSECONDS);
}
public static long getAccordRepairTimeoutNanos()
{
return conf.accord.repair_timeout.to(TimeUnit.NANOSECONDS);
}
public static boolean getAccordTransactionsEnabled()
{
return conf.accord.enabled;
@ -5405,29 +5386,43 @@ public class DatabaseDescriptor
return conf.accord.shrink_cache_entries_before_eviction;
}
public static long getAccordRecoverDelay(TimeUnit units)
public static String getAccordRecoverTxnDelay()
{
return conf.accord.recover_delay.to(units);
return conf.accord.recover_txn;
}
public static void setAccordRecoverDelay(long time, TimeUnit units)
public static void setAccordRecoverTxnDelay(String recoverTxnDelay)
{
conf.accord.recover_delay = new IntMillisecondsBound(time, units);
AccordWaitStrategies.setRecoverTxn(recoverTxnDelay);
conf.accord.recover_txn = recoverTxnDelay;
}
public static long getAccordRangeSyncPointRecoverDelay(TimeUnit units)
public static String getAccordExpireTxnDelay()
{
return conf.accord.range_syncpoint_recover_delay.to(units);
return conf.accord.expire_txn;
}
public static void setAccordRangeSyncPointRecoverDelay(long time, TimeUnit units)
public static void setAccordExpireTxnDelay(String expireTxnDelay)
{
conf.accord.range_syncpoint_recover_delay = new IntMillisecondsBound(time, units);
AccordWaitStrategies.setExpireTxn(expireTxnDelay);
conf.accord.expire_txn = expireTxnDelay;
}
public static String getAccordRecoverSyncPointDelay()
{
return conf.accord.recover_syncpoint;
}
public static void setAccordRecoverSyncPointDelay(String recoverSyncPointDelay)
{
AccordWaitStrategies.setRecoverSyncPoint(recoverSyncPointDelay);
conf.accord.recover_syncpoint = recoverSyncPointDelay;
}
public static long getAccordFastPathUpdateDelayMillis()
{
return conf.accord.fast_path_update_delay.to(TimeUnit.MILLISECONDS);
DurationSpec.IntSecondsBound bound = conf.accord.fast_path_update_delay;
return bound == null ? -1 : bound.to(TimeUnit.MILLISECONDS);
}
public static void setAccordFastPathUpdateDelaySeconds(long seconds)
@ -5470,26 +5465,6 @@ public class DatabaseDescriptor
conf.accord.global_durability_cycle = new DurationSpec.IntSecondsBound(seconds);
}
public static long getAccordDefaultDurabilityRetryDelay(TimeUnit unit)
{
return conf.accord.default_durability_retry_delay.to(unit);
}
public static void setAccordDefaultDurabilityRetryDelaySeconds(long seconds)
{
conf.accord.default_durability_retry_delay = new DurationSpec.IntSecondsBound(seconds);
}
public static long getAccordMaxDurabilityRetryDelay(TimeUnit unit)
{
return conf.accord.max_durability_retry_delay.to(unit);
}
public static void setAccordMaxDurabilityRetryDelaySeconds(long seconds)
{
conf.accord.max_durability_retry_delay = new DurationSpec.IntSecondsBound(seconds);
}
public static long getAccordShardDurabilityCycle(TimeUnit unit)
{
return conf.accord.shard_durability_cycle.to(unit);
@ -5837,6 +5812,11 @@ public class DatabaseDescriptor
return conf.cms_default_max_retry_backoff;
}
public static String getCMSRetryDelay()
{
return conf.cms_retry_delay;
}
public static DurationSpec getCmsAwaitTimeout()
{
return conf.cms_await_timeout;

11
src/java/org/apache/cassandra/config/RetrySpec.java
View File

@ -23,6 +23,10 @@ import java.util.Objects;
import javax.annotation.Nullable;
import org.apache.cassandra.config.DurationSpec.LongMillisecondsBound;
import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.service.RetryStrategy;
import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory;
import org.apache.cassandra.service.WaitStrategy;
public class RetrySpec
{
@ -153,6 +157,13 @@ public class RetrySpec
return !isEnabled() ? null : maxSleepTime;
}
public static WaitStrategy toStrategy(SharedContext ctx, RetrySpec spec)
{
if (!spec.isEnabled())
return WaitStrategy.None.INSTANCE;
return RetryStrategy.parse(spec.baseSleepTime.toMilliseconds() + "ms * 2^attempts <= " + spec.maxSleepTime.toMilliseconds() + "ms,retries=" + (spec.maxAttempts.value - 1), LatencySourceFactory.none());
}
@Override
public String toString()
{

16
src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
View File

@ -28,6 +28,7 @@ import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.primitives.Ranges;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
@ -48,7 +49,9 @@ import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.AccordTopology;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping;
import org.apache.cassandra.streaming.IncomingStream;
import org.apache.cassandra.streaming.StreamReceiver;
import org.apache.cassandra.streaming.StreamSession;
@ -58,8 +61,11 @@ import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.Refs;
import static accord.local.durability.DurabilityService.SyncLocal.Self;
import static accord.local.durability.DurabilityService.SyncRemote.NoRemote;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_MUTATION_REPAIR_ROWS_PER_BATCH;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
public class CassandraStreamReceiver implements StreamReceiver
{
@ -248,7 +254,15 @@ public class CassandraStreamReceiver implements StreamReceiver
if (session.streamOperation().requiresBarrierTransaction()
&& cfs.metadata().requiresAccordSupport()
&& CassandraVersion.CASSANDRA_5_0.compareTo(minVersion) >= 0)
accordService.postStreamReceivingBarrier(cfs, ranges);
{
Ranges accordRanges = AccordTopology.toAccordRanges(cfs.getTableId(), ranges);
long startedAtNanos = nanoTime();
long deadlineNanos = startedAtNanos + DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos();
// TODO (expected): use the source bounds for the streams to avoid waiting unnecessarily long
AccordService.getBlocking(accordService.maxConflict(accordRanges)
.flatMap(min -> accordService.sync("[Stream #" + session.planId() + ']', min, accordRanges, null, Self, NoRemote))
, accordRanges, new LatencyRequestBookkeeping(cfs.metric.accordPostStreamRepair), startedAtNanos, deadlineNanos);
}
boolean requiresWritePath = requiresWritePath(cfs);
Collection<SSTableReader> readers = sstables;

79
src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java
View File

@ -33,7 +33,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.RoutingKey;
import accord.impl.DurabilityScheduling;
import accord.impl.progresslog.DefaultProgressLog;
import accord.impl.progresslog.TxnStateKind;
import accord.local.CommandStore;
@ -41,6 +40,7 @@ import accord.local.CommandStores;
import accord.local.DurableBefore;
import accord.local.MaxConflicts;
import accord.local.RejectBefore;
import accord.local.durability.ShardDurability;
import accord.primitives.Status;
import accord.primitives.TxnId;
import accord.utils.Invariants;
@ -85,22 +85,22 @@ import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
public class AccordDebugKeyspace extends VirtualKeyspace
{
public static final String DURABILITY_SCHEDULING = "durability_scheduling";
public static final String DURABLE_BEFORE = "durable_before";
public static final String EXECUTOR_CACHE = "executor_cache";
public static final String MAX_CONFLICTS = "max_conflicts";
public static final String MIGRATION_STATE = "migration_state";
public static final String PROGRESS_LOG = "progress_log";
public static final String REDUNDANT_BEFORE = "redundant_before";
public static final String REJECT_BEFORE = "reject_before";
public static final String TXN_BLOCKED_BY = "txn_blocked_by";
public static final String DURABILITY_SERVICE = "durability_service";
public static final String DURABLE_BEFORE = "durable_before";
public static final String EXECUTOR_CACHE = "executor_cache";
public static final String MAX_CONFLICTS = "max_conflicts";
public static final String MIGRATION_STATE = "migration_state";
public static final String PROGRESS_LOG = "progress_log";
public static final String REDUNDANT_BEFORE = "redundant_before";
public static final String REJECT_BEFORE = "reject_before";
public static final String TXN_BLOCKED_BY = "txn_blocked_by";
public static final AccordDebugKeyspace instance = new AccordDebugKeyspace();
private AccordDebugKeyspace()
{
super(VIRTUAL_ACCORD_DEBUG, List.of(
new DurabilitySchedulingTable(),
new DurabilityServiceTable(),
new DurableBeforeTable(),
new ExecutorCacheTable(),
new MaxConflictsTable(),
@ -113,25 +113,33 @@ public class AccordDebugKeyspace extends VirtualKeyspace
}
// TODO (consider): use a different type for the three timestamps in micros
public static final class DurabilitySchedulingTable extends AbstractVirtualTable
public static final class DurabilityServiceTable extends AbstractVirtualTable
{
private DurabilitySchedulingTable()
private DurabilityServiceTable()
{
super(parse(VIRTUAL_ACCORD_DEBUG, DURABILITY_SCHEDULING,
"Accord per-Range Durability Scheduling State",
super(parse(VIRTUAL_ACCORD_DEBUG, DURABILITY_SERVICE,
"Accord per-Range Durability Service State",
"CREATE TABLE %s (\n" +
" keyspace_name text,\n" +
" table_name text,\n" +
" token_sort blob,\n" +
" token_start text,\n" +
" token_end text,\n" +
" last_started_at_micros bigint,\n" +
" cycle_started_at_micros bigint,\n" +
" retries int,\n" +
" min text,\n" +
" active text,\n" +
" waiting text,\n" +
" node_offset int,\n" +
" \"index\" int,\n" +
" number_of_splits int,\n" +
" range_started_at bigint,\n" +
" cycle_started_at bigint,\n" +
" retry_delay_micros bigint,\n" +
" is_defunct boolean,\n" +
" cycle_offset int,\n" +
" activeIndex int,\n" +
" nextIndex int,\n" +
" nextToIndex int,\n" +
" endIndex int,\n" +
" current_splits int,\n" +
" stopping boolean,\n" +
" stopped boolean,\n" +
" PRIMARY KEY (keyspace_name, table_name, token_start)" +
')', UTF8Type.instance));
}
@ -139,23 +147,30 @@ public class AccordDebugKeyspace extends VirtualKeyspace
@Override
public DataSet data()
{
DurabilityScheduling.ImmutableView view = ((AccordService) AccordService.instance()).durabilityScheduling();
ShardDurability.ImmutableView view = ((AccordService) AccordService.instance()).shardDurability();
SimpleDataSet ds = new SimpleDataSet(metadata());
while (view.advance())
{
TableId tableId = (TableId) view.range().start().prefix();
TableId tableId = (TableId) view.shard().range.start().prefix();
TableMetadata tableMetadata = tableMetadata(tableId);
ds.row(keyspace(tableMetadata), table(tableId, tableMetadata), sortToken(view.range().start()))
.column("start_token", printToken(view.range().start()))
.column("end_token", printToken(view.range().end()))
ds.row(keyspace(tableMetadata), table(tableId, tableMetadata), sortToken(view.shard().range.start()))
.column("start_token", printToken(view.shard().range.start()))
.column("end_token", printToken(view.shard().range.end()))
.column("last_started_at", approxTime.translate().toMillisSinceEpoch(view.lastStartedAtMicros() * 1000))
.column("cycle_started_at", approxTime.translate().toMillisSinceEpoch(view.cycleStartedAtMicros() * 1000))
.column("active", Objects.toString(view.active()))
.column("waiting", Objects.toString(view.waiting()))
.column("node_offset", view.nodeOffset())
.column("index", view.index())
.column("number_of_splits", view.numberOfSplits())
.column("range_started_at", view.rangeStartedAtMicros())
.column("cycle_started_at", view.cycleStartedAtMicros())
.column("retry_delay_micros", view.retryDelayMicros())
.column("is_defunct", view.isDefunct());
.column("cycle_offset", view.cycleOffset())
.column("activeIndex", view.activeIndex())
.column("nextIndex", view.nextIndex())
.column("nextToIndex", view.toIndex())
.column("endIndex", view.cycleLength())
.column("current_splits", view.currentSplits())
.column("stopping", view.stopping())
.column("stopping", view.stopping())
;
}
return ds;
}

73
src/java/org/apache/cassandra/hints/HintsDispatcher.java
View File

@ -27,6 +27,7 @@ import java.util.List;
import java.util.Queue;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
import javax.annotation.Nonnull;
@ -37,14 +38,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask;
import org.apache.cassandra.concurrent.ImmediateExecutor;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
@ -52,9 +51,7 @@ import org.apache.cassandra.metrics.HintsServiceMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.service.accord.IAccordService.IAccordResult;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitMutation;
@ -353,8 +350,8 @@ final class HintsDispatcher implements AutoCloseable
ClusterMetadata cm = ClusterMetadata.current();
SplitHint splitHint = splitHintIntoAccordAndNormal(cm, hint);
Mutation accordHintMutation = splitHint.accordMutation;
Dispatcher.RequestTime requestTime = null;
AsyncTxnResult accordTxnResult = null;
Dispatcher.RequestTime requestTime;
IAccordResult<TxnResult> accordTxnResult = null;
if (accordHintMutation != null)
{
requestTime = Dispatcher.RequestTime.forImmediateExecution();
@ -362,7 +359,7 @@ final class HintsDispatcher implements AutoCloseable
}
Hint normalHint = splitHint.normalHint;
Callback callback = new Callback(address, hint.creationTime, requestTime, accordTxnResult);
Callback callback = new Callback(address, hint.creationTime, accordTxnResult);
if (normalHint != null)
{
// We had a hint that was supposed to be done on Accord for the batch log (otherwise address would be non-null),
@ -438,7 +435,7 @@ final class HintsDispatcher implements AutoCloseable
return callback;
}
static final class Callback implements RequestCallback, Runnable
static final class Callback implements RequestCallback, BiConsumer<TxnResult, Throwable>
{
enum Outcome { SUCCESS, TIMEOUT, FAILURE, INTERRUPTED, RETRY_DIFFERENT_SYSTEM }
@ -449,23 +446,18 @@ final class HintsDispatcher implements AutoCloseable
@Nullable
private final InetAddressAndPort to;
private final long hintCreationNanoTime;
@Nullable
private final Dispatcher.RequestTime requestTime;
private final AsyncTxnResult accordTxnResult;
private Callback(@Nonnull InetAddressAndPort to, long hintCreationTimeMillisSinceEpoch)
{
this(to, hintCreationTimeMillisSinceEpoch, null, null);
this(to, hintCreationTimeMillisSinceEpoch, null);
}
private Callback(@Nullable InetAddressAndPort to, long hintCreationTimeMillisSinceEpoch, Dispatcher.RequestTime requestTime, @Nullable AsyncTxnResult accordTxnResult)
private Callback(@Nullable InetAddressAndPort to, long hintCreationTimeMillisSinceEpoch, @Nullable IAccordResult<TxnResult> accordTxnResult)
{
this.to = to != null ? to : ACCORD_HINT_ENDPOINT;
this.hintCreationNanoTime = approxTime.translate().fromMillisSinceEpoch(hintCreationTimeMillisSinceEpoch);
this.requestTime = requestTime;
this.accordTxnResult = accordTxnResult;
if (accordTxnResult != null)
accordTxnResult.addListener(this, ImmediateExecutor.INSTANCE);
accordTxnResult.addCallback(this);
else
accordOutcome = SUCCESS;
}
@ -533,34 +525,35 @@ final class HintsDispatcher implements AutoCloseable
}
@Override
public void run()
public void accept(TxnResult success, Throwable fail)
{
try
if (fail != null)
{
IAccordService accord = AccordService.instance();
TxnResult.Kind kind = accord.getTxnResult(accordTxnResult).kind();
if (kind == retry_new_protocol)
if (fail instanceof RequestExecutionException || fail instanceof RetryOnDifferentSystemException)
{
if (fail instanceof RetryOnDifferentSystemException)
accordOutcome = RETRY_DIFFERENT_SYSTEM;
else
accordOutcome = TIMEOUT;
String msg = "Accord hint delivery transaction failed retriably";
if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime()))
logger.error(msg, fail);
}
else
{
accordOutcome = FAILURE;
String msg = "Accord hint delivery transaction failed permanently";
if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime()))
logger.error(msg, fail);
}
}
else
{
if (success.kind() == retry_new_protocol)
accordOutcome = RETRY_DIFFERENT_SYSTEM;
else
accordOutcome = SUCCESS;
}
catch (WriteTimeoutException | WriteFailureException | RetryOnDifferentSystemException e)
{
if (e instanceof RetryOnDifferentSystemException)
accordOutcome = RETRY_DIFFERENT_SYSTEM;
else
accordOutcome = TIMEOUT;
String msg = "Accord hint delivery transaction failed retriably";
if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime()))
logger.error(msg, e);
}
catch (Exception e)
{
accordOutcome = FAILURE;
String msg = "Accord hint delivery transaction failed permanently";
if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime()))
logger.error(msg, e);
}
maybeSignal();
}
}

10
src/java/org/apache/cassandra/index/accord/RangeMemoryIndex.java
View File

@ -36,8 +36,8 @@ import java.util.TreeSet;
import java.util.stream.Collectors;
import javax.annotation.concurrent.GuardedBy;
import accord.primitives.Participants;
import accord.primitives.Routable;
import accord.primitives.Route;
import accord.primitives.Unseekable;
import org.apache.cassandra.cache.IMeasurableMemory;
import org.apache.cassandra.db.Clustering;
@ -52,7 +52,7 @@ import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.RTree;
import org.apache.cassandra.utils.RangeTree;
import static org.apache.cassandra.index.accord.RouteIndexFormat.deserializeParticipants;
import static org.apache.cassandra.index.accord.RouteIndexFormat.deserializeTouches;
public class RangeMemoryIndex
{
@ -107,10 +107,10 @@ public class RangeMemoryIndex
public synchronized long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
{
Route<?> route;
Participants<?> route;
try
{
route = deserializeParticipants(value);
route = deserializeTouches(value);
}
catch (IOException e)
{
@ -120,7 +120,7 @@ public class RangeMemoryIndex
return add(key, route);
}
public synchronized long add(DecoratedKey key, Route<?> route)
public synchronized long add(DecoratedKey key, Participants<?> route)
{
if (route == null || route.domain() != Routable.Domain.Range)
return 0;

13
src/java/org/apache/cassandra/index/accord/RouteIndexFormat.java
View File

@ -35,7 +35,6 @@ import com.google.common.collect.Maps;
import accord.local.StoreParticipants;
import accord.primitives.Participants;
import accord.primitives.Route;
import accord.primitives.TxnId;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
@ -72,7 +71,7 @@ public class RouteIndexFormat
{
public static final Supplier<Checksum> CHECKSUM_SUPPLIER = CRC32C::new;
static final LocalVersionedSerializer<Participants<?>> participants = localSerializer(KeySerializers.participants);
static final LocalVersionedSerializer<Participants<?>> touches = localSerializer(KeySerializers.participants);
private static <T> LocalVersionedSerializer<T> localSerializer(IVersionedSerializer<T> serializer)
{
return new LocalVersionedSerializer<>(AccordSerializerVersion.CURRENT, AccordSerializerVersion.serializer, serializer);
@ -80,23 +79,23 @@ public class RouteIndexFormat
public static ByteBuffer serialize(Participants<?> value) throws IOException
{
int size = Math.toIntExact(participants.serializedSize(value));
int size = Math.toIntExact(touches.serializedSize(value));
try (DataOutputBuffer buffer = new DataOutputBuffer(size))
{
participants.serialize(value, buffer);
touches.serialize(value, buffer);
return buffer.buffer(true);
}
}
static Route<?> deserializeParticipants(ByteBuffer bytes) throws IOException
static Participants<?> deserializeTouches(ByteBuffer bytes) throws IOException
{
if (bytes == null || ByteBufferAccessor.instance.isEmpty(bytes))
return null;
try (DataInputBuffer in = new DataInputBuffer(bytes, true))
{
MessageVersionProvider versionProvider = participants.deserializeVersion(in);
return KeySerializers.route.deserialize(in, versionProvider.messageVersion());
MessageVersionProvider versionProvider = touches.deserializeVersion(in);
return KeySerializers.participants.deserialize(in, versionProvider.messageVersion());
}
}

11
src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java
View File

@ -18,6 +18,8 @@
package org.apache.cassandra.metrics;
import javax.annotation.Nullable;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
@ -25,6 +27,8 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
public class AccordClientRequestMetrics extends ClientRequestMetrics
{
public final @Nullable ClientRequestMetrics shared;
public final Histogram keySize;
// During migration back to Paxos it's possible a transaction runs
@ -41,10 +45,13 @@ public class AccordClientRequestMetrics extends ClientRequestMetrics
public final Meter accordMigrationRejects;
public final Meter preempted;
public final Meter topologyMismatches;
public final boolean isWrite;
public AccordClientRequestMetrics(String scope)
public AccordClientRequestMetrics(String scope, ClientRequestMetrics shared, boolean isWrite)
{
super(scope);
this.shared = shared;
this.isWrite = isWrite;
keySize = Metrics.histogram(factory.createMetricName("KeySizeHistogram"), false);
migrationSkippedReads = Metrics.meter(factory.createMetricName("MigrationSkippedReads"));
@ -64,6 +71,6 @@ public class AccordClientRequestMetrics extends ClientRequestMetrics
Metrics.remove(factory.createMetricName("AccordMigrationRejects"));
Metrics.remove(factory.createMetricName("Preempted"));
Metrics.remove(factory.createMetricName("TopologyMismatches"));
}
}

8
src/java/org/apache/cassandra/metrics/AccordMetrics.java
View File

@ -21,7 +21,7 @@ package org.apache.cassandra.metrics;
import java.lang.reflect.Field;
import java.util.concurrent.TimeUnit;
import accord.api.EventsListener;
import accord.api.EventListener;
import accord.local.Command;
import accord.primitives.Deps;
import accord.primitives.PartialDeps;
@ -181,7 +181,7 @@ public class AccordMetrics
return builder.toString();
}
public static class Listener implements EventsListener
public static class Listener implements EventListener
{
public final static Listener instance = new Listener(AccordMetrics.readMetrics, AccordMetrics.writeMetrics);
@ -198,7 +198,7 @@ public class AccordMetrics
{
if (txnId.isWrite())
return writeMetrics;
else if (txnId.isRead())
else if (txnId.isSomeRead())
return readMetrics;
else
return null;
@ -289,6 +289,8 @@ public class AccordMetrics
@Override
public void onTimeout(TxnId txnId)
{
// TODO (required): we appear to be marking this twice, once in AccordResult and once here.
// why does AccordMetricsTest only see this one? remove duplication.
AccordMetrics metrics = forTransaction(txnId);
if (metrics != null)
metrics.timeouts.mark();

7
src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java
View File

@ -21,6 +21,7 @@ import java.util.EnumMap;
import java.util.Map;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.service.accord.ClientRequestBookkeeping;
public final class ClientRequestsMetricsHolder
{
@ -29,8 +30,10 @@ public final class ClientRequestsMetricsHolder
public static final CASClientWriteRequestMetrics casWriteMetrics = new CASClientWriteRequestMetrics("CASWrite");
public static final CASClientRequestMetrics casReadMetrics = new CASClientRequestMetrics("CASRead");
public static final ViewWriteMetrics viewWriteMetrics = new ViewWriteMetrics("ViewWrite");
public static final AccordClientRequestMetrics accordReadMetrics = new AccordClientRequestMetrics("AccordRead");
public static final AccordClientRequestMetrics accordWriteMetrics = new AccordClientRequestMetrics("AccordWrite");
public static final AccordClientRequestMetrics accordReadMetrics = new AccordClientRequestMetrics("AccordRead", readMetrics, false);
public static final AccordClientRequestMetrics accordWriteMetrics = new AccordClientRequestMetrics("AccordWrite", writeMetrics, true);
public static final ClientRequestBookkeeping accordReadBookkeeping = new ClientRequestBookkeeping(false, accordReadMetrics);
public static final ClientRequestBookkeeping accordWriteBookkeeping = new ClientRequestBookkeeping(true, accordWriteMetrics);
public static final Map<ConsistencyLevel, ClientRequestMetrics> readMetricsMap = new EnumMap<>(ConsistencyLevel.class);
public static final Map<ConsistencyLevel, ClientWriteRequestMetrics> writeMetricsMap = new EnumMap<>(ConsistencyLevel.class);

8
src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
View File

@ -104,9 +104,9 @@ public class KeyspaceMetrics
/** Latency for locally run key migrations **/
public final LatencyMetrics keyMigration;
/** Latency for range migrations run by locally coordinated Accord repairs **/
public final LatencyMetrics rangeMigration;
public final LatencyMetrics accordRepair;
public final LatencyMetrics accordPostStreamRepair;
public final Meter rangeMigrationUnexpectedFailures;
public final Meter rangeMigrationDependencyLimitFailures;
public final Meter mutationsRejectedOnWrongSystem;
public final Meter readsRejectedOnWrongSystem;
/** Writes failed ideal consistency **/
@ -256,9 +256,9 @@ public class KeyspaceMetrics
casPropose = createLatencyMetrics("CasPropose");
casCommit = createLatencyMetrics("CasCommit");
keyMigration = createLatencyMetrics("KeyMigration");
rangeMigration = createLatencyMetrics("RangeMigration");
accordRepair = createLatencyMetrics("AccordRepair");
accordPostStreamRepair = createLatencyMetrics("AccordPostStreamRepair");
rangeMigrationUnexpectedFailures = createKeyspaceMeter("RangeMigrationUnexpectedFailures");
rangeMigrationDependencyLimitFailures = createKeyspaceMeter("RangeMigratingDependencyLimitFailures");
mutationsRejectedOnWrongSystem = createKeyspaceMeter("MutationsRejectedOnWrongSystem");
readsRejectedOnWrongSystem = createKeyspaceMeter("ReadsRejectedOnWrongSystem");
writeFailedIdealCL = createKeyspaceCounter("WriteFailedIdealCL");

4
src/java/org/apache/cassandra/metrics/RepairMetrics.java
View File

@ -87,8 +87,8 @@ public class RepairMetrics
public static void retry(Verb verb, int attempt)
{
retries.update(attempt);
retriesByVerb.get(verb).update(attempt);
retries.update(attempt - 1);
retriesByVerb.get(verb).update(attempt - 1);
}
public static void retryTimeout(Verb verb)

6
src/java/org/apache/cassandra/metrics/TableMetrics.java
View File

@ -194,8 +194,8 @@ public class TableMetrics
public final LatencyMetrics keyMigration;
/** Latency for range migrations run by locally coordinated Accord repairs **/
public final LatencyMetrics accordRepair;
public final LatencyMetrics accordPostStreamRepair;
public final TableMeter accordRepairUnexpectedFailures;
public final TableMeter accordRepairDependencyLimitFailures;
public final TableMeter mutationsRejectedOnWrongSystem;
public final TableMeter readsRejectedOnWrongSystem;
/** percent of the data that is repaired */
@ -816,9 +816,9 @@ public class TableMetrics
casPropose = createLatencyMetrics("CasPropose", cfs.keyspace.metric.casPropose);
casCommit = createLatencyMetrics("CasCommit", cfs.keyspace.metric.casCommit);
keyMigration = createLatencyMetrics("KeyMigration", cfs.keyspace.metric.keyMigration, GLOBAL_KEY_MIGRATION_LATENCY);
accordRepair = createLatencyMetrics("AccordRepair", cfs.keyspace.metric.rangeMigration, GLOBAL_RANGE_MIGRATION_LATENCY);
accordRepair = createLatencyMetrics("AccordRepair", cfs.keyspace.metric.accordRepair, GLOBAL_RANGE_MIGRATION_LATENCY);
accordPostStreamRepair = createLatencyMetrics("AccordPostStreamRepair", cfs.keyspace.metric.accordPostStreamRepair);
accordRepairUnexpectedFailures = createTableMeter("AccordRepairUnexpectedFailures", cfs.keyspace.metric.rangeMigrationUnexpectedFailures);
accordRepairDependencyLimitFailures = createTableMeter("AccordRepairDependencyLimitFaiures", cfs.keyspace.metric.rangeMigrationDependencyLimitFailures);
mutationsRejectedOnWrongSystem = createTableMeter("MutationsRejectedOnWrongSystem", cfs.keyspace.metric.mutationsRejectedOnWrongSystem);
readsRejectedOnWrongSystem = createTableMeter("ReadsRejectedOnWrongSystem", cfs.keyspace.metric.readsRejectedOnWrongSystem);

35
src/java/org/apache/cassandra/net/MessageDelivery.java
View File

@ -22,16 +22,18 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.utils.Invariants;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.Backoff;
import org.apache.cassandra.service.WaitStrategy;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.Accumulator;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
@ -39,6 +41,7 @@ import org.apache.cassandra.utils.concurrent.CountDownLatch;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.Promise;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.net.MessageFlag.CALL_BACK_ON_FAILURE;
public interface MessageDelivery
@ -83,7 +86,7 @@ public interface MessageDelivery
public <REQ, RSP> void sendWithCallback(Message<REQ> message, InetAddressAndPort to, RequestCallback<RSP> cb, ConnectionType specifyConnection);
public <REQ, RSP> Future<Message<RSP>> sendWithResult(Message<REQ> message, InetAddressAndPort to);
public default <REQ, RSP> Future<Message<RSP>> sendWithRetries(Backoff backoff, RetryScheduler retryThreads,
public default <REQ, RSP> Future<Message<RSP>> sendWithRetries(WaitStrategy backoff, RetryScheduler retryThreads,
Verb verb, REQ request,
Iterator<InetAddressAndPort> candidates,
RetryPredicate shouldRetry,
@ -99,14 +102,14 @@ public interface MessageDelivery
return promise;
}
public default <REQ, RSP> void sendWithRetries(Backoff backoff, RetryScheduler retryThreads,
public default <REQ, RSP> void sendWithRetries(WaitStrategy backoff, RetryScheduler retryThreads,
Verb verb, REQ request,
Iterator<InetAddressAndPort> candidates,
OnResult<RSP> onResult,
RetryPredicate shouldRetry,
RetryErrorMessage errorMessage)
{
sendWithRetries(this, backoff, retryThreads, verb, request, candidates, onResult, shouldRetry, errorMessage, 0);
sendWithRetries(this, backoff, retryThreads, verb, request, candidates, onResult, shouldRetry, errorMessage, 1);
}
public <V> void respond(V response, Message<?> message);
public default void respondWithFailure(RequestFailureReason reason, Message<?> message)
@ -139,7 +142,7 @@ public interface MessageDelivery
}
private static <REQ, RSP> void sendWithRetries(MessageDelivery messaging,
Backoff backoff,
WaitStrategy backoff,
RetryScheduler retryThreads,
Verb verb, REQ request,
Iterator<InetAddressAndPort> candidates,
@ -148,6 +151,7 @@ public interface MessageDelivery
RetryErrorMessage errorMessage,
int attempt)
{
Invariants.require(backoff != null);
if (Thread.currentThread().isInterrupted())
{
onResult.result(attempt, null, new InterruptedException(errorMessage.apply(attempt, ResponseFailureReason.Interrupted, null, null)));
@ -169,10 +173,11 @@ public interface MessageDelivery
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
// TODO (required): we already have a separate retry predicate, backoff should not be taken into consideration when retrying
if (!backoff.mayRetry(attempt))
long retryDelay = backoff.computeWait(attempt + 1, NANOSECONDS);
// TODO (required): we already have a separate retry predicate, retries should not be taken into consideration when retrying
if (retryDelay < 0)
{
onResult.result(attempt, null, new MaxRetriesException(attempt, errorMessage.apply(attempt, ResponseFailureReason.MaxRetries, from, failure)));
onResult.result(attempt, null, new GivingUpException(attempt, errorMessage.apply(attempt, ResponseFailureReason.GiveUp, from, failure)));
return;
}
if (!shouldRetry.test(attempt, from, failure))
@ -183,7 +188,7 @@ public interface MessageDelivery
try
{
retryThreads.schedule(() -> sendWithRetries(messaging, backoff, retryThreads, verb, request, candidates, onResult, shouldRetry, errorMessage, attempt + 1),
backoff.computeWaitTime(attempt), backoff.unit());
retryDelay, NANOSECONDS);
}
catch (Throwable t)
{
@ -194,7 +199,7 @@ public interface MessageDelivery
messaging.sendWithCallback(Message.outWithFlag(verb, request, CALL_BACK_ON_FAILURE), candidates.next(), new Request());
}
enum ResponseFailureReason { MaxRetries, Rejected, NoMoreCandidates, Interrupted, FailedSchedule }
enum ResponseFailureReason {GiveUp, Rejected, NoMoreCandidates, Interrupted, FailedSchedule }
interface RetryScheduler
{
@ -212,7 +217,7 @@ public interface MessageDelivery
}
}
class NoMoreCandidatesException extends IllegalStateException
class NoMoreCandidatesException extends TimeoutException
{
public NoMoreCandidatesException(String s)
{
@ -220,7 +225,7 @@ public interface MessageDelivery
}
}
class FailedResponseException extends IllegalStateException
class FailedResponseException extends RuntimeException
{
public final InetAddressAndPort from;
public final RequestFailure failure;
@ -233,17 +238,17 @@ public interface MessageDelivery
}
}
class MaxRetriesException extends IllegalStateException
class GivingUpException extends TimeoutException
{
public final int attempts;
public MaxRetriesException(int attempts, String message)
public GivingUpException(int attempts, String message)
{
super(message);
this.attempts = attempts;
}
}
class FailedScheduleException extends IllegalStateException
class FailedScheduleException extends RuntimeException
{
public FailedScheduleException(String message, Throwable cause)
{

86
src/java/org/apache/cassandra/net/Verb.java
View File

@ -95,10 +95,11 @@ import org.apache.cassandra.service.accord.serializers.CommitSerializers;
import org.apache.cassandra.service.accord.serializers.EnumSerializer;
import org.apache.cassandra.service.accord.serializers.FetchSerializers;
import org.apache.cassandra.service.accord.serializers.GetEphmrlReadDepsSerializers;
import org.apache.cassandra.service.accord.serializers.GetMaxConflictSerializers;
import org.apache.cassandra.service.accord.serializers.InformDurableSerializers;
import org.apache.cassandra.service.accord.serializers.LatestDepsSerializers;
import org.apache.cassandra.service.accord.serializers.PreacceptSerializers;
import org.apache.cassandra.service.accord.serializers.QueryDurableBeforeSerializers;
import org.apache.cassandra.service.accord.serializers.GetDurableBeforeSerializers;
import org.apache.cassandra.service.accord.serializers.ReadDataSerializers;
import org.apache.cassandra.service.accord.serializers.RecoverySerializers;
import org.apache.cassandra.service.accord.serializers.SetDurableSerializers;
@ -318,51 +319,52 @@ public enum Verb
ACCORD_NOT_ACCEPT_REQ (124, P2, writeTimeout, IMMEDIATE, () -> AcceptSerializers.notAccept, AccordService::requestHandlerOrNoop, ACCORD_ACCEPT_RSP ),
ACCORD_READ_RSP (125, P2, readTimeout, IMMEDIATE, () -> ReadDataSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_READ_REQ (126, P2, readTimeout, IMMEDIATE, () -> ReadDataSerializers.readData, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ),
ACCORD_COMMIT_REQ (127, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ),
ACCORD_COMMIT_INVALIDATE_REQ (128, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.invalidate, AccordService::requestHandlerOrNoop ),
ACCORD_APPLY_RSP (129, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_APPLY_REQ (130, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.request, AccordService::requestHandlerOrNoop, ACCORD_APPLY_RSP ),
ACCORD_BEGIN_RECOVER_RSP (131, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_BEGIN_RECOVER_REQ (132, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.request, AccordService::requestHandlerOrNoop, ACCORD_BEGIN_RECOVER_RSP ),
ACCORD_BEGIN_INVALIDATE_RSP (133, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_BEGIN_INVALIDATE_REQ (134, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_BEGIN_INVALIDATE_RSP ),
ACCORD_AWAIT_RSP (136, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.syncReply, AccordService::responseHandlerOrNoop ),
ACCORD_AWAIT_REQ (135, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_AWAIT_RSP ),
ACCORD_AWAIT_ASYNC_RSP_REQ (137, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.asyncReply, AccordService::requestHandlerOrNoop ),
ACCORD_WAIT_UNTIL_APPLIED_REQ (138, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.waitUntilApplied, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ),
ACCORD_STABLE_THEN_READ_REQ (139, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.stableThenRead, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ),
ACCORD_INFORM_DURABLE_REQ (140, P2, writeTimeout, IMMEDIATE, () -> InformDurableSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ),
ACCORD_CHECK_STATUS_RSP (141, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_CHECK_STATUS_REQ (142, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_CHECK_STATUS_RSP ),
ACCORD_RECOVER_AWAIT_RSP (143, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.recoverReply, AccordService::responseHandlerOrNoop ),
ACCORD_RECOVER_AWAIT_REQ (144, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.recoverRequest, AccordService::requestHandlerOrNoop, ACCORD_RECOVER_AWAIT_RSP),
ACCORD_GET_LATEST_DEPS_RSP (167, P2, writeTimeout, IMMEDIATE, () -> LatestDepsSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_GET_LATEST_DEPS_REQ (168, P2, writeTimeout, IMMEDIATE, () -> LatestDepsSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_LATEST_DEPS_RSP),
ACCORD_GET_EPHMRL_READ_DEPS_RSP (161, P2, writeTimeout, IMMEDIATE, () -> GetEphmrlReadDepsSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_GET_EPHMRL_READ_DEPS_REQ (162, P2, writeTimeout, IMMEDIATE, () -> GetEphmrlReadDepsSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_EPHMRL_READ_DEPS_RSP),
ACCORD_FETCH_DATA_RSP (145, P2, writeTimeout, IMMEDIATE, () -> FetchSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_FETCH_DATA_REQ (146, P2, writeTimeout, IMMEDIATE, () -> FetchSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_FETCH_DATA_RSP ),
ACCORD_SET_SHARD_DURABLE_REQ (147, P2, writeTimeout, MISC, () -> SetDurableSerializers.shardDurable, AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ),
ACCORD_SET_GLOBALLY_DURABLE_REQ (148, P2, writeTimeout, MISC, () -> SetDurableSerializers.globallyDurable,AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ),
ACCORD_QUERY_DURABLE_BEFORE_RSP (149, P2, writeTimeout, IMMEDIATE, () -> QueryDurableBeforeSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_QUERY_DURABLE_BEFORE_REQ (150, P2, writeTimeout, IMMEDIATE, () -> QueryDurableBeforeSerializers.request,AccordService::requestHandlerOrNoop, ACCORD_QUERY_DURABLE_BEFORE_RSP ),
ACCORD_STABLE_THEN_READ_REQ (127, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.stableThenRead, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ),
ACCORD_COMMIT_REQ (128, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ),
ACCORD_COMMIT_INVALIDATE_REQ (129, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.invalidate, AccordService::requestHandlerOrNoop ),
ACCORD_APPLY_RSP (130, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_APPLY_REQ (131, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.request, AccordService::requestHandlerOrNoop, ACCORD_APPLY_RSP ),
ACCORD_APPLY_AND_WAIT_REQ (132, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.readData, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP),
ACCORD_BEGIN_RECOVER_RSP (133, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_BEGIN_RECOVER_REQ (134, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.request, AccordService::requestHandlerOrNoop, ACCORD_BEGIN_RECOVER_RSP ),
ACCORD_BEGIN_INVALIDATE_RSP (135, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_BEGIN_INVALIDATE_REQ (136, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_BEGIN_INVALIDATE_RSP ),
ACCORD_AWAIT_RSP (137, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.syncReply, AccordService::responseHandlerOrNoop ),
ACCORD_AWAIT_REQ (138, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_AWAIT_RSP ),
ACCORD_AWAIT_ASYNC_RSP_REQ (139, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.asyncReply, AccordService::requestHandlerOrNoop ),
ACCORD_WAIT_UNTIL_APPLIED_REQ (140, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.waitUntilApplied, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP ),
ACCORD_RECOVER_AWAIT_RSP (141, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.recoverReply, AccordService::responseHandlerOrNoop ),
ACCORD_RECOVER_AWAIT_REQ (142, P2, writeTimeout, IMMEDIATE, () -> AwaitSerializers.recoverRequest, AccordService::requestHandlerOrNoop, ACCORD_RECOVER_AWAIT_RSP),
ACCORD_INFORM_DURABLE_REQ (143, P2, writeTimeout, IMMEDIATE, () -> InformDurableSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ),
ACCORD_CHECK_STATUS_RSP (144, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_CHECK_STATUS_REQ (145, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_CHECK_STATUS_RSP ),
ACCORD_FETCH_DATA_RSP (146, P2, writeTimeout, IMMEDIATE, () -> FetchSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_FETCH_DATA_REQ (147, P2, writeTimeout, IMMEDIATE, () -> FetchSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_FETCH_DATA_RSP ),
ACCORD_GET_EPHMRL_READ_DEPS_RSP (148, P2, readTimeout, IMMEDIATE, () -> GetEphmrlReadDepsSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_GET_EPHMRL_READ_DEPS_REQ (149, P2, readTimeout, IMMEDIATE, () -> GetEphmrlReadDepsSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_EPHMRL_READ_DEPS_RSP),
ACCORD_GET_LATEST_DEPS_RSP (150, P2, readTimeout, IMMEDIATE, () -> LatestDepsSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_GET_LATEST_DEPS_REQ (151, P2, readTimeout, IMMEDIATE, () -> LatestDepsSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_LATEST_DEPS_RSP),
ACCORD_GET_MAX_CONFLICT_RSP (152, P2, readTimeout, IMMEDIATE, () -> GetMaxConflictSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_GET_MAX_CONFLICT_REQ (153, P2, readTimeout, IMMEDIATE, () -> GetMaxConflictSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_MAX_CONFLICT_RSP),
ACCORD_GET_DURABLE_BEFORE_RSP (154, P2, readTimeout, IMMEDIATE, () -> GetDurableBeforeSerializers.reply, AccordService::responseHandlerOrNoop ),
ACCORD_GET_DURABLE_BEFORE_REQ (155, P2, readTimeout, IMMEDIATE, () -> GetDurableBeforeSerializers.request, AccordService::requestHandlerOrNoop, ACCORD_GET_DURABLE_BEFORE_RSP ),
ACCORD_SET_SHARD_DURABLE_REQ (156, P2, rpcTimeout, MISC, () -> SetDurableSerializers.shardDurable, AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ),
ACCORD_SET_GLOBALLY_DURABLE_REQ (157, P2, rpcTimeout, MISC, () -> SetDurableSerializers.globallyDurable,AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP ),
ACCORD_SYNC_NOTIFY_RSP (151, P2, writeTimeout, MISC, () -> EnumSerializer.simpleReply, RESPONSE_HANDLER),
ACCORD_SYNC_NOTIFY_REQ (152, P2, writeTimeout, MISC, () -> Notification.listSerializer, () -> AccordSyncPropagator.verbHandler, ACCORD_SYNC_NOTIFY_RSP ),
ACCORD_SYNC_NOTIFY_RSP (158, P2, writeTimeout, MISC, () -> EnumSerializer.simpleReply, RESPONSE_HANDLER),
ACCORD_SYNC_NOTIFY_REQ (159, P2, writeTimeout, MISC, () -> Notification.listSerializer, () -> AccordSyncPropagator.verbHandler, ACCORD_SYNC_NOTIFY_RSP ),
ACCORD_APPLY_AND_WAIT_REQ (153, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.readData, AccordService::requestHandlerOrNoop, ACCORD_READ_RSP),
CONSENSUS_KEY_MIGRATION (154, P1, writeTimeout, MISC, () -> ConsensusKeyMigrationFinished.serializer,() -> ConsensusKeyMigrationState.consensusKeyMigrationFinishedHandler),
CONSENSUS_KEY_MIGRATION (160, P1, writeTimeout, MISC, () -> ConsensusKeyMigrationFinished.serializer,() -> ConsensusKeyMigrationState.consensusKeyMigrationFinishedHandler),
ACCORD_INTEROP_READ_RSP (155, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.replySerializer, AccordService::responseHandlerOrNoop),
ACCORD_INTEROP_READ_REQ (156, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_RSP),
ACCORD_INTEROP_STABLE_THEN_READ_REQ(157, P2, writeTimeout, IMMEDIATE, () -> AccordInteropStableThenRead.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_RSP),
ACCORD_INTEROP_READ_REPAIR_RSP (158, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.replySerializer, AccordService::responseHandlerOrNoop),
ACCORD_INTEROP_READ_REPAIR_REQ (159, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_REPAIR_RSP),
ACCORD_INTEROP_APPLY_REQ (160, P2, writeTimeout, IMMEDIATE, () -> AccordInteropApply.serializer, AccordService::requestHandlerOrNoop, ACCORD_APPLY_RSP),
// TODO (desired): swap verb order to make IDS sequential?
ACCORD_FETCH_MIN_EPOCH_RSP (166, P0, shortTimeout, FETCH_METADATA, () -> FetchMinEpoch.Response.serializer, RESPONSE_HANDLER),
ACCORD_FETCH_MIN_EPOCH_REQ (165, P0, shortTimeout, FETCH_METADATA, () -> FetchMinEpoch.serializer, () -> FetchMinEpoch.handler, ACCORD_FETCH_MIN_EPOCH_RSP),
ACCORD_INTEROP_READ_RSP (161, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.replySerializer, AccordService::responseHandlerOrNoop),
ACCORD_INTEROP_READ_REQ (162, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_RSP),
ACCORD_INTEROP_STABLE_THEN_READ_REQ(163, P2, writeTimeout, IMMEDIATE, () -> AccordInteropStableThenRead.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_RSP),
ACCORD_INTEROP_READ_REPAIR_RSP (164, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.replySerializer, AccordService::responseHandlerOrNoop),
ACCORD_INTEROP_READ_REPAIR_REQ (165, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.requestSerializer, AccordService::requestHandlerOrNoop, ACCORD_INTEROP_READ_REPAIR_RSP),
ACCORD_INTEROP_APPLY_REQ (166, P2, writeTimeout, IMMEDIATE, () -> AccordInteropApply.serializer, AccordService::requestHandlerOrNoop, ACCORD_APPLY_RSP),
ACCORD_FETCH_MIN_EPOCH_RSP (167, P0, shortTimeout, FETCH_METADATA, () -> FetchMinEpoch.Response.serializer, RESPONSE_HANDLER),
ACCORD_FETCH_MIN_EPOCH_REQ (168, P0, shortTimeout, FETCH_METADATA, () -> FetchMinEpoch.serializer, () -> FetchMinEpoch.handler, ACCORD_FETCH_MIN_EPOCH_RSP),
ACCORD_FETCH_TOPOLOGY_RSP (169, P0, shortTimeout, FETCH_METADATA, () -> FetchTopology.Response.serializer, RESPONSE_HANDLER),
ACCORD_FETCH_TOPOLOGY_REQ (170, P0, shortTimeout, FETCH_METADATA, () -> FetchTopology.serializer, () -> FetchTopology.handler, ACCORD_FETCH_TOPOLOGY_RSP),

4
src/java/org/apache/cassandra/repair/RepairJob.java
View File

@ -43,7 +43,6 @@ import accord.primitives.Ranges;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
@ -191,8 +190,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
requireAllEndpoints = true;
}
logger.info("{} {}.{} starting accord repair, require all endpoints {}", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily, requireAllEndpoints);
IPartitioner partitioner = metadata.partitioner;
AccordRepair repair = new AccordRepair(ctx, cfs, partitioner, desc.keyspace, desc.ranges, requireAllEndpoints, allEndpoints);
AccordRepair repair = new AccordRepair(ctx, cfs, desc.sessionId, desc.keyspace, desc.ranges, requireAllEndpoints, allEndpoints);
return repair.repair(taskExecutor);
}, taskExecutor);
}

12
src/java/org/apache/cassandra/repair/messages/RepairMessage.java
View File

@ -45,8 +45,8 @@ import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.service.WaitStrategy;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.Backoff;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.TimeUUID;
@ -134,11 +134,11 @@ public abstract class RepairMessage
void onFailure(Exception e);
}
private static Backoff backoff(SharedContext ctx, Verb verb)
private static WaitStrategy backoff(SharedContext ctx, Verb verb)
{
RepairRetrySpec retrySpec = DatabaseDescriptor.getRepairRetrySpec();
RetrySpec spec = verb == Verb.VALIDATION_RSP ? retrySpec.getMerkleTreeResponseSpec() : retrySpec;
return Backoff.fromConfig(ctx, spec);
return RetrySpec.toStrategy(ctx, spec);
}
public static Supplier<Boolean> notDone(Future<?> f)
@ -172,12 +172,12 @@ public abstract class RepairMessage
}
@VisibleForTesting
static <T> void sendMessageWithRetries(SharedContext ctx, Backoff backoff, Supplier<Boolean> allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback<T> finalCallback)
static <T> void sendMessageWithRetries(SharedContext ctx, WaitStrategy backoff, Supplier<Boolean> allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback<T> finalCallback)
{
if (!ALLOWS_RETRY.contains(verb))
throw new AssertionError("Repair verb " + verb + " does not support retry, but a request to send with retry was given!");
BiConsumer<Integer, RequestFailureReason > maybeRecordRetry = (attempt, reason) -> {
if (attempt <= 0)
if (attempt <= 1)
return;
// we don't know what the prefix kind is... so use NONE... this impacts logPrefix as it will cause us to use "repair" rather than "preview repair" which may not be correct... but close enough...
String prefix = PreviewKind.NONE.logPrefix(request.parentRepairSession());
@ -229,7 +229,7 @@ public abstract class RepairMessage
(attempt, retryReason, from, failure) -> {
switch (retryReason)
{
case MaxRetries:
case GiveUp:
maybeRecordRetry.accept(attempt, failure.reason);
finalCallback.onFailure(from, failure);
return null;

2
src/java/org/apache/cassandra/service/FailureRecordingCallback.java
View File

@ -66,7 +66,7 @@ public abstract class FailureRecordingCallback<T> implements RequestCallbackWith
public static <O> void push(AtomicReferenceFieldUpdater<O, FailureResponses> headUpdater, O owner, InetAddressAndPort from, RequestFailureReason reason)
{
push(headUpdater, owner, new FailureResponses(from, reason));
getAndPush(headUpdater, owner, new FailureResponses(from, reason));
}
public static <O> void pushExclusive(AtomicReferenceFieldUpdater<O, FailureResponses> headUpdater, O owner, InetAddressAndPort from, RequestFailureReason reason)

223
src/java/org/apache/cassandra/service/RetryStrategy.java
View File

@ -20,32 +20,38 @@ package org.apache.cassandra.service;
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.config.DatabaseDescriptor;
import accord.utils.Invariants;
import accord.utils.RandomSource;
import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory;
import org.apache.cassandra.service.TimeoutStrategy.Wait;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.function.DoubleSupplier;
import java.util.function.LongBinaryOperator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static java.lang.Math.*;
import static java.util.Arrays.stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static java.util.concurrent.TimeUnit.*;
import static org.apache.cassandra.service.TimeoutStrategy.parseWait;
import static org.apache.cassandra.service.TimeoutStrategy.parseInMicros;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
/**
* <p>A strategy for making retry timing decisions for operations.
* The strategy is defined by four factors: <ul>
* The strategy is defined by six factors: <ul>
* <li> {@link #minMinMicros}
* <li> {@link #maxMaxMicros}
* <li> {@link #min}
* <li> {@link #max}
* <li> {@link #spread}
* <li> {@link #waitRandomizer}
* <li> {@link #maxAttempts}
* </ul>
*
* <p>The first three represent time periods, and may be defined dynamically based on a simple calculation over: <ul>
* <p>The first two represent the absolute upper and lower bound times we are permitted to produce as constants</p>
* <p>The next two represent time periods, and may be defined dynamically based on a simple calculation over: <ul>
* <li> {@code pX()} recent experienced latency distribution for successful operations,
* e.g. {@code p50(rw)} the maximum of read and write median latencies,
* {@code p999(r)} the 99.9th percentile of read latencies
@ -59,16 +65,13 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime;
* <li> dynamic linear {@code pX() * constant * attempts}
* <li> dynamic exponential {@code pX() * constant ^ attempts}
*
* <p>Furthermore, the dynamic calculations can be bounded with a min/max, like so:
* {@code min[mu]s <= dynamic expr <= max[mu]s}
*
* e.g.
* <li> {@code 10ms <= p50(rw)*0.66}
* <li> {@code 10ms <= p50(rw)*0.66...p99(rw)}
* <li> {@code 10ms <= p95(rw)*1.8^attempts <= 100ms}
* <li> {@code 5ms <= p50(rw)*0.5}
*
* <p>These calculations are put together to construct a range from which we draw a random number.
* The period we wait for {@code X} will be drawn so that {@code min <= X < max}.
* The period we wait for {@code X} will be drawn so that {@code minMin <= min <= max <= maxMax}.
*
* <p>With the constraint that {@code max} must be {@code spread} greater than {@code min},
* but no greater than its expression-defined maximum. {@code max} will be increased up until
@ -87,19 +90,34 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime;
* The quantized exponential specifier partitions the range into {@code attempts} buckets, then applies the pure
* exponential approach to draw values from [0..attempts), before drawing a uniform value from the corresponding bucket
*/
public class RetryStrategy
public class RetryStrategy implements WaitStrategy
{
private static final Pattern RANDOMIZER = Pattern.compile(
"uniform|exp(onential)?[(](?<exp>[0-9.]+)[)]|q(uantized)?exp(onential)?[(](?<qexp>[0-9.]+)[)]");
final static WaitRandomizerFactory randomizers = new WaitRandomizerFactory(){};
static final Pattern PARSE = Pattern.compile(
"(\\s*(?<minmin>0|[0-9]+[mu]?s)\\s*<=)?" +
"(\\s*(?<min>[^=]+)\\s*[.]{3})?" +
"(\\s*(?<max>[^=]+))" +
"(\\s*<=\\s*(?<maxmax>0|[0-9]+[mu]?s))?");
protected interface WaitRandomizer
public static final WaitRandomizerFactory randomizers = new WaitRandomizerFactory(){};
public interface WaitRandomizer
{
long wait(long min, long max, int attempts);
}
interface WaitRandomizerFactory
public static WaitRandomizerFactory randomizers(RandomSource random)
{
return new WaitRandomizerFactory()
{
@Override public LongBinaryOperator uniformLongSupplier() { return random::nextLong; }
@Override public DoubleSupplier uniformDoubleSupplier() { return random::nextDouble; }
};
}
public interface WaitRandomizerFactory
{
default LongBinaryOperator uniformLongSupplier() { return (min, max) -> ThreadLocalRandom.current().nextLong(min, max); } // DO NOT USE METHOD HANDLES (want to fetch afresh each time)
default DoubleSupplier uniformDoubleSupplier() { return () -> ThreadLocalRandom.current().nextDouble(); }
@ -108,7 +126,7 @@ public class RetryStrategy
default WaitRandomizer exponential(double power) { return new Exponential(uniformLongSupplier(), uniformDoubleSupplier(), power); }
default WaitRandomizer quantizedExponential(double power) { return new QuantizedExponential(uniformLongSupplier(), uniformDoubleSupplier(), power); }
static class Uniform implements WaitRandomizer
class Uniform implements WaitRandomizer
{
final LongBinaryOperator uniformLong;
@ -124,7 +142,7 @@ public class RetryStrategy
}
}
static abstract class AbstractExponential implements WaitRandomizer
abstract class AbstractExponential implements WaitRandomizer
{
final LongBinaryOperator uniformLong;
final DoubleSupplier uniformDouble;
@ -138,7 +156,7 @@ public class RetryStrategy
}
}
static class Exponential extends AbstractExponential
class Exponential extends AbstractExponential
{
public Exponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power)
{
@ -158,7 +176,7 @@ public class RetryStrategy
}
}
static class QuantizedExponential extends AbstractExponential
class QuantizedExponential extends AbstractExponential
{
public QuantizedExponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power)
{
@ -180,108 +198,111 @@ public class RetryStrategy
}
public final WaitRandomizer waitRandomizer;
public final Wait min, max, spread;
public final long minMinMicros, maxMaxMicros;
public final @Nullable Wait min;
public final @Nonnull Wait max;
public final int maxAttempts;
public RetryStrategy(String waitRandomizer, String min, String max, String spread, LatencySourceFactory latencies)
{
this.waitRandomizer = parseWaitRandomizer(waitRandomizer);
this.min = parseBound(min, true, latencies);
this.max = parseBound(max, false, latencies);
this.spread = parseBound(spread, true, latencies);
}
protected RetryStrategy(WaitRandomizer waitRandomizer, Wait min, Wait max, Wait spread)
protected RetryStrategy(WaitRandomizer waitRandomizer, long minMinMicros, Wait min, Wait max, long maxMaxMicros, int retries)
{
this.waitRandomizer = waitRandomizer;
this.minMinMicros = minMinMicros;
this.min = min;
this.max = max;
this.spread = spread;
this.maxMaxMicros = maxMaxMicros;
this.maxAttempts = retries == Integer.MAX_VALUE ? Integer.MAX_VALUE : retries + 1;
Invariants.require(maxAttempts >= 1);
}
protected Wait parseBound(String spec, boolean isMin, LatencySourceFactory latencies)
public long computeWaitUntil(int attempts)
{
long defaultMaxMicros = DatabaseDescriptor.getRpcTimeout(MICROSECONDS);
return parseWait(spec, 0, defaultMaxMicros, isMin ? 0 : defaultMaxMicros, latencies);
long wait = computeWait(attempts, NANOSECONDS);
if (wait < 0)
return -1;
return nanoTime() + wait;
}
protected long computeWaitUntil(int attempts)
public long computeWait(int attempt, TimeUnit units)
{
long wait = computeWait(attempts);
return nanoTime() + MICROSECONDS.toNanos(wait);
}
if (attempt > maxAttempts)
return -1;
protected long computeWait(int attempts)
{
long minWaitMicros = min.get(attempts);
long maxWaitMicros = max.get(attempts);
long spreadMicros = spread.get(attempts);
if (minWaitMicros + spreadMicros > maxWaitMicros)
long result;
if (min == null)
{
maxWaitMicros = minWaitMicros + spreadMicros;
if (maxWaitMicros > this.max.max)
result = max.getMicros(attempt);
}
else
{
long min = this.min.getMicros(attempt);
long max = this.max.getMicros(attempt);
result = min >= max ? min : waitRandomizer.wait(min, max, attempt);
}
if (result > maxMaxMicros) result = maxMaxMicros;
if (result < minMinMicros) result = minMinMicros;
return units.convert(result, MICROSECONDS);
}
public static RetryStrategy parse(String spec, LatencySourceFactory latencies)
{
return parse(spec, latencies, null);
}
public static RetryStrategy parse(String spec, LatencySourceFactory latencies, WaitRandomizer randomizer)
{
String original = spec;
int retries = Integer.MAX_VALUE;
int end = spec.length();
{
int next;
while ((next = spec.lastIndexOf(',', end - 1)) >= 0)
{
maxWaitMicros = this.max.max;
minWaitMicros = max(this.min.min, min(this.min.max, maxWaitMicros - spreadMicros));
int mid = spec.indexOf('=', next + 1);
if (mid <= next || mid >= end)
throw new IllegalArgumentException("Invalid modifier specification: '" + spec.substring(next, end) + "'; expecting '=' for value assignment");
String key = spec.substring(next + 1, mid).trim();
String value = spec.substring(mid + 1, end).trim();
switch (key)
{
default: throw new IllegalArgumentException("Invalid modifier specification: unrecognised property '" + key + '\'');
case "retries":
retries = Integer.parseInt(value);
break;
case "rnd":
if (randomizer != null)
throw new IllegalArgumentException("Randomizer already specified, cannot re-specify: " + value);
randomizer = parseWaitRandomizer(value);
break;
}
end = next;
}
if (end != spec.length())
spec = spec.substring(0, end);
}
return waitRandomizer.wait(minWaitMicros, maxWaitMicros, attempts);
}
Matcher m = PARSE.matcher(spec);
if (!m.matches())
throw new IllegalArgumentException("Invalid specification: '" + spec + "'; does not match " + PARSE);
public static class ParsedStrategy
{
public final String waitRandomizer, min, max, spread;
public final RetryStrategy strategy;
protected ParsedStrategy(String waitRandomizer, String min, String max, String spread, RetryStrategy strategy)
{
this.waitRandomizer = waitRandomizer;
this.min = min;
this.max = max;
this.spread = spread;
this.strategy = strategy;
}
public String toString()
{
return "min=" + min + ",max=" + max + ",spread=" + spread + ",random=" + waitRandomizer;
}
long minMin = parseInMicros(m.group("minmin"), 0);
long maxMax = parseInMicros(m.group("maxmax"), Long.MAX_VALUE);
Wait max = TimeoutStrategy.parseWait(m.group("max"), latencies);
String minSpec = m.group("min");
Wait min = minSpec == null ? null : TimeoutStrategy.parseWait(minSpec, latencies);
if (min == null && randomizer != null)
throw new IllegalArgumentException("Invalid to specify randomiser when no range specified: '" + original + '\'');
if (min instanceof Wait.Constant && minMin != 0)
throw new IllegalArgumentException("Invalid to specify an absolute minimum constant when the min bound is itself a constant: '" + original + '\'');
if (max instanceof Wait.Constant && maxMax != Long.MAX_VALUE)
throw new IllegalArgumentException("Invalid to specify an absolute maximum constant when the max bound is itself a constant: '" + original + '\'');
if (randomizer == null)
randomizer = randomizers.uniform();
return new RetryStrategy(randomizer, minMin, min, max, maxMax, retries);
}
@VisibleForTesting
public static ParsedStrategy parseStrategy(String spec, LatencySourceFactory latencies, ParsedStrategy defaultStrategy)
{
String[] args = spec.split(",");
String waitRandomizer = find(args, "random");
String min = find(args, "min");
String max = find(args, "max");
String spread = find(args, "spread");
if (spread == null)
spread = find(args, "delta");
if (waitRandomizer == null) waitRandomizer = defaultStrategy.waitRandomizer;
if (min == null) min = defaultStrategy.min;
if (max == null) max = defaultStrategy.max;
if (spread == null) spread = defaultStrategy.spread;
RetryStrategy strategy = new RetryStrategy(waitRandomizer, min, max, spread, latencies);
return new ParsedStrategy(waitRandomizer, min, max, spread, strategy);
}
protected static String find(String[] args, String param)
{
return stream(args).filter(s -> s.startsWith(param + '='))
.map(s -> s.substring(param.length() + 1))
.findFirst().orElse(null);
}
static WaitRandomizer parseWaitRandomizer(String input)
{
return parseWaitRandomizer(input, randomizers);
}
static WaitRandomizer parseWaitRandomizer(String input, WaitRandomizerFactory randomizers)
protected static WaitRandomizer parseWaitRandomizer(String input)
{
Matcher m = RANDOMIZER.matcher(input);
if (!m.matches())

29
src/java/org/apache/cassandra/service/StorageProxy.java
View File

@ -137,7 +137,7 @@ import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableParams;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.service.accord.IAccordService.IAccordResult;
import org.apache.cassandra.service.accord.txn.TxnData;
import org.apache.cassandra.service.accord.txn.TxnDataKeyValue;
import org.apache.cassandra.service.accord.txn.TxnDataValue;
@ -1258,7 +1258,7 @@ public class StorageProxy implements StorageProxyMBean
{
SplitMutations splitMutations = splitMutationsIntoAccordAndNormal(cm, (List<IMutation>)mutations);
List<? extends IMutation> accordMutations = splitMutations.accordMutations();
AsyncTxnResult accordResult = accordMutations != null ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null;
IAccordResult<TxnResult> accordResult = accordMutations != null ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null;
List<? extends IMutation> normalMutations = splitMutations.normalMutations();
Tracing.trace("Split mutations into Accord {} and normal {}", accordMutations, normalMutations);
@ -1298,8 +1298,7 @@ public class StorageProxy implements StorageProxyMBean
{
if (accordResult != null)
{
IAccordService accord = AccordService.instance();
TxnResult.Kind kind = accord.getTxnResult(accordResult).kind();
TxnResult.Kind kind = accordResult.awaitAndGet().kind();
if (kind == retry_new_protocol && failure == null)
{
Tracing.trace("Accord returned retry new protocol");
@ -1464,7 +1463,7 @@ public class StorageProxy implements StorageProxyMBean
}
// Start Accord executing so it executes while the mutations are synchronously applied
AsyncTxnResult accordResult = !accordMutations.isEmpty() ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null;
IAccordResult<TxnResult> accordResult = !accordMutations.isEmpty() ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null;
Throwable failure = null;
try
@ -1512,8 +1511,7 @@ public class StorageProxy implements StorageProxyMBean
// the batch log.
if (accordResult != null)
{
IAccordService accord = AccordService.instance();
TxnResult.Kind kind = accord.getTxnResult(accordResult).kind();
TxnResult.Kind kind = accordResult.awaitAndGet().kind();
if (kind == retry_new_protocol && failure == null)
continue;
Tracing.trace("Successfully wrote Accord mutations");
@ -2205,7 +2203,7 @@ public class StorageProxy implements StorageProxyMBean
return null;
}
public static AsyncTxnResult readWithAccord(ClusterMetadata cm, PartitionRangeReadCommand command, AbstractBounds<PartitionPosition> range, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
public static IAccordResult<TxnResult> readWithAccord(ClusterMetadata cm, PartitionRangeReadCommand command, AbstractBounds<PartitionPosition> range, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
if (consistencyLevel != null && !IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS.contains(consistencyLevel))
throw new InvalidRequestException(consistencyLevel + " is not supported by Accord");
@ -2220,7 +2218,7 @@ public class StorageProxy implements StorageProxyMBean
return accordService.coordinateAsync(tableMetadata.epoch.getEpoch(), txn, consistencyLevel, requestTime);
}
private static AsyncTxnResult readWithAccordAsync(ClusterMetadata cm, SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
private static IAccordResult<TxnResult> readWithAccordAsync(ClusterMetadata cm, SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
if (consistencyLevel != null && !IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS.contains(consistencyLevel))
throw new InvalidRequestException(consistencyLevel + " is not supported by Accord");
@ -2238,16 +2236,16 @@ public class StorageProxy implements StorageProxyMBean
private static ConsensusAttemptResult readWithAccord(ClusterMetadata cm, SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
AsyncTxnResult asyncTxnResult = readWithAccordAsync(cm, group, consistencyLevel, requestTime);
return getConsensusAttemptResultFromAsyncTxnResult(asyncTxnResult, group.queries.size(), index -> group.queries.get(index).isReversed());
IAccordResult<TxnResult> accordResult = readWithAccordAsync(cm, group, consistencyLevel, requestTime);
return getConsensusAttemptResultFromAsyncTxnResult(accordResult, group.queries.size(), index -> group.queries.get(index).isReversed());
}
/*
* Used for both the SERIAL and non-SERIAL read path into Accord
*/
public static ConsensusAttemptResult getConsensusAttemptResultFromAsyncTxnResult(AsyncTxnResult asyncTxnResult, int numQueries, IntPredicate isQueryReversed)
public static ConsensusAttemptResult getConsensusAttemptResultFromAsyncTxnResult(IAccordResult<TxnResult> accordResult, int numQueries, IntPredicate isQueryReversed)
{
TxnResult txnResult = AccordService.instance().getTxnResult(asyncTxnResult);
TxnResult txnResult = accordResult.awaitAndGet();
// TODO (required): Converge on a single approach to RETRY_NEW_PROTOCOL, this works for now because reads don't support it anyways
if (txnResult.kind() == retry_new_protocol)
return RETRY_NEW_PROTOCOL;
@ -2385,7 +2383,7 @@ public class StorageProxy implements StorageProxyMBean
{
SplitReads splitReads = splitReadsIntoAccordAndNormal(cm, group, coordinator, requestTime);
SinglePartitionReadCommand.Group accordReads = splitReads.accordReads;
AsyncTxnResult accordResult = accordReads != null ? readWithAccordAsync(cm, accordReads, consistencyLevel, requestTime) : null;
IAccordResult<TxnResult> accordResult = accordReads != null ? readWithAccordAsync(cm, accordReads, consistencyLevel, requestTime) : null;
SinglePartitionReadCommand.Group normalReads = splitReads.normalReads;
Tracing.trace("Split reads into Accord {} and normal {}", accordReads, normalReads);
@ -3347,9 +3345,6 @@ public class StorageProxy implements StorageProxyMBean
public Long getTruncateRpcTimeout() { return DatabaseDescriptor.getTruncateRpcTimeout(MILLISECONDS); }
public void setTruncateRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setTruncateRpcTimeout(timeoutInMillis); }
public Long getTransactionTimeout() { return DatabaseDescriptor.getTransactionTimeout(MILLISECONDS); }
public void setTransactionTimeout(Long value) { DatabaseDescriptor.setTransactionTimeout(value); }
public Long getNativeTransportMaxConcurrentConnections() { return DatabaseDescriptor.getNativeTransportMaxConcurrentConnections(); }
public void setNativeTransportMaxConcurrentConnections(Long nativeTransportMaxConcurrentConnections) { DatabaseDescriptor.setNativeTransportMaxConcurrentConnections(nativeTransportMaxConcurrentConnections); }

2
src/java/org/apache/cassandra/service/StorageProxyMBean.java
View File

@ -51,8 +51,6 @@ public interface StorageProxyMBean
public void setRangeRpcTimeout(Long timeoutInMillis);
public Long getTruncateRpcTimeout();
public void setTruncateRpcTimeout(Long timeoutInMillis);
public Long getTransactionTimeout();
public void setTransactionTimeout(Long timeoutInMillis);
public void setNativeTransportMaxConcurrentConnections(Long nativeTransportMaxConcurrentConnections);
public Long getNativeTransportMaxConcurrentConnections();

11
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -1270,17 +1270,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return DatabaseDescriptor.getTruncateRpcTimeout(MILLISECONDS);
}
public void setTransactionTimeout(long value)
{
DatabaseDescriptor.setTransactionTimeout(value);
logger.info("set transaction timeout to {} ms", value);
}
public long getTransactionTimeout()
{
return DatabaseDescriptor.getTransactionTimeout(MILLISECONDS);
}
/** @deprecated See CASSANDRA-15234 */
@Deprecated(since = "4.1")
public void setStreamThroughputMbPerSec(int value)

3
src/java/org/apache/cassandra/service/StorageServiceMBean.java
View File

@ -791,9 +791,6 @@ public interface StorageServiceMBean extends NotificationEmitter
public void setTruncateRpcTimeout(long value);
public long getTruncateRpcTimeout();
public void setTransactionTimeout(long value);
public long getTransactionTimeout();
public void setStreamThroughputMbitPerSec(int value);
/**
* @return stream_throughput_outbound in megabits

255
src/java/org/apache/cassandra/service/TimeoutStrategy.java
View File

@ -24,25 +24,26 @@ import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import accord.utils.Invariants;
import com.codahale.metrics.Snapshot;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.metrics.ClientRequestMetrics;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.service.TimeoutStrategy.LatencySupplier.Constant;
import org.apache.cassandra.service.TimeoutStrategy.LatencySupplier.Percentile;
import static java.lang.Double.parseDouble;
import static java.lang.Integer.parseInt;
import static java.lang.Math.max;
import static java.lang.Math.min;
import static java.lang.Math.pow;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.cassandra.config.DatabaseDescriptor.getCasContentionTimeout;
import static org.apache.cassandra.config.DatabaseDescriptor.getReadRpcTimeout;
import static org.apache.cassandra.config.DatabaseDescriptor.getWriteRpcTimeout;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
/**
@ -74,21 +75,21 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime;
* TODO (expected): permit simple constant addition (e.g. p50+5ms)
* TODO (required): track separate stats per-DC as inputs to these decisions
*/
public class TimeoutStrategy
public class TimeoutStrategy implements WaitStrategy
{
private static final Logger logger = LoggerFactory.getLogger(TimeoutStrategy.class);
static final Pattern PARSE = Pattern.compile(
"(\\s*(?<min>0|[0-9]+[mu]?s)\\s*<=)?" +
"(\\s*(?<wait>[^=]+))" +
"(\\s*<=\\s*(?<max>0|[0-9]+[mu]?s))?");
static final Pattern BOUND = Pattern.compile(
"(?<const>0|[0-9]+[mu]s)" +
"|((?<min>0|[0-9]+[mu]s) *<= *)?" +
"(p(?<perc>[0-9]+)(\\((?<rw>r|w|rw|wr)\\))?|(?<constbase>0|[0-9]+[mu]s))" +
"\\s*([*]\\s*(?<mod>[0-9.]+)?\\s*(?<modkind>[*^]\\s*attempts)?)?" +
"( *<= *(?<max>0|[0-9]+[mu]s))?");
static final Pattern WAIT = Pattern.compile(
"\\s*(?<const>0|[0-9]+[mu]?s)" +
"|\\s*((p(?<perc>[0-9]+)(\\((?<rw>r|w|rw|wr)\\))?)?|(?<constbase>0|[0-9]+[mu]?s))" +
"\\s*(([*]\\s*(?<mod>[0-9.]+))?\\s*(?<modkind>[*^]\\s*attempts)?)?\\s*");
static final Pattern TIME = Pattern.compile(
"0|([0-9]+)ms|([0-9]+)us");
"0|[0-9]+[mu]?s");
// Factories can be useful for testing purposes, to supply custom implementations of selectors and modifiers.
final static LatencySupplierFactory selectors = new LatencySupplierFactory(){};
final static LatencyModifierFactory modifiers = new LatencyModifierFactory(){};
interface LatencyModifierFactory
@ -99,9 +100,64 @@ public class TimeoutStrategy
default LatencyModifier multiplyByAttemptsExp(double base) { return (l, a) -> saturatedCast(l * pow(base, a)); }
}
interface LatencySupplier
public interface Wait
{
long get();
long getMicros(int attempts);
class Constant implements Wait
{
final long micros;
public Constant(long micros) { this.micros = micros; }
@Override public long getMicros(int attempts) { return micros; }
}
class Modifying implements Wait
{
final LatencySupplier supplier;
final LatencyModifier modifier;
Modifying(LatencySupplier supplier, LatencyModifier modifier)
{
this.supplier = supplier;
this.modifier = modifier;
}
@Override
public long getMicros(int attempts)
{
return modifier.modify(supplier.getMicros(), attempts);
}
}
}
public interface LatencySupplier
{
long getMicros();
class Constant implements LatencySupplier
{
final long micros;
public Constant(long micros) {this.micros = micros; }
@Override public long getMicros() { return micros; }
}
class Percentile implements LatencySupplier
{
final LatencySource latencies;
final double percentile;
public Percentile(LatencySource latencies, double percentile)
{
this.latencies = latencies;
this.percentile = percentile;
}
@Override
public long getMicros()
{
return latencies.get(percentile);
}
}
}
public interface LatencySource
@ -109,12 +165,6 @@ public class TimeoutStrategy
long get(double percentile);
}
interface LatencySupplierFactory
{
default LatencySupplier constant(long latency) { return () -> latency; }
default LatencySupplier percentile(double percentile, LatencySource latencies) { return () -> latencies.get(percentile); }
}
public interface LatencySourceFactory
{
LatencySource source(String params);
@ -129,6 +179,11 @@ public class TimeoutStrategy
LatencySource source = new TimeLimitedLatencySupplier(latencies.latency::getSnapshot, 10, SECONDS);
return ignore -> source;
}
static LatencySourceFactory none()
{
return ignore -> ignore2 -> { throw new UnsupportedOperationException(); };
}
}
public static class ReadWriteLatencySourceFactory implements LatencySourceFactory
@ -209,58 +264,22 @@ public class TimeoutStrategy
}
}
public static class Wait
{
final long min, max, onFailure;
final LatencyModifier modifier;
final LatencySupplier supplier;
Wait(long min, long max, long onFailure, LatencyModifier modifier, LatencySupplier supplier)
{
Preconditions.checkArgument(min<=max, "min (%s) must be less than or equal to max (%s)", min, max);
this.min = min;
this.max = max;
this.onFailure = onFailure;
this.modifier = modifier;
this.supplier = supplier;
}
long get(int attempts)
{
try
{
long base = supplier.get();
return max(min, min(max, modifier.modify(base, attempts)));
}
catch (Throwable t)
{
NoSpamLogger.getLogger(logger, 1L, MINUTES).info("", t);
return onFailure;
}
}
public String toString()
{
return "Bound{" +
"min=" + min +
", max=" + max +
", onFailure=" + onFailure +
", modifier=" + modifier +
", supplier=" + supplier +
'}';
}
}
final Wait wait;
final long minMicros, maxMicros;
public TimeoutStrategy(String spec, LatencySourceFactory latencies)
public TimeoutStrategy(Wait wait, long minMicros, long maxMicros)
{
this.wait = parseWait(spec, latencies);
this.minMicros = minMicros;
this.maxMicros = maxMicros;
this.wait = wait;
}
public long computeWait(int attempts, TimeUnit units)
{
return units.convert(wait.get(attempts), MICROSECONDS);
long wait = this.wait.getMicros(attempts);
if (wait < minMicros) wait = minMicros;
else if (wait > maxMicros) wait = maxMicros;
return units.convert(wait, MICROSECONDS);
}
public long computeWaitUntil(int attempts)
@ -269,32 +288,29 @@ public class TimeoutStrategy
return nanoTime() + nanos;
}
protected Wait parseWait(String spec, LatencySourceFactory latencies)
{
long defaultMicros = DatabaseDescriptor.getRpcTimeout(MICROSECONDS);
return parseWait(spec, 0, defaultMicros, defaultMicros, latencies);
}
private static LatencySupplier parseLatencySupplier(Matcher m, LatencySupplierFactory selectors, LatencySourceFactory latenciesFactory)
private static LatencySupplier parseLatencySupplier(Matcher m, LatencySourceFactory latenciesFactory)
{
String perc = m.group("perc");
if (perc == null)
return selectors.constant(parseInMicros(m.group("constbase")));
return new Constant(parseInMicros(m.group("constbase")));
LatencySource latencies = latenciesFactory.source(m.group("rw"));
String rw = m.group("rw");
if (rw == null) rw = "rw";
LatencySource latencies = latenciesFactory.source(rw);
double percentile = parseDouble("0." + perc);
return selectors.percentile(percentile, latencies);
return new Percentile(latencies, percentile);
}
private static LatencyModifier parseLatencyModifier(Matcher m, LatencyModifierFactory modifiers)
private static @Nullable LatencyModifier parseLatencyModifier(String spec, Matcher m, LatencyModifierFactory modifiers)
{
String mod = m.group("mod");
if (mod == null)
return modifiers.identity();
double modifier = parseDouble(mod);
String modkind = m.group("modkind");
double modifier = 1.0;
if (mod != null) modifier = Double.parseDouble(mod);
else if (modkind == null) return null;
else if (!modkind.startsWith("*"))
throw new IllegalArgumentException("Invalid latency modifier specification: " + spec + ". Expect constant factor as base for exponent.");
if (modkind == null)
return modifiers.multiply(modifier);
@ -313,31 +329,46 @@ public class TimeoutStrategy
return (long) v;
}
public static Wait parseWait(String input, long defaultMin, long defaultMax, long onFailure, LatencySourceFactory latencies)
public static TimeoutStrategy parse(String input, LatencySourceFactory latencies)
{
return parseWait(input, defaultMin, defaultMax, onFailure, latencies, selectors, modifiers);
Matcher m = PARSE.matcher(input);
if (!m.matches())
throw new IllegalArgumentException("Invalid specification: '" + input + "'; does not match " + PARSE);
long min = parseInMicros(m.group("min"), 0);
long max = parseInMicros(m.group("max"), Long.MAX_VALUE);
Wait wait = TimeoutStrategy.parseWait(m.group("wait"), latencies);
return new TimeoutStrategy(wait, min, max);
}
public static Wait parseWait(String input, LatencySourceFactory latencies)
{
return parseWait(input, latencies, modifiers);
}
@VisibleForTesting
public static Wait parseWait(String input, long defaultMinMicros, long defaultMaxMicros, long onFailure, LatencySourceFactory latencies, LatencySupplierFactory selectors, LatencyModifierFactory modifiers)
static Wait parseWait(String input, LatencySourceFactory latencies, LatencyModifierFactory modifiers)
{
Matcher m = BOUND.matcher(input);
Matcher m = WAIT.matcher(input);
if (!m.matches())
throw new IllegalArgumentException(input + " does not match " + BOUND);
throw new IllegalArgumentException(input + " does not match " + WAIT);
String maybeConst = m.group("const");
if (maybeConst != null)
{
long v = parseInMicros(maybeConst);
return new Wait(v, v, v, modifiers.identity(), selectors.constant(v));
return new Wait.Constant(v);
}
long min = parseInMicros(m.group("min"), defaultMinMicros);
long max = parseInMicros(m.group("max"), defaultMaxMicros);
return new Wait(min, max, onFailure, parseLatencyModifier(m, modifiers), parseLatencySupplier(m, selectors, latencies));
LatencySupplier supplier = parseLatencySupplier(m, latencies);
LatencyModifier modifier = parseLatencyModifier(input, m, modifiers);
if (modifier == null && supplier instanceof LatencySupplier.Constant)
return new Wait.Constant(((Constant) supplier).micros);
if (modifier == null)
modifier = modifiers.identity();
return new Wait.Modifying(supplier, modifier);
}
private static long parseInMicros(String input, long orElse)
public static long parseInMicros(String input, long orElse)
{
if (input == null)
return orElse;
@ -345,19 +376,25 @@ public class TimeoutStrategy
return parseInMicros(input);
}
private static long parseInMicros(String input)
public static long parseInMicros(String text)
{
Matcher m = TIME.matcher(input);
Matcher m = TIME.matcher(text);
if (!m.matches())
throw new IllegalArgumentException(input + " does not match " + TIME);
throw new IllegalArgumentException(text + " does not match " + TIME);
String text;
if (null != (text = m.group(1)))
return parseInt(text) * 1000;
else if (null != (text = m.group(2)))
return parseInt(text);
else
if (text.length() == 1)
{
Invariants.require(text.charAt(0) == '0');
return 0;
}
char penultimate = text.charAt(text.length() - 2);
switch (penultimate)
{
default: return parseInt(text.substring(0, text.length() - 1)) * 1000_000L;
case 'm': return parseInt(text.substring(0, text.length() - 2)) * 1000L;
case 'u': return parseInt(text.substring(0, text.length() - 2));
}
}
private static String orElse(Supplier<String> get, String orElse)
@ -365,4 +402,10 @@ public class TimeoutStrategy
String result = get.get();
return result != null ? result : orElse;
}
@VisibleForTesting
public static long maxQueryTimeoutMicros()
{
return max(max(getCasContentionTimeout(MICROSECONDS), getWriteRpcTimeout(MICROSECONDS)), getReadRpcTimeout(MICROSECONDS));
}
}

30
src/java/org/apache/cassandra/service/accord/AccordConfiguration.java → src/java/org/apache/cassandra/service/WaitStrategy.java
View File

@ -16,18 +16,32 @@
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
package org.apache.cassandra.service;
import accord.api.LocalConfig;
import org.apache.cassandra.config.Config;
import java.util.concurrent.TimeUnit;
// TODO (expected): should this be merged with AccordSpec?
public class AccordConfiguration implements LocalConfig
public interface WaitStrategy
{
private final Config config;
// a value of below 0 means give up
long computeWaitUntil(int attempts);
// a value of below 0 means give up
long computeWait(int attempts, TimeUnit units);
public AccordConfiguration(Config config)
enum None implements WaitStrategy
{
this.config = config;
INSTANCE;
@Override
public long computeWait(int attempts, TimeUnit units)
{
return -1;
}
@Override
public long computeWaitUntil(int attempts)
{
return -1;
}
}
}

11
src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java
View File

@ -65,7 +65,7 @@ import org.apache.cassandra.utils.concurrent.Future;
import static org.apache.cassandra.service.accord.AccordTopology.tcmIdToAccord;
import static org.apache.cassandra.utils.Simulate.With.MONITORS;
// TODO: listen to FailureDetector and rearrange fast path accordingly
// TODO (desired): listen to FailureDetector and rearrange fast path accordingly
@Simulate(with=MONITORS)
public class AccordConfigurationService extends AbstractConfigurationService<AccordConfigurationService.EpochState, AccordConfigurationService.EpochHistory> implements AccordEndpointMapper, AccordSyncPropagator.Listener, Shutdownable
{
@ -389,9 +389,6 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
receiveRemoteSyncCompletePreListenerNotify(removed, oldEpoch);
listeners.forEach(l -> l.onRemoveNode(epoch, removed));
if (shareShard(current, removed, localId))
AccordService.instance().tryMarkRemoved(current, removed);
}
private long[] nonCompletedEpochsBefore(long max)
@ -467,7 +464,9 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
fetchTopologyAsync(epoch_,
(topology, throwable) -> {
if (topology != null)
{
future.setSuccess(topology);
}
else
{
Invariants.require(future == pendingTopologies.remove(epoch_));
@ -607,12 +606,12 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
}
@Override
public void reportEpochRedundant(Ranges ranges, long epoch)
public void reportEpochRetired(Ranges ranges, long epoch)
{
checkStarted();
// TODO (expected): ensure we aren't fetching a truncated epoch; otherwise this should be non-null
Topology topology = getTopologyForEpoch(epoch);
syncPropagator.reportRedundant(epoch, topology.nodes(), ranges);
syncPropagator.reportRetired(epoch, topology.nodes(), ranges);
}
@Override

10
src/java/org/apache/cassandra/service/accord/AccordDataStore.java
View File

@ -85,11 +85,15 @@ public class AccordDataStore implements DataStore
for (Map.Entry<TableId, SnapshotBounds> e : tables.entrySet())
{
// TODO (required): is it safe to ignore null table metadata / cfs?
TableMetadata tableMetadata = metadata.schema.getTableMetadata(e.getKey());
if (!tableMetadata.isAccordEnabled())
if (tableMetadata == null || !tableMetadata.isAccordEnabled())
continue;
ColumnFamilyStore cfs = Keyspace.openAndGetStoreIfExists(tableMetadata);
if (cfs == null)
continue;
// TODO (required): when we can safely map TxnId.hlc() -> local timestamp, consult Memtable timestamps
Memtable memtable = cfs.getCurrentMemtable();
e.getValue().id = memtable.getMemtableId();
@ -102,6 +106,10 @@ public class AccordDataStore implements DataStore
TableMetadata tableMetadata = metadata.schema.getTableMetadata(e.getKey());
SnapshotBounds bounds = e.getValue();
ColumnFamilyStore cfs = Keyspace.openAndGetStoreIfExists(tableMetadata);
// TODO (required): is it safe to ignore null cfs?
if (cfs == null) continue;
View view = cfs.getTracker().getView();
for (Memtable memtable : view.getAllMemtables())
{

18
src/java/org/apache/cassandra/service/accord/AccordExecutor.java
View File

@ -18,6 +18,7 @@
package org.apache.cassandra.service.accord;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
@ -28,6 +29,7 @@ import java.util.function.IntFunction;
import accord.api.Agent;
import accord.api.RoutingKey;
import accord.local.AgentExecutor;
import accord.local.Command;
import accord.local.cfk.CommandsForKey;
import accord.primitives.TxnId;
@ -40,6 +42,8 @@ import accord.utils.QuintConsumer;
import accord.utils.TriConsumer;
import accord.utils.TriFunction;
import accord.utils.UnhandledEnum;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
import accord.utils.async.Cancellable;
import org.agrona.collections.Object2ObjectHashMap;
import org.apache.cassandra.cache.CacheSize;
@ -60,7 +64,7 @@ import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LO
import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLoaded, AccordCacheEntry.OnSaved, Shutdownable
public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLoaded, AccordCacheEntry.OnSaved, Shutdownable, AgentExecutor
{
public interface AccordExecutorFactory
{
@ -314,6 +318,18 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
return true;
}
@Override
public Agent agent()
{
return agent;
}
@Override
public <T> AsyncChain<T> build(Callable<T> task)
{
return AsyncChains.ofCallable(this, task);
}
private void parkRangeLoad(AccordTask<?> task)
{
if (task.queued() != waitingToLoadRangeTxns)

3
src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java
View File

@ -163,6 +163,9 @@ public abstract class AccordFastPathCoordinator implements ChangeListener, Confi
AccordFastPath fastPath = cm.accordFastPath;
long updateDelayMillis = getAccordFastPathUpdateDelayMillis();
if (updateDelayMillis < 0)
return;
if (isShutdown(fastPath))
{
updateFastPath(localId, Status.NORMAL, Clock.Global.currentTimeMillis(), updateDelayMillis);

272
src/java/org/apache/cassandra/service/accord/AccordMessageSink.java
View File

@ -18,18 +18,13 @@
package org.apache.cassandra.service.accord;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -39,13 +34,10 @@ import accord.local.AgentExecutor;
import accord.local.Node;
import accord.messages.Callback;
import accord.messages.MessageType;
import accord.messages.ReadData;
import accord.messages.Reply;
import accord.messages.ReplyContext;
import accord.messages.Request;
import accord.messages.TxnRequest;
import accord.primitives.TxnId;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
@ -54,157 +46,107 @@ import org.apache.cassandra.net.MessageFlag;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.ResponseContext;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.TimeoutStrategy;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.utils.Clock;
import static accord.messages.MessageType.Kind.REMOTE;
import static accord.primitives.Routable.Domain.Range;
import static accord.messages.MessageType.StandardMessage.*;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.expire;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowPreaccept;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowRead;
public class AccordMessageSink implements MessageSink
{
private static final Logger logger = LoggerFactory.getLogger(AccordMessageSink.class);
public static final class AccordMessageType extends MessageType
public enum AccordMessageType implements MessageType
{
public static final AccordMessageType INTEROP_READ_REQ = remote("INTEROP_READ_REQ");
public static final AccordMessageType INTEROP_READ_RSP = remote("INTEROP_READ_RSP");
public static final AccordMessageType INTEROP_STABLE_THEN_READ_REQ = remote("INTEROP_STABLE_THEN_READ_REQ");
public static final AccordMessageType INTEROP_READ_REPAIR_REQ = remote("INTEROP_READ_REPAIR_REQ");
public static final AccordMessageType INTEROP_READ_REPAIR_RSP = remote("INTEROP_READ_REPAIR_RSP");
public static final AccordMessageType INTEROP_APPLY_MINIMAL_REQ = remote("INTEROP_APPLY_MINIMAL_REQ");
public static final AccordMessageType INTEROP_APPLY_MAXIMAL_REQ = remote("INTEROP_APPLY_MAXIMAL_REQ");
INTEROP_READ_REQ(Verb.ACCORD_INTEROP_READ_REQ),
INTEROP_STABLE_THEN_READ_REQ(Verb.ACCORD_INTEROP_STABLE_THEN_READ_REQ),
INTEROP_READ_RSP(Verb.ACCORD_INTEROP_READ_RSP),
INTEROP_READ_REPAIR_REQ(Verb.ACCORD_INTEROP_READ_REPAIR_REQ),
INTEROP_READ_REPAIR_RSP(Verb.ACCORD_INTEROP_READ_REPAIR_RSP),
INTEROP_APPLY_REQ(Verb.ACCORD_INTEROP_APPLY_REQ);
final Verb verb;
public static final List<MessageType> values;
static
AccordMessageType(Verb verb)
{
ImmutableList.Builder<MessageType> builder = ImmutableList.builder();
for (Field f : AccordMessageType.class.getDeclaredFields())
{
if (f.getType().equals(AccordMessageType.class) && Modifier.isStatic(f.getModifiers()))
{
try
{
builder.add((MessageType) f.get(null));
}
catch (IllegalAccessException e)
{
throw new RuntimeException(e);
}
}
}
values = builder.build();
}
protected static AccordMessageType remote(String name)
{
return new AccordMessageType(name, REMOTE);
}
private AccordMessageType(String name, MessageType.Kind kind)
{
super(name, kind);
this.verb = verb;
}
}
private static class VerbMapping
{
private static final VerbMapping instance = new VerbMapping();
private static final Map<Verb, Set<Verb>> overrideReplyVerbs = ImmutableMap.<Verb, Set<Verb>>builder()
// read takes Result | Nack
.put(Verb.ACCORD_FETCH_DATA_REQ, EnumSet.of(Verb.ACCORD_FETCH_DATA_RSP, Verb.ACCORD_READ_RSP /* nack */))
.put(Verb.ACCORD_INTEROP_STABLE_THEN_READ_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_RSP, Verb.ACCORD_READ_RSP))
.put(Verb.ACCORD_INTEROP_READ_REPAIR_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_REPAIR_RSP, Verb.ACCORD_READ_RSP))
.build();
private final Map<MessageType, Verb> mapping;
private final Map<Verb, Set<Verb>> overrideReplyVerbs = ImmutableMap.<Verb, Set<Verb>>builder()
// read takes Result | Nack
.put(Verb.ACCORD_FETCH_DATA_REQ, EnumSet.of(Verb.ACCORD_FETCH_DATA_RSP, Verb.ACCORD_READ_RSP /* nack */))
.put(Verb.ACCORD_INTEROP_STABLE_THEN_READ_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_RSP, Verb.ACCORD_READ_RSP))
.put(Verb.ACCORD_INTEROP_READ_REPAIR_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_REPAIR_RSP, Verb.ACCORD_READ_RSP))
.build();
private VerbMapping()
static
{
ImmutableMap.Builder<MessageType, Verb> builder = ImmutableMap.builder();
builder.put(MessageType.SIMPLE_RSP, Verb.ACCORD_SIMPLE_RSP);
builder.put(MessageType.PRE_ACCEPT_REQ, Verb.ACCORD_PRE_ACCEPT_REQ);
builder.put(MessageType.PRE_ACCEPT_RSP, Verb.ACCORD_PRE_ACCEPT_RSP);
builder.put(MessageType.ACCEPT_REQ, Verb.ACCORD_ACCEPT_REQ);
builder.put(MessageType.ACCEPT_RSP, Verb.ACCORD_ACCEPT_RSP);
builder.put(MessageType.NOT_ACCEPT_REQ, Verb.ACCORD_NOT_ACCEPT_REQ);
builder.put(MessageType.RECOVER_AWAIT_REQ, Verb.ACCORD_RECOVER_AWAIT_REQ);
builder.put(MessageType.RECOVER_AWAIT_RSP, Verb.ACCORD_RECOVER_AWAIT_RSP);
builder.put(MessageType.GET_LATEST_DEPS_REQ, Verb.ACCORD_GET_LATEST_DEPS_REQ);
builder.put(MessageType.GET_LATEST_DEPS_RSP, Verb.ACCORD_GET_LATEST_DEPS_RSP);
builder.put(MessageType.GET_EPHEMERAL_READ_DEPS_REQ, Verb.ACCORD_GET_EPHMRL_READ_DEPS_REQ);
builder.put(MessageType.GET_EPHEMERAL_READ_DEPS_RSP, Verb.ACCORD_GET_EPHMRL_READ_DEPS_RSP);
builder.put(MessageType.COMMIT_SLOW_PATH_REQ, Verb.ACCORD_COMMIT_REQ);
builder.put(MessageType.COMMIT_MAXIMAL_REQ, Verb.ACCORD_COMMIT_REQ);
builder.put(MessageType.STABLE_FAST_PATH_REQ, Verb.ACCORD_COMMIT_REQ);
builder.put(MessageType.STABLE_SLOW_PATH_REQ, Verb.ACCORD_COMMIT_REQ);
builder.put(MessageType.STABLE_MAXIMAL_REQ, Verb.ACCORD_COMMIT_REQ);
builder.put(MessageType.COMMIT_INVALIDATE_REQ, Verb.ACCORD_COMMIT_INVALIDATE_REQ);
builder.put(MessageType.APPLY_MINIMAL_REQ, Verb.ACCORD_APPLY_REQ);
builder.put(MessageType.APPLY_MAXIMAL_REQ, Verb.ACCORD_APPLY_REQ);
builder.put(MessageType.APPLY_RSP, Verb.ACCORD_APPLY_RSP);
builder.put(MessageType.READ_REQ, Verb.ACCORD_READ_REQ);
builder.put(MessageType.STABLE_THEN_READ_REQ, Verb.ACCORD_STABLE_THEN_READ_REQ);
builder.put(MessageType.READ_EPHEMERAL_REQ, Verb.ACCORD_READ_REQ);
builder.put(MessageType.READ_RSP, Verb.ACCORD_READ_RSP);
builder.put(MessageType.BEGIN_RECOVER_REQ, Verb.ACCORD_BEGIN_RECOVER_REQ);
builder.put(MessageType.BEGIN_RECOVER_RSP, Verb.ACCORD_BEGIN_RECOVER_RSP);
builder.put(MessageType.BEGIN_INVALIDATE_REQ, Verb.ACCORD_BEGIN_INVALIDATE_REQ);
builder.put(MessageType.BEGIN_INVALIDATE_RSP, Verb.ACCORD_BEGIN_INVALIDATE_RSP);
builder.put(MessageType.AWAIT_REQ, Verb.ACCORD_AWAIT_REQ);
builder.put(MessageType.AWAIT_RSP, Verb.ACCORD_AWAIT_RSP);
builder.put(MessageType.ASYNC_AWAIT_COMPLETE_REQ, Verb.ACCORD_AWAIT_ASYNC_RSP_REQ);
builder.put(MessageType.WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_WAIT_UNTIL_APPLIED_REQ);
builder.put(MessageType.APPLY_THEN_WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_APPLY_AND_WAIT_REQ);
builder.put(MessageType.INFORM_DURABLE_REQ, Verb.ACCORD_INFORM_DURABLE_REQ);
builder.put(MessageType.CHECK_STATUS_REQ, Verb.ACCORD_CHECK_STATUS_REQ);
builder.put(MessageType.CHECK_STATUS_RSP, Verb.ACCORD_CHECK_STATUS_RSP);
builder.put(MessageType.FETCH_DATA_REQ, Verb.ACCORD_FETCH_DATA_REQ);
builder.put(MessageType.FETCH_DATA_RSP, Verb.ACCORD_FETCH_DATA_RSP);
builder.put(MessageType.SET_SHARD_DURABLE_REQ, Verb.ACCORD_SET_SHARD_DURABLE_REQ);
builder.put(MessageType.SET_GLOBALLY_DURABLE_REQ, Verb.ACCORD_SET_GLOBALLY_DURABLE_REQ);
builder.put(MessageType.QUERY_DURABLE_BEFORE_REQ, Verb.ACCORD_QUERY_DURABLE_BEFORE_REQ);
builder.put(MessageType.QUERY_DURABLE_BEFORE_RSP, Verb.ACCORD_QUERY_DURABLE_BEFORE_RSP);
builder.put(AccordMessageType.INTEROP_READ_REQ, Verb.ACCORD_INTEROP_READ_REQ);
builder.put(AccordMessageType.INTEROP_READ_RSP, Verb.ACCORD_INTEROP_READ_RSP);
builder.put(AccordMessageType.INTEROP_STABLE_THEN_READ_REQ, Verb.ACCORD_INTEROP_STABLE_THEN_READ_REQ);
builder.put(AccordMessageType.INTEROP_READ_REPAIR_REQ, Verb.ACCORD_INTEROP_READ_REPAIR_REQ);
builder.put(AccordMessageType.INTEROP_READ_REPAIR_RSP, Verb.ACCORD_INTEROP_READ_REPAIR_RSP);
builder.put(AccordMessageType.INTEROP_APPLY_MINIMAL_REQ, Verb.ACCORD_INTEROP_APPLY_REQ);
builder.put(AccordMessageType.INTEROP_APPLY_MAXIMAL_REQ, Verb.ACCORD_INTEROP_APPLY_REQ);
mapping = builder.build();
for (MessageType type : Iterables.concat(AccordMessageType.values, MessageType.values))
{
// Any request can receive a generic failure response
if (type == MessageType.FAILURE_RSP || type.isLocal())
continue;
if (mapping.containsKey(type))
{
if (type.isLocal()) throw new AssertionError("Extraneous mapping for LOCAL Accord MessageType " + type);
}
else
{
if (type.isRemote()) throw new AssertionError("Missing mapping for REMOTE Accord MessageType " + type);
}
}
ImmutableMap.Builder<StandardMessage, Verb> builder = ImmutableMap.builder();
builder.put(SIMPLE_RSP, Verb.ACCORD_SIMPLE_RSP);
builder.put(PRE_ACCEPT_REQ, Verb.ACCORD_PRE_ACCEPT_REQ);
builder.put(PRE_ACCEPT_RSP, Verb.ACCORD_PRE_ACCEPT_RSP);
builder.put(ACCEPT_REQ, Verb.ACCORD_ACCEPT_REQ);
builder.put(ACCEPT_RSP, Verb.ACCORD_ACCEPT_RSP);
builder.put(NOT_ACCEPT_REQ, Verb.ACCORD_NOT_ACCEPT_REQ);
builder.put(RECOVER_AWAIT_REQ, Verb.ACCORD_RECOVER_AWAIT_REQ);
builder.put(RECOVER_AWAIT_RSP, Verb.ACCORD_RECOVER_AWAIT_RSP);
builder.put(GET_LATEST_DEPS_REQ, Verb.ACCORD_GET_LATEST_DEPS_REQ);
builder.put(GET_LATEST_DEPS_RSP, Verb.ACCORD_GET_LATEST_DEPS_RSP);
builder.put(GET_EPHEMERAL_READ_DEPS_REQ, Verb.ACCORD_GET_EPHMRL_READ_DEPS_REQ);
builder.put(GET_EPHEMERAL_READ_DEPS_RSP, Verb.ACCORD_GET_EPHMRL_READ_DEPS_RSP);
builder.put(GET_MAX_CONFLICT_REQ, Verb.ACCORD_GET_MAX_CONFLICT_REQ);
builder.put(GET_MAX_CONFLICT_RSP, Verb.ACCORD_GET_MAX_CONFLICT_RSP);
builder.put(COMMIT_REQ, Verb.ACCORD_COMMIT_REQ);
builder.put(COMMIT_INVALIDATE_REQ, Verb.ACCORD_COMMIT_INVALIDATE_REQ);
builder.put(APPLY_REQ, Verb.ACCORD_APPLY_REQ);
builder.put(APPLY_RSP, Verb.ACCORD_APPLY_RSP);
builder.put(READ_REQ, Verb.ACCORD_READ_REQ);
builder.put(STABLE_THEN_READ_REQ, Verb.ACCORD_STABLE_THEN_READ_REQ);
builder.put(READ_EPHEMERAL_REQ, Verb.ACCORD_READ_REQ);
builder.put(READ_RSP, Verb.ACCORD_READ_RSP);
builder.put(BEGIN_RECOVER_REQ, Verb.ACCORD_BEGIN_RECOVER_REQ);
builder.put(BEGIN_RECOVER_RSP, Verb.ACCORD_BEGIN_RECOVER_RSP);
builder.put(BEGIN_INVALIDATE_REQ, Verb.ACCORD_BEGIN_INVALIDATE_REQ);
builder.put(BEGIN_INVALIDATE_RSP, Verb.ACCORD_BEGIN_INVALIDATE_RSP);
builder.put(AWAIT_REQ, Verb.ACCORD_AWAIT_REQ);
builder.put(AWAIT_RSP, Verb.ACCORD_AWAIT_RSP);
builder.put(ASYNC_AWAIT_COMPLETE_REQ, Verb.ACCORD_AWAIT_ASYNC_RSP_REQ);
builder.put(WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_WAIT_UNTIL_APPLIED_REQ);
builder.put(APPLY_THEN_WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_APPLY_AND_WAIT_REQ);
builder.put(INFORM_DURABLE_REQ, Verb.ACCORD_INFORM_DURABLE_REQ);
builder.put(CHECK_STATUS_REQ, Verb.ACCORD_CHECK_STATUS_REQ);
builder.put(CHECK_STATUS_RSP, Verb.ACCORD_CHECK_STATUS_RSP);
builder.put(FETCH_DATA_REQ, Verb.ACCORD_FETCH_DATA_REQ);
builder.put(FETCH_DATA_RSP, Verb.ACCORD_FETCH_DATA_RSP);
builder.put(SET_SHARD_DURABLE_REQ, Verb.ACCORD_SET_SHARD_DURABLE_REQ);
builder.put(SET_GLOBALLY_DURABLE_REQ, Verb.ACCORD_SET_GLOBALLY_DURABLE_REQ);
builder.put(GET_DURABLE_BEFORE_REQ, Verb.ACCORD_GET_DURABLE_BEFORE_REQ);
builder.put(GET_DURABLE_BEFORE_RSP, Verb.ACCORD_GET_DURABLE_BEFORE_RSP);
builder.put(FAILURE_RSP, Verb.FAILURE_RSP);
Map<StandardMessage, Verb> mapping = builder.build();
StandardMessage.initialise(mapping);
}
}
private static Verb getVerb(MessageType type)
{
return VerbMapping.instance.mapping.get(type);
}
private static Verb getVerb(MessageType type)
{
if (type.getClass() == StandardMessage.class)
return (Verb) ((StandardMessage) type).mapToImplementation();
return ((AccordMessageType)type).verb;
}
private static Verb getVerb(Request request)
{
MessageType type = request.type();
if (type != null)
return getVerb(type);
return null;
private static Verb getVerb(Request request)
{
MessageType type = request.type();
if (type != null)
return getVerb(type);
return null;
}
}
private final AccordAgent agent;
@ -228,7 +170,7 @@ public class AccordMessageSink implements MessageSink
@Override
public void send(Node.Id to, Request request)
{
Verb verb = getVerb(request);
Verb verb = VerbMapping.getVerb(request);
Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type());
Message<Request> message = Message.out(verb, request);
InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to);
@ -236,59 +178,43 @@ public class AccordMessageSink implements MessageSink
messaging.send(message, endpoint);
}
private static boolean isRangeBarrier(Request request)
{
TxnId txnId = null;
if (request instanceof TxnRequest)
{
TxnRequest<?> txnRequest = (TxnRequest<?>) request;
txnId = txnRequest.txnId;
}
else if (request instanceof ReadData)
{
ReadData epochRequest = (ReadData)request;
txnId = epochRequest.txnId;
}
return txnId != null && txnId.isSyncPoint() && txnId.is(Range);
}
// TODO (expected): permit bulk send to save esp. on callback registration (and combine records)
@Override
public void send(Node.Id to, Request request, AgentExecutor executor, Callback callback)
public void send(Node.Id to, Request request, int attempt, AgentExecutor executor, Callback callback)
{
Verb verb = getVerb(request);
Verb verb = VerbMapping.getVerb(request);
Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type());
long nowNanos = Clock.Global.nanoTime();
long delayedAtNanos = Long.MAX_VALUE;
long expiresAtNanos;
if (isRangeBarrier(request))
expiresAtNanos = nowNanos + DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos();
else
expiresAtNanos = nowNanos + verb.expiresAfterNanos();
TxnId txnId = request.primaryTxnId();
long slowAtNanos = Long.MAX_VALUE;
long expiresAtNanos = nowNanos + expire(txnId, verb).computeWait(attempt, NANOSECONDS);
switch (verb)
{
case ACCORD_READ_REQ:
case ACCORD_STABLE_THEN_READ_REQ:
if (agent.slowRead() == null || isRangeBarrier(request))
break;
case ACCORD_CHECK_STATUS_REQ:
delayedAtNanos = nowNanos + agent.slowRead().computeWait(1, NANOSECONDS);
{
TimeoutStrategy slow = slowRead(txnId);
if (slow != null)
slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS);
break;
}
case ACCORD_PRE_ACCEPT_REQ:
if (agent.slowPreaccept() == null || isRangeBarrier(request))
break;
delayedAtNanos = nowNanos + agent.slowPreaccept().computeWait(1, NANOSECONDS);
{
TimeoutStrategy slow = slowPreaccept(txnId);
if (slow != null)
slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS);
break;
}
}
Message<Request> message = Message.out(verb, request, expiresAtNanos);
InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to);
logger.trace("Sending {} {} to {}", verb, message.payload, endpoint);
callbacks.registerAt(message.id(), executor, callback, to, nowNanos, delayedAtNanos, expiresAtNanos, NANOSECONDS);
callbacks.registerAt(message.id(), executor, callback, to, nowNanos, slowAtNanos, expiresAtNanos, NANOSECONDS);
messaging.send(message, endpoint);
}
@ -317,7 +243,7 @@ public class AccordMessageSink implements MessageSink
private static void checkReplyType(Reply reply, ResponseContext respondTo)
{
Verb verb = getVerb(reply.type());
Verb verb = VerbMapping.getVerb(reply.type());
Preconditions.checkNotNull(verb, "Verb is null for type %s", reply.type());
Set<Verb> allowedVerbs = expectedReplyTypes(respondTo.verb());
Preconditions.checkArgument(allowedVerbs.contains(verb), "Expected reply message with verbs %s but got %s; reply type was %s, request verb was %s", allowedVerbs, verb, reply.type(), respondTo.verb());
@ -325,7 +251,7 @@ public class AccordMessageSink implements MessageSink
private static Set<Verb> expectedReplyTypes(Verb verb)
{
Set<Verb> extra = VerbMapping.instance.overrideReplyVerbs.get(verb);
Set<Verb> extra = VerbMapping.overrideReplyVerbs.get(verb);
if (extra != null) return extra;
Verb v = verb.responseVerb;
return v == null ? Collections.emptySet() : Collections.singleton(v);

4
src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java
View File

@ -25,7 +25,6 @@ import accord.api.Key;
import accord.api.Result;
import accord.api.RoutingKey;
import accord.local.Command;
import accord.local.Command.WaitingOn;
import accord.local.ICommand;
import accord.local.Node;
import accord.local.StoreParticipants;
@ -79,7 +78,6 @@ import static accord.local.Command.NotAcceptedWithoutDefinition.notAccepted;
import static accord.local.Command.NotDefined.notDefined;
import static accord.local.Command.PreAccepted.preaccepted;
import static accord.local.Command.Truncated.*;
import static accord.local.StoreParticipants.empty;
import static accord.local.cfk.CommandsForKey.InternalStatus.ACCEPTED;
import static accord.primitives.Status.Durability.NotDurable;
import static accord.primitives.TxnId.NO_TXNIDS;
@ -400,7 +398,7 @@ public class AccordObjectSizes
return size;
}
private static long EMPTY_WAITING_ON_SIZE = measure(new WaitingOn(null, null, null, null, null));
private static long EMPTY_WAITING_ON_SIZE = measure(new WaitingOn(null, null, null, null));
private static long EMPTY_BIT_SET_SIZE = measure(new ImmutableBitSet(0));
private static long waitingOn(WaitingOn waitingOn)
{

229
src/java/org/apache/cassandra/service/accord/AccordResult.java Normal file
View File

@ -0,0 +1,229 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.function.BiConsumer;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.coordinate.CoordinationFailed;
import accord.coordinate.Exhausted;
import accord.coordinate.Preempted;
import accord.coordinate.Timeout;
import accord.coordinate.TopologyMismatch;
import accord.primitives.Seekable;
import accord.primitives.Seekables;
import accord.primitives.TxnId;
import accord.utils.UnhandledEnum;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.txn.RetryWithNewProtocolResult;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
public class AccordResult<V> extends AsyncFuture<V> implements BiConsumer<V, Throwable>, IAccordService.IAccordResult<V>
{
private static final Logger logger = LoggerFactory.getLogger(AccordResult.class);
final @Nullable TxnId txnId;
final Seekables<?, ?> keysOrRanges;
final RequestBookkeeping bookkeeping;
final long startedAtNanos, deadlineAtNanos;
final boolean isTxnRequest;
public AccordResult(@Nullable TxnId txnId, Seekables<?, ?> keysOrRanges, RequestBookkeeping bookkeeping, long startedAtNanos, long deadlineAtNanos, boolean isTxnRequest)
{
this.txnId = txnId;
this.keysOrRanges = keysOrRanges;
this.bookkeeping = bookkeeping;
this.startedAtNanos = startedAtNanos;
this.deadlineAtNanos = deadlineAtNanos;
this.isTxnRequest = isTxnRequest;
}
@Override
public V awaitAndGet() throws RequestExecutionException
{
try
{
if (!awaitUntil(deadlineAtNanos))
accept(null, new Timeout(txnId, null));
}
catch (InterruptedException e)
{
accept(null, e);
}
Throwable fail = fail();
if (fail != null)
throw (RequestExecutionException) fail;
return success();
}
@Override
public void accept(V success, Throwable fail)
{
if (fail != null)
{
RequestExecutionException report;
CoordinationFailed coordinationFailed = findCoordinationFailed(fail);
TxnId txnId = this.txnId;
if (coordinationFailed != null)
{
if (txnId == null && coordinationFailed.txnId() != null)
txnId = coordinationFailed.txnId();
if (coordinationFailed instanceof Timeout)
{
report = bookkeeping.newTimeout(txnId, keysOrRanges);
}
else if (coordinationFailed instanceof Preempted)
{
report = bookkeeping.newPreempted(txnId, keysOrRanges);
}
else if (coordinationFailed instanceof Exhausted)
{
report = bookkeeping.newExhausted(txnId, keysOrRanges);
}
else if (isTxnRequest && coordinationFailed instanceof TopologyMismatch)
{
// Excluding bugs topology mismatch can occur because a table was dropped in between creating the txn
// and executing it.
// It could also race with the table stopping/starting being managed by Accord.
// The caller can retry if the table indeed exists and is managed by Accord.
Set<TableId> txnDroppedTables = txnDroppedTables(keysOrRanges);
Tracing.trace("Accord returned topology mismatch: " + coordinationFailed.getMessage());
logger.debug("Accord returned topology mismatch", coordinationFailed);
bookkeeping.markTopologyMismatch();
// Throw IRE in case the caller fails to check if the table still exists
if (!txnDroppedTables.isEmpty())
{
Tracing.trace("Accord txn uses dropped tables {}", txnDroppedTables);
logger.debug("Accord txn uses dropped tables {}", txnDroppedTables);
throw new InvalidRequestException("Accord transaction uses dropped tables");
}
trySuccess((V) RetryWithNewProtocolResult.instance);
return;
}
else
{
report = bookkeeping.newFailed(txnId, keysOrRanges);
}
// this case happens when a non-timeout exception is seen, and we are unable to move forward
if (txnId != null && txnId.isSyncPoint())
AccordAgent.onFailedBarrier(txnId, fail);
}
else
{
report = bookkeeping.newFailed(txnId, keysOrRanges);
}
report.addSuppressed(fail);
tryFailure(report);
}
else
{
if (success == RetryWithNewProtocolResult.instance)
{
bookkeeping.markRetryDifferentSystem();
Tracing.trace("Got retry different system error from Accord, will retry");
}
trySuccess(success);
}
bookkeeping.markElapsedNanos(nanoTime() - startedAtNanos);
}
@Override
public boolean awaitUntil(long nanoTimeDeadline) throws InterruptedException
{
if (super.awaitUntil(nanoTimeDeadline))
return true;
accept(null, new Timeout(null, null));
return false;
}
public Throwable fail()
{
return cause();
}
public V success()
{
return getNow();
}
@Override
public AccordResult<V> addCallback(BiConsumer<? super V, Throwable> callback)
{
super.addCallback(callback);
return this;
}
private static CoordinationFailed findCoordinationFailed(Throwable fail)
{
while (fail != null)
{
if (fail instanceof CoordinationFailed)
return (CoordinationFailed) fail;
Throwable next = fail.getCause();
if (next == fail)
return null;
fail = next;
}
return null;
}
private static Set<TableId> txnDroppedTables(Seekables<?,?> keys)
{
Set<TableId> tables = new HashSet<>();
for (Seekable seekable : keys)
{
switch (seekable.domain())
{
default: UnhandledEnum.unknown(seekable.domain());
case Key:
tables.add(((PartitionKey) seekable).table());
break;
case Range:
tables.add(((TokenRange) seekable).table());
break;
}
}
Iterator<TableId> tablesIterator = tables.iterator();
while (tablesIterator.hasNext())
{
TableId table = tablesIterator.next();
if (Schema.instance.getTableMetadata(table) != null)
tablesIterator.remove();
}
return tables;
}
}

828
src/java/org/apache/cassandra/service/accord/AccordService.java
View File

File diff suppressed because it is too large Load Diff

24
src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java
View File

@ -88,7 +88,7 @@ public class AccordSyncPropagator
{
final long epoch;
ImmutableSet<Node.Id> syncComplete = ImmutableSet.of(); // TODO (desired): propagate ack's for other nodes
Ranges closed = Ranges.EMPTY, redundant = Ranges.EMPTY;
Ranges closed = Ranges.EMPTY, retired = Ranges.EMPTY;
PendingEpoch(long epoch)
{
@ -118,19 +118,19 @@ public class AccordSyncPropagator
return new Notification(epoch, Collections.emptySet(), addClosed, Ranges.EMPTY);
}
Notification redundant(Ranges addRedundant)
Notification retired(Ranges addRetired)
{
if (redundant.containsAll(addRedundant))
if (retired.containsAll(addRetired))
return null;
addRedundant = addRedundant.without(redundant);
redundant = redundant.with(addRedundant);
return new Notification(epoch, Collections.emptySet(), Ranges.EMPTY, addRedundant);
addRetired = addRetired.without(retired);
retired = retired.with(addRetired);
return new Notification(epoch, Collections.emptySet(), Ranges.EMPTY, addRetired);
}
boolean isEmpty()
{
return syncComplete.isEmpty() && closed.isEmpty() && redundant.isEmpty();
return syncComplete.isEmpty() && closed.isEmpty() && retired.isEmpty();
}
boolean ack(Notification notification)
@ -141,8 +141,8 @@ public class AccordSyncPropagator
else syncComplete = ImmutableSet.copyOf(Iterables.filter(syncComplete, v -> !notification.syncComplete.contains(v)));
}
closed = closed.without(notification.closed);
redundant = redundant.without(notification.redundant);
return syncComplete.isEmpty() && closed.isEmpty() && redundant.isEmpty();
retired = retired.without(notification.redundant);
return syncComplete.isEmpty() && closed.isEmpty() && retired.isEmpty();
}
@Override
@ -152,7 +152,7 @@ public class AccordSyncPropagator
"epoch=" + epoch +
", syncComplete=" + syncComplete +
", closed=" + closed +
", redundant=" + redundant +
", redundant=" + retired +
'}';
}
}
@ -277,9 +277,9 @@ public class AccordSyncPropagator
report(epoch, notify, PendingEpoch::closed, closed);
}
public void reportRedundant(long epoch, Collection<Node.Id> notify, Ranges redundant)
public void reportRetired(long epoch, Collection<Node.Id> notify, Ranges redundant)
{
report(epoch, notify, PendingEpoch::redundant, redundant);
report(epoch, notify, PendingEpoch::retired, redundant);
}
private synchronized <T> void report(long epoch, Collection<Node.Id> notify, ReportPending<T> report, T param)

4
src/java/org/apache/cassandra/service/accord/AccordTopology.java
View File

@ -42,7 +42,6 @@ import accord.topology.Topology;
import accord.utils.Invariants;
import accord.utils.SortedArrays.SortedArrayList;
import accord.utils.TinyEnumSet;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
@ -393,7 +392,8 @@ public class AccordTopology
try
{
ClusterMetadataService.instance().fetchLogFromCMS(epoch);
AccordService.instance().epochReady(epoch).get(DatabaseDescriptor.getTransactionTimeout(MILLISECONDS), MILLISECONDS);
IAccordService service = AccordService.instance();
service.epochReady(epoch).get(service.agent().expireEpochWait(MILLISECONDS), MILLISECONDS);
}
catch (InterruptedException e)
{

83
src/java/org/apache/cassandra/service/accord/ClientRequestBookkeeping.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.util.function.Function;
import com.codahale.metrics.Meter;
import org.apache.cassandra.metrics.AccordClientRequestMetrics;
import org.apache.cassandra.metrics.ClientRequestMetrics;
import static org.apache.cassandra.service.accord.RequestBookkeeping.ThrowsExceptionType.READ;
import static org.apache.cassandra.service.accord.RequestBookkeeping.ThrowsExceptionType.WRITE;
// TODO (expected): merge with AccordClientRequestMetrics instead
public class ClientRequestBookkeeping extends RequestBookkeeping
{
final AccordClientRequestMetrics metrics;
public ClientRequestBookkeeping(boolean isWrite, AccordClientRequestMetrics metrics)
{
super(isWrite ? WRITE : READ);
this.metrics = metrics;
}
@Override
final void markTimeout()
{
mark(metrics -> metrics.timeouts);
}
@Override
final void markPreempted()
{
metrics.preempted.mark();
}
final void markFailure()
{
mark(metrics -> metrics.failures);
}
@Override
final void markRetryDifferentSystem()
{
metrics.retryDifferentSystem.mark();
}
@Override
final void markTopologyMismatch()
{
metrics.topologyMismatches.mark();
}
private void mark(Function<ClientRequestMetrics, Meter> get)
{
get.apply(metrics).mark();
if (metrics.shared != null)
get.apply(metrics.shared).mark();
}
public final void markElapsedNanos(long nanos)
{
metrics.addNano(nanos);
if (metrics.shared != null)
metrics.shared.addNano(nanos);
}
}

8
src/java/org/apache/cassandra/service/accord/FetchMinEpoch.java
View File

@ -28,7 +28,6 @@ import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterators;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.io.IVersionedSerializer;
@ -39,13 +38,14 @@ import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.utils.Backoff;
import org.apache.cassandra.service.WaitStrategy;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
import static org.apache.cassandra.net.MessageDelivery.RetryErrorMessage;
import static org.apache.cassandra.net.MessageDelivery.RetryPredicate;
import static org.apache.cassandra.net.MessageDelivery.logger;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryFetchMinEpoch;
// TODO (required, efficiency): this can be simplified: we seem to always use "entire range"
public class FetchMinEpoch
@ -80,7 +80,7 @@ public class FetchMinEpoch
}
else
{
logger.error("Accord service is not started, resopnding with error to {}", message);
logger.error("Accord service is not started, responding with error to {}", message);
MessagingService.instance().respondWithFailure(RequestFailure.BOOTING, message);
}
};
@ -110,7 +110,7 @@ public class FetchMinEpoch
@VisibleForTesting
static Future<Long> fetch(SharedContext context, InetAddressAndPort to)
{
Backoff backoff = Backoff.fromConfig(context, DatabaseDescriptor.getAccord().minEpochSyncRetry);
WaitStrategy backoff = retryFetchMinEpoch();
return context.messaging().<FetchMinEpoch, Response>sendWithRetries(backoff,
context.optionalTasks()::schedule,
Verb.ACCORD_FETCH_MIN_EPOCH_REQ,

7
src/java/org/apache/cassandra/service/accord/FetchTopology.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.Collection;
import accord.topology.Topology;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.io.IVersionedSerializer;
@ -35,10 +34,12 @@ import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.MessagingUtils;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.service.WaitStrategy;
import org.apache.cassandra.service.accord.serializers.TopologySerializers;
import org.apache.cassandra.utils.Backoff;
import org.apache.cassandra.utils.concurrent.Future;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryFetchTopology;
public class FetchTopology
{
public String toString()
@ -127,7 +128,7 @@ public class FetchTopology
public static Future<Topology> fetch(SharedContext context, Collection<InetAddressAndPort> peers, long epoch)
{
FetchTopology request = new FetchTopology(epoch);
Backoff backoff = Backoff.fromConfig(context, DatabaseDescriptor.getAccord().fetchRetry);
WaitStrategy backoff = retryFetchTopology();
return context.messaging().<FetchTopology, Response>sendWithRetries(backoff,
context.optionalTasks()::schedule,
Verb.ACCORD_FETCH_TOPOLOGY_REQ,

167
src/java/org/apache/cassandra/service/accord/IAccordService.java
View File

@ -18,11 +18,13 @@
package org.apache.cassandra.service.accord;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.BiConsumer;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -30,7 +32,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.BarrierType;
import accord.local.durability.DurabilityService.SyncLocal;
import accord.local.durability.DurabilityService.SyncRemote;
import accord.local.CommandStores.RangesForEpoch;
import accord.local.DurableBefore;
import accord.local.Node;
@ -38,21 +41,19 @@ import accord.local.Node.Id;
import accord.local.RedundantBefore;
import accord.messages.Reply;
import accord.messages.Request;
import accord.primitives.Keys;
import accord.primitives.Ranges;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.topology.Topology;
import accord.topology.TopologyManager;
import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.journal.Params;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.schema.TableId;
@ -61,15 +62,10 @@ import org.apache.cassandra.service.accord.api.AccordScheduler;
import org.apache.cassandra.service.accord.api.AccordTopologySorter;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.transport.Dispatcher.RequestTime;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.ImmediateFuture;
import static com.google.common.base.Preconditions.checkNotNull;
// Avoid default methods that aren't just providing wrappers around other methods
// so it will be a compile error if DelegatingAccordService doesn't implement them
public interface IAccordService
@ -82,55 +78,21 @@ public interface IAccordService
IVerbHandler<? extends Request> requestHandler();
IVerbHandler<? extends Reply> responseHandler();
Seekables<?, ?> barrierWithRetries(Seekables<?, ?> keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException;
AsyncChain<Void> sync(Object requestedBy, @Nullable Timestamp minBound, Ranges ranges, @Nullable Collection<Id> include, SyncLocal syncLocal, SyncRemote syncRemote);
AsyncChain<Void> sync(@Nullable Timestamp minBound, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote);
AsyncChain<Timestamp> maxConflict(Ranges ranges);
Seekables<?, ?> barrier(@Nonnull Seekables<?, ?> keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite);
@Nonnull IAccordResult<TxnResult> coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime);
@Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime) throws RequestExecutionException;
Seekables<?, ?> repairWithRetries(Seekables<?, ?> keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints) throws InterruptedException;
Seekables<?, ?> repair(@Nonnull Seekables<?, ?> keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints);
default void postStreamReceivingBarrier(ColumnFamilyStore cfs, List<Range<Token>> ranges)
interface IAccordResult<V>
{
String ks = cfs.keyspace.getName();
Ranges accordRanges = AccordTopology.toAccordRanges(ks, ranges);
try
{
barrierWithRetries(accordRanges, Epoch.FIRST.getEpoch(), BarrierType.global_async, true);
}
catch (InterruptedException e)
{
throw new RuntimeException(e);
}
V success();
Throwable fail();
V awaitAndGet() throws RequestExecutionException;
IAccordResult<V> addCallback(BiConsumer<? super V, Throwable> callback);
}
@Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime);
class AsyncTxnResult extends AsyncPromise<TxnResult>
{
public final @Nonnull TxnId txnId;
public final long minEpoch;
@Nullable
public final ConsistencyLevel consistencyLevel;
public final boolean isWrite;
public final Dispatcher.RequestTime requestTime;
public AsyncTxnResult(@Nonnull TxnId txnId, long minEpoch, @Nullable ConsistencyLevel consistencyLevel, boolean isWrite, @Nonnull Dispatcher.RequestTime requestTime)
{
checkNotNull(txnId);
checkNotNull(requestTime);
this.txnId = txnId;
this.minEpoch = minEpoch;
this.consistencyLevel = consistencyLevel;
this.isWrite = isWrite;
this.requestTime = requestTime;
}
}
@Nonnull
AsyncTxnResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime);
TxnResult getTxnResult(AsyncTxnResult asyncTxnResult);
long currentEpoch();
void setCacheSize(long kb);
@ -197,8 +159,9 @@ public interface IAccordService
@Nullable
Long minEpoch();
void tryMarkRemoved(Topology topology, Node.Id node);
void awaitTableDrop(TableId id);
void awaitDone(TableId id, long epoch);
AccordConfigurationService configService();
Params journalConfiguration();
@ -225,43 +188,31 @@ public interface IAccordService
}
@Override
public Seekables<?, ?> barrierWithRetries(Seekables<?, ?> keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
{
throw new UnsupportedOperationException();
}
@Override
public Seekables<?, ?> barrier(@Nonnull Seekables<?, ?> keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite)
{
throw new UnsupportedOperationException("No accord barriers should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
public Seekables<?, ?> repairWithRetries(Seekables<?, ?> keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints) throws InterruptedException
{
return null;
}
@Override
public Seekables<?, ?> repair(@Nonnull Seekables<?, ?> keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints)
{
throw new UnsupportedOperationException("No accord repairs should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
public @Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull Dispatcher.RequestTime requestTime)
public AsyncChain<Void> sync(Object requestedBy, @Nullable Timestamp onOrAfter, Ranges ranges, @Nullable Collection<Id> include, SyncLocal syncLocal, SyncRemote syncRemote)
{
throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
public @Nonnull AsyncTxnResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
public AsyncChain<Void> sync(@Nullable Timestamp onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote)
{
throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult)
public AsyncChain<Timestamp> maxConflict(Ranges ranges)
{
throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
public @Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull RequestTime requestTime)
{
throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
public @Nonnull IAccordResult<TxnResult> coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime)
{
throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml");
}
@ -347,15 +298,15 @@ public interface IAccordService
}
@Override
public void tryMarkRemoved(Topology topology, Id node)
public void awaitDone(TableId id, long epoch)
{
}
@Override
public void awaitTableDrop(TableId id)
public AccordConfigurationService configService()
{
return null;
}
@Override
@ -399,33 +350,27 @@ public interface IAccordService
}
@Override
public Seekables<?, ?> barrierWithRetries(Seekables<?, ?> keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
public AsyncChain<Void> sync(Object requestedBy, @Nullable Timestamp onOrAfter, Ranges ranges, @Nullable Collection<Id> include, SyncLocal syncLocal, SyncRemote syncRemote)
{
return delegate.barrierWithRetries(keysOrRanges, minEpoch, barrierType, isForWrite);
return delegate.sync(requestedBy, onOrAfter, ranges, include, syncLocal, syncRemote);
}
@Override
public Seekables<?, ?> barrier(@Nonnull Seekables<?, ?> keysOrRanges, long minEpoch, RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite)
public AsyncChain<Void> sync(@Nullable Timestamp onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote)
{
return delegate.barrier(keysOrRanges, minEpoch, requestTime, timeoutNanos, barrierType, isForWrite);
return delegate.sync(onOrAfter, keys, syncLocal, syncRemote);
}
@Override
public Seekables<?, ?> repairWithRetries(Seekables<?, ?> keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints) throws InterruptedException
public AsyncChain<Timestamp> maxConflict(Ranges ranges)
{
return delegate.repairWithRetries(keysOrRanges, minEpoch, barrierType, isForWrite, allEndpoints);
return delegate.maxConflict(ranges);
}
@Override
public Seekables<?, ?> repair(@Nonnull Seekables<?, ?> keysOrRanges, long epoch, RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints)
public AccordConfigurationService configService()
{
return delegate.repair(keysOrRanges, epoch, requestTime, timeoutNanos, barrierType, isForWrite, allEndpoints);
}
@Override
public void postStreamReceivingBarrier(ColumnFamilyStore cfs, List<Range<Token>> ranges)
{
IAccordService.super.postStreamReceivingBarrier(cfs, ranges);
return delegate.configService();
}
@Nonnull
@ -437,17 +382,11 @@ public interface IAccordService
@Nonnull
@Override
public AsyncTxnResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime)
public IAccordResult<TxnResult> coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, RequestTime requestTime)
{
return delegate.coordinateAsync(minEpoch, txn, consistencyLevel, requestTime);
}
@Override
public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult)
{
return delegate.getTxnResult(asyncTxnResult);
}
@Override
public long currentEpoch()
{
@ -534,15 +473,9 @@ public interface IAccordService
}
@Override
public void tryMarkRemoved(Topology topology, Id node)
public void awaitDone(TableId id, long epoch)
{
delegate.tryMarkRemoved(topology, node);
}
@Override
public void awaitTableDrop(TableId id)
{
delegate.awaitTableDrop(id);
delegate.awaitDone(id, epoch);
}
@Override

113
src/java/org/apache/cassandra/service/accord/RequestBookkeeping.java Normal file
View File

@ -0,0 +1,113 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.util.Collections;
import javax.annotation.Nullable;
import accord.primitives.Seekables;
import accord.primitives.TxnId;
import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.exceptions.ReadFailureException;
import org.apache.cassandra.exceptions.ReadTimeoutException;
import org.apache.cassandra.exceptions.RequestFailureException;
import org.apache.cassandra.exceptions.RequestTimeoutException;
import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.service.accord.exceptions.AccordReadExhaustedException;
import org.apache.cassandra.service.accord.exceptions.AccordReadPreemptedException;
import org.apache.cassandra.service.accord.exceptions.AccordWriteExhaustedException;
import org.apache.cassandra.service.accord.exceptions.AccordWritePreemptedException;
import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
import static org.apache.cassandra.service.accord.RequestBookkeeping.ThrowsExceptionType.WRITE;
public abstract class RequestBookkeeping
{
public enum ThrowsExceptionType { READ, WRITE }
final ThrowsExceptionType exceptionType;
protected RequestBookkeeping(ThrowsExceptionType exceptionType)
{
this.exceptionType = exceptionType;
}
abstract void markFailure();
abstract void markTimeout();
abstract void markPreempted();
abstract void markRetryDifferentSystem();
abstract void markTopologyMismatch();
abstract void markElapsedNanos(long nanos);
public RequestTimeoutException newTimeout(@Nullable TxnId txnId, Seekables<?, ?> keysOrRanges)
{
markTimeout();
return newTimeout(txnId, exceptionType, keysOrRanges);
}
public RequestTimeoutException newPreempted(@Nullable TxnId txnId, Seekables<?, ?> keysOrRanges)
{
markPreempted();
return newPreempted(txnId, exceptionType, keysOrRanges);
}
public RequestTimeoutException newExhausted(@Nullable TxnId txnId, Seekables<?, ?> keysOrRanges)
{
markFailure();
return newExhausted(txnId, exceptionType, keysOrRanges);
}
public RequestFailureException newFailed(@Nullable TxnId txnId, Seekables<?, ?> keysOrRanges)
{
markFailure();
return newFailed(txnId, exceptionType, keysOrRanges);
}
private static RequestTimeoutException newTimeout(TxnId txnId, ThrowsExceptionType type, Seekables<?, ?> keysOrRanges)
{
// Client protocol doesn't handle null consistency level so use ANY
return type == WRITE ? new WriteTimeoutException(WriteType.CAS, SERIAL, 0, 0, describe(txnId, keysOrRanges))
: new ReadTimeoutException(SERIAL, 0, 0, false, describe(txnId, keysOrRanges));
}
private static RequestTimeoutException newPreempted(TxnId txnId, ThrowsExceptionType type, Seekables<?, ?> keysOrRanges)
{
return type == WRITE ? new AccordWritePreemptedException(0, 0, describe(txnId, keysOrRanges))
: new AccordReadPreemptedException(0, 0, false, describe(txnId, keysOrRanges));
}
private static RequestTimeoutException newExhausted(TxnId txnId, ThrowsExceptionType type, Seekables<?, ?> keysOrRanges)
{
return type == WRITE ? new AccordWriteExhaustedException(0, 0, describe(txnId, keysOrRanges))
: new AccordReadExhaustedException(0, 0, false, describe(txnId, keysOrRanges));
}
private static RequestFailureException newFailed(TxnId txnId, ThrowsExceptionType type, Seekables<?, ?> keysOrRanges)
{
// TODO (required): plumb in per-peer failure responses from Accord, and describe the txnId/keys
return type == WRITE ? new WriteFailureException(SERIAL, 0, 0, WriteType.CAS, Collections.emptyMap())
: new ReadFailureException(SERIAL, 0, 0, false, Collections.emptyMap());
}
private static String describe(TxnId txnId, Seekables<?, ?> keysOrRanges)
{
return txnId + ": " + keysOrRanges;
}
}

72
src/java/org/apache/cassandra/service/accord/TimeOnlyRequestBookkeeping.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import org.apache.cassandra.metrics.LatencyMetrics;
import static org.apache.cassandra.service.accord.RequestBookkeeping.ThrowsExceptionType.READ;
public abstract class TimeOnlyRequestBookkeeping extends RequestBookkeeping
{
public static class LatencyRequestBookkeeping extends TimeOnlyRequestBookkeeping
{
final LatencyMetrics latency;
public LatencyRequestBookkeeping(LatencyMetrics latency)
{
this.latency = latency;
}
public final void markElapsedNanos(long nanos)
{
if (latency != null)
latency.addNano(nanos);
}
}
TimeOnlyRequestBookkeeping()
{
super(READ);
}
@Override
final void markTimeout()
{
}
@Override
final void markPreempted()
{
}
final void markFailure()
{
}
@Override
final void markRetryDifferentSystem()
{
throw new UnsupportedOperationException();
}
@Override
void markTopologyMismatch()
{
throw new UnsupportedOperationException();
}
}

184
src/java/org/apache/cassandra/service/accord/api/AccordAgent.java
View File

@ -19,6 +19,7 @@
package org.apache.cassandra.service.accord.api;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import javax.annotation.Nullable;
@ -27,7 +28,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.EventsListener;
import accord.api.EventListener;
import accord.api.ProgressLog.BlockedUntil;
import accord.api.Result;
import accord.api.RoutingKey;
@ -35,29 +36,31 @@ import accord.local.Command;
import accord.local.Node;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.local.TimeService;
import accord.messages.ReplyContext;
import accord.primitives.Keys;
import accord.primitives.Participants;
import accord.primitives.Ranges;
import accord.primitives.Routable;
import accord.primitives.Seekables;
import accord.primitives.Status;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.Txn.Kind;
import accord.primitives.TxnId;
import accord.topology.Shard;
import accord.topology.Topologies;
import accord.utils.DefaultRandom;
import accord.utils.Invariants;
import accord.utils.RandomSource;
import accord.utils.SortedList;
import accord.utils.UnhandledEnum;
import org.apache.cassandra.config.AccordSpec;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.metrics.AccordMetrics;
import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
import org.apache.cassandra.net.ResponseContext;
import org.apache.cassandra.service.TimeoutStrategy;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.txn.TxnQuery;
import org.apache.cassandra.service.accord.txn.TxnRead;
@ -65,24 +68,38 @@ import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.JVMStabilityInspector;
import static accord.primitives.Routable.Domain.Key;
import static accord.primitives.Txn.Kind.Write;
import static accord.utils.SortedArrays.SortedArrayList.ofSorted;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.cassandra.config.DatabaseDescriptor.getAccordScheduleDurabilityTxnIdLag;
import static org.apache.cassandra.config.DatabaseDescriptor.getReadRpcTimeout;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.expireEpochWait;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.fetch;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.recover;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryBootstrap;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryDurability;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retrySyncPoint;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowTxnPreaccept;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowRead;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
// TODO (expected): merge with AccordService
public class AccordAgent implements Agent
{
private static final Logger logger = LoggerFactory.getLogger(AccordAgent.class);
// TODO (required): this should be configurable and have exponential back-off, escaping to operator input past a certain number of retries
private long retryBootstrapDelayMicros = SECONDS.toMicros(1L);
private final RandomSource random = new DefaultRandom();
private static BiConsumer<TxnId, Throwable> onFailedBarrier;
public static void setOnFailedBarrier(BiConsumer<TxnId, Throwable> newOnFailedBarrier) { onFailedBarrier = newOnFailedBarrier; }
public static void onFailedBarrier(TxnId txnId, Throwable cause)
{
BiConsumer<TxnId, Throwable> invoke = onFailedBarrier;
if (invoke != null) invoke.accept(txnId, cause);
}
// TODO (required): make hot property
private TimeoutStrategy slowPreaccept, slowRead;
private final RandomSource random = new DefaultRandom();
protected Node.Id self;
public AccordAgent()
@ -94,11 +111,6 @@ public class AccordAgent implements Agent
self = id;
}
public void setRetryBootstrapDelay(long delay, TimeUnit units)
{
retryBootstrapDelayMicros = units.toMicros(delay);
}
@Override
public void onRecover(Node node, Result success, Throwable fail)
{
@ -114,22 +126,17 @@ public class AccordAgent implements Agent
throw error;
}
public void onFailedBarrier(TxnId id, Seekables<?, ?> keysOrRanges, Throwable cause)
{
}
@Override
public void onFailedBootstrap(String phase, Ranges ranges, Runnable retry, Throwable failure)
public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Throwable failure)
{
logger.error("Failed bootstrap at {} for {}", phase, ranges, failure);
AccordService.instance().scheduler().once(retry, retryBootstrapDelayMicros, MICROSECONDS);
AccordService.instance().scheduler().once(retry, retryBootstrap.computeWait(attempts, MICROSECONDS), MICROSECONDS);
}
@Override
public void onStale(Timestamp staleSince, Ranges ranges)
{
// TODO (required): decide how to handle this - maybe do nothing besides log? Maybe configurably try some number of repair attempts to catch up.
logger.error("This replica has become stale for {} as of {}", ranges, staleSince);
}
@Override
@ -147,10 +154,18 @@ public class AccordAgent implements Agent
}
@Override
public long preAcceptTimeout()
public Topologies selectPreferred(Node.Id from, Topologies to)
{
// TODO: should distinguish between reads and writes (Aleksey: why? and why read rpc timeout is being used?)
return getReadRpcTimeout(MICROSECONDS);
SortedList<Node.Id> nodes = to.nodes();
int i = nodes.indexOf(from);
Node.Id node = i <= 0 ? nodes.get(nodes.size() - 1) : to.nodes().get(i - 1);
return to.select(ofSorted(node));
}
@Override
public boolean rejectPreAccept(TimeService time, TxnId txnId)
{
return time.now() - getReadRpcTimeout(MICROSECONDS) > txnId.hlc();
}
// TODO (expected): we probably want additional configuration here so we can prune on shorter time horizons when we have a lot of transactions on a single key
@ -190,13 +205,13 @@ public class AccordAgent implements Agent
}
@Override
public EventsListener metricsEventsListener()
public EventListener eventListener()
{
return AccordMetrics.Listener.instance;
}
@Override
public long attemptCoordinationDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int retryCount)
public long slowCoordinatorDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int retryCount)
{
SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId);
Invariants.nonNull(safeCommand);
@ -210,8 +225,7 @@ public class AccordAgent implements Agent
// TODO (expected): make this a configurable calculation on normal request latencies (like ContentionStrategy)
long oneSecond = SECONDS.toMicros(1L);
long startTime = mostRecentAttempt.hlc() + DatabaseDescriptor.getAccord().recoveryDelayFor(txnId, MICROSECONDS)
+ (retryCount == 0 ? 0 : random.nextLong(oneSecond << Math.min(retryCount, 4)));
long startTime = mostRecentAttempt.hlc() + recover(txnId).computeWait(retryCount, MICROSECONDS);
startTime = nonClashingStartTime(startTime, shard == null ? null : shard.nodes, node.id(), oneSecond, random);
long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis());
@ -244,36 +258,34 @@ public class AccordAgent implements Agent
}
@Override
public long seekProgressDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int retryCount, BlockedUntil blockedUntil, TimeUnit units)
public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil blockedUntil, TimeUnit units)
{
// TODO (required): make this configurable and dependent upon normal request latencies, and perhaps offset from txnId.hlc()
return units.convert((1L << Math.min(retryCount, 4)), SECONDS);
return fetch(txnId).computeWait(attempt, units);
}
@Override
public long retryAwaitTimeout(Node node, SafeCommandStore safeStore, TxnId txnId, int retryCount, BlockedUntil retrying, TimeUnit units)
public long slowAwaitDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil retrying, TimeUnit units)
{
// TODO (expected): integrate with contention backoff
return units.convert((1L << Math.min(retryCount, 4)), SECONDS);
// TODO (desired): separate config?
return fetch(txnId).computeWait(attempt, units);
}
@Override
public long localSlowAt(TxnId txnId, Status.Phase phase, TimeUnit unit)
public long retrySyncPointDelay(Node node, int attempt, TimeUnit units)
{
switch (phase)
{
default: throw new UnhandledEnum(phase);
case PreAccept: return unit.convert(slowPreaccept().computeWaitUntil(1), NANOSECONDS);
case Execute: return unit.convert(slowRead().computeWaitUntil(1), NANOSECONDS);
}
return retrySyncPoint.computeWait(attempt, units);
}
@Override
public long localExpiresAt(TxnId txnId, Status.Phase phase, TimeUnit unit)
public long retryDurabilityDelay(Node node, int attempt, TimeUnit units)
{
// TODO (expected): make this configurable
return txnId.is(Write) ? DatabaseDescriptor.getWriteRpcTimeout(unit)
: DatabaseDescriptor.getReadRpcTimeout(unit);
return retryDurability.computeWait(attempt, units);
}
@Override
public long expireEpochWait(TimeUnit units)
{
return expireEpochWait.computeWait(1, units);
}
@Override
@ -282,35 +294,59 @@ public class AccordAgent implements Agent
return unit.convert(((ResponseContext)replyContext).expiresAtNanos(), NANOSECONDS);
}
@Override
public long selfSlowAt(TxnId txnId, Status.Phase phase, TimeUnit unit)
{
switch (phase)
{
default: throw new UnhandledEnum(phase);
case PreAccept: return unit.convert(slowTxnPreaccept.computeWaitUntil(1), NANOSECONDS);
case Execute: return unit.convert(slowRead.computeWaitUntil(1), NANOSECONDS);
}
}
@Override
public long selfExpiresAt(TxnId txnId, Status.Phase phase, TimeUnit unit)
{
long delayNanos;
switch (txnId.kind())
{
default: throw new UnhandledEnum(txnId.kind());
case Write:
delayNanos = DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS);
break;
case EphemeralRead:
case Read:
delayNanos = DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS);
break;
case ExclusiveSyncPoint:
delayNanos = DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos();
}
return unit.convert(nanoTime() + delayNanos, NANOSECONDS);
}
@Override
public AsyncChain<TxnId> awaitStaleId(Node node, TxnId staleId, boolean isRequested)
{
long waitMicros = (staleId.hlc() + getAccordScheduleDurabilityTxnIdLag(MICROSECONDS)) - node.now();
if (waitMicros <= 0)
return AsyncChains.success(staleId);
logger.debug("Waiting {} micros for {} to be stale", waitMicros, staleId);
AsyncResult.Settable<TxnId> result = AsyncResults.settable();
node.scheduler().selfRecurring(() -> result.setSuccess(staleId), waitMicros, MICROSECONDS);
return result;
}
@Override
public long minStaleHlc(Node node, boolean requested)
{
return node.now() - (100 + getAccordScheduleDurabilityTxnIdLag(MICROSECONDS));
}
@Override
public void onViolation(String message, Participants<?> participants, @Nullable TxnId notWitnessed, @Nullable Timestamp notWitnessedExecuteAt, @Nullable TxnId by, @Nullable Timestamp byEexecuteAt)
{
logger.error(message);
}
public TimeoutStrategy slowRead()
{
if (slowRead == null)
{
synchronized (this)
{
AccordSpec config = DatabaseDescriptor.getAccord();
slowRead = new TimeoutStrategy(config.slowRead, TimeoutStrategy.LatencySourceFactory.of(ClientRequestsMetricsHolder.accordReadMetrics));
}
}
return slowRead;
}
public TimeoutStrategy slowPreaccept()
{
if (slowPreaccept == null)
{
synchronized (this)
{
AccordSpec config = DatabaseDescriptor.getAccord();
slowPreaccept = new TimeoutStrategy(config.slowPreAccept, TimeoutStrategy.LatencySourceFactory.of(ClientRequestsMetricsHolder.accordReadMetrics));
}
}
return slowPreaccept;
}
}

184
src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java Normal file
View File

@ -0,0 +1,184 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.api;
import javax.annotation.Nullable;
import accord.primitives.TxnId;
import org.apache.cassandra.config.AccordSpec;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.RetryStrategy;
import org.apache.cassandra.service.TimeoutStrategy;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics;
import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory.none;
import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory.of;
import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory.rw;
public class AccordWaitStrategies
{
static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead;
static TimeoutStrategy expireTxn, expireSyncPoint, expireDurability, expireEpochWait;
static TimeoutStrategy fetchTxn, fetchSyncPoint;
static RetryStrategy recoverTxn, recoverSyncPoint, retrySyncPoint, retryDurability, retryBootstrap;
static RetryStrategy retryFetchMinEpoch, retryFetchTopology;
public static @Nullable TimeoutStrategy slowRead(@Nullable TxnId txnId)
{
if (txnId != null && txnId.isSyncPoint())
return null;
return slowRead;
}
public static TimeoutStrategy expire(@Nullable TxnId txnId, Verb verb)
{
if (txnId == null || !txnId.isSyncPoint())
return expireTxn;
return verb == Verb.ACCORD_WAIT_UNTIL_APPLIED_REQ ? expireDurability : expireSyncPoint;
}
public static TimeoutStrategy fetch(@Nullable TxnId txnId)
{
if (txnId == null || !txnId.isSyncPoint())
return fetchTxn;
return fetchSyncPoint;
}
public static TimeoutStrategy slowPreaccept(TxnId txnId)
{
if (txnId.isSyncPoint())
return expireSyncPoint;
return slowTxnPreaccept;
}
public static RetryStrategy retryFetchMinEpoch()
{
return retryFetchMinEpoch;
}
public static RetryStrategy retryFetchTopology()
{
return retryFetchTopology;
}
static
{
AccordSpec config = DatabaseDescriptor.getAccord();
setSlowRead(config.slow_read);
setSlowTxnPreaccept(config.slow_txn_preaccept);
setSlowSyncPointPreaccept(config.slow_syncpoint_preaccept);
setExpireTxn(config.expire_txn);
setExpireSyncPoint(config.expire_syncpoint);
setExpireDurability(config.expire_durability);
setExpireEpochWait(config.expire_epoch_wait);
setFetchTxn(config.fetch_txn);
setFetchSyncPoint(config.fetch_syncpoint);
setRecoverTxn(config.recover_txn);
setRecoverSyncPoint(config.recover_syncpoint);
setRetrySyncPoint(config.retry_syncpoint);
setRetryDurability(config.retry_durability);
setRetryFetchMinEpoch(config.retry_fetch_min_epoch);
setRetryFetchTopology(config.retry_fetch_topology);
}
public static void setSlowRead(String spec)
{
slowRead = TimeoutStrategy.parse(spec, of(accordReadMetrics));
}
public static void setSlowTxnPreaccept(String spec)
{
slowTxnPreaccept = TimeoutStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics));
}
public static void setSlowSyncPointPreaccept(String spec)
{
slowSyncPointPreaccept = TimeoutStrategy.parse(spec, none());
}
public static void setExpireTxn(String spec)
{
expireTxn = TimeoutStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics));
}
public static void setExpireSyncPoint(String spec)
{
expireSyncPoint = TimeoutStrategy.parse(spec, none());
}
public static void setExpireDurability(String spec)
{
expireDurability = TimeoutStrategy.parse(spec, none());
}
public static void setExpireEpochWait(String spec)
{
expireEpochWait = TimeoutStrategy.parse(spec, none());
}
public static void setFetchTxn(String spec)
{
fetchTxn = TimeoutStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics));
}
public static void setFetchSyncPoint(String spec)
{
fetchSyncPoint = TimeoutStrategy.parse(spec, none());
}
public static void setRecoverTxn(String spec)
{
recoverTxn = RetryStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics));
}
public static void setRecoverSyncPoint(String spec)
{
recoverSyncPoint = RetryStrategy.parse(spec, none());
}
public static void setRetrySyncPoint(String spec)
{
retrySyncPoint = RetryStrategy.parse(spec, none());
}
public static void setRetryDurability(String spec)
{
retryDurability = RetryStrategy.parse(spec, none());
}
public static void setRetryFetchMinEpoch(String spec)
{
retryFetchMinEpoch = RetryStrategy.parse(spec, none());
}
public static void setRetryFetchTopology(String spec)
{
retryFetchTopology = RetryStrategy.parse(spec, none());
}
static RetryStrategy recover(TxnId txnId)
{
if (txnId.is(ExclusiveSyncPoint))
return recoverSyncPoint;
return recoverTxn;
}
}

16
src/java/org/apache/cassandra/service/accord/api/PartitionKey.java
View File

@ -25,7 +25,7 @@ import com.google.common.base.Preconditions;
import accord.api.Key;
import accord.primitives.Routable;
import org.apache.cassandra.config.DatabaseDescriptor;
import accord.utils.Invariants;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.TypeSizes;
@ -33,16 +33,15 @@ import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.ObjectSizes;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
// final in part because we refer to its class directly in AccordRoutableKey.compareTo
public final class PartitionKey extends AccordRoutableKey implements Key
{
@ -50,7 +49,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key
static
{
DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER);
DecoratedKey key = getPartitioner().decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER);
EMPTY_SIZE = ObjectSizes.measureDeep(new PartitionKey(null, key));
}
@ -140,6 +139,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key
int position = offset;
position += key.table().serializeCompact(dst, accessor, position);
ByteBuffer bytes = key.partitionKey().getKey();
Invariants.require(key.partitionKey().getPartitioner() == getPartitioner());
int numBytes = ByteBufferAccessor.instance.size(bytes);
Preconditions.checkState(numBytes <= Short.MAX_VALUE);
position += accessor.putShort(dst, position, (short) numBytes);
@ -159,8 +159,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key
public PartitionKey deserialize(DataInputPlus in, int version) throws IOException
{
TableId tableId = TableId.deserializeCompact(in).intern();
IPartitioner partitioner = Schema.instance.getExistingTablePartitioner(tableId);
DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in));
DecoratedKey key = getPartitioner().decorateKey(ByteBufferUtil.readWithShortLength(in));
return new PartitionKey(tableId, key);
}
@ -168,12 +167,11 @@ public final class PartitionKey extends AccordRoutableKey implements Key
{
TableId tableId = TableId.deserializeCompact(src, accessor, offset).intern();
offset += tableId.serializedCompactSize();
TableMetadata metadata = Schema.instance.getTableMetadata(tableId);
int numBytes = accessor.getShort(src, offset);
offset += TypeSizes.SHORT_SIZE;
ByteBuffer bytes = ByteBuffer.allocate(numBytes);
accessor.copyTo(src, offset, bytes, ByteBufferAccessor.instance, 0, numBytes);
DecoratedKey key = metadata.partitioner.decorateKey(bytes);
DecoratedKey key = getPartitioner().decorateKey(bytes);
return new PartitionKey(tableId, key);
}

31
src/java/org/apache/cassandra/service/accord/api/TokenKey.java
View File

@ -296,6 +296,7 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran
return result;
}
// WARNING: consumes buffer!
public TokenKey deserialize(ByteBuffer buffer)
{
return deserialize(buffer, getPartitioner());
@ -303,8 +304,16 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran
public TokenKey deserialize(ByteBuffer buffer, IPartitioner partitioner)
{
TableId tableId = TableId.deserializeCompactComparable(buffer, ByteBufferAccessor.instance, 0);
int offset = tableId.serializedCompactComparableSize();
return deserializeWithPrefix(tableId, buffer.remaining() - offset, buffer, ByteBufferAccessor.instance, offset, partitioner);
}
// WARNING: consumes buffer!
public TokenKey deserializeAndConsume(ByteBuffer buffer, IPartitioner partitioner)
{
TableId tableId = TableId.deserializeCompactComparable(buffer, ByteBufferAccessor.instance, 0);
int offset = buffer.position();
TableId tableId = TableId.deserializeCompactComparable(buffer, ByteBufferAccessor.instance, offset);
buffer.position(offset + tableId.serializedCompactComparableSize());
return deserializeWithPrefix(tableId, buffer.remaining(), buffer, partitioner);
}
@ -403,21 +412,13 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran
return new TokenKey((TableId) tableId, sentinel, token);
}
public <V> TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, ByteBuffer buffer)
{
return deserializeWithPrefixAndImpliedLength(tableId, buffer, getPartitioner());
}
public <V> TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, ByteBuffer buffer, IPartitioner partitioner)
// WARNING: consumes buffer!
public TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, ByteBuffer buffer, IPartitioner partitioner)
{
return deserializeWithPrefix(tableId, buffer.remaining(), buffer, partitioner);
}
public TokenKey deserializeWithPrefix(Object tableId, int length, ByteBuffer buffer)
{
return deserializeWithPrefix(tableId, length, buffer, getPartitioner());
}
// WARNING: consumes buffer!
public TokenKey deserializeWithPrefix(Object tableId, int length, ByteBuffer buffer, IPartitioner partitioner)
{
byte sentinel = buffer.get();
@ -590,6 +591,12 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran
return subSplitter.splitRange(range, from, to, numSplits);
}
@Override
public Ranges selectFirstSubRanges(Range range, Ranges subRanges, int totalSplits)
{
return subSplitter.selectFirstSubRanges(range, subRanges, totalSplits);
}
@Override
public int numberOfSplitsPossible(Range range)
{

16
src/java/org/apache/cassandra/service/accord/exceptions/ReadExhaustedException.java → src/java/org/apache/cassandra/service/accord/exceptions/AccordReadExhaustedException.java
View File

@ -18,15 +18,19 @@
package org.apache.cassandra.service.accord.exceptions;
import com.google.common.collect.ImmutableMap;
import org.apache.cassandra.exceptions.ReadTimeoutException;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.exceptions.ReadFailureException;
import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
public class ReadExhaustedException extends ReadFailureException
public class AccordReadExhaustedException extends ReadTimeoutException
{
public ReadExhaustedException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent, String msg)
public AccordReadExhaustedException(int received, int blockFor, boolean dataPresent)
{
super(msg, consistency, received, blockFor, dataPresent, ImmutableMap.of());
super(SERIAL, received, blockFor, dataPresent);
}
public AccordReadExhaustedException(int received, int blockFor, boolean dataPresent, String msg)
{
super(SERIAL, received, blockFor, dataPresent, msg);
}
}

13
src/java/org/apache/cassandra/service/accord/exceptions/ReadPreemptedException.java → src/java/org/apache/cassandra/service/accord/exceptions/AccordReadPreemptedException.java
View File

@ -18,19 +18,20 @@
package org.apache.cassandra.service.accord.exceptions;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.exceptions.ReadTimeoutException;
import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
// shim to allow tests to tell the difference between preemption and other protocol timeouts
public class ReadPreemptedException extends ReadTimeoutException
public class AccordReadPreemptedException extends ReadTimeoutException
{
public ReadPreemptedException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent)
public AccordReadPreemptedException(int received, int blockFor, boolean dataPresent)
{
super(consistency, received, blockFor, dataPresent);
super(SERIAL, received, blockFor, dataPresent);
}
public ReadPreemptedException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent, String msg)
public AccordReadPreemptedException(int received, int blockFor, boolean dataPresent, String msg)
{
super(consistency, received, blockFor, dataPresent, msg);
super(SERIAL, received, blockFor, dataPresent, msg);
}
}

37
src/java/org/apache/cassandra/service/accord/exceptions/AccordWriteExhaustedException.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.exceptions;
import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
public class AccordWriteExhaustedException extends WriteTimeoutException
{
public AccordWriteExhaustedException(int received, int blockFor)
{
super(WriteType.CAS, SERIAL, received, blockFor);
}
public AccordWriteExhaustedException(int received, int blockFor, String msg)
{
super(WriteType.CAS, SERIAL, received, blockFor, msg);
}
}

13
src/java/org/apache/cassandra/service/accord/exceptions/WritePreemptedException.java → src/java/org/apache/cassandra/service/accord/exceptions/AccordWritePreemptedException.java
View File

@ -18,20 +18,21 @@
package org.apache.cassandra.service.accord.exceptions;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
// quick hack to allow tests to tell the difference between preemption and other protocol timeouts
public class WritePreemptedException extends WriteTimeoutException
public class AccordWritePreemptedException extends WriteTimeoutException
{
public WritePreemptedException(WriteType writeType, ConsistencyLevel consistency, int received, int blockFor)
public AccordWritePreemptedException(int received, int blockFor)
{
super(writeType, consistency, received, blockFor);
super(WriteType.CAS, SERIAL, received, blockFor);
}
public WritePreemptedException(WriteType writeType, ConsistencyLevel consistency, int received, int blockFor, String msg)
public AccordWritePreemptedException(int received, int blockFor, String msg)
{
super(writeType, consistency, received, blockFor, msg);
super(WriteType.CAS, SERIAL, received, blockFor, msg);
}
}

4
src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java
View File

@ -53,7 +53,7 @@ import static accord.messages.Apply.Kind.Minimal;
public class AccordInteropAdapter extends TxnAdapter
{
private static final Logger logger = LoggerFactory.getLogger(AccordInteropAdapter.class);
public static final class AccordInteropFactory implements CoordinationAdapter.Factory
public static final class AccordInteropFactory extends DefaultFactory
{
final AccordInteropAdapter standard, recovery;
@ -67,6 +67,8 @@ public class AccordInteropAdapter extends TxnAdapter
@Override
public <R> CoordinationAdapter<R> get(TxnId txnId, Kind step)
{
if (txnId.isSyncPoint())
return super.get(txnId, step);
return (CoordinationAdapter<R>) (step == Kind.Recovery ? recovery : standard);
}
};

7
src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java
View File

@ -221,12 +221,7 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
@Override
public MessageType type()
{
switch (kind)
{
case Minimal: return AccordMessageType.INTEROP_APPLY_MINIMAL_REQ;
case Maximal: return AccordMessageType.INTEROP_APPLY_MAXIMAL_REQ;
default: throw new IllegalStateException();
}
return AccordMessageType.INTEROP_APPLY_REQ;
}
@Override

172
src/java/org/apache/cassandra/service/accord/repair/AccordRepair.java
View File

@ -18,37 +18,43 @@
package org.apache.cassandra.service.accord.repair;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.BarrierType;
import accord.api.RoutingKey;
import accord.local.durability.DurabilityService;
import accord.local.Node;
import accord.primitives.Ranges;
import accord.primitives.Seekables;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.dht.AccordSplitter;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.LatencyMetrics;
import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.AccordTopology;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.RequestBookkeeping;
import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.cassandra.config.CassandraRelevantProperties.ACCORD_REPAIR_RANGE_STEP_UPDATE_INTERVAL;
import static accord.local.durability.DurabilityService.SyncLocal.NoLocal;
import static accord.local.durability.DurabilityService.SyncRemote.All;
import static accord.local.durability.DurabilityService.SyncRemote.Quorum;
import static accord.primitives.Timestamp.mergeMax;
import static accord.primitives.Timestamp.minForEpoch;
import static org.apache.cassandra.config.DatabaseDescriptor.getAccordRepairTimeoutNanos;
/*
* Accord repair consists of creating a barrier transaction for all the ranges which ensure that all Accord transactions
@ -56,33 +62,28 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.ACCORD_REP
*/
public class AccordRepair
{
private static final Logger logger = LoggerFactory.getLogger(AccordRepair.class);
public static final BigInteger TWO = BigInteger.valueOf(2);
private final SharedContext ctx;
private final ColumnFamilyStore cfs;
private final TimeUUID repairId;
private final Ranges ranges;
private final AccordSplitter splitter;
private final boolean requireAllEndpoints;
private final List<InetAddressAndPort> endpoints;
private BigInteger rangeStep;
private final Epoch minEpoch = ClusterMetadata.current().epoch;
private volatile Throwable shouldAbort = null;
private volatile Thread waiting;
public AccordRepair(SharedContext ctx, ColumnFamilyStore cfs, IPartitioner partitioner, String keyspace, Collection<Range<Token>> ranges, boolean requireAllEndpoints, List<InetAddressAndPort> endpoints)
public AccordRepair(SharedContext ctx, ColumnFamilyStore cfs, TimeUUID repairId, String keyspace, Collection<Range<Token>> ranges, boolean requireAllEndpoints, List<InetAddressAndPort> endpoints)
{
this.ctx = ctx;
this.cfs = cfs;
this.repairId = repairId;
this.requireAllEndpoints = requireAllEndpoints;
this.endpoints = endpoints;
this.ranges = AccordTopology.toAccordRanges(keyspace, ranges);
this.splitter = partitioner.accordSplitter().apply(this.ranges);
}
public Epoch minEpoch()
@ -117,100 +118,65 @@ public class AccordRepair
protected void abort(@Nullable Throwable reason)
{
shouldAbort = reason == null ? new RuntimeException("Abort") : reason;
Thread thread = waiting;
if (thread != null)
thread.interrupt();
}
private List<accord.primitives.Range> repairRange(TokenRange range) throws Throwable
{
List<accord.primitives.Range> repairedRanges = new ArrayList<>();
int rangeStepUpdateInterval = ACCORD_REPAIR_RANGE_STEP_UPDATE_INTERVAL.getInt();
RoutingKey remainingStart = range.start();
// TODO (expected): repair ranges should have a configurable lower limit of split size so already small repairs aren't broken up into excessively tiny ones
// TODO (expected): we should support lower range divisions for accord repair
BigInteger rangeSize = splitter.sizeOf(range);
if (rangeStep == null)
List<Node.Id> ids = endpoints == null ? null : endpoints.stream().map(AccordService.instance().configService()::mappedId).collect(Collectors.toList());
DurabilityService.SyncRemote syncRemote = requireAllEndpoints ? All : Quorum;
if (shouldAbort != null)
throw shouldAbort;
LatencyMetrics latency = null;
{
BigInteger divide = splitter.divide(rangeSize, 10000);
rangeStep = divide.equals(BigInteger.ZERO) ? rangeSize : BigInteger.ONE.max(divide);
TableMetadata metadata = Schema.instance.getTableMetadata(range.table());
if (metadata != null)
{
ColumnFamilyStore cfs = Keyspace.openAndGetStore(metadata);
if (cfs != null)
latency = cfs.metric.accordRepair;
}
}
BigInteger offset = BigInteger.ZERO;
TokenRange lastRepaired = null;
int iteration = 0;
while (true)
long start = ctx.clock().nanoTime();
try
{
IAccordService service = AccordService.instance();
Ranges ranges = AccordService.intersecting(Ranges.of(range));
waiting = Thread.currentThread();
RequestBookkeeping bookkeeping = new LatencyRequestBookkeeping(latency);
AccordService.getBlocking(service.maxConflict(ranges).flatMap(conflict -> {
conflict = mergeMax(conflict, minForEpoch(this.minEpoch.getEpoch()));
return service.sync("[repairId #" + repairId + ']', conflict, Ranges.of(range), ids, NoLocal, syncRemote);
}), ranges, bookkeeping, start, start + getAccordRepairTimeoutNanos());
waiting = null;
if (shouldAbort != null)
throw shouldAbort;
iteration++;
if (iteration % rangeStepUpdateInterval == 0)
rangeStep = rangeStep.multiply(TWO);
BigInteger remaining = rangeSize.subtract(offset);
BigInteger length = remaining.min(rangeStep);
long start = ctx.clock().nanoTime();
boolean dependencyOverflow = false;
try
{
// Splitter is approximate so it can't work right up to the end
TokenRange toRepair;
if (splitter.compare(offset, rangeSize) >= 0)
{
if (remainingStart.equals(range.end()))
{
logger.info("Completed barriers for {} in {} iterations", range, iteration - 1);
return repairedRanges;
}
// Final repair is whatever remains
toRepair = range.newRange(remainingStart, range.end());
}
else
{
toRepair = splitter.subRange(range, offset, splitter.add(offset, length));
checkState(iteration > 1 || toRepair.start().equals(range.start()));
}
checkState(!toRepair.equals(lastRepaired), "Shouldn't repair the same range twice");
checkState(lastRepaired == null || toRepair.start().equals(lastRepaired.end()), "Next range should directly follow previous range");
lastRepaired = toRepair;
Ranges barrieredRanges;
if (requireAllEndpoints)
barrieredRanges = (Ranges)AccordService.instance().repairWithRetries(Seekables.of(toRepair), minEpoch.getEpoch(), BarrierType.global_sync, false, endpoints);
else
barrieredRanges = (Ranges)AccordService.instance().barrierWithRetries(Seekables.of(toRepair), minEpoch.getEpoch(), BarrierType.global_sync, false);
for (accord.primitives.Range barrieredRange : barrieredRanges)
repairedRanges.add(barrieredRange);
remainingStart = toRepair.end();
}
catch (RuntimeException e)
{
cfs.metric.accordRepairUnexpectedFailures.mark();
throw e;
}
catch (Throwable t)
{
cfs.metric.accordRepairUnexpectedFailures.mark();
throw new RuntimeException(t);
}
finally
{
long end = ctx.clock().nanoTime();
cfs.metric.accordRepair.addNano(end - start);
}
// TODO when dependency limits are added to Accord need to test repair overflow
if (dependencyOverflow)
{
offset = offset.subtract(rangeStep);
if (rangeStep.equals(BigInteger.ONE))
throw new IllegalStateException("Unable to repair without overflowing with range step of 1");
rangeStep = BigInteger.ONE.max(rangeStep.divide(TWO));
continue;
}
offset = offset.add(length);
for (accord.primitives.Range r : ranges)
repairedRanges.add(r);
}
catch (Throwable t)
{
cfs.metric.accordRepairUnexpectedFailures.mark();
if (shouldAbort != null)
{
shouldAbort.addSuppressed(t);
throw shouldAbort;
}
throw t;
}
finally
{
long end = ctx.clock().nanoTime();
cfs.metric.accordRepair.addNano(end - start);
}
return repairedRanges;
}
}

80
src/java/org/apache/cassandra/service/accord/repair/RepairSyncPointAdapter.java
View File

@ -1,80 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.repair;
import java.util.Collection;
import java.util.function.BiConsumer;
import com.google.common.collect.ImmutableSet;
import accord.api.Result;
import accord.coordinate.CoordinationAdapter;
import accord.coordinate.ExecuteFlag;
import accord.coordinate.ExecutePath;
import accord.coordinate.ExecuteSyncPoint;
import accord.local.Node;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.primitives.Unseekable;
import accord.primitives.Writes;
import accord.topology.Topologies;
/**
* Sync point adapter used for accord-only repairs.
*
* Repair has the requirement that all client writes begun before the repair will be fully replicated once repair
* has completed. In the case of accord, repairs that compare data on disk satisfy this requirement by running
* a sync point as part of streaming if differences are found. For accord-only repairs, the barrier used by normal
* repairs is not sufficient since it only requires a quorum of nodes to respond before completing. This sync point
* adapter requires responses from all of the supplied endpoints before completing. Note that shards only block on the
* intersection of the provided replicas and their own endpoints.
*/
public class RepairSyncPointAdapter<U extends Unseekable> extends CoordinationAdapter.Adapters.AsyncInclusiveSyncPointAdapter<U>
{
private final ImmutableSet<Node.Id> requiredResponses;
public RepairSyncPointAdapter(Collection<Node.Id> requiredResponses)
{
this.requiredResponses = ImmutableSet.copyOf(requiredResponses);
}
@Override
public void execute(Node node, Topologies all, FullRoute<?> route, ExecutePath path, ExecuteFlag.ExecuteFlags executeFlags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
RequiredResponseTracker tracker = new RequiredResponseTracker(requiredResponses, all);
ExecuteSyncPoint.ExecuteInclusive<U> execute = new ExecuteSyncPoint.ExecuteInclusive<>(node, new SyncPoint<>(txnId, executeAt, stableDeps, (FullRoute<U>) route), tracker, executeAt);
execute.addCallback(callback);
execute.start();
}
@Override
public void persist(Node node, Topologies all, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
throw new UnsupportedOperationException();
}
public static <U extends Unseekable> CoordinationAdapter<SyncPoint<U>> create(Collection<Node.Id> requiredResponses)
{
return new RepairSyncPointAdapter<>(requiredResponses);
}
}

36
src/java/org/apache/cassandra/service/accord/serializers/DepsSerializers.java
View File

@ -72,7 +72,7 @@ public class DepsSerializers
this.tokenRange = tokenRange;
}
abstract D deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version) throws IOException;
abstract D deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, DataInputPlus in, int version) throws IOException;
@Override
public void serialize(D deps, DataOutputPlus out, int version) throws IOException
@ -144,16 +144,6 @@ public class DepsSerializers
for (int i = 0; i < rangesToTxnIdsCount; i++)
out.writeUnsignedVInt32(rangesToTxnIds(rangeDeps, i));
}
{
RoutingKeys keys = deps.directKeyDeps.keys();
boolean isSubset = isSubset(keys, deps.keyDeps.keys());
out.writeBoolean(isSubset);
if (isSubset) serializeSubset(keys, deps.keyDeps.keys(), out);
else KeySerializers.routingKeys.serialize(keys, out, version);
serializeKeyDepsWithoutKeys(deps.directKeyDeps, out, version);
}
}
private void serializeKeyDepsWithoutKeys(KeyDeps keyDeps, DataOutputPlus out, int version) throws IOException
@ -193,14 +183,7 @@ public class DepsSerializers
rangeDeps = RangeDeps.SerializerSupport.create(ranges, txnIds, rangesToTxnIds);
}
KeyDeps directKeyDeps;
{
boolean isSubset = in.readBoolean();
RoutingKeys directKeys = isSubset ? (RoutingKeys) deserializeSubset(keys, in) : KeySerializers.routingKeys.deserialize(in, version);
directKeyDeps = deserializeKeyDeps(directKeys, in, version);
}
return deserialize(keyDeps, rangeDeps, directKeyDeps, in, version);
return deserialize(keyDeps, rangeDeps, in, version);
}
private long serializedSizeWithoutKeys(D deps, int version)
@ -224,13 +207,6 @@ public class DepsSerializers
for (int i = 0; i < rangesToTxnIdsCount; i++)
size += sizeofUnsignedVInt(rangesToTxnIds(rangeDeps, i));
}
{
boolean isSubset = isSubset(deps.directKeyDeps.keys(), deps.keyDeps.keys());
size += 1;
size += isSubset ? serializedSubsetSize(deps.directKeyDeps.keys(), deps.keyDeps.keys()) : KeySerializers.routingKeys.serializedSize(deps.directKeyDeps.keys(), version);
size += serializedSizeOfKeyDepsWithoutKeys(deps.directKeyDeps, version);
}
return size;
}
}
@ -250,19 +226,19 @@ public class DepsSerializers
this.deps = new DepsSerializer<>(tokenRange)
{
@Override
Deps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version)
Deps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, DataInputPlus in, int version)
{
return new Deps(keyDeps, rangeDeps, directKeyDeps);
return new Deps(keyDeps, rangeDeps);
}
};
this.nullableDeps = NullableSerializer.wrap(deps);
this.partialDeps = new DepsSerializer<>(tokenRange)
{
@Override
PartialDeps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version) throws IOException
PartialDeps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, DataInputPlus in, int version) throws IOException
{
Participants<?> covering = KeySerializers.participants.deserialize(in, version);
return new PartialDeps(covering, keyDeps, rangeDeps, directKeyDeps);
return new PartialDeps(covering, keyDeps, rangeDeps);
}
@Override

20
src/java/org/apache/cassandra/service/accord/serializers/QueryDurableBeforeSerializers.java → src/java/org/apache/cassandra/service/accord/serializers/GetDurableBeforeSerializers.java
View File

@ -19,33 +19,31 @@ package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import accord.messages.QueryDurableBefore;
import accord.messages.QueryDurableBefore.DurableBeforeReply;
import org.apache.cassandra.db.TypeSizes;
import accord.messages.GetDurableBefore;
import accord.messages.GetDurableBefore.DurableBeforeReply;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
public class QueryDurableBeforeSerializers
public class GetDurableBeforeSerializers
{
public static final IVersionedSerializer<QueryDurableBefore> request = new IVersionedSerializer<QueryDurableBefore>()
public static final IVersionedSerializer<GetDurableBefore> request = new IVersionedSerializer<GetDurableBefore>()
{
@Override
public void serialize(QueryDurableBefore msg, DataOutputPlus out, int version) throws IOException
public void serialize(GetDurableBefore msg, DataOutputPlus out, int version) throws IOException
{
out.writeLong(msg.waitForEpoch());
}
@Override
public QueryDurableBefore deserialize(DataInputPlus in, int version) throws IOException
public GetDurableBefore deserialize(DataInputPlus in, int version) throws IOException
{
return new QueryDurableBefore(in.readLong());
return new GetDurableBefore();
}
@Override
public long serializedSize(QueryDurableBefore msg, int version)
public long serializedSize(GetDurableBefore msg, int version)
{
return TypeSizes.LONG_SIZE;
return 0;
}
};

89
src/java/org/apache/cassandra/service/accord/serializers/GetMaxConflictSerializers.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import accord.messages.GetMaxConflict;
import accord.messages.GetMaxConflict.GetMaxConflictOk;
import accord.primitives.Route;
import accord.primitives.Timestamp;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
public class GetMaxConflictSerializers
{
public static final IVersionedSerializer<GetMaxConflict> request = new IVersionedSerializer<>()
{
@Override
public void serialize(GetMaxConflict msg, DataOutputPlus out, int version) throws IOException
{
KeySerializers.route.serialize(msg.scope, out, version);
out.writeUnsignedVInt(msg.waitForEpoch);
out.writeUnsignedVInt(msg.minEpoch);
out.writeUnsignedVInt(msg.executionEpoch);
}
@Override
public GetMaxConflict deserialize(DataInputPlus in, int version) throws IOException
{
Route<?> scope = KeySerializers.route.deserialize(in, version);
long waitForEpoch = in.readUnsignedVInt();
long minEpoch = in.readUnsignedVInt();
long executionEpoch = in.readUnsignedVInt();
return GetMaxConflict.SerializationSupport.create(scope, waitForEpoch, minEpoch, executionEpoch);
}
@Override
public long serializedSize(GetMaxConflict msg, int version)
{
return KeySerializers.route.serializedSize(msg.scope(), version)
+ TypeSizes.sizeofUnsignedVInt(msg.waitForEpoch)
+ TypeSizes.sizeofUnsignedVInt(msg.minEpoch)
+ TypeSizes.sizeofUnsignedVInt(msg.executionEpoch);
}
};
public static final IVersionedSerializer<GetMaxConflictOk> reply = new IVersionedSerializer<>()
{
@Override
public void serialize(GetMaxConflictOk reply, DataOutputPlus out, int version) throws IOException
{
CommandSerializers.timestamp.serialize(reply.maxConflict, out, version);
out.writeUnsignedVInt(reply.latestEpoch);
}
@Override
public GetMaxConflictOk deserialize(DataInputPlus in, int version) throws IOException
{
Timestamp maxConflict = CommandSerializers.timestamp.deserialize(in, version);
long latestEpoch = in.readUnsignedVInt();
return new GetMaxConflictOk(maxConflict, latestEpoch);
}
@Override
public long serializedSize(GetMaxConflictOk reply, int version)
{
return CommandSerializers.timestamp.serializedSize(reply.maxConflict, version)
+ TypeSizes.sizeofUnsignedVInt(reply.latestEpoch);
}
};
}

4
src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java
View File

@ -52,6 +52,7 @@ import accord.primitives.Seekable;
import accord.primitives.Seekables;
import accord.primitives.Unseekables;
import accord.primitives.Unseekables.UnseekablesKind;
import accord.utils.Invariants;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
@ -382,8 +383,7 @@ public class KeySerializers
case 5: kind = UnseekablesKind.PartialRangeRoute; result = (RS)partialRangeRoute.deserialize(in, version); break;
case 6: kind = UnseekablesKind.FullRangeRoute; result = (RS)fullRangeRoute.deserialize(in, version); break;
}
if (!permitted.contains(kind))
throw new IllegalStateException();
Invariants.require(permitted.contains(kind));
return result;
}

6
src/java/org/apache/cassandra/service/accord/serializers/WaitingOnSerializer.java
View File

@ -23,7 +23,6 @@ import java.io.IOException;
import accord.impl.CommandChange.WaitingOnProvider;
import accord.local.Command;
import accord.local.Command.WaitingOn;
import accord.primitives.KeyDeps;
import accord.primitives.PartialDeps;
import accord.primitives.RangeDeps;
import accord.primitives.RoutingKeys;
@ -76,12 +75,11 @@ public class WaitingOnSerializer
{
RoutingKeys keys = deps.keyDeps.keys();
RangeDeps directRangeDeps = deps.rangeDeps;
KeyDeps directKeyDeps = deps.directKeyDeps;
int txnIdCount = directRangeDeps.txnIdCount() + directKeyDeps.txnIdCount();
int txnIdCount = directRangeDeps.txnIdCount();
Invariants.require(waitingOn.size()/64 == (txnIdCount + keys.size() + 63) / 64);
Invariants.require(appliedOrInvalidated == null || (appliedOrInvalidated.size()/64 == (txnIdCount + 63)/64));
WaitingOn result = new WaitingOn(keys, directRangeDeps, directKeyDeps, waitingOn, appliedOrInvalidated);
WaitingOn result = new WaitingOn(keys, directRangeDeps, waitingOn, appliedOrInvalidated);
if (executeAtLeast != null) return new Command.WaitingOnWithExecuteAt(result, executeAtLeast);
else if (uniqueHlc != 0) return new Command.WaitingOnWithMinUniqueHlc(result, uniqueHlc);
return result;

6
src/java/org/apache/cassandra/service/accord/txn/TxnDataKeyValue.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.service.accord.txn;
import java.io.IOException;
import accord.utils.Invariants;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.partitions.FilteredPartition;
import org.apache.cassandra.db.rows.DeserializationHelper;
@ -48,10 +49,9 @@ public class TxnDataKeyValue extends FilteredPartition implements TxnDataValue
}
@Override
public TxnDataValue merge(TxnDataValue other)
public TxnDataValue merge(TxnDataValue that)
{
// TODO (review): In Accord, for keys, these should always be identical and really there shouldn't be duplicates
// so we could return either
Invariants.require(this.equals(that));
return this;
}

8
src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java
View File

@ -49,6 +49,7 @@ import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.ObjectSizes;
import static accord.api.Data.NOOP_DATA;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.CAS_READ;
@ -210,7 +211,7 @@ public abstract class TxnQuery implements Query
ClusterMetadata clusterMetadata = ClusterMetadata.current();
checkState(clusterMetadata.epoch.getEpoch() >= executeAt.epoch(), "TCM epoch %d is < executeAt epoch %d", clusterMetadata.epoch.getEpoch(), executeAt.epoch());
boolean reads = read != null && !read.keys().isEmpty();
if (transactionShouldBeBlocked(clusterMetadata, reads, keys))
if (transactionShouldBeBlocked(clusterMetadata, reads, keys, data, update))
{
if (txnId.isWrite())
ClientRequestsMetricsHolder.accordWriteMetrics.accordMigrationRejects.mark();
@ -258,8 +259,11 @@ public abstract class TxnQuery implements Query
}
};
private static boolean transactionShouldBeBlocked(ClusterMetadata clusterMetadata, boolean reads, Seekables<?, ?> keys)
private static boolean transactionShouldBeBlocked(ClusterMetadata clusterMetadata, boolean reads, Seekables<?, ?> keys, Data data, Update update)
{
if (data == NOOP_DATA && (update == null || update.keys().isEmpty()))
return false;
// TxnQuery needs to be smart enough to allow blind writes through for the non-transactional use cases during migration
// This also allows blind write TransactionStatement to run before TransactionStatemetns with reads can run,
// but this is harmless since we only promise that TransactionStatement works when migrated to Accord.

13
src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java
View File

@ -110,6 +110,11 @@ public class TxnUpdate extends AccordUpdate
this.preserveTimestamps = preserveTimestamps;
}
public static TxnUpdate empty()
{
return new TxnUpdate(Collections.emptyList(), TxnCondition.none(), null, false);
}
@Override
public long estimatedSizeOnHeap()
{
@ -221,6 +226,9 @@ public class TxnUpdate extends AccordUpdate
if (!checkCondition(data))
return TxnWrite.EMPTY_CONDITION_FAILED;
if (keys.isEmpty())
return new TxnWrite(Collections.emptyList(), true);
List<TxnWrite.Fragment> fragments = deserialize(this.fragments, TxnWrite.Fragment.serializer);
List<TxnWrite.Update> updates = new ArrayList<>(fragments.size());
QueryOptions options = QueryOptions.forProtocolVersion(ProtocolVersion.CURRENT);
@ -365,8 +373,9 @@ public class TxnUpdate extends AccordUpdate
if (conditionResult != null)
return conditionResult;
TxnCondition condition = AccordSerializers.deserialize(this.condition, TxnCondition.serializer);
conditionResult = condition.applies((TxnData) data);
return conditionResult;
if (condition == TxnCondition.none())
return conditionResult = true;
return conditionResult = condition.applies((TxnData) data);
}
@Override

49
src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java
View File

@ -21,7 +21,6 @@ package org.apache.cassandra.service.consensus.migration;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.SortedSet;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
@ -33,9 +32,8 @@ import com.google.common.primitives.Ints;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.BarrierType;
import accord.primitives.Keys;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
@ -62,6 +60,9 @@ import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.RequestBookkeeping;
import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.paxos.AbstractPaxosRepair.Failure;
import org.apache.cassandra.service.paxos.AbstractPaxosRepair.Result;
@ -70,11 +71,15 @@ import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Collectors3;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.UUIDSerializer;
import static accord.local.durability.DurabilityService.SyncLocal.Self;
import static accord.local.durability.DurabilityService.SyncRemote.NoRemote;
import static accord.local.durability.DurabilityService.SyncRemote.Quorum;
import static org.apache.cassandra.config.DatabaseDescriptor.getReadRpcTimeout;
import static org.apache.cassandra.config.DatabaseDescriptor.getWriteRpcTimeout;
import static org.apache.cassandra.net.Verb.CONSENSUS_KEY_MIGRATION;
import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget.paxos;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@ -290,27 +295,23 @@ public abstract class ConsensusKeyMigrationState
boolean isForWrite)
{
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
if (isForWrite)
ClientRequestsMetricsHolder.casWriteMetrics.accordKeyMigrations.mark();
else
ClientRequestsMetricsHolder.casReadMetrics.accordKeyMigrations.mark();
if (isForWrite) ClientRequestsMetricsHolder.casWriteMetrics.accordKeyMigrations.mark();
else ClientRequestsMetricsHolder.casReadMetrics.accordKeyMigrations.mark();
// Global will always create a transaction to effect the barrier so all replicas
// will soon be ready to execute, but only waits for the local replica to be ready
// Local will only create a transaction if it can't find an existing one to wait on
Keys partitionKeys = AccordService.intersecting(Keys.of(keys, k -> new PartitionKey(tableId, k)));
if (partitionKeys.isEmpty())
throw new RetryOnDifferentSystemException();
IAccordService accord = AccordService.instance();
long start = nanoTime();
try
{
// Global will always create a transaction to effect the barrier so all replicas
// will soon be ready to execute, but only waits for the local replica to be ready
// Local will only create a transaction if it can't find an existing one to wait on
BarrierType barrierType = global ? BarrierType.global_async : BarrierType.local;
SortedSet<PartitionKey> partitionKeys = keys.stream().map(key -> new PartitionKey(tableId, key)).collect(Collectors3.toSortedSet());
Seekables keysOrRanges = AccordService.instance().barrier(new Keys(partitionKeys), minEpoch, requestTime, DatabaseDescriptor.getTransactionTimeout(TimeUnit.NANOSECONDS), barrierType, isForWrite);
if (keysOrRanges.isEmpty())
throw new RetryOnDifferentSystemException();
// We don't save the state to the cache here. Accord will notify the agent every time a barrier happens.
}
finally
{
cfs.metric.keyMigration.addNano(nanoTime() - start);
}
long deadline = requestTime.computeDeadline(isForWrite ? getWriteRpcTimeout(TimeUnit.NANOSECONDS) : getReadRpcTimeout(TimeUnit.NANOSECONDS));
RequestBookkeeping bookkeeping = new LatencyRequestBookkeeping(cfs == null ? null : cfs.metric.keyMigration);
AccordService.getBlocking(accord.sync(Timestamp.minForEpoch(minEpoch), partitionKeys, Self, global ? Quorum : NoRemote),
partitionKeys, bookkeeping, start, deadline);
maybeSaveAccordKeyMigrationLocally((PartitionKey) partitionKeys.get(0), Epoch.create(minEpoch));
}
static void repairKeyPaxos(EndpointsForToken naturalReplicas,

22
src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationMutationHelper.java
View File

@ -32,7 +32,6 @@ import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.coordinate.CoordinationFailed;
import accord.primitives.Keys;
import accord.primitives.Routable.Domain;
import accord.primitives.Txn;
@ -52,12 +51,13 @@ import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableParams;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.service.accord.IAccordService.IAccordResult;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.txn.TxnCondition;
import org.apache.cassandra.service.accord.txn.TxnQuery;
import org.apache.cassandra.service.accord.txn.TxnRead;
import org.apache.cassandra.service.accord.txn.TxnReferenceOperations;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.service.accord.txn.TxnUpdate;
import org.apache.cassandra.service.accord.txn.TxnWrite;
import org.apache.cassandra.service.consensus.TransactionalMode;
@ -183,7 +183,7 @@ public class ConsensusMigrationMutationHelper
void consume(@Nullable T accordMutation, @Nullable T normalMutation, List<T> mutations, int mutationIndex);
}
public static <T extends IMutation, N> SplitMutations<T> splitMutationsIntoAccordAndNormal(ClusterMetadata cm, List<T> mutations)
public static <T extends IMutation> SplitMutations<T> splitMutationsIntoAccordAndNormal(ClusterMetadata cm, List<T> mutations)
{
SplitMutations<T> splitMutations = new SplitMutations<>();
splitMutationsIntoAccordAndNormal(cm, mutations, splitMutations);
@ -233,12 +233,12 @@ public class ConsensusMigrationMutationHelper
return new SplitMutation(accordMutation, normalMutation);
}
public AsyncTxnResult mutateWithAccordAsync(ClusterMetadata cm, Mutation mutation, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
public IAccordResult<TxnResult> mutateWithAccordAsync(ClusterMetadata cm, Mutation mutation, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
return mutateWithAccordAsync(cm, ImmutableList.of(mutation), consistencyLevel, requestTime);
}
public static AsyncTxnResult mutateWithAccordAsync(ClusterMetadata cm, Collection<? extends IMutation> mutations, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
public static IAccordResult<TxnResult> mutateWithAccordAsync(ClusterMetadata cm, Collection<? extends IMutation> mutations, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
if (consistencyLevel != null && !IAccordService.SUPPORTED_COMMIT_CONSISTENCY_LEVELS.contains(consistencyLevel))
throw new InvalidRequestException(consistencyLevel + " is not supported by Accord");
@ -260,17 +260,7 @@ public class ConsensusMigrationMutationHelper
ConsistencyLevel clForCommit = consistencyLevelForCommit(cm, mutations, consistencyLevel);
TxnUpdate update = new TxnUpdate(fragments, TxnCondition.none(), clForCommit, true);
Txn.InMemory txn = new Txn.InMemory(Keys.of(partitionKeys), TxnRead.empty(Domain.Key), TxnQuery.NONE, update);
IAccordService accordService = AccordService.instance();
try
{
return accordService.coordinateAsync(minEpoch, txn, clForCommit, requestTime);
}
catch (CoordinationFailed coordinationFailed)
{
AsyncTxnResult failure = new AsyncTxnResult(coordinationFailed.txnId(), minEpoch, clForCommit, true, requestTime);
failure.setFailure(coordinationFailed.wrap());
return failure;
}
return AccordService.instance().coordinateAsync(minEpoch, txn, clForCommit, requestTime);
}
public static void validateSafeToExecuteNonTransactionally(IMutation mutation) throws RetryOnDifferentSystemException

251
src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java
View File

@ -19,6 +19,7 @@
package org.apache.cassandra.service.paxos;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import org.apache.cassandra.config.DatabaseDescriptor;
@ -32,70 +33,38 @@ import org.apache.cassandra.service.TimeoutStrategy.ReadWriteLatencySourceFactor
import org.apache.cassandra.service.TimeoutStrategy.Wait;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.NoSpamLogger;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.lang.Math.*;
import static java.util.concurrent.TimeUnit.*;
import static java.lang.Math.max;
import static java.lang.Math.min;
import static java.util.Arrays.stream;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.cassandra.config.DatabaseDescriptor.*;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics;
import static org.apache.cassandra.service.TimeoutStrategy.parseInMicros;
import static org.apache.cassandra.utils.Clock.waitUntil;
import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized;
/**
* See {@link RetryStrategy}
* TODO (expected): deprecate in favour of pure RetryStrategy
*/
public class ContentionStrategy extends RetryStrategy
{
private static final Logger logger = LoggerFactory.getLogger(RetryStrategy.class);
private static final String DEFAULT_WAIT_RANDOMIZER = "uniform";
private static final String DEFAULT_MIN = "0";
private static final String DEFAULT_MAX = "100ms";
private static final String DEFAULT_SPREAD = "100ms";
private static final LatencySourceFactory LATENCIES = new ReadWriteLatencySourceFactory(casReadMetrics, casWriteMetrics);
private static volatile ContentionStrategy current;
private static volatile ParsedStrategy currentParsed;
private static final RetryStrategy.ParsedStrategy defaultStrategy;
static
{
defaultStrategy = new ParsedStrategy(DEFAULT_WAIT_RANDOMIZER, DEFAULT_MIN, DEFAULT_MAX, DEFAULT_SPREAD, Integer.MAX_VALUE,
new ContentionStrategy(DEFAULT_WAIT_RANDOMIZER, DEFAULT_MIN, DEFAULT_MAX, DEFAULT_SPREAD, Integer.MAX_VALUE));
String waitRandomizer = orElse(DatabaseDescriptor::getPaxosContentionWaitRandomizer, DEFAULT_WAIT_RANDOMIZER);
String min = orElse(DatabaseDescriptor::getPaxosContentionMinWait, DEFAULT_MIN);
String max = orElse(DatabaseDescriptor::getPaxosContentionMaxWait, DEFAULT_MAX);
String spread = orElse(DatabaseDescriptor::getPaxosContentionMinDelta, DEFAULT_SPREAD);
current = new ContentionStrategy(waitRandomizer, min, max, spread, Integer.MAX_VALUE);
currentParsed = new ParsedStrategy(waitRandomizer, min, max, spread, Integer.MAX_VALUE, current);
}
final int traceAfterAttempts;
public ContentionStrategy(String waitRandomizer, String min, String max, String spread, int traceAfterAttempts)
{
super(waitRandomizer, min, max, spread, LATENCIES);
this.traceAfterAttempts = traceAfterAttempts;
}
public ContentionStrategy(WaitRandomizer waitRandomizer, Wait min, Wait max, Wait spread, int traceAfterAttempts)
{
super(waitRandomizer, min, max, spread);
this.traceAfterAttempts = traceAfterAttempts;
}
@Override
protected Wait parseBound(String spec, boolean isMin, LatencySourceFactory latencies)
{
return TimeoutStrategy.parseWait(spec, 0, maxQueryTimeoutMicros(), isMin ? 0 : maxQueryTimeoutMicros(), latencies);
}
public enum Type
{
READ("Contended Paxos Read"), WRITE("Contended Paxos Write"), REPAIR("Contended Paxos Repair");
@ -110,6 +79,71 @@ public class ContentionStrategy extends RetryStrategy
}
}
static final Pattern LEGACY = Pattern.compile(
"(?<const>0|[0-9]+[mu]?s)" +
"|((?<min>0|[0-9]+[mu]?s) *<= *)?" +
"(?<delegate>[^=]+)?" +
"( *<= *(?<max>0|[0-9]+[mu]?s))?");
private static final String DEFAULT_WAIT_RANDOMIZER = "uniform";
private static final String DEFAULT_MIN = "0";
private static final String DEFAULT_MAX = "100ms";
private static final String DEFAULT_SPREAD = "100ms";
private static final String MAX_INT = "" + Integer.MAX_VALUE;
private static final LatencySourceFactory LATENCIES = new ReadWriteLatencySourceFactory(casReadMetrics, casWriteMetrics);
private static volatile ContentionStrategy current;
private static volatile ParsedStrategy currentParsed;
static
{
String waitRandomizer = orElse(DatabaseDescriptor::getPaxosContentionWaitRandomizer, DEFAULT_WAIT_RANDOMIZER);
String min = orElse(DatabaseDescriptor::getPaxosContentionMinWait, DEFAULT_MIN);
String max = orElse(DatabaseDescriptor::getPaxosContentionMaxWait, DEFAULT_MAX);
String spread = orElse(DatabaseDescriptor::getPaxosContentionMinDelta, DEFAULT_SPREAD);
current = parse(waitRandomizer, min, max, spread, MAX_INT, MAX_INT);
currentParsed = new ParsedStrategy(waitRandomizer, min, max, spread, MAX_INT, MAX_INT, current);
}
final @Nullable LegacyWait spread;
final int traceAfterAttempts;
public ContentionStrategy(WaitRandomizer waitRandomizer, LegacyWait min, LegacyWait max, LegacyWait spread, int retries, int traceAfterAttempts)
{
super(waitRandomizer, min.min, min, max, max.max, retries);
this.traceAfterAttempts = traceAfterAttempts;
this.spread = spread;
}
public long computeWait(int attempt, TimeUnit units)
{
if (attempt > maxAttempts)
return -1;
long minWaitMicros = min.getMicros(attempt);
long maxWaitMicros = max.getMicros(attempt);
long spreadMicros = spread == null ? 0 : spread.getMicros(attempt);
if (minWaitMicros + spreadMicros >= maxWaitMicros)
{
if (spreadMicros == 0)
return units.convert(maxWaitMicros, MICROSECONDS);
if (maxWaitMicros < minWaitMicros)
maxWaitMicros = minWaitMicros;
long newMaxWaitMicros = minWaitMicros + spreadMicros;
if (newMaxWaitMicros > maxMaxMicros)
{
newMaxWaitMicros = maxMaxMicros;
minWaitMicros = max(this.minMinMicros, maxWaitMicros - spreadMicros);
}
maxWaitMicros = newMaxWaitMicros;
if (minWaitMicros >= maxWaitMicros)
return minWaitMicros;
}
return units.convert(waitRandomizer.wait(minWaitMicros, maxWaitMicros, attempt), MICROSECONDS);
}
long computeWaitUntilForContention(int attempts, TableMetadata table, DecoratedKey partitionKey, ConsistencyLevel consistency, Type type)
{
if (attempts >= traceAfterAttempts && !Tracing.isTracing())
@ -162,41 +196,71 @@ public class ContentionStrategy extends RetryStrategy
return current.computeWaitUntilForContention(attempts, table, partitionKey, consistency, type);
}
public static class ParsedStrategy extends RetryStrategy.ParsedStrategy
public static class ParsedStrategy
{
public final int trace;
public final String waitRandomizer, min, max, spread, retries, trace;
public final ContentionStrategy strategy;
ParsedStrategy(String waitRandomizer, String min, String max, String minDelta, int trace, ContentionStrategy strategy)
protected ParsedStrategy(String waitRandomizer, String min, String max, String spread, String retries, String trace, ContentionStrategy strategy)
{
super(waitRandomizer, min, max, minDelta, strategy);
this.waitRandomizer = waitRandomizer;
this.min = min;
this.max = max;
this.spread = spread;
this.retries = retries;
this.trace = trace;
this.strategy = strategy;
}
@Override
public String toString()
{
return super.toString() + (trace == Integer.MAX_VALUE ? "" : ",trace=" + current.traceAfterAttempts);
return "min=" + min + ",max=" + max + ",spread=" + spread + ",retries=" + retries
+ ",random=" + waitRandomizer + ",trace=" + current.traceAfterAttempts;
}
}
@VisibleForTesting
public static ParsedStrategy parseStrategy(String spec)
public static ParsedStrategy parseStrategy(String spec, ParsedStrategy defaultStrategy)
{
RetryStrategy.ParsedStrategy parsed = RetryStrategy.parseStrategy(spec, LATENCIES, defaultStrategy);
String[] args = spec.split(",");
String waitRandomizer = find(args, "random");
String min = find(args, "min");
String max = find(args, "max");
String spread = find(args, "spread");
String trace = find(args, "trace");
int traceAfterAttempts = trace == null ? current.traceAfterAttempts: Integer.parseInt(trace);
if (spread == null)
spread = find(args, "delta");
String retries = find(args, "retries");
ContentionStrategy strategy = new ContentionStrategy(parsed.strategy.waitRandomizer, parsed.strategy.min, parsed.strategy.max, parsed.strategy.spread, traceAfterAttempts);
return new ParsedStrategy(parsed.waitRandomizer, parsed.min, parsed.max, parsed.spread, traceAfterAttempts, strategy);
if (waitRandomizer == null) waitRandomizer = defaultStrategy.waitRandomizer;
if (min == null) min = defaultStrategy.min;
if (max == null) max = defaultStrategy.max;
if (spread == null) spread = defaultStrategy.spread;
if (retries == null) retries = defaultStrategy.retries;
if (trace == null) trace = defaultStrategy.trace;
ContentionStrategy strategy = parse(waitRandomizer, min, max, spread, retries, trace);
return new ParsedStrategy(waitRandomizer, min, max, spread, retries, trace, strategy);
}
private static ContentionStrategy parse(String waitRandomizerString, String minString, String maxString, String spreadString, String retriesString, String traceString)
{
return new ContentionStrategy(parseWaitRandomizer(waitRandomizerString),
parseLegacy(minString, true), parseLegacy(maxString, false), parseLegacy(spreadString, false),
Integer.parseInt(retriesString), Integer.parseInt(traceString));
}
private static String find(String[] args, String param)
{
return stream(args).filter(s -> s.startsWith(param + '='))
.map(s -> s.substring(param.length() + 1))
.findFirst().orElse(null);
}
public static synchronized void setStrategy(String spec)
{
ParsedStrategy parsed = parseStrategy(spec);
ParsedStrategy parsed = parseStrategy(spec, currentParsed);
currentParsed = parsed;
current = parsed.strategy;
setPaxosContentionWaitRandomizer(parsed.waitRandomizer);
@ -210,15 +274,74 @@ public class ContentionStrategy extends RetryStrategy
return currentParsed.toString();
}
@VisibleForTesting
static long maxQueryTimeoutMicros()
{
return max(max(getCasContentionTimeout(MICROSECONDS), getWriteRpcTimeout(MICROSECONDS)), getReadRpcTimeout(MICROSECONDS));
}
private static String orElse(Supplier<String> get, String orElse)
{
String result = get.get();
return result != null ? result : orElse;
}
@VisibleForTesting
static LegacyWait parseLegacy(String spec, boolean isMin)
{
long defaultMaxMicros = getRpcTimeout(MICROSECONDS);
return parseLegacy(spec, 0, defaultMaxMicros, isMin ? 0 : defaultMaxMicros, LATENCIES);
}
public static LegacyWait parseLegacy(String input, long defaultMinMicros, long defaultMaxMicros, long onFailure, LatencySourceFactory latencies)
{
Matcher m = LEGACY.matcher(input);
if (!m.matches())
throw new IllegalArgumentException(input + " does not match " + LEGACY);
String maybeConst = m.group("const");
if (maybeConst != null)
{
long v = parseInMicros(maybeConst);
return new LegacyWait(v, v, v, new Wait.Constant(v));
}
long min = parseInMicros(m.group("min"), defaultMinMicros);
long max = parseInMicros(m.group("max"), defaultMaxMicros);
return new LegacyWait(min, max, onFailure, TimeoutStrategy.parseWait(m.group("delegate"), latencies));
}
private static class LegacyWait implements Wait
{
final long min, max, onFailure;
final Wait delegate;
LegacyWait(long min, long max, long onFailure, Wait delegate)
{
Preconditions.checkArgument(min <= max, "min (%s) must be less than or equal to max (%s)", min, max);
this.min = min;
this.max = max;
this.onFailure = onFailure;
this.delegate = delegate;
}
public long getMicros(int attempts)
{
try
{
return max(min, min(max, delegate.getMicros(attempts)));
}
catch (Throwable t)
{
NoSpamLogger.getLogger(logger, 1L, MINUTES).info("", t);
return onFailure;
}
}
public String toString()
{
return "Bound{" +
"min=" + min +
", max=" + max +
", onFailure=" + onFailure +
", delegate=" + delegate +
'}';
}
}
}

2
src/java/org/apache/cassandra/service/paxos/cleanup/PaxosRepairState.java
View File

@ -130,7 +130,7 @@ public class PaxosRepairState
private static void add(SharedContext ctx, AtomicReference<PendingCleanup> pendingCleanup, Message<PaxosCleanupHistory> message)
{
PendingCleanup next = new PendingCleanup(message);
PendingCleanup prev = IntrusiveStack.push(AtomicReference::get, AtomicReference::compareAndSet, pendingCleanup, next);
PendingCleanup prev = IntrusiveStack.getAndPush(AtomicReference::get, AtomicReference::compareAndSet, pendingCleanup, next);
if (prev == null)
Stage.MISC.execute(() -> cleanup(ctx, pendingCleanup));
}

17
src/java/org/apache/cassandra/service/reads/range/AccordRangeResponse.java
View File

@ -20,30 +20,25 @@ package org.apache.cassandra.service.reads.range;
import java.util.function.IntPredicate;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.service.accord.IAccordService.IAccordResult;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.utils.AbstractIterator;
public class AccordRangeResponse extends AbstractIterator<RowIterator> implements PartitionIterator
{
private final AsyncTxnResult asyncTxnResult;
private final IAccordResult<TxnResult> accordResult;
// Range queries don't support reverse, but dutifully threading it through anyways
private final boolean reversed;
private final ConsistencyLevel cl;
private final Dispatcher.RequestTime requestTime;
private PartitionIterator result;
public AccordRangeResponse(AsyncTxnResult asyncTxnResult, boolean reversed, ConsistencyLevel cl, Dispatcher.RequestTime requestTime)
public AccordRangeResponse(IAccordResult<TxnResult> accordResult, boolean reversed)
{
this.asyncTxnResult = asyncTxnResult;
this.cl = cl;
this.requestTime = requestTime;
this.accordResult = accordResult;
this.reversed = reversed;
}
@ -54,7 +49,7 @@ public class AccordRangeResponse extends AbstractIterator<RowIterator> implement
IntPredicate alwaysTrue = ignored -> true;
IntPredicate alwaysFalse = ignored -> false;
// TODO (required): Handle retry on different system
ConsensusAttemptResult consensusAttemptResult = StorageProxy.getConsensusAttemptResultFromAsyncTxnResult(asyncTxnResult, 1, reversed ? alwaysTrue : alwaysFalse);
ConsensusAttemptResult consensusAttemptResult = StorageProxy.getConsensusAttemptResultFromAsyncTxnResult(accordResult, 1, reversed ? alwaysTrue : alwaysFalse);
if (consensusAttemptResult.shouldRetryOnNewConsensusProtocol)
throw new RetryOnDifferentSystemException();
result = consensusAttemptResult.serialReadResult;

9
src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
View File

@ -51,7 +51,8 @@ import org.apache.cassandra.locator.ReplicaPlan;
import org.apache.cassandra.metrics.ClientRangeRequestMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.RangeReadTarget;
import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.RangeReadWithTarget;
@ -195,9 +196,9 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
private PartitionIterator executeAccord(ClusterMetadata cm, PartitionRangeReadCommand rangeCommand, ConsistencyLevel cl)
{
//TODO (nicetohave): https://issues.apache.org/jira/browse/CASSANDRA-20210 More efficient reads across command stores
AsyncTxnResult result = StorageProxy.readWithAccord(cm, rangeCommand, rangeCommand.dataRange().keyRange(), cl, requestTime);
return new AccordRangeResponse(result, rangeCommand.isReversed(), cl, requestTime);
//TODO (expected): https://issues.apache.org/jira/browse/CASSANDRA-20210 More efficient reads across command stores
IAccordService.IAccordResult<TxnResult> result = StorageProxy.readWithAccord(cm, rangeCommand, rangeCommand.dataRange().keyRange(), cl, requestTime);
return new AccordRangeResponse(result, rangeCommand.isReversed());
}
private SingleRangeResponse executeNormal(ReplicaPlan.ForRangeRead replicaPlan, PartitionRangeReadCommand rangeCommand, ReadCoordinator readCoordinator)

17
src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java
View File

@ -18,7 +18,6 @@
package org.apache.cassandra.tcm;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import org.slf4j.Logger;
@ -50,9 +49,9 @@ public abstract class AbstractLocalProcessor implements Processor
* the time when this method returns.
*/
@Override
public final Commit.Result commit(Entry.Id entryId, Transformation transform, final Epoch lastKnown, Retry.Deadline retryPolicy)
public final Commit.Result commit(Entry.Id entryId, Transformation transform, final Epoch lastKnown, Retry retryPolicy)
{
while (!retryPolicy.reachedMax())
while (!retryPolicy.hasExpired())
{
ClusterMetadata previous = log.waitForHighestConsecutive();
if (!previous.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort()))
@ -112,7 +111,8 @@ public abstract class AbstractLocalProcessor implements Processor
}
else
{
retryPolicy.maybeSleep();
if (!retryPolicy.maybeSleep())
break;
// TODO: could also add epoch from mis-application from [applied].
fetchLogAndWait(null, retryPolicy);
}
@ -121,13 +121,12 @@ public abstract class AbstractLocalProcessor implements Processor
{
logger.error("Caught error while trying to perform a local commit", e);
JVMStabilityInspector.inspectThrowable(e);
retryPolicy.maybeSleep();
if (!retryPolicy.maybeSleep())
break;
}
}
return Commit.Result.failed(SERVER_ERROR,
String.format("Could not perform commit after %d/%d tries. Time remaining: %dms",
retryPolicy.tries, retryPolicy.maxTries,
TimeUnit.NANOSECONDS.toMillis(retryPolicy.remainingNanos())));
String.format("Could not perform commit; policy %s gave up", retryPolicy));
}
public Commit.Result maybeFailure(Entry.Id entryId, Epoch lastKnown, Supplier<Commit.Result.Failure> orElse)
@ -196,6 +195,6 @@ public abstract class AbstractLocalProcessor implements Processor
return logState;
}
public abstract ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy);
public abstract ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy);
protected abstract boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch);
}

4
src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java
View File

@ -76,7 +76,7 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor
}
@Override
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retry)
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retry)
{
return log.waitForHighestConsecutive();
}
@ -109,7 +109,7 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor
}
@Override
public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry.Deadline retryPolicy)
public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry retryPolicy)
{
return getLocalState(lowEpoch, highEpoch, includeSnapshot);
}

15
src/java/org/apache/cassandra/tcm/ClusterMetadataService.java
View File

@ -78,6 +78,7 @@ import org.apache.cassandra.utils.concurrent.ImmediateFuture;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.stream.Collectors.toSet;
import static org.apache.cassandra.config.CassandraRelevantProperties.TCM_SKIP_CMS_RECONFIGURATION_AFTER_TOPOLOGY_CHANGE;
import static org.apache.cassandra.config.DatabaseDescriptor.getCmsAwaitTimeout;
import static org.apache.cassandra.tcm.ClusterMetadataService.State.GOSSIP;
import static org.apache.cassandra.tcm.ClusterMetadataService.State.LOCAL;
import static org.apache.cassandra.tcm.ClusterMetadataService.State.REMOTE;
@ -667,8 +668,7 @@ public class ClusterMetadataService
if (ourEpoch.isEqualOrAfter(awaitAtLeast))
return metadata;
Retry.Deadline deadline = Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS),
new Retry.Jitter(TCMMetrics.instance.fetchLogRetries));
Retry deadline = Retry.untilElapsed(getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.fetchLogRetries);
// responses for ALL withhout knowing we have pending
metadata = processor.fetchLogAndWait(awaitAtLeast, deadline);
if (metadata.epoch.isBefore(awaitAtLeast))
@ -876,9 +876,9 @@ public class ClusterMetadataService
}
@Override
public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy)
public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy)
{
while (!retryPolicy.reachedMax())
while (true)
{
try
{
@ -891,14 +891,15 @@ public class ClusterMetadataService
}
catch (NotCMSException e)
{
retryPolicy.maybeSleep();
if (!retryPolicy.maybeSleep())
break;
}
}
return Commit.Result.failed(ExceptionCode.SERVER_ERROR, "Could not commit " + transform.kind() + " at epoch " + lastKnown);
}
@Override
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy)
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy)
{
return delegate().fetchLogAndWait(waitFor, retryPolicy);
}
@ -910,7 +911,7 @@ public class ClusterMetadataService
}
@Override
public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy)
public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy)
{
return delegate().getLogState(start, end, includeSnapshot, retryPolicy);
}

2
src/java/org/apache/cassandra/tcm/Commit.java
View File

@ -364,7 +364,7 @@ public class Commit
{
checkCMSState();
logger.info("Received commit request {} from {}", message.payload, message.from());
Retry.Deadline retryPolicy = Retry.Deadline.at(message.expiresAtNanos(), new Retry.Jitter(TCMMetrics.instance.commitRetries));
Retry retryPolicy = Retry.until(message.expiresAtNanos(), TCMMetrics.instance.commitRetries);
Result result = processor.commit(message.payload.entryId, message.payload.transform, message.payload.lastKnown, retryPolicy);
if (result.isSuccess())
{

6
src/java/org/apache/cassandra/tcm/FetchCMSLog.java
View File

@ -25,7 +25,6 @@ import java.util.function.Supplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
@ -37,6 +36,8 @@ import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.tcm.log.LogState;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.config.DatabaseDescriptor.getCmsAwaitTimeout;
public class FetchCMSLog
{
public static final Serializer serializer = new Serializer();
@ -115,8 +116,7 @@ public class FetchCMSLog
// If both we and the other node believe it should be caught up with a linearizable read
boolean consistentFetch = request.consistentFetch && !ClusterMetadataService.instance().isCurrentMember(message.from());
Retry.Deadline retry = Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS),
new Retry.Jitter(TCMMetrics.instance.fetchLogRetries));
Retry retry = Retry.untilElapsed(getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.fetchLogRetries);
LogState delta;
if (consistentFetch)
delta = processor.get().getLogState(message.payload.lowerBound, Epoch.MAX, false, retry);

15
src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java
View File

@ -70,7 +70,7 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor
}
@Override
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy)
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy)
{
ClusterMetadata metadata = log.waitForHighestConsecutive();
@ -106,7 +106,7 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor
for (Replica peer : replicas)
requests.add(new FetchLogRequest(peer, MessagingService.instance(), metadata.epoch));
while (!retryPolicy.reachedMax())
while (!retryPolicy.hasExpired())
{
Iterator<FetchLogRequest> iter = requests.iterator();
boolean hasRequestToSelf = false;
@ -153,7 +153,8 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor
if (collected.size() < blockFor)
{
retryPolicy.maybeSleep();
if (!retryPolicy.maybeSleep())
break;
continue;
}
@ -174,9 +175,9 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor
}
@Override
public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy)
public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy)
{
while (!retryPolicy.reachedMax())
while (true)
{
if (Thread.currentThread().isInterrupted())
throw new RuntimeException("Can not reconstruct during shutdown", new InterruptedException());
@ -186,10 +187,10 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor
}
catch (RuntimeException e) // honestly best to only retry timeouts, but everything gets wrapped in a RuntimeException...
{
retryPolicy.maybeSleep();
if (!retryPolicy.maybeSleep())
throw new RuntimeException(String.format("Could not reconstruct range %d, %d", start.getEpoch(), end.getEpoch()), new TimeoutException());
}
}
throw new RuntimeException(String.format("Could not reconstruct range %d, %d", start.getEpoch(), end.getEpoch()), new TimeoutException());
}
private static <T> T unwrap(Promise<T> promise)

3
src/java/org/apache/cassandra/tcm/PeerLogFetcher.java
View File

@ -95,8 +95,7 @@ public class PeerLogFetcher
Verb.TCM_FETCH_PEER_LOG_REQ,
new FetchPeerLog(before),
new RemoteProcessor.CandidateIterator(Collections.singletonList(remote), false),
Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS),
new Retry.Jitter(TCMMetrics.instance.fetchLogRetries)));
Retry.untilElapsed(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.fetchLogRetries));
return fetchRes.map((logState) -> {
log.append(logState);

43
src/java/org/apache/cassandra/tcm/Processor.java
View File

@ -26,11 +26,14 @@ import java.util.concurrent.TimeUnit;
import com.codahale.metrics.Meter;
import accord.utils.Invariants;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.metrics.TCMMetrics;
import org.apache.cassandra.service.WaitStrategy;
import org.apache.cassandra.tcm.log.Entry;
import org.apache.cassandra.tcm.log.LogState;
import org.apache.cassandra.utils.Clock;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.config.DatabaseDescriptor.getCmsAwaitTimeout;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
public interface Processor
{
@ -49,8 +52,7 @@ public interface Processor
}
return commit(entryId, transform, lastKnown,
Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS),
new Retry.Jitter(TCMMetrics.instance.commitRetries)));
Retry.untilElapsed(getCmsAwaitTimeout().to(NANOSECONDS), TCMMetrics.instance.commitRetries));
}
/**
@ -58,33 +60,27 @@ public interface Processor
* to overflow the long, since messaging is using only 32 bits for deadlines. To achieve that, we are
* giving `timeoutNanos` every time we retry, but will retry indefinitely.
*/
private static Retry.Deadline unsafeRetryIndefinitely()
private static Retry unsafeRetryIndefinitely()
{
long timeoutNanos = DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS);
long timeoutNanos = getCmsAwaitTimeout().to(NANOSECONDS);
Meter retryMeter = TCMMetrics.instance.commitRetries;
return new Retry.Deadline(Clock.Global.nanoTime() + timeoutNanos,
new Retry.Jitter(retryMeter))
return Retry.withNoTimeLimit(retryMeter, new WaitStrategy()
{
@Override
public boolean reachedMax()
public long computeWaitUntil(int attempts)
{
return false;
return nanoTime() + timeoutNanos;
}
@Override
public long remainingNanos()
public long computeWait(int attempts, TimeUnit units)
{
return timeoutNanos;
return units.convert(timeoutNanos, NANOSECONDS);
}
public String toString()
{
return String.format("RetryIndefinitely{tries=%d}", currentTries());
}
};
});
}
Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy);
Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy);
/**
* Fetches log from CMS up to the highest currently known epoch.
@ -103,11 +99,10 @@ public interface Processor
default ClusterMetadata fetchLogAndWait(Epoch waitFor)
{
return fetchLogAndWait(waitFor,
Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS),
new Retry.Jitter(TCMMetrics.instance.fetchLogRetries)));
Retry.untilElapsed(getCmsAwaitTimeout().to(NANOSECONDS), TCMMetrics.instance.fetchLogRetries));
}
ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy);
ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy);
/**
* Queries node's _local_ state. It is not guaranteed to be contiguous, but can be used for restoring CMS state/
@ -117,12 +112,12 @@ public interface Processor
/**
* Queries global log state.
*/
LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy);
LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy);
/**
* Reconstructs
*/
default List<ClusterMetadata> reconstruct(Epoch lowEpoch, Epoch highEpoch, Retry.Deadline retryPolicy)
default List<ClusterMetadata> reconstruct(Epoch lowEpoch, Epoch highEpoch, Retry retryPolicy)
{
LogState logState = getLogState(lowEpoch, highEpoch, true, retryPolicy);
if (logState.isEmpty()) return Collections.emptyList();

6
src/java/org/apache/cassandra/tcm/ReconstructLogState.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
@ -34,6 +33,8 @@ import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.tcm.log.LogState;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.config.DatabaseDescriptor.getCmsAwaitTimeout;
public class ReconstructLogState
{
public static final Serializer serializer = new Serializer();
@ -97,8 +98,7 @@ public class ReconstructLogState
throw new NotCMSException("This node is not in the CMS, can't generate a consistent log fetch response to " + message.from());
LogState result = processor.get().getLogState(request.lowerBound, request.higherBound, request.includeSnapshot,
Retry.Deadline.after(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS),
new Retry.Jitter(TCMMetrics.instance.fetchLogRetries)));
Retry.untilElapsed(getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.fetchLogRetries));
MessagingService.instance().send(message.responseWith(result), message.from());
}

22
src/java/org/apache/cassandra/tcm/RemoteProcessor.java
View File

@ -44,12 +44,12 @@ import org.apache.cassandra.net.MessageDelivery;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.RequestCallbackWithFailure;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.WaitStrategy;
import org.apache.cassandra.tcm.Discovery.DiscoveredNodes;
import org.apache.cassandra.tcm.log.Entry;
import org.apache.cassandra.tcm.log.LocalLog;
import org.apache.cassandra.tcm.log.LogState;
import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.Backoff;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
@ -73,7 +73,7 @@ public final class RemoteProcessor implements Processor
@Override
@SuppressWarnings("resource")
public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy)
public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy)
{
try
{
@ -126,7 +126,7 @@ public final class RemoteProcessor implements Processor
}
@Override
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy)
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy)
{
// Synchonous, non-debounced call if we are waiting for the highest epoch (without knowing/caring what it is).
// Should be used sparingly.
@ -158,7 +158,7 @@ public final class RemoteProcessor implements Processor
}
@Override
public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry.Deadline retryPolicy)
public LogState getLogState(Epoch lowEpoch, Epoch highEpoch, boolean includeSnapshot, Retry retryPolicy)
{
try
{
@ -203,7 +203,7 @@ public final class RemoteProcessor implements Processor
Verb.TCM_FETCH_CMS_LOG_REQ,
new FetchCMSLog(currentEpoch, ClusterMetadataService.state() == REMOTE),
candidates,
new Retry.Backoff(TCMMetrics.instance.fetchLogRetries));
Retry.withNoTimeLimit(TCMMetrics.instance.fetchLogRetries));
return remoteRequest.map((replay) -> {
if (!replay.isEmpty())
{
@ -217,12 +217,12 @@ public final class RemoteProcessor implements Processor
}
// todo rename to send with retries or something
public static <REQ, RSP> RSP sendWithCallback(Verb verb, REQ request, CandidateIterator candidates, Retry retryPolicy)
public static <REQ, RSP> RSP sendWithCallback(Verb verb, REQ request, CandidateIterator candidates, WaitStrategy backoff)
{
try
{
Promise<RSP> promise = new AsyncPromise<>();
sendWithCallbackAsync(promise, verb, request, candidates, retryPolicy);
sendWithCallbackAsync(promise, verb, request, candidates, backoff);
return promise.await().get();
}
catch (InterruptedException | ExecutionException e)
@ -231,10 +231,10 @@ public final class RemoteProcessor implements Processor
}
}
public static <REQ, RSP> void sendWithCallbackAsync(Promise<RSP> promise, Verb verb, REQ request, CandidateIterator candidates, Retry retryPolicy)
public static <REQ, RSP> void sendWithCallbackAsync(Promise<RSP> promise, Verb verb, REQ request, CandidateIterator candidates, WaitStrategy backoff)
{
//TODO (now): the retry defines how long to wait for a retry, but the old behavior scheduled the message right away... should this be delayed as well?
MessagingService.instance().<REQ, RSP>sendWithRetries(Backoff.fromRetry(retryPolicy), MessageDelivery.ImmediateRetryScheduler.instance,
MessagingService.instance().<REQ, RSP>sendWithRetries(backoff, MessageDelivery.ImmediateRetryScheduler.instance,
verb, request, candidates,
(attempt, success, failure) -> {
if (failure != null) promise.tryFailure(failure);
@ -263,8 +263,8 @@ public final class RemoteProcessor implements Processor
{
case NoMoreCandidates:
return String.format("Ran out of candidates while sending %s: %s", verb, candidates);
case MaxRetries:
return String.format("Could not succeed sending %s to %s after %d tries", verb, candidates, retryPolicy.tries);
case GiveUp:
return String.format("Could not succeed sending %s to %s; policy %s gave up", verb, candidates, backoff);
case Interrupted:
case FailedSchedule:
return null;

290
src/java/org/apache/cassandra/tcm/Retry.java
View File

@ -18,235 +18,143 @@
package org.apache.cassandra.tcm;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.function.DoubleSupplier;
import com.codahale.metrics.Meter;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.service.RetryStrategy;
import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory;
import org.apache.cassandra.service.WaitStrategy;
import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
public abstract class Retry
// TODO (expected): unwrap this, use RetryStrategy directly
public class Retry implements WaitStrategy
{
protected static final int MAX_TRIES = DatabaseDescriptor.getCmsDefaultRetryMaxTries();
private static final int DEFAULT_BACKOFF_MS = DatabaseDescriptor.getDefaultRetryBackoff().toMilliseconds();
private static final int DEFAULT_MAX_BACKOFF_MS = DatabaseDescriptor.getDefaultMaxRetryBackoff().toMilliseconds();
private static final WaitStrategy DEFAULT_STRATEGY;
static
{
DurationSpec.IntMillisecondsBound defaultBackoff = DatabaseDescriptor.getDefaultRetryBackoff();
DurationSpec.IntMillisecondsBound defaultMaxBackoff = DatabaseDescriptor.getDefaultMaxRetryBackoff();
String defaultSpec = DatabaseDescriptor.getCMSRetryDelay();
if (defaultSpec == null || (defaultBackoff != null || defaultMaxBackoff != null))
{
defaultSpec = (defaultBackoff == null ? "50ms" : defaultBackoff.toMilliseconds() + "ms")
+ "*attempts <=" + (defaultMaxBackoff == null ? "10s" : defaultMaxBackoff.toMilliseconds() + "ms")
+ ",retries=" + DatabaseDescriptor.getCmsDefaultRetryMaxTries();
}
DEFAULT_STRATEGY = RetryStrategy.parse(defaultSpec, LatencySourceFactory.none());
}
protected final int maxTries;
protected int tries;
public final long deadlineNanos;
protected Meter retryMeter;
private final WaitStrategy delegate;
int attempts = 1;
public Retry(Meter retryMeter)
public Retry(long deadlineNanos, Meter retryMeter, WaitStrategy delegate)
{
this(MAX_TRIES, retryMeter);
}
public Retry(int maxTries, Meter retryMeter)
{
this.maxTries = maxTries;
this.deadlineNanos = deadlineNanos;
this.retryMeter = retryMeter;
this.delegate = delegate;
}
public int currentTries()
public Retry(long deadlineNanos, Meter retryMeter)
{
return tries;
this(deadlineNanos, retryMeter, DEFAULT_STRATEGY);
}
public boolean reachedMax()
public int attempts()
{
return tries >= maxTries;
return attempts;
}
public void maybeSleep()
public boolean hasExpired()
{
sleepUninterruptibly(computeSleepFor(), TimeUnit.MILLISECONDS);
return nanoTime() >= deadlineNanos;
}
public long computeSleepFor()
public boolean maybeSleep()
{
long wait = computeWait(attempts, TimeUnit.MILLISECONDS);
if (wait < 0)
return false;
sleepUninterruptibly(wait, TimeUnit.MILLISECONDS);
return true;
}
@Override
public long computeWaitUntil(int attempts)
{
long wait = computeWaitInternal(attempts, TimeUnit.NANOSECONDS);
if (wait < 0)
return -1;
long now = nanoTime();
if (now >= deadlineNanos)
return -1;
return Math.min(deadlineNanos, wait + now);
}
@Override
public long computeWait(int attempts, TimeUnit units)
{
long wait = computeWaitInternal(attempts, TimeUnit.NANOSECONDS);
if (wait < 0)
return -1;
if (deadlineNanos == Long.MAX_VALUE)
return wait;
long now = nanoTime();
wait = Math.min(deadlineNanos - now, wait);
if (wait <= 0)
return -1;
return units.convert(wait, TimeUnit.NANOSECONDS);
}
private long computeWaitInternal(int attempts, TimeUnit units)
{
tries++;
retryMeter.mark();
return sleepFor();
attempts = Math.max(attempts, ++this.attempts);
return delegate.computeWait(attempts, units);
}
protected abstract long sleepFor();
protected abstract long maxWait();
public static class Jitter extends Retry
// imposes attempt limit
public static Retry withNoTimeLimit(Meter retryMeter)
{
private final Random random;
private final int maxJitterMs;
public Jitter(Meter retryMeter)
{
this(MAX_TRIES, DEFAULT_BACKOFF_MS, new Random(), retryMeter);
}
private Jitter(int maxTries, int maxJitterMs, Random random, Meter retryMeter)
{
super(maxTries, retryMeter);
this.random = random;
this.maxJitterMs = maxJitterMs;
}
public long sleepFor()
{
int actualBackoff = ThreadLocalRandom.current().nextInt(maxJitterMs / 2, maxJitterMs);
return random.nextInt(actualBackoff);
}
@Override
protected long maxWait()
{
return maxJitterMs;
}
@Override
public String toString()
{
return "Jitter{" +
", maxTries=" + maxTries +
", tries=" + tries +
", maxJitterMs=" + maxJitterMs +
'}';
}
return new Retry(Long.MAX_VALUE, retryMeter, DEFAULT_STRATEGY);
}
public static class Backoff extends Retry
public static Retry withNoTimeLimit(Meter retryMeter, WaitStrategy delegate)
{
protected final int backoffMs;
public Backoff(Meter retryMeter)
{
this(MAX_TRIES, DEFAULT_BACKOFF_MS, retryMeter);
}
public Backoff(int maxTries, int backoffMs, Meter retryMeter)
{
super(maxTries, retryMeter);
this.backoffMs = backoffMs;
}
public long sleepFor()
{
return (long) tries * backoffMs;
}
@Override
protected long maxWait()
{
return backoffMs;
}
@Override
public String toString()
{
return "Backoff{" +
"backoffMs=" + backoffMs +
", maxTries=" + maxTries +
", tries=" + tries +
'}';
}
return new Retry(Long.MAX_VALUE, retryMeter, delegate);
}
public static class ExponentialBackoff extends Retry
public static Retry until(long deadlineNanos, Meter retryMeter)
{
private final long baseSleepTimeMillis;
private final long maxSleepMillis;
private final DoubleSupplier randomSource;
public ExponentialBackoff(int maxAttempts, long baseSleepTimeMillis, long maxSleepMillis, DoubleSupplier randomSource, Meter retryMeter)
{
super(maxAttempts, retryMeter);
this.baseSleepTimeMillis = baseSleepTimeMillis;
this.maxSleepMillis = maxSleepMillis;
this.randomSource = randomSource;
}
public ExponentialBackoff(Meter retryMeter)
{
this(MAX_TRIES, DEFAULT_BACKOFF_MS, DEFAULT_MAX_BACKOFF_MS, ThreadLocalRandom.current()::nextDouble, retryMeter);
}
@Override
protected long sleepFor()
{
return org.apache.cassandra.utils.Backoff.ExponentialBackoff.computeWaitTime(tries, baseSleepTimeMillis, maxSleepMillis, randomSource);
}
@Override
protected long maxWait()
{
return maxSleepMillis;
}
return new Retry(deadlineNanos, retryMeter, DEFAULT_STRATEGY);
}
public static class Deadline extends Retry
public static Retry untilElapsed(long timeoutNanos, Meter retryMeter)
{
public final long deadlineNanos;
protected final Retry delegate;
public Deadline(long deadlineNanos, Retry delegate)
{
super(delegate.maxTries, delegate.retryMeter);
assert deadlineNanos > 0 : String.format("Deadline should be strictly positive but was %d.", deadlineNanos);
this.deadlineNanos = deadlineNanos;
this.delegate = delegate;
}
public static Deadline at(long deadlineNanos, Retry delegate)
{
return new Deadline(deadlineNanos, delegate);
}
public static Deadline after(long timeoutNanos, Retry delegate)
{
return new Deadline(Clock.Global.nanoTime() + timeoutNanos, delegate);
}
public static Deadline wrap(Retry delegate)
{
long deadlineMillis = delegate.maxTries * delegate.maxWait();
return new Deadline(Clock.Global.nanoTime() + TimeUnit.MILLISECONDS.toNanos(deadlineMillis), delegate);
}
@Override
public boolean reachedMax()
{
return delegate.reachedMax() || Clock.Global.nanoTime() > deadlineNanos;
}
public long remainingNanos()
{
return Math.max(0, deadlineNanos - Clock.Global.nanoTime());
}
@Override
public int currentTries()
{
return delegate.currentTries();
}
@Override
public long sleepFor()
{
return delegate.sleepFor();
}
@Override
protected long maxWait()
{
return deadlineNanos;
}
public String toString()
{
return String.format("Deadline{remainingMs=%d, tries=%d/%d}", TimeUnit.NANOSECONDS.toMillis(remainingNanos()), currentTries(), delegate.maxTries);
}
return new Retry(nanoTime() + timeoutNanos, retryMeter, DEFAULT_STRATEGY);
}
public static Retry untilElapsed(long timeoutNanos, Meter retryMeter, WaitStrategy waitStrategy)
{
return new Retry(nanoTime() + timeoutNanos, retryMeter, waitStrategy);
}
public String toString()
{
if (deadlineNanos == Long.MAX_VALUE)
return "RetryIndefinitely{attempts=" + attempts + '}';
return String.format("Retry{remainingMs=%d, attempts=%d}", TimeUnit.NANOSECONDS.toMillis(remainingNanos()), attempts());
}
public long remainingNanos()
{
return Math.max(0, deadlineNanos - nanoTime());
}
}

6
src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java
View File

@ -142,13 +142,13 @@ public class StubClusterMetadataService extends ClusterMetadataService
private StubProcessor() {}
@Override
public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy)
public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy)
{
throw new UnsupportedOperationException();
}
@Override
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy)
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy)
{
throw new UnsupportedOperationException();
}
@ -160,7 +160,7 @@ public class StubClusterMetadataService extends ClusterMetadataService
}
@Override
public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy)
public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy)
{
throw new UnsupportedOperationException();
}

6
src/java/org/apache/cassandra/tcm/migration/GossipProcessor.java
View File

@ -30,13 +30,13 @@ import org.apache.cassandra.tcm.log.LogState;
public class GossipProcessor implements Processor
{
@Override
public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry.Deadline retryPolicy)
public Commit.Result commit(Entry.Id entryId, Transformation transform, Epoch lastKnown, Retry retryPolicy)
{
throw new IllegalStateException("Can't commit transformations when running in gossip mode. Enable the ClusterMetadataService with `nodetool cms initialize`.");
}
@Override
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy)
public ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy)
{
return ClusterMetadata.current();
}
@ -48,7 +48,7 @@ public class GossipProcessor implements Processor
}
@Override
public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry.Deadline retryPolicy)
public LogState getLogState(Epoch start, Epoch end, boolean includeSnapshot, Retry retryPolicy)
{
throw new IllegalStateException("Can't reconstruct log state when running in gossip mode. Enable the ClusterMetadataService with `nodetool addtocms`.");
}

2
src/java/org/apache/cassandra/tcm/sequences/DropAccordTable.java
View File

@ -175,7 +175,7 @@ public class DropAccordTable extends MultiStepOperation<Epoch>
// There are retries baked into this call, but trying to handle timeouts more broadly is put on hold as there is active work to define a EpochSyncPoint that should be far cheaper
// which would avoid the timeout issues
// NOTE: ExclusiveSyncPoint must find all keys in the range, then make sure nothing is blocking them... this causes a lot of IO. EpochSyncPoint just needs to validate that the last txn processed is in the newer epoch, this can work with in-memory state.
AccordService.instance().awaitTableDrop(table.id);
AccordService.instance().awaitDone(table.id, metadata.epoch.getEpoch());
long awaitEndNanos = nanoTime();
logger.info("Wait for Accord to see the drop table was success. " +
"Took {} to wait for Accord to learn about the change, then {} to process everything",

33
src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java
View File

@ -50,6 +50,8 @@ import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.RequestCallbackWithFailure;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.service.RetryStrategy;
import org.apache.cassandra.service.WaitStrategy;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.Retry;
@ -58,6 +60,13 @@ import org.apache.cassandra.tcm.membership.Location;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import static org.apache.cassandra.config.DatabaseDescriptor.getCmsDefaultRetryMaxTries;
import static org.apache.cassandra.config.DatabaseDescriptor.getProgressBarrierBackoff;
import static org.apache.cassandra.config.DatabaseDescriptor.getProgressBarrierDefaultConsistencyLevel;
import static org.apache.cassandra.config.DatabaseDescriptor.getProgressBarrierMinConsistencyLevel;
import static org.apache.cassandra.config.DatabaseDescriptor.getProgressBarrierTimeout;
import static org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory.none;
/**
* ProgressBarrier is responsible for ensuring that epoch visibility plays together with quorum consistency.
*
@ -73,10 +82,16 @@ import org.apache.cassandra.utils.concurrent.AsyncPromise;
public class ProgressBarrier
{
private static final Logger logger = LoggerFactory.getLogger(ProgressBarrier.class);
private static final ConsistencyLevel MIN_CL = DatabaseDescriptor.getProgressBarrierMinConsistencyLevel();
private static final ConsistencyLevel DEFAULT_CL = DatabaseDescriptor.getProgressBarrierDefaultConsistencyLevel();
private static final long TIMEOUT_MILLIS = DatabaseDescriptor.getProgressBarrierTimeout(TimeUnit.MILLISECONDS);
private static final long BACKOFF_MILLIS = DatabaseDescriptor.getProgressBarrierBackoff(TimeUnit.MILLISECONDS);
private static final ConsistencyLevel MIN_CL = getProgressBarrierMinConsistencyLevel();
private static final ConsistencyLevel DEFAULT_CL = getProgressBarrierDefaultConsistencyLevel();
private static final long TIMEOUT_MILLIS = getProgressBarrierTimeout(TimeUnit.MILLISECONDS);
private static final long BACKOFF_MILLIS = getProgressBarrierBackoff(TimeUnit.MILLISECONDS);
private static final WaitStrategy WAIT_STRATEGY;
static
{
WAIT_STRATEGY = RetryStrategy.parse(BACKOFF_MILLIS + "ms" + "*attempts <=" + TIMEOUT_MILLIS + "ms,retries="
+ getCmsDefaultRetryMaxTries(), none());
}
public final Epoch waitFor;
// Location of the affected node; used for LOCAL_QUORUM
@ -195,11 +210,8 @@ public class ProgressBarrier
requests.add(new WatermarkRequest(peer, messagingService, waitFor));
long start = Clock.Global.nanoTime();
Retry.Deadline deadline = Retry.Deadline.after(TimeUnit.MILLISECONDS.toNanos(TIMEOUT_MILLIS),
new Retry.Backoff(DatabaseDescriptor.getCmsDefaultRetryMaxTries(),
(int) BACKOFF_MILLIS,
TCMMetrics.instance.progressBarrierRetries));
while (!deadline.reachedMax())
Retry deadline = Retry.untilElapsed(TimeUnit.MILLISECONDS.toNanos(TIMEOUT_MILLIS), TCMMetrics.instance.progressBarrierRetries, WAIT_STRATEGY);
while (!deadline.hasExpired())
{
for (WatermarkRequest request : requests)
request.retry();
@ -219,7 +231,8 @@ public class ProgressBarrier
// No need to try processing until we collect enough nodes to pass all conditions
if (collected.size() < maxWaitFor)
{
deadline.maybeSleep();
if (!deadline.maybeSleep())
break;
continue;
}

7
src/java/org/apache/cassandra/tcm/sequences/ReconfigureCMS.java
View File

@ -343,13 +343,13 @@ public class ReconfigureCMS extends MultiStepOperation<AdvanceCMSReconfiguration
// paxos repair at the beginning of each step, before streaming where applicable, will ensure that the
// overlapping quorums invariant holds.
Retry.Backoff retry = new Retry.Backoff(TCMMetrics.instance.repairPaxosTopologyRetries);
Retry retry = Retry.withNoTimeLimit(TCMMetrics.instance.repairPaxosTopologyRetries);
List<Supplier<Future<?>>> remaining = ActiveRepairService.instance()
.repairPaxosForTopologyChangeAsync(SchemaConstants.METADATA_KEYSPACE_NAME,
Collections.singletonList(entireRange),
"CMS reconfiguration");
while (!retry.reachedMax())
while (true)
{
Map<Supplier<Future<?>>, Future<?>> tasks = new HashMap<>();
for (Supplier<Future<?>> supplier : remaining)
@ -377,7 +377,8 @@ public class ReconfigureCMS extends MultiStepOperation<AdvanceCMSReconfiguration
if (remaining.isEmpty())
return;
retry.maybeSleep();
if (!retry.maybeSleep())
break;
}
logger.error("Added node as a CMS, but failed to repair paxos topology after this operation.");
}

162
src/java/org/apache/cassandra/utils/Backoff.java
View File

@ -1,162 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils;
import java.util.concurrent.TimeUnit;
import java.util.function.DoubleSupplier;
import org.apache.cassandra.config.RetrySpec;
import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.tcm.Retry;
public interface Backoff
{
// TODO (required): backoff should not have retries
boolean mayRetry(int attempt);
long computeWaitTime(int attempt);
TimeUnit unit();
static Backoff fromRetry(Retry retry)
{
return new Backoff()
{
@Override
public boolean mayRetry(int attempt)
{
return !retry.reachedMax();
}
@Override
public long computeWaitTime(int retryCount)
{
return retry.computeSleepFor();
}
@Override
public TimeUnit unit()
{
return TimeUnit.MILLISECONDS;
}
};
}
static Backoff fromConfig(SharedContext ctx, RetrySpec spec)
{
if (!spec.isEnabled())
return Backoff.None.INSTANCE;
return new Backoff.ExponentialBackoff(spec.maxAttempts.value, spec.baseSleepTime.toMilliseconds(), spec.maxSleepTime.toMilliseconds(), ctx.random().get()::nextDouble);
}
enum None implements Backoff
{
INSTANCE;
@Override
public boolean mayRetry(int attempt)
{
return false;
}
@Override
public long computeWaitTime(int retryCount)
{
throw new UnsupportedOperationException();
}
@Override
public TimeUnit unit()
{
throw new UnsupportedOperationException();
}
}
enum NO_OP implements Backoff
{
INSTANCE;
@Override
public boolean mayRetry(int attempt)
{
return true;
}
@Override
public long computeWaitTime(int retryCount)
{
return 0;
}
@Override
public TimeUnit unit()
{
return TimeUnit.NANOSECONDS;
}
}
class ExponentialBackoff implements Backoff
{
private final int maxAttempts;
private final long baseSleepTimeMillis;
private final long maxSleepMillis;
private final DoubleSupplier randomSource;
public ExponentialBackoff(int maxAttempts, long baseSleepTimeMillis, long maxSleepMillis, DoubleSupplier randomSource)
{
this.maxAttempts = maxAttempts;
this.baseSleepTimeMillis = baseSleepTimeMillis;
this.maxSleepMillis = maxSleepMillis;
this.randomSource = randomSource;
}
public int maxAttempts()
{
return maxAttempts;
}
@Override
public boolean mayRetry(int attempt)
{
return attempt < maxAttempts;
}
@Override
public long computeWaitTime(int retryCount)
{
return computeWaitTime(retryCount, baseSleepTimeMillis, maxSleepMillis, randomSource);
}
public static long computeWaitTime(int retryCount, long baseSleepTimeMillis, long maxSleepMillis, DoubleSupplier randomSource)
{
long baseTimeMillis = baseSleepTimeMillis * (1L << retryCount);
// it's possible that this overflows, so fall back to max;
if (baseTimeMillis <= 0)
baseTimeMillis = maxSleepMillis;
// now make sure this is capped to target max
baseTimeMillis = Math.min(baseTimeMillis, maxSleepMillis);
return (long) (baseTimeMillis * (randomSource.getAsDouble() + 0.5));
}
@Override
public TimeUnit unit()
{
return TimeUnit.MILLISECONDS;
}
}
}

5
src/java/org/apache/cassandra/utils/HeapUtils.java
View File

@ -24,6 +24,7 @@ import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.nio.file.FileStore;
import java.nio.file.Path;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import javax.management.MBeanServer;
@ -37,6 +38,7 @@ import com.sun.management.HotSpotDiagnosticMXBean;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.PathUtils;
import org.apache.cassandra.utils.NoSpamLogger.NoSpamLogStatement;
import static org.apache.cassandra.config.CassandraRelevantEnv.JAVA_HOME;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
@ -48,6 +50,7 @@ import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
public final class HeapUtils
{
private static final Logger logger = LoggerFactory.getLogger(HeapUtils.class);
private static final NoSpamLogStatement disabledStatement = NoSpamLogger.getStatement(logger, "Heap dump creation on uncaught exceptions is disabled.", 1L, TimeUnit.MINUTES);
private static final Lock DUMP_LOCK = new ReentrantLock();
@ -130,7 +133,7 @@ public final class HeapUtils
}
else
{
logger.debug("Heap dump creation on uncaught exceptions is disabled.");
disabledStatement.debug();
}
}
catch (Throwable e)

81
src/java/org/apache/cassandra/utils/concurrent/AsyncFuture.java
View File

@ -19,14 +19,18 @@
package org.apache.cassandra.utils.concurrent;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.concurrent.locks.LockSupport;
import java.util.function.Function;
import javax.annotation.Nullable;
import com.google.common.util.concurrent.AsyncFunction;
import com.google.common.util.concurrent.ListenableFuture; // checkstyle: permit this import
import accord.utils.Invariants;
import io.netty.util.concurrent.GenericFutureListener;
import org.apache.cassandra.utils.concurrent.ListenerList.Waiting;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
/**
* Our default {@link Future} implementation, with all state being managed without locks (except those used by the JVM).
@ -45,11 +49,6 @@ import io.netty.util.concurrent.GenericFutureListener;
*/
public class AsyncFuture<V> extends AbstractFuture<V>
{
@SuppressWarnings({ "rawtypes" })
private static final AtomicReferenceFieldUpdater<AsyncFuture, WaitQueue> waitingUpdater = AtomicReferenceFieldUpdater.newUpdater(AsyncFuture.class, WaitQueue.class, "waiting");
@SuppressWarnings({ "unused" })
private volatile WaitQueue waiting;
public AsyncFuture()
{
super();
@ -100,10 +99,7 @@ public class AsyncFuture<V> extends AbstractFuture<V>
if (resultUpdater.compareAndSet(this, current, v))
{
if (v != UNCANCELLABLE)
{
ListenerList.notify(listenersUpdater, this);
AsyncAwaitable.signalAll(waitingUpdater, this);
}
return true;
}
}
@ -122,6 +118,17 @@ public class AsyncFuture<V> extends AbstractFuture<V>
ListenerList.notify(listenersUpdater, this);
}
/**
* Logically append {@code newListener} to {@link #listeners}
* (at this stage it is a stack, so we actually prepend)
*
* @param newListener must be either a {@link ListenerList} or {@link GenericFutureListener}
*/
boolean appendListenerIfNotNotifying(ListenerList<V> newListener)
{
return ListenerList.pushIfNotNotifying(listenersUpdater, this, newListener);
}
/**
* Support {@link com.google.common.util.concurrent.Futures#transform} natively
*
@ -161,14 +168,64 @@ public class AsyncFuture<V> extends AbstractFuture<V>
@Override
public AsyncFuture<V> await() throws InterruptedException
{
//noinspection unchecked
return AsyncAwaitable.await(waitingUpdater, Future::isDone, this);
if (isDone())
return this;
Waiting<V> waiting = new Waiting<>();
if (!appendListenerIfNotNotifying(waiting))
{
Invariants.require(isDone());
return this;
}
while (true)
{
if (isDone())
return this;
LockSupport.park();
if (Thread.interrupted())
{
waiting.cancel();
throw new InterruptedException();
}
}
}
@Override
public boolean awaitUntil(long nanoTimeDeadline) throws InterruptedException
{
return AsyncAwaitable.awaitUntil(waitingUpdater, Future::isDone, this, nanoTimeDeadline);
if (isDone())
return true;
Waiting<V> waiting = new Waiting<>();
if (!appendListenerIfNotNotifying(waiting))
{
Invariants.require(isDone());
return true;
}
while (true)
{
if (isDone())
return true;
long wait = nanoTimeDeadline - nanoTime();
if (wait <= 0)
{
waiting.cancel();
return false;
}
LockSupport.parkNanos(wait);
if (Thread.interrupted())
{
waiting.cancel();
throw new InterruptedException();
}
}
}
}

2
src/java/org/apache/cassandra/utils/concurrent/ConcurrentLinkedStack.java
View File

@ -38,7 +38,7 @@ public class ConcurrentLinkedStack<T>
public void push(T value)
{
IntrusiveStack.push(headUpdater, this, (Node)new Node<>(value));
IntrusiveStack.getAndPush(headUpdater, this, (Node)new Node<>(value));
}
public boolean isEmpty()

26
src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java
View File

@ -65,39 +65,51 @@ public class IntrusiveStack<T extends IntrusiveStack<T>> implements Iterable<T>
T next;
@Inline
protected static <O, T extends IntrusiveStack<T>> T push(AtomicReferenceFieldUpdater<? super O, T> headUpdater, O owner, T prepend)
protected static <O, T extends IntrusiveStack<T>> T getAndPush(AtomicReferenceFieldUpdater<? super O, T> headUpdater, O owner, T prepend)
{
return push(headUpdater, owner, prepend, (prev, next) -> {
return getAndPush(headUpdater, owner, prepend, (prev, next) -> {
next.next = prev;
return next;
});
}
protected static <O, T extends IntrusiveStack<T>> T push(AtomicReferenceFieldUpdater<O, T> headUpdater, O owner, T prepend, BiFunction<T, T, T> combine)
protected static <O, T extends IntrusiveStack<T>> T getAndPush(AtomicReferenceFieldUpdater<O, T> headUpdater, O owner, T prepend, BiFunction<T, T, T> combine)
{
while (true)
{
T head = headUpdater.get(owner);
if (headUpdater.compareAndSet(owner, head, combine.apply(head, prepend)))
T newHead = combine.apply(head, prepend);
if (headUpdater.compareAndSet(owner, head, newHead))
return head;
}
}
protected static <O, T extends IntrusiveStack<T>> T pushAndGet(AtomicReferenceFieldUpdater<O, T> headUpdater, O owner, T prepend, BiFunction<T, T, T> combine)
{
while (true)
{
T head = headUpdater.get(owner);
T newHead = combine.apply(head, prepend);
if (head == newHead || headUpdater.compareAndSet(owner, head, newHead))
return newHead;
}
}
protected interface Setter<O, T>
{
public boolean compareAndSet(O owner, T expect, T update);
}
@Inline
protected static <O, T extends IntrusiveStack<T>> T push(Function<O, T> getter, Setter<O, T> setter, O owner, T prepend)
protected static <O, T extends IntrusiveStack<T>> T getAndPush(Function<O, T> getter, Setter<O, T> setter, O owner, T prepend)
{
return push(getter, setter, owner, prepend, (prev, next) -> {
return getAndPush(getter, setter, owner, prepend, (prev, next) -> {
next.next = prev;
return next;
});
}
protected static <O, T extends IntrusiveStack<T>> T push(Function<O, T> getter, Setter<O, T> setter, O owner, T prepend, BiFunction<T, T, T> combine)
protected static <O, T extends IntrusiveStack<T>> T getAndPush(Function<O, T> getter, Setter<O, T> setter, O owner, T prepend, BiFunction<T, T, T> combine)
{
while (true)
{

60
src/java/org/apache/cassandra/utils/concurrent/ListenerList.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.utils.concurrent;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.concurrent.locks.LockSupport;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import javax.annotation.Nullable;
@ -50,13 +51,22 @@ abstract class ListenerList<V> extends IntrusiveStack<ListenerList<V>>
{
Notifying result = new Notifying();
result.next = next;
next.next = prev == NOTIFYING ? null : prev;
next.next = prev.next;
return result;
}
next.next = prev;
return next;
}
static ListenerList pushHeadIfNotNotifying(ListenerList prev, ListenerList next)
{
if (prev instanceof Notifying<?>)
return prev;
next.next = prev;
return next;
}
/**
* Logically append {@code newListener} to {@link #listeners}
* (at this stage it is a stack, so we actually prepend)
@ -66,7 +76,12 @@ abstract class ListenerList<V> extends IntrusiveStack<ListenerList<V>>
@Inline
static <T> void push(AtomicReferenceFieldUpdater<? super T, ListenerList> updater, T in, ListenerList newListener)
{
IntrusiveStack.push(updater, in, newListener, ListenerList::pushHead);
IntrusiveStack.getAndPush(updater, in, newListener, ListenerList::pushHead);
}
static <T> boolean pushIfNotNotifying(AtomicReferenceFieldUpdater<? super T, ListenerList> updater, T in, ListenerList newListener)
{
return newListener == IntrusiveStack.pushAndGet(updater, in, newListener, ListenerList::pushHeadIfNotNotifying);
}
/**
@ -91,13 +106,22 @@ abstract class ListenerList<V> extends IntrusiveStack<ListenerList<V>>
if (updater.compareAndSet(in, listeners, NOTIFYING))
{
while (true)
try
{
notifyExclusive(listeners, in);
if (updater.compareAndSet(in, NOTIFYING, null))
return;
while (true)
{
notifyExclusive(listeners, in);
if (updater.compareAndSet(in, NOTIFYING, null))
return;
listeners = updater.getAndSet(in, NOTIFYING);
listeners = updater.getAndSet(in, NOTIFYING);
}
}
catch (Throwable t)
{
Thread thread = Thread.currentThread();
try { thread.getUncaughtExceptionHandler().uncaughtException(thread, t); }
catch (Throwable t2) { t.addSuppressed(t2); t.printStackTrace(); }
}
}
}
@ -354,6 +378,7 @@ abstract class ListenerList<V> extends IntrusiveStack<ListenerList<V>>
static class Notifying<V> extends ListenerList<V>
{
static final Notifying NOTIFYING = new Notifying();
static final Notifying DONE = new Notifying();
@Override
void notifySelf(Executor notifyExecutor, Future<V> future)
@ -361,6 +386,27 @@ abstract class ListenerList<V> extends IntrusiveStack<ListenerList<V>>
}
}
static class Waiting<V> extends ListenerList<V>
{
volatile Thread waiting = Thread.currentThread();
@Override
void notifySelf(Executor notifyExecutor, Future<V> future)
{
Thread thread = waiting;
if (thread != null)
{
LockSupport.unpark(thread);
waiting = null;
}
}
void cancel()
{
waiting = null;
}
}
/**
* @return true iff the invoking thread is executing {@code executor}
*/

2
test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
View File

@ -101,7 +101,7 @@ public class InstanceConfig implements IInstanceConfig
.set("accord.journal_directory", accord.journal_directory)
.set("accord.queue_shard_count", accord.queue_shard_count.toString())
.set("accord.command_store_shard_count", accord.command_store_shard_count.toString())
.set("accord.recover_delay", accord.recover_delay.toString())
.set("accord.expire_txn", accord.expire_txn)
.set("accord.enable_virtual_debug_only_keyspace", "true")
.set("partitioner", "org.apache.cassandra.dht.Murmur3Partitioner")
.set("start_native_transport", true)

4
test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
View File

@ -490,12 +490,12 @@ public class ClusterUtils
public static Callable<Void> pauseBeforeEnacting(IInvokableInstance instance, long epoch)
{
return pauseBeforeEnacting(instance, Epoch.create(epoch), 10, TimeUnit.SECONDS);
return pauseBeforeEnacting(instance, Epoch.create(epoch), 30, TimeUnit.SECONDS);
}
public static Callable<Void> pauseBeforeEnacting(IInvokableInstance instance, Epoch epoch)
{
return pauseBeforeEnacting(instance, epoch, 10, TimeUnit.SECONDS);
return pauseBeforeEnacting(instance, epoch, 30, TimeUnit.SECONDS);
}
protected static Callable<Void> pauseBeforeEnacting(IInvokableInstance instance,

3
test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java
View File

@ -66,8 +66,7 @@ public class QueriesTableTest extends TestBaseImpl
{
SHARED_CLUSTER = init(Cluster.build(1).withInstanceInitializer(QueryDelayHelper::install)
.withConfig(c -> c.with(Feature.NATIVE_PROTOCOL, Feature.GOSSIP)
.set("write_request_timeout", "10s")
.set("transaction_timeout", "15s")).start());
.set("write_request_timeout", "10s")).start());
DRIVER_CLUSTER = JavaDriverUtils.create(SHARED_CLUSTER);
SESSION = DRIVER_CLUSTER.connect();

5
test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java
View File

@ -106,7 +106,10 @@ public class ShortReadProtectionTest extends TestBaseImpl
// but maybe that is out of scope and is covered by the dedicated BRR tests?
cluster = init(Cluster.build()
.withNodes(NUM_NODES)
.withConfig(config -> config.set("hinted_handoff_enabled", false))
.withConfig(config ->
config.set("hinted_handoff_enabled", false)
.set("accord.shard_durability_target_splits", 4)
)
.start());
}

16
test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java
View File

@ -109,7 +109,7 @@ public class AccordBootstrapTest extends TestBaseImpl
Assertions.assertThat(completed)
.describedAs("Epoch %s did not become ready within timeout on %s -> %s",
epoch, FBUtilities.getBroadcastAddressAndPort(),
service().configurationService().getEpochSnapshot(epoch))
service().configService().getEpochSnapshot(epoch))
.isTrue();
}
catch (InterruptedException e)
@ -122,7 +122,7 @@ public class AccordBootstrapTest extends TestBaseImpl
{
try
{
AccordConfigurationService configService = service().configurationService();
AccordConfigurationService configService = service().configService();
boolean completed = configService.unsafeLocalSyncNotified(epoch).await(30, TimeUnit.SECONDS);
Assert.assertTrue(String.format("Local sync notification for epoch %s did not become ready within timeout on %s",
epoch, FBUtilities.getBroadcastAddressAndPort()), completed);
@ -195,7 +195,7 @@ public class AccordBootstrapTest extends TestBaseImpl
Assert.assertEquals(initialMax, ClusterMetadata.current().epoch.getEpoch());
System.out.println("Awaiting " + initialMax);
awaitEpoch(initialMax);
AccordConfigurationService configService = service().configurationService();
AccordConfigurationService configService = service().configService();
long minEpoch = configService.minEpoch();
Assert.assertEquals(initialMax, configService.maxEpoch());
@ -222,7 +222,7 @@ public class AccordBootstrapTest extends TestBaseImpl
node.runOnInstance(() -> {
ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(schemaChangeMax));
awaitEpoch(schemaChangeMax);
AccordConfigurationService configService = service().configurationService();
AccordConfigurationService configService = service().configService();
for (long epoch = initialMax + 1; epoch <= schemaChangeMax; epoch++)
{
@ -260,7 +260,7 @@ public class AccordBootstrapTest extends TestBaseImpl
Assert.assertEquals(bootstrapMax, ClusterMetadata.current().epoch.getEpoch());
AccordService service = (AccordService) AccordService.instance();
awaitEpoch(bootstrapMax);
AccordConfigurationService configService = service.configurationService();
AccordConfigurationService configService = service.configService();
awaitLocalSyncNotification(bootstrapMax);
Assert.assertEquals(EpochSnapshot.completed(bootstrapMax), configService.getEpochSnapshot(bootstrapMax));
@ -381,7 +381,7 @@ public class AccordBootstrapTest extends TestBaseImpl
node.runOnInstance(() -> {
Assert.assertEquals(initialMax, ClusterMetadata.current().epoch.getEpoch());
awaitEpoch(initialMax);
AccordConfigurationService configService = service().configurationService();
AccordConfigurationService configService = service().configService();
long minEpoch = configService.minEpoch();
Assert.assertEquals(initialMax, configService.maxEpoch());
@ -404,7 +404,7 @@ public class AccordBootstrapTest extends TestBaseImpl
Assert.assertEquals(schemaChangeMax, ClusterMetadata.current().epoch.getEpoch());
AccordService service = (AccordService) AccordService.instance();
awaitEpoch(schemaChangeMax);
AccordConfigurationService configService = service.configurationService();
AccordConfigurationService configService = service.configService();
for (long epoch = initialMax + 1; epoch <= schemaChangeMax; epoch++)
{
@ -439,7 +439,7 @@ public class AccordBootstrapTest extends TestBaseImpl
Assert.assertEquals(moveMax, ClusterMetadata.current().epoch.getEpoch());
AccordService service = (AccordService) AccordService.instance();
awaitEpoch(moveMax);
AccordConfigurationService configService = service.configurationService();
AccordConfigurationService configService = service.configService();
awaitLocalSyncNotification(moveMax);
Assert.assertEquals(EpochSnapshot.completed(moveMax), configService.getEpochSnapshot(moveMax));

5
test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropKeyspaceTest.java
View File

@ -39,8 +39,9 @@ public class AccordDropKeyspaceTest extends AccordDropTableBase
try (Cluster cluster = Cluster.build(3)
.withoutVNodes()
.withConfig(c -> c.with(GOSSIP, NETWORK, NATIVE_PROTOCOL)
.set("auto_snapshot", false))
.start())
.set("auto_snapshot", false)
.set("accord.shard_durability_target_splits", 4)
).start())
{
fixDistributedSchemas(cluster);
for (int i = 0; i < examples; i++)

1
test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableTest.java
View File

@ -39,6 +39,7 @@ public class AccordDropTableTest extends AccordDropTableBase
try (Cluster cluster = Cluster.build(3)
.withoutVNodes()
.withConfig(c -> c.with(GOSSIP, NETWORK, NATIVE_PROTOCOL)
.set("accord.shard_durability_target_splits", 4)
.set("auto_snapshot", false))
.start())
{

Some files were not shown because too many files have changed in this diff Show More