natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Accord migration and interop correctness 

Patch by Ariel Weisberg; Reviewed by Blake Eggleston for CASSANDRA-19744
This commit is contained in:
Ariel Weisberg 2024-03-18 15:38:31 -04:00 committed by David Capwell
parent c377a066af
commit 8ab5003118
110 changed files with 4461 additions and 1380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitmodules vendored
View File

@ -1,4 +1,4 @@
[submodule "modules/accord"]
path = modules/accord
url = https://github.com/apache/cassandra-accord.git
url = ../cassandra-accord.git
branch = trunk

2
modules/accord

@ -1 +1 @@
Subproject commit 4aa0a8aeb6b12036660695e3fb89c69b5d40f345
Subproject commit 129a4862df43fdc5893687922a77bb0288f8cb83

229
src/java/org/apache/cassandra/batchlog/BatchlogManager.java
View File

@ -27,25 +27,25 @@ import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.RateLimiter;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Future;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.UntypedResultSet.Row;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.Keyspace;
@ -55,6 +55,7 @@ import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.gms.FailureDetector;
@ -70,26 +71,41 @@ import org.apache.cassandra.locator.Replicas;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessageFlag;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.WriteResponseHandler;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitMutations;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Future;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.config.CassandraRelevantProperties.BATCHLOG_REPLAY_TIMEOUT_IN_MS;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithPaging;
import static org.apache.cassandra.hints.HintsService.RETRY_ON_DIFFERENT_SYSTEM_UUID;
import static org.apache.cassandra.net.Verb.MUTATION_REQ;
import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.retry_new_protocol;
import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.mutateWithAccordAsync;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
public class BatchlogManager implements BatchlogManagerMBean
{
public static final String MBEAN_NAME = "org.apache.cassandra.db:type=BatchlogManager";
private static final long REPLAY_INTERVAL = 10 * 1000; // milliseconds
static final int DEFAULT_PAGE_SIZE = 128;
private static final Logger logger = LoggerFactory.getLogger(BatchlogManager.class);
@ -104,6 +120,8 @@ public class BatchlogManager implements BatchlogManagerMBean
private final RateLimiter rateLimiter = RateLimiter.create(Double.MAX_VALUE);
private final AtomicBoolean isBatchlogReplayPaused = new AtomicBoolean(false);
public BatchlogManager()
{
batchlogTasks = executorFactory().scheduled(false, "BatchlogTasks");
@ -115,7 +133,7 @@ public class BatchlogManager implements BatchlogManagerMBean
batchlogTasks.scheduleWithFixedDelay(this::replayFailedBatches,
StorageService.RING_DELAY_MILLIS,
REPLAY_INTERVAL,
CassandraRelevantProperties.BATCHLOG_REPLAY_INTERVAL_MS.getLong(),
MILLISECONDS);
}
@ -184,7 +202,9 @@ public class BatchlogManager implements BatchlogManagerMBean
public void forceBatchlogReplay() throws Exception
{
logger.debug("Forcing batchlog replay");
startBatchlogReplay().get();
logger.debug("Finished forcing batchlog replay");
}
public Future<?> startBatchlogReplay()
@ -193,14 +213,25 @@ public class BatchlogManager implements BatchlogManagerMBean
return batchlogTasks.submit(this::replayFailedBatches);
}
void performInitialReplay() throws InterruptedException, ExecutionException
public void pauseReplay()
{
// Invokes initial replay. Used for testing only.
batchlogTasks.submit(this::replayFailedBatches).get();
logger.debug("Paused batchlog replay");
isBatchlogReplayPaused.set(true);
}
public void resumeReplay()
{
logger.debug("Resumed batchlog replay");
isBatchlogReplayPaused.set(false);
}
private void replayFailedBatches()
{
if (isBatchlogReplayPaused.get())
{
logger.debug("Batch log replay is paused, skipping replay");
return;
}
logger.trace("Started replayFailedBatches");
// rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml).
@ -223,6 +254,7 @@ public class BatchlogManager implements BatchlogManagerMBean
SchemaConstants.SYSTEM_KEYSPACE_NAME,
SystemKeyspace.BATCHES);
UntypedResultSet batches = executeInternalWithPaging(query, pageSize, lastReplayedUuid, limitUuid);
processBatchlogEntries(batches, pageSize, rateLimiter);
lastReplayedUuid = limitUuid;
logger.trace("Finished replayFailedBatches");
@ -276,16 +308,7 @@ public class BatchlogManager implements BatchlogManagerMBean
int version = row.getInt("version");
try
{
ReplayingBatch batch = new ReplayingBatch(id, version, row.getList("mutations", BytesType.instance));
if (batch.replay(rateLimiter, hintedNodes) > 0)
{
unfinishedBatches.add(batch);
}
else
{
remove(id); // no write mutations were sent (either expired or all CFs involved truncated).
++totalBatchesReplayed;
}
dispatchBatch(rateLimiter, row, id, version, hintedNodes, unfinishedBatches);
}
catch (IOException e)
{
@ -307,6 +330,8 @@ public class BatchlogManager implements BatchlogManagerMBean
// finalize the incomplete last page of batches
if (positionInPage > 0)
finishAndClearBatches(unfinishedBatches, hintedNodes, replayedBatches);
else
logger.trace("Had no batches to replay");
if (caughtException != null)
logger.warn(String.format("Encountered %d unexpected exceptions while sending out batches", skipped), caughtException);
@ -318,6 +343,35 @@ public class BatchlogManager implements BatchlogManagerMBean
replayedBatches.forEach(BatchlogManager::remove);
}
private void dispatchBatch(RateLimiter rateLimiter, Row row, TimeUUID id, int version, Set<UUID> hintedNodes, ArrayList<ReplayingBatch> unfinishedBatches) throws IOException
{
while (true)
{
ClusterMetadata cm = ClusterMetadata.current();
try
{
ReplayingBatch batch = new ReplayingBatch(id, version, row.getList("mutations", BytesType.instance), cm);
if (batch.replay(rateLimiter, hintedNodes))
{
unfinishedBatches.add(batch);
}
else
{
remove(id); // no write mutations were sent (either expired or all CFs involved truncated).
++totalBatchesReplayed;
}
}
catch (RetryOnDifferentSystemException e)
{
// Self apply can throw retry on different system
// Barring bugs we should already have the latest cluster metadata needed to correctly
// split the batch and retry since that is what was used to generate the exception
continue;
}
break;
}
}
private void finishAndClearBatches(ArrayList<ReplayingBatch> batches, Set<UUID> hintedNodes, Set<TimeUUID> replayedBatches)
{
// schedule hints for timed out deliveries
@ -340,61 +394,112 @@ public class BatchlogManager implements BatchlogManagerMBean
{
private final TimeUUID id;
private final long writtenAt;
private final List<Mutation> mutations;
private final int unsplitGcGs;
private final List<Mutation> normalMutations;
private final List<Mutation> accordMutations;
private final int replayedBytes;
private final ClusterMetadata cm;
private List<ReplayWriteResponseHandler<Mutation>> replayHandlers;
private List<ReplayWriteResponseHandler<Mutation>> replayHandlers = ImmutableList.of();
private AsyncTxnResult accordResult;
@Nullable
private Dispatcher.RequestTime accordTxnStart;
ReplayingBatch(TimeUUID id, int version, List<ByteBuffer> serializedMutations) throws IOException
ReplayingBatch(TimeUUID id, int version, List<ByteBuffer> serializedMutations, ClusterMetadata cm) throws IOException
{
this.id = id;
this.writtenAt = id.unix(MILLISECONDS);
this.mutations = new ArrayList<>(serializedMutations.size());
this.replayedBytes = addMutations(version, serializedMutations);
List<Mutation> unsplitMutations = new ArrayList<>(serializedMutations.size());
this.replayedBytes = addMutations(unsplitMutations, writtenAt, version, serializedMutations);
unsplitGcGs = gcgs(unsplitMutations);
SplitMutations<Mutation> splitMutations = ConsensusMigrationMutationHelper.splitMutationsIntoAccordAndNormal(cm, unsplitMutations);
logger.trace("Replaying batch with Accord {} and normal {}", splitMutations.accordMutations(), splitMutations.normalMutations());
normalMutations = splitMutations.normalMutations();
accordMutations = splitMutations.accordMutations();
if (accordMutations != null)
accordTxnStart = new Dispatcher.RequestTime(Clock.Global.nanoTime());
this.cm = cm;
}
public int replay(RateLimiter rateLimiter, Set<UUID> hintedNodes) throws IOException
public boolean replay(RateLimiter rateLimiter, Set<UUID> hintedNodes) throws IOException
{
logger.trace("Replaying batch {}", id);
if (mutations.isEmpty())
return 0;
if ((normalMutations == null || normalMutations.isEmpty()) && (accordMutations == null || accordMutations.isEmpty()))
return false;
int gcgs = gcgs(mutations);
if (MILLISECONDS.toSeconds(writtenAt) + gcgs <= FBUtilities.nowInSeconds())
return 0;
if (MILLISECONDS.toSeconds(writtenAt) + unsplitGcGs <= FBUtilities.nowInSeconds())
return false;
replayHandlers = sendReplays(mutations, writtenAt, hintedNodes);
if (accordMutations != null)
{
accordTxnStart = accordTxnStart.withStartedAt(Clock.Global.nanoTime());
accordResult = accordMutations != null ? mutateWithAccordAsync(cm, accordMutations, null, accordTxnStart) : null;
}
if (normalMutations != null)
replayHandlers = sendReplays(normalMutations, writtenAt, hintedNodes);
rateLimiter.acquire(replayedBytes); // acquire afterwards, to not mess up ttl calculation.
return replayHandlers.size();
return replayHandlers.size() > 0 || accordMutations != null;
}
public void finish(Set<UUID> hintedNodes)
{
for (int i = 0; i < replayHandlers.size(); i++)
Throwable failure = null;
// Check if the Accord mutations succeeded asynchronously
try
{
ReplayWriteResponseHandler<Mutation> handler = replayHandlers.get(i);
try
if (accordResult != null)
{
handler.get();
IAccordService accord = AccordService.instance();
TxnResult.Kind kind = accord.getTxnResult(accordResult, true, ConsistencyLevel.QUORUM, accordTxnStart).kind();
if (kind == retry_new_protocol)
throw new RetryOnDifferentSystemException();
}
catch (WriteTimeoutException|WriteFailureException e)
}
catch (WriteTimeoutException|WriteFailureException|RetryOnDifferentSystemException e)
{
logger.trace("Failed replaying a batched mutation on Accord, will write a hint");
logger.trace("Failure was : {}", e.getMessage());
writeHintsForUndeliveredAccordTxns(hintedNodes);
}
catch (Exception e)
{
failure = Throwables.merge(failure, e);
}
try
{
for (int i = 0; i < replayHandlers.size(); i++)
{
if (logger.isTraceEnabled())
ReplayWriteResponseHandler<Mutation> handler = replayHandlers.get(i);
try
{
handler.get();
}
catch (WriteTimeoutException|WriteFailureException|RetryOnDifferentSystemException e)
{
logger.trace("Failed replaying a batched mutation to a node, will write a hint");
logger.trace("Failure was : {}", e.getMessage());
// writing hints for the rest to hints, starting from i
writeHintsForUndeliveredEndpoints(i, hintedNodes);
break;
}
// writing hints for the rest to hints, starting from i
writeHintsForUndeliveredEndpoints(i, hintedNodes);
return;
}
}
catch (Exception e)
{
logger.debug("Unexpected batchlog replay exception", e);
failure = Throwables.merge(failure, e);
}
if (failure != null)
throw Throwables.unchecked(failure);
}
private int addMutations(int version, List<ByteBuffer> serializedMutations) throws IOException
private static int addMutations(List<Mutation> unsplitMutations, long writtenAt, int version, List<ByteBuffer> serializedMutations) throws IOException
{
int ret = 0;
for (ByteBuffer serializedMutation : serializedMutations)
@ -402,7 +507,7 @@ public class BatchlogManager implements BatchlogManagerMBean
ret += serializedMutation.remaining();
try (DataInputBuffer in = new DataInputBuffer(serializedMutation, true))
{
addMutation(Mutation.serializer.deserialize(in, version));
addMutation(unsplitMutations, writtenAt, Mutation.serializer.deserialize(in, version));
}
}
@ -412,19 +517,41 @@ public class BatchlogManager implements BatchlogManagerMBean
// Remove CFs that have been truncated since. writtenAt and SystemTable#getTruncatedAt() both return millis.
// We don't abort the replay entirely b/c this can be considered a success (truncated is same as delivered then
// truncated.
private void addMutation(Mutation mutation)
private static void addMutation(List<Mutation> unsplitMutations, long writtenAt, Mutation mutation)
{
for (TableId tableId : mutation.getTableIds())
if (writtenAt <= SystemKeyspace.getTruncatedAt(tableId))
mutation = mutation.without(tableId);
if (!mutation.isEmpty())
mutations.add(mutation);
if (mutation != null)
unsplitMutations.add(mutation);
}
// Write the hint assuming that when it is replayed it will probably be replayed
// as an Accord transaction so no reason to record per endpoint hints for all the endpoints
// Hints will still have to split and re-route on replay
private void writeHintsForUndeliveredAccordTxns(Set<UUID> hintedNodes)
{
if (accordMutations == null)
return;
int gcgs = gcgs(accordMutations);
// expired
if (MILLISECONDS.toSeconds(writtenAt) + gcgs <= FBUtilities.nowInSeconds())
return;
for (Mutation m : accordMutations)
HintsService.instance.write(ImmutableList.of(RETRY_ON_DIFFERENT_SYSTEM_UUID), Hint.create(m, writtenAt));
hintedNodes.add(RETRY_ON_DIFFERENT_SYSTEM_UUID);
}
private void writeHintsForUndeliveredEndpoints(int startFrom, Set<UUID> hintedNodes)
{
int gcgs = gcgs(mutations);
if (normalMutations == null)
return;
int gcgs = gcgs(normalMutations);
// expired
if (MILLISECONDS.toSeconds(writtenAt) + gcgs <= FBUtilities.nowInSeconds())
@ -434,7 +561,7 @@ public class BatchlogManager implements BatchlogManagerMBean
for (int i = startFrom; i < replayHandlers.size(); i++)
{
ReplayWriteResponseHandler<Mutation> handler = replayHandlers.get(i);
Mutation undeliveredMutation = mutations.get(i);
Mutation undeliveredMutation = normalMutations.get(i);
if (handler != null)
{

3
src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
View File

@ -61,6 +61,7 @@ public enum CassandraRelevantProperties
AUTOCOMPACTION_ON_STARTUP_ENABLED("cassandra.autocompaction_on_startup_enabled", "true"),
AUTO_BOOTSTRAP("cassandra.auto_bootstrap"),
AUTO_REPAIR_FREQUENCY_SECONDS("cassandra.auto_repair_frequency_seconds", convertToString(TimeUnit.MINUTES.toSeconds(5))),
BATCHLOG_REPLAY_INTERVAL_MS("cassandra.batchlog.replay_interval_ms", "10000"),
BATCHLOG_REPLAY_TIMEOUT_IN_MS("cassandra.batchlog.replay_timeout_in_ms"),
BATCH_COMMIT_LOG_SYNC_INTERVAL("cassandra.batch_commitlog_sync_interval_millis", "1000"),
/**
@ -271,6 +272,8 @@ public enum CassandraRelevantProperties
*/
GOSSIP_SETTLE_POLL_SUCCESSES_REQUIRED("cassandra.gossip_settle_poll_success_required", "3"),
HINT_DISPATCH_INTERVAL_MS("cassandra.hint_dispatch_interval_ms", "10000"),
IGNORED_SCHEMA_CHECK_ENDPOINTS("cassandra.skip_schema_check_for_endpoints"),
IGNORED_SCHEMA_CHECK_VERSIONS("cassandra.skip_schema_check_for_versions"),
IGNORE_CORRUPTED_SCHEMA_TABLES("cassandra.ignore_corrupted_schema_tables"),

6
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -80,6 +80,7 @@ import org.apache.cassandra.config.Config.CommitLogSync;
import org.apache.cassandra.config.Config.DiskAccessMode;
import org.apache.cassandra.config.Config.PaxosOnLinearizabilityViolation;
import org.apache.cassandra.config.Config.PaxosStatePurging;
import org.apache.cassandra.config.DurationSpec.IntMillisecondsBound;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.CommitLog;
@ -4047,6 +4048,11 @@ public class DatabaseDescriptor
return conf.hints_flush_period.toMilliseconds();
}
public static void setHintsFlushPeriodInMS(int milliseconds)
{
conf.hints_flush_period = new IntMillisecondsBound(milliseconds);
}
public static long getMaxHintsFileSize()
{
return conf.max_hints_file_size.toBytesInLong();

4
src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
View File

@ -354,7 +354,9 @@ public class BatchStatement implements CQLStatement.CompositeCQLStatement
ClientWarn.instance.warn(MessageFormatter.arrayFormat(LOGGED_BATCH_LOW_GCGS_WARNING, new Object[] { suffix, tablesWithZeroGcGs })
.getMessage());
}
return collector.toMutations(state);
// local is either executeWithoutConditions modifying a virtual table (doesn't support txns) or executeLocal
// which is called by test or internal things that are bypassing distributed system modification/checks
return collector.toMutations(state, local);
}
/**

16
src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java
View File

@ -137,14 +137,14 @@ final class BatchUpdatesCollector implements UpdatesCollector
* @return a collection containing all the mutations.
*/
@Override
public List<IMutation> toMutations(ClientState state)
public List<IMutation> toMutations(ClientState state, boolean allowPotentialTxnConflicts)
{
List<IMutation> ms = new ArrayList<>();
for (Map<ByteBuffer, IMutationBuilder> ksMap : mutationBuilders.values())
{
for (IMutationBuilder builder : ksMap.values())
{
IMutation mutation = builder.build();
IMutation mutation = builder.build(allowPotentialTxnConflicts);
mutation.validateIndexedColumns(state);
mutation.validateSize(MessagingService.current_version, CommitLogSegment.ENTRY_OVERHEAD_SIZE);
ms.add(mutation);
@ -182,7 +182,7 @@ final class BatchUpdatesCollector implements UpdatesCollector
/**
* Build the immutable mutation
*/
IMutation build();
IMutation build(boolean allowPotentialTxnConflicts);
/**
* Get the builder for the given tableId
@ -215,7 +215,7 @@ final class BatchUpdatesCollector implements UpdatesCollector
return this;
}
public Mutation build()
public Mutation build(boolean allowPotentialTxnConflicts)
{
ImmutableMap.Builder<TableId, PartitionUpdate> updates = new ImmutableMap.Builder<>();
for (Map.Entry<TableId, PartitionUpdate.Builder> updateEntry : modifications.entrySet())
@ -223,7 +223,7 @@ final class BatchUpdatesCollector implements UpdatesCollector
PartitionUpdate update = updateEntry.getValue().build();
updates.put(updateEntry.getKey(), update);
}
return new Mutation(keyspaceName, key, updates.build(), createdAt, false);
return new Mutation(keyspaceName, key, updates.build(), createdAt, allowPotentialTxnConflicts);
}
public PartitionUpdate.Builder get(TableId tableId)
@ -263,9 +263,9 @@ final class BatchUpdatesCollector implements UpdatesCollector
return mutationBuilder.add(builder);
}
public IMutation build()
public IMutation build(boolean allowPotentialTxnConflicts)
{
return new CounterMutation(mutationBuilder.build(), cl);
return new CounterMutation(mutationBuilder.build(allowPotentialTxnConflicts), cl);
}
public PartitionUpdate.Builder get(TableId id)
@ -297,7 +297,7 @@ final class BatchUpdatesCollector implements UpdatesCollector
}
@Override
public VirtualMutation build()
public VirtualMutation build(boolean allowPotentialTxnConflicts)
{
ImmutableMap.Builder<TableId, PartitionUpdate> updates = new ImmutableMap.Builder<>();
modifications.forEach((tableId, updateBuilder) -> updates.put(tableId, updateBuilder.build()));

5
src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
View File

@ -511,10 +511,11 @@ public class CQL3CasRequest implements CASRequest
private Update createUpdate(ClientState clientState, ConsistencyLevel commitConsistencyLevel)
{
// Potentially ignore commit consistency level if non-SERIAL write strategy is Accord
// Potentially ignore commit consistency level if TransactionalMode is full
// since it is safe to match what non-SERIAL writes do
commitConsistencyLevel = metadata.params.transactionalMode.commitCLForStrategy(commitConsistencyLevel);
return new TxnUpdate(createWriteFragments(clientState), createCondition(), commitConsistencyLevel);
// CAS requires using the new txn timestamp to correctly linearize some kinds of updates
return new TxnUpdate(createWriteFragments(clientState), createCondition(), commitConsistencyLevel, false);
}
private TxnCondition createCondition()

10
src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
View File

@ -44,10 +44,9 @@ import org.apache.cassandra.cql3.Attributes;
import org.apache.cassandra.cql3.CQLStatement;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.Ordering;
import org.apache.cassandra.cql3.terms.Constants;
import org.apache.cassandra.cql3.Operation;
import org.apache.cassandra.cql3.Operations;
import org.apache.cassandra.cql3.Ordering;
import org.apache.cassandra.cql3.QualifiedName;
import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
@ -66,6 +65,7 @@ import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
import org.apache.cassandra.cql3.selection.ResultSetBuilder;
import org.apache.cassandra.cql3.selection.Selection;
import org.apache.cassandra.cql3.selection.Selection.Selectors;
import org.apache.cassandra.cql3.terms.Constants;
import org.apache.cassandra.cql3.transactions.ReferenceOperation;
import org.apache.cassandra.db.CBuilder;
import org.apache.cassandra.db.Clustering;
@ -869,14 +869,16 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
{
SingleTableSinglePartitionUpdatesCollector collector = new SingleTableSinglePartitionUpdatesCollector(metadata, updatedColumns);
addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, requestTime, constructingAccordBaseUpdate);
return collector.toMutations(state);
// local means this is test or internal things that are bypassing distributed system modification/checks
return collector.toMutations(state, local);
}
else
{
HashMultiset<ByteBuffer> perPartitionKeyCounts = HashMultiset.create(keys);
SingleTableUpdatesCollector collector = new SingleTableUpdatesCollector(metadata, updatedColumns, perPartitionKeyCounts);
addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, requestTime, constructingAccordBaseUpdate);
return collector.toMutations(state);
// local means this is test or internal things that are bypassing distributed system modification/checks
return collector.toMutations(state, local);
}
}

8
src/java/org/apache/cassandra/cql3/statements/SingleTableSinglePartitionUpdatesCollector.java
View File

@ -78,16 +78,16 @@ final class SingleTableSinglePartitionUpdatesCollector implements UpdatesCollect
* Returns a collection containing all the mutations.
*/
@Override
public List<IMutation> toMutations(ClientState state)
public List<IMutation> toMutations(ClientState state, boolean allowPotentialTxnConflicts)
{
// it is possible that a modification statement does not create any mutations
// for example: DELETE FROM some_table WHERE part_key = 1 AND clust_key < 3 AND clust_key > 5
if (builder == null)
return Collections.emptyList();
return Collections.singletonList(createMutation(state, builder));
return Collections.singletonList(createMutation(state, builder, allowPotentialTxnConflicts));
}
private IMutation createMutation(ClientState state, PartitionUpdate.Builder builder)
private IMutation createMutation(ClientState state, PartitionUpdate.Builder builder, boolean allowPotentialTxnConflicts)
{
IMutation mutation;
@ -96,7 +96,7 @@ final class SingleTableSinglePartitionUpdatesCollector implements UpdatesCollect
else if (metadata.isCounter())
mutation = new CounterMutation(new Mutation(builder.build()), counterConsistencyLevel);
else
mutation = new Mutation(builder.build());
mutation = new Mutation(builder.build(), allowPotentialTxnConflicts);
mutation.validateIndexedColumns(state);
mutation.validateSize(MessagingService.current_version, CommitLogSegment.ENTRY_OVERHEAD_SIZE);

10
src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java
View File

@ -95,24 +95,24 @@ final class SingleTableUpdatesCollector implements UpdatesCollector
* @return a collection containing all the mutations.
*/
@Override
public List<IMutation> toMutations(ClientState state)
public List<IMutation> toMutations(ClientState state, boolean allowPotentialTxnConflicts)
{
if (puBuilders.size() == 1)
{
PartitionUpdate.Builder builder = puBuilders.values().iterator().next();
return Collections.singletonList(createMutation(state, builder));
return Collections.singletonList(createMutation(state, builder, allowPotentialTxnConflicts));
}
List<IMutation> ms = new ArrayList<>(puBuilders.size());
for (PartitionUpdate.Builder builder : puBuilders.values())
{
IMutation mutation = createMutation(state, builder);
IMutation mutation = createMutation(state, builder, allowPotentialTxnConflicts);
ms.add(mutation);
}
return ms;
}
private IMutation createMutation(ClientState state, PartitionUpdate.Builder builder)
private IMutation createMutation(ClientState state, PartitionUpdate.Builder builder, boolean allowPotentialTxnConflicts)
{
IMutation mutation;
@ -121,7 +121,7 @@ final class SingleTableUpdatesCollector implements UpdatesCollector
else if (metadata.isCounter())
mutation = new CounterMutation(new Mutation(builder.build()), counterConsistencyLevel);
else
mutation = new Mutation(builder.build());
mutation = new Mutation(builder.build(), allowPotentialTxnConflicts);
mutation.validateIndexedColumns(state);
mutation.validateSize(MessagingService.current_version, CommitLogSegment.ENTRY_OVERHEAD_SIZE);

2
src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java
View File

@ -310,7 +310,7 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
AccordUpdate createUpdate(ClientState state, QueryOptions options, Map<TxnDataName, NamedSelect> autoReads, Consumer<Key> keyConsumer)
{
return new TxnUpdate(createWriteFragments(state, options, autoReads, keyConsumer), createCondition(options), null);
return new TxnUpdate(createWriteFragments(state, options, autoReads, keyConsumer), createCondition(options), null, false);
}
Keys toKeys(SortedSet<Key> keySet)

2
src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java
View File

@ -30,5 +30,5 @@ import org.apache.cassandra.service.ClientState;
public interface UpdatesCollector
{
PartitionUpdate.Builder getPartitionUpdateBuilder(TableMetadata metadata, DecoratedKey dk, ConsistencyLevel consistency);
List<IMutation> toMutations(ClientState state);
List<IMutation> toMutations(ClientState state, boolean allowPotentialTxnConflicts);
}

10
src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java
View File

@ -62,6 +62,7 @@ import org.apache.cassandra.schema.MemtableParams;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableParams;
import org.apache.cassandra.schema.TableParams.Option;
import org.apache.cassandra.schema.UserFunctions;
import org.apache.cassandra.schema.ViewMetadata;
import org.apache.cassandra.schema.Views;
@ -594,7 +595,10 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
boolean modeChange = prev.transactionalMode != next.transactionalMode;
boolean wasMigrating = prev.transactionalMigrationFrom.isMigrating();
boolean forceMigrationChange = prev.transactionalMigrationFrom != next.transactionalMigrationFrom;
boolean explicitlySetMigrationFrom = attrs.hasOption(Option.TRANSACTIONAL_MIGRATION_FROM);
// set table to migrating
TransactionalMigrationFromMode newMigrateFrom = TransactionalMigrationFromMode.fromMode(prev.transactionalMode, next.transactionalMode);
boolean forceMigrationChange = modeChange && explicitlySetMigrationFrom && next.transactionalMigrationFrom != newMigrateFrom;
if (modeChange && next.transactionalMode.accordIsEnabled && !DatabaseDescriptor.getAccordTransactionsEnabled())
throw ire(format("Cannot change transactional mode to %s for %s.%s with accord_transactions_enabled set to false",
@ -617,9 +621,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
prev.transactionalMode, next.transactionalMode,
keyspaceName, tableName));
// set table to migrating
TransactionalMigrationFromMode migrateFrom = TransactionalMigrationFromMode.fromMode(prev.transactionalMode, next.transactionalMode);
return next.unbuild().transactionalMigrationFrom(migrateFrom).build();
return next.unbuild().transactionalMigrationFrom(newMigrateFrom).build();
}

23
src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java
View File

@ -27,18 +27,23 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.exceptions.CoordinatorBehindException;
import org.apache.cassandra.exceptions.InvalidRoutingException;
import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.TCMMetrics;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.NoSpamLogger;
import static org.apache.cassandra.exceptions.RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM;
public abstract class AbstractMutationVerbHandler<T extends IMutation> implements IVerbHandler<T>
{
private static final Logger logger = LoggerFactory.getLogger(AbstractMutationVerbHandler.class);
@ -57,7 +62,17 @@ public abstract class AbstractMutationVerbHandler<T extends IMutation> implement
metadata = checkTokenOwnership(metadata, message, respondTo);
metadata = checkSchemaVersion(metadata, message, respondTo);
}
applyMutation(message, respondTo);
try
{
applyMutation(message, respondTo);
}
catch (RetryOnDifferentSystemException e)
{
logger.debug("Responding with retry on different system");
MessagingService.instance().respondWithFailure(RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM, message);
Tracing.trace("Payload application resulted in RetryOnDifferentSysten");
}
}
abstract void applyMutation(Message<T> message, InetAddressAndPort respondToAddress);
@ -85,9 +100,7 @@ public abstract class AbstractMutationVerbHandler<T extends IMutation> implement
}
}
// Mutations may intentionally be sent against an older Epoch so out of range checking doesn't work
// and could cause data to not end up where it needs to be for future operations
if (!message.payload.allowsOutOfRangeMutations() && !forToken.get().containsSelf())
if (!forToken.get().containsSelf())
{
StorageService.instance.incOutOfRangeOperationCount();
Keyspace.open(message.payload.getKeyspaceName()).metric.outOfRangeTokenWrites.inc();
@ -95,7 +108,7 @@ public abstract class AbstractMutationVerbHandler<T extends IMutation> implement
throw InvalidRoutingException.forWrite(respondTo, key.getToken(), metadata.epoch, message.payload);
}
if (!message.payload.allowsOutOfRangeMutations() && forToken.lastModified().isAfter(message.epoch()))
if (forToken.lastModified().isAfter(message.epoch()))
{
TCMMetrics.instance.coordinatorBehindPlacements.mark();
throw new CoordinatorBehindException(String.format("Routing is correct, but coordinator needs to catch-up at least to epoch %s to maintain consistency. Current coordinator epoch is %s",

4
src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java
View File

@ -86,10 +86,10 @@ public class CassandraKeyspaceWriteHandler implements KeyspaceWriteHandler
Set<TableId> ids = new HashSet<>();
for (PartitionUpdate update : mutation.getPartitionUpdates())
{
if (update.metadata().params.memtable.factory().writesShouldSkipCommitLog())
if (!update.metadata().params.memtable.factory().writesShouldSkipCommitLog())
ids.add(update.metadata().id);
}
mutation = mutation.without(ids);
mutation = mutation.filter(ids::contains);
}
}
// Note: It may be a good idea to precalculate none/all for the set of all tables in the keyspace,

52
src/java/org/apache/cassandra/db/CounterMutation.java
View File

@ -18,10 +18,15 @@
package org.apache.cassandra.db;
import java.io.IOException;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.function.Predicate;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import com.google.common.base.Function;
import com.google.common.base.Objects;
@ -31,11 +36,16 @@ import com.google.common.collect.PeekingIterator;
import com.google.common.util.concurrent.Striped;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.ColumnData;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.rows.UnfilteredRowIterators;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
@ -44,10 +54,11 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.CounterId;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.btree.BTreeSet;
import static java.util.concurrent.TimeUnit.*;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.net.MessagingService.VERSION_40;
import static org.apache.cassandra.net.MessagingService.VERSION_50;
import static org.apache.cassandra.net.MessagingService.VERSION_51;
@ -83,6 +94,12 @@ public class CounterMutation implements IMutation
return mutation.getPartitionUpdates();
}
@Override
public boolean hasUpdateForTable(TableId tableId)
{
return mutation.hasUpdateForTable(tableId);
}
@Override
public Supplier<Mutation> hintOnFailure()
{
@ -156,6 +173,27 @@ public class CounterMutation implements IMutation
applyCounterMutation();
}
@Override
public @Nullable CounterMutation filter(Predicate<TableId> test)
{
Mutation m = mutation.filter(test);
if (m == null)
return null;
if (m == mutation)
return this;
return new CounterMutation(m, consistency);
}
/*
* Accord currently doesn't support interoperability with counters so no Accord transactions should read them
* anyways and it's safe to continue non-transactionally updating them
*/
@Override
public boolean allowsPotentialTransactionConflicts()
{
return true;
}
private void grabCounterLocks(Keyspace keyspace, List<Lock> locks) throws WriteTimeoutException
{
long startTime = nanoTime();

20
src/java/org/apache/cassandra/db/IMutation.java
View File

@ -19,7 +19,9 @@ package org.apache.cassandra.db;
import java.util.Collection;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.partitions.PartitionUpdate;
@ -37,6 +39,7 @@ public interface IMutation
long getTimeout(TimeUnit unit);
String toString(boolean shallow);
Collection<PartitionUpdate> getPartitionUpdates();
boolean hasUpdateForTable(TableId tableId);
Supplier<Mutation> hintOnFailure();
default void validateIndexedColumns(ClientState state)
@ -71,8 +74,23 @@ public interface IMutation
return size;
}
default boolean allowsOutOfRangeMutations()
/**
* True if this mutation is being applied by a transaction system or doesn't need to be
* and conflicts between this mutation and transactions systems that are managing all or part of this table
* should be assumed to be handled already (by either Paxos or Accord) and the mutation should be applied.
*
* This causes mutations against tables to fail if they are from a non-transaction sub-system such as mutations,
* logged and unlogged batches, hints, and read repair against tables that are being managed by a transaction system
* like Accord that can't safely read data that is written non-transactionally.
*
*/
default boolean allowsPotentialTransactionConflicts()
{
return false;
}
// Construct replacement mutation that is identical except it only includes updates for the specified tables
@Nullable IMutation filter(Predicate<TableId> predicate);
default void clearCachedSerializationsForRetry() {}
}

6
src/java/org/apache/cassandra/db/Keyspace.java
View File

@ -47,8 +47,8 @@ import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.metrics.KeyspaceMetrics;
import org.apache.cassandra.repair.KeyspaceRepairManager;
@ -58,14 +58,15 @@ import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaProvider;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import org.apache.cassandra.utils.concurrent.Promise;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
@ -545,6 +546,7 @@ public class Keyspace
}
try (WriteContext ctx = getWriteHandler().beginWrite(mutation, makeDurable))
{
ConsensusMigrationMutationHelper.validateSafeToExecuteNonTransactionally(mutation);
for (PartitionUpdate upd : mutation.getPartitionUpdates())
{
ColumnFamilyStore cfs = columnFamilyStores.get(upd.metadata().id);

139
src/java/org/apache/cassandra/db/Mutation.java
View File

@ -19,15 +19,17 @@ package org.apache.cassandra.db;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLongFieldUpdater;
import java.util.function.Predicate;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableCollection;
@ -54,6 +56,7 @@ import org.apache.cassandra.service.AbstractWriteResponseHandler;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.concurrent.Future;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.cassandra.net.MessagingService.VERSION_40;
import static org.apache.cassandra.net.MessagingService.VERSION_50;
import static org.apache.cassandra.net.MessagingService.VERSION_51;
@ -62,7 +65,8 @@ import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
public class Mutation implements IMutation, Supplier<Mutation>
{
public static final MutationSerializer serializer = new MutationSerializer();
public static final int ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG = 0x01;
public static final int ALLOW_POTENTIAL_TRANSACTION_CONFLICTS = 0x01;
// todo this is redundant
// when we remove it, also restore SerializationsTest.testMutationRead to not regenerate new Mutations each test
@ -84,33 +88,41 @@ public class Mutation implements IMutation, Supplier<Mutation>
private static final int SERIALIZATION_VERSION_COUNT = MessagingService.Version.values().length;
// Contains serialized representations of this mutation.
// Note: there is no functionality to clear/remove serialized instances, because a mutation must never
// be modified (e.g. calling add(PartitionUpdate)) when it's being serialized.
// Note: The cached serializations can be cleared when CoordinatorBehindException is being retried
private final Serialization[] cachedSerializations = new Serialization[SERIALIZATION_VERSION_COUNT];
/** @see CassandraRelevantProperties#CACHEABLE_MUTATION_SIZE_LIMIT */
private static final long CACHEABLE_MUTATION_SIZE_LIMIT = CassandraRelevantProperties.CACHEABLE_MUTATION_SIZE_LIMIT.getLong();
private boolean allowOutOfRangeMutations;
// Paxos & Accord manage conflicts directly and needs to apply mutations to tables/ranges
// that are only safe to write to from a transaction system.
// Don't refuse to apply this mutation because it should go through a transaction system
// because it is being applied by one or in a context where transaction conflicts don't occur
private boolean allowPotentialTransactionConflicts;
public Mutation(PartitionUpdate update)
{
this(update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now(), update.metadata().params.cdc);
this(update, false);
}
public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, PartitionUpdate> modifications, long approxCreatedAtNanos, boolean allowOutOfRangeMutations)
public Mutation(PartitionUpdate update, boolean allowPotentialTransactionConflicts)
{
this(keyspaceName, key, modifications, approxCreatedAtNanos, cdcEnabled(modifications.values()), allowOutOfRangeMutations);
this(update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now(), update.metadata().params.cdc, allowPotentialTransactionConflicts);
}
public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, PartitionUpdate> modifications, long approxCreatedAtNanos, boolean cdcEnabled, boolean allowOutOfRangeMutations)
public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, PartitionUpdate> modifications, long approxCreatedAtNanos, boolean allowPotentialTransactionConflicts)
{
this(keyspaceName, key, modifications, approxCreatedAtNanos, cdcEnabled(modifications.values()), allowPotentialTransactionConflicts);
}
public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, PartitionUpdate> modifications, long approxCreatedAtNanos, boolean cdcEnabled, boolean allowPotentialTransactionConflicts)
{
this.keyspaceName = keyspaceName;
this.key = key;
this.modifications = modifications;
this.cdcEnabled = cdcEnabled;
this.approxCreatedAtNanos = approxCreatedAtNanos;
this.allowOutOfRangeMutations = allowOutOfRangeMutations;
this.allowPotentialTransactionConflicts = allowPotentialTransactionConflicts;
}
private static boolean cdcEnabled(Iterable<PartitionUpdate> modifications)
@ -121,26 +133,35 @@ public class Mutation implements IMutation, Supplier<Mutation>
return cdc;
}
public Mutation without(Set<TableId> tableIds)
@Override
public @Nullable Mutation filter(Predicate<TableId> predicate)
{
if (tableIds.isEmpty())
boolean allMatch = true;
boolean noneMatch = true;
for (TableId tableId : modifications.keySet())
{
boolean test = predicate.test(tableId);
allMatch &= test;
noneMatch &= !test;
}
if (allMatch)
return this;
if (noneMatch)
return null;
ImmutableMap.Builder<TableId, PartitionUpdate> builder = new ImmutableMap.Builder<>();
for (Map.Entry<TableId, PartitionUpdate> update : modifications.entrySet())
{
if (!tableIds.contains(update.getKey()))
{
if (predicate.test(update.getKey()))
builder.put(update);
}
}
return new Mutation(keyspaceName, key, builder.build(), approxCreatedAtNanos, allowOutOfRangeMutations);
Map<TableId, PartitionUpdate> updates = builder.build();
checkState(!updates.isEmpty(), "Updates should not be empty");
return new Mutation(keyspaceName, key, builder.build(), approxCreatedAtNanos, allowPotentialTransactionConflicts);
}
public Mutation without(TableId tableId)
public @Nullable Mutation without(TableId tableId)
{
return without(Collections.singleton(tableId));
return filter(otherTableId -> !tableId.equals(otherTableId));
}
public String getKeyspaceName()
@ -168,6 +189,12 @@ public class Mutation implements IMutation, Supplier<Mutation>
return approxCreatedAtNanos;
}
@Override
public boolean hasUpdateForTable(TableId tableId)
{
return modifications.containsKey(tableId);
}
@Override
public Supplier<Mutation> hintOnFailure()
{
@ -215,18 +242,18 @@ public class Mutation implements IMutation, Supplier<Mutation>
{
assert !mutations.isEmpty();
if (mutations.size() == ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG)
if (mutations.size() == 1)
return mutations.iterator().next();
Set<TableId> updatedTables = new HashSet<>();
String ks = null;
DecoratedKey key = null;
Boolean allowOutOfRangeMutations = null;
Boolean allowPotentialTransactionConflicts = null;
for (Mutation mutation : mutations)
{
if (allowOutOfRangeMutations != null && allowOutOfRangeMutations != mutation.allowOutOfRangeMutations)
throw new IllegalArgumentException("Can't merge mutations with differing policies on allowing out of range mutations");
allowOutOfRangeMutations = mutation.allowOutOfRangeMutations;
if (allowPotentialTransactionConflicts != null && allowPotentialTransactionConflicts != mutation.allowPotentialTransactionConflicts)
throw new IllegalArgumentException("Can't merge mutations with differing policies on allowing potential transaction conflicts");
allowPotentialTransactionConflicts = mutation.allowPotentialTransactionConflicts;
updatedTables.addAll(mutation.modifications.keySet());
if (ks != null && !ks.equals(mutation.keyspaceName))
throw new IllegalArgumentException();
@ -250,10 +277,10 @@ public class Mutation implements IMutation, Supplier<Mutation>
if (updates.isEmpty())
continue;
modifications.put(table, updates.size() == ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG ? updates.get(0) : PartitionUpdate.merge(updates));
modifications.put(table, updates.size() == 1 ? updates.get(0) : PartitionUpdate.merge(updates));
updates.clear();
}
return new Mutation(ks, key, modifications.build(), approxTime.now(), allowOutOfRangeMutations);
return new Mutation(ks, key, modifications.build(), approxTime.now(), allowPotentialTransactionConflicts);
}
public Future<?> applyFuture()
@ -310,25 +337,26 @@ public class Mutation implements IMutation, Supplier<Mutation>
return cdcEnabled;
}
public Mutation allowOutOfRangeMutations()
public void allowPotentialTransactionConflicts()
{
allowOutOfRangeMutations = true;
return this;
allowPotentialTransactionConflicts = true;
Arrays.fill(cachedSerializations, null);
}
public boolean allowsOutOfRangeMutations()
@Override
public boolean allowsPotentialTransactionConflicts()
{
return allowOutOfRangeMutations;
return allowPotentialTransactionConflicts;
}
private static int allowsOutOfRangeMutationsFlag(boolean allowOutOfRangeMutations)
private static int allowPotentialTransactionConflictsFlag(boolean allowPotentialTransactionConflicts)
{
return allowOutOfRangeMutations ? ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG : 0;
return allowPotentialTransactionConflicts ? ALLOW_POTENTIAL_TRANSACTION_CONFLICTS : 0;
}
private static boolean allowsOutOfRangeMutations(int flags)
public static boolean allowPotentialTransactionConflicts(int flags)
{
return (flags & ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG) != 0;
return (flags & ALLOW_POTENTIAL_TRANSACTION_CONFLICTS) != 0;
}
public String toString()
@ -336,6 +364,12 @@ public class Mutation implements IMutation, Supplier<Mutation>
return toString(false);
}
@Override
public void clearCachedSerializationsForRetry()
{
Arrays.fill(cachedSerializations, null);
}
public String toString(boolean shallow)
{
StringBuilder buff = new StringBuilder("Mutation(");
@ -404,6 +438,13 @@ public class Mutation implements IMutation, Supplier<Mutation>
*/
public interface SimpleBuilder
{
/**
* Assume any potential transaction conflicts that might occur by applying this mutation are already
* being handled by the caller
* @return this builder
*/
public SimpleBuilder allowPotentialTransactionConflicts();
/**
* Sets the timestamp to use for the following additions to this builder or any derived (update or row) builder.
*
@ -517,7 +558,11 @@ public class Mutation implements IMutation, Supplier<Mutation>
Map<TableId, PartitionUpdate> modifications = mutation.modifications;
if (version >= VERSION_51)
out.write(allowsOutOfRangeMutationsFlag(mutation.allowsOutOfRangeMutations()));
{
int flags = 0;
flags |= allowPotentialTransactionConflictsFlag(mutation.allowPotentialTransactionConflicts);
out.write(flags);
}
/* serialize the modifications in the mutation */
int size = modifications.size();
@ -538,11 +583,11 @@ public class Mutation implements IMutation, Supplier<Mutation>
{
teeIn = new TeeDataInputPlus(in, dob, CACHEABLE_MUTATION_SIZE_LIMIT);
boolean allowsOutOfRangeMutations = false;
boolean allowPotentialTransactionConflicts = false;
if (version >= VERSION_51)
{
int flags = teeIn.readByte();
allowsOutOfRangeMutations = allowsOutOfRangeMutations(flags);
allowPotentialTransactionConflicts = allowPotentialTransactionConflicts(flags);
}
int size = teeIn.readUnsignedVInt32();
assert size > 0;
@ -550,7 +595,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
PartitionUpdate update = PartitionUpdate.serializer.deserialize(teeIn, version, flag);
if (size == 1)
{
m = new Mutation(update);
m = new Mutation(update, allowPotentialTransactionConflicts);
}
else
{
@ -563,7 +608,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
update = PartitionUpdate.serializer.deserialize(teeIn, version, flag);
modifications.put(update.metadata().id, update);
}
m = new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now(), allowsOutOfRangeMutations);
m = new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now(), allowPotentialTransactionConflicts);
}
//Only cache serializations that don't hit the limit
@ -642,7 +687,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
if (size == 0L)
{
if (version >= VERSION_51)
size += TypeSizes.sizeof((byte)ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG); // flags
size += TypeSizes.sizeof((byte)ALLOW_POTENTIAL_TRANSACTION_CONFLICTS); // flags
size += TypeSizes.sizeofUnsignedVInt(mutation.modifications.size());
for (PartitionUpdate partitionUpdate : mutation.modifications.values())
size += serializer.serializedSize(partitionUpdate, version);
@ -663,10 +708,18 @@ public class Mutation implements IMutation, Supplier<Mutation>
private final long approxCreatedAtNanos = approxTime.now();
private boolean empty = true;
private boolean allowPotentialTransactionConflicts;
public PartitionUpdateCollector(String keyspaceName, DecoratedKey key)
{
this(keyspaceName, key, false);
}
public PartitionUpdateCollector(String keyspaceName, DecoratedKey key, boolean allowPotentialTransactionConflicts)
{
this.keyspaceName = keyspaceName;
this.key = key;
this.allowPotentialTransactionConflicts = allowPotentialTransactionConflicts;
}
public PartitionUpdateCollector add(PartitionUpdate partitionUpdate)
@ -700,7 +753,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
public Mutation build()
{
return new Mutation(keyspaceName, key, modifications.build(), approxCreatedAtNanos, false);
return new Mutation(keyspaceName, key, modifications.build(), approxCreatedAtNanos, allowPotentialTransactionConflicts);
}
}
}

33
src/java/org/apache/cassandra/db/SimpleBuilders.java
View File

@ -18,21 +18,30 @@
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.db.marshal.ListType;
import org.apache.cassandra.db.marshal.MapType;
import org.apache.cassandra.db.marshal.SetType;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.BTreeRow;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CounterId;
import org.apache.cassandra.utils.FBUtilities;
@ -111,12 +120,20 @@ public abstract class SimpleBuilders
private final Map<TableId, PartitionUpdateBuilder> updateBuilders = new HashMap<>();
private boolean allowPotentialTransactionConflicts = false;
public MutationBuilder(String keyspaceName, DecoratedKey key)
{
this.keyspaceName = keyspaceName;
this.key = key;
}
public MutationBuilder allowPotentialTransactionConflicts()
{
allowPotentialTransactionConflicts = true;
return this;
}
public PartitionUpdate.SimpleBuilder update(TableMetadata metadata)
{
assert metadata.keyspace.equals(keyspaceName);
@ -145,9 +162,9 @@ public abstract class SimpleBuilders
assert !updateBuilders.isEmpty() : "Cannot create empty mutation";
if (updateBuilders.size() == 1)
return new Mutation(updateBuilders.values().iterator().next().build());
return new Mutation(updateBuilders.values().iterator().next().build(), allowPotentialTransactionConflicts);
Mutation.PartitionUpdateCollector mutationBuilder = new Mutation.PartitionUpdateCollector(keyspaceName, key);
Mutation.PartitionUpdateCollector mutationBuilder = new Mutation.PartitionUpdateCollector(keyspaceName, key, allowPotentialTransactionConflicts);
for (PartitionUpdateBuilder builder : updateBuilders.values())
mutationBuilder.add(builder.build());
return mutationBuilder.build();

23
src/java/org/apache/cassandra/db/virtual/VirtualMutation.java
View File

@ -19,7 +19,9 @@ package org.apache.cassandra.db.virtual;
import java.util.Collection;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableMap;
@ -107,6 +109,12 @@ public final class VirtualMutation implements IMutation
return modifications.values();
}
@Override
public boolean hasUpdateForTable(TableId tableId)
{
return modifications.containsKey(tableId);
}
@Override
public Supplier<Mutation> hintOnFailure()
{
@ -123,4 +131,19 @@ public final class VirtualMutation implements IMutation
{
// no-op
}
@Override
public @Nullable VirtualMutation filter(Predicate<TableId> test)
{
throw new UnsupportedOperationException();
}
/*
* Accord doesn't support reading/writing virtual tables yet so updating them non-transactionally is always safe
*/
@Override
public boolean allowsPotentialTransactionConflicts()
{
return true;
}
}

13
src/java/org/apache/cassandra/exceptions/RequestFailure.java
View File

@ -35,7 +35,10 @@ import static org.apache.cassandra.exceptions.ExceptionSerializer.nullableRemote
/**
* Allow inclusion of a serialized exception in failure response messages
* This continues to use the same verb as the old failure response (whether a message payload or parameter)
* and has a nullable failure field that may contain a serialized in later versions.
* and has a nullable failure field that may contain a serialized exception in later versions.
*
* It's important to note RequestFailure is not a singleton for each type, unlike RequestFailureReason,
* since it might include a stack trace so don't compare using identity.
*/
public class RequestFailure
{
@ -50,6 +53,7 @@ public class RequestFailure
public static final RequestFailure INDEX_NOT_AVAILABLE = new RequestFailure(RequestFailureReason.INDEX_NOT_AVAILABLE);
public static final RequestFailure COORDINATOR_BEHIND = new RequestFailure(RequestFailureReason.COORDINATOR_BEHIND);
public static final RequestFailure READ_TOO_MANY_INDEXES = new RequestFailure(RequestFailureReason.READ_TOO_MANY_INDEXES);
public static final RequestFailure RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM = new RequestFailure(RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM);
static
{
@ -114,6 +118,12 @@ public class RequestFailure
if (t instanceof InvalidRoutingException)
return INVALID_ROUTING;
if (t instanceof RetryOnDifferentSystemException)
return RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM;
if (t instanceof CoordinatorBehindException)
return COORDINATOR_BEHIND;
return UNKNOWN;
}
@ -133,6 +143,7 @@ public class RequestFailure
case INDEX_NOT_AVAILABLE: return INDEX_NOT_AVAILABLE;
case COORDINATOR_BEHIND: return COORDINATOR_BEHIND;
case READ_TOO_MANY_INDEXES: return READ_TOO_MANY_INDEXES;
case RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM: return RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM;
}
}

26
src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
View File

@ -34,21 +34,23 @@ import static org.apache.cassandra.net.MessagingService.VERSION_40;
public enum RequestFailureReason
{
UNKNOWN (0),
READ_TOO_MANY_TOMBSTONES (1),
TIMEOUT (2),
INCOMPATIBLE_SCHEMA (3),
READ_SIZE (4),
UNKNOWN (0),
READ_TOO_MANY_TOMBSTONES (1),
TIMEOUT (2),
INCOMPATIBLE_SCHEMA (3),
READ_SIZE (4),
// below reason is only logged, but it does not have associated exception
NODE_DOWN (5),
INDEX_NOT_AVAILABLE (6),
NODE_DOWN (5),
INDEX_NOT_AVAILABLE (6),
// below reason does not have an associated exception
READ_TOO_MANY_INDEXES (7),
NOT_CMS (8),
INVALID_ROUTING (9),
COORDINATOR_BEHIND (10),
READ_TOO_MANY_INDEXES (7),
NOT_CMS (8),
INVALID_ROUTING (9),
COORDINATOR_BEHIND (10),
// The following codes have been ported from an external fork, where they were offset explicitly to avoid conflicts.
INDEX_BUILD_IN_PROGRESS (503);
INDEX_BUILD_IN_PROGRESS (503),
RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM (504),
;
static
{

30
src/java/org/apache/cassandra/exceptions/RetryOnDifferentSystemException.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.exceptions;
/**
* Thrown when a non-transactional operation is attempted when the operation needs to be done transactionally (or vice versa)
* and it could interfere with operations performed transactionally or can't be applied by the chosen transaction system.
*
* The correct way to handle this is to forward the error the originator of the operation who can then retry it on
* the correct system.
*/
public class RetryOnDifferentSystemException extends RuntimeException
{
}

16
src/java/org/apache/cassandra/hints/HintDiagnostics.java
View File

@ -37,44 +37,44 @@ final class HintDiagnostics
{
if (isEnabled(HintEventType.DISPATCHER_CREATED))
service.publish(new HintEvent(HintEventType.DISPATCHER_CREATED, dispatcher,
dispatcher.hostId, dispatcher.address, null, null, null, null));
dispatcher.hostId, dispatcher.address, null, null, null, null, null));
}
static void dispatcherClosed(HintsDispatcher dispatcher)
{
if (isEnabled(HintEventType.DISPATCHER_CLOSED))
service.publish(new HintEvent(HintEventType.DISPATCHER_CLOSED, dispatcher,
dispatcher.hostId, dispatcher.address, null, null, null, null));
dispatcher.hostId, dispatcher.address, null, null, null, null, null));
}
static void dispatchPage(HintsDispatcher dispatcher)
{
if (isEnabled(HintEventType.DISPATCHER_PAGE))
service.publish(new HintEvent(HintEventType.DISPATCHER_PAGE, dispatcher,
dispatcher.hostId, dispatcher.address, null, null, null, null));
dispatcher.hostId, dispatcher.address, null, null, null, null, null));
}
static void abortRequested(HintsDispatcher dispatcher)
{
if (isEnabled(HintEventType.ABORT_REQUESTED))
service.publish(new HintEvent(HintEventType.ABORT_REQUESTED, dispatcher,
dispatcher.hostId, dispatcher.address, null, null, null, null));
dispatcher.hostId, dispatcher.address, null, null, null, null, null));
}
static void pageSuccessResult(HintsDispatcher dispatcher, long success, long failures, long timeouts)
static void pageSuccessResult(HintsDispatcher dispatcher, long success, long failures, long timeouts, long retryDifferentSystem)
{
if (isEnabled(HintEventType.DISPATCHER_HINT_RESULT))
service.publish(new HintEvent(HintEventType.DISPATCHER_HINT_RESULT, dispatcher,
dispatcher.hostId, dispatcher.address, HintResult.PAGE_SUCCESS,
success, failures, timeouts));
success, failures, timeouts, retryDifferentSystem));
}
static void pageFailureResult(HintsDispatcher dispatcher, long success, long failures, long timeouts)
static void pageFailureResult(HintsDispatcher dispatcher, long success, long failures, long timeouts, long retryDifferentSystem)
{
if (isEnabled(HintEventType.DISPATCHER_HINT_RESULT))
service.publish(new HintEvent(HintEventType.DISPATCHER_HINT_RESULT, dispatcher,
dispatcher.hostId, dispatcher.address, HintResult.PAGE_FAILURE,
success, failures, timeouts));
success, failures, timeouts, retryDifferentSystem));
}
private static boolean isEnabled(HintEventType type)

7
src/java/org/apache/cassandra/hints/HintEvent.java
View File

@ -64,10 +64,13 @@ final class HintEvent extends DiagnosticEvent
private final Long pageHintsFailed;
@Nullable
private final Long pageHintsTimeout;
@Nullable
private final Long pageHintsRetryDifferentSystem;
HintEvent(HintEventType type, HintsDispatcher dispatcher, UUID targetHostId, InetAddressAndPort targetAddress,
@Nullable HintResult dispatchResult, @Nullable Long pageHintsSuccessful,
@Nullable Long pageHintsFailed, @Nullable Long pageHintsTimeout)
@Nullable Long pageHintsFailed, @Nullable Long pageHintsTimeout,
@Nullable Long pageHintsRetryDifferentSystem)
{
this.type = type;
this.dispatcher = dispatcher;
@ -77,6 +80,7 @@ final class HintEvent extends DiagnosticEvent
this.pageHintsSuccessful = pageHintsSuccessful;
this.pageHintsFailed = pageHintsFailed;
this.pageHintsTimeout = pageHintsTimeout;
this.pageHintsRetryDifferentSystem = pageHintsRetryDifferentSystem;
}
public Enum<HintEventType> getType()
@ -96,6 +100,7 @@ final class HintEvent extends DiagnosticEvent
ret.put("hint.page.hints_succeeded", pageHintsSuccessful);
ret.put("hint.page.hints_failed", pageHintsFailed);
ret.put("hint.page.hints_timed_out", pageHintsTimeout);
ret.put("hint.page.hints_retry_different_system", pageHintsRetryDifferentSystem);
}
return ret;
}

14
src/java/org/apache/cassandra/hints/HintVerbHandler.java
View File

@ -24,6 +24,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
@ -34,6 +35,8 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeId;
import static org.apache.cassandra.exceptions.RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM;
/**
* Verb handler used both for hint dispatch and streaming.
*
@ -100,8 +103,15 @@ public final class HintVerbHandler implements IVerbHandler<HintMessage>
}
else
{
// the common path - the node is both the destination and a valid replica for the hint.
hint.applyFuture().addCallback(o -> respond(message), e -> logger.debug("Failed to apply hint", e));
try
{
// the common path - the node is both the destination and a valid replica for the hint.
hint.applyFuture().addCallback(o -> respond(message), e -> logger.debug("Failed to apply hint", e));
}
catch (RetryOnDifferentSystemException e)
{
MessagingService.instance().respondWithFailure(RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM, message);
}
}
}

9
src/java/org/apache/cassandra/hints/HintsBufferPool.java
View File

@ -21,6 +21,8 @@ import java.io.Closeable;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
@ -133,6 +135,13 @@ final class HintsBufferPool implements Closeable
return HintsBuffer.create(bufferSize);
}
@VisibleForTesting
public void clearUnsafe()
{
if (currentBuffer != null)
currentBuffer = currentBuffer.recycle();
}
public void close()
{
currentBuffer.free();

13
src/java/org/apache/cassandra/hints/HintsCatalog.java
View File

@ -20,7 +20,11 @@ package org.apache.cassandra.hints;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Stream;
import javax.annotation.Nullable;
@ -30,10 +34,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.NativeLibrary;
import org.apache.cassandra.utils.SyncUtil;
@ -110,6 +114,11 @@ final class HintsCatalog
return stores.get(hostId);
}
void deleteAllHintsUnsafe()
{
stores.values().forEach(HintsStore::deleteAllHintsUnsafe);
}
/**
* Delete all hints for all host ids.
*

15
src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java
View File

@ -26,9 +26,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BooleanSupplier;
import java.util.function.Predicate;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -39,10 +39,11 @@ import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.hints.HintsService.RETRY_ON_DIFFERENT_SYSTEM_UUID;
/**
* A multi-threaded (by default) executor for dispatching hints.
@ -273,7 +274,7 @@ final class HintsDispatchExecutor
logger.trace("Dispatching hints file {}", descriptor.hintsFileName);
InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
if (address != null)
if (address != null || hostId == RETRY_ON_DIFFERENT_SYSTEM_UUID)
return deliver(descriptor, address);
// address == null means the target no longer exist; find new home for each hint entry.
@ -281,12 +282,12 @@ final class HintsDispatchExecutor
return true;
}
private boolean deliver(HintsDescriptor descriptor, InetAddressAndPort address)
private boolean deliver(HintsDescriptor descriptor, @Nullable InetAddressAndPort address)
{
File file = descriptor.file(hintsDirectory);
InputPosition offset = store.getDispatchOffset(descriptor);
BooleanSupplier shouldAbort = () -> !isAlive.test(address) || isPaused.get();
BooleanSupplier shouldAbort = () -> (!hostId.equals(RETRY_ON_DIFFERENT_SYSTEM_UUID) && (address == null || !isAlive.test(address)) || isPaused.get());
try (HintsDispatcher dispatcher = HintsDispatcher.create(file, rateLimiter, address, descriptor.hostId, shouldAbort))
{
if (offset != null)
@ -298,7 +299,7 @@ final class HintsDispatchExecutor
{
store.delete(descriptor);
store.cleanUp(descriptor);
logger.info("Finished hinted handoff of file {} to endpoint {}: {}", descriptor.fileName(), address, hostId);
logger.info("Finished hinted handoff of file {} to destination {}: {}", descriptor.fileName(), dispatcher.destination(), hostId);
return true;
}
else
@ -322,7 +323,7 @@ final class HintsDispatchExecutor
{
store.markDispatchOffset(descriptor, dispatcher.dispatchPosition());
store.offerFirst(descriptor);
logger.info("Finished hinted handoff of file {} to endpoint {}: {}, partially", descriptor.fileName(), address, hostId);
logger.info("Finished hinted handoff of file {} to destination {}: {}, partially", descriptor.fileName(), dispatcher.destination(), hostId);
}
// for each hint in the hints file for a node that isn't part of the ring anymore, write RF hints for each replica

347
src/java/org/apache/cassandra/hints/HintsDispatcher.java
View File

@ -19,31 +19,64 @@ package org.apache.cassandra.hints;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask;
import org.apache.cassandra.concurrent.ImmediateExecutor;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.metrics.HintsServiceMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitMutation;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.MonotonicClock;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.concurrent.Condition;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.cassandra.hints.HintsDispatcher.Callback.Outcome.FAILURE;
import static org.apache.cassandra.hints.HintsDispatcher.Callback.Outcome.INTERRUPTED;
import static org.apache.cassandra.hints.HintsDispatcher.Callback.Outcome.RETRY_DIFFERENT_SYSTEM;
import static org.apache.cassandra.hints.HintsDispatcher.Callback.Outcome.SUCCESS;
import static org.apache.cassandra.hints.HintsDispatcher.Callback.Outcome.TIMEOUT;
import static org.apache.cassandra.hints.HintsService.RETRY_ON_DIFFERENT_SYSTEM_UUID;
import static org.apache.cassandra.metrics.HintsServiceMetrics.ACCORD_HINT_ENDPOINT;
import static org.apache.cassandra.metrics.HintsServiceMetrics.updateDelayMetrics;
import static org.apache.cassandra.net.Verb.HINT_REQ;
import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.retry_new_protocol;
import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition;
@ -56,19 +89,28 @@ import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeConditio
final class HintsDispatcher implements AutoCloseable
{
private static final Logger logger = LoggerFactory.getLogger(HintsDispatcher.class);
private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
private enum Action { CONTINUE, ABORT }
private final HintsReader reader;
final UUID hostId;
@Nullable
final InetAddressAndPort address;
private final int messagingVersion;
private final BooleanSupplier abortRequested;
private InputPosition currentPagePosition;
private HintsDispatcher(HintsReader reader, UUID hostId, InetAddressAndPort address, int messagingVersion, BooleanSupplier abortRequested)
// Hints from the batch log that were attempted on Accord don't have a list of hosts that need hinting
// since Accord doesn't expose that on failure. If Accord no longer manages the range for this hint then we need
// to send the hint to all replicas after the page succeeds
private final Queue<Hint> hintsNeedingRehinting = new LinkedList<>();
private HintsDispatcher(HintsReader reader, UUID hostId, @Nullable InetAddressAndPort address, int messagingVersion, BooleanSupplier abortRequested)
{
checkArgument(address != null ^ hostId.equals(RETRY_ON_DIFFERENT_SYSTEM_UUID), "address must be nonnull or hostId must be " + RETRY_ON_DIFFERENT_SYSTEM_UUID);
currentPagePosition = null;
this.reader = reader;
@ -78,9 +120,9 @@ final class HintsDispatcher implements AutoCloseable
this.abortRequested = abortRequested;
}
static HintsDispatcher create(File file, RateLimiter rateLimiter, InetAddressAndPort address, UUID hostId, BooleanSupplier abortRequested)
static HintsDispatcher create(File file, RateLimiter rateLimiter, @Nullable InetAddressAndPort address, UUID hostId, BooleanSupplier abortRequested)
{
int messagingVersion = MessagingService.instance().versions.get(address);
int messagingVersion = address == null ? MessagingService.current_version : MessagingService.instance().versions.get(address);
HintsDispatcher dispatcher = new HintsDispatcher(HintsReader.open(file, rateLimiter), hostId, address, messagingVersion, abortRequested);
HintDiagnostics.dispatcherCreated(dispatcher);
return dispatcher;
@ -92,6 +134,11 @@ final class HintsDispatcher implements AutoCloseable
reader.close();
}
String destination()
{
return address == null ? "RETRY_ON_DIFFERENT_SYSTEM" : address.toString();
}
void seek(InputPosition position)
{
reader.seek(position);
@ -130,7 +177,19 @@ final class HintsDispatcher implements AutoCloseable
private Action sendHintsAndAwait(HintsReader.Page page)
{
Collection<Callback> callbacks = new ArrayList<>();
try
{
return doSendHintsAndAwait(page, null);
}
finally
{
hintsNeedingRehinting.clear();
}
}
private Action doSendHintsAndAwait(HintsReader.Page page, @Nullable BitSet hintsFilter)
{
List<Callback> callbacks = new ArrayList<>();
/*
* If hints file messaging version matches the version of the target host, we'll use the optimised path -
@ -138,50 +197,135 @@ final class HintsDispatcher implements AutoCloseable
*
* If that is not the case, we'll need to perform conversion to a newer (or an older) format, and decoding the hint
* is an unavoidable intermediate step.
*
* If these hints are from the batchlog and were originally attempted on Accord then
* we also need to decode so we can route the Hint contents appropriately.
*
* If filtering of hints is requested, because this is retrying a page that had some retry on different system
* errors, then also don't go down the sendEncodedHints path since it won't re-route the mutation and will trigger
* the same retry on different system error.
*/
Action action = reader.descriptor().messagingVersion() == messagingVersion
? sendHints(page.buffersIterator(), callbacks, this::sendEncodedHint)
: sendHints(page.hintsIterator(), callbacks, this::sendHint);
boolean isBatchLogHints = hostId.equals(RETRY_ON_DIFFERENT_SYSTEM_UUID);
boolean sendEncodedHints = reader.descriptor().messagingVersion() == messagingVersion && !isBatchLogHints && hintsFilter == null;
// If the hints filter is set then splitting the hints is needed and encoded hints can't do that
checkState(!sendEncodedHints || hintsFilter == null, "Should not send encoded hints if hints filter is set");
Action action = sendEncodedHints
? sendHints(page.buffersIterator(), null, callbacks, this::sendEncodedHint)
: sendHints(page.hintsIterator(), hintsFilter, callbacks, this::sendHint);
if (action == Action.ABORT)
return action;
long success = 0, failures = 0, timeouts = 0;
for (Callback cb : callbacks)
BitSet retryDifferentSystemHints = new BitSet(callbacks.size());
long success = 0, failures = 0, timeouts = 0, retryDifferentSystem = 0;
for (int i = 0; i < callbacks.size(); i++)
{
Callback cb = callbacks.get(i);
Callback.Outcome outcome = cb.await();
if (outcome == Callback.Outcome.SUCCESS) success++;
else if (outcome == Callback.Outcome.FAILURE) failures++;
else if (outcome == Callback.Outcome.TIMEOUT) timeouts++;
else if (outcome == RETRY_DIFFERENT_SYSTEM)
{
retryDifferentSystemHints.set(i);
retryDifferentSystem++;
}
else throw new IllegalStateException("Unhandled outcome: " + outcome);
}
updateMetrics(success, failures, timeouts);
updateMetrics(success, failures, timeouts, retryDifferentSystem);
if (failures > 0 || timeouts > 0)
// If the only errors were retryDifferentSystem and we aren't already filtering the hints then retry
// immediately otherwise we will repeat the page later including any successful hints we may have already delivered
// Hints for the batch log can hit RETRY_DIFFERENT_SYSTEM but don't need to be retried here and it could result
// in the same hint ending up in hintsNeedingRehinting twice
boolean failedRetryDifferentSystem = false;
if (retryDifferentSystem > 0 && failures < 1 && timeouts < 1 && hintsFilter == null && !isBatchLogHints)
{
HintDiagnostics.pageFailureResult(this, success, failures, timeouts);
reader.seek(currentPagePosition);
Action retryResult = doSendHintsAndAwait(page, retryDifferentSystemHints);
if (retryResult != Action.CONTINUE)
failedRetryDifferentSystem = true;
}
if (failures > 0 || timeouts > 0 || failedRetryDifferentSystem)
{
HintDiagnostics.pageFailureResult(this, success, failures, timeouts, retryDifferentSystem);
return Action.ABORT;
}
else
{
HintDiagnostics.pageSuccessResult(this, success, failures, timeouts);
HintDiagnostics.pageSuccessResult(this, success, failures, timeouts, retryDifferentSystem);
rehintHintsNeedingRehinting();
return Action.CONTINUE;
}
}
private void updateMetrics(long success, long failures, long timeouts)
private void rehintHintsNeedingRehinting()
{
ClusterMetadata cm = ClusterMetadata.current();
Hint hint;
while ((hint = hintsNeedingRehinting.poll()) != null)
{
HintsService.instance.writeForAllReplicas(hint);
Mutation mutation = hint.mutation;
// Also may need to apply locally because it's possible this is from the batchlog
// and we never applied it locally
// TODO (review): Additional error handling necessary? Hints are lossy
DataPlacement dataPlacement = cm.placements.get(cm.schema.getKeyspace(mutation.getKeyspaceName()).getMetadata().params.replication);
VersionedEndpoints.ForToken forToken = dataPlacement.writes.forToken(mutation.key().getToken());
Replica self = forToken.get().selfIfPresent();
if (self != null)
{
Stage.MUTATION.maybeExecuteImmediately(new RunnableDebuggableTask()
{
private final long approxCreationTimeNanos = MonotonicClock.Global.approxTime.now();
private volatile long approxStartTimeNanos;
@Override
public void run()
{
approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
mutation.apply();
}
@Override
public long creationTimeNanos()
{
return approxCreationTimeNanos;
}
@Override
public long startTimeNanos()
{
return approxStartTimeNanos;
}
@Override
public String description()
{
return "HintsService rehinting Accord txn";
}
});
}
}
}
private void updateMetrics(long success, long failures, long timeouts, long retryDifferentSystem)
{
HintsServiceMetrics.hintsSucceeded.mark(success);
HintsServiceMetrics.hintsFailed.mark(failures);
HintsServiceMetrics.hintsTimedOut.mark(timeouts);
HintsServiceMetrics.hintsRetryDifferentSystem.mark(retryDifferentSystem);
}
/*
* Sending hints in compatibility mode.
*/
private <T> Action sendHints(Iterator<T> hints, Collection<Callback> callbacks, Function<T, Callback> sendFunction)
private <T> Action sendHints(Iterator<T> hints, @Nullable BitSet hintsFilter, Collection<Callback> callbacks, Function<T, Callback> sendFunction)
{
int hintIndex = -1;
while (hints.hasNext())
{
if (abortRequested.getAsBoolean())
@ -189,19 +333,95 @@ final class HintsDispatcher implements AutoCloseable
HintDiagnostics.abortRequested(this);
return Action.ABORT;
}
callbacks.add(sendFunction.apply(hints.next()));
T hint = hints.next();
hintIndex++;
if (hintsFilter != null && !hintsFilter.get(hintIndex))
continue;
callbacks.add(sendFunction.apply(hint));
}
return Action.CONTINUE;
}
private Callback sendHint(Hint hint)
{
Callback callback = new Callback(hint.creationTime);
Message<?> message = Message.out(HINT_REQ, new HintMessage(hostId, hint));
MessagingService.instance().sendWithCallback(message, address, callback);
ClusterMetadata cm = ClusterMetadata.current();
SplitHint splitHint = splitHintIntoAccordAndNormal(cm, hint);
Mutation accordHintMutation = splitHint.accordMutation;
Dispatcher.RequestTime requestTime = null;
AsyncTxnResult accordTxnResult = null;
if (accordHintMutation != null)
{
requestTime = Dispatcher.RequestTime.forImmediateExecution();
accordTxnResult = accordHintMutation != null ? ConsensusMigrationMutationHelper.instance().mutateWithAccordAsync(cm, accordHintMutation, null, requestTime) : null;
}
Hint normalHint = splitHint.normalHint;
Callback callback = new Callback(address, hint.creationTime, requestTime, accordTxnResult);
if (normalHint != null)
{
// We had a hint that was supposed to be done on Accord for the batch log (otherwise address would be non-null),
// but Accord no longer manages that table/range and now we don't know which nodes (if any) are missing the Mutation.
// Convert them to per replica hints *after* all the hints in this page have been applied so we can be reasonably sure
// this page isn't going to be played again thus avoiding any futher amplification from the same hint being
// replayed and repeatedly converted to per replica hints
if (address == null)
{
checkState(hostId.equals(RETRY_ON_DIFFERENT_SYSTEM_UUID), "If there is no address to send the hint to then the host ID should be BATCHLOG_ACCORD_HINT_UUID");
callback.onResponse(null);
hintsNeedingRehinting.add(normalHint);
}
else
{
Message<?> message = Message.out(HINT_REQ, new HintMessage(hostId, normalHint));
MessagingService.instance().sendWithCallback(message, address, callback);
}
}
else
{
// Don't wait for a normal response that will never come since no hints were sent
callback.onResponse(null);
}
return callback;
}
/**
* Result of splitting a hint across Accord and non-transactional boundaries
*/
private class SplitHint
{
private final Mutation accordMutation;
private final Hint normalHint;
public SplitHint(Mutation accordMutation, Hint normalHint)
{
this.accordMutation = accordMutation;
this.normalHint = normalHint;
}
@Override
public String toString()
{
return "SplitHint{" +
"accordMutation=" + accordMutation +
", normalHint=" + normalHint +
'}';
}
}
private SplitHint splitHintIntoAccordAndNormal(ClusterMetadata cm, Hint hint)
{
SplitMutation<Mutation> splitMutation = ConsensusMigrationMutationHelper.instance().splitMutationIntoAccordAndNormal(hint.mutation, cm);
if (splitMutation.accordMutation == null)
return new SplitHint(null, hint);
if (splitMutation.normalMutation == null)
return new SplitHint(splitMutation.accordMutation, null);
Hint normalHint = Hint.create(splitMutation.normalMutation, hint.creationTime, splitMutation.normalMutation.smallestGCGS());
return new SplitHint(splitMutation.accordMutation, normalHint);
}
/*
* Sending hints in raw mode.
*/
@ -209,23 +429,41 @@ final class HintsDispatcher implements AutoCloseable
private Callback sendEncodedHint(ByteBuffer hint)
{
HintMessage.Encoded message = new HintMessage.Encoded(hostId, hint, messagingVersion);
Callback callback = new Callback(message.getHintCreationTime());
Callback callback = new Callback(address, message.getHintCreationTime());
MessagingService.instance().sendWithCallback(Message.out(HINT_REQ, message), address, callback);
return callback;
}
static final class Callback implements RequestCallback
static final class Callback implements RequestCallback, Runnable
{
enum Outcome { SUCCESS, TIMEOUT, FAILURE, INTERRUPTED }
enum Outcome { SUCCESS, TIMEOUT, FAILURE, INTERRUPTED, RETRY_DIFFERENT_SYSTEM }
private final long start = approxTime.now();
private final Condition condition = newOneTimeCondition();
private volatile Outcome outcome;
private Outcome normalOutcome;
private Outcome accordOutcome;
@Nullable
private final InetAddressAndPort to;
private final long hintCreationNanoTime;
@Nullable
private final Dispatcher.RequestTime requestTime;
private final AsyncTxnResult accordTxnResult;
private Callback(long hintCreationTimeMillisSinceEpoch)
private Callback(@Nonnull InetAddressAndPort to, long hintCreationTimeMillisSinceEpoch)
{
this(to, hintCreationTimeMillisSinceEpoch, null, null);
}
private Callback(@Nullable InetAddressAndPort to, long hintCreationTimeMillisSinceEpoch, Dispatcher.RequestTime requestTime, @Nullable AsyncTxnResult accordTxnResult)
{
this.to = to != null ? to : ACCORD_HINT_ENDPOINT;
this.hintCreationNanoTime = approxTime.translate().fromMillisSinceEpoch(hintCreationTimeMillisSinceEpoch);
this.requestTime = requestTime;
this.accordTxnResult = accordTxnResult;
if (accordTxnResult != null)
accordTxnResult.addListener(this, ImmediateExecutor.INSTANCE);
else
accordOutcome = SUCCESS;
}
Outcome await()
@ -240,8 +478,31 @@ final class HintsDispatcher implements AutoCloseable
logger.warn("Hint dispatch was interrupted", e);
return INTERRUPTED;
}
normalOutcome = timedOut ? TIMEOUT : normalOutcome;
return timedOut ? TIMEOUT : outcome;
return outcome();
}
private Outcome outcome()
{
checkState((normalOutcome != null && accordOutcome != null) || (normalOutcome != SUCCESS || accordOutcome != SUCCESS), "Outcome for both normal and accord hint delivery should be known");
if (normalOutcome == RETRY_DIFFERENT_SYSTEM || accordOutcome == RETRY_DIFFERENT_SYSTEM)
return RETRY_DIFFERENT_SYSTEM;
if (normalOutcome == TIMEOUT || accordOutcome == TIMEOUT)
return TIMEOUT;
if (normalOutcome == FAILURE || accordOutcome == FAILURE)
return FAILURE;
checkState(normalOutcome == SUCCESS && accordOutcome == SUCCESS, "Hint delivery should have been successful");
return SUCCESS;
}
private synchronized void maybeSignal()
{
if ((normalOutcome != null && accordOutcome != null) || normalOutcome == FAILURE || accordOutcome == FAILURE)
{
updateDelayMetrics(to, approxTime.now() - this.hintCreationNanoTime);
condition.signalAll();
}
}
@Override
@ -253,16 +514,40 @@ final class HintsDispatcher implements AutoCloseable
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failureMessage)
{
outcome = FAILURE;
condition.signalAll();
if (failureMessage.reason == RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM)
normalOutcome = RETRY_DIFFERENT_SYSTEM;
else
normalOutcome = FAILURE;
maybeSignal();
}
@Override
public void onResponse(Message msg)
{
updateDelayMetrics(msg.from(), approxTime.now() - this.hintCreationNanoTime);
outcome = SUCCESS;
condition.signalAll();
normalOutcome = SUCCESS;
maybeSignal();
}
@Override
public void run()
{
try
{
IAccordService accord = AccordService.instance();
TxnResult.Kind kind = accord.getTxnResult(accordTxnResult, true, null, requestTime).kind();
if (kind == retry_new_protocol)
accordOutcome = RETRY_DIFFERENT_SYSTEM;
else
accordOutcome = SUCCESS;
}
catch (Exception e)
{
accordOutcome = e instanceof WriteTimeoutException ? TIMEOUT : FAILURE;
String msg = "Accord hint delivery transaction failed";
if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime()))
logger.error(msg, e);
}
maybeSignal();
}
}
}

47
src/java/org/apache/cassandra/hints/HintsService.java
View File

@ -33,30 +33,32 @@ import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.ReplicaLayout;
import org.apache.cassandra.utils.concurrent.Future;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.EndpointsForToken;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.ReplicaLayout;
import org.apache.cassandra.metrics.HintedHandoffMetrics;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.MBeanWrapper;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static com.google.common.collect.Iterables.filter;
import static com.google.common.collect.Iterables.transform;
import static org.apache.cassandra.config.CassandraRelevantProperties.HINT_DISPATCH_INTERVAL_MS;
/**
* A singleton-ish wrapper over various hints components:
@ -70,12 +72,35 @@ import static com.google.common.collect.Iterables.transform;
*/
public final class HintsService implements HintsServiceMBean
{
private static final Logger logger = LoggerFactory.getLogger(HintsService.class);
// Dummy address to use for storing metrics for hints that will be retried on a different transaction system
// and aren't being sent to a specific node
public static final InetAddressAndPort RETRY_ON_DIFFERENT_SYSTEM_ADDRESS;
static
{
try
{
RETRY_ON_DIFFERENT_SYSTEM_ADDRESS = InetAddressAndPort.getByNameOverrideDefaults("0.0.0.0", 65535);
}
catch (UnknownHostException e)
{
throw new RuntimeException(e);
}
}
// Batch log replay may need to route some mutations to Accord which may fail and Hints are used for retry by the batch log.
// Write them to this endpoint which indicates that on replay hints will need to calculate the endpoints
// to deliver to since it's not really a per node hint, but part of a batch that needs replaying.
// This can also occur with regular mutations as well when some replicas return a retry error but quorum
// is reached so hinting is used to bring the other replicas up to date
public static final UUID RETRY_ON_DIFFERENT_SYSTEM_UUID = TimeUUID.atUnixMicrosWithLsbAsUUID(-1, -1);
public static HintsService instance = new HintsService();
public static final String MBEAN_NAME = "org.apache.cassandra.hints:type=HintsService";
private static final Logger logger = LoggerFactory.getLogger(HintsService.class);
private static final int MIN_BUFFER_SIZE = 32 << 20;
static final ImmutableMap<String, Object> EMPTY_PARAMS = ImmutableMap.of();
@ -226,7 +251,8 @@ public final class HintsService implements HintsServiceMBean
HintsDispatchTrigger trigger = new HintsDispatchTrigger(catalog, writeExecutor, dispatchExecutor, isDispatchPaused);
// triggering hint dispatch is now very cheap, so we can do it more often - every 10 seconds vs. every 10 minutes,
// previously; this reduces mean time to delivery, and positively affects batchlog delivery latencies, too
triggerDispatchFuture = ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(trigger, 10, 10, TimeUnit.SECONDS);
long hintDispatchIntervalMs = HINT_DISPATCH_INTERVAL_MS.getLong();
triggerDispatchFuture = ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(trigger, hintDispatchIntervalMs, hintDispatchIntervalMs, TimeUnit.MILLISECONDS);
}
public void pauseDispatch()
@ -322,6 +348,13 @@ public final class HintsService implements HintsServiceMBean
catalog.deleteAllHints();
}
@VisibleForTesting
public void deleteAllHintsUnsafe()
{
catalog.deleteAllHintsUnsafe();
bufferPool.clearUnsafe();
}
/**
* Deletes all hints for the provided destination. Doesn't make snapshots - should be used with care.
*

17
src/java/org/apache/cassandra/hints/HintsStore.java
View File

@ -18,6 +18,7 @@
package org.apache.cassandra.hints;
import java.io.IOException;
import java.nio.channels.ClosedChannelException;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
@ -174,6 +175,8 @@ final class HintsStore
boolean isLive()
{
if (hostId.equals(HintsService.RETRY_ON_DIFFERENT_SYSTEM_UUID))
return true;
InetAddressAndPort address = address();
return address != null && FailureDetector.instance.isAlive(address);
}
@ -193,6 +196,20 @@ final class HintsStore
dispatchDequeue.offerLast(descriptor);
}
void deleteAllHintsUnsafe()
{
try
{
closeWriter();
}
catch (FSWriteError e)
{
if (!(e.getCause() instanceof ClosedChannelException))
throw e;
}
deleteAllHints();
}
void deleteAllHints()
{
HintsDescriptor descriptor;

7
src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java
View File

@ -39,6 +39,8 @@ public class AccordClientRequestMetrics extends ClientRequestMetrics
// Number of times a query was rejected by Accord in TxnQuery due to a migration back to Paxos
public final Meter accordMigrationRejects;
public final Meter preempted;
public final Meter topologyMismatches;
public AccordClientRequestMetrics(String scope)
{
@ -48,6 +50,8 @@ public class AccordClientRequestMetrics extends ClientRequestMetrics
migrationSkippedReads = Metrics.meter(factory.createMetricName("MigrationSkippedReads"));
paxosKeyMigrations = Metrics.meter(factory.createMetricName("PaxosKeyMigrations"));
accordMigrationRejects = Metrics.meter(factory.createMetricName("AccordMigrationRejects"));
preempted = Metrics.meter(factory.createMetricName("Preempted"));
topologyMismatches = Metrics.meter(factory.createMetricName("TopologyMismatches"));
}
@Override
@ -58,5 +62,8 @@ public class AccordClientRequestMetrics extends ClientRequestMetrics
Metrics.remove(factory.createMetricName("MigrationSkippedReads"));
Metrics.remove(factory.createMetricName("PaxosKeyMigrations"));
Metrics.remove(factory.createMetricName("AccordMigrationRejects"));
Metrics.remove(factory.createMetricName("Preempted"));
Metrics.remove(factory.createMetricName("TopologyMismatches"));
}
}

6
src/java/org/apache/cassandra/metrics/ClientRequestMetrics.java
View File

@ -40,6 +40,8 @@ public class ClientRequestMetrics extends LatencyMetrics
public final Meter readSizeAborts;
public final Meter localRequests;
public final Meter remoteRequests;
public final Meter retryDifferentSystem;
public final Meter retryCoordinatorBehind;
public ClientRequestMetrics(String scope)
{
@ -53,6 +55,8 @@ public class ClientRequestMetrics extends LatencyMetrics
readSizeAborts = Metrics.meter(factory.createMetricName("ReadSizeAborts"));
localRequests = Metrics.meter(factory.createMetricName("LocalRequests"));
remoteRequests = Metrics.meter(factory.createMetricName("RemoteRequests"));
retryDifferentSystem = Metrics.meter(factory.createMetricName("RetryDifferentSystem"));
retryCoordinatorBehind = Metrics.meter(factory.createMetricName("RetryCoordinatorBehind"));
}
public void markAbort(Throwable cause)
@ -81,5 +85,7 @@ public class ClientRequestMetrics extends LatencyMetrics
Metrics.remove(factory.createMetricName("ReadSizeAborts"));
Metrics.remove(factory.createMetricName("LocalRequests"));
Metrics.remove(factory.createMetricName("RemoteRequests"));
Metrics.remove(factory.createMetricName("RetryDifferentSystem"));
Metrics.remove(factory.createMetricName("RetryCoordinatorBehind"));
}
}

25
src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.cassandra.metrics;
import java.net.UnknownHostException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -35,6 +37,22 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
public final class HintsServiceMetrics
{
public static final String TYPE_NAME = "HintsService";
// Hint metrics are by address and hints that are for Accord need an address
public static final InetAddressAndPort ACCORD_HINT_ENDPOINT;
static
{
try
{
ACCORD_HINT_ENDPOINT = InetAddressAndPort.getByNameOverrideDefaults("0.0.0.0", 0);
}
catch (UnknownHostException e)
{
throw new RuntimeException(e);
}
}
private static final Logger logger = LoggerFactory.getLogger(HintsServiceMetrics.class);
private static final MetricNameFactory factory = new DefaultNameFactory(TYPE_NAME);
@ -42,6 +60,8 @@ public final class HintsServiceMetrics
public static final Meter hintsSucceeded = Metrics.meter(factory.createMetricName("HintsSucceeded"));
public static final Meter hintsFailed = Metrics.meter(factory.createMetricName("HintsFailed"));
public static final Meter hintsTimedOut = Metrics.meter(factory.createMetricName("HintsTimedOut"));
public static final Meter hintsRetryDifferentSystem = Metrics.meter(factory.createMetricName("HintsRetryDifferentSystem"));
/** Histogram of all hint delivery delays */
private static final Histogram globalDelayHistogram = Metrics.histogram(factory.createMetricName("Hint_delays"), false);
@ -62,4 +82,9 @@ public final class HintsServiceMetrics
globalDelayHistogram.update(delay);
delayByEndpoint.get(endpoint).update(delay);
}
public static long getDelayCount(InetAddressAndPort endpoint)
{
return delayByEndpoint.get(endpoint).getCount();
}
}

2
src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
View File

@ -107,6 +107,7 @@ public class KeyspaceMetrics
public final LatencyMetrics rangeMigration;
public final Meter rangeMigrationUnexpectedFailures;
public final Meter rangeMigrationDependencyLimitFailures;
public final Meter mutationsRejectedOnWrongSystem;
/** Writes failed ideal consistency **/
public final Counter writeFailedIdealCL;
/** Ideal CL write latency metrics */
@ -257,6 +258,7 @@ public class KeyspaceMetrics
rangeMigration = createLatencyMetrics("RangeMigration");
rangeMigrationUnexpectedFailures = createKeyspaceMeter("RangeMigrationUnexpectedFailures");
rangeMigrationDependencyLimitFailures = createKeyspaceMeter("RangeMigratingDependencyLimitFailures");
mutationsRejectedOnWrongSystem = createKeyspaceMeter("MutationsRejectedOnWrongSystem");
writeFailedIdealCL = createKeyspaceCounter("WriteFailedIdealCL");
idealCLWriteLatency = createLatencyMetrics("IdealCLWrite");

2
src/java/org/apache/cassandra/metrics/TableMetrics.java
View File

@ -196,6 +196,7 @@ public class TableMetrics
public final LatencyMetrics accordRepair;
public final TableMeter accordRepairUnexpectedFailures;
public final TableMeter accordRepairDependencyLimitFailures;
public final TableMeter mutationsRejectedOnWrongSystem;
/** percent of the data that is repaired */
public final Gauge<Double> percentRepaired;
/** Reports the size of sstables in repaired, unrepaired, and any ongoing repair buckets */
@ -817,6 +818,7 @@ public class TableMetrics
accordRepair = createLatencyMetrics("AccordRepair", cfs.keyspace.metric.rangeMigration, GLOBAL_RANGE_MIGRATION_LATENCY);
accordRepairUnexpectedFailures = createTableMeter("AccordRepairUnexpectedFailures", cfs.keyspace.metric.rangeMigrationUnexpectedFailures);
accordRepairDependencyLimitFailures = createTableMeter("AccordRepairDependencyLimitFaiures", cfs.keyspace.metric.rangeMigrationDependencyLimitFailures);
mutationsRejectedOnWrongSystem = createTableMeter("MutationsRejectedOnWrongSystem", cfs.keyspace.metric.mutationsRejectedOnWrongSystem);
repairsStarted = createTableCounter("RepairJobsStarted");
repairsCompleted = createTableCounter("RepairJobsCompleted");

9
src/java/org/apache/cassandra/net/ResponseVerbHandler.java
View File

@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tracing.Tracing;
@ -33,6 +34,7 @@ import org.apache.cassandra.utils.FBUtilities;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.exceptions.RequestFailureReason.COORDINATOR_BEHIND;
import static org.apache.cassandra.exceptions.RequestFailureReason.INVALID_ROUTING;
import static org.apache.cassandra.exceptions.RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM;
import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
class ResponseVerbHandler implements IVerbHandler
@ -101,8 +103,11 @@ class ResponseVerbHandler implements IVerbHandler
// Gossip stage is single-threaded, so we may end up in a deadlock with after-commit hook
// that executes something on the gossip stage as well.
if (message.isFailureResponse() &&
(message.payload == COORDINATOR_BEHIND || message.payload == INVALID_ROUTING) &&
boolean isFailureResponse = message.isFailureResponse();
// RequestFailure is not a singleton so we need to extract and compare against the reason
RequestFailureReason reason = isFailureResponse ? ((RequestFailure)message.payload).reason : null;
if (isFailureResponse &&
(reason == COORDINATOR_BEHIND || reason == INVALID_ROUTING || reason == RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM) &&
// Gossip stage is single-threaded, so we may end up in a deadlock with after-commit hook
// that executes something on the gossip stage as well.
!Stage.GOSSIP.executor().inExecutor())

14
src/java/org/apache/cassandra/repair/RepairJob.java
View File

@ -39,6 +39,7 @@ import com.google.common.util.concurrent.FutureCallback;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.primitives.Ranges;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
@ -68,6 +69,7 @@ import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
import org.apache.cassandra.utils.concurrent.ImmediateFuture;
import static com.google.common.util.concurrent.Futures.getUnchecked;
import static org.apache.cassandra.config.DatabaseDescriptor.paxosRepairEnabled;
import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME;
import static org.apache.cassandra.service.paxos.Paxos.useV2;
@ -193,7 +195,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
return;
}
Future<Void> accordRepair;
Future<Ranges> accordRepair;
if (doAccordRepair)
{
accordRepair = paxosRepair.flatMap(unused -> {
@ -213,12 +215,12 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
if (session.accordOnly)
{
accordRepair.addCallback(new FutureCallback<Void>()
accordRepair.addCallback(new FutureCallback<>()
{
public void onSuccess(Void ignored)
public void onSuccess(Ranges barrieredRanges)
{
logger.info("{} {}.{} accord repair completed", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily);
trySuccess(new RepairResult(desc, Collections.emptyList(), ConsensusMigrationRepairResult.fromAccordOnlyRepair(repairStartingEpoch, session.excludedDeadNodes)));
trySuccess(new RepairResult(desc, Collections.emptyList(), ConsensusMigrationRepairResult.fromAccordOnlyRepair(repairStartingEpoch, barrieredRanges, session.excludedDeadNodes)));
}
public void onFailure(Throwable t)
@ -282,7 +284,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}
cfs.metric.repairsCompleted.inc();
logger.info("Completing repair with excludedDeadNodes {}", session.excludedDeadNodes);
trySuccess(new RepairResult(desc, stats, ConsensusMigrationRepairResult.fromRepair(repairStartingEpoch, doPaxosRepair, doAccordRepair, session.excludedDeadNodes)));
trySuccess(new RepairResult(desc, stats, ConsensusMigrationRepairResult.fromRepair(repairStartingEpoch, getUnchecked(accordRepair), doPaxosRepair, doAccordRepair, session.excludedDeadNodes)));
}
/**
@ -307,7 +309,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}, taskExecutor);
}
private Future<List<SyncTask>> createSyncTasks(Future<Void> accordRepair, Future<?> allSnapshotTasks, List<InetAddressAndPort> allEndpoints)
private Future<List<SyncTask>> createSyncTasks(Future<Ranges> accordRepair, Future<?> allSnapshotTasks, List<InetAddressAndPort> allEndpoints)
{
Future<List<TreeResponse>> treeResponses;
if (allSnapshotTasks != null)

29
src/java/org/apache/cassandra/schema/DistributedSchema.java
View File

@ -18,20 +18,8 @@
package org.apache.cassandra.schema;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import org.apache.cassandra.auth.AuthKeyspace;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.functions.UserFunction;
@ -47,6 +35,18 @@ import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.tracing.TraceKeyspace;
import org.apache.cassandra.utils.FBUtilities;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import static org.apache.cassandra.db.TypeSizes.sizeof;
/**
@ -130,6 +130,11 @@ public class DistributedSchema implements MetadataValue<DistributedSchema>
return keyspaces.get(keyspace).get();
}
public Optional<KeyspaceMetadata> maybeGetKeyspaceMetadata(String keyspace)
{
return keyspaces.get(keyspace);
}
public TableMetadata getTableMetadata(TableId id)
{
return tables.get(id);

67
src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
View File

@ -25,7 +25,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.slf4j.Logger;
@ -36,8 +35,10 @@ import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.IMutation;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.exceptions.CoordinatorBehindException;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.locator.EndpointsForToken;
@ -51,6 +52,7 @@ import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.concurrent.Condition;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static java.lang.Long.MAX_VALUE;
import static java.lang.Math.min;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
@ -59,6 +61,8 @@ import static java.util.stream.Collectors.toList;
import static org.apache.cassandra.config.DatabaseDescriptor.getCounterWriteRpcTimeout;
import static org.apache.cassandra.config.DatabaseDescriptor.getWriteRpcTimeout;
import static org.apache.cassandra.db.WriteType.COUNTER;
import static org.apache.cassandra.exceptions.RequestFailureReason.COORDINATOR_BEHIND;
import static org.apache.cassandra.exceptions.RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM;
import static org.apache.cassandra.locator.Replicas.countInOurDc;
import static org.apache.cassandra.schema.Schema.instance;
import static org.apache.cassandra.service.StorageProxy.WritePerformer;
@ -79,6 +83,10 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
private static final AtomicIntegerFieldUpdater<AbstractWriteResponseHandler> failuresUpdater =
AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "failures");
private volatile int failures = 0;
private static final AtomicIntegerFieldUpdater<AbstractWriteResponseHandler> alreadyHintedForRetryOnDifferentSystemUpdater =
AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "alreadyHintedForRetryOnDifferentSystem");
// Only write a hint to be applied as a transaction once
private volatile int alreadyHintedForRetryOnDifferentSystem = 0;
private volatile Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint;
private final Dispatcher.RequestTime requestTime;
private @Nullable final Supplier<Mutation> hintOnFailure;
@ -111,7 +119,7 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
this.requestTime = requestTime;
}
public void get() throws WriteTimeoutException, WriteFailureException
public void get() throws WriteTimeoutException, WriteFailureException, RetryOnDifferentSystemException
{
long timeoutNanos = currentTimeoutNanos();
@ -128,14 +136,41 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
if (!signaled)
throwTimeout();
if (blockFor() + failures > candidateReplicaCount())
int candidateReplicaCount = candidateReplicaCount();
if (blockFor() + failures > candidateReplicaCount)
{
if (RequestCallback.isTimeout(this.getFailureReasonByEndpointMap().keySet().stream()
.filter(this::waitingFor) // DatacenterWriteResponseHandler filters errors from remote DCs
.collect(Collectors.toMap(Function.identity(), this.getFailureReasonByEndpointMap()::get))))
// failures keeps incrementing, and this.failureReasonByEndpoint keeps getting new entries after signaling.
// Simpler to reason about what happened by copying this.failureReasonByEndpoint and then inferring
// failures from it
final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint = getFailureReasonByEndpointMap().keySet().stream()
.filter(this::waitingFor) // DatacenterWriteResponseHandler filters errors from remote DCs
.collect(toImmutableMap(Function.identity(), getFailureReasonByEndpointMap()::get));
final int failures = failureReasonByEndpoint.size();
if (RequestCallback.isTimeout(failureReasonByEndpoint))
throwTimeout();
throw new WriteFailureException(replicaPlan.consistencyLevel(), ackCount(), blockFor(), writeType, this.getFailureReasonByEndpointMap());
int transactionRetryErrors = 0;
int coordinatorBehindErrors = 0;
for (RequestFailureReason reason : failureReasonByEndpoint.values())
{
if (reason == RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM)
transactionRetryErrors++;
if (reason == COORDINATOR_BEHIND)
coordinatorBehindErrors++;
}
int totalRetriableFailures = transactionRetryErrors + coordinatorBehindErrors;
// Retrying might fix this
if (candidateReplicaCount - failures + totalRetriableFailures >= blockFor())
{
// Doesn't matter which we throw really but for clarity/metrics be specific
// Retrying on the correct system might make this write succeed
if (transactionRetryErrors > 0)
throw new RetryOnDifferentSystemException();
throw new CoordinatorBehindException("Write request failed due to coordinator behind");
}
throw new WriteFailureException(replicaPlan.consistencyLevel(), ackCount(), blockFor(), writeType, getFailureReasonByEndpointMap());
}
if (replicaPlan.stillAppliesTo(ClusterMetadata.current()))
@ -298,7 +333,7 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
logger.trace("Got failure from {}", from);
logger.trace("Got failure {} from {}", failure, from);
int n = waitingFor(from)
? failuresUpdater.incrementAndGet(this)
@ -317,8 +352,20 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
if (blockFor() + n > candidateReplicaCount())
signal();
if (hintOnFailure != null && StorageProxy.shouldHint(replicaPlan.lookup(from)) && requestTime.shouldSendHints())
StorageProxy.submitHint(hintOnFailure.get(), replicaPlan.lookup(from), null);
// If the failure was RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM then we only want to hint once
// and not for each instance since odds are it will be applied as a transaction at all replicas
if (hintOnFailure != null && StorageProxy.shouldHint(replicaPlan.lookup(from)) )
{
if (failure.reason == RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM)
{
if (alreadyHintedForRetryOnDifferentSystemUpdater.compareAndSet(this, 0, 1))
StorageProxy.submitHintForRetryOnDifferentSystem(hintOnFailure.get());
}
else
{
StorageProxy.submitHint(hintOnFailure.get(), replicaPlan.lookup(from), null);
}
}
}
@Override

15
src/java/org/apache/cassandra/service/BatchlogResponseHandler.java
View File

@ -29,9 +29,10 @@ import org.apache.cassandra.transport.Dispatcher;
public class BatchlogResponseHandler<T> extends AbstractWriteResponseHandler<T>
{
AbstractWriteResponseHandler<T> wrapped;
BatchlogCleanup cleanup;
final AbstractWriteResponseHandler<T> wrapped;
final BatchlogCleanup cleanup;
protected volatile int requiredBeforeFinish;
private static final AtomicIntegerFieldUpdater<BatchlogResponseHandler> requiredBeforeFinishUpdater
= AtomicIntegerFieldUpdater.newUpdater(BatchlogResponseHandler.class, "requiredBeforeFinish");
@ -104,6 +105,11 @@ public class BatchlogResponseHandler<T> extends AbstractWriteResponseHandler<T>
this.callback = callback;
}
public BatchlogCleanup(BatchlogCleanupCallback callback)
{
this.callback = callback;
}
public int decrement()
{
return mutationsWaitingForUpdater.decrementAndGet(this);
@ -114,6 +120,11 @@ public class BatchlogResponseHandler<T> extends AbstractWriteResponseHandler<T>
if (decrement() == 0)
callback.invoke();
}
public void setMutationsWaitingFor(int mutationsWaitingFor)
{
mutationsWaitingForUpdater.lazySet(this, mutationsWaitingFor);
}
}
public interface BatchlogCleanupCallback

523
src/java/org/apache/cassandra/service/StorageProxy.java
View File

@ -31,7 +31,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@ -42,14 +41,13 @@ import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheLoader;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.primitives.Keys;
import accord.primitives.Txn;
import accord.utils.Invariants;
import org.apache.cassandra.batchlog.Batch;
@ -58,7 +56,6 @@ import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
@ -88,6 +85,7 @@ import org.apache.cassandra.db.view.ViewUtils;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.CasWriteTimeoutException;
import org.apache.cassandra.exceptions.CasWriteUnknownResultException;
import org.apache.cassandra.exceptions.CoordinatorBehindException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.IsBootstrappingException;
import org.apache.cassandra.exceptions.OverloadedException;
@ -98,6 +96,7 @@ import org.apache.cassandra.exceptions.ReadTimeoutException;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.exceptions.RequestFailureException;
import org.apache.cassandra.exceptions.RequestTimeoutException;
import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
import org.apache.cassandra.exceptions.UnavailableException;
import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
@ -131,15 +130,14 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.txn.TxnCondition;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.service.accord.txn.TxnData;
import org.apache.cassandra.service.accord.txn.TxnQuery;
import org.apache.cassandra.service.accord.txn.TxnRead;
import org.apache.cassandra.service.accord.txn.TxnReferenceOperations;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.service.accord.txn.TxnUpdate;
import org.apache.cassandra.service.accord.txn.TxnWrite;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitConsumer;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitMutations;
import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
import org.apache.cassandra.service.paxos.Ballot;
import org.apache.cassandra.service.paxos.Commit;
@ -165,8 +163,10 @@ import org.apache.cassandra.utils.MBeanWrapper;
import org.apache.cassandra.utils.MonotonicClock;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static accord.primitives.Txn.Kind.EphemeralRead;
@ -197,6 +197,8 @@ import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.R
import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.casResult;
import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.serialReadResult;
import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.retry_new_protocol;
import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.mutateWithAccordAsync;
import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.splitMutationsIntoAccordAndNormal;
import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision;
import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL;
import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL;
@ -207,6 +209,8 @@ import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.LocalizeString.toUpperCaseLocalized;
import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
import static org.apache.cassandra.utils.Throwables.getStackTraceAsToString;
import static org.apache.cassandra.utils.Throwables.unchecked;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
import static org.apache.commons.lang3.StringUtils.join;
@ -1184,54 +1188,6 @@ public class StorageProxy implements StorageProxyMBean
}
}
private static ConsistencyLevel consistencyLevelForCommit(Collection<? extends IMutation> mutations, ConsistencyLevel consistencyLevel)
{
ConsistencyLevel result = null;
for (IMutation mutation : mutations)
{
for (TableId tableId : mutation.getTableIds())
{
TransactionalMode mode = Schema.instance.getTableMetadata(tableId).params.transactionalMode;
ConsistencyLevel commitCL = mode.commitCLForStrategy(consistencyLevel);
if (result == null || commitCL.compareTo(result) > 0)
result = commitCL;
}
}
return result;
}
private static boolean writesThroughAccord(List<? extends IMutation> mutations, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
boolean accordWrite = false;
boolean normalWrite = false;
for (int i=0,mi=mutations.size(); i<mi; i++)
{
IMutation mutation = mutations.get(i);
for (TableId tableId : mutation.getTableIds())
{
if (ConsensusRequestRouter.instance.shouldWriteThroughAccordAndMaybeMigrate(mutation.key(),
tableId,
consistencyLevel,
requestTime,
DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS),
true))
{
accordWrite = true;
}
else
{
normalWrite = true;
}
}
}
if (!accordWrite && !normalWrite)
throw new IllegalStateException("Mutation is neither a normal write nor an accord write");
if (accordWrite && normalWrite)
throw new InvalidRequestException("Cannot mix writes to accord and normal tables");
return accordWrite;
}
@SuppressWarnings("unchecked")
public static void mutateWithTriggers(List<? extends IMutation> mutations,
ConsistencyLevel consistencyLevel,
@ -1258,53 +1214,140 @@ public class StorageProxy implements StorageProxyMBean
}
}
Collection<Mutation> augmented = TriggerExecutor.instance.execute(mutations);
List<Mutation> augmented = TriggerExecutor.instance.execute(mutations);
String keyspaceName = mutations.iterator().next().getKeyspaceName();
boolean updatesView = Keyspace.open(mutations.iterator().next().getKeyspaceName())
boolean updatesView = Keyspace.open(keyspaceName)
.viewManager
.updatesAffectView(mutations, true);
long size = IMutation.dataSize(mutations);
long size = IMutation.dataSize(augmented != null ? augmented : mutations);
writeMetrics.mutationSize.update(size);
writeMetricsForLevel(consistencyLevel).mutationSize.update(size);
if (writesThroughAccord(mutations, consistencyLevel, requestTime))
{
Preconditions.checkState(!SchemaConstants.getSystemKeyspaces().contains(keyspaceName));
mutateWithAccord(augmented != null ? augmented : mutations, consistencyLevel, requestTime);
}
else if (augmented != null)
mutateAtomically(augmented, consistencyLevel, updatesView, requestTime);
if (augmented != null || mutateAtomically || updatesView)
mutateAtomically(augmented != null ? augmented : (List<Mutation>)mutations, consistencyLevel, updatesView, requestTime);
else
dispatchMutationsWithRetryOnDifferentSystem(mutations, consistencyLevel, requestTime);
}
public static void dispatchMutationsWithRetryOnDifferentSystem(List<? extends IMutation> mutations, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
while (true)
{
if (mutateAtomically || updatesView)
mutateAtomically((Collection<Mutation>) mutations, consistencyLevel, updatesView, requestTime);
else
mutate(mutations, consistencyLevel, requestTime);
ClusterMetadata cm = ClusterMetadata.current();
try
{
SplitMutations splitMutations = splitMutationsIntoAccordAndNormal(cm, (List<IMutation>)mutations);
List<? extends IMutation> accordMutations = splitMutations.accordMutations();
AsyncTxnResult accordResult = accordMutations != null ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null;
List<? extends IMutation> normalMutations = splitMutations.normalMutations();
Tracing.trace("Split mutations into Accord {} and normal {}", accordMutations, normalMutations);
Throwable failure = null;
try
{
if (normalMutations != null)
{
mutate(normalMutations, consistencyLevel, requestTime);
Tracing.trace("Successfully wrote normal mutations");
}
}
catch (RetryOnDifferentSystemException e)
{
writeMetrics.retryDifferentSystem.mark();
writeMetricsForLevel(consistencyLevel).retryDifferentSystem.mark();
logger.debug("Retrying mutations on different system because some mutations were misrouted");
Tracing.trace("Got {} from normal mutations, will retry", e);
continue;
}
catch (CoordinatorBehindException e)
{
writeMetrics.retryCoordinatorBehind.mark();
writeMetricsForLevel(consistencyLevel).retryCoordinatorBehind.mark();
mutations.forEach(IMutation::clearCachedSerializationsForRetry);
logger.debug("Retrying mutations now that coordinator has caught up to cluster metadata");
Tracing.trace("Got {} from normal mutations, will retry", e);
continue;
}
catch (Exception e)
{
failure = Throwables.merge(failure, e);
}
// Check if the Accord mutations succeeded asynchronously
try
{
if (accordResult != null)
{
IAccordService accord = AccordService.instance();
TxnResult.Kind kind = accord.getTxnResult(accordResult, true, consistencyLevel, requestTime).kind();
if (kind == retry_new_protocol)
continue;
Tracing.trace("Successfully wrote Accord mutations");
}
}
catch (Exception e)
{
failure = Throwables.merge(failure, e);
}
if (failure != null)
throw unchecked(failure);
}
catch (Exception t)
{
// Unexpected error so it would be helpful to have details
Tracing.trace("{}", getStackTraceAsToString(t));
throw t;
}
break;
}
}
private static void mutateWithAccord(Collection<? extends IMutation> mutations, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
private static ConsistencyLevel consistencyLevelForBatchLog(ConsistencyLevel consistencyLevel, boolean requireQuorumForRemove)
{
int fragmentIndex = 0;
List<TxnWrite.Fragment> fragments = new ArrayList<>(mutations.size());
List<PartitionKey> partitionKeys = new ArrayList<>(mutations.size());
for (IMutation mutation : mutations)
// If we are requiring quorum nodes for removal, we upgrade consistency level to QUORUM unless we already
// require ALL, or EACH_QUORUM. This is so that *at least* QUORUM nodes see the update.
ConsistencyLevel batchConsistencyLevel = requireQuorumForRemove
? ConsistencyLevel.QUORUM
: consistencyLevel;
switch (consistencyLevel)
{
for (PartitionUpdate update : mutation.getPartitionUpdates())
{
PartitionKey pk = PartitionKey.of(update);
partitionKeys.add(pk);
fragments.add(new TxnWrite.Fragment(PartitionKey.of(update), fragmentIndex++, update, TxnReferenceOperations.empty()));
}
case ALL:
case EACH_QUORUM:
batchConsistencyLevel = consistencyLevel;
}
return batchConsistencyLevel;
}
private static void doFallibleWriteWithMetricTracking(Runnable r, ConsistencyLevel consistencyLevel)
{
try
{
r.run();
}
catch (UnavailableException e)
{
writeMetrics.unavailables.mark();
writeMetricsForLevel(consistencyLevel).unavailables.mark();
Tracing.trace("Unavailable");
throw e;
}
catch (WriteTimeoutException e)
{
writeMetrics.timeouts.mark();
writeMetricsForLevel(consistencyLevel).timeouts.mark();
Tracing.trace("Write timeout; received {} of {} required replies", e.received, e.blockFor);
throw e;
}
catch (WriteFailureException e)
{
writeMetrics.failures.mark();
writeMetricsForLevel(consistencyLevel).failures.mark();
Tracing.trace("Write failure; received {} of {} required replies", e.received, e.blockFor);
throw e;
}
// Potentially ignore commit consistency level if the strategy specifies accord and not migration
ConsistencyLevel clForCommit = consistencyLevelForCommit(mutations, consistencyLevel);
TxnUpdate update = new TxnUpdate(fragments, TxnCondition.none(), clForCommit);
Txn.InMemory txn = new Txn.InMemory(Keys.of(partitionKeys), TxnRead.EMPTY, TxnQuery.EMPTY, update);
IAccordService accordService = AccordService.instance();
accordService.coordinate(txn, consistencyLevel, requestTime);
}
/**
@ -1314,92 +1357,171 @@ public class StorageProxy implements StorageProxyMBean
* After: remove the batchlog entry (after writing hints for the batch rows, if necessary).
*
* @param mutations the Mutations to be applied across the replicas
* @param consistency_level the consistency level for the operation
* @param consistencyLevel the consistency level for the operation
* @param requireQuorumForRemove at least a quorum of nodes will see update before deleting batchlog
* @param requestTime object holding times when request got enqueued and started execution
*/
public static void mutateAtomically(Collection<Mutation> mutations,
ConsistencyLevel consistency_level,
public static void mutateAtomically(List<Mutation> mutations,
ConsistencyLevel consistencyLevel,
boolean requireQuorumForRemove,
Dispatcher.RequestTime requestTime)
throws UnavailableException, OverloadedException, WriteTimeoutException
{
Tracing.trace("Determining replicas for atomic batch");
long startTime = nanoTime();
List<WriteResponseHandlerWrapper> wrappers = new ArrayList<>(mutations.size());
boolean attributeNonAccordLatency = true;
long nonAccordEndTime = -1;
if (mutations.stream().anyMatch(mutation -> Keyspace.open(mutation.getKeyspaceName()).getReplicationStrategy().hasTransientReplicas()))
throw new AssertionError("Logged batches are unsupported with transient replication");
try
{
// If we are requiring quorum nodes for removal, we upgrade consistency level to QUORUM unless we already
// require ALL, or EACH_QUORUM. This is so that *at least* QUORUM nodes see the update.
ConsistencyLevel batchConsistencyLevel = requireQuorumForRemove
? ConsistencyLevel.QUORUM
: consistency_level;
switch (consistency_level)
{
case ALL:
case EACH_QUORUM:
batchConsistencyLevel = consistency_level;
}
ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forBatchlogWrite(batchConsistencyLevel == ConsistencyLevel.ANY);
ConsistencyLevel batchConsistencyLevel = consistencyLevelForBatchLog(consistencyLevel, requireQuorumForRemove);
// This can't be updated for each iteration because cleanup has to go to the correct replicas which is where the batchlog is originally written
ReplicaPlan.ForWrite batchlogReplicaPlan = ReplicaPlans.forBatchlogWrite(ClusterMetadata.current(), batchConsistencyLevel == ConsistencyLevel.ANY);
final TimeUUID batchUUID = nextTimeUUID();
BatchlogCleanup cleanup = new BatchlogCleanup(mutations.size(),
() -> asyncRemoveFromBatchlog(replicaPlan, batchUUID, requestTime));
// add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
for (Mutation mutation : mutations)
boolean wroteToBatchLog = false;
while (true)
{
WriteResponseHandlerWrapper wrapper = wrapBatchResponseHandler(mutation,
consistency_level,
batchConsistencyLevel,
WriteType.BATCH,
cleanup,
requestTime);
// exit early if we can't fulfill the CL at this time.
wrappers.add(wrapper);
// In case we hit an error in before/during splitting
attributeNonAccordLatency = true;
ClusterMetadata cm = ClusterMetadata.current();
List<WriteResponseHandlerWrapper> wrappers = new ArrayList<>(mutations.size());
List<Mutation> accordMutations = new ArrayList<>(mutations.size());
BatchlogCleanup cleanup = new BatchlogCleanup(() -> asyncRemoveFromBatchlog(batchlogReplicaPlan, batchUUID, requestTime));
// add a handler for each mutation that will not be written on Accord - includes checking availability, but doesn't initiate any writes, yet
SplitConsumer<Mutation> splitConsumer = (accordMutation, normalMutation, originalMutations, mutationIndex) -> {
Mutation eitherMutation = normalMutation != null ? normalMutation : accordMutation;
Keyspace keyspace = Keyspace.open(eitherMutation.getKeyspaceName());
Token tk = eitherMutation.key().getToken();
if (accordMutation != null)
accordMutations.add(accordMutation);
if (normalMutation == null)
return;
// Always construct the replica plan to check availability
ReplicaPlan.ForWrite dataReplicaPlan = ReplicaPlans.forWrite(cm, keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal);
if (dataReplicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null)
writeMetrics.localRequests.mark();
else
writeMetrics.remoteRequests.mark();
WriteResponseHandlerWrapper wrapper = wrapBatchResponseHandler(normalMutation,
dataReplicaPlan,
batchConsistencyLevel,
WriteType.BATCH,
cleanup,
requestTime);
wrappers.add(wrapper);
};
splitMutationsIntoAccordAndNormal(cm, mutations, splitConsumer);
attributeNonAccordLatency = !wrappers.isEmpty();
cleanup.setMutationsWaitingFor(wrappers.size() + (accordMutations.isEmpty() ? 0 : 1));
Tracing.trace("Split batch into Accord {} and normal {}", accordMutations, wrappers);
// If the entire batch can execute on Accord then we can skip the batch log entirely
// Write to the batch log first in case it fails so we don't end up with Accord applying
// part of the batch independently
if (!wrappers.isEmpty() && !wroteToBatchLog)
{
// write to the batchlog, including writes that will be routed to Accord to preserve the behavior
// of the batch log where if part of a batch is visible then eventually the entire batch is visible.
// If the Accord routed mutations depend on the Accord txn succeeding then it is no longer consistent
// with the mutations delivered by the batch log since an unacknowledged Accord txn won't be retried
// unless those mutations are also written to the batch log
// Only write to the log once and reuse the batchUUID for every attempt to route the mutations correctly
doFallibleWriteWithMetricTracking(() -> syncWriteToBatchlog(mutations, batchlogReplicaPlan, batchUUID, requestTime), consistencyLevel);
Tracing.trace("Successfully wrote to batchlog");
wroteToBatchLog = true;
}
// Start Accord executing so it executes while the mutations are synchronously applied
AsyncTxnResult accordResult = !accordMutations.isEmpty() ? mutateWithAccordAsync(cm, accordMutations, consistencyLevel, requestTime) : null;
Throwable failure = null;
try
{
// now actually perform the writes and wait for them to complete
if (!wrappers.isEmpty())
{
doFallibleWriteWithMetricTracking(() -> syncWriteBatchedMutations(wrappers, Stage.MUTATION, requestTime), consistencyLevel);
Tracing.trace("Successfully wrote normal mutations");
}
}
catch (RetryOnDifferentSystemException e)
{
writeMetrics.retryDifferentSystem.mark();
writeMetricsForLevel(consistencyLevel).retryDifferentSystem.mark();
logger.debug("Retrying batch txn on different system because some mutations were misrouted");
Tracing.trace("Got {} from normal mutations, will retry", e);
continue;
}
catch (CoordinatorBehindException e)
{
writeMetrics.retryCoordinatorBehind.mark();
writeMetricsForLevel(consistencyLevel).retryCoordinatorBehind.mark();
mutations.forEach(IMutation::clearCachedSerializationsForRetry);
logger.debug("Retrying batch now that coordinator has caught up to cluster metadata");
Tracing.trace("Got {} from normal mutations, will retry", e);
continue;
}
catch (Exception e)
{
failure = Throwables.merge(failure, e);
}
finally
{
// Try to exclude most of the Accord time
nonAccordEndTime = nanoTime();
}
// Check if the Accord mutations succeeded asynchronously
try
{
// It's notable here that the Accord portion of the batch will not be hinted
// while the regular mutations are hinted on failure and also going to be replayed later from
// the batch log. It wouldn't be difficult to add hinting here, but it does seem redundant with
// the batch log.
if (accordResult != null)
{
IAccordService accord = AccordService.instance();
TxnResult.Kind kind = accord.getTxnResult(accordResult, true, consistencyLevel, requestTime).kind();
if (kind == retry_new_protocol && failure == null)
continue;
Tracing.trace("Successfully wrote Accord mutations");
cleanup.ackMutation();
}
}
catch (Exception e)
{
failure = Throwables.merge(failure, e);
}
if (failure != null)
throw unchecked(failure);
break;
}
// write to the batchlog
syncWriteToBatchlog(mutations, replicaPlan, batchUUID, requestTime);
// now actually perform the writes and wait for them to complete
syncWriteBatchedMutations(wrappers, Stage.MUTATION, requestTime);
}
catch (UnavailableException e)
catch (Exception t)
{
writeMetrics.unavailables.mark();
writeMetricsForLevel(consistency_level).unavailables.mark();
Tracing.trace("Unavailable");
throw e;
}
catch (WriteTimeoutException e)
{
writeMetrics.timeouts.mark();
writeMetricsForLevel(consistency_level).timeouts.mark();
Tracing.trace("Write timeout; received {} of {} required replies", e.received, e.blockFor);
throw e;
}
catch (WriteFailureException e)
{
writeMetrics.failures.mark();
writeMetricsForLevel(consistency_level).failures.mark();
Tracing.trace("Write failure; received {} of {} required replies", e.received, e.blockFor);
throw e;
// Unexpected error so it would be helpful to have details
Tracing.trace("{}", getStackTraceAsToString(t));
throw t;
}
finally
{
long latency = nanoTime() - startTime;
writeMetrics.addNano(latency);
writeMetricsForLevel(consistency_level).addNano(latency);
updateCoordinatorWriteLatencyTableMetric(mutations, latency);
if (attributeNonAccordLatency)
{
// On the exception path nonAccordEndTime will be -1
long latency = nonAccordEndTime != -1 ? nonAccordEndTime : nanoTime() - startTime;
writeMetrics.addNano(latency);
writeMetricsForLevel(consistencyLevel).addNano(latency);
updateCoordinatorWriteLatencyTableMetric(mutations, latency);
}
}
}
@ -1501,7 +1623,7 @@ public class StorageProxy implements StorageProxyMBean
}
}
private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, Stage stage, Dispatcher.RequestTime requestTime)
private static void syncWriteBatchedMutations(Iterable<WriteResponseHandlerWrapper> wrappers, Stage stage, Dispatcher.RequestTime requestTime)
throws WriteTimeoutException, OverloadedException
{
String localDataCenter = DatabaseDescriptor.getLocator().local().datacenter;
@ -1559,22 +1681,12 @@ public class StorageProxy implements StorageProxyMBean
// same as performWrites except does not initiate writes (but does perform availability checks).
private static WriteResponseHandlerWrapper wrapBatchResponseHandler(Mutation mutation,
ConsistencyLevel consistencyLevel,
ReplicaPlan.ForWrite replicaPlan,
ConsistencyLevel batchConsistencyLevel,
WriteType writeType,
BatchlogResponseHandler.BatchlogCleanup cleanup,
Dispatcher.RequestTime requestTime)
{
Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
Token tk = mutation.key().getToken();
ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal);
if (replicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null)
writeMetrics.localRequests.mark();
else
writeMetrics.remoteRequests.mark();
AbstractReplicationStrategy rs = replicaPlan.replicationStrategy();
AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(replicaPlan, null, writeType, mutation, requestTime);
BatchlogResponseHandler<IMutation> batchHandler = new BatchlogResponseHandler<>(writeHandler, batchConsistencyLevel.blockFor(rs), cleanup, requestTime);
@ -1603,13 +1715,17 @@ public class StorageProxy implements StorageProxyMBean
}
// used by atomic_batch_mutate to decouple availability check from the write itself, caches consistency level and endpoints.
private static class WriteResponseHandlerWrapper
public static class WriteResponseHandlerWrapper
{
final BatchlogResponseHandler<IMutation> handler;
final Mutation mutation;
@Nonnull
public final BatchlogResponseHandler<IMutation> handler;
@Nonnull
public final Mutation mutation;
WriteResponseHandlerWrapper(BatchlogResponseHandler<IMutation> handler, Mutation mutation)
public WriteResponseHandlerWrapper(@Nonnull BatchlogResponseHandler<IMutation> handler, @Nonnull Mutation mutation)
{
checkNotNull(handler);
checkNotNull(mutation);
this.handler = handler;
this.mutation = mutation;
}
@ -1844,7 +1960,7 @@ public class StorageProxy implements StorageProxyMBean
}
catch (Exception ex)
{
if (!(ex instanceof WriteTimeoutException))
if (!(ex instanceof WriteTimeoutException) && !(ex instanceof RetryOnDifferentSystemException))
logger.error("Failed to apply mutation locally : ", ex);
handler.onFailure(FBUtilities.getBroadcastAddressAndPort(), RequestFailure.forException(ex));
}
@ -2807,17 +2923,18 @@ public class StorageProxy implements StorageProxyMBean
long timeTakenNanos = now - startTimeNanos();
MessagingService.instance().metrics.recordSelfDroppedMessage(Verb.MUTATION_REQ, timeTakenNanos, NANOSECONDS);
if (requestTime.shouldSendHints())
// Don't submit a hint if this replica is transient
if (localReplica.isTransient())
return;
HintRunnable runnable = new HintRunnable(ImmutableSet.of(localReplica.endpoint()))
{
HintRunnable runnable = new HintRunnable(EndpointsForToken.of(localReplica.range().right, localReplica))
protected void runMayThrow() throws Exception
{
protected void runMayThrow() throws Exception
{
LocalMutationRunnable.this.runMayThrow();
}
};
submitHint(runnable);
}
LocalMutationRunnable.this.runMayThrow();
}
};
submitHint(runnable);
return;
}
@ -2872,9 +2989,9 @@ public class StorageProxy implements StorageProxyMBean
*/
private abstract static class HintRunnable implements Runnable
{
public final EndpointsForToken targets;
public final Set<InetAddressAndPort> targets;
protected HintRunnable(EndpointsForToken targets)
protected HintRunnable(Set<InetAddressAndPort> targets)
{
this.targets = targets;
}
@ -2892,7 +3009,7 @@ public class StorageProxy implements StorageProxyMBean
finally
{
StorageMetrics.totalHintsInProgress.dec(targets.size());
for (InetAddressAndPort target : targets.endpoints())
for (InetAddressAndPort target : targets)
getHintsInProgressFor(target).decrementAndGet();
}
}
@ -2938,25 +3055,43 @@ public class StorageProxy implements StorageProxyMBean
}
}
public static void submitHint(Mutation mutation, Replica target, AbstractWriteResponseHandler<IMutation> responseHandler)
public static void submitHintForRetryOnDifferentSystem(Mutation mutation)
{
submitHint(mutation, EndpointsForToken.of(target.range().right, target), responseHandler);
submitHint(mutation, ImmutableSet.of(HintsService.RETRY_ON_DIFFERENT_SYSTEM_ADDRESS), null);
}
private static void submitHint(Mutation mutation,
EndpointsForToken targets,
AbstractWriteResponseHandler<IMutation> responseHandler)
public static Future<Void> submitHint(Mutation mutation, Replica target, AbstractWriteResponseHandler<IMutation> responseHandler)
{
return submitHint(mutation, EndpointsForToken.of(target.range().right, target), responseHandler);
}
private static Future<Void> submitHint(Mutation mutation,
EndpointsForToken targets,
AbstractWriteResponseHandler<IMutation> responseHandler)
{
// hints should not be written for transient replicas because there is no point if they didn't contribute
// to quorum, they would eventually be removed anyways after running incremental repair.
// This logic assumes we don't always write to transient replicas to minimize incremental repair mismatches
// so we may want to walk this back when revisiting transient replication
Replicas.assertFull(targets);
return submitHint(mutation, targets.endpoints(), responseHandler);
}
private static Future<Void> submitHint(Mutation mutation,
Set<InetAddressAndPort> targets,
AbstractWriteResponseHandler<IMutation> responseHandler)
{
Replicas.assertFull(targets); // hints should not be written for transient replicas
HintRunnable runnable = new HintRunnable(targets)
{
public void runMayThrow()
{
Set<InetAddressAndPort> validTargets = new HashSet<>(targets.size());
Set<UUID> hostIds = new HashSet<>(targets.size());
for (InetAddressAndPort target : targets.endpoints())
for (InetAddressAndPort target : targets)
{
UUID hostId = StorageService.instance.getHostIdForEndpoint(target);
UUID hostId = target == HintsService.RETRY_ON_DIFFERENT_SYSTEM_ADDRESS ?
HintsService.RETRY_ON_DIFFERENT_SYSTEM_UUID :
StorageService.instance.getHostIdForEndpoint(target);
if (hostId != null)
{
hostIds.add(hostId);
@ -2981,14 +3116,14 @@ public class StorageProxy implements StorageProxyMBean
}
};
submitHint(runnable);
return submitHint(runnable);
}
private static Future<Void> submitHint(HintRunnable runnable)
{
StorageMetrics.totalHintsInProgress.inc(runnable.targets.size());
for (Replica target : runnable.targets)
getHintsInProgressFor(target.endpoint()).incrementAndGet();
for (InetAddressAndPort target : runnable.targets)
getHintsInProgressFor(target).incrementAndGet();
return (Future<Void>) Stage.MUTATION.submit(runnable);
}
@ -3181,9 +3316,9 @@ public class StorageProxy implements StorageProxyMBean
RowIterator casResult;
@Nonnull
PartitionIterator serialReadResult;
public final PartitionIterator serialReadResult;
boolean shouldRetryOnNewConsensusProtocol;
public final boolean shouldRetryOnNewConsensusProtocol;
private ConsensusAttemptResult(@Nullable RowIterator casResult, @Nullable PartitionIterator serialReadResult, boolean shouldRetryOnNewConsensusProtocol)
{

22
src/java/org/apache/cassandra/service/accord/AccordJournal.java
View File

@ -36,6 +36,7 @@ import com.google.common.primitives.Ints;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.coordinate.Timeout;
import accord.local.Command;
import accord.local.Node;
import accord.messages.AbstractEpochRequest;
@ -660,7 +661,26 @@ public class AccordJournal implements IJournal, Shutdownable
if (l.isEmpty())
waitForEpochs.pushLong(waitForEpoch);
l.add(context);
node.withEpoch(waitForEpoch, this::runOnce);
BiConsumer<Void, Throwable> withEpochCallback = new BiConsumer<>()
{
@Override
public void accept(Void unused, Throwable withEpochFailure)
{
if (withEpochFailure != null)
{
// Nothing to do but keep waiting
if (withEpochFailure instanceof Timeout)
{
node.withEpoch(waitForEpoch, this);
return;
}
else
throw new RuntimeException(withEpochFailure);
}
runOnce();
}
};
node.withEpoch(waitForEpoch, withEpochCallback);
}
// Next, process all delayed epochs

273
src/java/org/apache/cassandra/service/accord/AccordService.java
View File

@ -27,49 +27,37 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.Ints;
import accord.coordinate.Barrier;
import accord.coordinate.CoordinateSyncPoint;
import accord.coordinate.Exhausted;
import accord.coordinate.FailureAccumulator;
import accord.coordinate.TopologyMismatch;
import accord.impl.CoordinateDurabilityScheduling;
import accord.local.CommandStores;
import accord.primitives.SyncPoint;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.cql3.statements.RequestValidations;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.accord.exceptions.ReadExhaustedException;
import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.service.accord.repair.RepairSyncPointAdapter;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.service.accord.api.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.BarrierType;
import accord.api.Result;
import accord.config.LocalConfig;
import accord.coordinate.Barrier;
import accord.coordinate.CoordinateSyncPoint;
import accord.coordinate.CoordinationFailed;
import accord.coordinate.Exhausted;
import accord.coordinate.FailureAccumulator;
import accord.coordinate.Preempted;
import accord.coordinate.Timeout;
import accord.coordinate.TopologyMismatch;
import accord.impl.AbstractConfigurationService;
import accord.impl.CoordinateDurabilityScheduling;
import accord.impl.SimpleProgressLog;
import accord.impl.SizeOfIntersectionSorter;
import accord.local.CommandStores;
import accord.local.DurableBefore;
import accord.local.Node;
import accord.local.Node.Id;
@ -78,7 +66,11 @@ import accord.local.RedundantBefore;
import accord.local.ShardDistributor.EvenSplit;
import accord.messages.LocalRequest;
import accord.messages.Request;
import accord.primitives.Keys;
import accord.primitives.Ranges;
import accord.primitives.Seekable;
import accord.primitives.Seekables;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.Txn.Kind;
@ -91,25 +83,46 @@ import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.concurrent.Shutdownable;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.statements.RequestValidations;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.exceptions.ReadTimeoutException;
import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.RequestTimeoutException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.AccordClientRequestMetrics;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessageDelivery;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordSyncPropagator.Notification;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.KeyspaceSplitter;
import org.apache.cassandra.service.accord.api.AccordScheduler;
import org.apache.cassandra.service.accord.api.AccordTopologySorter;
import org.apache.cassandra.service.accord.api.CompositeTopologySorter;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.exceptions.ReadExhaustedException;
import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException;
import org.apache.cassandra.service.accord.exceptions.WritePreemptedException;
import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory;
import org.apache.cassandra.service.accord.repair.RepairSyncPointAdapter;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.consensus.migration.TableMigrationState;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.Blocking;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.ExecutorUtils;
@ -120,7 +133,10 @@ import org.apache.cassandra.utils.concurrent.ImmediateFuture;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static accord.messages.SimpleReply.Ok;
import static accord.primitives.Routable.Domain.Key;
import static accord.primitives.Routable.Domain.Range;
import static accord.utils.Invariants.checkState;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
@ -161,15 +177,21 @@ public class AccordService implements IAccordService, Shutdownable
}
@Override
public long barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite)
public Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
{
throw new UnsupportedOperationException();
}
@Override
public Seekables barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite)
{
throw new UnsupportedOperationException("No accord barriers should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
public long barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
public Seekables repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints)
{
throw new UnsupportedOperationException();
throw new UnsupportedOperationException("No accord repairs should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
@ -179,9 +201,15 @@ public class AccordService implements IAccordService, Shutdownable
}
@Override
public long repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints)
public @Nonnull AsyncTxnResult coordinateAsync(@Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
throw new UnsupportedOperationException("No accord repairs should be executed when accord.enabled = false in cassandra.yaml");
throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult, boolean isWrite, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml");
}
@Override
@ -328,7 +356,7 @@ public class AccordService implements IAccordService, Shutdownable
this::handleLocalRequest,
configService,
AccordService::uniqueNow,
NodeTimeService.unixWrapper(TimeUnit.MICROSECONDS, AccordService::uniqueNow),
NodeTimeService.elapsedWrapperFromMonotonicSource(NANOSECONDS, Clock.Global::nanoTime),
() -> dataStore,
new KeyspaceSplitter(new EvenSplit<>(DatabaseDescriptor.getAccordShardCount(), getPartitioner().accordSplitter())),
agent,
@ -369,8 +397,17 @@ public class AccordService implements IAccordService, Shutdownable
return requestHandler;
}
private <S extends Seekables<?, ?>> long barrier(@Nonnull S keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, BiFunction<Node, S, AsyncResult<SyncPoint<S>>> syncPoint)
private <S extends Seekables<?, ?>> Seekables barrier(@Nonnull S keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, BiFunction<Node, S, AsyncResult<SyncPoint<S>>> syncPoint)
{
Stopwatch sw = Stopwatch.createStarted();
keysOrRanges = (S)intersectionWithAccordManagedRanges(keysOrRanges);
// It's possible none of them were Accord managed and we aren't going to treat that as an error
if (keysOrRanges.isEmpty())
{
logger.info("Skipping barrier because there are no ranges managed by Accord");
return keysOrRanges;
}
AccordClientRequestMetrics metrics = isForWrite ? accordWriteMetrics : accordReadMetrics;
TxnId txnId = null;
try
@ -382,14 +419,15 @@ public class AccordService implements IAccordService, Shutdownable
: Barrier.barrier(node, keysOrRanges, epoch, barrierType, syncPoint);
long deadlineNanos = requestTime.startedAtNanos() + timeoutNanos;
Timestamp barrierExecuteAt = AsyncChains.getBlocking(asyncResult, deadlineNanos - nanoTime(), NANOSECONDS);
logger.debug("Completed in {}ms barrier key: {} epoch: {} barrierType: {} isForWrite {}",
logger.debug("Completed barrier attempt in {}ms, {}ms since attempts start, barrier key: {} epoch: {} barrierType: {} isForWrite {}",
sw.elapsed(MILLISECONDS),
NANOSECONDS.toMillis(nanoTime() - requestTime.startedAtNanos()),
keysOrRanges, epoch, barrierType, isForWrite);
return barrierExecuteAt.epoch();
return keysOrRanges;
}
catch (ExecutionException e)
{
Throwable cause = e.getCause();
Throwable cause = Throwables.getRootCause(e);
if (cause instanceof Timeout)
{
metrics.timeouts.mark();
@ -431,7 +469,7 @@ public class AccordService implements IAccordService, Shutdownable
}
@Override
public long barrier(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite)
public Seekables barrier(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite)
{
return barrier(keysOrRanges, epoch, requestTime, timeoutNanos, barrierType, isForWrite, null);
}
@ -442,12 +480,59 @@ public class AccordService implements IAccordService, Shutdownable
}
@Override
public long repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints)
public Seekables repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints)
{
Set<Node.Id> allNodes = allEndpoints.stream().map(configService::mappedId).collect(Collectors.toUnmodifiableSet());
return barrier(keysOrRanges, epoch, requestTime, timeoutNanos, barrierType, isForWrite, repairSyncPoint(allNodes));
}
private static <S extends Seekables<?,?>> Seekables intersectionWithAccordManagedRanges(Seekables<?, ?> keysOrRanges)
{
TableId tableId = null;
for (Seekable seekable : keysOrRanges)
{
TableId newTableId;
if (keysOrRanges.domain() == Key)
newTableId = ((PartitionKey)seekable).table();
else if (keysOrRanges.domain() == Range)
newTableId = ((TokenRange)seekable).table();
else
throw new IllegalStateException("Unexpected domain " + keysOrRanges.domain());
if (tableId == null)
tableId = newTableId;
else if (!tableId.equals(newTableId))
throw new IllegalArgumentException("Currently only one table is handled here.");
}
ClusterMetadata cm = ClusterMetadata.current();
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
TableMetadata tm = cfs.metadata();
// Barriers can be needed just because it's an Accord managed range, but it could also be a migration back to Paxos
// in which case we do want to barrier the migrating/migrated ranges even though the target for the migration is not Accord
// In either case Accord should be aware of those ranges and not generate a topology mismatch
if (tm.params.transactionalMode != TransactionalMode.off || tm.params.transactionalMigrationFrom.from != TransactionalMode.off)
{
TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tm.id);
// null is fine could be completely migrated or was always an Accord table on creation
if (tms == null)
return keysOrRanges;
Ranges migratingAndMigratedRanges = AccordTopology.toAccordRanges(tms.tableId, tms.migratingAndMigratedRanges);
return keysOrRanges.slice(migratingAndMigratedRanges);
}
switch (keysOrRanges.domain())
{
case Key:
return Keys.EMPTY;
case Range:
return Ranges.EMPTY;
default:
throw new IllegalStateException("Only keys and ranges are supported");
}
}
@VisibleForTesting
static ReadTimeoutException newBarrierTimeout(TxnId txnId, boolean global)
{
@ -474,21 +559,27 @@ public class AccordService implements IAccordService, Shutdownable
}
@VisibleForTesting
static long doWithRetries(Blocking blocking, LongSupplier action, int retryAttempts, long initialBackoffMillis, long maxBackoffMillis) throws InterruptedException
static Seekables doWithRetries(Blocking blocking, Supplier<Seekables> action, int retryAttempts, long initialBackoffMillis, long maxBackoffMillis) throws InterruptedException
{
// Since we could end up having the barrier transaction or the transaction it listens to invalidated
Throwable existingFailures = null;
Long success = null;
Seekables success = null;
long backoffMillis = initialBackoffMillis;
for (int attempt = 0; attempt < retryAttempts; attempt++)
{
try
{
success = action.getAsLong();
success = action.get();
break;
}
catch (TopologyMismatch topologyMismatch)
{
// Retry topology mismatch immediately because we should be able calculate the correct ranges immediately
backoffMillis = 0;
}
catch (RequestExecutionException | CoordinationFailed newFailures)
{
logger.error("Had failure on barrier", newFailures);
existingFailures = FailureAccumulator.append(existingFailures, newFailures, AccordService::isTimeout);
try
@ -514,6 +605,7 @@ public class AccordService implements IAccordService, Shutdownable
}
if (success == null)
{
logger.error("Ran out of retries for barrier");
checkState(existingFailures != null, "Didn't have success, but also didn't have failures");
Throwables.throwIfUnchecked(existingFailures);
throw new RuntimeException(existingFailures);
@ -522,7 +614,7 @@ public class AccordService implements IAccordService, Shutdownable
}
@Override
public long barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
public Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
{
return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().barrier(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeBarrierTimeoutNanos(), barrierType, isForWrite),
DatabaseDescriptor.getAccordBarrierRetryAttempts(),
@ -531,7 +623,7 @@ public class AccordService implements IAccordService, Shutdownable
}
@Override
public long repairWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints) throws InterruptedException
public Seekables repairWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints) throws InterruptedException
{
return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().repair(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeBarrierTimeoutNanos(), barrierType, isForWrite, allEndpoints),
DatabaseDescriptor.getAccordBarrierRetryAttempts(),
@ -558,38 +650,86 @@ public class AccordService implements IAccordService, Shutdownable
@Override
public @Nonnull TxnResult coordinate(@Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
AsyncTxnResult asyncTxnResult = coordinateAsync(txn, consistencyLevel, requestTime);
return getTxnResult(asyncTxnResult, txn.isWrite(), consistencyLevel, requestTime);
}
@Override
public @Nonnull AsyncTxnResult coordinateAsync(Txn txn, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
TxnId txnId = node.nextTxnId(txn.kind(), txn.keys().domain());
AccordClientRequestMetrics metrics = txn.isWrite() ? accordWriteMetrics : accordReadMetrics;
TxnId txnId = null;
try
{
metrics.keySize.update(txn.keys().size());
txnId = node.nextTxnId(txn.kind(), txn.keys().domain());
long deadlineNanos = requestTime.startedAtNanos() + DatabaseDescriptor.getTransactionTimeout(NANOSECONDS);
AsyncResult<Result> asyncResult = node.coordinate(txnId, txn);
Result result = AsyncChains.getBlocking(asyncResult, deadlineNanos - nanoTime(), NANOSECONDS);
return (TxnResult) result;
}
catch (ExecutionException e)
{
Throwable cause = e.getCause();
metrics.keySize.update(txn.keys().size());
AsyncResult<Result> asyncResult = node.coordinate(txnId, txn);
AsyncTxnResult asyncTxnResult = new AsyncTxnResult(txnId);
asyncResult.addCallback((success, failure) -> {
long durationNanos = nanoTime() - requestTime.startedAtNanos();
metrics.addNano(durationNanos);
Throwable cause = failure != null ? Throwables.getRootCause(failure) : null;
if (success != null)
{
if (((TxnResult)success).kind() == TxnResult.Kind.retry_new_protocol)
{
metrics.retryDifferentSystem.mark();
Tracing.trace("Got retry different system error from Accord, will retry");
}
asyncTxnResult.trySuccess((TxnResult)success);
return;
}
if (cause instanceof Timeout)
{
metrics.timeouts.mark();
throw newTimeout(txnId, txn, consistencyLevel);
// Don't mark the metric here, should be done in getTxnResult to ensure it only happens once
// since both Accord and the thread blocked on the result can trigger a timeout
asyncTxnResult.tryFailure(newTimeout(txnId, txn.isWrite(), consistencyLevel));
return;
}
if (cause instanceof Preempted)
{
metrics.preempted.mark();
//TODO need to improve
// Coordinator "could" query the accord state to see whats going on but that doesn't exist yet.
// Protocol also doesn't have a way to denote "unknown" outcome, so using a timeout as the closest match
throw newPreempted(txnId, txn, consistencyLevel);
asyncTxnResult.tryFailure(newPreempted(txnId, txn.isWrite(), consistencyLevel));
return;
}
if (cause instanceof TopologyMismatch)
{
throw RequestValidations.invalidRequest(cause.getMessage());
metrics.topologyMismatches.mark();
asyncTxnResult.tryFailure(RequestValidations.invalidRequest(cause.getMessage()));
return;
}
metrics.failures.mark();
throw new RuntimeException(cause);
asyncTxnResult.tryFailure(new RuntimeException(cause));
});
return asyncTxnResult;
}
@Override
public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult, boolean isWrite, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
AccordClientRequestMetrics metrics = isWrite ? accordWriteMetrics : accordReadMetrics;
try
{
long deadlineNanos = requestTime.startedAtNanos() + DatabaseDescriptor.getTransactionTimeout(NANOSECONDS);
TxnResult result = asyncTxnResult.get(deadlineNanos - nanoTime(), NANOSECONDS);
return result;
}
catch (ExecutionException e)
{
// Metrics except timeout have already been handled
Throwable cause = e.getCause();
if (cause instanceof RequestTimeoutException)
{
// Mark here instead of in coordinate async since this is where the request timeout actually occurs
metrics.timeouts.mark();
cause.addSuppressed(e);
throw (RequestTimeoutException) cause;
}
else if (cause instanceof RuntimeException)
throw (RuntimeException)cause;
else
throw new RuntimeException(cause);
}
catch (InterruptedException e)
{
@ -599,11 +739,7 @@ public class AccordService implements IAccordService, Shutdownable
catch (TimeoutException e)
{
metrics.timeouts.mark();
throw newTimeout(txnId, txn, consistencyLevel);
}
finally
{
metrics.addNano(nanoTime() - requestTime.startedAtNanos());
throw newTimeout(asyncTxnResult.txnId, isWrite, consistencyLevel);
}
}
@ -614,15 +750,20 @@ public class AccordService implements IAccordService, Shutdownable
journal.processLocalRequest(request, callback);
}
private static RequestTimeoutException newTimeout(TxnId txnId, Txn txn, ConsistencyLevel consistencyLevel)
private static RequestTimeoutException newTimeout(TxnId txnId, boolean isWrite, ConsistencyLevel consistencyLevel)
{
throw txn.isWrite() ? new WriteTimeoutException(WriteType.CAS, consistencyLevel, 0, 0, txnId.toString())
// Client protocol doesn't handle null consistency level so use ANY
if (consistencyLevel == null)
consistencyLevel = ConsistencyLevel.ANY;
return isWrite ? new WriteTimeoutException(WriteType.CAS, consistencyLevel, 0, 0, txnId.toString())
: new ReadTimeoutException(consistencyLevel, 0, 0, false, txnId.toString());
}
private static RuntimeException newPreempted(TxnId txnId, Txn txn, ConsistencyLevel consistencyLevel)
private static RuntimeException newPreempted(TxnId txnId, boolean isWrite, ConsistencyLevel consistencyLevel)
{
throw txn.isWrite() ? new WritePreemptedException(WriteType.CAS, consistencyLevel, 0, 0, txnId.toString())
if (consistencyLevel == null)
consistencyLevel = ConsistencyLevel.ANY;
return isWrite ? new WritePreemptedException(WriteType.CAS, consistencyLevel, 0, 0, txnId.toString())
: new ReadPreemptedException(consistencyLevel, 0, 0, false, txnId.toString());
}

6
src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java
View File

@ -67,6 +67,10 @@ public class AccordVerbHandler<T extends Request> implements IVerbHandler<T>
if (node.topology().hasEpoch(waitForEpoch))
request.process(node, fromNodeId, message);
else
node.withEpoch(waitForEpoch, () -> request.process(node, fromNodeId, message));
node.withEpoch(waitForEpoch, (ignored, withEpochFailure) -> {
if (withEpochFailure != null)
throw new RuntimeException("Timed out waiting for epoch when processing message from " + fromNodeId + " to " + node + " message " + message, withEpochFailure);
request.process(node, fromNodeId, message);
});
}
}

49
src/java/org/apache/cassandra/service/accord/IAccordService.java
View File

@ -18,6 +18,15 @@
package org.apache.cassandra.service.accord;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.collect.ImmutableSet;
import accord.api.BarrierType;
import accord.local.CommandStores;
import accord.local.DurableBefore;
@ -27,8 +36,8 @@ import accord.messages.Request;
import accord.primitives.Ranges;
import accord.primitives.Seekables;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.topology.TopologyManager;
import com.google.common.collect.ImmutableSet;
import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
@ -37,19 +46,14 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.AccordScheduler;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
import javax.annotation.Nonnull;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkNotNull;
public interface IAccordService
{
@ -58,25 +62,21 @@ public interface IAccordService
IVerbHandler<? extends Request> verbHandler();
long barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException;
Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException;
long barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite);
Seekables barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite);
default long repairWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints) throws InterruptedException
default Seekables repairWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints) throws InterruptedException
{
throw new UnsupportedOperationException();
}
long repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints);
Seekables repair(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints);
default void postStreamReceivingBarrier(ColumnFamilyStore cfs, List<Range<Token>> ranges)
{
String ks = cfs.keyspace.getName();
Ranges accordRanges = Ranges.of(ranges
.stream()
.map(r -> new TokenRange(new TokenKey(cfs.getTableId(), r.left), new TokenKey(cfs.getTableId(), r.right)))
.collect(Collectors.toList())
.toArray(new accord.primitives.Range[0]));
Ranges accordRanges = AccordTopology.toAccordRanges(ks, ranges);
try
{
barrierWithRetries(accordRanges, Epoch.FIRST.getEpoch(), BarrierType.global_async, true);
@ -89,6 +89,21 @@ public interface IAccordService
@Nonnull TxnResult coordinate(@Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime);
class AsyncTxnResult extends AsyncPromise<TxnResult>
{
public final @Nonnull TxnId txnId;
public AsyncTxnResult(@Nonnull TxnId txnId)
{
checkNotNull(txnId);
this.txnId = txnId;
}
}
@Nonnull
AsyncTxnResult coordinateAsync(@Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime);
TxnResult getTxnResult(AsyncTxnResult asyncTxnResult, boolean isWrite, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime);
long currentEpoch();
void setCacheSize(long kb);

8
src/java/org/apache/cassandra/service/accord/api/AccordAgent.java
View File

@ -132,10 +132,14 @@ public class AccordAgent implements Agent
return 32;
}
/**
* Create an empty transaction that Accord can use for its internal transactions. This is not suitable
* for tests since it skips validation done by regular transactions.
*/
@Override
public Txn emptyTxn(Kind kind, Seekables<?, ?> seekables)
public Txn emptySystemTxn(Kind kind, Seekables<?, ?> seekables)
{
return new Txn.InMemory(kind, seekables, TxnRead.EMPTY, TxnQuery.EMPTY, null);
return new Txn.InMemory(kind, seekables, TxnRead.EMPTY, TxnQuery.UNSAFE_EMPTY, null);
}
@Override

22
src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java
View File

@ -28,9 +28,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import accord.messages.ReadTxnData;
import accord.primitives.Ballot;
import org.apache.cassandra.schema.TableId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -43,6 +40,8 @@ import accord.local.Node;
import accord.local.Node.Id;
import accord.messages.Commit;
import accord.messages.Commit.Kind;
import accord.messages.ReadTxnData;
import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
import accord.primitives.Participants;
@ -74,6 +73,7 @@ import org.apache.cassandra.metrics.AccordClientRequestMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.accord.AccordEndpointMapper;
import org.apache.cassandra.service.accord.TokenRange;
@ -239,9 +239,9 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen
@Override
public void sendReadRepairMutation(Message<Mutation> message, InetAddressAndPort to, RequestCallback<Object> callback)
{
checkArgument(message.payload.allowsPotentialTransactionConflicts());
Node.Id id = endpointMapper.mappedId(to);
Mutation mutation = message.payload;
AccordInteropReadRepair readRepair = new AccordInteropReadRepair(id, executes, txnId, readScope, executeAt.epoch(), mutation);
AccordInteropReadRepair readRepair = new AccordInteropReadRepair(id, executes, txnId, readScope, executeAt.epoch(), message.payload);
node.send(id, readRepair, executor, new AccordInteropReadRepair.ReadRepairCallback(id, to, message, callback, this));
}
@ -260,8 +260,8 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen
// This should only rarely occur when coordinators start a transaction in a migrating range
// because they haven't yet updated their cluster metadata.
// It would be harmless to do the read, but we can respond faster skipping it
// and getting the transaction on the correct protocol
// It would be harmless to do the read, because it will be rejected in `TxnQuery` anyways,
// but it's faster to skip the read
TableMigrationState tms = ConsensusTableMigration.getTableMigrationState(command.metadata().id);
AccordClientRequestMetrics metrics = txn.kind().isWrite() ? accordWriteMetrics : accordReadMetrics;
if (ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(command.metadata(), tms, command.partitionKey()))
@ -376,13 +376,7 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen
return readCommand.allowOutOfRangeReads();
}
@Override
public Mutation maybeAllowOutOfRangeMutations(Mutation m)
{
return m.allowOutOfRangeMutations();
}
// Prrovide request callbacks with a way to send maximal commits on Insufficient responses
// Provide request callbacks with a way to send maximal commits on Insufficient responses
@Override
public void sendMaximalCommit(Id to)
{

11
src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java
View File

@ -20,12 +20,14 @@ package org.apache.cassandra.service.accord.interop;
import javax.annotation.Nonnull;
import accord.coordinate.Timeout;
import accord.local.Node;
import accord.messages.Callback;
import accord.messages.ReadData.ReadOk;
import accord.messages.ReadData.ReadReply;
import accord.utils.Invariants;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.RequestCallback;
@ -78,7 +80,14 @@ public abstract class AccordInteropReadCallback<T> implements Callback<ReadReply
public void onFailure(Node.Id from, Throwable failure)
{
wrapped.onFailure(endpoint, RequestFailure.UNKNOWN);
RequestFailure requestFailure;
// Convert from Accord's timeout exception to our failure reason because timeout is something
// that is useful for metrics and can be handled differently
if (failure instanceof Timeout)
requestFailure = new RequestFailure(RequestFailureReason.TIMEOUT, failure);
else
requestFailure = new RequestFailure(RequestFailureReason.UNKNOWN, failure);
wrapped.onFailure(endpoint, requestFailure);
}
public void onCallbackFailure(Node.Id from, Throwable failure)

30
src/java/org/apache/cassandra/service/accord/repair/AccordRepair.java
View File

@ -19,6 +19,7 @@
package org.apache.cassandra.service.accord.repair;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.Executor;
@ -89,20 +90,21 @@ public class AccordRepair
return minEpoch;
}
public void repair() throws Throwable
public Ranges repair() throws Throwable
{
List<accord.primitives.Range> repairedRanges = new ArrayList<>();
for (accord.primitives.Range range : ranges)
repairRange((TokenRange)range);
repairedRanges.addAll(repairRange((TokenRange)range));
return Ranges.of(repairedRanges.toArray(new accord.primitives.Range[0]));
}
public Future<Void> repair(Executor executor)
public Future<Ranges> repair(Executor executor)
{
AsyncPromise<Void> future = new AsyncPromise<>();
AsyncPromise<Ranges> future = new AsyncPromise<>();
executor.execute(() -> {
try
{
repair();
future.trySuccess(null);
future.trySuccess(repair());
}
catch (Throwable e)
{
@ -117,8 +119,9 @@ public class AccordRepair
shouldAbort = reason == null ? new RuntimeException("Abort") : reason;
}
private void repairRange(TokenRange range) throws Throwable
private List<accord.primitives.Range> repairRange(TokenRange range) throws Throwable
{
List<accord.primitives.Range> repairedRanges = new ArrayList<>();
int rangeStepUpdateInterval = ACCORD_REPAIR_RANGE_STEP_UPDATE_INTERVAL.getInt();
RoutingKey remainingStart = range.start();
BigInteger rangeSize = splitter.sizeOf(range);
@ -154,7 +157,7 @@ public class AccordRepair
if (remainingStart.equals(range.end()))
{
logger.info("Completed barriers for {} in {} iterations", range, iteration - 1);
return;
return repairedRanges;
}
// Final repair is whatever remains
@ -169,14 +172,13 @@ public class AccordRepair
checkState(lastRepaired == null || toRepair.start().equals(lastRepaired.end()), "Next range should directly follow previous range");
lastRepaired = toRepair;
Ranges barrieredRanges;
if (requireAllEndpoints)
{
AccordService.instance().repairWithRetries(Seekables.of(toRepair), minEpoch.getEpoch(), BarrierType.global_sync, false, endpoints);
}
barrieredRanges = (Ranges)AccordService.instance().repairWithRetries(Seekables.of(toRepair), minEpoch.getEpoch(), BarrierType.global_sync, false, endpoints);
else
{
AccordService.instance().barrierWithRetries(Seekables.of(toRepair), minEpoch.getEpoch(), BarrierType.global_sync, false);
}
barrieredRanges = (Ranges)AccordService.instance().barrierWithRetries(Seekables.of(toRepair), minEpoch.getEpoch(), BarrierType.global_sync, false);
for (accord.primitives.Range barrieredRange : barrieredRanges)
repairedRanges.add(barrieredRange);
remainingStart = toRepair.end();
}

26
src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java
View File

@ -50,6 +50,11 @@ import static org.apache.cassandra.service.accord.txn.TxnRead.CAS_READ;
public abstract class TxnQuery implements Query
{
/**
* Used by transaction statements which will have Accord pass back to the C* coordinator code all the data that is
* read even if it is not returned as part of the result to the client. TxnDataName.returning() will fetch the data
* that is returned from TxnData.
*/
public static final TxnQuery ALL = new TxnQuery()
{
@Override
@ -65,6 +70,10 @@ public abstract class TxnQuery implements Query
}
};
/**
* For transactions that return no results but do still care that they don't apply if the tokens/ranges
* are not owned/managed by Accord
*/
public static final TxnQuery NONE = new TxnQuery()
{
@Override
@ -80,6 +89,9 @@ public abstract class TxnQuery implements Query
}
};
/**
* For supporting CQL CAS compatible transactions
*/
public static final TxnQuery CONDITION = new TxnQuery()
{
@Override
@ -114,7 +126,13 @@ public abstract class TxnQuery implements Query
}
};
public static final TxnQuery EMPTY = new TxnQuery()
/**
* UNSAFE_EMPTY doesn't validate that the range is owned by Accord so you want to be careful and use NONE
* if your transaction simply doesn't have results because that will validate that Accord owns the range
* for things like blind writes. Empty is used by Accord for things like sync points which may need to exeucte
* for ranges Accord used to manage, but no longer does.
*/
public static final TxnQuery UNSAFE_EMPTY = new TxnQuery()
{
@Override
@ -171,7 +189,7 @@ public abstract class TxnQuery implements Query
@Override
public void serialize(TxnQuery query, DataOutputPlus out, int version) throws IOException
{
Preconditions.checkArgument(query == null | query == ALL | query == NONE | query == CONDITION | query == EMPTY);
Preconditions.checkArgument(query == null | query == ALL | query == NONE | query == CONDITION | query == UNSAFE_EMPTY);
out.writeByte(query == null ? 0 : query.type());
}
@ -185,14 +203,14 @@ public abstract class TxnQuery implements Query
case 1: return ALL;
case 2: return NONE;
case 3: return CONDITION;
case 4: return EMPTY;
case 4: return UNSAFE_EMPTY;
}
}
@Override
public long serializedSize(TxnQuery query, int version)
{
Preconditions.checkArgument(query == null | query == ALL | query == NONE | query == CONDITION | query == EMPTY);
Preconditions.checkArgument(query == null | query == ALL | query == NONE | query == CONDITION | query == UNSAFE_EMPTY);
return TypeSizes.sizeof((byte)2);
}
};

36
src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java
View File

@ -69,7 +69,8 @@ import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSi
public class TxnUpdate extends AccordUpdate
{
private static final long EMPTY_SIZE = ObjectSizes.measure(new TxnUpdate(null, new ByteBuffer[0], null, null));
private static final long EMPTY_SIZE = ObjectSizes.measure(new TxnUpdate(null, new ByteBuffer[0], null, null, false));
private static final int FLAG_PRESERVE_TIMESTAMPS = 0x1;
private final Keys keys;
private final ByteBuffer[] fragments;
@ -78,10 +79,15 @@ public class TxnUpdate extends AccordUpdate
@Nullable
private final ConsistencyLevel cassandraCommitCL;
// Hints and batchlog want to write with the lower timestamp they generated when applying their writes via Accord
// so they don't resurrect data if they are applied at a later time. Accord should be fine with this because
// the writes are still deterministic from the perspective of coordinators/recovery coordinators.
private final boolean preserveTimestamps;
// Memoize computation of condition
private Boolean conditionResult;
public TxnUpdate(List<TxnWrite.Fragment> fragments, TxnCondition condition, @Nullable ConsistencyLevel cassandraCommitCL)
public TxnUpdate(List<TxnWrite.Fragment> fragments, TxnCondition condition, @Nullable ConsistencyLevel cassandraCommitCL, boolean preserveTimestamps)
{
checkArgument(cassandraCommitCL == null || IAccordService.SUPPORTED_COMMIT_CONSISTENCY_LEVELS.contains(cassandraCommitCL));
// TODO: Figure out a way to shove keys into TxnCondition, and have it implement slice/merge.
@ -90,14 +96,16 @@ public class TxnUpdate extends AccordUpdate
this.fragments = toSerializedValuesArray(keys, fragments, fragment -> fragment.key, TxnWrite.Fragment.serializer);
this.condition = serialize(condition, TxnCondition.serializer);
this.cassandraCommitCL = cassandraCommitCL;
this.preserveTimestamps = preserveTimestamps;
}
private TxnUpdate(Keys keys, ByteBuffer[] fragments, ByteBuffer condition, ConsistencyLevel cassandraCommitCL)
private TxnUpdate(Keys keys, ByteBuffer[] fragments, ByteBuffer condition, ConsistencyLevel cassandraCommitCL, boolean preserveTimestamps)
{
this.keys = keys;
this.fragments = fragments;
this.condition = condition;
this.cassandraCommitCL = cassandraCommitCL;
this.preserveTimestamps = preserveTimestamps;
}
@Override
@ -141,12 +149,19 @@ public class TxnUpdate extends AccordUpdate
return keys;
}
// Batch log and hints want to keep their lower timestamp for the applied writes to avoid resurrecting old data
// when they are applied later, possibly after further updates have already been acknowledged.
public boolean preserveTimestamps()
{
return preserveTimestamps;
}
@Override
public Update slice(Ranges ranges)
{
Keys keys = this.keys.slice(ranges);
// TODO: Slice the condition.
return new TxnUpdate(keys, select(this.keys, keys, fragments), condition, cassandraCommitCL);
return new TxnUpdate(keys, select(this.keys, keys, fragments), condition, cassandraCommitCL, preserveTimestamps);
}
@Override
@ -154,7 +169,7 @@ public class TxnUpdate extends AccordUpdate
{
Keys keys = this.keys.intersecting(participants);
// TODO: Slice the condition.
return new TxnUpdate(keys, select(this.keys, keys, fragments), condition, cassandraCommitCL);
return new TxnUpdate(keys, select(this.keys, keys, fragments), condition, cassandraCommitCL, preserveTimestamps);
}
private static ByteBuffer[] select(Keys in, Keys out, ByteBuffer[] from)
@ -176,7 +191,7 @@ public class TxnUpdate extends AccordUpdate
TxnUpdate that = (TxnUpdate) update;
Keys mergedKeys = this.keys.with(that.keys);
ByteBuffer[] mergedFragments = merge(this.keys, that.keys, this.fragments, that.fragments, mergedKeys.size());
return new TxnUpdate(mergedKeys, mergedFragments, condition, cassandraCommitCL);
return new TxnUpdate(mergedKeys, mergedFragments, condition, cassandraCommitCL, preserveTimestamps);
}
private static ByteBuffer[] merge(Keys leftKeys, Keys rightKeys, ByteBuffer[] left, ByteBuffer[] right, int outputSize)
@ -207,7 +222,6 @@ public class TxnUpdate extends AccordUpdate
QueryOptions options = QueryOptions.forProtocolVersion(ProtocolVersion.CURRENT);
AccordUpdateParameters parameters = new AccordUpdateParameters((TxnData) data, options);
// First completes all fragments and join them with the repairs pending for those partitions
for (TxnWrite.Fragment fragment : fragments)
// Filter out fragments that already constitute complete updates to avoid persisting them via TxnWrite:
if (!fragment.isComplete())
@ -233,6 +247,7 @@ public class TxnUpdate extends AccordUpdate
@Override
public void serialize(TxnUpdate update, DataOutputPlus out, int version) throws IOException
{
out.writeByte(update.preserveTimestamps ? FLAG_PRESERVE_TIMESTAMPS : 0);
KeySerializers.keys.serialize(update.keys, out, version);
writeWithVIntLength(update.condition, out);
serializeArray(update.fragments, out, version, ByteBufferUtil.byteBufferSerializer);
@ -242,17 +257,20 @@ public class TxnUpdate extends AccordUpdate
@Override
public TxnUpdate deserialize(DataInputPlus in, int version) throws IOException
{
int flags = in.readByte();
boolean preserveTimestamps = (FLAG_PRESERVE_TIMESTAMPS & flags) == 1;
Keys keys = KeySerializers.keys.deserialize(in, version);
ByteBuffer condition = readWithVIntLength(in);
ByteBuffer[] fragments = deserializeArray(in, version, ByteBufferUtil.byteBufferSerializer, ByteBuffer[]::new);
ConsistencyLevel consistencyLevel = deserializeNullable(in, version, consistencyLevelSerializer);
return new TxnUpdate(keys, fragments, condition, consistencyLevel);
return new TxnUpdate(keys, fragments, condition, consistencyLevel, preserveTimestamps);
}
@Override
public long serializedSize(TxnUpdate update, int version)
{
long size = KeySerializers.keys.serializedSize(update.keys, version);
long size = 1; // flags
size += KeySerializers.keys.serializedSize(update.keys, version);
size += serializedSizeWithVIntLength(update.condition);
size += serializedArraySize(update.fragments, version, ByteBufferUtil.byteBufferSerializer);
size += serializedNullableSize(update.cassandraCommitCL, version, consistencyLevelSerializer);

15
src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java
View File

@ -38,8 +38,8 @@ import accord.api.DataStore;
import accord.api.Key;
import accord.api.Write;
import accord.impl.AbstractSafeCommandStore;
import accord.impl.TimestampsForKeys;
import accord.impl.TimestampsForKey;
import accord.impl.TimestampsForKeys;
import accord.local.SafeCommandStore;
import accord.primitives.PartialTxn;
import accord.primitives.RoutableKey;
@ -138,10 +138,12 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
'}';
}
public AsyncChain<Void> write(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long timestamp, int nowInSeconds)
public AsyncChain<Void> write(boolean preserveTimestamps, @Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long timestamp, int nowInSeconds)
{
PartitionUpdate update = new PartitionUpdate.Builder(get(), 0).updateTimesAndPathsForAccord(cellToMaybeNewListPath, timestamp, nowInSeconds).build();
Mutation mutation = new Mutation(update);
PartitionUpdate update = get();
if (!preserveTimestamps)
update = new PartitionUpdate.Builder(get(), 0).updateTimesAndPathsForAccord(cellToMaybeNewListPath, timestamp, nowInSeconds).build();
Mutation mutation = new Mutation(update, true);
return AsyncChains.ofRunnable(Stage.MUTATION.executor(), mutation::apply);
}
@ -383,10 +385,11 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
List<AsyncChain<Void>> results = new ArrayList<>();
boolean preserveTimestamps = ((TxnUpdate)txn.update()).preserveTimestamps();
// Apply updates not specified fully by the client but built from fragments completed by data from reads.
// This occurs, for example, when an UPDATE statement uses a value assigned by a LET statement.
Function<Cell, CellPath> accordListPathSuppler = accordListPathSupplier(timestamp);
forEachWithKey((PartitionKey) key, write -> results.add(write.write(accordListPathSuppler, timestamp, nowInSeconds)));
forEachWithKey((PartitionKey) key, write -> results.add(write.write(preserveTimestamps, accordListPathSuppler, timestamp, nowInSeconds)));
if (isConditionMet)
{
@ -396,7 +399,7 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
TxnUpdate txnUpdate = (TxnUpdate) txn.update();
assert txnUpdate != null : "PartialTxn should contain an update if we're applying a write!";
List<Update> updates = txnUpdate.completeUpdatesForKey((RoutableKey) key);
updates.forEach(update -> results.add(update.write(accordListPathSuppler, timestamp, nowInSeconds)));
updates.forEach(update -> results.add(update.write(preserveTimestamps, accordListPathSuppler, timestamp, nowInSeconds)));
}
if (results.isEmpty())

6
src/java/org/apache/cassandra/service/accord/txn/UnrecoverableRepairUpdate.java
View File

@ -47,7 +47,7 @@ import org.apache.cassandra.service.reads.repair.BlockingReadRepair;
/**
* This update is used to support blocking read repair from non-transactional Cassandra reads. Cassandra creates
* a read repair mutation per node and this enables some partitiosn to be readable that would otherwise run into messages
* a read repair mutation per node and this enables some partitions to be readable that would otherwise run into messages
* size limits.
*
* This update is used during the `Execute` phase to apply the repair mutations directly in AccordInteropExecution similar
@ -58,7 +58,8 @@ import org.apache.cassandra.service.reads.repair.BlockingReadRepair;
* The state for this update is always kept in memory and is never serialized. Only the Id is propagated so the cache
* can evict the update and then load it back. We don't need to persist it or have it be recoverable because if the original
* coordinator fails to complete the transaction then the dependent Cassandra read that triggered the read repair will
* also fail and it doesn't matter if the read repair is partially applied or not applied at all.
* also fail and it doesn't matter if the read repair is partially applied or not applied at all since it doesn't propose
* new values.
*/
public class UnrecoverableRepairUpdate<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>> extends AccordUpdate
{
@ -117,6 +118,7 @@ public class UnrecoverableRepairUpdate<E extends Endpoints<E>, P extends Replica
this.keys = keys;
this.dk = dk;
this.mutations = mutations;
mutations.values().forEach(Mutation::allowPotentialTransactionConflicts);
this.writePlan = writePlan;
this.updateKey = new Key(nodeId.id, nextCounter.getAndIncrement());
}

4
src/java/org/apache/cassandra/service/consensus/TransactionalMode.java
View File

@ -113,6 +113,10 @@ public enum TransactionalMode
return consistencyLevel;
}
// TODO (required): This won't work for migration directly from none to full because there is no safe system to read from
// during the first phase (repair). Accord won't read correctly beacuse it won't honor the CL and miss non-transactional writes that haven't been repaired and non-transactional
// reads will miss all the writes being routed through Accord since they occur asynchronously. Something has to give here where either writes routed through are Accord are synchronous at CL
// or reads are routed through Accord and read at quorum as long as the range has not completed the first phase (repair).
public ConsistencyLevel readCLForStrategy(ConsistencyLevel consistencyLevel)
{
if (ignoresSuppliedConsistencyLevel)

8
src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java
View File

@ -46,6 +46,7 @@ import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.exceptions.CasWriteTimeoutException;
import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@ -173,7 +174,7 @@ public abstract class ConsensusKeyMigrationState
// TODO (desired): Better query start time
TableMigrationState tms = tableMigrationState;
repairKeyAccord(key, tms.keyspaceName, tms.tableId, tms.minMigrationEpoch(key.getToken()).getEpoch(), Dispatcher.RequestTime.forImmediateExecution(), false, isForWrite);
repairKeyAccord(key, tms.tableId, tms.minMigrationEpoch(key.getToken()).getEpoch(), Dispatcher.RequestTime.forImmediateExecution(), false, isForWrite);
}
private boolean paxosReadSatisfiedByKeyMigration()
@ -264,7 +265,6 @@ public abstract class ConsensusKeyMigrationState
* Trigger a distributed repair of Accord state for this key.
*/
static void repairKeyAccord(DecoratedKey key,
String keyspace,
TableId tableId,
long minEpoch,
Dispatcher.RequestTime requestTime,
@ -283,7 +283,9 @@ public abstract class ConsensusKeyMigrationState
// will soon be ready to execute, but only waits for the local replica to be ready
// Local will only create a transaction if it can't find an existing one to wait on
BarrierType barrierType = global ? BarrierType.global_async : BarrierType.local;
AccordService.instance().barrier(Seekables.of(new PartitionKey(tableId, key)), minEpoch, requestTime, DatabaseDescriptor.getTransactionTimeout(TimeUnit.NANOSECONDS), barrierType, isForWrite);
Seekables keysOrRanges = AccordService.instance().barrier(Seekables.of(new PartitionKey(tableId, key)), minEpoch, requestTime, DatabaseDescriptor.getTransactionTimeout(TimeUnit.NANOSECONDS), barrierType, isForWrite);
if (keysOrRanges.isEmpty())
throw new RetryOnDifferentSystemException();
// We don't save the state to the cache here. Accord will notify the agent every time a barrier happens.
}
finally

359
src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationMutationHelper.java Normal file
View File

@ -0,0 +1,359 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.consensus.migration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.coordinate.CoordinationFailed;
import accord.primitives.Keys;
import accord.primitives.Txn;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.IMutation;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.txn.TxnCondition;
import org.apache.cassandra.service.accord.txn.TxnQuery;
import org.apache.cassandra.service.accord.txn.TxnRead;
import org.apache.cassandra.service.accord.txn.TxnReferenceOperations;
import org.apache.cassandra.service.accord.txn.TxnUpdate;
import org.apache.cassandra.service.accord.txn.TxnWrite;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.Dispatcher;
import static com.google.common.base.Preconditions.checkState;
import static java.util.function.Predicate.not;
import static org.apache.cassandra.dht.Range.isInNormalizedRanges;
import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.getTableMetadata;
/**
* Applying mutations can fail with RetryOnDifferentSystemException if a
* mutation conflicts with a table and range that needs to be managed
* transactionally. This impacts mutations, logged/unlogged batches, hints,and blocking read repair.
*
* This class contains the logic needed for managing these retry loops and splitting the mutations up
*/
public class ConsensusMigrationMutationHelper
{
private static final Logger logger = LoggerFactory.getLogger(ConsensusMigrationMutationHelper.class);
private static ConsensusMigrationMutationHelper instance = new ConsensusMigrationMutationHelper();
public static ConsensusMigrationMutationHelper instance()
{
return instance;
}
@VisibleForTesting
public static void replaceInstanceForTest(ConsensusMigrationMutationHelper testInstance)
{
instance = testInstance;
}
@VisibleForTesting
public static void resetInstanceForTest()
{
instance = new ConsensusMigrationMutationHelper();
}
public ConsensusMigrationMutationHelper() {}
private static ConsistencyLevel consistencyLevelForCommit(ClusterMetadata cm, Collection<? extends IMutation> mutations, @Nullable ConsistencyLevel consistencyLevel)
{
// Null means no specific consistency behavior is required from Accord, it's functionally similar to ANY
// if you aren't reading the result back via Accord
if (consistencyLevel == null)
return null;
for (IMutation mutation : mutations)
{
for (TableId tableId : mutation.getTableIds())
{
TransactionalMode mode = getTableMetadata(cm, tableId).params.transactionalMode;
// commitCLForStrategy should return either null or the supplied consistency level
// in which case we will commit everything at that CL since Accord doesn't support per table
// commit consistency
ConsistencyLevel commitCL = mode.commitCLForStrategy(consistencyLevel);
if (commitCL != null)
return commitCL;
}
}
return null;
}
/**
* Result of splitting mutations across Accord and non-transactional boundaries
*/
public static class SplitMutations<T extends IMutation> implements SplitConsumer<T>
{
@Nullable
private List<T> accordMutations;
@Nullable
private List<T> normalMutations;
private SplitMutations() {}
public List<T> accordMutations()
{
return accordMutations;
}
public List<T> normalMutations()
{
return normalMutations;
}
@Override
public void consume(@Nullable T accordMutation, @Nullable T normalMutation, List<T> mutations, int mutationIndex)
{
// Avoid allocating an ArrayList in common single mutation single system case
if (mutations.size() == 1 && (accordMutation != null ^ normalMutation != null))
{
if (accordMutation != null)
accordMutations = mutations;
else
normalMutations = mutations;
return;
}
if (accordMutation != null)
{
if (accordMutations == null)
accordMutations = new ArrayList<>(Math.min(mutations.size(), 10));
accordMutations.add(accordMutation);
}
if (normalMutation != null)
{
if (normalMutations == null)
normalMutations = new ArrayList<>(Math.min(mutations.size(), 10));
normalMutations.add(normalMutation);
}
}
}
public interface SplitConsumer<T extends IMutation>
{
void consume(@Nullable T accordMutation, @Nullable T normalMutation, List<T> mutations, int mutationIndex);
}
public static <T extends IMutation, N> SplitMutations<T> splitMutationsIntoAccordAndNormal(ClusterMetadata cm, List<T> mutations)
{
SplitMutations<T> splitMutations = new SplitMutations<>();
splitMutationsIntoAccordAndNormal(cm, mutations, splitMutations);
return splitMutations;
}
public static <T extends IMutation> void splitMutationsIntoAccordAndNormal(ClusterMetadata cm, List<T> mutations, SplitConsumer<T> splitConsumer)
{
for (int i=0,mi=mutations.size(); i<mi; i++)
{
SplitMutation<T> splitMutation = instance.splitMutationIntoAccordAndNormal(mutations.get(i), cm);
splitConsumer.consume(splitMutation.accordMutation, splitMutation.normalMutation, mutations, i);
}
}
/**
* Result of splitting a mutation across Accord and non-transactional boundaries
*/
public static class SplitMutation<T extends IMutation>
{
@Nullable
public final T accordMutation;
@Nullable
public final T normalMutation;
public SplitMutation(@Nullable T accordMutation, @Nullable T normalMutation)
{
this.accordMutation = accordMutation;
this.normalMutation = normalMutation;
}
}
public <T extends IMutation> SplitMutation<T> splitMutationIntoAccordAndNormal(T mutation, ClusterMetadata cm)
{
if (mutation.allowsPotentialTransactionConflicts())
return new SplitMutation<>(null, mutation);
Token token = mutation.key().getToken();
Predicate<TableId> isAccordUpdate = tableId -> tokenShouldBeWrittenThroughAccord(cm, tableId, token);
T accordMutation = (T)mutation.filter(isAccordUpdate);
T normalMutation = (T)mutation.filter(not(isAccordUpdate));
for (PartitionUpdate pu : mutation.getPartitionUpdates())
checkState((accordMutation == null ? false : accordMutation.hasUpdateForTable(pu.metadata().id))
|| (normalMutation == null ? false : normalMutation.hasUpdateForTable(pu.metadata().id)),
"All partition updates should still be present after splitting");
return new SplitMutation(accordMutation, normalMutation);
}
public AsyncTxnResult mutateWithAccordAsync(ClusterMetadata cm, Mutation mutation, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
return mutateWithAccordAsync(cm, ImmutableList.of(mutation), consistencyLevel, requestTime);
}
public static AsyncTxnResult mutateWithAccordAsync(ClusterMetadata cm, Collection<? extends IMutation> mutations, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
int fragmentIndex = 0;
List<TxnWrite.Fragment> fragments = new ArrayList<>(mutations.size());
List<PartitionKey> partitionKeys = new ArrayList<>(mutations.size());
for (IMutation mutation : mutations)
{
for (PartitionUpdate update : mutation.getPartitionUpdates())
{
PartitionKey pk = PartitionKey.of(update);
partitionKeys.add(pk);
fragments.add(new TxnWrite.Fragment(PartitionKey.of(update), fragmentIndex++, update, TxnReferenceOperations.empty()));
}
}
// Potentially ignore commit consistency level if the TransactionalMode specifies full
ConsistencyLevel clForCommit = consistencyLevelForCommit(cm, mutations, consistencyLevel);
TxnUpdate update = new TxnUpdate(fragments, TxnCondition.none(), clForCommit, true);
Txn.InMemory txn = new Txn.InMemory(Keys.of(partitionKeys), TxnRead.EMPTY, TxnQuery.NONE, update);
IAccordService accordService = AccordService.instance();
try
{
return accordService.coordinateAsync(txn, consistencyLevel, requestTime);
}
catch (CoordinationFailed coordinationFailed)
{
AsyncTxnResult failure = new AsyncTxnResult(coordinationFailed.txnId());
failure.setFailure(coordinationFailed.wrap());
return failure;
}
}
public static void validateSafeToExecuteNonTransactionally(IMutation mutation) throws RetryOnDifferentSystemException
{
if (mutation.allowsPotentialTransactionConflicts())
return;
// System keyspaces are never managed by Accord
if (SchemaConstants.isSystemKeyspace(mutation.getKeyspaceName()))
return;
// Local keyspaces are never managed by Accord
if (Schema.instance.localKeyspaces().containsKeyspace(mutation.getKeyspaceName()))
return;
ClusterMetadata cm = ClusterMetadata.current();
DecoratedKey dk = mutation.key();
// Check all the partition updates and if any can't be done return an error response
// and the coordinator can retry with things correctly routed
boolean throwRetryOnDifferentSystem = false;
// Track CFS so we only mark each one once
Set<TableId> markedColumnFamilies = null;
for (PartitionUpdate pu : mutation.getPartitionUpdates())
{
TableId tableId = pu.metadata().id;
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
if (tokenShouldBeWrittenThroughAccord(cm, tableId, dk.getToken()))
{
throwRetryOnDifferentSystem = true;
if (markedColumnFamilies == null)
markedColumnFamilies = new HashSet<>();
if (markedColumnFamilies.add(tableId))
cfs.metric.mutationsRejectedOnWrongSystem.mark();
logger.debug("Rejecting mutation on wrong system to table {}.{}", cfs.keyspace.getName(), cfs.name);
Tracing.trace("Rejecting mutation on wrong system to table {}.{} token {}", cfs.keyspace.getName(), cfs.name, dk.getToken());
}
}
if (throwRetryOnDifferentSystem)
throw new RetryOnDifferentSystemException();
}
public static boolean tokenShouldBeWrittenThroughAccord(@Nonnull ClusterMetadata cm, @Nonnull TableId tableId, @Nonnull Token token)
{
TableMetadata tm = getTableMetadata(cm, tableId);
if (tm == null)
return false;
boolean transactionalModeWritesThroughAccord = tm.params.transactionalMode.writesThroughAccord;
TransactionalMigrationFromMode transactionalMigrationFromMode = tm.params.transactionalMigrationFrom;
boolean migrationFromWritesThroughAccord = transactionalMigrationFromMode.writesThroughAccord();
if (transactionalModeWritesThroughAccord && migrationFromWritesThroughAccord)
return true;
// Could be migrating or could be completely migrated, if it's migrating check if the key for this mutation
if (transactionalModeWritesThroughAccord || migrationFromWritesThroughAccord)
{
TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tm.id);
if (tms == null)
{
if (transactionalMigrationFromMode == TransactionalMigrationFromMode.none)
// There is no migration and no TMS so do what the schema says since no migration should be required
return transactionalModeWritesThroughAccord;
else
// If we are migrating from something and there is no migration state the migration hasn't begun
// so continue to do what we are migrating from does until the range is marked as migrating
return migrationFromWritesThroughAccord;
}
// This logic is driven by the fact that Paxos is not picky about how data is written since it's txn recovery
// is deterministic in the face of non-deterministic reads because consensus is agreeing on the writes that will be done to the database
// Accord agrees on what computation will produce those writes and then asynchronously executes those computations, potentially multiple times
// with different results if Accord reads non-transactionally written data that could be seen differently by different coordinators
// If the current mode writes through Accord then we should always write though Accord for ranges managed by Accord.
// Accord needs to do synchronous commit and respect the consistency level so that Accord will later be able to
// read its own writes
if (transactionalModeWritesThroughAccord)
return isInNormalizedRanges(token, tms.migratingAndMigratedRanges);
// If we are migrating from a mode that used to write to Accord then any range that isn't migrating/migrated
// should continue to write through Accord.
// It's not completely symmetrical because Paxos is able to read Accord's writes by performing a single key barrier
// and regular mutations will be able to do the same thing (needs to be added along with non-transactional reads)
// This means that migrating ranges don't need to be written through Accord because we are running Paxos now
// and not Accord. When migrating to Accord we need to do all the writes through Accord even if we aren't
// reading through Accord so that repair + Accord metadata is sufficient for Accord to be able to read
// safely and deterministically from any coordinator
if (migrationFromWritesThroughAccord)
return !isInNormalizedRanges(token, tms.migratingAndMigratedRanges);
}
return false;
}
}

24
src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationRepairResult.java
View File

@ -18,40 +18,46 @@
package org.apache.cassandra.service.consensus.migration;
import javax.annotation.Nullable;
import accord.primitives.Ranges;
import org.apache.cassandra.tcm.Epoch;
import static com.google.common.base.Preconditions.checkArgument;
public class ConsensusMigrationRepairResult
{
private static final ConsensusMigrationRepairResult INELIGIBLE = new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.ineligible, Epoch.EMPTY);
private static final ConsensusMigrationRepairResult INELIGIBLE = new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.ineligible, Epoch.EMPTY, null);
public final ConsensusMigrationRepairType type;
public final Epoch minEpoch;
@Nullable
public final Ranges barrieredRanges;
private ConsensusMigrationRepairResult(ConsensusMigrationRepairType type, Epoch minEpoch)
private ConsensusMigrationRepairResult(ConsensusMigrationRepairType type, Epoch minEpoch, @Nullable Ranges barrieredRanges)
{
this.type = type;
this.minEpoch = minEpoch;
this.barrieredRanges = barrieredRanges;
}
public static ConsensusMigrationRepairResult fromRepair(Epoch minEpoch, boolean paxosAndDataRepaired, boolean accordRepaired, boolean deadNodesExcluded)
public static ConsensusMigrationRepairResult fromRepair(Epoch minEpoch, Ranges barrieredRanges, boolean paxosAndDataRepaired, boolean accordRepaired, boolean deadNodesExcluded)
{
checkArgument((!paxosAndDataRepaired && !accordRepaired) || minEpoch.isAfter(Epoch.EMPTY), "Epoch should not be empty if Paxos and regular repairs were performed");
if (deadNodesExcluded) return INELIGIBLE;
if (paxosAndDataRepaired && accordRepaired) return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.either, minEpoch);
if (paxosAndDataRepaired) return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.paxos, minEpoch);
if (accordRepaired) return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.accord, minEpoch);
if (paxosAndDataRepaired && accordRepaired) return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.either, minEpoch, barrieredRanges);
if (paxosAndDataRepaired) return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.paxos, minEpoch, barrieredRanges);
if (accordRepaired) return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.accord, minEpoch, barrieredRanges);
return INELIGIBLE;
}
public static ConsensusMigrationRepairResult fromPaxosOnlyRepair(Epoch minEpoch, boolean deadNodesExcluded)
{
return fromRepair(minEpoch, false, false, deadNodesExcluded);
return fromRepair(minEpoch, null, false, false, deadNodesExcluded);
}
public static ConsensusMigrationRepairResult fromAccordOnlyRepair(Epoch minEpoch, boolean deadNodesExcluded)
public static ConsensusMigrationRepairResult fromAccordOnlyRepair(Epoch minEpoch, Ranges barrieredRanges, boolean deadNodesExcluded)
{
return fromRepair(minEpoch, false, true, deadNodesExcluded);
return fromRepair(minEpoch, barrieredRanges, false, true, deadNodesExcluded);
}
}

16
src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationRepairType.java
View File

@ -24,16 +24,22 @@ import org.apache.cassandra.utils.LocalizeString;
public enum ConsensusMigrationRepairType
{
ineligible(0),
paxos(1),
accord(2),
either(3);
ineligible(0, false, false),
paxos(1, false, true),
accord(2, true, false),
either(3, true, true);
public final byte value;
ConsensusMigrationRepairType(int value)
public final boolean accordMigrationEligible;
public final boolean paxosMigrationEligible;
ConsensusMigrationRepairType(int value, boolean accordMigrationEligible, boolean paxosMigrationEligible)
{
this.value = SignedBytes.checkedCast(value);
this.accordMigrationEligible = accordMigrationEligible;
this.paxosMigrationEligible = paxosMigrationEligible;
}
public static ConsensusMigrationRepairType fromString(String repairType)

61
src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java
View File

@ -18,6 +18,7 @@
package org.apache.cassandra.service.consensus.migration;
import java.util.Optional;
import javax.annotation.Nonnull;
import com.google.common.annotations.VisibleForTesting;
@ -26,9 +27,12 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.locator.EndpointsForToken;
import org.apache.cassandra.locator.ReplicaLayout;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.consensus.TransactionalMode;
@ -41,12 +45,11 @@ import org.apache.cassandra.utils.FBUtilities;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.getConsensusMigratedAt;
import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget.paxos;
import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.accord;
import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.paxosV1;
import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.paxosV2;
import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget.paxos;
/**
* Helper class to decide where to route a request that requires consensus, migrating a key if necessary
* before rerouting.
@ -84,13 +87,30 @@ public class ConsensusRequestRouter
return pickPaxos();
}
/*
* Accord never handles local tables, but if the table doesn't exist then we need to generate the correct
* InvalidRequestException.
*/
private static TableMetadata metadata(ClusterMetadata cm, String keyspace, String table)
{
KeyspaceMetadata ksm = cm.schema.getKeyspaceMetadata(keyspace);
TableMetadata tbm = ksm != null ? ksm.getTableOrViewNullable(table) : null;
Optional<KeyspaceMetadata> ksm = cm.schema.maybeGetKeyspaceMetadata(keyspace);
if (ksm.isEmpty())
{
// It's a non-distributed table which is fine, but we want to error if it doesn't exist
// We should never actually reach here unless there is a race with dropping the table
Keyspaces localKeyspaces = Schema.instance.localKeyspaces();
KeyspaceMetadata ksm2 = localKeyspaces.getNullable(keyspace);
if (ksm2 == null)
throw new InvalidRequestException("Keyspace " + keyspace + " does not exist");
// Explicitly including views in case they get used in non-distributed tables
TableMetadata tbm2 = ksm2.getTableOrViewNullable(table);
if (tbm2 == null)
throw new InvalidRequestException("Table " + keyspace + "." + table + " does not exist");
return null;
}
TableMetadata tbm = ksm.get().getTableNullable(table);
if (tbm == null)
throw new IllegalStateException("Can't route consensus request to nonexistent CFS %s.%s".format(keyspace, table));
throw new InvalidRequestException("Table " + keyspace + "." + table + " does not exist");
return tbm;
}
@ -99,18 +119,39 @@ public class ConsensusRequestRouter
{
ClusterMetadata cm = ClusterMetadata.current();
TableMetadata metadata = metadata(cm, keyspace, table);
// Non-distributed tables always take the Paxos path
if (metadata == null)
return pickPaxos();
return routeAndMaybeMigrate(cm, metadata, key, consistencyLevel, requestTime, timeoutNanos, isForWrite);
}
public ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull TableId tableId, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite)
{
ClusterMetadata cm = ClusterMetadata.current();
TableMetadata metadata = cm.schema.getTableMetadata(tableId);
TableMetadata metadata = getTableMetadata(cm, tableId);
// Non-distributed tables always take the Paxos path
if (metadata == null)
throw new IllegalStateException("Can't route consensus request for nonexistent table %s".format(tableId.toString()));
pickPaxos();
return routeAndMaybeMigrate(cm, metadata, key, consistencyLevel, requestTime, timeoutNanos, isForWrite);
}
public static TableMetadata getTableMetadata(ClusterMetadata cm, TableId tableId)
{
TableMetadata tm = cm.schema.getTableMetadata(tableId);
if (tm == null)
{
// It's a non-distributed table which is fine, but we want to error if it doesn't exist
// We should never actually reach here unless there is a race with dropping the table
Keyspaces localKeyspaces = Schema.instance.localKeyspaces();
TableMetadata tm2 = localKeyspaces.getTableOrViewNullable(tableId);
if (tm2 == null)
throw new InvalidRequestException("Table with id " + tableId + " does not exist");
return null;
}
return tm;
}
protected static boolean mayWriteThroughAccord(TableMetadata metadata)
{
return metadata.params.transactionalMode.writesThroughAccord || metadata.params.transactionalMigrationFrom.writesThroughAccord();
@ -170,7 +211,7 @@ public class ConsensusRequestRouter
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tmd.id);
if (cfs == null)
throw new IllegalStateException("Can't route consensus request to nonexistent CFS %s.%s".format(tmd.keyspace, tmd.name));
throw new InvalidRequestException("Can't route consensus request to nonexistent CFS %s.%s".format(tmd.keyspace, tmd.name));
// If it is locally replicated we can check our local migration state to see if it was already migrated
EndpointsForToken naturalReplicas = ReplicaLayout.forNonLocalStrategyTokenRead(cm, cfs.keyspace.getMetadata(), key.getToken());
boolean isLocallyReplicated = naturalReplicas.lookup(FBUtilities.getBroadcastAddressAndPort()) != null;
@ -191,7 +232,7 @@ public class ConsensusRequestRouter
// barrier transactions to accomplish the migration
// They still might need to go through the fast local path for barrier txns
// at each replica, but they won't create their own txn since we created it here
ConsensusKeyMigrationState.repairKeyAccord(key, tms.keyspaceName, tms.tableId, tms.minMigrationEpoch(key.getToken()).getEpoch(), requestTime, true, isForWrite);
ConsensusKeyMigrationState.repairKeyAccord(key, tms.tableId, tms.minMigrationEpoch(key.getToken()).getEpoch(), requestTime, true, isForWrite);
return paxosV2;
}
// Fall through for repairKeyPaxos

21
src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigration.java
View File

@ -46,6 +46,7 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.paxos.Paxos;
import org.apache.cassandra.streaming.PreviewKind;
@ -56,6 +57,7 @@ import org.apache.cassandra.tcm.serialization.MetadataSerializer;
import org.apache.cassandra.tcm.transformations.BeginConsensusMigrationForTableAndRange;
import org.apache.cassandra.tcm.transformations.MaybeFinishConsensusMigrationForTableAndRange;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
@ -97,9 +99,24 @@ public abstract class ConsensusTableMigration
if (!tms.targetProtocol.isMigratedBy(repairResult.consensusMigrationRepairResult.type))
return;
List<Range<Token>> paxosRepairedRanges = ImmutableList.of();
if (migrationResult.type.paxosMigrationEligible)
// Paxos always repairs all ranges requested by the repair although there should be nothing
// repaired in the migrated and Accord managed ranges
paxosRepairedRanges = ImmutableList.copyOf(desc.ranges);
List<Range<Token>> accordBarrieredRanges = ImmutableList.of();
if (migrationResult.type.accordMigrationEligible)
// Accord only barriers ranges it thinks it manages and repair collects which it barriered
// precisely which doesn't have to match what the entire repair covers
accordBarrieredRanges = migrationResult.barrieredRanges.stream()
.map(range -> ((TokenRange)range).toKeyspaceRange())
.collect(toImmutableList());
accordBarrieredRanges = Range.normalize(accordBarrieredRanges);
ClusterMetadataService.instance().commit(
new MaybeFinishConsensusMigrationForTableAndRange(
desc.keyspace, desc.columnFamily, ImmutableList.copyOf(desc.ranges),
desc.keyspace, desc.columnFamily, paxosRepairedRanges, accordBarrieredRanges,
migrationResult.minEpoch, migrationResult.type));
}
@ -283,7 +300,7 @@ public abstract class ConsensusTableMigration
.map(tableName -> {
TableMetadata tm = Schema.instance.getTableMetadata(keyspaceName, tableName);
if (tm == null)
throw new IllegalArgumentException("Unknown table %s.%s".format(keyspaceName, tableName));
throw new IllegalArgumentException(format("Unknown table %s.%s", keyspaceName, tableName));
return tm.id;
})
.collect(toImmutableList()));

12
src/java/org/apache/cassandra/service/paxos/Commit.java
View File

@ -23,25 +23,25 @@ package org.apache.cassandra.service.paxos;
import java.io.IOException;
import java.util.function.BiFunction;
import javax.annotation.Nullable;
import com.google.common.base.Objects;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.DeserializationHelper;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableMetadata;
import static org.apache.cassandra.db.SystemKeyspace.*;
import static org.apache.cassandra.db.SystemKeyspace.legacyPaxosTtlSec;
import static org.apache.cassandra.service.paxos.Commit.CompareResult.AFTER;
import static org.apache.cassandra.service.paxos.Commit.CompareResult.BEFORE;
import static org.apache.cassandra.service.paxos.Commit.CompareResult.IS_REPROPOSAL;
import static org.apache.cassandra.service.paxos.Commit.CompareResult.WAS_REPROPOSED_BY;
import static org.apache.cassandra.service.paxos.Commit.CompareResult.SAME;
import static org.apache.cassandra.service.paxos.Commit.CompareResult.WAS_REPROPOSED_BY;
import static org.apache.cassandra.utils.FBUtilities.nowInSeconds;
public class Commit
@ -314,7 +314,7 @@ public class Commit
public Mutation makeMutation()
{
return new Mutation(update);
return new Mutation(update, true);
}
@Override

4
src/java/org/apache/cassandra/service/reads/ReadCoordinator.java
View File

@ -71,9 +71,9 @@ public interface ReadCoordinator
void sendReadCommand(Message<ReadCommand> message, InetAddressAndPort to, RequestCallback<ReadResponse> callback);
default void notifyOfInitialContacts(EndpointsForToken fullDataRequests, EndpointsForToken transientRequests, EndpointsForToken digestRequests) {}
void sendReadRepairMutation(Message<Mutation> message, InetAddressAndPort to, RequestCallback<Object> callback);
default Mutation maybeAllowOutOfRangeMutations(Mutation m)
default boolean allowsPotentialTransactionConflicts()
{
return m;
return !isEventuallyConsistent();
}
boolean isEventuallyConsistent();
}

9
src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java
View File

@ -52,6 +52,7 @@ import org.apache.cassandra.utils.concurrent.AsyncFuture;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
@ -154,6 +155,7 @@ public class BlockingPartitionRepair
@VisibleForTesting
protected void sendRR(Message<Mutation> message, InetAddressAndPort endpoint)
{
checkArgument(message.payload.allowsPotentialTransactionConflicts() == coordinator.allowsPotentialTransactionConflicts(), "Mutation allowing transaction conflicts should match coordinator");
coordinator.sendReadRepairMutation(message, endpoint, this);
}
@ -165,8 +167,8 @@ public class BlockingPartitionRepair
for (Map.Entry<Replica, Mutation> entry: pendingRepairs.entrySet())
{
Replica destination = entry.getKey();
Preconditions.checkArgument(destination.isFull(), "Can't send repairs to transient replicas: %s", destination);
Mutation mutation = coordinator.maybeAllowOutOfRangeMutations(entry.getValue());
checkArgument(destination.isFull(), "Can't send repairs to transient replicas: %s", destination);
Mutation mutation = entry.getValue();
TableId tableId = extractUpdate(mutation).metadata().id;
Tracing.trace("Sending read-repair-mutation to {}", destination);
@ -239,7 +241,7 @@ public class BlockingPartitionRepair
if (mutation == null)
{
mutation = BlockingReadRepairs.createRepairMutation(update, repairPlan.consistencyLevel(), replica.endpoint(), true);
mutation = BlockingReadRepairs.createRepairMutation(update, repairPlan.consistencyLevel(), replica.endpoint(), true, coordinator.allowsPotentialTransactionConflicts());
versionedMutations[versionIdx] = mutation;
}
@ -250,7 +252,6 @@ public class BlockingPartitionRepair
continue;
}
mutation = coordinator.maybeAllowOutOfRangeMutations(mutation);
Tracing.trace("Sending speculative read-repair-mutation to {}", replica);
sendRR(Message.out(READ_REPAIR_REQ, mutation), replica.endpoint());
ReadRepairDiagnostics.speculatedWrite(this, replica.endpoint(), mutation);

133
src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java
View File

@ -18,7 +18,6 @@
package org.apache.cassandra.service.reads.repair;
import java.util.Collection;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
@ -34,13 +33,11 @@ import accord.primitives.Keys;
import accord.primitives.Txn;
import com.codahale.metrics.Meter;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.ReadCommand;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
import org.apache.cassandra.exceptions.ReadTimeoutException;
import org.apache.cassandra.locator.Endpoints;
@ -54,6 +51,7 @@ import org.apache.cassandra.service.accord.txn.TxnQuery;
import org.apache.cassandra.service.accord.txn.TxnRead;
import org.apache.cassandra.service.accord.txn.TxnResult;
import org.apache.cassandra.service.accord.txn.UnrecoverableRepairUpdate;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper;
import org.apache.cassandra.service.reads.ReadCoordinator;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tracing.Tracing;
@ -195,70 +193,83 @@ public class BlockingReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.Fo
@Override
public void repairPartition(DecoratedKey dk, Map<Replica, Mutation> mutations, ReplicaPlan.ForWrite writePlan)
{
TransactionalMode transactionalMode = command.metadata().params.transactionalMode;
if (coordinator.isEventuallyConsistent() && transactionalMode.blockingReadRepairThroughAccord)
// non-Accord reads only ever touch one table and key so all mutations need to be applied either transactionally
// or non-transactionally (not a mix). There is no retry loop here because read repair is relatively rare so it racing
// with changes to migrating ranges should also be pretty rare so it isn't worth the added complexity. If you were
// to add a retry loop you would need to be careful to correctly set/unset allowPotentialTransactionConflicts in the mutations
// since that is set if it is routed to Accord
//
// If this is an Accord transaction that is in interoperability mode and executing a read repair
// then we take the non-transactional path and the mutations are intercepted in ReadCoordinator.sendRepairMutation
// which will ensure the repair mutation runs in the command store thread after preceding transactions are done
ClusterMetadata cm = ClusterMetadata.current();
if (coordinator.isEventuallyConsistent() && ConsensusMigrationMutationHelper.tokenShouldBeWrittenThroughAccord(cm, command.metadata().id, dk.getToken()))
repairTransactionally(dk, mutations, writePlan);
else
repairNonTransactionally(dk, mutations, writePlan);
}
private void repairTransactionally(DecoratedKey dk, Map<Replica, Mutation> accordMutations, ForWrite writePlan)
{
checkState(coordinator.isEventuallyConsistent(), "Should only repair transactionally for an eventually consistent read coordinator");
PartitionKey partitionKey = new PartitionKey(command.metadata().id, dk);
Keys key = Keys.of(partitionKey);
// This is going create a new BlockingReadRepair inside an Accord transaction which will go down
// the !isEventuallyConsistent path and apply the repairs through Accord command stores using AccordInteropExecution
UnrecoverableRepairUpdate<E, P> repairUpdate = UnrecoverableRepairUpdate.create(AccordService.instance().nodeId(), this, key, dk, accordMutations, writePlan);
Future<TxnResult> repairFuture;
try
{
Collection<PartitionUpdate> partitionUpdates = Mutation.merge(mutations.values()).getPartitionUpdates();
checkState(partitionUpdates.size() == 1, "Expect only one PartitionUpdate");
PartitionUpdate update = partitionUpdates.iterator().next();
PartitionKey partitionKey = PartitionKey.of(update);
Keys key = Keys.of(partitionKey);
// This is going create a new BlockingReadRepair inside an Accord transaction which will go down
// the !isEventuallyConsistent path and apply the repairs through Accord command stores using AccordInteropExecution
UnrecoverableRepairUpdate<E, P> repairUpdate = UnrecoverableRepairUpdate.create(AccordService.instance().nodeId(), this, key, dk, mutations, writePlan);
Future<TxnResult> repairFuture;
try
{
Txn txn = new Txn.InMemory(Txn.Kind.Read, key, TxnRead.createNoOpRead(key), TxnQuery.NONE, repairUpdate);
repairFuture = Stage.ACCORD_MIGRATION.submit(() -> {
try
{
return AccordService.instance().coordinate(txn, ConsistencyLevel.ANY, requestTime);
}
finally
{
// If we successfully ran the repair txn then the update should definitely
// be there for us to clear which means we are sure it was there to be sent
checkNotNull(UnrecoverableRepairUpdate.removeInflightUpdate(repairUpdate.updateKey));
}
});
}
catch (Throwable t)
{
UnrecoverableRepairUpdate.removeInflightUpdate(repairUpdate.updateKey);
throw t;
}
repairs.add(new PendingPartitionRepair()
{
@Override
public boolean awaitRepairs(long remaining, TimeUnit timeUnit) throws InterruptedException, ExecutionException
Txn txn = new Txn.InMemory(Txn.Kind.Read, key, TxnRead.createNoOpRead(key), TxnQuery.NONE, repairUpdate);
repairFuture = Stage.ACCORD_MIGRATION.submit(() -> {
try
{
try
{
repairFuture.get(remaining, timeUnit);
return true;
}
catch (TimeoutException e)
{
return false;
}
return AccordService.instance().coordinate(txn, ConsistencyLevel.ANY, requestTime);
}
@Override
public ForWrite repairPlan()
finally
{
return writePlan;
// If we successfully ran the repair txn then the update should definitely
// be there for us to clear which means we are sure it was there to be sent
checkNotNull(UnrecoverableRepairUpdate.removeInflightUpdate(repairUpdate.updateKey));
}
});
}
else
catch (Throwable t)
{
BlockingPartitionRepair blockingRepair = new BlockingPartitionRepair(coordinator, dk, mutations, writePlan);
blockingRepair.sendInitialRepairs();
repairs.add(blockingRepair);
UnrecoverableRepairUpdate.removeInflightUpdate(repairUpdate.updateKey);
throw t;
}
repairs.add(new PendingPartitionRepair()
{
@Override
public boolean awaitRepairs(long remaining, TimeUnit timeUnit) throws InterruptedException, ExecutionException
{
try
{
repairFuture.get(remaining, timeUnit);
return true;
}
catch (TimeoutException e)
{
return false;
}
}
@Override
public ForWrite repairPlan()
{
return writePlan;
}
});
}
private void repairNonTransactionally(DecoratedKey dk, Map<Replica, Mutation> mutations, ForWrite writePlan)
{
BlockingPartitionRepair blockingRepair = new BlockingPartitionRepair(coordinator, dk, mutations, writePlan);
blockingRepair.sendInitialRepairs();
repairs.add(blockingRepair);
}
public void repairPartitionDirectly(ReadCoordinator readCoordinator, DecoratedKey dk, Map<Replica, Mutation> mutations, ForWrite writePlan)
@ -268,4 +279,10 @@ public class BlockingReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.Fo
delegateRR.maybeSendAdditionalWrites();
delegateRR.awaitWrites();
}
@Override
public boolean coordinatorAllowsPotentialTransactionConflicts()
{
return coordinator.allowsPotentialTransactionConflicts();
}
}

4
src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java
View File

@ -46,13 +46,13 @@ public class BlockingReadRepairs
* Create a read repair mutation from the given update, if the mutation is not larger than the maximum
* mutation size, otherwise return null. Or, if we're configured to be strict, throw an exception.
*/
public static Mutation createRepairMutation(PartitionUpdate update, ConsistencyLevel consistency, InetAddressAndPort destination, boolean suppressException)
public static Mutation createRepairMutation(PartitionUpdate update, ConsistencyLevel consistency, InetAddressAndPort destination, boolean suppressException, boolean allowPotentialTransactionConflicts)
{
if (update == null)
return null;
DecoratedKey key = update.partitionKey();
Mutation mutation = new Mutation(update);
Mutation mutation = new Mutation(update, allowPotentialTransactionConflicts);
int messagingVersion = MessagingService.instance().versions.get(destination);
try

5
src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java
View File

@ -100,4 +100,9 @@ public interface ReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRea
* Repairs a partition using the provided read coordinator
*/
void repairPartitionDirectly(ReadCoordinator coordinator, DecoratedKey partitionKey, Map<Replica, Mutation> mutations, ReplicaPlan.ForWrite writePlan);
default boolean coordinatorAllowsPotentialTransactionConflicts()
{
return false;
}
}

2
src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java
View File

@ -390,7 +390,7 @@ public class RowIteratorMergeListener<E extends Endpoints<E>>
else if (repairs[i] != null)
update = repairs[i].build();
Mutation mutation = BlockingReadRepairs.createRepairMutation(update, readPlan.consistencyLevel(), replica.endpoint(), false);
Mutation mutation = BlockingReadRepairs.createRepairMutation(update, readPlan.consistencyLevel(), replica.endpoint(), false, readRepair.coordinatorAllowsPotentialTransactionConflicts());
if (mutation == null)
continue;

44
src/java/org/apache/cassandra/tcm/transformations/MaybeFinishConsensusMigrationForTableAndRange.java
View File

@ -38,8 +38,8 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableParams;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationRepairType;
import org.apache.cassandra.service.consensus.migration.ConsensusTableMigration;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
import org.apache.cassandra.service.consensus.migration.ConsensusTableMigration;
import org.apache.cassandra.service.consensus.migration.TableMigrationState;
import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode;
import org.apache.cassandra.tcm.ClusterMetadata;
@ -56,7 +56,8 @@ import static java.lang.String.format;
import static org.apache.cassandra.dht.Range.intersects;
import static org.apache.cassandra.dht.Range.normalize;
import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget.accord;
import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget.paxos;
public class MaybeFinishConsensusMigrationForTableAndRange implements Transformation
{
@ -71,7 +72,10 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma
public final String cf;
@Nonnull
public final List<Range<Token>> repairedRanges;
public final List<Range<Token>> paxosRepairedRanges;
@Nonnull
public final List<Range<Token>> accordBarrieredRanges;
@Nonnull
public final Epoch minEpoch;
@ -81,21 +85,23 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma
public MaybeFinishConsensusMigrationForTableAndRange(@Nonnull String keyspace,
@Nonnull String cf,
@Nonnull List<Range<Token>> repairedRanges,
@Nonnull List<Range<Token>> paxosRepairedRanges,
@Nonnull List<Range<Token>> accordBarrieredRanges,
@Nonnull Epoch minEpoch,
@Nonnull ConsensusMigrationRepairType repairType)
{
checkNotNull(keyspace, "keyspace should not be null");
checkNotNull(cf, "cf should not be null");
checkNotNull(repairedRanges, "repairedRanges should not be null");
checkArgument(!repairedRanges.isEmpty(), "repairedRanges should not be empty");
checkNotNull(paxosRepairedRanges, "paxosRepairedRanges should not be null");
checkNotNull(accordBarrieredRanges, "accordBarrierRanges should not be null");
checkNotNull(minEpoch, "minEpoch should not be null");
checkArgument(minEpoch.isAfter(Epoch.EMPTY), "minEpoch should not be empty");
checkNotNull(repairType, "repairType is null");
checkArgument(repairType != ConsensusMigrationRepairType.ineligible, "Shouldn't attempt to finish migration with ineligible repair");
this.keyspace = keyspace;
this.cf = cf;
this.repairedRanges = repairedRanges;
this.paxosRepairedRanges = paxosRepairedRanges;
this.accordBarrieredRanges = accordBarrieredRanges;
this.minEpoch = minEpoch;
this.repairType = repairType;
}
@ -125,7 +131,7 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma
public Result execute(@Nonnull ClusterMetadata metadata)
{
logger.info("Completed repair {} ranges {}", repairType, repairedRanges);
logger.info("Completed repair eligibiliy '{}' paxos repaired ranges {}, accord repaired ranges {}", repairType, paxosRepairedRanges, accordBarrieredRanges);
checkNotNull(metadata, "clusterMetadata should not be null");
String ksAndCF = keyspace + "." + cf;
TableMetadata tbm = Schema.instance.getTableMetadata(keyspace, cf);
@ -138,8 +144,15 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma
return new Rejected(INVALID, format("Table %s is not currently performing consensus migration", ksAndCF));
if (!tms.targetProtocol.isMigratedBy(repairType))
return new Rejected(INVALID, format("Table %s is not currently performing consensus migration to %s and the repair was a %s repair", ksAndCF, tms.targetProtocol, repairType));
return new Rejected(INVALID, format("Table %s has a target protocol of %s and is the repair type %s is not eligible/needed to progress the migration", ksAndCF, tms.targetProtocol, repairType));
List<Range<Token>> repairedRanges;
if (tms.targetProtocol == accord)
repairedRanges = paxosRepairedRanges;
else if (tms.targetProtocol == paxos)
repairedRanges = accordBarrieredRanges;
else
throw new IllegalStateException("Unhandled migration target " + tms.targetProtocol);
List<Range<Token>> normalizedRepairedRanges = normalize(repairedRanges);
// Bail out if repair doesn't actually intersect with any migrating ranges
@ -148,6 +161,8 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma
Transformer next = metadata.transformer();
ConsensusMigrationState migrationState = metadata.consensusMigrationState.withRangesRepairedAtEpoch(tbm, normalizedRepairedRanges, minEpoch);
logger.debug("Original migration state {}");
logger.debug("New migration state {}");
next = next.with(migrationState);
// reset the migration value on the table if the migration has completed
@ -165,7 +180,8 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma
MaybeFinishConsensusMigrationForTableAndRange v = (MaybeFinishConsensusMigrationForTableAndRange)t;
out.writeUTF(v.keyspace);
out.writeUTF(v.cf);
ConsensusTableMigration.rangesSerializer.serialize(v.repairedRanges, out, version);
ConsensusTableMigration.rangesSerializer.serialize(v.paxosRepairedRanges, out, version);
ConsensusTableMigration.rangesSerializer.serialize(v.accordBarrieredRanges, out, version);
Epoch.serializer.serialize(v.minEpoch, out, version);
out.write(v.repairType.value);
}
@ -174,10 +190,11 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma
{
String keyspace = in.readUTF();
String cf = in.readUTF();
List<Range<Token>> repairedRanges = ConsensusTableMigration.rangesSerializer.deserialize(in, version);
List<Range<Token>> paxosRepairedRanges = ConsensusTableMigration.rangesSerializer.deserialize(in, version);
List<Range<Token>> accordBarrieredRanges = ConsensusTableMigration.rangesSerializer.deserialize(in, version);
Epoch minEpoch = Epoch.serializer.deserialize(in, version);
ConsensusMigrationRepairType repairType = ConsensusMigrationRepairType.fromValue(in.readByte());
return new MaybeFinishConsensusMigrationForTableAndRange(keyspace, cf, repairedRanges, minEpoch, repairType);
return new MaybeFinishConsensusMigrationForTableAndRange(keyspace, cf, paxosRepairedRanges, accordBarrieredRanges, minEpoch, repairType);
}
public long serializedSize(Transformation t, Version version)
@ -185,7 +202,8 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma
MaybeFinishConsensusMigrationForTableAndRange v = (MaybeFinishConsensusMigrationForTableAndRange)t;
return TypeSizes.sizeof(v.keyspace)
+ TypeSizes.sizeof(v.cf)
+ ConsensusTableMigration.rangesSerializer.serializedSize(v.repairedRanges, version)
+ ConsensusTableMigration.rangesSerializer.serializedSize(v.paxosRepairedRanges, version)
+ ConsensusTableMigration.rangesSerializer.serializedSize(v.accordBarrieredRanges, version)
+ Epoch.serializer.serializedSize(v.minEpoch)
+ TypeSizes.sizeof(v.repairType.value);
}

7
src/java/org/apache/cassandra/transport/Dispatcher.java
View File

@ -151,6 +151,11 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
return new RequestTime(MonotonicClock.Global.preciseTime.now());
}
public RequestTime withStartedAt(long startedAtNanos)
{
return new RequestTime(enqueuedAtNanos, startedAtNanos);
}
public long startedAtNanos()
{
return startedAtNanos;
@ -426,7 +431,7 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
connection.applyStateTransition(request.type, response.type);
return response;
}
/**
* Note: this method may be executed on the netty event loop.
*/

16
src/java/org/apache/cassandra/triggers/TriggerExecutor.java
View File

@ -19,13 +19,18 @@
package org.apache.cassandra.triggers;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.TimeUnit;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.slf4j.Logger;
@ -34,7 +39,10 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.CounterMutation;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.IMutation;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.exceptions.CassandraException;
import org.apache.cassandra.exceptions.InvalidRequestException;
@ -120,7 +128,7 @@ public class TriggerExecutor
* @throws InvalidRequestException if additional mutations were generated, but
* the initial mutations contains counter updates
*/
public Collection<Mutation> execute(Collection<? extends IMutation> mutations) throws InvalidRequestException
public List<Mutation> execute(Collection<? extends IMutation> mutations) throws InvalidRequestException
{
boolean hasCounters = false;
List<Mutation> augmentedMutations = null;
@ -156,7 +164,7 @@ public class TriggerExecutor
return mergeMutations(Iterables.concat(originalMutations, augmentedMutations));
}
private Collection<Mutation> mergeMutations(Iterable<Mutation> mutations)
private List<Mutation> mergeMutations(Iterable<Mutation> mutations)
{
ListMultimap<Pair<String, ByteBuffer>, Mutation> groupedMutations = ArrayListMultimap.create();

5
src/java/org/apache/cassandra/utils/NoSpamLogger.java
View File

@ -21,11 +21,10 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import com.google.common.annotations.VisibleForTesting;
import org.cliffc.high_scale_lib.NonBlockingHashMap;
import org.slf4j.Logger;
import com.google.common.annotations.VisibleForTesting;
import static org.apache.cassandra.utils.Clock.Global;
/**
@ -78,7 +77,7 @@ public class NoSpamLogger
this.minIntervalNanos = minIntervalNanos;
}
private boolean shouldLog(long nowNanos)
public boolean shouldLog(long nowNanos)
{
long expected = get();
return nowNanos >= expected && compareAndSet(expected, nowNanos + minIntervalNanos);

31
src/java/org/apache/cassandra/utils/Throwables.java
View File

@ -39,6 +39,8 @@ import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static com.google.common.base.Throwables.getStackTraceAsString;
public final class Throwables
{
public enum FileOpType { READ, WRITE }
@ -48,6 +50,23 @@ public final class Throwables
void perform() throws E;
}
public interface ThrowingRunnable
{
void run() throws Exception;
}
public static void runUnchecked(ThrowingRunnable runnable)
{
try
{
runnable.run();
}
catch (Exception e)
{
throwAsUncheckedException(e);
}
}
public static boolean isCausedBy(Throwable t, Predicate<Throwable> cause)
{
return cause.test(t) || (t.getCause() != null && cause.test(t.getCause()));
@ -340,4 +359,16 @@ public final class Throwables
if (Arrays.stream(causeClasses).noneMatch(c -> anyCauseMatches(err, c::isInstance)))
throw new AssertionError("The exception is not caused by any of " + Arrays.toString(causeClasses), err);
}
public static Object getStackTraceAsToString(Throwable t)
{
return new Object()
{
@Override
public String toString()
{
return getStackTraceAsString(t);
}
};
}
}

5
test/distributed/org/apache/cassandra/distributed/api/ICoordinator.java
View File

@ -18,13 +18,13 @@
package org.apache.cassandra.distributed.api;
import org.apache.cassandra.distributed.shared.FutureUtils;
import java.util.Iterator;
import java.util.UUID;
import java.util.concurrent.Future;
import java.util.function.BiConsumer;
import org.apache.cassandra.distributed.shared.FutureUtils;
// The cross-version API requires that a Coordinator can be constructed without any constructor arguments
public interface ICoordinator
{
@ -82,6 +82,7 @@ public interface ICoordinator
}
Future<SimpleQueryResult> asyncExecuteWithTracingWithResult(UUID sessionId, String query, ConsistencyLevel consistencyLevel, Object... boundValues);
Future<SimpleQueryResult> asyncExecuteWithResult(String query, ConsistencyLevel consistencyLevel, Object... boundValues);
default Object[][] executeWithTracing(UUID sessionId, String query, ConsistencyLevel consistencyLevel, Object... boundValues)
{

6
test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java
View File

@ -102,6 +102,12 @@ public class Coordinator implements ICoordinator
}).call();
}
@Override
public Future<SimpleQueryResult> asyncExecuteWithResult(String query, ConsistencyLevel consistencyLevelOrigin, Object... boundValues)
{
return instance.async(() -> unsafeExecuteInternal(query, consistencyLevelOrigin, boundValues)).call();
}
public static org.apache.cassandra.db.ConsistencyLevel toCassandraCL(ConsistencyLevel cl)
{
try

101
test/distributed/org/apache/cassandra/distributed/impl/TestChangeListener.java
View File

@ -18,10 +18,8 @@
package org.apache.cassandra.distributed.impl;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.function.Predicate;
import java.util.NavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -30,7 +28,9 @@ import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.listeners.ChangeListener;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
import org.apache.cassandra.utils.concurrent.WaitQueue;
import org.apache.cassandra.utils.concurrent.WaitQueue.Signal;
public class TestChangeListener implements ChangeListener
{
@ -43,77 +43,68 @@ public class TestChangeListener implements ChangeListener
ClusterMetadataService.instance().log().addListener(instance);
}
private final List<Predicate<Epoch>> preCommitPredicates = new ArrayList<>();
private final List<Predicate<Epoch>> postCommitPredicates = new ArrayList<>();
NavigableMap<Epoch, CommitBarrier> preCommitBarriers = new ConcurrentSkipListMap<>();
NavigableMap<Epoch, CommitBarrier> postCommitBarriers = new ConcurrentSkipListMap<>();
private final WaitQueue waiters = WaitQueue.newWaitQueue();
private class CommitBarrier
{
private final CountDownLatch waiting = CountDownLatch.newCountDownLatch(1);
private final Runnable onPaused;
private final String desc;
private CommitBarrier(Runnable onPaused, String desc)
{
this.onPaused = onPaused;
this.desc = desc;
}
private void await()
{
logger.debug("Notifying paused: {}", desc);
Signal s = waiters.register();
waiting.decrement();
onPaused.run();
s.awaitUninterruptibly();
logger.debug("Unpaused: {}", desc);
}
}
@Override
public void notifyPreCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot)
{
Iterator<Predicate<Epoch>> iter = preCommitPredicates.iterator();
while (iter.hasNext())
{
if (iter.next().test(next.epoch))
{
logger.debug("Epoch matches pre-commit predicate, pausing");
pause();
iter.remove();
}
}
CommitBarrier commitBarrier = preCommitBarriers.remove(next.epoch);
if (commitBarrier != null)
commitBarrier.await();
}
@Override
public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot)
{
Iterator<Predicate<Epoch>> iter = postCommitPredicates.iterator();
while (iter.hasNext())
{
if (iter.next().test(next.epoch))
{
logger.debug("Epoch matches post-commit predicate, pausing");
pause();
iter.remove();
}
}
CommitBarrier commitBarrier = postCommitBarriers.remove(next.epoch);
if (commitBarrier != null)
commitBarrier.await();
}
public void pauseBefore(Epoch epoch, Runnable onMatch)
public void pauseBefore(Epoch epoch, Runnable onPaused)
{
logger.debug("Requesting pause before enacting {}", epoch);
preCommitPredicates.add((e) -> {
if (e.is(epoch))
{
onMatch.run();
return true;
}
return false;
});
preCommitBarriers.put(epoch, new CommitBarrier(onPaused, "pre-commit " + epoch));
}
public void pauseAfter(Epoch epoch, Runnable onMatch)
public void pauseAfter(Epoch epoch, Runnable onPaused)
{
logger.debug("Requesting pause after enacting {}", epoch);
postCommitPredicates.add((e) -> {
if (e.is(epoch))
{
onMatch.run();
return true;
}
return false;
});
}
public void pause()
{
WaitQueue.Signal signal = waiters.register();
logger.debug("Log follower is paused, waiting...");
signal.awaitUninterruptibly();
logger.debug("Resumed log follower...");
postCommitBarriers.put(epoch, new CommitBarrier(onPaused, "post-commit " + epoch));
}
public void unpause()
{
logger.debug("Unpausing log follower");
logger.info("Unpausing all precommit and post commit barriers");
waiters.signalAll();
}
public void clearAndUnpause()
{
preCommitBarriers.clear();
postCommitBarriers.clear();
waiters.signalAll();
}
}

5
test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
View File

@ -600,6 +600,11 @@ public class ClusterUtils
instance.runOnInstance(() -> TestChangeListener.instance.unpause());
}
public static void clearAndUnpause(IInvokableInstance instance)
{
instance.runOnInstance(() -> TestChangeListener.instance.clearAndUnpause());
}
public static boolean isMigrating(IInvokableInstance instance)
{
return instance.callOnInstance(() -> ClusterMetadataService.instance().isMigrating());

4
test/distributed/org/apache/cassandra/distributed/test/CASAddTest.java
View File

@ -72,13 +72,13 @@ public class CASAddTest extends TestBaseImpl
assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.SERIAL), row(1, null, null));
// this section is testing current limitations... if they start to fail due to the limitations going away... update this test to include those cases
Assertions.assertThatThrownBy(() -> cluster.coordinator(1).execute(batch(
Assertions.assertThatThrownBy(() -> cluster.coordinator(1).execute(unloggedBatch(
"INSERT INTO " + KEYSPACE + ".tbl (pk, a, b) VALUES (1, 0, '') IF NOT EXISTS",
"UPDATE " + KEYSPACE + ".tbl SET a = a + 1, b = b + 'success' WHERE pk = 1 IF EXISTS"
), ConsistencyLevel.QUORUM))
.is(AssertionUtils.is(InvalidRequestException.class))
.hasMessage("Cannot mix IF EXISTS and IF NOT EXISTS conditions for the same row");
Assertions.assertThatThrownBy(() -> cluster.coordinator(1).execute(batch(
Assertions.assertThatThrownBy(() -> cluster.coordinator(1).execute(unloggedBatch(
"INSERT INTO " + KEYSPACE + ".tbl (pk, a, b) VALUES (1, 0, '') IF NOT EXISTS",
"UPDATE " + KEYSPACE + ".tbl SET a = a + 1, b = b + 'success' WHERE pk = 1"

12
test/distributed/org/apache/cassandra/distributed/test/CASMultiDCTest.java
View File

@ -21,6 +21,7 @@ package org.apache.cassandra.distributed.test;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
@ -36,7 +37,10 @@ import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.service.paxos.PaxosCommit;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.*;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.LOCAL_QUORUM;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.LOCAL_SERIAL;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL;
public class CASMultiDCTest
{
@ -75,6 +79,12 @@ public class CASMultiDCTest
}));
}
@AfterClass
public static void afterClass() throws Throwable
{
CLUSTER.close();
}
@Before
public void setUp()
{

3
test/distributed/org/apache/cassandra/distributed/test/MessageFiltersTest.java
View File

@ -275,7 +275,8 @@ public class MessageFiltersTest extends TestBaseImpl
try (Cluster cluster = init(builder().withNodes(3)
.withConfig(config -> config.with(GOSSIP)
.with(NETWORK)
.set("hinted_handoff_enabled", true))
.set("hinted_handoff_enabled", true)
.set("accord.enabled", false))
.start()))
{
cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k int PRIMARY KEY, v int)"));

3
test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java
View File

@ -146,7 +146,6 @@ public class QueriesTableTest extends TestBaseImpl
// Issue a read to unblock the read generated by the original CAS operation:
SESSION.executeAsync("SELECT * FROM " + KEYSPACE + ".cas_tbl WHERE k = 0");
waitForQueriesToFinish();
}
@ -174,7 +173,7 @@ public class QueriesTableTest extends TestBaseImpl
@Test
public void shouldExposeTransaction() throws Throwable
{
SHARED_CLUSTER.schemaChange("CREATE TABLE " + KEYSPACE + ".accord_tbl (k int primary key, v int) WITH transactional_mode='full'");
SHARED_CLUSTER.schemaChange("CREATE TABLE " + KEYSPACE + ".accord_tbl (k int primary key, v int) WITH transactional_mode='mixed_reads'");
// Disable recovery to make sure only one local read occurs:
for (IInvokableInstance instance : SHARED_CLUSTER)

33
test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
View File

@ -30,44 +30,45 @@ import java.util.stream.Stream;
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Assert;
import org.apache.cassandra.Util;
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.EndpointsForToken;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.ReplicaLayout;
import org.apache.cassandra.locator.ReplicaUtils;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.Throwables;
import org.junit.Test;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
import org.apache.cassandra.Util;
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.ReadCommand;
import org.apache.cassandra.db.ReadExecutionController;
import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.StatsComponent;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.EndpointsForToken;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.ReplicaLayout;
import org.apache.cassandra.locator.ReplicaUtils;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageProxy.LocalReadRunnable;
import org.apache.cassandra.service.reads.ReadCoordinator;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.DiagnosticSnapshotService;
import org.apache.cassandra.utils.Throwables;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
@ -440,7 +441,7 @@ public class RepairDigestTrackingTest extends TestBaseImpl
.load(classLoader, ClassLoadingStrategy.Default.INJECTION);
new ByteBuddy().rebase(ReplicaLayout.class)
.method(named("forTokenReadSorted").and(takesArguments(ClusterMetadata.class, Keyspace.class, AbstractReplicationStrategy.class, Token.class)))
.method(named("forTokenReadSorted").and(takesArguments(ClusterMetadata.class, Keyspace.class, AbstractReplicationStrategy.class, TableId.class, Token.class, ReadCoordinator.class)))
.intercept(MethodDelegation.to(BBHelper.class))
.make()
.load(classLoader, ClassLoadingStrategy.Default.INJECTION);
@ -475,7 +476,7 @@ public class RepairDigestTrackingTest extends TestBaseImpl
}
@SuppressWarnings({ "unused" })
public static ReplicaLayout.ForTokenRead forTokenReadSorted(ClusterMetadata metadata, Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, Token token)
public static ReplicaLayout.ForTokenRead forTokenReadSorted(ClusterMetadata metadata, Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, TableId tableId, Token token, org.apache.cassandra.service.reads.ReadCoordinator coordinator)
{
try
{

3
test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java
View File

@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.IntStream;
import com.google.common.collect.ImmutableList;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@ -90,7 +91,7 @@ public class ShortReadProtectionTest extends TestBaseImpl
public static Collection<Object[]> data()
{
List<Object[]> result = new ArrayList<>();
for (TransactionalMode mode : TransactionalMode.values())
for (TransactionalMode mode : ImmutableList.of(TransactionalMode.mixed_reads, TransactionalMode.off))
for (ConsistencyLevel readConsistencyLevel : Arrays.asList(ALL, QUORUM, SERIAL))
for (boolean flush : BOOLEANS)
for (boolean paging : BOOLEANS)

44
test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
View File

@ -33,6 +33,8 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListenableFutureTask;
import org.junit.After;
import org.junit.BeforeClass;
@ -58,8 +60,11 @@ import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.ICoordinator;
import org.apache.cassandra.distributed.api.IInstance;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.shared.DistributedTestBase;
import org.apache.cassandra.service.accord.AccordStateCache;
@ -69,6 +74,7 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING;
import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_GC_INSPECTOR;
import static org.apache.cassandra.distributed.action.GossipHelper.withProperty;
import static org.assertj.core.api.Assertions.fail;
// checkstyle: suppress below 'blockSystemPropertyUsage'
public class TestBaseImpl extends DistributedTestBase
@ -140,10 +146,17 @@ public class TestBaseImpl extends DistributedTestBase
return tupleType.pack(bbs, ByteBufferAccessor.instance);
}
public static String batch(String... queries)
public static String unloggedBatch(String... queries)
{
return batch(false, queries);
}
public static String batch(boolean logged, String... queries)
{
StringBuilder sb = new StringBuilder();
sb.append("BEGIN UNLOGGED BATCH\n");
sb.append("BEGIN ");
sb.append(logged ? "" : "UNLOGGED ");
sb.append("BATCH\n");
for (String q : queries)
sb.append(q).append(";\n");
sb.append("APPLY BATCH;");
@ -253,4 +266,31 @@ public class TestBaseImpl extends DistributedTestBase
for (int i = 1; i < cluster.size() + 1; i++)
cluster.get(i).nodetool("disableautocompaction", keyspace, table);
}
public static String nodetool(IInstance instance, String... commandAndArgs)
{
NodeToolResult nodetoolResult = instance.nodetoolResult(commandAndArgs);
if (!nodetoolResult.getStdout().isEmpty())
System.out.println(nodetoolResult.getStdout());
if (!nodetoolResult.getStderr().isEmpty())
System.err.println(nodetoolResult.getStderr());
if (nodetoolResult.getError() != null)
fail("Failed nodetool " + Arrays.asList(commandAndArgs), nodetoolResult.getError());
// TODO why does standard out end up in stderr in nodetool?
return nodetoolResult.getStdout();
}
public static String nodetool(ICoordinator coordinator, String... commandAndArgs)
{
return nodetool(coordinator.instance(), commandAndArgs);
}
public static ListenableFuture<String> nodetoolAsync(ICoordinator coordinator, String... commandAndArgs)
{
ListenableFutureTask<String> task = ListenableFutureTask.create(() -> nodetool(coordinator, commandAndArgs));
Thread asyncThread = new Thread(task, "NodeTool: " + Arrays.asList(commandAndArgs));
asyncThread.setDaemon(true);
asyncThread.start();
return task;
}
}

942
test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java
View File

File diff suppressed because it is too large Load Diff

31
test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java
View File

@ -36,12 +36,12 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.local.cfk.CommandsForKey;
import accord.impl.SimpleProgressLog;
import accord.local.Node;
import accord.local.PreLoadContext;
import accord.local.SafeCommand;
import accord.local.Status;
import accord.local.cfk.CommandsForKey;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
@ -51,6 +51,7 @@ import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.gms.FailureDetector;
@ -147,13 +148,15 @@ public class AccordIncrementalRepairTest extends AccordTestBase
public static void setupClass() throws Throwable
{
CassandraRelevantProperties.ACCORD_AGENT_CLASS.setString(BarrierRecordingAgent.class.getName());
// setupCluster(opt -> opt.withConfig(conf -> conf.with(Feature.NETWORK, Feature.GOSSIP)), 3);
setupCluster(opt -> opt, 3);
setupCluster(opt -> opt.withConfig(conf -> conf.with(Feature.NETWORK, Feature.GOSSIP)), 3);
// setupCluster(opt -> opt, 3);
}
@After
public void tearDown()
{
for (IInvokableInstance instance : SHARED_CLUSTER)
instance.runOnInstance(() -> SimpleProgressLog.PAUSE_FOR_TEST = false);
SHARED_CLUSTER.filters().reset();
}
@ -248,16 +251,16 @@ public class AccordIncrementalRepairTest extends AccordTestBase
@Test
public void txnRepairTest() throws Throwable
{
SHARED_CLUSTER.schemaChange(format("CREATE TABLE %s.%s (k int primary key, v int) WITH transactional_mode='full' AND fast_path={'size':2};", KEYSPACE, tableName));
SHARED_CLUSTER.schemaChange(format("CREATE TABLE %s.%s (k int primary key, v int) WITH transactional_mode='full' AND fast_path={'size':2};", KEYSPACE, accordTableName));
final String keyspace = KEYSPACE;
final String table = tableName;
final String table = accordTableName;
SHARED_CLUSTER.filters().allVerbs().to(3).drop();
awaitEndpointDown(SHARED_CLUSTER.get(1), SHARED_CLUSTER.get(3));
executeWithRetry(SHARED_CLUSTER, format("BEGIN TRANSACTION\n" +
"INSERT INTO %s (k, v) VALUES (1, 1);\n" +
"COMMIT TRANSACTION", qualifiedTableName));
"COMMIT TRANSACTION", qualifiedAccordTableName));
SHARED_CLUSTER.get(1, 2).forEach(instance -> instance.runOnInstance(() -> {
TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table);
@ -291,7 +294,7 @@ public class AccordIncrementalRepairTest extends AccordTestBase
});
SHARED_CLUSTER.filters().reset();
awaitEndpointUp(SHARED_CLUSTER.get(1), SHARED_CLUSTER.get(3));
SHARED_CLUSTER.get(1).nodetool("repair", KEYSPACE);
nodetool(SHARED_CLUSTER.get(1), "repair", KEYSPACE);
SHARED_CLUSTER.forEach(instance -> {
instance.runOnInstance(() -> {
@ -307,9 +310,9 @@ public class AccordIncrementalRepairTest extends AccordTestBase
private void testSingleNodeWrite(TransactionalMode mode)
{
SHARED_CLUSTER.schemaChange(format("CREATE TABLE %s.%s (k int primary key, v int) WITH transactional_mode='%s';", KEYSPACE, tableName, mode));
SHARED_CLUSTER.schemaChange(format("CREATE TABLE %s.%s (k int primary key, v int) WITH transactional_mode='%s';", KEYSPACE, accordTableName, mode));
final String keyspace = KEYSPACE;
final String table = tableName;
final String table = accordTableName;
SHARED_CLUSTER.get(3).runOnInstance(() -> {
QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (1, 2);", keyspace, table));
@ -344,7 +347,7 @@ public class AccordIncrementalRepairTest extends AccordTestBase
agent().reset();
}));
SHARED_CLUSTER.get(1).nodetool("repair", KEYSPACE);
nodetool(SHARED_CLUSTER.get(1), "repair", KEYSPACE);
SHARED_CLUSTER.forEach(instance -> instance.runOnInstance(() -> {
Assert.assertFalse( agent().executedBarriers().isEmpty());
ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
@ -382,9 +385,9 @@ public class AccordIncrementalRepairTest extends AccordTestBase
@Test
public void onlyAccordTest()
{
SHARED_CLUSTER.schemaChange(format("CREATE TABLE %s.%s (k int primary key, v int) WITH transactional_mode='full' AND fast_path={'size':2};", KEYSPACE, tableName));
SHARED_CLUSTER.schemaChange(format("CREATE TABLE %s.%s (k int primary key, v int) WITH transactional_mode='full' AND fast_path={'size':2};", KEYSPACE, accordTableName));
final String keyspace = KEYSPACE;
final String table = tableName;
final String table = accordTableName;
SHARED_CLUSTER.filters().allVerbs().to(3).drop();
awaitEndpointDown(SHARED_CLUSTER.get(1), SHARED_CLUSTER.get(3));
@ -392,7 +395,7 @@ public class AccordIncrementalRepairTest extends AccordTestBase
executeWithRetry(SHARED_CLUSTER, format("BEGIN TRANSACTION\n" +
"INSERT INTO %s (k, v) VALUES (1, 1);\n" +
"COMMIT TRANSACTION", qualifiedTableName));
"COMMIT TRANSACTION", qualifiedAccordTableName));
SHARED_CLUSTER.get(1, 2).forEach(instance -> instance.runOnInstance(() -> {
TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table);
@ -403,7 +406,7 @@ public class AccordIncrementalRepairTest extends AccordTestBase
SHARED_CLUSTER.filters().reset();
awaitEndpointUp(SHARED_CLUSTER.get(1), SHARED_CLUSTER.get(3));
SHARED_CLUSTER.get(1).nodetool("repair", "--accord-only", KEYSPACE);
nodetool(SHARED_CLUSTER.get(1), "repair", "--accord-only", KEYSPACE);
SHARED_CLUSTER.forEach(instance -> {
logger().info("checking instance {}", instance.broadcastAddress());

20
test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java
View File

@ -59,10 +59,10 @@ public class AccordIntegrationTest extends AccordTestBase
IMessageFilters.Filter lostCommit = cluster.filters().verbs(Verb.ACCORD_COMMIT_REQ.id).to(2).drop();
String query = "BEGIN TRANSACTION\n" +
" LET row1 = (SELECT v FROM " + qualifiedTableName + " WHERE k=0 AND c=0);\n" +
" LET row1 = (SELECT v FROM " + qualifiedAccordTableName + " WHERE k=0 AND c=0);\n" +
" SELECT row1.v;\n" +
" IF row1 IS NULL THEN\n" +
" INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, 0, 1);\n" +
" INSERT INTO " + qualifiedAccordTableName + " (k, c, v) VALUES (0, 0, 1);\n" +
" END IF\n" +
"COMMIT TRANSACTION";
// row1.v shouldn't have existed when the txn's SELECT was executed
@ -73,24 +73,24 @@ public class AccordIntegrationTest extends AccordTestBase
// Querying again should trigger recovery...
query = "BEGIN TRANSACTION\n" +
" LET row1 = (SELECT v FROM " + qualifiedTableName + " WHERE k=0 AND c=0);\n" +
" LET row1 = (SELECT v FROM " + qualifiedAccordTableName + " WHERE k=0 AND c=0);\n" +
" SELECT row1.v;\n" +
" IF row1.v = 1 THEN\n" +
" UPDATE " + qualifiedTableName + " SET v=2 WHERE k = 0 AND c = 0;\n" +
" UPDATE " + qualifiedAccordTableName + " SET v=2 WHERE k = 0 AND c = 0;\n" +
" END IF\n" +
"COMMIT TRANSACTION";
assertRowEqualsWithPreemptedRetry(cluster, new Object[] { 1 }, query);
String check = "BEGIN TRANSACTION\n" +
" SELECT * FROM " + qualifiedTableName + " WHERE k = ? AND c = ?;\n" +
" SELECT * FROM " + qualifiedAccordTableName + " WHERE k = ? AND c = ?;\n" +
"COMMIT TRANSACTION";
assertRowEqualsWithPreemptedRetry(cluster, new Object[] {0, 0, 2}, check, 0, 0);
query = "BEGIN TRANSACTION\n" +
" LET row1 = (SELECT v FROM " + qualifiedTableName + " WHERE k=0 AND c=0);\n" +
" LET row1 = (SELECT v FROM " + qualifiedAccordTableName + " WHERE k=0 AND c=0);\n" +
" SELECT row1.v;\n" +
" IF row1 IS NULL THEN\n" +
" INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, 0, 3);\n" +
" INSERT INTO " + qualifiedAccordTableName + " (k, c, v) VALUES (0, 0, 3);\n" +
" END IF\n" +
"COMMIT TRANSACTION";
assertRowEqualsWithPreemptedRetry(cluster, new Object[] { 2 }, query);
@ -113,16 +113,16 @@ public class AccordIntegrationTest extends AccordTestBase
})).drop();
String query = "BEGIN TRANSACTION\n" +
" LET row1 = (SELECT * FROM " + qualifiedTableName + " WHERE k = 0 AND c = 0);\n" +
" LET row1 = (SELECT * FROM " + qualifiedAccordTableName + " WHERE k = 0 AND c = 0);\n" +
" SELECT row1.v;\n" +
" IF row1 IS NULL THEN\n" +
" INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, 0, 1);\n" +
" INSERT INTO " + qualifiedAccordTableName + " (k, c, v) VALUES (0, 0, 1);\n" +
" END IF\n" +
"COMMIT TRANSACTION";
assertRowEqualsWithPreemptedRetry(cluster, new Object[] { null }, query);
String check = "BEGIN TRANSACTION\n" +
" SELECT * FROM " + qualifiedTableName + " WHERE k = ? AND c = ?;\n" +
" SELECT * FROM " + qualifiedAccordTableName + " WHERE k = ? AND c = ?;\n" +
"COMMIT TRANSACTION";
assertRowEqualsWithPreemptedRetry(cluster, new Object[] { 0, 0, 1 }, check, 0, 0);
});

12
test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteroperabilityTest.java
View File

@ -48,15 +48,15 @@ public class AccordInteroperabilityTest extends AccordTestBase
@Test
public void testSerialReadDescending() throws Throwable
{
test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY(k, c)) WITH transactional_mode='full'",
test("CREATE TABLE " + qualifiedAccordTableName + " (k int, c int, v int, PRIMARY KEY(k, c)) WITH transactional_mode='full'",
cluster -> {
ICoordinator coordinator = cluster.coordinator(1);
for (int i = 1; i <= 10; i++)
coordinator.execute("INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, ?, ?) USING TIMESTAMP 0;", ConsistencyLevel.ALL, i, i * 10);
assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedTableName + " WHERE k=0 ORDER BY c DESC LIMIT 1", AssertUtils.row(10, 100));
assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedTableName + " WHERE k=0 ORDER BY c DESC LIMIT 2", AssertUtils.row(10, 100), AssertUtils.row(9, 90));
assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedTableName + " WHERE k=0 ORDER BY c DESC LIMIT 3", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80));
assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedTableName + " WHERE k=0 ORDER BY c DESC LIMIT 4", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80), AssertUtils.row(7, 70));
coordinator.execute("INSERT INTO " + qualifiedAccordTableName + " (k, c, v) VALUES (0, ?, ?) USING TIMESTAMP 0;", ConsistencyLevel.ALL, i, i * 10);
assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedAccordTableName + " WHERE k=0 ORDER BY c DESC LIMIT 1", AssertUtils.row(10, 100));
assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedAccordTableName + " WHERE k=0 ORDER BY c DESC LIMIT 2", AssertUtils.row(10, 100), AssertUtils.row(9, 90));
assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedAccordTableName + " WHERE k=0 ORDER BY c DESC LIMIT 3", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80));
assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedAccordTableName + " WHERE k=0 ORDER BY c DESC LIMIT 4", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80), AssertUtils.row(7, 70));
}
);
}

10
test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java
View File

@ -64,7 +64,7 @@ public class AccordLoadTest extends AccordTestBase
@Test
public void testLoad() throws Exception
{
test("CREATE TABLE " + qualifiedTableName + " (k int, v int, PRIMARY KEY(k)) WITH transactional_mode = 'full'",
test("CREATE TABLE " + qualifiedAccordTableName + " (k int, v int, PRIMARY KEY(k)) WITH transactional_mode = 'full'",
cluster -> {
final ConcurrentHashMap<Verb, AtomicInteger> verbs = new ConcurrentHashMap<>();
@ -86,7 +86,7 @@ public class AccordLoadTest extends AccordTestBase
final float readChance = 0.33f;
long nextRepairAt = repairInterval;
for (int i = 1; i <= keyCount; i++)
coordinator.execute("INSERT INTO " + qualifiedTableName + " (k, v) VALUES (0, 0) USING TIMESTAMP 0;", ConsistencyLevel.ALL, i);
coordinator.execute("INSERT INTO " + qualifiedAccordTableName + " (k, v) VALUES (0, 0) USING TIMESTAMP 0;", ConsistencyLevel.ALL, i);
Random random = new Random();
// CopyOnWriteArrayList<Throwable> exceptions = new CopyOnWriteArrayList<>();
@ -109,7 +109,7 @@ public class AccordLoadTest extends AccordTestBase
inFlight.release();
if (fail == null) histogram.add(NANOSECONDS.toMicros(System.nanoTime() - commandStart));
// else exceptions.add(fail);
}, "SELECT * FROM " + qualifiedTableName + " WHERE k = ?;", ConsistencyLevel.SERIAL, random.nextInt(keyCount));
}, "SELECT * FROM " + qualifiedAccordTableName + " WHERE k = ?;", ConsistencyLevel.SERIAL, random.nextInt(keyCount));
}
else
{
@ -117,7 +117,7 @@ public class AccordLoadTest extends AccordTestBase
inFlight.release();
if (fail == null) histogram.add(NANOSECONDS.toMicros(System.nanoTime() - commandStart));
// else exceptions.add(fail);
}, "UPDATE " + qualifiedTableName + " SET v += 1 WHERE k = ? IF EXISTS;", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, random.nextInt(keyCount));
}, "UPDATE " + qualifiedAccordTableName + " SET v += 1 WHERE k = ? IF EXISTS;", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, random.nextInt(keyCount));
}
}
@ -125,7 +125,7 @@ public class AccordLoadTest extends AccordTestBase
{
nextRepairAt += repairInterval;
System.out.println("repairing...");
cluster.coordinator(1).instance().nodetool("repair", qualifiedTableName);
cluster.coordinator(1).instance().nodetool("repair", qualifiedAccordTableName);
}
final Date date = new Date();

8
test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java
View File

@ -73,16 +73,16 @@ public class AccordMetricsTest extends AccordTestBase
String writeCql()
{
return "BEGIN TRANSACTION\n" +
" LET val = (SELECT v FROM " + qualifiedTableName + " WHERE k=? AND c=?);\n" +
" LET val = (SELECT v FROM " + qualifiedAccordTableName + " WHERE k=? AND c=?);\n" +
" SELECT val.v;\n" +
" UPDATE " + qualifiedTableName + " SET v = v + 1 WHERE k=? AND c=?;\n" +
" UPDATE " + qualifiedAccordTableName + " SET v = v + 1 WHERE k=? AND c=?;\n" +
"COMMIT TRANSACTION";
}
String readCql()
{
return "BEGIN TRANSACTION\n" +
" LET val = (SELECT v FROM " + qualifiedTableName + " WHERE k=? AND c=?);\n" +
" LET val = (SELECT v FROM " + qualifiedAccordTableName + " WHERE k=? AND c=?);\n" +
" SELECT val.v;\n" +
"COMMIT TRANSACTION";
}
@ -93,7 +93,7 @@ public class AccordMetricsTest extends AccordTestBase
public void beforeTest()
{
SHARED_CLUSTER.filters().reset();
SHARED_CLUSTER.schemaChange("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH " + TransactionalMode.full.asCqlParam());
SHARED_CLUSTER.schemaChange("CREATE TABLE " + qualifiedAccordTableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH " + TransactionalMode.full.asCqlParam());
}
@Test

771
test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationRaceTestBase.java Normal file
View File

@ -0,0 +1,771 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.accord;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import com.google.common.base.Stopwatch;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListenableFutureTask;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.RoutingKey;
import accord.messages.PreAccept;
import accord.primitives.PartialKeyRoute;
import accord.primitives.PartialRoute;
import accord.primitives.Routable.Domain;
import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.Util;
import org.apache.cassandra.batchlog.BatchlogManager;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.Config.PaxosVariant;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.ICoordinator;
import org.apache.cassandra.distributed.api.IInstance;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IMessage;
import org.apache.cassandra.distributed.api.IMessageSink;
import org.apache.cassandra.distributed.api.QueryResults;
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.distributed.impl.Instance;
import org.apache.cassandra.distributed.impl.TestChangeListener;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.exceptions.CoordinatorBehindException;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.hints.HintsService;
import org.apache.cassandra.metrics.AccordClientRequestMetrics;
import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
import org.apache.cassandra.metrics.HintsServiceMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState;
import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Promise;
import org.eclipse.jetty.util.ConcurrentHashSet;
import static java.lang.String.format;
import static org.apache.cassandra.Util.expectException;
import static org.apache.cassandra.Util.spinAssertEquals;
import static org.apache.cassandra.config.CassandraRelevantProperties.HINT_DISPATCH_INTERVAL_MS;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
import static org.apache.cassandra.distributed.shared.ClusterUtils.getNextEpoch;
import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseAfterEnacting;
import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseBeforeEnacting;
import static org.apache.cassandra.distributed.shared.ClusterUtils.unpauseEnactment;
import static org.apache.cassandra.distributed.test.accord.AccordMigrationRaceTestBase.Scenario.BATCHLOG_FAILED_ROUTING_THEN_HINT;
import static org.apache.cassandra.distributed.test.accord.AccordMigrationRaceTestBase.Scenario.BATCHLOG_FAILED_TIMEOUT_THEN_HINT;
import static org.apache.cassandra.distributed.test.accord.AccordMigrationRaceTestBase.Scenario.BATCHLOG_SUCCESSFUL_ROUTING;
import static org.apache.cassandra.distributed.test.accord.AccordMigrationRaceTestBase.Scenario.HINT;
import static org.apache.cassandra.distributed.test.accord.AccordMigrationRaceTestBase.Scenario.MUTATION;
import static org.apache.cassandra.distributed.util.QueryResultUtil.assertThat;
import static org.apache.cassandra.exceptions.RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM;
import static org.apache.cassandra.utils.Throwables.runUnchecked;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/*
* Test that non-transactional write operations such as regular mutations, batch log, and hints
* all detect when a migration is in progress, and then retry on the correct system.
*/
public abstract class AccordMigrationRaceTestBase extends AccordTestBase
{
private static final Logger logger = LoggerFactory.getLogger(AccordMigrationRaceTestBase.class);
private static final int CLUSTERING_VALUE = 1;
private static final String TABLE_FMT = "CREATE TABLE %s (id int, c int, v int, PRIMARY KEY ((id), c));";
public static final int PKEY_ACCORD = 3;
public static final int PKEY_NORMAL = 0;
private static IPartitioner partitioner;
private static Token minToken;
private static Token maxToken;
private static Token midToken;
private static Token upperMidToken;
private static Token lowerMidToken;
private static ICoordinator coordinator;
private final static TestMessageSink messageSink = new TestMessageSink();
private static class TestMessageSink implements IMessageSink
{
private final Queue<Pair<InetSocketAddress,IMessage>> messages = new ConcurrentLinkedQueue<>();
private final Set<InetSocketAddress> blackholed = new ConcurrentHashSet<>();
public void reset()
{
messages.clear();
blackholed.clear();
}
@Override
public void accept(InetSocketAddress to, IMessage message) {
messages.offer(Pair.create(to,message));
IInstance i = SHARED_CLUSTER.get(to);
if (blackholed.contains(to) || blackholed.contains(message.from()))
return;
if (i != null)
i.receiveMessage(message);
}
}
enum Scenario
{
// Apply the mutation from the coordinator directly without going through hinting
MUTATION(false, false, false, false, false),
// Hint from the initial mutation coordination
HINT(true, false, true, false, true),
// Apply the mutation from the batchlog directly
BATCHLOG_SUCCESSFUL_ROUTING(false, true, true, true, false),
// Have the batchlog use hints to apply the mutation after failing to route, migrating back from Accord this is a timeout because you can't get Accord to fail at routing
// it either executes correctly in the old epoch or times out waiting for the new one to arrive
BATCHLOG_FAILED_ROUTING_THEN_HINT(false, true, true, true, true),
// Have the batchlog use hints to apply the mutation after a timeout
BATCHLOG_FAILED_TIMEOUT_THEN_HINT(false, true, true, true, true),
;
final boolean initiallyEnableHints;
final boolean initiallyEnableBatchlogReplay;
final boolean initiallyBlockTestKeyspaceMutations;
final boolean passesThroughBatchlog;
final boolean deliversViaHint;
Scenario(boolean initiallyEnableHints, boolean initiallyEnableBatchlogReplay, boolean initiallyBlockTestKeyspaceMutations, boolean passesThroughBatchlog, boolean deliversViaHint)
{
this.initiallyEnableHints = initiallyEnableHints;
this.initiallyEnableBatchlogReplay = initiallyEnableBatchlogReplay;
this.initiallyBlockTestKeyspaceMutations = initiallyBlockTestKeyspaceMutations;
this.passesThroughBatchlog = passesThroughBatchlog;
this.deliversViaHint = deliversViaHint;
}
}
private final boolean migrateAwayFromAccord;
protected AccordMigrationRaceTestBase()
{
this.migrateAwayFromAccord = migratingAwayFromAccord();
}
protected abstract boolean migratingAwayFromAccord();
@Override
protected Logger logger()
{
return logger;
}
@BeforeClass
public static void setupClass() throws IOException
{
HINT_DISPATCH_INTERVAL_MS.setLong(100);
ServerTestUtils.daemonInitialization();
// Otherwise repair complains if you don't specify a keyspace
CassandraRelevantProperties.SYSTEM_TRACES_DEFAULT_RF.setInt(3);
AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("paxos_variant", PaxosVariant.v2.name())
.set("write_request_timeout", "2s")
.set("accord.range_migration", "explicit")), 3);
partitioner = FBUtilities.newPartitioner(SHARED_CLUSTER.get(1).callsOnInstance(() -> DatabaseDescriptor.getPartitioner().getClass().getSimpleName()).call());
StorageService.instance.setPartitionerUnsafe(partitioner);
ServerTestUtils.prepareServerNoRegister();
minToken = partitioner.getMinimumToken();
maxToken = partitioner.getMaximumTokenForSplitting();
midToken = partitioner.midpoint(minToken, maxToken);
upperMidToken = partitioner.midpoint(midToken, maxToken);
lowerMidToken = partitioner.midpoint(minToken, midToken);
coordinator = SHARED_CLUSTER.coordinator(1);
SHARED_CLUSTER.setMessageSink(messageSink);
}
@AfterClass
public static void tearDownClass()
{
StorageService.instance.resetPartitionerUnsafe();
}
@After
public void tearDown() throws Exception
{
messageSink.reset();
forEach(() -> {
BatchlogManager.instance.resumeReplay();
HintsService.instance.deleteAllHintsUnsafe();
HintsService.instance.resumeDispatch();
});
SHARED_CLUSTER.forEach(ClusterUtils::clearAndUnpause);
super.tearDown();
// Reset migration state
forEach(() -> {
ConsensusRequestRouter.resetInstance();
ConsensusKeyMigrationState.reset();
});
truncateSystemTables();
}
private ListenableFuture<Void> alterTableTransactionalModeAsync(TransactionalMode mode)
{
ListenableFutureTask<Void> task = ListenableFutureTask.create(() -> {
coordinator.execute(format("ALTER TABLE %s WITH %s", qualifiedAccordTableName, mode.asCqlParam()), ALL);
}, null);
Thread asyncThread = new Thread(task, "Alter table transaction mode " + mode);
asyncThread.setDaemon(true);
asyncThread.start();
return task;
}
@Test
public void testSplitAndRetryNonSerialUnloggedBatchTwoTablesOnePkey() throws Throwable
{
testSplitAndRetryMutationCoordination(twoTableBatchInsert(false, PKEY_ACCORD, PKEY_ACCORD, 1), validateTwoTable(PKEY_ACCORD));
}
@Test
public void testSplitAndRetryNonSerialUnloggedBatchTwoTablesOnePkeyHinting() throws Throwable
{
// Accord doesn't hint if a write times out
if (!migrateAwayFromAccord)
testSplitAndRetryHintDelivery(twoTableBatchInsert(false, PKEY_ACCORD, PKEY_ACCORD, 1), validateTwoTable(PKEY_ACCORD));
}
@Test
public void testSplitAndRetryNonSerialUnloggedBatchTwoTablesTwoPkey() throws Throwable
{
testSplitAndRetryMutationCoordination(twoTableBatchInsert(false, PKEY_ACCORD, PKEY_NORMAL, 1), validateTwoTable(PKEY_NORMAL));
}
@Test
public void testSplitAndRetryNonSerialUnloggedBatchTwoTablesHinting() throws Throwable
{
// Accord doesn't hint if a write times out
if (!migrateAwayFromAccord)
testSplitAndRetryHintDelivery(twoTableBatchInsert(false, PKEY_ACCORD, PKEY_NORMAL, 1), validateTwoTable(PKEY_NORMAL));
}
@Test
public void testSplitAndRetryNonSerialUnloggedBatchSingleTable() throws Throwable
{
testSplitAndRetryMutationCoordination(singleTableBatchInsert(false, PKEY_ACCORD, PKEY_NORMAL, 1), this::validateSingleTable);
}
@Test
public void testSplitAndRetryNonSerialUnloggedBatchSingleTableHinting() throws Throwable
{
// Accord doesn't hint if a write times out
if (!migrateAwayFromAccord)
testSplitAndRetryHintDelivery(singleTableBatchInsert(false, PKEY_ACCORD, PKEY_NORMAL, 1), this::validateSingleTable);
}
/*
* This doesn't really test much since on top of testSplitAndRetryNonSerialUnloggedBatchTwoTablesOnePkey since it is
* a single table & key and will be converted to an unlogged batch
*/
@Test
public void testSplitAndRetryNonSerialLoggedBatchTwoTablesOnePkey() throws Throwable
{
testSplitAndRetryMutationCoordination(twoTableBatchInsert(true, PKEY_ACCORD, PKEY_ACCORD, 1), validateTwoTable(PKEY_ACCORD));
}
@Test
public void testSplitAndRetryNonSerialLoggedBatchTwoTablesTwoPkey() throws Throwable
{
testSplitAndRetryMutationCoordination(twoTableBatchInsert(true, PKEY_ACCORD, PKEY_NORMAL, 1), validateTwoTable(PKEY_NORMAL));
}
@Test
public void testSplitAndRetryNonSerialLoggedBatchTwoTablesTwoPkeyDeliverViaBatchLog() throws Throwable
{
testSplitAndRetryBatchlogDelivery(twoTableBatchInsert(true, PKEY_ACCORD, PKEY_NORMAL, 1), validateTwoTable(PKEY_NORMAL));
}
@Test
public void testSplitAndRetryNonSerialLoggedBatchTwoTablesTwoPkeyHintedViaBatchLogTimeout() throws Throwable
{
testSplitAndRetryHintDeliveryAfterBatchlogTimeout(twoTableBatchInsert(true, PKEY_ACCORD, PKEY_NORMAL, 1), validateTwoTable(PKEY_NORMAL));
}
@Test
public void testSplitAndRetryNonSerialLoggedBatchTwoTablesTwoPkeyHintedViaBatchLogRoutingFailure() throws Throwable
{
testSplitAndRetryHintDeliveryAfterBatchlogRoutingFailure(twoTableBatchInsert(true, PKEY_ACCORD, PKEY_NORMAL, 1), validateTwoTable(PKEY_NORMAL));
}
/*
* Test that a logged batch writing to a migrating table and a non-migrating table can
*/
@Test
public void testSplitAndRetryNonSerialLoggedBatchSingleTable() throws Throwable
{
testSplitAndRetryBatchlogDelivery(singleTableBatchInsert(true, PKEY_ACCORD, PKEY_NORMAL, 1), this::validateSingleTable);
}
@Test
public void testSplitAndRetryNonSerialLoggedBatchSingleTableDeliverViaBatchLog() throws Throwable
{
testSplitAndRetryBatchlogDelivery(singleTableBatchInsert(true, PKEY_ACCORD, PKEY_NORMAL, 1), this::validateSingleTable);
}
@Test
public void testSplitAndRetryNonSerialLoggedBatchSingleTableHintedViaBatchLogTimeout() throws Throwable
{
testSplitAndRetryHintDeliveryAfterBatchlogTimeout(singleTableBatchInsert(true, PKEY_ACCORD, PKEY_NORMAL, 1), this::validateSingleTable);
}
@Test
public void testSplitAndRetryNonSerialLoggedBatchSingleTableHintedViaBatchLogRoutingFailure() throws Throwable
{
testSplitAndRetryHintDeliveryAfterBatchlogRoutingFailure(singleTableBatchInsert(true, PKEY_ACCORD, PKEY_NORMAL, 1), this::validateSingleTable);
}
private void testSplitAndRetryMutationCoordination(String batchCQL, Consumer<Cluster> validation) throws Throwable
{
testSplitAndRetry(batchCQL, validation, MUTATION);
}
private void testSplitAndRetryBatchlogDelivery(String batchCQL, Consumer<Cluster> validation) throws Throwable
{
testSplitAndRetry(batchCQL, validation, BATCHLOG_SUCCESSFUL_ROUTING);
}
private void testSplitAndRetryHintDeliveryAfterBatchlogTimeout(String batchCQL, Consumer<Cluster> validation) throws Throwable
{
testSplitAndRetry(batchCQL, validation, BATCHLOG_FAILED_TIMEOUT_THEN_HINT);
}
private void testSplitAndRetryHintDeliveryAfterBatchlogRoutingFailure(String batchCQL, Consumer<Cluster> validation) throws Throwable
{
testSplitAndRetry(batchCQL, validation, BATCHLOG_FAILED_ROUTING_THEN_HINT);
}
private void testSplitAndRetryHintDelivery(String batchCQL, Consumer<Cluster> validation) throws Throwable
{
testSplitAndRetry(batchCQL, validation, HINT);
}
private void validateSingleTable(Cluster cluster)
{
SimpleQueryResult expected = QueryResults.builder()
.columns("id", "c", "v")
.row(PKEY_NORMAL, 1, 1)
.row(PKEY_ACCORD, 1, 1)
.build();
cluster.forEach(instance -> {
assertThat(instance.executeInternalWithResult("SELECT * FROM " + qualifiedAccordTableName)).isEqualTo(expected);
});
}
private Consumer<Cluster> validateTwoTable(int secondPkey)
{
return cluster -> {
SimpleQueryResult expectedAccord = QueryResults.builder()
.columns("id", "c", "v")
.row(PKEY_ACCORD, 1, 1)
.build();
cluster.forEach(instance -> assertThat(instance.executeInternalWithResult("SELECT * FROM " + qualifiedAccordTableName)).isEqualTo(expectedAccord));
SimpleQueryResult expectedNormal = QueryResults.builder()
.columns("id", "c", "v")
.row(secondPkey, 1, 1)
.build();
cluster.forEach(instance -> assertThat(instance.executeInternalWithResult("SELECT * FROM " + qualifiedRegularTableName)).isEqualTo(expectedNormal));
};
}
/*
* Test if the coordinator is behind that the request can be re-split and routed to the correct systems
* without surfacing an error
*/
private void testSplitAndRetry(String batchCQL, Consumer<Cluster> validation, Scenario scenario) throws Throwable
{
test(createTables(TABLE_FMT, qualifiedRegularTableName, qualifiedAccordTableName),
cluster -> {
// Only enable these when testing it works from a specific instance
forEach(() -> BatchlogManager.instance.pauseReplay());
forEach(() -> HintsService.instance.pauseDispatch());
// Node 3 is always the out of sync node
IInvokableInstance outOfSyncInstance = setUpOutOfSyncNode(cluster);
// Force the batchlog Accord txn to run after this write txn in the new epoch where it
// will trigger RetryDifferentSystem
if (scenario == BATCHLOG_FAILED_ROUTING_THEN_HINT && migrateAwayFromAccord)
writeAccordRowViaAccord();
// Need to be able to block writing to the test keyspace forcing batchlog replay
// without also failing writes to the batch log
if (scenario.initiallyBlockTestKeyspaceMutations)
cluster.filters().outbound().messagesMatching((from, to, message) -> {
if (message.verb() == Verb.MUTATION_REQ.id)
{
String keyspace = cluster.get(to).callsOnInstance(() -> ((Message<Mutation>)Instance.deserializeMessage(message)).payload.getKeyspaceName()).call();
if (keyspace.equals(KEYSPACE))
return true;
}
if (message.verb() == Verb.ACCORD_PRE_ACCEPT_REQ.id && !migrateAwayFromAccord)
{
boolean drop = cluster.get(to).callsOnInstance(() -> {
PreAccept preAccept = (PreAccept)Instance.deserializeMessage(message).payload;
PartialRoute<?> route = preAccept.scope;
if (route.domain() == Domain.Key)
for (RoutingKey key : (PartialKeyRoute)route)
{
AccordRoutingKey routingKey = (AccordRoutingKey)key;
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(routingKey.table());
if (cfs.getKeyspaceName().equals(KEYSPACE))
return true;
}
return false;
}).call();
if (drop)
return true;
}
return false;
}).drop();
forEach(() -> BatchlogManager.instance.pauseReplay());
// If testing batch log delivery the coordinator needs to be a node other than the node that is behind on
// topology updates so that the batch log writes (and thus replay) can be done on the node that is out of sync
int coordinatorIndex = scenario.initiallyEnableBatchlogReplay ? 2 : 3;
IInvokableInstance instance = cluster.get(coordinatorIndex);
ICoordinator coordinator = instance.coordinator();
int startRetryCount = getRetryOnDifferentSystemCount(coordinatorIndex);
// If testing routing at mutation coordination then Node 1 and 2 will both rejected the mutation because it is in a migrating range
int startRejectedCount = getMutationsRejectedOnWrongSystemCount();
logger.info("Executing batch insert");
Future<SimpleQueryResult> resultFuture = coordinator.asyncExecuteWithResult(batchCQL, ALL);
// Testing either batch log delivery or hint delivery via batchlog
if (scenario.initiallyBlockTestKeyspaceMutations)
{
// Expect initial write failure
expectException(() -> {
try
{
return resultFuture.get();
}
catch (ExecutionException e)
{
throw (Exception) e.getCause();
}
}, WriteTimeoutException.class);
}
if (scenario.passesThroughBatchlog)
{
// At this stage we want the batch log to fail because it misrouted the queries to the wrong system
// not because it timed out not getting a response. We only did that with mutations as a quick
// way to populate the batch log. Could almost as easily have constructed the mutation and put it
// in the batch log directly
if (scenario == Scenario.BATCHLOG_FAILED_ROUTING_THEN_HINT || scenario == BATCHLOG_SUCCESSFUL_ROUTING)
cluster.filters().reset();
// We only want the batch log to have access to the correct topology if we are testing its
// ability to handle misrouted things
if (scenario == Scenario.BATCHLOG_SUCCESSFUL_ROUTING)
unpauseEnactment(outOfSyncInstance);
// Unfortunately the batch won't be replayed until some time has passed because the starting time
// for replay is the current time - timeout
// Don't wait here for the batchlog if we need to spin on the creation of the Accord transaction
// and then unpause to test Accord routing failure
boolean unpauseAfterBatchLogCreatesTransaction = migrateAwayFromAccord && scenario == BATCHLOG_FAILED_ROUTING_THEN_HINT;
if (!unpauseAfterBatchLogCreatesTransaction)
Thread.sleep(BatchlogManager.BATCHLOG_REPLAY_TIMEOUT + DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.MILLISECONDS));
messageSink.reset();
// Force batch log delivery (or hint delivery) on the node that was out of sync, but should be in sync once we unpause
// This demonstrates it can split the mutation correctly or forward it to hinting if it fails
outOfSyncInstance.runOnInstance(() -> runUnchecked(() -> {
// We don't want hints for any reason that might apply the mutation and make the test look like it succeeded
assertTrue(HintsService.instance.isDispatchPaused());
// The failed write will have written hints
HintsService.instance.deleteAllHintsUnsafe();
assertFalse(hasPendingHints());
BatchlogManager.instance.resumeReplay();
// Unpausing needs to be done async because it waits for the batch log replay
Promise<Void> unpaused = new AsyncPromise<>();
if (unpauseAfterBatchLogCreatesTransaction)
{
logger.info("Creating thread to unpause after batchlog creates Accord transaction");
new Thread(() ->
{
try
{
// Unpause so it can route incorrectly instead of timing out waiting to fetch the epoch, need the transaction to be created first
// otherwise it will just be routed straight to non-Accord.
logger.info("Spinning waiting on a transaction");
Util.spinUntilTrue(() -> !((AccordService)AccordService.instance()).node().coordinating().isEmpty(), 20);
logger.info("Foudn transaction, unpausing");
TestChangeListener.instance.unpause();
unpaused.trySuccess(null);
}
catch (Throwable t)
{
unpaused.tryFailure(t);
}
}).start();
}
else
{
// Force replay so mosts tests don't have to wait
BatchlogManager.instance.forceBatchlogReplay();
unpaused.trySuccess(null);
}
// Fetch errors
unpaused.get();
// Ensure the batch log did or didn't create pending hints depending on the test scenario
spinAssertEquals(scenario == BATCHLOG_FAILED_TIMEOUT_THEN_HINT || scenario == BATCHLOG_FAILED_ROUTING_THEN_HINT, () -> hasPendingHints(), 20);
}));
}
// Mutation successfully applied from the coordinator after retrying scenario
if (scenario == MUTATION)
{
// Don't want to mistakenly have hints applying the mutation
forEach(() -> assertTrue(HintsService.instance.isDispatchPaused()));
// Check for the error differently depending on what system should be seeing an error
if (migrateAwayFromAccord)
{
// Accord will block until we unpause enactment so to test the routing we wait until the transaction
// has started so the epoch it is created in is the old one
Util.spinUntilTrue(() -> outOfSyncInstance.callOnInstance(() -> !((AccordService)AccordService.instance()).node().coordinating().isEmpty()), 20);
logger.info("Accord node is now coordinating something");
try
{
validation.accept(cluster);
throw new AssertionError("Expected validation to fail");
}
catch (AssertionError e)
{
//ignored
}
}
else
{
Stopwatch sw = Stopwatch.createStarted();
spinAssertEquals(startRejectedCount + 2, 10, () -> getMutationsRejectedOnWrongSystemCount() - startRejectedCount);
logger.info("Took {}ms to get mutations rejected on wrong system", sw.elapsed(TimeUnit.MILLISECONDS));
}
logger.info("Unpausing out of sync instance");
// Testing regular mutation coordination retry loop let coordinator get up to date and retry
unpauseEnactment(outOfSyncInstance);
try
{
resultFuture.get();
}
catch (ExecutionException e)
{
// This is expected when inverting the migration
if (migrateAwayFromAccord && e.getCause() instanceof CoordinatorBehindException)
throw e;
throw e;
}
if (!migrateAwayFromAccord)
{
int endRetryCount = getRetryOnDifferentSystemCount(coordinatorIndex);
int endRejectedCount = getMutationsRejectedOnWrongSystemCount();
assertEquals(1, endRetryCount - startRetryCount);
// Expect only two nodes to reject since they enacted the new epoch
assertEquals(2, endRejectedCount - startRejectedCount);
}
}
// Anything related to making sure hints are delivered goes here
if (scenario.deliversViaHint)
{
// Don't want to mistakenly have hints applying the mutation before we enable it on just one instance
forEach(() -> assertTrue(HintsService.instance.isDispatchPaused()));
// The filters wouldn't have been reset yet if they were needed to make the batchlog or original mutation time out
// Need to reset so Hints can use Accord txns
cluster.filters().reset();
long startingAccordTimeouts = outOfSyncInstance.callOnInstance(() -> ClientRequestsMetricsHolder.accordWriteMetrics.timeouts.getCount());
long startingAccordPreempted = outOfSyncInstance.callOnInstance(() -> ClientRequestsMetricsHolder.accordWriteMetrics.preempted.getCount());
long startingAccordMigrationRejects = outOfSyncInstance.callOnInstance(() -> ClientRequestsMetricsHolder.accordWriteMetrics.accordMigrationRejects.getCount());
long startingHintTimeouts = outOfSyncInstance.callOnInstance(() -> HintsServiceMetrics.hintsTimedOut.getCount());
outOfSyncInstance.runOnInstance(() -> HintsService.instance.resumeDispatch());
// The initial hinting attempt should fail, unless it's a batchlog routing failure in which
// case the coordinator has already caught up so the hint will succeed on the first try
// Can only really have this case for BATCHLOG_FAILED_TIMEOUT_THEN_HINT becuase Accord timeouts don't
// write hints so there is nothing to test
if (migrateAwayFromAccord && scenario == BATCHLOG_FAILED_TIMEOUT_THEN_HINT)
{
Callable<Boolean> test = () -> outOfSyncInstance.callOnInstance(() -> {
logger.info("startingAccordTimeouts {}, startingAccordPreempts {}, startingAccordMigrationRejects {}, startingHintTimeouts {}, accord timeouts {}, accordPreempts {}, accordMigrationRejects {}, hint timeouts {}", startingAccordTimeouts, startingAccordPreempted, startingAccordMigrationRejects, startingHintTimeouts, ClientRequestsMetricsHolder.accordWriteMetrics.timeouts.getCount(), ClientRequestsMetricsHolder.accordWriteMetrics.preempted.getCount(), ClientRequestsMetricsHolder.accordWriteMetrics.accordMigrationRejects.getCount(), HintsServiceMetrics.hintsTimedOut.getCount());
AccordClientRequestMetrics accordMetrics = ClientRequestsMetricsHolder.accordWriteMetrics;
return accordMetrics.timeouts.getCount() >= (startingAccordTimeouts + 1) && HintsServiceMetrics.hintsTimedOut.getCount() >= (startingHintTimeouts + 1);
});
Util.spinUntilTrue(test, 40);
}
else if (!migrateAwayFromAccord)
{
// Expect two retry on different system responses when migrating from Paxos to Accord, one from each
// node that knows it is on the wrong system
Util.spinUntilTrue(() -> messageSink.messages.stream().filter(p -> {
if (p.right.verb() != Verb.FAILURE_RSP.id)
return false;
if (!p.left.equals(outOfSyncInstance.broadcastAddress()))
return false;
RequestFailureReason reason = ((RequestFailure) Instance.deserializeMessage(p.right).payload).reason;
if (reason == RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM)
return true;
return false;
}).count() == 2, 20);
}
// After this hints should deliver and the final validation should succeed
// if we don't unpause enactment
unpauseEnactment(outOfSyncInstance);
}
// Accord commit is async and might take a while, but the data should end up as expected
Util.spinUntilSuccess(() -> validation.accept(cluster));
});
}
/*
* Set up 3 to be behind and unaware of the migration while 1 and 2 are aware
*/
private IInvokableInstance setUpOutOfSyncNode(Cluster cluster) throws Throwable
{
IInvokableInstance i1 = cluster.get(1);
IInvokableInstance i2 = cluster.get(2);
IInvokableInstance i3 = cluster.get(3);
alterTableTransactionalMode(TransactionalMode.full);
Epoch nextEpoch = getNextEpoch(i1);
// Node 3 will coordinate the query and not be aware that the migration has begun
Callable<?> pausedBeforeEnacting = pauseBeforeEnacting(i3, nextEpoch);
// In batch log delivery cases i2 will be the coordinator and we need to be sure that it has enacted the latest epoch
Callable<?> i2PausedAfterEnacting = pauseAfterEnacting(i2, nextEpoch);
ListenableFuture<?> result = nodetoolAsync(coordinator, "consensus_admin", "begin-migration", "-st", midToken.toString(), "-et", maxToken.toString(), "-tp", "accord", KEYSPACE, accordTableName);
if (migrateAwayFromAccord)
{
pausedBeforeEnacting.call();
i2PausedAfterEnacting.call();
unpauseEnactment(i2);
unpauseEnactment(i3);
result.get();
long migratingEpoch = nextEpoch.getEpoch();
Util.spinUntilTrue(() -> cluster.stream().allMatch(instance -> instance.callOnInstance(() -> ClusterMetadata.current().epoch.equals(Epoch.create(migratingEpoch)))), 10);
nextEpoch = getNextEpoch(i1);
pausedBeforeEnacting = pauseBeforeEnacting(i3, nextEpoch);
i2PausedAfterEnacting = pauseAfterEnacting(i2, nextEpoch);
// In the reverse direction doing the alter automatically reverses the migratin without a need to call begin migration on any ranges
result = alterTableTransactionalModeAsync(TransactionalMode.off);
}
// Wait for everyone to get to where they are supposed to be
try
{
pausedBeforeEnacting.call();
}
catch (Throwable t)
{
if (result.isDone())
{
try
{
result.get();
}
catch (ExecutionException e)
{
t.addSuppressed(e);
throw t;
}
}
throw t;
}
i2PausedAfterEnacting.call();
// Unpause on 1 and 2 where we want them aware of the migration
unpauseEnactment(i1);
unpauseEnactment(i2);
// nodetool should be able to complete now
result.get();
return i3;
}
private String twoTableBatchInsert(boolean logged, int pkey1, int pkey2, int value)
{
return batch(logged,
insertCQL(qualifiedAccordTableName, pkey1, value),
insertCQL(qualifiedRegularTableName, pkey2, value));
}
private String singleTableBatchInsert(boolean logged, int pkey1, int pkey2, int value)
{
return batch(logged,
insertCQL(qualifiedAccordTableName, pkey1, value),
insertCQL(qualifiedAccordTableName, pkey2, value));
}
private static String insertCQL(String qualifiedTableName, int pkey, int value)
{
return format("INSERT INTO %s ( id, c, v ) VALUES ( %d, %d, %d )", qualifiedTableName, pkey, CLUSTERING_VALUE, value);
}
// Prevents the creation of transactions in an older epoch because later writes need to order after earlier
private void writeAccordRowViaAccord()
{
logger.info("Initiating Accord row write");
SHARED_CLUSTER.coordinator(1).execute(insertCQL(qualifiedAccordTableName, PKEY_ACCORD, 99), ConsistencyLevel.QUORUM);
logger.info("Finished Accord row write");
}
}

70
test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java
View File

@ -21,7 +21,6 @@ package org.apache.cassandra.distributed.test.accord;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@ -52,14 +51,16 @@ import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.Mutation.SimpleBuilder;
import org.apache.cassandra.db.SimpleBuilders.PartitionUpdateBuilder;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.ICoordinator;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.api.Row;
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.gms.EndpointState;
@ -97,8 +98,8 @@ import org.yaml.snakeyaml.Yaml;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
import static org.apache.cassandra.Util.dk;
import static org.apache.cassandra.Util.spinUntilSuccess;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.db.SystemKeyspace.CONSENSUS_MIGRATION_STATE;
import static org.apache.cassandra.db.SystemKeyspace.PAXOS;
import static org.apache.cassandra.dht.Range.normalize;
@ -160,7 +161,7 @@ public class AccordMigrationTest extends AccordTestBase
ServerTestUtils.daemonInitialization();
// Otherwise repair complains if you don't specify a keyspace
CassandraRelevantProperties.SYSTEM_TRACES_DEFAULT_RF.setInt(3);
AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("paxos_variant", PaxosVariant.v2.name())
AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.with(Feature.NETWORK).set("paxos_variant", PaxosVariant.v2.name())
.set("accord.range_migration", "explicit")), 3);
partitioner = FBUtilities.newPartitioner(SHARED_CLUSTER.get(1).callsOnInstance(() -> DatabaseDescriptor.getPartitioner().getClass().getSimpleName()).call());
StorageService.instance.setPartitionerUnsafe(partitioner);
@ -192,19 +193,6 @@ public class AccordMigrationTest extends AccordTestBase
SHARED_CLUSTER.coordinators().forEach(coordinator -> coordinator.execute(format("TRUNCATE TABLE %s.%s", SYSTEM_KEYSPACE_NAME, PAXOS), ALL));
}
private static String nodetool(ICoordinator coordinator, String... commandAndArgs)
{
NodeToolResult nodetoolResult = coordinator.instance().nodetoolResult(commandAndArgs);
if (!nodetoolResult.getStdout().isEmpty())
System.out.println(nodetoolResult.getStdout());
if (!nodetoolResult.getStderr().isEmpty())
System.err.println(nodetoolResult.getStderr());
if (nodetoolResult.getError() != null)
fail("Failed nodetool " + Arrays.asList(commandAndArgs), nodetoolResult.getError());
// TODO why does standard out end up in stderr in nodetool?
return nodetoolResult.getStdout();
}
private static int getKeyBetweenTokens(Token left, Token right)
{
return getKeysBetweenTokens(left, right).next();
@ -398,10 +386,10 @@ public class AccordMigrationTest extends AccordTestBase
@Test
public void testPaxosToAccordCAS() throws Exception
{
test(format(TABLE_FMT, qualifiedTableName),
test(format(TABLE_FMT, qualifiedAccordTableName),
cluster -> {
List<Pair<ByteBuffer, UUID>> expectedKeyMigrations = new ArrayList<>();
String table = tableName;
String table = accordTableName;
UUID tableUUID = cluster.get(1).callOnInstance(() -> ColumnFamilyStore.getIfExists(KEYSPACE, table).getTableId().asUUID());
cluster.forEach(node -> node.runOnInstance(() -> {
TableMetadata tbl = Schema.instance.getTableMetadata(KEYSPACE, table);
@ -409,7 +397,7 @@ public class AccordMigrationTest extends AccordTestBase
Assert.assertEquals(TransactionalMigrationFromMode.none, tbl.params.transactionalMigrationFrom);
}));
cluster.schemaChange(format("ALTER TABLE %s.%s WITH transactional_mode='%s'", KEYSPACE, tableName, TransactionalMode.full));
cluster.schemaChange(format("ALTER TABLE %s.%s WITH transactional_mode='%s'", KEYSPACE, accordTableName, TransactionalMode.full));
cluster.forEach(node -> node.runOnInstance(() -> {
TableMetadata tbl = Schema.instance.getTableMetadata(KEYSPACE, table);
@ -417,11 +405,11 @@ public class AccordMigrationTest extends AccordTestBase
Assert.assertEquals(TransactionalMigrationFromMode.off, tbl.params.transactionalMigrationFrom);
}));
String casCQL = format(CAS_FMT, qualifiedTableName, CLUSTERING_VALUE);
String casCQL = format(CAS_FMT, qualifiedAccordTableName, CLUSTERING_VALUE);
Consumer<Integer> runCasNoApply = key -> assertRowEquals(cluster, new Object[]{false}, casCQL, key);
Consumer<Integer> runCasApplies = key -> assertRowEquals(cluster, new Object[]{true}, casCQL, key);
Consumer<Integer> runCasOnSecondNode = key -> assertEquals( "[applied]", cluster.coordinator(2).executeWithResult(casCQL, ANY, key).names().get(0));
String tableName = qualifiedTableName.split("\\.")[1];
String tableName = qualifiedAccordTableName.split("\\.")[1];
int migratingKey = getKeyBetweenTokens(midToken, maxToken);
int notMigratingKey = getKeyBetweenTokens(minToken, midToken);
Range<Token> migratingRange = new Range(midToken, maxToken);
@ -493,8 +481,15 @@ public class AccordMigrationTest extends AccordTestBase
// Update inserted row so the condition can apply, if the condition check doesn't apply
// then it won't get to propose/accept
migratingKey = testingKeys.next();
String query = "UPDATE " + qualifiedTableName + " SET v = 42 WHERE id = ? AND c = ?";
Consumer<Integer> makeCASApply = key -> cluster.forEach(instance -> instance.runOnInstance(() -> executeInternal(query, key, CLUSTERING_VALUE)));
String keyspace = KEYSPACE;
Integer clusteringValue = CLUSTERING_VALUE;
String mutationTableName = accordTableName;
Consumer<Integer> makeCASApply = key -> cluster.forEach(instance -> instance.runOnInstance(() -> {
SimpleBuilder mutationBuilder = Mutation.simpleBuilder(keyspace, dk(key)).allowPotentialTransactionConflicts();
mutationBuilder.update(mutationTableName).row(clusteringValue).add("v", 42);
Mutation m = mutationBuilder.build();
m.applyUnsafe();
}));
makeCASApply.accept(migratingKey);
// This will force the request to run on Paxos up to Accept
@ -552,13 +547,13 @@ public class AccordMigrationTest extends AccordTestBase
@Test
public void testPaxosToAccordSerialRead() throws Exception
{
test(format(TABLE_FMT, qualifiedTableName),
test(format(TABLE_FMT, qualifiedAccordTableName),
cluster -> {
String table = tableName;
String table = accordTableName;
UUID tableUUID = cluster.get(1).callOnInstance(() -> ColumnFamilyStore.getIfExists(KEYSPACE, table).getTableId().asUUID());
List<Pair<ByteBuffer, UUID>> expectedKeyMigrations = new ArrayList<>();
cluster.schemaChange(format("ALTER TABLE %s.%s WITH transactional_mode='%s'", KEYSPACE, tableName, TransactionalMode.full));
String readCQL = format("SELECT * FROM %s WHERE id = ? and c = %s", qualifiedTableName, CLUSTERING_VALUE);
cluster.schemaChange(format("ALTER TABLE %s.%s WITH transactional_mode='%s'", KEYSPACE, accordTableName, TransactionalMode.full));
String readCQL = format("SELECT * FROM %s WHERE id = ? and c = %s", qualifiedAccordTableName, CLUSTERING_VALUE);
Function<Integer, Object[][]> runRead = key -> cluster.coordinator(1).execute(readCQL, SERIAL, key);
Range<Token> migratingRange = new Range<>(new LongToken(Long.MIN_VALUE + 1), new LongToken(Long.MIN_VALUE));
List<Range<Token>> migratingRanges = ImmutableList.of(migratingRange);
@ -566,15 +561,15 @@ public class AccordMigrationTest extends AccordTestBase
assertTargetAccordRead(runRead, 1, key, expectedKeyMigrations, 0, 1, 0, 0, 0);
// Mark wrap around range as migrating
nodetool(coordinator, "consensus_admin", "begin-migration", "-st", String.valueOf(Long.MIN_VALUE + 1), "-et", String.valueOf(Long.MIN_VALUE), "-tp", "accord", KEYSPACE, tableName);
assertMigrationState(tableName, ConsensusMigrationTarget.accord, emptyList(), migratingRanges, 1);
nodetool(coordinator, "consensus_admin", "begin-migration", "-st", String.valueOf(Long.MIN_VALUE + 1), "-et", String.valueOf(Long.MIN_VALUE), "-tp", "accord", KEYSPACE, accordTableName);
assertMigrationState(accordTableName, ConsensusMigrationTarget.accord, emptyList(), migratingRanges, 1);
// Should run directly on accord, migrate the key, and perform a quorum read from Accord, Paxos repair will run prepare once
addExpectedMigratedKey(expectedKeyMigrations, key, tableUUID);
assertTargetAccordRead(runRead, 1, key, expectedKeyMigrations, 1, 1, 1, 0, 0);
key++;
// Should run up to accept with both nodes refusing to accept
savePromisedAndCommittedPaxosProposal(tableName, key);
savePromisedAndCommittedPaxosProposal(accordTableName, key);
cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new PaxosToAccordMigrationNotHappeningUpToBegin()));
addExpectedMigratedKey(expectedKeyMigrations, key, tableUUID);
assertTargetAccordRead(runRead, 1, key, expectedKeyMigrations, 1, 2, 1, 0, 1);
@ -582,11 +577,6 @@ public class AccordMigrationTest extends AccordTestBase
});
}
private void alterTableTransactionalMode(TransactionalMode mode)
{
SHARED_CLUSTER.schemaChange(format("ALTER TABLE %s WITH %s", qualifiedTableName, mode.asCqlParam()));
}
private void assertTransactionalModes(String keyspace, String table, TransactionalMode mode, TransactionalMigrationFromMode migration)
{
forEach(() -> {
@ -598,17 +588,17 @@ public class AccordMigrationTest extends AccordTestBase
private void assertTransactionalModes(TransactionalMode mode, TransactionalMigrationFromMode migration)
{
assertTransactionalModes(KEYSPACE, tableName, mode, migration);
assertTransactionalModes(KEYSPACE, accordTableName, mode, migration);
}
@Test
public void testAccordToPaxos() throws Exception
{
test(format(TABLE_FMT, qualifiedTableName),
test(format(TABLE_FMT, qualifiedAccordTableName),
cluster -> {
String casCQL = format(CAS_FMT, qualifiedTableName, CLUSTERING_VALUE);
String casCQL = format(CAS_FMT, qualifiedAccordTableName, CLUSTERING_VALUE);
Consumer<Integer> runCasNoApply = key -> assertRowEquals(cluster, new Object[]{false}, casCQL, key);
String tableName = qualifiedTableName.split("\\.")[1];
String tableName = qualifiedAccordTableName.split("\\.")[1];
UUID tableUUID = cluster.get(1).callOnInstance(() -> ColumnFamilyStore.getIfExists(KEYSPACE, tableName).getTableId().asUUID());
alterTableTransactionalMode(TransactionalMode.mixed_reads);

148
test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java
View File

@ -24,17 +24,17 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import accord.coordinate.Invalidated;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.primitives.Ints;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@ -42,15 +42,25 @@ import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.RoutingKey;
import accord.coordinate.Invalidated;
import accord.impl.SimpleProgressLog;
import accord.messages.PreAccept;
import accord.primitives.PartialKeyRoute;
import accord.primitives.PartialRoute;
import accord.primitives.Routable.Domain;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
import net.bytebuddy.implementation.bind.annotation.This;
import org.apache.cassandra.batchlog.BatchlogManager;
import org.apache.cassandra.cql3.statements.ModificationStatement;
import org.apache.cassandra.cql3.statements.TransactionStatement;
import org.apache.cassandra.cql3.transactions.ReferenceValue;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.Cluster.Builder;
@ -60,24 +70,37 @@ import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableRunnable;
import org.apache.cassandra.distributed.api.QueryResults;
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.distributed.impl.Instance;
import org.apache.cassandra.distributed.shared.AssertUtils;
import org.apache.cassandra.distributed.shared.Metrics;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.distributed.util.QueryResultUtil;
import org.apache.cassandra.hints.HintsService;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.AccordTestUtils;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException;
import org.apache.cassandra.service.accord.exceptions.WritePreemptedException;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.utils.AssertionUtils;
import org.apache.cassandra.utils.FailingConsumer;
import static java.lang.String.format;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static org.apache.cassandra.db.SystemKeyspace.CONSENSUS_MIGRATION_STATE;
import static org.apache.cassandra.db.SystemKeyspace.PAXOS;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME;
import static org.junit.Assert.assertArrayEquals;
public abstract class AccordTestBase extends TestBaseImpl
@ -89,8 +112,10 @@ public abstract class AccordTestBase extends TestBaseImpl
protected static Cluster SHARED_CLUSTER;
protected String tableName;
protected String qualifiedTableName;
protected String accordTableName;
protected String qualifiedAccordTableName;
protected String regularTableName;
protected String qualifiedRegularTableName;
public static void setupCluster(Function<Builder, Builder> options, int nodes) throws IOException
{
@ -107,8 +132,10 @@ public abstract class AccordTestBase extends TestBaseImpl
@Before
public void setup()
{
tableName = "tbl" + COUNTER.getAndIncrement();
qualifiedTableName = KEYSPACE + '.' + tableName;
accordTableName = "accordtbl" + COUNTER.getAndIncrement();
qualifiedAccordTableName = KEYSPACE + '.' + accordTableName;
regularTableName = "regulartbl" + COUNTER.getAndIncrement();
qualifiedRegularTableName = KEYSPACE + '.' + regularTableName;
}
@After
@ -135,6 +162,14 @@ public abstract class AccordTestBase extends TestBaseImpl
test(Collections.singletonList(tableDDL), fn);
}
protected List<String> createTables(String tableFormat, String... qualifiedTables)
{
ImmutableList.Builder<String> builder = ImmutableList.builder();
for (String qualifiedTable : qualifiedTables)
builder.add(format(tableFormat, qualifiedTable));
return builder.build();
}
public static void ensureTableIsAccordManaged(Cluster cluster, String ksname, String tableName)
{
cluster.get(1).runOnInstance(() -> {
@ -165,7 +200,7 @@ public abstract class AccordTestBase extends TestBaseImpl
protected void test(FailingConsumer<Cluster> fn) throws Exception
{
test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", fn);
test("CREATE TABLE " + qualifiedAccordTableName + " (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", fn);
}
protected static ConsensusMigrationState getMigrationStateSnapshot(IInvokableInstance instance) throws IOException
@ -198,6 +233,19 @@ public abstract class AccordTestBase extends TestBaseImpl
return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.Latency.CASWrite"));
}
protected static int getRetryOnDifferentSystemCount(int coordinatorIndex)
{
return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.RetryDifferentSystem.Write"));
}
protected int getMutationsRejectedOnWrongSystemCount()
{
long sum = 0;
for (IInvokableInstance instance : SHARED_CLUSTER)
sum += instance.metrics().getCounter("org.apache.cassandra.metrics.Table.MutationsRejectedOnWrongSystem." + qualifiedAccordTableName);
return Ints.checkedCast(sum);
}
protected static int getCasPrepareCount(int coordinatorIndex)
{
return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.keyspace.CasPrepareLatency.distributed_test_keyspace"));
@ -279,7 +327,7 @@ public abstract class AccordTestBase extends TestBaseImpl
// disable vnode for now, but should enable before trunk
Cluster.Builder builder = Cluster.build(nodes)
.withoutVNodes()
.withConfig(c -> c.with(Feature.NETWORK, Feature.GOSSIP).set("write_request_timeout", "10s")
.withConfig(c -> c.with(Feature.GOSSIP).set("write_request_timeout", "10s")
.set("transaction_timeout", "15s")
.set("transaction_timeout", "15s"))
.withInstanceInitializer(EnforceUpdateDoesNotPerformRead::install);
@ -464,4 +512,86 @@ public abstract class AccordTestBase extends TestBaseImpl
}
protected abstract Logger logger();
protected void alterTableTransactionalMode(TransactionalMode mode)
{
SHARED_CLUSTER.schemaChange(format("ALTER TABLE %s WITH %s", qualifiedAccordTableName, mode.asCqlParam()));
}
protected static void pauseHints()
{
forEach(() -> HintsService.instance.pauseDispatch());
}
protected static void deleteAllHints()
{
forEach(() -> HintsService.instance.deleteAllHintsUnsafe());
}
protected static void pauseBatchlog()
{
forEach(() -> BatchlogManager.instance.pauseReplay());
}
protected static void unpauseHints()
{
forEach(() -> HintsService.instance.resumeDispatch());
}
protected static void unpauseBatchlog()
{
forEach(() -> BatchlogManager.instance.resumeReplay());
}
protected static void blockMutationAndPreAccept(Cluster cluster)
{
cluster.filters().outbound().messagesMatching((from, to, message) -> {
if (message.verb() == Verb.MUTATION_REQ.id)
{
String keyspace = cluster.get(to).callsOnInstance(() -> ((Message<Mutation>) Instance.deserializeMessage(message)).payload.getKeyspaceName()).call();
if (keyspace.equals(KEYSPACE))
return true;
}
if (message.verb() == Verb.ACCORD_PRE_ACCEPT_REQ.id)
{
boolean drop = cluster.get(to).callsOnInstance(() -> {
PreAccept preAccept = (PreAccept)Instance.deserializeMessage(message).payload;
PartialRoute<?> route = preAccept.scope;
if (route.domain() == Domain.Key)
for (RoutingKey key : (PartialKeyRoute)route)
{
AccordRoutingKey routingKey = (AccordRoutingKey)key;
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(routingKey.table());
if (cfs.getKeyspaceName().equals(KEYSPACE))
return true;
}
return false;
}).call();
if (drop)
return true;
}
return false;
}).drop();
}
protected static void truncateSystemTables()
{
SHARED_CLUSTER.coordinator(1).execute("TRUNCATE " + SYSTEM_KEYSPACE_NAME + "." + SystemKeyspace.BATCHES, ALL);
SHARED_CLUSTER.coordinator(1).execute(format("TRUNCATE TABLE %s.%s", SYSTEM_KEYSPACE_NAME, CONSENSUS_MIGRATION_STATE), ALL);
SHARED_CLUSTER.coordinator(1).execute(format("TRUNCATE TABLE %s.%s", SYSTEM_KEYSPACE_NAME, PAXOS), ALL);
}
protected static Stream<UUID> hostIds()
{
return Stream.concat(ClusterMetadata.current().directory.peerIds()
.stream()
.map(ClusterMetadata.current().directory::hostId),
Stream.of(HintsService.RETRY_ON_DIFFERENT_SYSTEM_UUID));
}
protected static boolean hasPendingHints()
{
return hostIds().map(HintsService.instance::getTotalHintsSize)
.anyMatch(size -> size > 0);
}
}

209
test/distributed/org/apache/cassandra/distributed/test/accord/AccordTimestampPreservationTest.java Normal file
View File

@ -0,0 +1,209 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.accord;
import java.io.IOException;
import java.nio.ByteBuffer;
import com.google.common.collect.ImmutableList;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.SimpleBuilders.MutationBuilder;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.distributed.api.ICoordinator;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.locator.ReplicaPlan;
import org.apache.cassandra.locator.ReplicaPlans;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState;
import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
import static java.lang.String.format;
import static org.apache.cassandra.Util.spinAssertEquals;
import static org.apache.cassandra.config.CassandraRelevantProperties.HINT_DISPATCH_INTERVAL_MS;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/*
* Test that non-transactional updates have their timestamps preserved when written through Accord so that
* `USING TIMESTAMP` continues to work and so that hints and batch log retry attempts are inserted with their
* original timestamp and not a later Accord timestamp which could cause data resurrection.
*/
public class AccordTimestampPreservationTest extends AccordTestBase
{
private static final Logger logger = LoggerFactory.getLogger(AccordTimestampPreservationTest.class);
private static final int CLUSTERING_VALUE = 1;
private static final String NORMAL_TABLE_FMT = "CREATE TABLE %s (id int, c int, v int, PRIMARY KEY ((id), c))";
private static final String ACCORD_TABLE_FMT = NORMAL_TABLE_FMT + " WITH transactional_mode='full'";
private static ICoordinator coordinator;
private static final String expectedResult = "[[42]]";
private static final int PKEY1 = 77;
private static final int PKEY2 = 78;
private static final int VALUE = 66;
private static final long TIMESTAMP = 42;
@Override
protected Logger logger()
{
return logger;
}
@BeforeClass
public static void setupClass() throws IOException
{
HINT_DISPATCH_INTERVAL_MS.setLong(100);
ServerTestUtils.daemonInitialization();
// Otherwise repair complains if you don't specify a keyspace
CassandraRelevantProperties.SYSTEM_TRACES_DEFAULT_RF.setInt(3);
AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("write_request_timeout", "2s")), 3);
ServerTestUtils.prepareServerNoRegister();
coordinator = SHARED_CLUSTER.coordinator(1);
}
@After
public void tearDown() throws Exception
{
unpauseBatchlog();
deleteAllHints();
unpauseHints();
super.tearDown();
// Reset migration state
forEach(() -> {
ConsensusRequestRouter.resetInstance();
ConsensusKeyMigrationState.reset();
});
truncateSystemTables();
}
@Test
public void testMutationPreservesTimestamp() throws Exception
{
test(createTables(ACCORD_TABLE_FMT, qualifiedAccordTableName), cluster -> {
long startCount = getAccordCoordinateCount();
coordinator.executeWithResult(insertCQL(qualifiedAccordTableName, PKEY1, VALUE), ALL);
assertEquals(startCount + 1, getAccordWriteCount());
int id = 1;
for (IInvokableInstance instance : cluster)
{
logger.info("Checking instance " + id);
id++;
spinAssertEquals(expectedResult, () -> instance.executeInternalWithResult(checkCQL()).toString(), 20);
}
});
}
@Test
public void testBatchlogPreservesTimestamp() throws Exception
{
test(ImmutableList.of(format(NORMAL_TABLE_FMT, qualifiedRegularTableName), format(ACCORD_TABLE_FMT, qualifiedAccordTableName)), cluster -> {
pauseHints();
blockMutationAndPreAccept(cluster);
try
{
// Insert must span both Accord and non-Accord ranges or tables otherwise it bypasses the batchlog entirely
coordinator.executeWithResult(batchInsert(true, PKEY1, PKEY2, VALUE), ALL);
fail("Should have thrown WTE");
}
catch (Throwable t)
{
assertEquals(t.getClass().getName(), WriteTimeoutException.class.getName());
}
cluster.filters().reset();
int id = 1;
for (IInvokableInstance instance : cluster)
{
logger.info("Checking instance " + id);
id++;
spinAssertEquals(expectedResult, () -> instance.executeInternalWithResult(checkCQL()).toString(), 20);
}
});
}
@Test
public void testHintsPreservesTimestamp() throws Exception
{
test(createTables(ACCORD_TABLE_FMT, qualifiedAccordTableName), cluster -> {
String keyspace = KEYSPACE;
int pkey1 = PKEY1;
long timestamp = TIMESTAMP;
int clustering = CLUSTERING_VALUE;
String tableName = accordTableName;
cluster.get(1).runOnInstance(() -> {
ByteBuffer keyBuf = Int32Type.instance.fromString(Integer.toString(pkey1));
DecoratedKey dk = DatabaseDescriptor.getPartitioner().decorateKey(keyBuf);
MutationBuilder mutationBuilder = new MutationBuilder(KEYSPACE, dk);
mutationBuilder.timestamp(timestamp);
mutationBuilder.update(tableName).row(clustering).add("v", VALUE);
Mutation m = mutationBuilder.build();
ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(Keyspace.open(keyspace), ConsistencyLevel.ALL, dk.getToken(), ReplicaPlans.writeAll);
for (Replica replica : plan.live().withoutSelf())
StorageProxy.submitHint(m, replica, null);
});
for (int i = 2; i <= 3; i++)
{
int instance = i;
spinAssertEquals(expectedResult, () -> cluster.get(instance).executeInternalWithResult(checkCQL()).toString(), 20);
}
});
}
private String batchInsert(boolean logged, int pkey1, int pkey2, int value)
{
return batch(logged,
insertCQL(qualifiedAccordTableName, pkey1, value),
insertCQL(qualifiedRegularTableName, pkey2, value));
}
private String insertCQL(String table, int pkey, int value)
{
return insertCQL(table, pkey, value, false);
}
private String insertCQL(String table, int pkey, int value, boolean cas)
{
return format("INSERT INTO %s ( id, c, v ) VALUES ( %d, %d, %d )%s USING TIMESTAMP %d", table, pkey, CLUSTERING_VALUE, value, cas ? " IF NOT EXISTS" : "", TIMESTAMP);
}
private String checkCQL()
{
return format("SELECT WRITETIME(v) from %s WHERE id = %d", qualifiedAccordTableName, PKEY1);
}
}

27
test/distributed/org/apache/cassandra/distributed/test/accord/MigrationFromAccordRaceTest.java Normal file
View File

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.accord;
public class MigrationFromAccordRaceTest extends AccordMigrationRaceTestBase
{
protected boolean migratingAwayFromAccord()
{
return true;
}
}

27
test/distributed/org/apache/cassandra/distributed/test/accord/MigrationToAccordRaceTest.java Normal file
View File

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.accord;
public class MigrationToAccordRaceTest extends AccordMigrationRaceTestBase
{
protected boolean migratingAwayFromAccord()
{
return false;
}
}

7
test/distributed/org/apache/cassandra/distributed/test/metrics/HintsServiceMetricsTest.java
View File

@ -142,6 +142,7 @@ public class HintsServiceMetricsTest extends TestBaseImpl
assertThat(countHintsSucceeded(node)).isEqualTo(0);
assertThat(countHintsFailed(node)).isEqualTo(0);
assertThat(countHintsTimedOut(node)).isEqualTo(0);
assertThat(countHintsRetryDifferentSystem(node)).isEqualTo(0);
assertThat(countGlobalDelays(node)).isEqualTo(0);
cluster.forEach(target -> assertThat(countEndpointDelays(node, target)).isEqualTo(0));
}
@ -180,6 +181,12 @@ public class HintsServiceMetricsTest extends TestBaseImpl
return node.callOnInstance(() -> HintsServiceMetrics.hintsTimedOut.getCount());
}
@SuppressWarnings("Convert2MethodRef")
private static Long countHintsRetryDifferentSystem(IInvokableInstance node)
{
return node.callOnInstance(() -> HintsServiceMetrics.hintsRetryDifferentSystem.getCount());
}
private static Long countGlobalDelays(IInvokableInstance node)
{
return getHistogramCount(node, "org.apache.cassandra.metrics.HintsService.Hint_delays");

Some files were not shown because too many files have changed in this diff Show More