diff --git a/.gitmodules b/.gitmodules index 616dacf610..6e00943162 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "modules/accord"] path = modules/accord - url = https://github.com/apache/cassandra-accord.git + url = https://github.com/apache/cassandra-accord branch = trunk diff --git a/build.xml b/build.xml index 563ac70ea2..7544c664c1 100644 --- a/build.xml +++ b/build.xml @@ -328,6 +328,7 @@ -XX:-CMSClassUnloadingEnabled -Dio.netty.tryReflectionSetAccessible=true + -XX:MaxMetaspaceSize=2G @@ -1155,6 +1156,7 @@ + @@ -1174,7 +1176,7 @@ - + @@ -1353,7 +1355,7 @@ - @@ -1664,12 +1666,14 @@ + + testtag="@{testtag}" showoutput="@{showoutput}" + maxmemory="@{maxmemory}"> @@ -1804,7 +1808,7 @@ - + diff --git a/ide/idea/workspace.xml b/ide/idea/workspace.xml index 1260f8613e..08a8a73ae4 100644 --- a/ide/idea/workspace.xml +++ b/ide/idea/workspace.xml @@ -213,7 +213,6 @@ -XX:CICompilerCount=1 -XX:HeapDumpPath=build/test -XX:MaxMetaspaceSize=2G - -Xmx4G -XX:ReservedCodeCacheSize=256M -XX:Tier4CompileThreshold=1000 -ea" /> diff --git a/modules/accord b/modules/accord index 3056d13bc8..6c6872270e 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 3056d13bc8c45a22ec794e0979d02f469cc4e209 +Subproject commit 6c6872270e16d2e777f1fa2c510b8f15396be3f3 diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java index 808dc34b68..135b5d078e 100644 --- a/src/java/org/apache/cassandra/concurrent/Stage.java +++ b/src/java/org/apache/cassandra/concurrent/Stage.java @@ -47,6 +47,7 @@ public enum Stage MUTATION (true, "MutationStage", "request", DatabaseDescriptor::getConcurrentWriters, DatabaseDescriptor::setConcurrentWriters, Stage::multiThreadedLowSignalStage), COUNTER_MUTATION (true, "CounterMutationStage", "request", DatabaseDescriptor::getConcurrentCounterWriters, DatabaseDescriptor::setConcurrentCounterWriters, Stage::multiThreadedLowSignalStage), VIEW_MUTATION (true, "ViewMutationStage", "request", DatabaseDescriptor::getConcurrentViewWriters, DatabaseDescriptor::setConcurrentViewWriters, Stage::multiThreadedLowSignalStage), + ACCORD_MIGRATION (false, "AccordMigrationReadStage", "request", DatabaseDescriptor::getConcurrentAccordOps, DatabaseDescriptor::setConcurrentAccordOps, Stage::multiThreadedLowSignalStage), GOSSIP (true, "GossipStage", "internal", () -> 1, null, Stage::singleThreadedStage), REQUEST_RESPONSE (false, "RequestResponseStage", "request", FBUtilities::getAvailableProcessors, null, Stage::multiThreadedLowSignalStage), ANTI_ENTROPY (false, "AntiEntropyStage", "internal", () -> 1, null, Stage::singleThreadedStage), diff --git a/src/java/org/apache/cassandra/concurrent/SyncFutureTask.java b/src/java/org/apache/cassandra/concurrent/SyncFutureTask.java index 422da99fb8..8176913de7 100644 --- a/src/java/org/apache/cassandra/concurrent/SyncFutureTask.java +++ b/src/java/org/apache/cassandra/concurrent/SyncFutureTask.java @@ -71,7 +71,11 @@ public class SyncFutureTask extends SyncFuture implements RunnableFuture SENSITIVE_KEYS = new HashSet() {{ add("client_encryption_options"); add("server_encryption_options"); diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 8912803564..757407ad7b 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -78,6 +78,8 @@ import org.apache.cassandra.auth.INetworkAuthorizer; import org.apache.cassandra.auth.IRoleManager; import org.apache.cassandra.config.Config.CommitLogSync; import org.apache.cassandra.config.Config.DiskAccessMode; +import org.apache.cassandra.config.Config.LWTStrategy; +import org.apache.cassandra.config.Config.NonSerialWriteStrategy; import org.apache.cassandra.config.Config.PaxosOnLinearizabilityViolation; import org.apache.cassandra.config.Config.PaxosStatePurging; import org.apache.cassandra.db.ConsistencyLevel; @@ -167,7 +169,7 @@ import static org.apache.cassandra.utils.Clock.Global.logInitializationOutcome; public class DatabaseDescriptor { public static final String NO_ACCORD_PAXOS_STRATEGY_WITH_ACCORD_DISABLED_MESSAGE = - "Cannot use legacy_paxos_strategy \"accord\" while Accord transactions are disabled."; + "Cannot use lwt_strategy \"accord\" while Accord transactions are disabled."; static { @@ -226,6 +228,7 @@ public class DatabaseDescriptor private static long keyCacheSizeInMiB; private static long paxosCacheSizeInMiB; + private static long consensusMigrationCacheSizeInMiB; private static long counterCacheSizeInMiB; private static long indexSummaryCapacityInMiB; @@ -651,6 +654,9 @@ public class DatabaseDescriptor if (conf.concurrent_counter_writes < 2) throw new ConfigurationException("concurrent_counter_writes must be at least 2, but was " + conf.concurrent_counter_writes, false); + if (conf.concurrent_accord_operations < 1) + throw new ConfigurationException("concurrent_accord_operations must be at least 1, but was " + conf.concurrent_accord_operations, false); + if (conf.networking_cache_size == null) conf.networking_cache_size = new DataStorageSpec.IntMebibytesBound(Math.min(128, (int) (Runtime.getRuntime().maxMemory() / (16 * 1048576)))); @@ -718,7 +724,6 @@ public class DatabaseDescriptor conf.commitlog_directory = storagedirFor("commitlog"); } - if (conf.accord.journal_directory == null) initializeCommitLogDiskAccessMode(); if (commitLogWriteDiskAccessMode != conf.commitlog_disk_access_mode) logger.info("commitlog_disk_access_mode resolved to: {}", commitLogWriteDiskAccessMode); @@ -959,6 +964,22 @@ public class DatabaseDescriptor + conf.paxos_cache_size + "', supported values are >= 0.", false); } + try + { + // if consensusMigrationCacheSizeInMiB option was set to "auto" then size of the cache should be "min(1% of Heap (in MB), 50MB) + consensusMigrationCacheSizeInMiB = (conf.consensus_migration_cache_size == null) + ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.01 / 1024 / 1024)), 50) + : conf.consensus_migration_cache_size.toMebibytes(); + + if (consensusMigrationCacheSizeInMiB < 0) + throw new NumberFormatException(); // to escape duplicating error message + } + catch (NumberFormatException e) + { + throw new ConfigurationException("consensus_migration_cache_size option was set incorrectly to '" + + conf.consensus_migration_cache_size + "', supported values are >= 0.", false); + } + // we need this assignment for the Settings virtual table - CASSANDRA-17735 conf.counter_cache_size = new DataStorageSpec.LongMebibytesBound(counterCacheSizeInMiB); @@ -1146,8 +1167,13 @@ public class DatabaseDescriptor if (conf.audit_logging_options != null) setAuditLoggingOptions(conf.audit_logging_options); - if (conf.legacy_paxos_strategy == Config.LegacyPaxosStrategy.accord && !conf.accord.enabled) - throw new ConfigurationException(NO_ACCORD_PAXOS_STRATEGY_WITH_ACCORD_DISABLED_MESSAGE); + if (conf.lwt_strategy == LWTStrategy.accord) + { + if (!conf.accord.enabled) + throw new ConfigurationException(NO_ACCORD_PAXOS_STRATEGY_WITH_ACCORD_DISABLED_MESSAGE); + if (conf.non_serial_write_strategy == Config.NonSerialWriteStrategy.normal) + throw new ConfigurationException("If Accord is used for LWTs then regular writes needs to be routed through Accord for interoperability by setting non_serial_write_strategy to \"accord\" or \"migration\""); + } } @VisibleForTesting @@ -2697,6 +2723,20 @@ public class DatabaseDescriptor conf.concurrent_materialized_view_writes = concurrent_materialized_view_writes; } + public static int getConcurrentAccordOps() + { + return conf.concurrent_accord_operations; + } + + public static void setConcurrentAccordOps(int concurrent_operations) + { + if (concurrent_operations < 0) + { + throw new IllegalArgumentException("Concurrent accord operations must be non-negative"); + } + conf.concurrent_accord_operations = concurrent_operations; + } + public static int getFlushWriters() { return conf.memtable_flush_writers; @@ -3613,9 +3653,45 @@ public class DatabaseDescriptor return conf.paxos_topology_repair_strict_each_quorum; } - public static Config.LegacyPaxosStrategy getLegacyPaxosStrategy() + // TODO (desired): This configuration should come out of TrM to force the cluster to agree on it + public static LWTStrategy getLWTStrategy() { - return conf.legacy_paxos_strategy; + return conf.lwt_strategy; + } + + public static void setLWTStrategy(LWTStrategy lwtStrategy) + { + conf.lwt_strategy = lwtStrategy; + } + + public static Config.NonSerialWriteStrategy getNonSerialWriteStrategy() + { + return conf.non_serial_write_strategy; + } + + public static void setNonSerialWriteStrategy(NonSerialWriteStrategy nonSerialWriteStrategy) + { + conf.non_serial_write_strategy = nonSerialWriteStrategy; + } + + public static int getAccordBarrierRetryAttempts() + { + return conf.accord_barrier_retry_attempts; + } + + public static long getAccordBarrierRetryInitialBackoffMillis() + { + return conf.accord_barrier_retry_inital_backoff_millis.toMilliseconds(); + } + + public static long getAccordBarrierRetryMaxBackoffMillis() + { + return conf.accord_barrier_max_backoff.toMilliseconds(); + } + + public static long getAccordRangeBarrierTimeoutNanos() + { + return conf.accord_range_barrier_timeout.to(TimeUnit.NANOSECONDS); } public static void setNativeTransportMaxRequestDataInFlightPerIpInBytes(long maxRequestDataInFlightInBytes) @@ -4205,6 +4281,11 @@ public class DatabaseDescriptor return paxosCacheSizeInMiB; } + public static long getConsensusMigrationCacheSizeInMiB() + { + return consensusMigrationCacheSizeInMiB; + } + public static long getCounterCacheSizeInMiB() { return counterCacheSizeInMiB; diff --git a/src/java/org/apache/cassandra/cql3/UpdateParameters.java b/src/java/org/apache/cassandra/cql3/UpdateParameters.java index 5ec6d44d31..8b0e5cdf7c 100644 --- a/src/java/org/apache/cassandra/cql3/UpdateParameters.java +++ b/src/java/org/apache/cassandra/cql3/UpdateParameters.java @@ -20,14 +20,26 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; import java.util.Map; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionPurger; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.LivenessInfo; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.db.guardrails.Guardrails; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.partitions.Partition; -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.utils.TimeUUID; @@ -39,6 +51,7 @@ public class UpdateParameters public final TableMetadata metadata; public final ClientState clientState; public final QueryOptions options; + public final boolean constructingAccordBaseUpdate; private final long nowInSec; private final long timestamp; @@ -62,6 +75,18 @@ public class UpdateParameters long nowInSec, int ttl, Map prefetchedRows) throws InvalidRequestException + { + this(metadata, clientState, options, timestamp, nowInSec, ttl, prefetchedRows, false); + } + + public UpdateParameters(TableMetadata metadata, + ClientState clientState, + QueryOptions options, + long timestamp, + long nowInSec, + int ttl, + Map prefetchedRows, + boolean constructingAccordBaseUpdate) throws InvalidRequestException { this.metadata = metadata; this.clientState = clientState; @@ -79,6 +104,8 @@ public class UpdateParameters // it to avoid potential confusion. if (timestamp == Long.MIN_VALUE) throw new InvalidRequestException(String.format("Out of bound timestamp, must be in [%d, %d]", Long.MIN_VALUE + 1, Long.MAX_VALUE)); + + this.constructingAccordBaseUpdate = constructingAccordBaseUpdate; } public void newRow(Clustering clustering) throws InvalidRequestException diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java index 939d7df767..65c7f56662 100644 --- a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java @@ -343,7 +343,7 @@ public class BatchStatement implements CQLStatement.CompositeCQLStatement } QueryOptions statementOptions = options.forStatement(i); long timestamp = attrs.getTimestamp(batchTimestamp, statementOptions); - statement.addUpdates(collector, partitionKeys.get(i), state, statementOptions, local, timestamp, nowInSeconds, requestTime); + statement.addUpdates(collector, partitionKeys.get(i), state, statementOptions, local, timestamp, nowInSeconds, requestTime, false); } if (tablesWithZeroGcGs != null) diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java index 521cd2afa6..aabcecec72 100644 --- a/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java +++ b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java @@ -223,7 +223,7 @@ final class BatchUpdatesCollector implements UpdatesCollector PartitionUpdate update = updateEntry.getValue().build(); updates.put(updateEntry.getKey(), update); } - return new Mutation(keyspaceName, key, updates.build(), createdAt); + return new Mutation(keyspaceName, key, updates.build(), createdAt, false); } public PartitionUpdate.Builder get(TableId tableId) diff --git a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java index e511af3190..b2edb6cc33 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java +++ b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java @@ -31,14 +31,18 @@ import java.util.TreeMap; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import accord.api.Update; import accord.primitives.Txn; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.UpdateParameters; import org.apache.cassandra.cql3.conditions.ColumnCondition; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.Columns; +import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.SinglePartitionReadCommand; @@ -53,7 +57,6 @@ import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.db.partitions.Partition; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; -import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.service.CASRequest; import org.apache.cassandra.service.ClientState; @@ -63,6 +66,7 @@ import org.apache.cassandra.service.accord.txn.TxnDataName; import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.service.accord.txn.TxnReference; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.accord.txn.TxnUpdate; import org.apache.cassandra.service.accord.txn.TxnWrite; import org.apache.cassandra.service.paxos.Ballot; @@ -70,13 +74,21 @@ import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.TimeUUID; import static com.google.common.base.Preconditions.checkState; -import static org.apache.cassandra.service.accord.txn.TxnDataName.Kind.USER; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.RETRY_NEW_PROTOCOL; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.casResult; +import static org.apache.cassandra.service.accord.txn.TxnDataName.Kind.CAS_READ; +import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.retry_new_protocol; + /** * Processed CAS conditions and update on potentially multiple rows of the same partition. */ public class CQL3CasRequest implements CASRequest { + @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(CQL3CasRequest.class); + public final TableMetadata metadata; public final DecoratedKey key; private final RegularAndStaticColumns conditionColumns; @@ -410,7 +422,7 @@ public class CQL3CasRequest implements CASRequest public TxnCondition asTxnCondition() { - TxnDataName txnDataName = new TxnDataName(USER, clustering, TxnRead.SERIAL_READ_NAME); + TxnDataName txnDataName = new TxnDataName(CAS_READ, clustering, TxnRead.CAS_READ_NAME); TxnReference txnReference = new TxnReference(txnDataName, null); return new TxnCondition.Exists(txnReference, TxnCondition.Kind.IS_NULL); } @@ -436,7 +448,7 @@ public class CQL3CasRequest implements CASRequest public TxnCondition asTxnCondition() { - TxnDataName txnDataName = new TxnDataName(USER, clustering, TxnRead.SERIAL_READ_NAME); + TxnDataName txnDataName = new TxnDataName(CAS_READ, clustering, TxnRead.CAS_READ_NAME); TxnReference txnReference = new TxnReference(txnDataName, null); return new TxnCondition.Exists(txnReference, TxnCondition.Kind.IS_NOT_NULL); } @@ -484,20 +496,26 @@ public class CQL3CasRequest implements CASRequest } @Override - public Txn toAccordTxn(ClientState clientState, long nowInSecs) + public Txn toAccordTxn(ConsistencyLevel consistencyLevel, ConsistencyLevel commitConsistencyLevel, ClientState clientState, long nowInSecs) { SinglePartitionReadCommand readCommand = readCommand(nowInSecs); - Update update = createUpdate(clientState); - // In a CAS request only one key is supported and writes + Update update = createUpdate(clientState, commitConsistencyLevel); + // If the write strategy is sending all writes through Accord there is no need to use the supplied consistency + // level since Accord will manage reading safely + consistencyLevel = DatabaseDescriptor.getNonSerialWriteStrategy().readCLForStrategy(consistencyLevel); + TxnRead read = TxnRead.createCasRead(readCommand, consistencyLevel); + // In a CAS requesting only one key is supported and writes // can't be dependent on any data that is read (only conditions) // so the only relevant keys are the read key - TxnRead read = TxnRead.createSerialRead(readCommand); return new Txn.InMemory(read.keys(), read, TxnQuery.CONDITION, update); } - private Update createUpdate(ClientState clientState) + private Update createUpdate(ClientState clientState, ConsistencyLevel commitConsistencyLevel) { - return new TxnUpdate(createWriteFragments(clientState), createCondition()); + // Potentially ignore commit consistency level if non-SERIAL write strategy is Accord + // since it is safe to match what non-SERIAL writes do + commitConsistencyLevel = DatabaseDescriptor.getNonSerialWriteStrategy().commitCLForStrategy(commitConsistencyLevel); + return new TxnUpdate(createWriteFragments(clientState), createCondition(), commitConsistencyLevel); } private TxnCondition createCondition() @@ -528,13 +546,23 @@ public class CQL3CasRequest implements CASRequest TxnWrite.Fragment fragment = modification.getTxnWriteFragment(idx++, state, options); fragments.add(fragment); } + for (RangeDeletion rangeDeletion : rangeDeletions) + { + ModificationStatement modification = rangeDeletion.stmt; + QueryOptions options = rangeDeletion.options; + TxnWrite.Fragment fragment = modification.getTxnWriteFragment(idx++, state, options); + fragments.add(fragment); + } return fragments; } @Override - public RowIterator toCasResult(TxnData txnData) + public ConsensusAttemptResult toCasResult(TxnResult txnResult) { - FilteredPartition partition = txnData.get(TxnRead.SERIAL_READ); - return partition != null ? partition.rowIterator() : null; + if (txnResult.kind() == retry_new_protocol) + return RETRY_NEW_PROTOCOL; + TxnData txnData = (TxnData)txnResult; + FilteredPartition partition = txnData.get(TxnRead.CAS_READ); + return casResult(partition != null ? partition.rowIterator(false) : null); } } diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java index 51f51e6407..19556a766b 100644 --- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java @@ -57,19 +57,10 @@ import org.apache.cassandra.cql3.UpdateParameters; import org.apache.cassandra.cql3.Validation; import org.apache.cassandra.cql3.VariableSpecifications; import org.apache.cassandra.cql3.WhereClause; -import org.apache.cassandra.cql3.constraints.ConstraintViolationException; -import org.apache.cassandra.db.guardrails.Guardrails; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.locator.ReplicaLayout; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.ViewMetadata; import org.apache.cassandra.cql3.conditions.ColumnCondition; import org.apache.cassandra.cql3.conditions.ColumnConditions; import org.apache.cassandra.cql3.conditions.Conditions; +import org.apache.cassandra.cql3.constraints.ConstraintViolationException; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.restrictions.StatementRestrictions; import org.apache.cassandra.cql3.selection.ResultSetBuilder; @@ -94,6 +85,7 @@ import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.filter.DataLimits; import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.guardrails.Guardrails; import org.apache.cassandra.db.marshal.BooleanType; import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.db.partitions.Partition; @@ -102,11 +94,19 @@ import org.apache.cassandra.db.partitions.PartitionIterators; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.view.View; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.RequestValidationException; import org.apache.cassandra.exceptions.UnauthorizedException; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.metrics.ClientRequestSizeMetrics; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.ViewMetadata; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageProxy; @@ -641,7 +641,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa false, options.getTimestamp(queryState), options.getNowInSeconds(queryState), - requestTime); + requestTime, + false); if (!mutations.isEmpty()) { StorageProxy.mutateWithTriggers(mutations, cl, false, requestTime); @@ -806,7 +807,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa { long timestamp = options.getTimestamp(queryState); long nowInSeconds = options.getNowInSeconds(queryState); - for (IMutation mutation : getMutations(queryState.getClientState(), options, true, timestamp, nowInSeconds, requestTime)) + for (IMutation mutation : getMutations(queryState.getClientState(), options, true, timestamp, nowInSeconds, requestTime, false)) mutation.apply(); return null; } @@ -834,7 +835,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa } if (!request.appliesTo(current)) - return current.rowIterator(); + return current.rowIterator(false); PartitionUpdate updates = request.makeUpdates(current, state, ballot); updates = TriggerExecutor.instance.execute(updates); @@ -859,19 +860,22 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa boolean local, long timestamp, long nowInSeconds, - Dispatcher.RequestTime requestTime) + Dispatcher.RequestTime requestTime, + boolean constructingAccordBaseUpdate) { List keys = buildPartitionKeyNames(options, state); - if(keys.size() == 1) + + if (keys.size() == 1) { SingleTableSinglePartitionUpdatesCollector collector = new SingleTableSinglePartitionUpdatesCollector(metadata, updatedColumns); - addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, requestTime); + addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, requestTime, constructingAccordBaseUpdate); return collector.toMutations(state); - } else + } + else { HashMultiset perPartitionKeyCounts = HashMultiset.create(keys); SingleTableUpdatesCollector collector = new SingleTableUpdatesCollector(metadata, updatedColumns, perPartitionKeyCounts); - addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, requestTime); + addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, requestTime, constructingAccordBaseUpdate); return collector.toMutations(state); } } @@ -879,7 +883,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa @VisibleForTesting public PartitionUpdate getTxnUpdate(ClientState state, QueryOptions options) { - List mutations = getMutations(state, options, false, 0, 0, new Dispatcher.RequestTime(0, 0)); + List mutations = getMutations(state, options, false, 0, 0, new Dispatcher.RequestTime(0, 0), true); if (mutations.size() != 1) throw new IllegalArgumentException("When running withing a transaction, modification statements may only mutate a single partition"); return Iterables.getOnlyElement(mutations.get(0).getPartitionUpdates()); @@ -942,7 +946,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa boolean local, long timestamp, long nowInSeconds, - Dispatcher.RequestTime requestTime) + Dispatcher.RequestTime requestTime, + boolean constructingAccordBaseUpdate) { if (hasSlices()) { @@ -960,7 +965,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa local, timestamp, nowInSeconds, - requestTime); + requestTime, + constructingAccordBaseUpdate); for (ByteBuffer key : keys) { Validation.validateKey(metadata(), key); @@ -984,7 +990,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa if (restrictions.hasClusteringColumnsRestrictions() && clusterings.isEmpty()) return; - UpdateParameters params = makeUpdateParameters(keys, clusterings, state, options, local, timestamp, nowInSeconds, requestTime); + UpdateParameters params = makeUpdateParameters(keys, clusterings, state, options, local, timestamp, nowInSeconds, requestTime, constructingAccordBaseUpdate); for (ByteBuffer key : keys) { @@ -1042,7 +1048,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa boolean local, long timestamp, long nowInSeconds, - Dispatcher.RequestTime requestTime) + Dispatcher.RequestTime requestTime, + boolean constructingAccordBaseUpdate) { if (clusterings.contains(Clustering.STATIC_CLUSTERING)) return makeUpdateParameters(keys, @@ -1053,7 +1060,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa local, timestamp, nowInSeconds, - requestTime); + requestTime, + constructingAccordBaseUpdate); return makeUpdateParameters(keys, new ClusteringIndexNamesFilter(clusterings, false), @@ -1063,7 +1071,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa local, timestamp, nowInSeconds, - requestTime); + requestTime, + constructingAccordBaseUpdate); } private UpdateParameters makeUpdateParameters(Collection keys, @@ -1074,7 +1083,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa boolean local, long timestamp, long nowInSeconds, - Dispatcher.RequestTime requestTime) + Dispatcher.RequestTime requestTime, + boolean constructingAccordBaseUpdate) { // Some lists operation requires reading Map lists = @@ -1092,7 +1102,8 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa getTimestamp(timestamp, options), nowInSeconds, getTimeToLive(options), - lists); + lists, + constructingAccordBaseUpdate); } public static abstract class Parsed extends QualifiedStatement diff --git a/src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java b/src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java index e56fd1ec1d..cff94f9526 100644 --- a/src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java @@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory; import accord.api.Key; import accord.primitives.Keys; import accord.primitives.Txn; -import accord.utils.Invariants; import org.apache.cassandra.audit.AuditLogContext; import org.apache.cassandra.audit.AuditLogEntryType; import org.apache.cassandra.config.DatabaseDescriptor; @@ -63,12 +62,11 @@ import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.SinglePartitionReadQuery; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.partitions.FilteredPartition; -import org.apache.cassandra.exceptions.ExceptionCode; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.accord.AccordService; -import org.apache.cassandra.service.accord.api.AccordRoutableKey; +import org.apache.cassandra.service.accord.txn.AccordUpdate; import org.apache.cassandra.service.accord.txn.TxnCondition; import org.apache.cassandra.service.accord.txn.TxnData; import org.apache.cassandra.service.accord.txn.TxnDataName; @@ -76,18 +74,18 @@ import org.apache.cassandra.service.accord.txn.TxnNamedRead; import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.service.accord.txn.TxnReference; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.accord.txn.TxnUpdate; import org.apache.cassandra.service.accord.txn.TxnWrite; import org.apache.cassandra.transport.Dispatcher; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.ClusterMetadataService; -import org.apache.cassandra.tcm.transformations.AddAccordKeyspace; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse; import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull; import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue; +import static org.apache.cassandra.service.accord.txn.TxnRead.createTxnRead; +import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.retry_new_protocol; public class TransactionStatement implements CQLStatement.CompositeCQLStatement, CQLStatement.ReturningCQLStatement { @@ -302,9 +300,9 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement, return fragments; } - TxnUpdate createUpdate(ClientState state, QueryOptions options, Map autoReads, Consumer keyConsumer) + AccordUpdate createUpdate(ClientState state, QueryOptions options, Map autoReads, Consumer keyConsumer) { - return new TxnUpdate(createWriteFragments(state, options, autoReads, keyConsumer), createCondition(options)); + return new TxnUpdate(createWriteFragments(state, options, autoReads, keyConsumer), createCondition(options), null); } Keys toKeys(SortedSet keySet) @@ -323,16 +321,16 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement, Preconditions.checkState(conditions.isEmpty(), "No condition should exist without updates present"); List reads = createNamedReads(options, state, ImmutableMap.of(), keySet::add); Keys txnKeys = toKeys(keySet); - TxnRead read = new TxnRead(reads, txnKeys); + TxnRead read = createTxnRead(reads, txnKeys, null); return new Txn.InMemory(txnKeys, read, TxnQuery.ALL); } else { Map autoReads = new HashMap<>(); - TxnUpdate update = createUpdate(state, options, autoReads, keySet::add); + AccordUpdate update = createUpdate(state, options, autoReads, keySet::add); List reads = createNamedReads(options, state, autoReads, keySet::add); Keys txnKeys = toKeys(keySet); - TxnRead read = new TxnRead(reads, txnKeys); + TxnRead read = createTxnRead(reads, txnKeys, null); return new Txn.InMemory(txnKeys, read, TxnQuery.ALL, update); } } @@ -357,40 +355,6 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement, return select.getLimit(options) != 1; } - private void maybeConvertTablesToAccord(Txn txn) - { - Set allKeyspaces = new HashSet<>(); - Set newKeyspaces = new HashSet<>(); - txn.keys().forEach(key -> { - String keyspace = ((AccordRoutableKey) key).keyspace(); - if (allKeyspaces.add(keyspace) && !AccordService.instance().isAccordManagedKeyspace(keyspace)) - newKeyspaces.add(keyspace); - }); - - if (newKeyspaces.isEmpty()) - return; - - for (String keyspace : newKeyspaces) - { - ClusterMetadataService.instance().commit(new AddAccordKeyspace(keyspace), - metadata -> null, - (code, message) -> { - Invariants.checkState(code == ExceptionCode.ALREADY_EXISTS, - "Expected %s, got %s", ExceptionCode.ALREADY_EXISTS, code); - return null; - }); - } - - // we need to avoid creating a txnId in an epoch when no one has any ranges - FBUtilities.waitOnFuture(AccordService.instance().epochReady(ClusterMetadata.current().epoch)); - - for (String keyspace : allKeyspaces) - { - if (!AccordService.instance().isAccordManagedKeyspace(keyspace)) - throw new IllegalStateException(keyspace + " is not an accord managed keyspace"); - } - } - @Override public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime) { @@ -407,9 +371,12 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement, Txn txn = createTxn(state.getClientState(), options); - maybeConvertTablesToAccord(txn); + AccordService.instance().maybeConvertKeyspacesToAccord(txn); - TxnData data = AccordService.instance().coordinate(txn, options.getConsistency()); + TxnResult txnResult = AccordService.instance().coordinate(txn, options.getConsistency(), requestTime); + if (txnResult.kind() == retry_new_protocol) + throw new IllegalStateException("Transaction statement should never be required to switch consensus protocols"); + TxnData data = (TxnData)txnResult; if (returningSelect != null) { @@ -420,8 +387,9 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement, if (selectQuery.queries.size() == 1) { FilteredPartition partition = data.get(TxnDataName.returning()); + boolean reversed = selectQuery.queries.get(0).isReversed(); if (partition != null) - returningSelect.select.processPartition(partition.rowIterator(), options, result, FBUtilities.nowInSeconds()); + returningSelect.select.processPartition(partition.rowIterator(reversed), options, result, FBUtilities.nowInSeconds()); } else { @@ -429,8 +397,9 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement, for (int i = 0; i < selectQuery.queries.size(); i++) { FilteredPartition partition = data.get(TxnDataName.returning(i)); + boolean reversed = selectQuery.queries.get(i).isReversed(); if (partition != null) - returningSelect.select.processPartition(partition.rowIterator(), options, result, nowInSec); + returningSelect.select.processPartition(partition.rowIterator(reversed), options, result, nowInSec); } } return new ResultMessage.Rows(result.build()); diff --git a/src/java/org/apache/cassandra/cql3/terms/Lists.java b/src/java/org/apache/cassandra/cql3/terms/Lists.java index 1533162267..e82f0f5ac0 100644 --- a/src/java/org/apache/cassandra/cql3/terms/Lists.java +++ b/src/java/org/apache/cassandra/cql3/terms/Lists.java @@ -27,6 +27,10 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.cql3.AssignmentTestable; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.ColumnSpecification; @@ -34,21 +38,25 @@ import org.apache.cassandra.cql3.Operation; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.UpdateParameters; import org.apache.cassandra.cql3.VariableSpecifications; +import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.guardrails.Guardrails; -import org.apache.cassandra.db.marshal.MultiElementType; -import org.apache.cassandra.schema.ColumnMetadata; -import com.google.common.annotations.VisibleForTesting; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.MultiElementType; +import org.apache.cassandra.db.marshal.TimeUUIDType; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.db.rows.ComplexColumnData; +import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.TimeUUID; -import static org.apache.cassandra.cql3.terms.Constants.UNSET_VALUE; import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse; import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; +import static org.apache.cassandra.cql3.terms.Constants.UNSET_VALUE; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.TimeUUID.Generator.atUnixMillisAsBytes; @@ -57,6 +65,16 @@ import static org.apache.cassandra.utils.TimeUUID.Generator.atUnixMillisAsBytes; */ public abstract class Lists { + @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(Lists.class); + + /** + * Sentinel value indicating the cell path should be replaced by Accord with one based on the transaction executeAt + */ + private static final TimeUUID ACCORD_CELL_PATH_SENTINEL_UUID = TimeUUID.atUnixMicrosWithLsb(0, 0); + public static final CellPath ACCORD_DUMMY_CELL_PATH = CellPath.create(ACCORD_CELL_PATH_SENTINEL_UUID.toBytes()); + private static final long ACCORD_CELL_PATH_SENTINEL_MSB = ACCORD_CELL_PATH_SENTINEL_UUID.msb(); + private Lists() {} public static ColumnSpecification indexSpecOf(ColumnSpecification column) @@ -142,6 +160,33 @@ public abstract class Lists return type == null ? null : ListType.getInstance(type, false); } + /** + * Return a function that given a cell with an ACCORD_CELL_PATH_SENTINEL_MSB will + * return a new CellPath with a TimeUUID that increases monotonically every time it is called or + * the existing cell path if path does not contain ACCORD_CELL_PATH_SENTINEL_MSB. + * + * Only intended to work with list cell paths where list append needs a timestamp based on the executeAt + * of the Accord transaction appending the cell. + * @param timestampMicros executeAt timestamp to use as the MSB for generated cell paths + */ + public static com.google.common.base.Function accordListPathSupplier(long timestampMicros) + { + return new com.google.common.base.Function() + { + final long timeUuidMsb = TimeUUID.unixMicrosToMsb(timestampMicros); + long cellIndex = 0; + @Override + public CellPath apply(Cell cell) + { + CellPath path = cell.path(); + if (ACCORD_CELL_PATH_SENTINEL_MSB == path.get(0).getLong(0)) + return CellPath.create(ByteBuffer.wrap(TimeUUID.toBytes(timeUuidMsb, TimeUUIDType.signedBytesToNativeLong(cellIndex++)))); + else + return path; + } + }; + } + public static class Literal extends Term.Raw { private final List elements; @@ -406,11 +451,18 @@ public abstract class Lists // during SSTable write. Guardrails.itemsPerCollection.guard(type.collectionSize(elements), column.name.toString(), false, params.clientState); + long cellIndex = 0; int dataSize = 0; for (ByteBuffer buffer : elements) { - ByteBuffer uuid = ByteBuffer.wrap(params.nextTimeUUIDAsBytes()); - Cell cell = params.addCell(column, CellPath.create(uuid), buffer); + ByteBuffer cellPath; + // Accord will need to replace this value later once it knows the executeAt timestamp + // so just put a TimeUUID with MSB sentinel for now + if (params.constructingAccordBaseUpdate) + cellPath = TimeUUID.atUnixMicrosWithLsb(0, cellIndex++).toBytes(); + else + cellPath = ByteBuffer.wrap(params.nextTimeUUIDAsBytes()); + Cell cell = params.addCell(column, CellPath.create(cellPath), buffer); dataSize += cell.dataSize(); } Guardrails.collectionListSize.guard(dataSize, column.name.toString(), false, params.clientState); diff --git a/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java b/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java index fe3acdba06..76b765ae70 100644 --- a/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java +++ b/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java @@ -85,7 +85,9 @@ public abstract class AbstractMutationVerbHandler implement } } - if (!forToken.get().containsSelf()) + // Mutations may intentionally be sent against an older Epoch so out of range checking doesn't work + // and could cause data to not end up where it needs to be for future operations + if (!message.payload.allowsOutOfRangeMutations() && !forToken.get().containsSelf()) { StorageService.instance.incOutOfRangeOperationCount(); Keyspace.open(message.payload.getKeyspaceName()).metric.outOfRangeTokenWrites.inc(); @@ -93,7 +95,7 @@ public abstract class AbstractMutationVerbHandler implement throw InvalidRoutingException.forWrite(respondTo, key.getToken(), metadata.epoch, message.payload); } - if (forToken.lastModified().isAfter(message.epoch())) + if (!message.payload.allowsOutOfRangeMutations() && forToken.lastModified().isAfter(message.epoch())) { TCMMetrics.instance.coordinatorBehindPlacements.mark(); throw new CoordinatorBehindException(String.format("Routing is correct, but coordinator needs to catch-up at least to epoch %s to maintain consistency. Current coordinator epoch is %s", diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index d47a651a06..0fcdb192fd 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -3339,4 +3339,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner { return metric; } + + public TableId getTableId() + { + return metadata().id; + } } diff --git a/src/java/org/apache/cassandra/db/IMutation.java b/src/java/org/apache/cassandra/db/IMutation.java index 1998e2c035..ba8d586dea 100644 --- a/src/java/org/apache/cassandra/db/IMutation.java +++ b/src/java/org/apache/cassandra/db/IMutation.java @@ -70,4 +70,9 @@ public interface IMutation } return size; } + + default boolean allowsOutOfRangeMutations() + { + return false; + } } diff --git a/src/java/org/apache/cassandra/db/Mutation.java b/src/java/org/apache/cassandra/db/Mutation.java index 0861bb64c4..1f9d2c86be 100644 --- a/src/java/org/apache/cassandra/db/Mutation.java +++ b/src/java/org/apache/cassandra/db/Mutation.java @@ -18,7 +18,13 @@ package org.apache.cassandra.db; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.function.Supplier; @@ -56,6 +62,7 @@ import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; public class Mutation implements IMutation, Supplier { public static final MutationSerializer serializer = new MutationSerializer(); + public static final int ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG = 0x01; // todo this is redundant // when we remove it, also restore SerializationsTest.testMutationRead to not regenerate new Mutations each test @@ -84,23 +91,26 @@ public class Mutation implements IMutation, Supplier /** @see CassandraRelevantProperties#CACHEABLE_MUTATION_SIZE_LIMIT */ private static final long CACHEABLE_MUTATION_SIZE_LIMIT = CassandraRelevantProperties.CACHEABLE_MUTATION_SIZE_LIMIT.getLong(); + private boolean allowOutOfRangeMutations; + public Mutation(PartitionUpdate update) { this(update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now(), update.metadata().params.cdc); } - public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap modifications, long approxCreatedAtNanos) + public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap modifications, long approxCreatedAtNanos, boolean allowOutOfRangeMutations) { - this(keyspaceName, key, modifications, approxCreatedAtNanos, cdcEnabled(modifications.values())); + this(keyspaceName, key, modifications, approxCreatedAtNanos, cdcEnabled(modifications.values()), allowOutOfRangeMutations); } - public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap modifications, long approxCreatedAtNanos, boolean cdcEnabled) + public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap modifications, long approxCreatedAtNanos, boolean cdcEnabled, boolean allowOutOfRangeMutations) { this.keyspaceName = keyspaceName; this.key = key; this.modifications = modifications; this.cdcEnabled = cdcEnabled; this.approxCreatedAtNanos = approxCreatedAtNanos; + this.allowOutOfRangeMutations = allowOutOfRangeMutations; } private static boolean cdcEnabled(Iterable modifications) @@ -125,7 +135,7 @@ public class Mutation implements IMutation, Supplier } } - return new Mutation(keyspaceName, key, builder.build(), approxCreatedAtNanos); + return new Mutation(keyspaceName, key, builder.build(), approxCreatedAtNanos, allowOutOfRangeMutations); } public Mutation without(TableId tableId) @@ -201,18 +211,22 @@ public class Mutation implements IMutation, Supplier * @throws IllegalArgumentException if not all the mutations are on the same * keyspace and key. */ - public static Mutation merge(List mutations) + public static Mutation merge(Collection mutations) { assert !mutations.isEmpty(); - if (mutations.size() == 1) - return mutations.get(0); + if (mutations.size() == ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG) + return mutations.iterator().next(); Set updatedTables = new HashSet<>(); String ks = null; DecoratedKey key = null; + Boolean allowOutOfRangeMutations = null; for (Mutation mutation : mutations) { + if (allowOutOfRangeMutations != null && allowOutOfRangeMutations != mutation.allowOutOfRangeMutations) + throw new IllegalArgumentException("Can't merge mutations with differing policies on allowing out of range mutations"); + allowOutOfRangeMutations = mutation.allowOutOfRangeMutations; updatedTables.addAll(mutation.modifications.keySet()); if (ks != null && !ks.equals(mutation.keyspaceName)) throw new IllegalArgumentException(); @@ -236,10 +250,10 @@ public class Mutation implements IMutation, Supplier if (updates.isEmpty()) continue; - modifications.put(table, updates.size() == 1 ? updates.get(0) : PartitionUpdate.merge(updates)); + modifications.put(table, updates.size() == ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG ? updates.get(0) : PartitionUpdate.merge(updates)); updates.clear(); } - return new Mutation(ks, key, modifications.build(), approxTime.now()); + return new Mutation(ks, key, modifications.build(), approxTime.now(), allowOutOfRangeMutations); } public Future applyFuture() @@ -296,6 +310,27 @@ public class Mutation implements IMutation, Supplier return cdcEnabled; } + public Mutation allowOutOfRangeMutations() + { + allowOutOfRangeMutations = true; + return this; + } + + public boolean allowsOutOfRangeMutations() + { + return allowOutOfRangeMutations; + } + + private static int allowsOutOfRangeMutationsFlag(boolean allowOutOfRangeMutations) + { + return allowOutOfRangeMutations ? ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG : 0; + } + + private static boolean allowsOutOfRangeMutations(int flags) + { + return (flags & ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG) != 0; + } + public String toString() { return toString(false); @@ -481,6 +516,9 @@ public class Mutation implements IMutation, Supplier { Map modifications = mutation.modifications; + if (version >= VERSION_51) + out.write(allowsOutOfRangeMutationsFlag(mutation.allowsOutOfRangeMutations())); + /* serialize the modifications in the mutation */ int size = modifications.size(); out.writeUnsignedVInt32(size); @@ -500,6 +538,12 @@ public class Mutation implements IMutation, Supplier { teeIn = new TeeDataInputPlus(in, dob, CACHEABLE_MUTATION_SIZE_LIMIT); + boolean allowsOutOfRangeMutations = false; + if (version >= VERSION_51) + { + int flags = in.readByte(); + allowsOutOfRangeMutations = allowsOutOfRangeMutations(flags); + } int size = teeIn.readUnsignedVInt32(); assert size > 0; @@ -519,7 +563,7 @@ public class Mutation implements IMutation, Supplier update = PartitionUpdate.serializer.deserialize(teeIn, version, flag); modifications.put(update.metadata().id, update); } - m = new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now()); + m = new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now(), allowsOutOfRangeMutations); } //Only cache serializations that don't hit the limit @@ -597,7 +641,9 @@ public class Mutation implements IMutation, Supplier long size = this.size; if (size == 0L) { - size = TypeSizes.sizeofUnsignedVInt(mutation.modifications.size()); + if (version >= VERSION_51) + size += ALLOW_OUT_OF_RANGE_MUTATIONS_FLAG; // flags + size += TypeSizes.sizeofUnsignedVInt(mutation.modifications.size()); for (PartitionUpdate partitionUpdate : mutation.modifications.values()) size += serializer.serializedSize(partitionUpdate, version); this.size = size; @@ -650,7 +696,7 @@ public class Mutation implements IMutation, Supplier public Mutation build() { - return new Mutation(keyspaceName, key, modifications.build(), approxCreatedAtNanos); + return new Mutation(keyspaceName, key, modifications.build(), approxCreatedAtNanos, false); } } } diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java index 4926061cb8..6a5301946a 100644 --- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java +++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java @@ -71,6 +71,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR boolean isDigest, int digestVersion, boolean acceptsTransient, + boolean allowOutOfRangeReads, TableMetadata metadata, long nowInSec, ColumnFilter columnFilter, @@ -80,7 +81,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR Index.QueryPlan indexQueryPlan, boolean trackWarnings) { - super(serializedAtEpoch, Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange); + super(serializedAtEpoch, Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, allowOutOfRangeReads, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange); this.requestedSlices = dataRange.clusteringIndexFilter.getSlices(metadata()); } @@ -88,6 +89,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR boolean isDigest, int digestVersion, boolean acceptsTransient, + boolean allowsOutOfRangeReads, TableMetadata metadata, long nowInSec, ColumnFilter columnFilter, @@ -115,6 +117,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR isDigest, digestVersion, acceptsTransient, + allowsOutOfRangeReads, metadata, nowInSec, columnFilter, @@ -136,6 +139,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR false, 0, false, + false, metadata, nowInSec, columnFilter, @@ -160,6 +164,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR false, 0, false, + false, metadata, nowInSec, ColumnFilter.all(metadata), @@ -206,6 +211,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR isDigestQuery(), digestVersion(), acceptsTransient(), + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -222,6 +228,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR isDigestQuery(), digestVersion(), acceptsTransient(), + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -239,6 +246,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR true, digestVersion(), false, + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -256,6 +264,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR false, 0, true, + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -273,6 +282,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR isDigestQuery(), digestVersion(), acceptsTransient(), + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -290,6 +300,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR isDigestQuery(), digestVersion(), acceptsTransient(), + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -525,6 +536,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR boolean isDigest, int digestVersion, boolean acceptsTransient, + boolean allowsOutOfRangeReads, TableMetadata metadata, long nowInSec, ColumnFilter columnFilter, @@ -534,7 +546,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR throws IOException { DataRange range = DataRange.serializer.deserialize(in, version, metadata); - return PartitionRangeReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, range, indexQueryPlan, false); + return PartitionRangeReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, allowsOutOfRangeReads, metadata, nowInSec, columnFilter, rowFilter, limits, range, indexQueryPlan, false); } } @@ -552,7 +564,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR Index.QueryPlan indexQueryPlan, boolean trackWarnings) { - super(metadata.epoch, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, indexQueryPlan, trackWarnings); + super(metadata.epoch, isDigest, digestVersion, acceptsTransient, true, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, indexQueryPlan, trackWarnings); } @Override diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index e4ea5f12d7..d6b0f4734d 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -18,44 +18,62 @@ package org.apache.cassandra.db; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; -import java.util.function.LongPredicate; import java.util.function.Function; +import java.util.function.LongPredicate; import java.util.stream.Collectors; - import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.netty.util.concurrent.FastThreadLocal; -import org.apache.cassandra.config.*; -import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DataStorageSpec; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.filter.ClusteringIndexFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.filter.LocalReadSizeTooLargeException; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.filter.TombstoneOverwhelmingException; +import org.apache.cassandra.db.partitions.PurgeFunction; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; import org.apache.cassandra.db.transform.BasePartitions; import org.apache.cassandra.db.transform.BaseRows; +import org.apache.cassandra.db.transform.RTBoundCloser; +import org.apache.cassandra.db.transform.RTBoundValidator; +import org.apache.cassandra.db.transform.RTBoundValidator.Stage; +import org.apache.cassandra.db.transform.StoppingTransformation; +import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.exceptions.CoordinatorBehindException; import org.apache.cassandra.exceptions.QueryCancelledException; +import org.apache.cassandra.exceptions.UnknownIndexException; import org.apache.cassandra.exceptions.UnknownTableException; import org.apache.cassandra.metrics.TCMMetrics; import org.apache.cassandra.net.MessageFlag; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.ParamType; import org.apache.cassandra.net.Verb; -import org.apache.cassandra.db.partitions.*; -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.db.transform.RTBoundCloser; -import org.apache.cassandra.db.transform.RTBoundValidator; -import org.apache.cassandra.db.transform.RTBoundValidator.Stage; -import org.apache.cassandra.db.transform.StoppingTransformation; -import org.apache.cassandra.db.transform.Transformation; -import org.apache.cassandra.exceptions.UnknownIndexException; import org.apache.cassandra.index.Index; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -67,16 +85,16 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SchemaProvider; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.SchemaProvider; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.CassandraUInt; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.utils.CassandraUInt; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.NoSpamLogger; import org.apache.cassandra.utils.ObjectSizes; @@ -84,8 +102,8 @@ import org.apache.cassandra.utils.TimeUUID; import static com.google.common.collect.Iterables.any; import static com.google.common.collect.Iterables.filter; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.db.partitions.UnfilteredPartitionIterators.MergeListener.NOOP; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; /** @@ -110,6 +128,7 @@ public abstract class ReadCommand extends AbstractReadQuery private final boolean isDigestQuery; private final boolean acceptsTransient; private final Epoch serializedAtEpoch; + private boolean allowsOutOfRangeReads; // if a digest query, the version for which the digest is expected. Ignored if not a digest. private int digestVersion; @@ -128,6 +147,7 @@ public abstract class ReadCommand extends AbstractReadQuery boolean isDigest, int digestVersion, boolean acceptsTransient, + boolean allowsOutOfRangeReads, TableMetadata metadata, long nowInSec, ColumnFilter columnFilter, @@ -154,6 +174,7 @@ public abstract class ReadCommand extends AbstractReadQuery boolean isDigestQuery, int digestVersion, boolean acceptsTransient, + boolean allowsOutOfRangeReads, TableMetadata metadata, long nowInSec, ColumnFilter columnFilter, @@ -172,6 +193,7 @@ public abstract class ReadCommand extends AbstractReadQuery this.digestVersion = digestVersion; this.acceptsTransient = acceptsTransient; this.indexQueryPlan = indexQueryPlan; + this.allowsOutOfRangeReads = allowsOutOfRangeReads; this.trackWarnings = trackWarnings; this.serializedAtEpoch = serializedAtEpoch; this.dataRange = dataRange; @@ -528,6 +550,17 @@ public abstract class ReadCommand extends AbstractReadQuery return ReadExecutionController.forCommand(this, false); } + public ReadCommand allowOutOfRangeReads() + { + allowsOutOfRangeReads = true; + return this; + } + + public boolean allowsOutOfRangeReads() + { + return allowsOutOfRangeReads; + } + /** * Wraps the provided iterator so that metrics on what is scanned by the command are recorded. * This also log warning/trow TombstoneOverwhelmingException if appropriate. @@ -874,7 +907,7 @@ public abstract class ReadCommand extends AbstractReadQuery // Skip purgeable tombstones. We do this because it's safe to do (post-merge of the memtable and sstable at least), it // can save us some bandwith, and avoid making us throw a TombstoneOverwhelmingException for purgeable tombstones (which // are to some extend an artefact of compaction lagging behind and hence counting them is somewhat unintuitive). - protected UnfilteredPartitionIterator withoutPurgeableTombstones(UnfilteredPartitionIterator iterator, + protected UnfilteredPartitionIterator withoutPurgeableTombstones(UnfilteredPartitionIterator iterator, ColumnFamilyStore cfs, ReadExecutionController controller) { @@ -1217,6 +1250,7 @@ public abstract class ReadCommand extends AbstractReadQuery private static final int HAS_INDEX = 0x04; private static final int ACCEPTS_TRANSIENT = 0x08; private static final int NEEDS_RECONCILIATION = 0x10; + private static final int ALLOWS_OUT_OF_RANGE_READS = 0x20; private final SchemaProvider schema; @@ -1281,6 +1315,16 @@ public abstract class ReadCommand extends AbstractReadQuery return (flags & NEEDS_RECONCILIATION) != 0; } + private static int allowsOutOfRangeReadsFlag(boolean allowsOutOfRangeReads) + { + return allowsOutOfRangeReads ? ALLOWS_OUT_OF_RANGE_READS: 0; + } + + private static boolean allowsOutOfRangeReads(int flags) + { + return (flags & ALLOWS_OUT_OF_RANGE_READS) != 0; + } + public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException { out.writeByte(command.kind.ordinal()); @@ -1289,6 +1333,7 @@ public abstract class ReadCommand extends AbstractReadQuery | indexFlag(null != command.indexQueryPlan()) | acceptsTransientFlag(command.acceptsTransient()) | needsReconciliationFlag(command.rowFilter().needsReconciliation()) + | allowsOutOfRangeReadsFlag(command.allowsOutOfRangeReads) ); if (command.isDigestQuery()) out.writeUnsignedVInt32(command.digestVersion()); @@ -1314,6 +1359,7 @@ public abstract class ReadCommand extends AbstractReadQuery int flags = in.readByte(); boolean isDigest = isDigest(flags); boolean acceptsTransient = acceptsTransient(flags); + boolean allowsOutOfRangeReads = allowsOutOfRangeReads(flags); // Shouldn't happen or it's a user error (see comment above) but // better complain loudly than doing the wrong thing. if (isForThrift(flags)) @@ -1359,7 +1405,7 @@ public abstract class ReadCommand extends AbstractReadQuery indexQueryPlan = indexGroup.queryPlanFor(rowFilter); } - return kind.selectionDeserializer.deserialize(in, version, schemaVersion, isDigest, digestVersion, acceptsTransient, tableMetadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan); + return kind.selectionDeserializer.deserialize(in, version, schemaVersion, isDigest, digestVersion, acceptsTransient, allowsOutOfRangeReads, tableMetadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan); } private IndexMetadata deserializeIndexMetadata(DataInputPlus in, int version, TableMetadata metadata) throws IOException diff --git a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java index 5d430f32cf..0094a0df92 100644 --- a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java +++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java @@ -22,21 +22,21 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; -import org.apache.cassandra.exceptions.CoordinatorBehindException; -import org.apache.cassandra.exceptions.InvalidRoutingException; -import org.apache.cassandra.exceptions.QueryCancelledException; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.CoordinatorBehindException; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.exceptions.InvalidRoutingException; +import org.apache.cassandra.exceptions.QueryCancelledException; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.metrics.TCMMetrics; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; @@ -49,6 +49,18 @@ public class ReadCommandVerbHandler implements IVerbHandler private static final Logger logger = LoggerFactory.getLogger(ReadCommandVerbHandler.class); + public ReadResponse doRead(ReadCommand command, boolean trackRepairedData) + { + ReadResponse response; + try (ReadExecutionController controller = command.executionController(trackRepairedData); + UnfilteredPartitionIterator iterator = command.executeLocally(controller)) + { + response = command.createResponse(iterator, controller.getRepairedDataInfo()); + } + + return response; + } + public void doVerb(Message message) { if (message.epoch().isAfter(Epoch.EMPTY)) @@ -68,10 +80,9 @@ public class ReadCommandVerbHandler implements IVerbHandler command.trackWarnings(); ReadResponse response; - try (ReadExecutionController controller = command.executionController(message.trackRepairedData()); - UnfilteredPartitionIterator iterator = command.executeLocally(controller)) + try { - response = command.createResponse(iterator, controller.getRepairedDataInfo()); + response = doRead(command, message.trackRepairedData()); } catch (RejectException e) { @@ -147,15 +158,21 @@ public class ReadCommandVerbHandler implements IVerbHandler private ClusterMetadata checkTokenOwnership(ClusterMetadata metadata, Message message) { ReadCommand command = message.payload; + if (command.metadata().isVirtual()) return metadata; + // Some read commands may be sent using an older Epoch intentionally so validating using the current Epoch + // doesn't work + if (command.allowsOutOfRangeReads()) + return metadata; + if (command.isTopK()) return metadata; if (command instanceof SinglePartitionReadCommand) { - Token token = ((SinglePartitionReadCommand) command).partitionKey().getToken(); + Token token = ((SinglePartitionReadCommand)command).partitionKey().getToken(); Replica localReplica = getLocalReplica(metadata, token, command.metadata().keyspace); if (localReplica == null) { diff --git a/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java b/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java index 8ca29eba13..d40359c144 100644 --- a/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java +++ b/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java @@ -25,9 +25,14 @@ public class ReadRepairVerbHandler extends AbstractMutationVerbHandler { public static final ReadRepairVerbHandler instance = new ReadRepairVerbHandler(); + public void applyMutation(Mutation mutation) + { + mutation.apply(); + } + void applyMutation(Message message, InetAddressAndPort respondToAddress) { - message.payload.apply(); + applyMutation(message.payload); MessagingService.instance().send(message.emptyResponse(), respondToAddress); } } diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java index ab346bd47b..921d90a47b 100644 --- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java +++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java @@ -98,6 +98,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar boolean isDigest, int digestVersion, boolean acceptsTransient, + boolean allowsOutOfRangeReads, TableMetadata metadata, long nowInSec, ColumnFilter columnFilter, @@ -109,7 +110,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar boolean trackWarnings, DataRange dataRange) { - super(serializedAtEpoch, Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange); + super(serializedAtEpoch, Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, allowsOutOfRangeReads, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange); assert partitionKey.getPartitioner() == metadata.partitioner; this.partitionKey = partitionKey; this.clusteringIndexFilter = clusteringIndexFilter; @@ -119,6 +120,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar boolean isDigest, int digestVersion, boolean acceptsTransient, + boolean allowsOutOfRangeReads, TableMetadata metadata, long nowInSec, ColumnFilter columnFilter, @@ -152,6 +154,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar isDigest, digestVersion, acceptsTransient, + allowsOutOfRangeReads, metadata, nowInSec, columnFilter, @@ -191,6 +194,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar false, 0, false, + false, metadata, nowInSec, columnFilter, @@ -369,6 +373,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar isDigestQuery(), digestVersion(), acceptsTransient(), + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -387,6 +392,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar true, digestVersion(), acceptsTransient(), + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -405,6 +411,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar false, 0, true, + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -423,6 +430,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar isDigestQuery(), digestVersion(), acceptsTransient(), + allowsOutOfRangeReads(), metadata(), nowInSec(), columnFilter(), @@ -440,6 +448,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar isDigestQuery(), digestVersion(), acceptsTransient(), + allowsOutOfRangeReads(), metadata(), nowInSec, columnFilter(), @@ -1342,6 +1351,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar boolean isDigest, int digestVersion, boolean acceptsTransient, + boolean allowsOutOfRangeReads, TableMetadata metadata, long nowInSec, ColumnFilter columnFilter, @@ -1352,7 +1362,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar { DecoratedKey key = metadata.partitioner.decorateKey(metadata.partitionKeyType.readBuffer(in, DatabaseDescriptor.getMaxValueSize())); ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata); - return SinglePartitionReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false); + return SinglePartitionReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, allowsOutOfRangeReads, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false); } } @@ -1400,7 +1410,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar boolean trackWarnings, DataRange dataRange) { - super(metadata.epoch, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, + super(metadata.epoch, isDigest, digestVersion, acceptsTransient, true, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter, indexQueryPlan, trackWarnings, dataRange); } diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index db82fe2386..8f155773c8 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -136,6 +136,8 @@ import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithNowInSec; import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigratedAt; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; import static org.apache.cassandra.gms.ApplicationState.DC; import static org.apache.cassandra.gms.ApplicationState.HOST_ID; import static org.apache.cassandra.gms.ApplicationState.INTERNAL_ADDRESS_AND_PORT; @@ -162,6 +164,7 @@ public final class SystemKeyspace public static final String BATCHES = "batches"; public static final String PAXOS = "paxos"; + public static final String CONSENSUS_MIGRATION_STATE = "consensus_migration_state"; public static final String PAXOS_REPAIR_HISTORY = "paxos_repair_history"; public static final String PAXOS_REPAIR_STATE = "_paxos_repair_state"; public static final String BUILT_INDEXES = "IndexInfo"; @@ -190,6 +193,7 @@ public final class SystemKeyspace */ public static final Set TABLES_SPLIT_ACROSS_MULTIPLE_DISKS = ImmutableSet.of(BATCHES, PAXOS, + CONSENSUS_MIGRATION_STATE, COMPACTION_HISTORY, PREPARED_STATEMENTS, REPAIRS); @@ -215,14 +219,14 @@ public final class SystemKeyspace TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY, AVAILABLE_RANGES_V2, TRANSFERRED_RANGES_V2, VIEW_BUILDS_IN_PROGRESS, BUILT_VIEWS, PREPARED_STATEMENTS, REPAIRS, TOP_PARTITIONS, LEGACY_PEERS, LEGACY_PEER_EVENTS, LEGACY_TRANSFERRED_RANGES, LEGACY_AVAILABLE_RANGES, LEGACY_SIZE_ESTIMATES, LEGACY_SSTABLE_ACTIVITY, - METADATA_LOG, SNAPSHOT_TABLE_NAME); + METADATA_LOG, SNAPSHOT_TABLE_NAME, CONSENSUS_MIGRATION_STATE); public static final Set TABLE_NAMES = ImmutableSet.of( - BATCHES, PAXOS, PAXOS_REPAIR_HISTORY, BUILT_INDEXES, LOCAL, PEERS_V2, PEER_EVENTS_V2, - COMPACTION_HISTORY, SSTABLE_ACTIVITY_V2, TABLE_ESTIMATES, AVAILABLE_RANGES_V2, TRANSFERRED_RANGES_V2, VIEW_BUILDS_IN_PROGRESS, - BUILT_VIEWS, PREPARED_STATEMENTS, REPAIRS, TOP_PARTITIONS, LEGACY_PEERS, LEGACY_PEER_EVENTS, + BATCHES, PAXOS, PAXOS_REPAIR_HISTORY, BUILT_INDEXES, LOCAL, PEERS_V2, PEER_EVENTS_V2, + COMPACTION_HISTORY, SSTABLE_ACTIVITY_V2, TABLE_ESTIMATES, AVAILABLE_RANGES_V2, TRANSFERRED_RANGES_V2, VIEW_BUILDS_IN_PROGRESS, + BUILT_VIEWS, PREPARED_STATEMENTS, REPAIRS, TOP_PARTITIONS, LEGACY_PEERS, LEGACY_PEER_EVENTS, LEGACY_TRANSFERRED_RANGES, LEGACY_AVAILABLE_RANGES, LEGACY_SIZE_ESTIMATES, LEGACY_SSTABLE_ACTIVITY, - METADATA_LOG, SNAPSHOT_TABLE_NAME); + METADATA_LOG, SNAPSHOT_TABLE_NAME, CONSENSUS_MIGRATION_STATE); public static final TableMetadata Batches = parse(BATCHES, @@ -255,6 +259,25 @@ public final class SystemKeyspace .indexes(PaxosUncommittedIndex.indexes()) .build(); + private static final TableMetadata ConsensusMigrationState = + parse(CONSENSUS_MIGRATION_STATE, + "Keys that have been migrated to another consensus protocol", + "CREATE TABLE %s (" + + "row_key blob, " + + "cf_id UUID, " + + "consensus_migrated_at_epoch bigint, " + + "consensus_target tinyint, " + + "PRIMARY KEY ((row_key), cf_id, consensus_migrated_at_epoch)) " + + "WITH CLUSTERING ORDER BY (cf_id ASC, consensus_migrated_at_epoch DESC)") + .compaction(CompactionParams.twcs( + ImmutableMap.of( + "compaction_window_unit", "MINUTES", + "compaction_window_size", + // 7 days divided into 30 windows + String.valueOf((7 * 24 * 60) / 30)))) + .defaultTimeToLive((int)TimeUnit.DAYS.toSeconds(7)) + .build(); + private static final TableMetadata BuiltIndexes = parse(BUILT_INDEXES, "built column indexes", @@ -602,7 +625,8 @@ public final class SystemKeyspace Repairs, TopPartitions, LocalMetadataLog, - Snapshots); + Snapshots, + ConsensusMigrationState); } private static volatile Map> truncationRecords; @@ -1598,6 +1622,27 @@ public final class SystemKeyspace return PaxosRepairHistory.fromTupleBufferList(keyspace, table, points); } + public static void saveConsensusKeyMigrationState(ByteBuffer partitionKey, UUID cfId, ConsensusMigratedAt consensusMigratedAt) + { + String cql = "UPDATE system." + CONSENSUS_MIGRATION_STATE + " SET consensus_target = ? WHERE row_key = ? AND cf_id = ? AND consensus_migrated_at_epoch = ?"; + executeInternal(cql, consensusMigratedAt.migratedAtTarget.value, partitionKey, cfId, consensusMigratedAt.migratedAtEpoch.getEpoch()); + } + + public static ConsensusMigratedAt loadConsensusKeyMigrationState(ByteBuffer partitionKey, UUID cfId) + { + String cql = "SELECT consensus_migrated_at_epoch, consensus_target FROM system." + CONSENSUS_MIGRATION_STATE + " WHERE row_key = ? AND cf_id = ? LIMIT 1"; + UntypedResultSet results = executeInternal(cql, partitionKey, cfId); + + if (results.isEmpty()) + return null; + + UntypedResultSet.Row row = results.one(); + // TODO Period won't be necessary eventually + Epoch migratedAtEpoch = Epoch.create(row.getLong("consensus_migrated_at_epoch")); + ConsensusMigrationTarget target = ConsensusMigrationTarget.fromValue(row.getByte("consensus_target")); + return new ConsensusMigratedAt(migratedAtEpoch, target); + } + /** * Returns a RestorableMeter tracking the average read rate of a particular SSTable, restoring the last-seen rate * from values in system.sstable_activity if present. diff --git a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java index 857e0dfde9..923ef54ab6 100644 --- a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java +++ b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java @@ -24,12 +24,32 @@ import java.util.NavigableSet; import com.google.common.collect.Iterators; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionInfo; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.MutableDeletionInfo; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ColumnFilter; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowAndDeletionMergeIterator; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.SearchIterator; import org.apache.cassandra.utils.btree.BTree; +import org.apache.cassandra.utils.btree.BTree.Dir; import static org.apache.cassandra.utils.btree.BTree.Dir.desc; @@ -403,9 +423,15 @@ public abstract class AbstractBTreePartition implements Partition, Iterable return BTree.size(holder().tree); } + @Override public Iterator iterator() { - return BTree.iterator(holder().tree); + return iterator(false); + } + + public Iterator iterator(boolean reverse) + { + return BTree.iterator(holder().tree, reverse ? Dir.DESC : Dir.ASC); } public Row lastRow() diff --git a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java index c9035befbd..994ef1ac7b 100644 --- a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java +++ b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java @@ -25,12 +25,17 @@ import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import com.google.common.annotations.VisibleForTesting; import org.apache.cassandra.index.transactions.UpdateTransaction; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionInfo; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.filter.ColumnFilter; -import org.apache.cassandra.db.rows.*; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.concurrent.OpOrder; import org.apache.cassandra.utils.memory.Cloner; @@ -223,9 +228,9 @@ public final class AtomicBTreePartition extends AbstractBTreePartition } @Override - public Iterator iterator() + public Iterator iterator(boolean reverse) { - return allocator.ensureOnHeap().applyToPartition(super.iterator()); + return allocator.ensureOnHeap().applyToPartition(super.iterator(reverse)); } private boolean shouldLock(OpOrder.Group writeOp) diff --git a/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java b/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java index 138c853224..a66781dd7e 100644 --- a/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java +++ b/src/java/org/apache/cassandra/db/partitions/FilteredPartition.java @@ -19,11 +19,12 @@ package org.apache.cassandra.db.partitions; import java.util.Iterator; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionInfo; import org.apache.cassandra.db.RegularAndStaticColumns; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.btree.BTree; public class FilteredPartition extends ImmutableBTreePartition @@ -49,9 +50,9 @@ public class FilteredPartition extends ImmutableBTreePartition return BTree.findByIndex(holder.tree, idx); } - public RowIterator rowIterator() + public RowIterator rowIterator(boolean reverse) { - final Iterator iter = iterator(); + final Iterator iter = iterator(reverse); return new RowIterator() { public TableMetadata metadata() @@ -61,7 +62,7 @@ public class FilteredPartition extends ImmutableBTreePartition public boolean isReverseOrder() { - return false; + return reverse; } public RegularAndStaticColumns columns() diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java index cb0fdfb9ff..1b543bd2c7 100644 --- a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java +++ b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java @@ -24,17 +24,45 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import javax.annotation.Nonnull; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.Columns; +import org.apache.cassandra.db.CounterMutation; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionInfo; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.MutableDeletionInfo; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.SimpleBuilders; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ColumnFilter; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.db.rows.ColumnData; +import org.apache.cassandra.db.rows.ComplexColumnData; +import org.apache.cassandra.db.rows.DeserializationHelper; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.rows.RowIterators; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; import org.apache.cassandra.exceptions.CoordinatorBehindException; import org.apache.cassandra.exceptions.UnknownTableException; import org.apache.cassandra.index.IndexRegistry; @@ -1109,11 +1137,11 @@ public class PartitionUpdate extends AbstractBTreePartition return this; } - public Builder updateAllTimestampAndLocalDeletionTime(long newTimestamp, int newLocalDeletionTime) + public Builder updateTimesAndPathsForAccord(@Nonnull Function cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime) { deletionInfo.updateAllTimestampAndLocalDeletionTime(newTimestamp - 1, newLocalDeletionTime); - tree = BTree.transformAndFilter(tree, (x) -> x.updateAllTimestampAndLocalDeletionTime(newTimestamp, newLocalDeletionTime)); - staticRow = this.staticRow.updateAllTimestampAndLocalDeletionTime(newTimestamp, newLocalDeletionTime); + tree = BTree.transformAndFilter(tree, (x) -> x.updateTimesAndPathsForAccord(cellToMaybeNewListPath, newTimestamp, newLocalDeletionTime)); + staticRow = this.staticRow.updateTimesAndPathsForAccord(cellToMaybeNewListPath, newTimestamp, newLocalDeletionTime); return this; } @@ -1130,6 +1158,5 @@ public class PartitionUpdate extends AbstractBTreePartition ", isBuilt=" + isBuilt + '}'; } - } } diff --git a/src/java/org/apache/cassandra/db/rows/AbstractCell.java b/src/java/org/apache/cassandra/db/rows/AbstractCell.java index c3df806b8c..d30489a109 100644 --- a/src/java/org/apache/cassandra/db/rows/AbstractCell.java +++ b/src/java/org/apache/cassandra/db/rows/AbstractCell.java @@ -19,9 +19,12 @@ package org.apache.cassandra.db.rows; import java.nio.ByteBuffer; import java.util.Objects; +import javax.annotation.Nonnull; + +import com.google.common.base.Function; -import org.apache.cassandra.db.Digest; import org.apache.cassandra.db.DeletionPurger; +import org.apache.cassandra.db.Digest; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.db.marshal.AbstractType; @@ -118,12 +121,19 @@ public abstract class AbstractCell extends Cell } @Override - public ColumnData updateAllTimestampAndLocalDeletionTime(long newTimestamp, int newLocalDeletionTime) + public ColumnData updateTimesAndPathsForAccord(@Nonnull Function cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime) { long localDeletionTime = localDeletionTime() != NO_DELETION_TIME ? newLocalDeletionTime : NO_DELETION_TIME; return new BufferCell(column, isTombstone() ? newTimestamp - 1 : newTimestamp, ttl(), localDeletionTime, buffer(), path()); } + @Override + public Cell updateAllTimesWithNewCellPathForComplexColumnData(@Nonnull CellPath maybeNewPath, long newTimestamp, int newLocalDeletionTime) + { + long localDeletionTime = localDeletionTime() != NO_DELETION_TIME ? newLocalDeletionTime : NO_DELETION_TIME; + return new BufferCell(column, isTombstone() ? newTimestamp - 1 : newTimestamp, ttl(), localDeletionTime, buffer(), maybeNewPath); + } + public int dataSize() { CellPath path = path(); diff --git a/src/java/org/apache/cassandra/db/rows/BTreeRow.java b/src/java/org/apache/cassandra/db/rows/BTreeRow.java index 2fbee6a24a..ed445e8b6b 100644 --- a/src/java/org/apache/cassandra/db/rows/BTreeRow.java +++ b/src/java/org/apache/cassandra/db/rows/BTreeRow.java @@ -18,7 +18,6 @@ package org.apache.cassandra.db.rows; import java.nio.ByteBuffer; - import java.util.AbstractCollection; import java.util.Arrays; import java.util.Collection; @@ -28,9 +27,10 @@ import java.util.Iterator; import java.util.Map; import java.util.function.BiConsumer; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.Predicate; +import javax.annotation.Nonnull; +import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.Iterators; import com.google.common.primitives.Ints; @@ -40,15 +40,13 @@ import org.apache.cassandra.db.Columns; import org.apache.cassandra.db.DeletionPurger; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.LivenessInfo; +import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.TableMetadata; - -import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.DroppedColumn; - +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.AbstractIterator; import org.apache.cassandra.utils.BiLongAccumulator; import org.apache.cassandra.utils.BulkIterator; @@ -446,7 +444,7 @@ public class BTreeRow extends AbstractRow } @Override - public Row updateAllTimestampAndLocalDeletionTime(long newTimestamp, int newLocalDeletionTime) + public Row updateTimesAndPathsForAccord(@Nonnull Function cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime) { LivenessInfo newInfo = primaryKeyLivenessInfo.isEmpty() ? primaryKeyLivenessInfo : primaryKeyLivenessInfo.withUpdatedTimestampAndLocalDeletionTime(newTimestamp, newLocalDeletionTime); // If the deletion is shadowable and the row has a timestamp, we'll forced the deletion timestamp to be less than the row one, so we @@ -454,7 +452,7 @@ public class BTreeRow extends AbstractRow Deletion newDeletion = deletion.isLive() || (deletion.isShadowable() && !primaryKeyLivenessInfo.isEmpty()) ? Deletion.LIVE : new Deletion(DeletionTime.build(newTimestamp - 1, newLocalDeletionTime), deletion.isShadowable()); - return transformAndFilter(newInfo, newDeletion, (cd) -> cd.updateAllTimestampAndLocalDeletionTime(newTimestamp, newLocalDeletionTime)); + return transformAndFilter(newInfo, newDeletion, (cd) -> cd.updateTimesAndPathsForAccord(cellToMaybeNewListPath, newTimestamp, newLocalDeletionTime)); } public Row withRowDeletion(DeletionTime newDeletion) diff --git a/src/java/org/apache/cassandra/db/rows/ColumnData.java b/src/java/org/apache/cassandra/db/rows/ColumnData.java index 18530b2d39..8f055e1d14 100644 --- a/src/java/org/apache/cassandra/db/rows/ColumnData.java +++ b/src/java/org/apache/cassandra/db/rows/ColumnData.java @@ -18,13 +18,16 @@ package org.apache.cassandra.db.rows; import java.util.Comparator; +import javax.annotation.Nonnull; + +import com.google.common.base.Function; import org.apache.cassandra.cache.IMeasurableMemory; -import org.apache.cassandra.db.Digest; -import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.db.DeletionPurger; import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Digest; import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.utils.btree.BTree; import org.apache.cassandra.utils.btree.UpdateFunction; @@ -284,7 +287,19 @@ public abstract class ColumnData implements IMeasurableMemory * This exists for the Paxos path, see {@link PartitionUpdate#updateAllTimestamp} for additional details. */ public abstract ColumnData updateAllTimestamp(long newTimestamp); - public abstract ColumnData updateAllTimestampAndLocalDeletionTime(long newTimestamp, int newLocalDeletionTime); + + /** + * @param cellToMaybeNewListPath If the cell is a list append cell a new cell path is returned generated based on the Accord executeAt timestamp + */ + public abstract ColumnData updateTimesAndPathsForAccord(@Nonnull Function cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime); + + /** + * List paths are time UUIDs that increment for each item in the list and for Accord and Paxos + * should be based on the transaction's ballot/timestamp. + * + * @param maybeNewPath If this cell is a list append for a non-frozen list (multi-cell) then it will be new path generated using the executeAt timestamp, otherwise it will be the existing path + */ + public abstract ColumnData updateAllTimesWithNewCellPathForComplexColumnData(@Nonnull CellPath maybeNewPath, long newTimestamp, int newLocalDeletionTime); public abstract ColumnData markCounterLocalToBeCleared(); diff --git a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java index f668edfd7a..f032fd5b6d 100644 --- a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java +++ b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java @@ -20,6 +20,7 @@ package org.apache.cassandra.db.rows; import java.nio.ByteBuffer; import java.util.Iterator; import java.util.Objects; +import javax.annotation.Nonnull; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; @@ -30,6 +31,7 @@ import org.apache.cassandra.db.Digest; import org.apache.cassandra.db.LivenessInfo; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.ByteType; +import org.apache.cassandra.db.marshal.ListType; import org.apache.cassandra.db.marshal.SetType; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.DroppedColumn; @@ -265,10 +267,21 @@ public class ComplexColumnData extends ColumnData implements Iterable> } @Override - public ColumnData updateAllTimestampAndLocalDeletionTime(long newTimestamp, int newLocalDeletionTime) + public ColumnData updateTimesAndPathsForAccord(@Nonnull Function cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime) { DeletionTime newDeletion = complexDeletion.isLive() ? complexDeletion : DeletionTime.build(newTimestamp - 1, newLocalDeletionTime); - return transformAndFilter(newDeletion, (cell) -> (Cell) cell.updateAllTimestampAndLocalDeletionTime(newTimestamp, newLocalDeletionTime)); + Function maybeNewListPath; + if (column.type instanceof ListType && column.type.isMultiCell()) + maybeNewListPath = cellToMaybeNewListPath; + else + maybeNewListPath = cell -> cell.path(); + return transformAndFilter(newDeletion, (cell) -> (Cell) cell.updateAllTimesWithNewCellPathForComplexColumnData(maybeNewListPath.apply(cell), newTimestamp, newLocalDeletionTime)); + } + + @Override + public ColumnData updateAllTimesWithNewCellPathForComplexColumnData(@Nonnull CellPath maybeNewPath, long newTimestamp, int newLocalDeletionTime) + { + throw new UnsupportedOperationException(); } public long maxTimestamp() diff --git a/src/java/org/apache/cassandra/db/rows/Row.java b/src/java/org/apache/cassandra/db/rows/Row.java index ee836446d4..3820e8c3a4 100644 --- a/src/java/org/apache/cassandra/db/rows/Row.java +++ b/src/java/org/apache/cassandra/db/rows/Row.java @@ -17,13 +17,26 @@ */ package org.apache.cassandra.db.rows; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; import java.util.function.BiConsumer; import java.util.function.Consumer; -import java.util.function.Function; +import javax.annotation.Nonnull; + +import com.google.common.base.Function; import org.apache.cassandra.cache.IMeasurableMemory; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.DeletionPurger; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Digest; +import org.apache.cassandra.db.LivenessInfo; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; @@ -299,7 +312,7 @@ public interface Row extends Unfiltered, Iterable, IMeasurableMemory */ public Row updateAllTimestamp(long newTimestamp); - public Row updateAllTimestampAndLocalDeletionTime(long newTimestamp, int newLocalDeletionTime); + public Row updateTimesAndPathsForAccord(@Nonnull Function cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime); /** * Returns a copy of this row with the new deletion as row deletion if it is more recent diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java index 7572749d37..dc24a3bc0f 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java @@ -49,6 +49,7 @@ public class CassandraOutgoingFile implements OutgoingStream private final boolean shouldStreamEntireSSTable; private final StreamOperation operation; private final CassandraStreamHeader header; + private final List> ranges; public CassandraOutgoingFile(StreamOperation operation, Ref ref, List sections, List> normalizedRanges, @@ -60,6 +61,7 @@ public class CassandraOutgoingFile implements OutgoingStream this.ref = ref; this.estimatedKeys = estimatedKeys; this.sections = sections; + this.ranges = normalizedRanges; SSTableReader sstable = ref.get(); @@ -131,6 +133,12 @@ public class CassandraOutgoingFile implements OutgoingStream return shouldStreamEntireSSTable ? header.componentManifest.components().size() : 1; } + @Override + public List> ranges() + { + return ranges; + } + @Override public long getRepairedAt() { diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java index 6940f11b57..505bd6b928 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java @@ -81,9 +81,9 @@ public class CassandraStreamManager implements TableStreamManager } @Override - public StreamReceiver createStreamReceiver(StreamSession session, int totalStreams) + public StreamReceiver createStreamReceiver(StreamSession session, List> ranges, int totalStreams) { - return new CassandraStreamReceiver(cfs, session, totalStreams); + return new CassandraStreamReceiver(cfs, session, ranges, totalStreams); } @Override diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 50f87c799e..62af761277 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -41,18 +41,24 @@ import org.apache.cassandra.db.rows.ThrottledUnfilteredIterator; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.view.View; import org.apache.cassandra.dht.Bounds; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.streaming.IncomingStream; import org.apache.cassandra.streaming.StreamReceiver; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.CassandraVersion; import org.apache.cassandra.utils.CloseableIterator; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.Refs; +import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_MUTATION_REPAIR_ROWS_PER_BATCH; public class CassandraStreamReceiver implements StreamReceiver @@ -74,14 +80,17 @@ public class CassandraStreamReceiver implements StreamReceiver private final boolean requiresWritePath; + private final List> ranges; - public CassandraStreamReceiver(ColumnFamilyStore cfs, StreamSession session, int totalFiles) + + public CassandraStreamReceiver(ColumnFamilyStore cfs, StreamSession session, List> ranges, int totalFiles) { this.cfs = cfs; this.session = session; // this is an "offline" transaction, as we currently manually expose the sstables once done; // this should be revisited at a later date, so that LifecycleTransaction manages all sstable state changes this.txn = LifecycleTransaction.offline(OperationType.STREAM); + this.ranges = ranges; this.sstables = new ArrayList<>(totalFiles); this.requiresWritePath = requiresWritePath(cfs); } @@ -233,6 +242,14 @@ public class CassandraStreamReceiver implements StreamReceiver @Override public void finished() { + CassandraVersion minVersion = ClusterMetadata.current().directory.clusterMinVersion.cassandraVersion; + checkNotNull(minVersion, "Unable to determine minimum cluster version"); + IAccordService accordService = AccordService.instance(); + if (session.streamOperation().requiresBarrierTransaction() + && accordService.isAccordManagedKeyspace(cfs.keyspace.getName()) + && CassandraVersion.CASSANDRA_5_0.compareTo(minVersion) >= 0) + accordService.postStreamReceivingBarrier(cfs, ranges); + boolean requiresWritePath = requiresWritePath(cfs); Collection readers = sstables; diff --git a/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java b/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java index 1c8c24b171..0012382bd9 100644 --- a/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java +++ b/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java @@ -144,7 +144,7 @@ public class LocalRepairTables result.column("options_primary_range", state.options.isPrimaryRange()); result.column("options_trace", state.options.isTraced()); result.column("options_job_threads", state.options.getJobThreads()); - result.column("options_subrange_repair", state.options.isSubrangeRepair()); + result.column("options_subrange_repair", false); result.column("options_pull_repair", state.options.isPullRepair()); result.column("options_force_repair", state.options.isForcedRepair()); result.column("options_preview_kind", state.options.getPreviewKind().name()); @@ -183,6 +183,10 @@ public class LocalRepairTables default: throw new AssertionError("Unknown preview kind: " + state.options.getPreviewKind()); } } + else if (state.options.accordRepair()) + { + return "accord repair"; + } else if (state.options.isIncremental()) { return "incremental"; diff --git a/src/java/org/apache/cassandra/dht/AccordSplitter.java b/src/java/org/apache/cassandra/dht/AccordSplitter.java index c5971dc89f..b0868b47e8 100644 --- a/src/java/org/apache/cassandra/dht/AccordSplitter.java +++ b/src/java/org/apache/cassandra/dht/AccordSplitter.java @@ -45,7 +45,7 @@ public abstract class AccordSplitter implements ShardDistributor.EvenSplit.Split } @Override - public accord.primitives.Range subRange(accord.primitives.Range range, BigInteger startOffset, BigInteger endOffset) + public TokenRange subRange(accord.primitives.Range range, BigInteger startOffset, BigInteger endOffset) { AccordRoutingKey startBound = (AccordRoutingKey)range.start(); AccordRoutingKey endBound = (AccordRoutingKey)range.end(); diff --git a/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java b/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java index 9b3f63b820..9e3c3cf848 100644 --- a/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java +++ b/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java @@ -17,26 +17,6 @@ */ package org.apache.cassandra.dht; -import accord.primitives.Ranges; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.db.BufferDecoratedKey; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.bytecomparable.ByteComparable; -import org.apache.cassandra.utils.bytecomparable.ByteSource; -import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Hex; -import org.apache.cassandra.utils.ObjectSizes; -import org.apache.cassandra.utils.Pair; - -import org.apache.commons.lang3.ArrayUtils; - import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -48,6 +28,25 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.function.Function; import com.google.common.collect.Maps; +import org.apache.commons.lang3.ArrayUtils; + +import accord.primitives.Ranges; +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Hex; +import org.apache.cassandra.utils.ObjectSizes; +import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.bytecomparable.ByteComparable; +import org.apache.cassandra.utils.bytecomparable.ByteSource; +import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; public class ByteOrderedPartitioner implements IPartitioner { @@ -194,6 +193,8 @@ public class ByteOrderedPartitioner implements IPartitioner } } + private ByteOrderedPartitioner() {} + public BytesToken getToken(ByteBuffer key) { if (key.remaining() == 0) diff --git a/src/java/org/apache/cassandra/dht/LocalPartitioner.java b/src/java/org/apache/cassandra/dht/LocalPartitioner.java index b0b8d558ad..c2886fd539 100644 --- a/src/java/org/apache/cassandra/dht/LocalPartitioner.java +++ b/src/java/org/apache/cassandra/dht/LocalPartitioner.java @@ -21,12 +21,13 @@ import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Random; import java.util.function.Function; import accord.primitives.Ranges; -import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.CachedHashDecoratedKey; +import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.utils.ByteBufferUtil; @@ -140,6 +141,21 @@ public class LocalPartitioner implements IPartitioner return comparator; } + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LocalPartitioner that = (LocalPartitioner) o; + return comparator.equals(that.comparator) && tokenFactory.equals(that.tokenFactory); + } + + @Override + public int hashCode() + { + return Objects.hash(comparator, tokenFactory); + } + public class LocalToken extends ComparableObjectToken { static final long serialVersionUID = 8437543776403014875L; diff --git a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java index f80d6d4843..410dbfcd0b 100644 --- a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java +++ b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java @@ -21,28 +21,33 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; -import java.util.*; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Function; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.primitives.Longs; + import accord.primitives.Ranges; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PreHashedDecoratedKey; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.PartitionerDefinedOrder; import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.db.marshal.PartitionerDefinedOrder; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.MurmurHash; +import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; -import org.apache.cassandra.utils.MurmurHash; -import org.apache.cassandra.utils.ObjectSizes; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.primitives.Longs; /** * This class generates a BigIntegerToken using a Murmur3 hash. @@ -85,6 +90,8 @@ public class Murmur3Partitioner implements IPartitioner } }; + protected Murmur3Partitioner() {} + public DecoratedKey decorateKey(ByteBuffer key) { long[] hash = getHash(key); diff --git a/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java b/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java index d2419049db..741a2b0c7f 100644 --- a/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java +++ b/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java @@ -20,14 +20,18 @@ package org.apache.cassandra.dht; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Function; import accord.api.RoutingKey; import accord.primitives.Ranges; -import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.CachedHashDecoratedKey; +import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.ConfigurationException; @@ -70,6 +74,8 @@ public class OrderPreservingPartitioner implements IPartitioner public static final OrderPreservingPartitioner instance = new OrderPreservingPartitioner(); + private OrderPreservingPartitioner() {} + public DecoratedKey decorateKey(ByteBuffer key) { return new CachedHashDecoratedKey(getToken(key), key); diff --git a/src/java/org/apache/cassandra/dht/RandomPartitioner.java b/src/java/org/apache/cassandra/dht/RandomPartitioner.java index 44f1893f0b..a21815c0c5 100644 --- a/src/java/org/apache/cassandra/dht/RandomPartitioner.java +++ b/src/java/org/apache/cassandra/dht/RandomPartitioner.java @@ -22,28 +22,33 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; import java.security.MessageDigest; -import java.util.*; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; import accord.primitives.Ranges; import org.apache.cassandra.db.CachedHashDecoratedKey; -import org.apache.cassandra.db.marshal.ByteArrayAccessor; -import org.apache.cassandra.db.marshal.ByteBufferAccessor; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.ByteArrayAccessor; +import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.db.marshal.IntegerType; import org.apache.cassandra.db.marshal.PartitionerDefinedOrder; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.bytecomparable.ByteComparable; -import org.apache.cassandra.utils.bytecomparable.ByteSource; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.GuidGenerator; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.bytecomparable.ByteComparable; +import org.apache.cassandra.utils.bytecomparable.ByteSource; /** * This class generates a BigIntegerToken using MD5 hash. @@ -108,6 +113,8 @@ public class RandomPartitioner implements IPartitioner } }; + private RandomPartitioner() {} + public DecoratedKey decorateKey(ByteBuffer key) { return new CachedHashDecoratedKey(getToken(key), key); diff --git a/src/java/org/apache/cassandra/dht/Range.java b/src/java/org/apache/cassandra/dht/Range.java index b5d06967ac..9100254508 100644 --- a/src/java/org/apache/cassandra/dht/Range.java +++ b/src/java/org/apache/cassandra/dht/Range.java @@ -19,13 +19,27 @@ package org.apache.cassandra.dht; import java.io.IOException; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; import java.util.function.Predicate; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; import org.apache.commons.lang3.ObjectUtils; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.Token.TokenFactory; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.net.MessagingService; @@ -34,6 +48,10 @@ import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.Pair; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Collections.emptyList; +import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_RANGE_EXPENSIVE_CHECKS; + /** * A representation of the range that a node is responsible for on the DHT ring. * @@ -48,6 +66,34 @@ public class Range> extends AbstractBounds implemen public static final Serializer serializer = new Serializer(); public static final long serialVersionUID = 1L; + public static final boolean EXPENSIVE_CHECKS = TEST_RANGE_EXPENSIVE_CHECKS.getBoolean(); + + public static final IPartitionerDependentSerializer rangeSerializer = new RangeSerializer(); + + public static class RangeSerializer> implements IPartitionerDependentSerializer> + { + @Override + public void serialize(Range range, DataOutputPlus out, int version) throws IOException + { + Token.compactSerializer.serialize(range.left.getToken(), out, version); + Token.compactSerializer.serialize(range.right.getToken(), out, version); + } + + @Override + public Range deserialize(DataInputPlus in, IPartitioner p, int version) throws IOException + { + return new Range(Token.compactSerializer.deserialize(in, p, version), + Token.compactSerializer.deserialize(in, p, version)); + } + + @Override + public long serializedSize(Range range, int version) + { + return Token.compactSerializer.serializedSize(range.left.getToken(), version) + + Token.compactSerializer.serializedSize(range.right.getToken(), version); + } + } + public Range(T left, T right) { super(left, right); @@ -349,6 +395,43 @@ public class Range> extends AbstractBounds implemen return right.compareTo(rhs.right); } + /* + * Compares ranges by right token. Used for intersecting normalized ranges. + * + * Assumes no wrap around ranges except for RHS = minValue which is essentialy synonymous with the maximal value. + * This shows up coming out of unwrap because Range is not left inclusive so the only way to include minValue + * in the range is by wrapping from maxValue. + */ + private int compareNormalized(Range rhs) + { + // otherwise compare by right. + int cmp = right.compareTo(rhs.right); + // minValue on the RHS is maxValue, but doesn't work with compare so check for it explicitly + boolean rhsRMin = rhs.right.isMinimum(); + boolean lhsRMin = right.isMinimum(); + + if (rhsRMin && lhsRMin) + return 0; + + if (cmp < 0) + { + if (lhsRMin) + { + return 1; + } + return -1; + } + else if (cmp > 0) + { + if (rhsRMin) + { + return -1; + } + return 1; + } + return 0; + } + /** * Subtracts a portion of this range. * @param contained The range to subtract from this. It must be totally @@ -361,7 +444,7 @@ public class Range> extends AbstractBounds implemen // both ranges cover the entire ring, their difference is an empty set if(isFull(left, right) && isFull(contained.left, contained.right)) { - return Collections.emptyList(); + return emptyList(); } // a range is subtracted from another range that covers the entire ring @@ -472,6 +555,190 @@ public class Range> extends AbstractBounds implemen return false; } + private static final Comparator NORMALIZED_TOKEN_RANGE_COMPARATOR = (o1, o2) -> { + Range range = (Range)o1; + RingPosition key = (RingPosition) o2; + boolean rangeRightIsMin = range.right.isMinimum(); + boolean keyIsMinimum = key.isMinimum(); + + if (keyIsMinimum & rangeRightIsMin) + return 0; + + int lc = key.compareTo(range.left); + int rc = key.compareTo(range.right); + if ((lc < 0 & !keyIsMinimum) | lc == 0) return 1; + if (rc > 0 & !rangeRightIsMin) return -1; + return 0; + }; + + public static > boolean isInNormalizedRanges(T token, List> ranges) + { + if (ranges.size() == 1 && ranges.get(0).isFull()) + return true; + boolean isIn = Collections.binarySearch((List)ranges, token, NORMALIZED_TOKEN_RANGE_COMPARATOR) >= 0; + if (EXPENSIVE_CHECKS) + checkState(isInRanges(token, ranges) == isIn); + return isIn; + } + + public static > List> subtractNormalizedRanges(List> a, List> b) + { + if (b.size() == 1 && b.get(0).isFull()) + return emptyList(); + + if (a.size() == 1 && a.get(0).isFull()) + return invertNormalizedRanges(b); + + List> remaining = new ArrayList<>(); + Iterator> aIter = a.iterator(); + Iterator> bIter = b.iterator(); + Range aRange = aIter.hasNext() ? aIter.next() : null; + Range bRange = bIter.hasNext() ? bIter.next() : null; + while (aRange != null && bRange != null) + { + boolean aRMin = aRange.right.isMinimum(); + boolean bRMin = bRange.right.isMinimum(); + + if (aRMin && bRMin) + { + if (aRange.left.compareTo(bRange.left) < 0) + remaining.add(new Range<>(aRange.left, bRange.left)); + checkState(!aIter.hasNext() && !bIter.hasNext()); + aRange = null; + break; + } + + if (!aRMin && aRange.right.compareTo(bRange.left) <= 0) + { + remaining.add(aRange); + aRange = aIter.hasNext() ? aIter.next() : null; + } + else if (!bRMin && aRange.left.compareTo(bRange.right) >= 0) + { + bRange = bIter.hasNext() ? bIter.next() : null; + } + else + { + // Handle what remains to the left of the intersection + if (aRange.left.compareTo(bRange.left) < 0) + { + remaining.add(new Range(aRange.left, bRange.left)); + } + + // Handle what remains to the right of the intersection + if (!aRMin && (aRange.right.compareTo(bRange.right) <= 0 | bRMin)) + aRange = aIter.hasNext() ? aIter.next() : null; + else + aRange = new Range(bRange.right, aRange.right); + } + } + + while (aRange != null) + { + remaining.add(aRange); + aRange = aIter.hasNext() ? aIter.next() : null; + } + + List> result = ImmutableList.copyOf(normalize(remaining)); + if (EXPENSIVE_CHECKS) + checkState(result.equals(normalize(subtract(a, b)))); + return result; + } + + private boolean isFull() + { + return isFull(left, right); + } + + @VisibleForTesting + static > List> invertNormalizedRanges(List> ranges) + { + if (ranges.isEmpty()) + return ranges; + + List> result = new ArrayList<>(ranges.size() + 2); + T minValue = ranges.get(0).left.minValue(); + T left = minValue; + for (Range r : ranges) + { + if (!r.left.equals(left)) + { + result.add(new Range<>(left, r.left)); + } + left = r.right; + } + + // Loop doesn't add the range to the right of the last one + Range last = ranges.get(ranges.size() - 1); + if (!last.right.isMinimum()) + result.add(new Range<>(last.right, minValue)); + + result = normalize(result); + if (EXPENSIVE_CHECKS) + checkState(result.equals(normalize(subtract(ImmutableList.of(new Range<>(minValue, minValue)), ranges)))); + return result; + } + + public static > List> intersectionOfNormalizedRanges(List> a, List> b) + { + if (a.size() == 1 && a.get(0).isFull()) + return b; + if (b.size() == 1 && b.get(0).isFull()) + return a; + + List> merged = new ArrayList<>(); + PeekingIterator> aIter = Iterators.peekingIterator(a.iterator()); + PeekingIterator> bIter = Iterators.peekingIterator(b.iterator()); + while (aIter.hasNext() && bIter.hasNext()) + { + Range aRange = aIter.peek(); + Range bRange = bIter.peek(); + + int cmp = aRange.compareNormalized(bRange); + if (aRange.intersects(bRange)) + { + merged.addAll(aRange.intersectionWith(bRange)); + if (cmp == 0) + { + aIter.next(); + bIter.next(); + } + else if(cmp < 0) + { + aIter.next(); + } + else + { + bIter.next(); + } + } + else + { + if (cmp <= 0) + aIter.next(); + if (cmp >= 0) + bIter.next(); + } + } + + List> result = ImmutableList.copyOf(normalize(merged)); + + if (EXPENSIVE_CHECKS) + { + List> expensiveResult = new ArrayList<>(); + for (Range r1 : a) + { + for (Range r2 : b) + { + expensiveResult.addAll(r1.intersectionWith(r2)); + } + } + checkState(result.equals(normalize(expensiveResult))); + } + + return result; + } + @Override public boolean equals(Object o) { @@ -670,6 +937,26 @@ public class Range> extends AbstractBounds implemen } } + public static > boolean equals(Collection> a, Collection> b) + { + return normalize(a).equals(normalize(b)); + } + + // Helper to convert a range string to POJO so you can copy toString from a debugger + public static Range fromString(String value) + { + return fromString(value, DatabaseDescriptor.getPartitioner()); + } + + public static Range fromString(String value, IPartitioner partitioner) + { + TokenFactory tokenFactory = partitioner.getTokenFactory(); + String[] parts = value.split(","); + Token left = tokenFactory.fromString(parts[0].substring(1)); + Token right = tokenFactory.fromString(parts[1].substring(0, parts[1].length() -1)); + return new Range<>(left, right); + } + public static > void assertNormalized(List> ranges) { Range lastRange = null; diff --git a/src/java/org/apache/cassandra/exceptions/RequestFailure.java b/src/java/org/apache/cassandra/exceptions/RequestFailure.java index 39700d795c..d2c8a2e61c 100644 --- a/src/java/org/apache/cassandra/exceptions/RequestFailure.java +++ b/src/java/org/apache/cassandra/exceptions/RequestFailure.java @@ -67,7 +67,7 @@ public class RequestFailure public void serialize(RequestFailure t, DataOutputPlus out, int version) throws IOException { RequestFailureReason.serializer.serialize(t.reason, out, version); - if (version >= MessagingService.VERSION_50) + if (version >= MessagingService.VERSION_51) nullableRemoteExceptionSerializer.serialize(t.failure, out, version); } @@ -76,7 +76,7 @@ public class RequestFailure { RequestFailureReason reason = RequestFailureReason.serializer.deserialize(in, version); Throwable failure = null; - if (version >= MessagingService.VERSION_50) + if (version >= MessagingService.VERSION_51) failure = nullableRemoteExceptionSerializer.deserialize(in, version); if (failure == null) return forReason(reason); @@ -88,7 +88,7 @@ public class RequestFailure public long serializedSize(RequestFailure t, int version) { long size = RequestFailureReason.serializer.serializedSize(t.reason, version); - if (version >= MessagingService.VERSION_50) + if (version >= MessagingService.VERSION_51) size += nullableRemoteExceptionSerializer.serializedSize(t.failure, version); return size; } diff --git a/src/java/org/apache/cassandra/locator/ReplicaLayout.java b/src/java/org/apache/cassandra/locator/ReplicaLayout.java index 30a52be73a..6b464237d0 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaLayout.java +++ b/src/java/org/apache/cassandra/locator/ReplicaLayout.java @@ -26,6 +26,7 @@ import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.utils.FBUtilities; @@ -356,11 +357,11 @@ public abstract class ReplicaLayout> * @return the read layout for a token - this includes natural replicas, i.e. those that are not pending. * They are reverse sorted by the badness score of the configured snitch */ - static ReplicaLayout.ForTokenRead forTokenReadSorted(ClusterMetadata metadata, Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, Token token) + static ReplicaLayout.ForTokenRead forTokenReadSorted(ClusterMetadata metadata, Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, Token token, ReadCoordinator coordinator) { EndpointsForToken replicas = keyspace.getMetadata().params.replication.isLocal() ? forLocalStrategyToken(metadata, replicationStrategy, token) - : forNonLocalStrategyTokenRead(metadata, keyspace.getMetadata(), token); + : coordinator.forNonLocalStrategyTokenRead(metadata, keyspace.getMetadata(), token); replicas = DatabaseDescriptor.getNodeProximity().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas); @@ -386,7 +387,7 @@ public abstract class ReplicaLayout> return metadata.placements.get(keyspace.params.replication).reads.forRange(range.right.getToken()).get(); } - static EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token) + public static EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token) { return metadata.placements.get(keyspace.params.replication).reads.forToken(token).get(); } diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlans.java b/src/java/org/apache/cassandra/locator/ReplicaPlans.java index 53b32797c6..5aef13a6d3 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaPlans.java +++ b/src/java/org/apache/cassandra/locator/ReplicaPlans.java @@ -63,6 +63,7 @@ import org.apache.cassandra.index.Index; import org.apache.cassandra.index.IndexStatusManager; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.service.reads.AlwaysSpeculativeRetryPolicy; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; @@ -533,7 +534,7 @@ public class ReplicaPlans } - public static ReplicaPlan.ForWrite forReadRepair(ReplicaPlan forRead, ClusterMetadata metadata, Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, Predicate isAlive) throws UnavailableException + public static ReplicaPlan.ForWrite forReadRepair(ReplicaPlan forRead, ClusterMetadata metadata, Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, Predicate isAlive, ReadCoordinator coordinator) throws UnavailableException { AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); Selector selector = writeReadRepair(forRead); @@ -550,7 +551,7 @@ public class ReplicaPlans liveAndDown.all(), live.all(), contacts, - (newClusterMetadata) -> forReadRepair(forRead, newClusterMetadata, keyspace, consistencyLevel, token, isAlive), + (newClusterMetadata) -> forReadRepair(forRead, newClusterMetadata, keyspace, consistencyLevel, token, isAlive, coordinator), metadata.epoch); } @@ -882,9 +883,10 @@ public class ReplicaPlans Token token, @Nullable Index.QueryPlan indexQueryPlan, ConsistencyLevel consistencyLevel, - SpeculativeRetryPolicy retry) + SpeculativeRetryPolicy retry, + ReadCoordinator coordinator) { - return forRead(ClusterMetadata.current(), keyspace, token, indexQueryPlan, consistencyLevel, retry, false); + return forRead(ClusterMetadata.current(), keyspace, token, indexQueryPlan, consistencyLevel, retry, coordinator, false); } public static ReplicaPlan.ForTokenRead forRead(ClusterMetadata metadata, @@ -892,9 +894,10 @@ public class ReplicaPlans Token token, @Nullable Index.QueryPlan indexQueryPlan, ConsistencyLevel consistencyLevel, - SpeculativeRetryPolicy retry) + SpeculativeRetryPolicy retry, + ReadCoordinator coordinator) { - return forRead(metadata, keyspace, token, indexQueryPlan, consistencyLevel, retry, true); + return forRead(metadata, keyspace, token, indexQueryPlan, consistencyLevel, retry, coordinator, true); } private static ReplicaPlan.ForTokenRead forRead(ClusterMetadata metadata, @@ -903,10 +906,11 @@ public class ReplicaPlans @Nullable Index.QueryPlan indexQueryPlan, ConsistencyLevel consistencyLevel, SpeculativeRetryPolicy retry, + ReadCoordinator coordinator, boolean throwOnInsufficientLiveReplicas) { AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); - ReplicaLayout.ForTokenRead forTokenReadLiveAndDown = ReplicaLayout.forTokenReadSorted(metadata, keyspace, replicationStrategy, token); + ReplicaLayout.ForTokenRead forTokenReadLiveAndDown = ReplicaLayout.forTokenReadSorted(metadata, keyspace, replicationStrategy, token, coordinator); ReplicaLayout.ForTokenRead forTokenReadLive = forTokenReadLiveAndDown.filter(FailureDetector.isReplicaAlive); EndpointsForToken candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, forTokenReadLive.all()); EndpointsForToken contacts = contactForRead(metadata.locator, replicationStrategy, consistencyLevel, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE), candidates); @@ -915,8 +919,8 @@ public class ReplicaPlans assureSufficientLiveReplicasForRead(metadata.locator, replicationStrategy, consistencyLevel, contacts); return new ReplicaPlan.ForTokenRead(keyspace, replicationStrategy, consistencyLevel, candidates, contacts, forTokenReadLiveAndDown.all(), - (newClusterMetadata) -> forRead(newClusterMetadata, keyspace, token, indexQueryPlan, consistencyLevel, retry, false), - (self) -> forReadRepair(self, metadata, keyspace, consistencyLevel, token, FailureDetector.isReplicaAlive), + (newClusterMetadata) -> forRead(newClusterMetadata, keyspace, token, indexQueryPlan, consistencyLevel, retry, coordinator, false), + (self) -> forReadRepair(self, metadata, keyspace, consistencyLevel, token, FailureDetector.isReplicaAlive, coordinator), metadata.epoch); } @@ -962,7 +966,7 @@ public class ReplicaPlans forRangeReadLiveAndDown.all(), vnodeCount, (newClusterMetadata) -> forRangeRead(newClusterMetadata, keyspace, indexQueryPlan, consistencyLevel, range, vnodeCount, false), - (self, token) -> forReadRepair(self, metadata, keyspace, consistencyLevel, token, FailureDetector.isReplicaAlive), + (self, token) -> forReadRepair(self, metadata, keyspace, consistencyLevel, token, FailureDetector.isReplicaAlive, ReadCoordinator.DEFAULT), metadata.epoch); } @@ -1041,7 +1045,7 @@ public class ReplicaPlans (self, token) -> { // It might happen that the ring has moved forward since the operation has started, but because we'll be recomputing a quorum // after the operation is complete, we will catch inconsistencies either way. - return forReadRepair(self, ClusterMetadata.current(), keyspace, consistencyLevel, token, FailureDetector.isReplicaAlive); + return forReadRepair(self, ClusterMetadata.current(), keyspace, consistencyLevel, token, FailureDetector.isReplicaAlive, ReadCoordinator.DEFAULT); }, left.epoch); } diff --git a/src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java b/src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java index a9d1f28c47..33f7e8f201 100644 --- a/src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordClientRequestMetrics.java @@ -19,6 +19,7 @@ package org.apache.cassandra.metrics; import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; @@ -26,11 +27,27 @@ public class AccordClientRequestMetrics extends ClientRequestMetrics { public final Histogram keySize; + // During migration back to Paxos it's possible a transaction runs + // in an Epoch where Accord is no longer accepting transactions + // and we still run it to completion, but we do skip the read from Cassandra + // although it would be harmless. This should only occur briefly when coordinators + // start transactions on the wrong protocol due to temporarily out of data cluster metadata. + public final Meter migrationSkippedReads; + + // Number of times a key had to be run through PaxosRepair for migration to Accord + public final Meter paxosKeyMigrations; + + // Number of times a query was rejected by Accord in TxnQuery due to a migration back to Paxos + public final Meter accordMigrationRejects; + public AccordClientRequestMetrics(String scope) { super(scope); keySize = Metrics.histogram(factory.createMetricName("KeySizeHistogram"), false); + migrationSkippedReads = Metrics.meter(factory.createMetricName("MigrationSkippedReads")); + paxosKeyMigrations = Metrics.meter(factory.createMetricName("PaxosKeyMigrations")); + accordMigrationRejects = Metrics.meter(factory.createMetricName("AccordMigrationRejects")); } @Override @@ -38,5 +55,8 @@ public class AccordClientRequestMetrics extends ClientRequestMetrics { super.release(); Metrics.remove(factory.createMetricName("KeySizeHistogram")); + Metrics.remove(factory.createMetricName("MigrationSkippedReads")); + Metrics.remove(factory.createMetricName("PaxosKeyMigrations")); + Metrics.remove(factory.createMetricName("AccordMigrationRejects")); } } diff --git a/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java b/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java index 654bb059d1..f3ae6c8924 100644 --- a/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java +++ b/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java @@ -29,6 +29,12 @@ public class CASClientRequestMetrics extends ClientRequestMetrics public final Histogram contention; public final Counter unfinishedCommit; public final Meter unknownResult; + // CAS request rejected after Prepare/Promise due to migration from Paxos to Accord + public final Meter beginMigrationRejects; + // Number of times a CAS request was rejected after Propose/Accept due to migration from Paxos to Accord + public final Meter acceptMigrationRejects; + // Number of times a key was migrated from Accord to Paxos + public final Meter accordKeyMigrations; public CASClientRequestMetrics(String scope) { @@ -36,6 +42,9 @@ public class CASClientRequestMetrics extends ClientRequestMetrics contention = Metrics.histogram(factory.createMetricName("ContentionHistogram"), false); unfinishedCommit = Metrics.counter(factory.createMetricName("UnfinishedCommit")); unknownResult = Metrics.meter(factory.createMetricName("UnknownResult")); + beginMigrationRejects = Metrics.meter(factory.createMetricName("PaxosBeginMigrationRejects")); + acceptMigrationRejects = Metrics.meter(factory.createMetricName("PaxosAcceptMigrationRejects")); + accordKeyMigrations = Metrics.meter(factory.createMetricName("AccordKeyMigrations")); } public void release() @@ -44,5 +53,8 @@ public class CASClientRequestMetrics extends ClientRequestMetrics Metrics.remove(factory.createMetricName("ContentionHistogram")); Metrics.remove(factory.createMetricName("UnfinishedCommit")); Metrics.remove(factory.createMetricName("UnknownResult")); + Metrics.remove(factory.createMetricName("PaxosBeginMigrationRejects")); + Metrics.remove(factory.createMetricName("PaxosAcceptMigrationRejects")); + Metrics.remove(factory.createMetricName("AccordKeyMigrations")); } } diff --git a/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java b/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java index 26f2913263..05d17a0fda 100644 --- a/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java +++ b/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java @@ -29,6 +29,8 @@ public final class ClientRequestsMetricsHolder public static final CASClientWriteRequestMetrics casWriteMetrics = new CASClientWriteRequestMetrics("CASWrite"); public static final CASClientRequestMetrics casReadMetrics = new CASClientRequestMetrics("CASRead"); public static final ViewWriteMetrics viewWriteMetrics = new ViewWriteMetrics("ViewWrite"); + public static final AccordClientRequestMetrics accordReadMetrics = new AccordClientRequestMetrics("AccordRead"); + public static final AccordClientRequestMetrics accordWriteMetrics = new AccordClientRequestMetrics("AccordWrite"); public static final Map readMetricsMap = new EnumMap<>(ConsistencyLevel.class); public static final Map writeMetricsMap = new EnumMap<>(ConsistencyLevel.class); diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index e603381aff..c290c321f3 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -101,6 +101,12 @@ public class KeyspaceMetrics public final LatencyMetrics casPropose; /** CAS Commit metrics */ public final LatencyMetrics casCommit; + /** Latency for locally run key migrations **/ + public final LatencyMetrics keyMigration; + /** Latency for range migrations run by locally coordinated Accord repairs **/ + public final LatencyMetrics rangeMigration; + public final Meter rangeMigrationUnexpectedFailures; + public final Meter rangeMigrationDependencyLimitFailures; /** Writes failed ideal consistency **/ public final Counter writeFailedIdealCL; /** Ideal CL write latency metrics */ @@ -247,6 +253,10 @@ public class KeyspaceMetrics casPrepare = createLatencyMetrics("CasPrepare"); casPropose = createLatencyMetrics("CasPropose"); casCommit = createLatencyMetrics("CasCommit"); + keyMigration = createLatencyMetrics("KeyMigration"); + rangeMigration = createLatencyMetrics("RangeMigration"); + rangeMigrationUnexpectedFailures = createKeyspaceMeter("RangeMigrationUnexpectedFailures"); + rangeMigrationDependencyLimitFailures = createKeyspaceMeter("RangeMigratingDependencyLimitFailures"); writeFailedIdealCL = createKeyspaceCounter("WriteFailedIdealCL"); idealCLWriteLatency = createLatencyMetrics("IdealCLWrite"); diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index fd3a6bac2e..3729fe6074 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -87,6 +87,8 @@ public class TableMetrics public final static LatencyMetrics GLOBAL_READ_LATENCY = new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "Read"); public final static LatencyMetrics GLOBAL_WRITE_LATENCY = new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "Write"); public final static LatencyMetrics GLOBAL_RANGE_LATENCY = new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "Range"); + public final static LatencyMetrics GLOBAL_KEY_MIGRATION_LATENCY = new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "KeyMigration"); + public final static LatencyMetrics GLOBAL_RANGE_MIGRATION_LATENCY = new LatencyMetrics(GLOBAL_FACTORY, GLOBAL_ALIAS_FACTORY, "RangeMigration"); /** Total amount of data stored in the memtable that resides on-heap, including column related overhead and partitions overwritten. */ public final Gauge memtableOnHeapDataSize; @@ -188,6 +190,12 @@ public class TableMetrics public final LatencyMetrics casPropose; /** CAS Commit metrics */ public final LatencyMetrics casCommit; + /** Latency for locally run key migrations **/ + public final LatencyMetrics keyMigration; + /** Latency for range migrations run by locally coordinated Accord repairs **/ + public final LatencyMetrics rangeMigration; + public final TableMeter rangeMigrationUnexpectedFailures; + public final TableMeter rangeMigrationDependencyLimitFailures; /** percent of the data that is repaired */ public final Gauge percentRepaired; /** Reports the size of sstables in repaired, unrepaired, and any ongoing repair buckets */ @@ -624,6 +632,7 @@ public class TableMetrics readLatency = createLatencyMetrics("Read", cfs.keyspace.metric.readLatency, GLOBAL_READ_LATENCY); writeLatency = createLatencyMetrics("Write", cfs.keyspace.metric.writeLatency, GLOBAL_WRITE_LATENCY); rangeLatency = createLatencyMetrics("Range", cfs.keyspace.metric.rangeLatency, GLOBAL_RANGE_LATENCY); + pendingFlushes = createTableCounter("PendingFlushes"); bytesFlushed = createTableCounter("BytesFlushed"); flushSizeOnDisk = ExpMovingAverage.decayBy1000(); @@ -804,6 +813,10 @@ public class TableMetrics casPrepare = createLatencyMetrics("CasPrepare", cfs.keyspace.metric.casPrepare); casPropose = createLatencyMetrics("CasPropose", cfs.keyspace.metric.casPropose); casCommit = createLatencyMetrics("CasCommit", cfs.keyspace.metric.casCommit); + keyMigration = createLatencyMetrics("KeyMigration", cfs.keyspace.metric.keyMigration, GLOBAL_KEY_MIGRATION_LATENCY); + rangeMigration = createLatencyMetrics("RangeMigration", cfs.keyspace.metric.rangeMigration, GLOBAL_RANGE_MIGRATION_LATENCY); + rangeMigrationUnexpectedFailures = createTableMeter("RangeMigrationUnexpectedFailures", cfs.keyspace.metric.rangeMigrationUnexpectedFailures); + rangeMigrationDependencyLimitFailures = createTableMeter("RangeMigrationDependencyLimitFaiures", cfs.keyspace.metric.rangeMigrationDependencyLimitFailures); repairsStarted = createTableCounter("RepairJobsStarted"); repairsCompleted = createTableCounter("RepairJobsCompleted"); diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java index 146f8c1119..57359cc039 100644 --- a/src/java/org/apache/cassandra/net/Message.java +++ b/src/java/org/apache/cassandra/net/Message.java @@ -303,6 +303,7 @@ public class Message implements ReplyContext * Used by the {@code MultiRangeReadCommand} to split multi-range responses from a replica * into single-range responses. */ + @VisibleForTesting public static Message remoteResponse(InetAddressAndPort from, Verb verb, T payload) { assert verb.isResponse(); @@ -574,6 +575,11 @@ public class Message implements ReplyContext return MessageFlag.TRACK_WARNINGS.isIn(flags); } + boolean isFinal() + { + return !MessageFlag.NOT_FINAL.isIn(flags); + } + @Nullable ForwardingInfo forwardTo() { diff --git a/src/java/org/apache/cassandra/net/MessageFlag.java b/src/java/org/apache/cassandra/net/MessageFlag.java index 1c2db557c3..4c5762f979 100644 --- a/src/java/org/apache/cassandra/net/MessageFlag.java +++ b/src/java/org/apache/cassandra/net/MessageFlag.java @@ -31,7 +31,10 @@ public enum MessageFlag /** allow creating warnings or aborting queries based off query - see CASSANDRA-16850 */ TRACK_WARNINGS(2), /** whether this message should be sent on an URGENT channel despite its Verb default priority */ - URGENT(3); + URGENT(3), + /** Allow a single callback to receive multiple responses until a final response is received **/ + NOT_FINAL(4) + ; private final int id; diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index 12d3f17cd4..2457f2dbce 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -500,9 +500,9 @@ public class MessagingService extends MessagingServiceMBeanImpl implements Messa } @Override - public void onFailure(InetAddressAndPort from, RequestFailure failureReason) + public void onFailure(InetAddressAndPort from, RequestFailure failure) { - future.setFailure(new RuntimeException(failureReason.toString())); + future.setFailure(new RuntimeException(failure.toString())); } }); diff --git a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java index 6cecd2a415..f89362715b 100644 --- a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java +++ b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java @@ -58,7 +58,11 @@ class ResponseVerbHandler implements IVerbHandler @Override public void doVerb(Message message) { - RequestCallbacks.CallbackInfo callbackInfo = MessagingService.instance().callbacks.remove(message.id(), message.from()); + RequestCallbacks.CallbackInfo callbackInfo; + if (message.header.isFinal()) + callbackInfo = MessagingService.instance().callbacks.remove(message.id(), message.from()); + else + callbackInfo = MessagingService.instance().callbacks.get(message.id(), message.from()); if (callbackInfo == null) { String msg = "Callback already removed for {} (from {})"; diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java index 54849389f0..97e77eccbb 100644 --- a/src/java/org/apache/cassandra/net/Verb.java +++ b/src/java/org/apache/cassandra/net/Verb.java @@ -81,6 +81,10 @@ import org.apache.cassandra.service.SnapshotVerbHandler; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordSyncPropagator; import org.apache.cassandra.service.accord.AccordSyncPropagator.Notification; +import org.apache.cassandra.service.accord.interop.AccordInteropApply; +import org.apache.cassandra.service.accord.interop.AccordInteropCommit; +import org.apache.cassandra.service.accord.interop.AccordInteropRead; +import org.apache.cassandra.service.accord.interop.AccordInteropReadRepair; import org.apache.cassandra.service.accord.serializers.AcceptSerializers; import org.apache.cassandra.service.accord.serializers.ApplySerializers; import org.apache.cassandra.service.accord.serializers.BeginInvalidationSerializers; @@ -98,6 +102,8 @@ import org.apache.cassandra.service.accord.serializers.ReadDataSerializers; import org.apache.cassandra.service.accord.serializers.RecoverySerializers; import org.apache.cassandra.service.accord.serializers.SetDurableSerializers; import org.apache.cassandra.service.accord.serializers.WaitOnCommitSerializer; +import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.ConsensusKeyMigrationFinished; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.service.paxos.Commit.Agreed; import org.apache.cassandra.service.paxos.PaxosCommit; @@ -260,7 +266,7 @@ public enum Verb PAXOS2_PREPARE_REQ (40, P2, writeTimeout, MUTATION, () -> PaxosPrepare.requestSerializer, () -> PaxosPrepare.requestHandler, PAXOS2_PREPARE_RSP ), PAXOS2_PREPARE_REFRESH_RSP (51, P2, writeTimeout, REQUEST_RESPONSE, () -> PaxosPrepareRefresh.responseSerializer, RESPONSE_HANDLER ), PAXOS2_PREPARE_REFRESH_REQ (41, P2, writeTimeout, MUTATION, () -> PaxosPrepareRefresh.requestSerializer, () -> PaxosPrepareRefresh.requestHandler, PAXOS2_PREPARE_REFRESH_RSP ), - PAXOS2_PROPOSE_RSP (52, P2, writeTimeout, REQUEST_RESPONSE, () -> PaxosPropose.responseSerializer, RESPONSE_HANDLER ), + PAXOS2_PROPOSE_RSP (52, P2, writeTimeout, REQUEST_RESPONSE, () -> PaxosPropose.ACCEPT_RESULT_SERIALIZER, RESPONSE_HANDLER ), PAXOS2_PROPOSE_REQ (42, P2, writeTimeout, MUTATION, () -> PaxosPropose.requestSerializer, () -> PaxosPropose.requestHandler, PAXOS2_PROPOSE_RSP ), PAXOS2_COMMIT_AND_PREPARE_RSP (53, P2, writeTimeout, REQUEST_RESPONSE, () -> PaxosPrepare.responseSerializer, RESPONSE_HANDLER ), PAXOS2_COMMIT_AND_PREPARE_REQ (43, P2, writeTimeout, MUTATION, () -> PaxosCommitAndPrepare.requestSerializer, () -> PaxosCommitAndPrepare.requestHandler, PAXOS2_COMMIT_AND_PREPARE_RSP ), @@ -300,40 +306,51 @@ public enum Verb // accord ACCORD_SIMPLE_RSP (119, P2, writeTimeout, REQUEST_RESPONSE, () -> EnumSerializer.simpleReply, RESPONSE_HANDLER ), - ACCORD_PRE_ACCEPT_RSP (121, P2, writeTimeout, REQUEST_RESPONSE, () -> PreacceptSerializers.reply, RESPONSE_HANDLER ), - ACCORD_PRE_ACCEPT_REQ (120, P2, writeTimeout, IMMEDIATE, () -> PreacceptSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_PRE_ACCEPT_RSP ), - ACCORD_ACCEPT_RSP (124, P2, writeTimeout, REQUEST_RESPONSE, () -> AcceptSerializers.reply, RESPONSE_HANDLER ), - ACCORD_ACCEPT_REQ (122, P2, writeTimeout, IMMEDIATE, () -> AcceptSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_ACCEPT_RSP ), - ACCORD_ACCEPT_INVALIDATE_REQ (123, P2, writeTimeout, IMMEDIATE, () -> AcceptSerializers.invalidate, AccordService::verbHandlerOrNoop, ACCORD_ACCEPT_RSP ), - ACCORD_READ_RSP (126, P2, writeTimeout, REQUEST_RESPONSE, () -> ReadDataSerializers.reply, RESPONSE_HANDLER ), - ACCORD_READ_REQ (125, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_READ_RSP ), - ACCORD_COMMIT_REQ (127, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_READ_RSP ), - ACCORD_COMMIT_INVALIDATE_REQ (128, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.invalidate, AccordService::verbHandlerOrNoop ), - ACCORD_APPLY_RSP (130, P2, writeTimeout, REQUEST_RESPONSE, () -> ApplySerializers.reply, RESPONSE_HANDLER ), - ACCORD_APPLY_REQ (129, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.request, AccordService::verbHandlerOrNoop, ACCORD_APPLY_RSP ), - ACCORD_BEGIN_RECOVER_RSP (132, P2, writeTimeout, REQUEST_RESPONSE, () -> RecoverySerializers.reply, RESPONSE_HANDLER ), - ACCORD_BEGIN_RECOVER_REQ (131, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.request, AccordService::verbHandlerOrNoop, ACCORD_BEGIN_RECOVER_RSP ), - ACCORD_BEGIN_INVALIDATE_RSP (134, P2, writeTimeout, REQUEST_RESPONSE, () -> BeginInvalidationSerializers.reply, RESPONSE_HANDLER ), - ACCORD_BEGIN_INVALIDATE_REQ (133, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_BEGIN_INVALIDATE_RSP ), + ACCORD_PRE_ACCEPT_RSP (120, P2, writeTimeout, REQUEST_RESPONSE, () -> PreacceptSerializers.reply, RESPONSE_HANDLER ), + ACCORD_PRE_ACCEPT_REQ (121, P2, writeTimeout, IMMEDIATE, () -> PreacceptSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_PRE_ACCEPT_RSP ), + ACCORD_ACCEPT_RSP (122, P2, writeTimeout, REQUEST_RESPONSE, () -> AcceptSerializers.reply, RESPONSE_HANDLER ), + ACCORD_ACCEPT_REQ (123, P2, writeTimeout, IMMEDIATE, () -> AcceptSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_ACCEPT_RSP ), + ACCORD_ACCEPT_INVALIDATE_REQ (124, P2, writeTimeout, IMMEDIATE, () -> AcceptSerializers.invalidate, AccordService::verbHandlerOrNoop, ACCORD_ACCEPT_RSP ), + ACCORD_READ_RSP (125, P2, writeTimeout, REQUEST_RESPONSE, () -> ReadDataSerializers.reply, RESPONSE_HANDLER ), + ACCORD_READ_REQ (126, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.readData, AccordService::verbHandlerOrNoop, ACCORD_READ_RSP ), + ACCORD_COMMIT_REQ (127, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_READ_RSP ), + ACCORD_COMMIT_INVALIDATE_REQ (128, P2, writeTimeout, IMMEDIATE, () -> CommitSerializers.invalidate, AccordService::verbHandlerOrNoop ), + ACCORD_APPLY_RSP (129, P2, writeTimeout, REQUEST_RESPONSE, () -> ApplySerializers.reply, RESPONSE_HANDLER ), + ACCORD_APPLY_REQ (130, P2, writeTimeout, IMMEDIATE, () -> ApplySerializers.request, AccordService::verbHandlerOrNoop, ACCORD_APPLY_RSP ), + ACCORD_BEGIN_RECOVER_RSP (131, P2, writeTimeout, REQUEST_RESPONSE, () -> RecoverySerializers.reply, RESPONSE_HANDLER ), + ACCORD_BEGIN_RECOVER_REQ (132, P2, writeTimeout, IMMEDIATE, () -> RecoverySerializers.request, AccordService::verbHandlerOrNoop, ACCORD_BEGIN_RECOVER_RSP ), + ACCORD_BEGIN_INVALIDATE_RSP (133, P2, writeTimeout, REQUEST_RESPONSE, () -> BeginInvalidationSerializers.reply, RESPONSE_HANDLER ), + ACCORD_BEGIN_INVALIDATE_REQ (134, P2, writeTimeout, IMMEDIATE, () -> BeginInvalidationSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_BEGIN_INVALIDATE_RSP ), ACCORD_WAIT_ON_COMMIT_RSP (136, P2, writeTimeout, REQUEST_RESPONSE, () -> WaitOnCommitSerializer.reply, RESPONSE_HANDLER ), - ACCORD_WAIT_ON_COMMIT_REQ (135, P2, writeTimeout, IMMEDIATE, () -> WaitOnCommitSerializer.request, AccordService::verbHandlerOrNoop, ACCORD_WAIT_ON_COMMIT_RSP ), - ACCORD_WAIT_ON_APPLY_REQ (137, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.waitOnApply, AccordService::verbHandlerOrNoop, ACCORD_READ_RSP ), - ACCORD_INFORM_OF_TXN_REQ (138, P2, writeTimeout, IMMEDIATE, () -> InformOfTxnIdSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), - ACCORD_INFORM_HOME_DURABLE_REQ (139, P2, writeTimeout, IMMEDIATE, () -> InformHomeDurableSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), - ACCORD_INFORM_DURABLE_REQ (140, P2, writeTimeout, IMMEDIATE, () -> InformDurableSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), - ACCORD_CHECK_STATUS_RSP (142, P2, writeTimeout, REQUEST_RESPONSE, () -> CheckStatusSerializers.reply, RESPONSE_HANDLER ), - ACCORD_CHECK_STATUS_REQ (141, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_CHECK_STATUS_RSP ), - ACCORD_GET_DEPS_RSP (144, P2, writeTimeout, REQUEST_RESPONSE, () -> GetDepsSerializers.reply, RESPONSE_HANDLER ), - ACCORD_GET_DEPS_REQ (143, P2, writeTimeout, IMMEDIATE, () -> GetDepsSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_GET_DEPS_RSP ), - ACCORD_FETCH_DATA_RSP (146, P2, repairTimeout,REQUEST_RESPONSE, () -> FetchSerializers.reply, RESPONSE_HANDLER ), - ACCORD_FETCH_DATA_REQ (145, P2, repairTimeout,IMMEDIATE, () -> FetchSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_FETCH_DATA_RSP ), - ACCORD_SET_SHARD_DURABLE_REQ (147, P2, writeTimeout, IMMEDIATE, () -> SetDurableSerializers.shardDurable, AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), - ACCORD_SET_GLOBALLY_DURABLE_REQ (148, P2, writeTimeout, IMMEDIATE, () -> SetDurableSerializers.globallyDurable,AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), - ACCORD_QUERY_DURABLE_BEFORE_RSP (150, P2, writeTimeout, REQUEST_RESPONSE, () -> QueryDurableBeforeSerializers.reply, RESPONSE_HANDLER ), - ACCORD_QUERY_DURABLE_BEFORE_REQ (149, P2, writeTimeout, IMMEDIATE, () -> QueryDurableBeforeSerializers.request,AccordService::verbHandlerOrNoop, ACCORD_QUERY_DURABLE_BEFORE_RSP), + ACCORD_WAIT_ON_COMMIT_REQ (135, P2, writeTimeout, IMMEDIATE, () -> WaitOnCommitSerializer.request, AccordService::verbHandlerOrNoop, ACCORD_WAIT_ON_COMMIT_RSP ), + ACCORD_WAIT_UNTIL_APPLIED_REQ (137, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.waitUntilApplied, AccordService::verbHandlerOrNoop, ACCORD_READ_RSP ), + ACCORD_INFORM_OF_TXN_REQ (138, P2, writeTimeout, IMMEDIATE, () -> InformOfTxnIdSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), + ACCORD_INFORM_HOME_DURABLE_REQ (139, P2, writeTimeout, IMMEDIATE, () -> InformHomeDurableSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), + ACCORD_INFORM_DURABLE_REQ (140, P2, writeTimeout, IMMEDIATE, () -> InformDurableSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), + ACCORD_CHECK_STATUS_RSP (141, P2, writeTimeout, REQUEST_RESPONSE, () -> CheckStatusSerializers.reply, RESPONSE_HANDLER ), + ACCORD_CHECK_STATUS_REQ (142, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_CHECK_STATUS_RSP ), + ACCORD_GET_DEPS_RSP (143, P2, writeTimeout, REQUEST_RESPONSE, () -> GetDepsSerializers.reply, RESPONSE_HANDLER ), + ACCORD_GET_DEPS_REQ (144, P2, writeTimeout, IMMEDIATE, () -> GetDepsSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_GET_DEPS_RSP ), + ACCORD_FETCH_DATA_RSP (145, P2, repairTimeout,REQUEST_RESPONSE, () -> FetchSerializers.reply, RESPONSE_HANDLER ), + ACCORD_FETCH_DATA_REQ (146, P2, repairTimeout,IMMEDIATE, () -> FetchSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_FETCH_DATA_RSP ), + ACCORD_SET_SHARD_DURABLE_REQ (147, P2, writeTimeout, IMMEDIATE, () -> SetDurableSerializers.shardDurable, AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), + ACCORD_SET_GLOBALLY_DURABLE_REQ (148, P2, writeTimeout, IMMEDIATE, () -> SetDurableSerializers.globallyDurable,AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP ), + ACCORD_QUERY_DURABLE_BEFORE_RSP (149, P2, writeTimeout, REQUEST_RESPONSE, () -> QueryDurableBeforeSerializers.reply, RESPONSE_HANDLER ), + ACCORD_QUERY_DURABLE_BEFORE_REQ (150, P2, writeTimeout, IMMEDIATE, () -> QueryDurableBeforeSerializers.request,AccordService::verbHandlerOrNoop, ACCORD_QUERY_DURABLE_BEFORE_RSP ), ACCORD_SYNC_NOTIFY_REQ (151, P2, writeTimeout, IMMEDIATE, () -> Notification.listSerializer, () -> AccordSyncPropagator.verbHandler, ACCORD_SIMPLE_RSP ), + ACCORD_APPLY_AND_WAIT_UNTIL_APPLIED_REQ(152, P2, writeTimeout, IMMEDIATE, () -> ReadDataSerializers.readData,() -> AccordSyncPropagator.verbHandler, ACCORD_READ_RSP), + + CONSENSUS_KEY_MIGRATION (153, P1, writeTimeout, MUTATION, () -> ConsensusKeyMigrationFinished.serializer,() -> ConsensusKeyMigrationState.consensusKeyMigrationFinishedHandler), + + ACCORD_INTEROP_READ_RSP (154, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.replySerializer, RESPONSE_HANDLER), + ACCORD_INTEROP_READ_REQ (155, P2, writeTimeout, IMMEDIATE, () -> AccordInteropRead.requestSerializer, () -> AccordService.instance().verbHandler(), ACCORD_INTEROP_READ_RSP), + ACCORD_INTEROP_COMMIT_REQ (156, P2, writeTimeout, IMMEDIATE, () -> AccordInteropCommit.serializer, () -> AccordService.instance().verbHandler(), ACCORD_INTEROP_READ_RSP), + ACCORD_INTEROP_READ_REPAIR_RSP (157, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.replySerializer, RESPONSE_HANDLER), + ACCORD_INTEROP_READ_REPAIR_REQ (158, P2, writeTimeout, IMMEDIATE, () -> AccordInteropReadRepair.requestSerializer, () -> AccordService.instance().verbHandler(), ACCORD_INTEROP_READ_REPAIR_RSP), + ACCORD_INTEROP_APPLY_REQ (160, P2, writeTimeout, IMMEDIATE, () -> AccordInteropApply.serializer, AccordService::verbHandlerOrNoop, ACCORD_APPLY_RSP), + // generic failure response FAILURE_RSP (99, P0, noTimeout, REQUEST_RESPONSE, () -> RequestFailure.serializer, RESPONSE_HANDLER ), diff --git a/src/java/org/apache/cassandra/repair/AbstractRepairJob.java b/src/java/org/apache/cassandra/repair/AbstractRepairJob.java new file mode 100644 index 0000000000..7c4346e3bf --- /dev/null +++ b/src/java/org/apache/cassandra/repair/AbstractRepairJob.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.concurrent.Executor; +import javax.annotation.Nullable; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.repair.state.JobState; +import org.apache.cassandra.utils.concurrent.AsyncFuture; + +public abstract class AbstractRepairJob extends AsyncFuture implements Runnable +{ + private final SharedContext ctx; + public final JobState state; + protected final RepairJobDesc desc; + protected final RepairSession session; + protected final Executor taskExecutor; + + protected final Keyspace ks; + protected final ColumnFamilyStore cfs; + + /** + * Create repair job to run on specific columnfamily + * @param session RepairSession that this RepairJob belongs + * @param columnFamily name of the ColumnFamily to repair + */ + public AbstractRepairJob(RepairSession session, String columnFamily) + { + this.ctx = session.ctx; + this.session = session; + this.taskExecutor = session.taskExecutor; + this.desc = new RepairJobDesc(session.state.parentRepairSession, session.getId(), session.state.keyspace, columnFamily, session.state.commonRange.ranges); + this.state = new JobState(ctx.clock(), desc, session.state.commonRange.endpoints); + this.ks = Keyspace.open(desc.keyspace); + this.cfs = ks.getColumnFamilyStore(columnFamily); + } + + public void run() + { + state.phase.start(); + cfs.metric.repairsStarted.inc(); + runRepair(); + } + + abstract protected void runRepair(); + + abstract void abort(@Nullable Throwable reason); +} diff --git a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java index f27e72deb1..e6ba28aee6 100644 --- a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java +++ b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java @@ -24,7 +24,6 @@ import java.util.Objects; import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +58,7 @@ public abstract class AbstractRepairTask implements RepairTask ExecutorPlus executor, Scheduler validationScheduler, List commonRanges, + boolean excludedDeadNodes, String... cfnames) { List futures = new ArrayList<>(options.getRanges().size()); @@ -68,6 +68,7 @@ public abstract class AbstractRepairTask implements RepairTask logger.info("Starting RepairSession for {}", commonRange); RepairSession session = coordinator.ctx.repair().submitRepairSession(parentSession, commonRange, + excludedDeadNodes, keyspace, options.getParallelism(), isIncremental, @@ -77,6 +78,7 @@ public abstract class AbstractRepairTask implements RepairTask options.repairPaxos(), options.paxosOnly(), options.dontPurgeTombstones(), + options.accordRepair(), executor, validationScheduler, cfnames); @@ -93,9 +95,10 @@ public abstract class AbstractRepairTask implements RepairTask ExecutorPlus executor, Scheduler validationScheduler, List commonRanges, + boolean excludedDeadNodes, String... cfnames) { - List allSessions = submitRepairSessions(parentSession, isIncremental, executor, validationScheduler, commonRanges, cfnames); + List allSessions = submitRepairSessions(parentSession, isIncremental, executor, validationScheduler, commonRanges, excludedDeadNodes, cfnames); List>> ranges = Lists.transform(allSessions, RepairSession::ranges); Future> f = FutureCombiner.successfulOf(allSessions); return f.map(results -> { diff --git a/src/java/org/apache/cassandra/repair/AccordRepairJob.java b/src/java/org/apache/cassandra/repair/AccordRepairJob.java new file mode 100644 index 0000000000..1736d799fa --- /dev/null +++ b/src/java/org/apache/cassandra/repair/AccordRepairJob.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.math.BigInteger; +import java.util.List; +import javax.annotation.Nullable; + +import accord.api.BarrierType; +import accord.api.RoutingKey; +import accord.primitives.Ranges; +import accord.primitives.Seekables; +import org.apache.cassandra.dht.AccordSplitter; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.TokenRange; +import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationRepairResult; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Collections.emptyList; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +/* + * Accord repair consists of creating a barrier transaction for all the ranges which ensure that all Accord transactions + * before the Epoch and point in time at which the repair started have their side effects visible to Paxos and regular quorum reads. + */ +public class AccordRepairJob extends AbstractRepairJob +{ + public static final BigInteger TWO = BigInteger.valueOf(2); + + private final Ranges ranges; + + private final AccordSplitter splitter; + + private BigInteger rangeStep; + + private Epoch minEpoch = ClusterMetadata.current().epoch; + + public AccordRepairJob(RepairSession repairSession, String cfname) + { + super(repairSession, cfname); + List> normalizedRanges = Range.normalize(desc.ranges); + IPartitioner partitioner = normalizedRanges.get(0).left.getPartitioner(); + TokenRange[] tokenRanges = new TokenRange[normalizedRanges.size()]; + for (int i = 0; i < normalizedRanges.size(); i++) + tokenRanges[i] = new TokenRange(new TokenKey(ks.getName(), normalizedRanges.get(i).left), new TokenKey(ks.getName(), normalizedRanges.get(i).right)); + this.ranges = Ranges.of(tokenRanges); + this.splitter = partitioner.accordSplitter().apply(Ranges.of(tokenRanges)); + } + + @Override + protected void runRepair() + { + try + { + for (accord.primitives.Range range : ranges) + repairRange((TokenRange)range); + state.phase.success(); + cfs.metric.repairsCompleted.inc(); + trySuccess(new RepairResult(desc, emptyList(), ConsensusMigrationRepairResult.fromAccordRepair(minEpoch))); + } + catch (Throwable t) + { + state.phase.fail(t); + cfs.metric.repairsCompleted.inc(); + tryFailure(t); + } + } + + @Override + void abort(@Nullable Throwable reason) + { + throw new UnsupportedOperationException("Have not implemented this yet, and the job runs synchronously so it isn't abortable"); + } + + private void repairRange(TokenRange range) + { + RoutingKey remainingStart = range.start(); + BigInteger rangeSize = splitter.sizeOf(range); + if (rangeStep == null) + rangeStep = BigInteger.ONE.max(splitter.divide(rangeSize, 1000)); + + BigInteger offset = BigInteger.ZERO; + + TokenRange lastRepaired = null; + int iteration = 0; + while (true) + { + iteration++; + if (iteration % 100 == 0) + rangeStep = rangeStep.multiply(TWO); + + BigInteger remaining = rangeSize.subtract(offset); + BigInteger length = remaining.min(rangeStep); + + long start = nanoTime(); + boolean dependencyOverflow = false; + try + { + // Splitter is approximate so it can't work right up to the end + TokenRange toRepair; + if (splitter.compare(offset, rangeSize) >= 0) + { + if (remainingStart.equals(range.end())) + return; + + // Final repair is whatever remains + toRepair = range.newRange(remainingStart, range.end()); + } + else + { + toRepair = splitter.subRange(range, offset, splitter.add(offset, length)); + checkState(iteration > 1 || toRepair.start().equals(range.start())); + } + checkState(!toRepair.equals(lastRepaired), "Shouldn't repair the same range twice"); + checkState(lastRepaired == null || toRepair.start().equals(lastRepaired.end()), "Next range should directly follow previous range"); + lastRepaired = toRepair; + AccordService.instance().barrierWithRetries(Seekables.of(toRepair), minEpoch.getEpoch(), BarrierType.global_sync, false); + remainingStart = toRepair.end(); + } + catch (RuntimeException e) + { + // TODO Placeholder for dependency limit overflow +// dependencyOverflow = true; + cfs.metric.rangeMigrationDependencyLimitFailures.mark(); + throw e; + } + catch (Throwable t) + { + // unexpected error + cfs.metric.rangeMigrationUnexpectedFailures.mark(); + throw new RuntimeException(t); + } + finally + { + cfs.metric.rangeMigration.addNano(start); + } + + // TODO when dependency limits are added to Accord need to test repair overflow + if (dependencyOverflow) + { + offset = offset.subtract(rangeStep); + if (rangeStep.equals(BigInteger.ONE)) + throw new IllegalStateException("Unable to repair without overflowing with range step of 1"); + rangeStep = BigInteger.ONE.max(rangeStep.divide(TWO)); + continue; + } + + offset = offset.add(length); + } + } +} diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/CassandraRepairJob.java similarity index 95% rename from src/java/org/apache/cassandra/repair/RepairJob.java rename to src/java/org/apache/cassandra/repair/CassandraRepairJob.java index 7b60df9416..95a55cd4d5 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/CassandraRepairJob.java @@ -17,7 +17,14 @@ */ package org.apache.cassandra.repair; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; import java.util.function.Function; @@ -28,18 +35,11 @@ import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import com.google.common.util.concurrent.*; - -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.repair.state.JobState; -import org.apache.cassandra.utils.concurrent.AsyncFuture; +import com.google.common.util.concurrent.FutureCallback; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; @@ -47,9 +47,15 @@ import org.apache.cassandra.repair.asymmetric.DifferenceHolder; import org.apache.cassandra.repair.asymmetric.HostDifferences; import org.apache.cassandra.repair.asymmetric.PreferedNodeFilter; import org.apache.cassandra.repair.asymmetric.ReduceHelper; +import org.apache.cassandra.repair.state.JobState; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SystemDistributedKeyspace; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationRepairResult; import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTrees; @@ -65,9 +71,9 @@ import static org.apache.cassandra.service.paxos.Paxos.useV2; /** * RepairJob runs repair on given ColumnFamily. */ -public class RepairJob extends AsyncFuture implements Runnable +public class CassandraRepairJob extends AbstractRepairJob { - private static final Logger logger = LoggerFactory.getLogger(RepairJob.class); + private static final Logger logger = LoggerFactory.getLogger(CassandraRepairJob.class); private final SharedContext ctx; public final JobState state; @@ -87,8 +93,9 @@ public class RepairJob extends AsyncFuture implements Runnable * @param session RepairSession that this RepairJob belongs * @param columnFamily name of the ColumnFamily to repair */ - public RepairJob(RepairSession session, String columnFamily) + public CassandraRepairJob(RepairSession session, String columnFamily) { + super(session, columnFamily); this.ctx = session.ctx; this.session = session; this.taskExecutor = session.taskExecutor; @@ -116,17 +123,16 @@ public class RepairJob extends AsyncFuture implements Runnable * This sets up necessary task and runs them on given {@code taskExecutor}. * After submitting all tasks, waits until validation with replica completes. */ - public void run() + @Override + protected void runRepair() { - state.phase.start(); - Keyspace ks = Keyspace.open(desc.keyspace); - ColumnFamilyStore cfs = ks.getColumnFamilyStore(desc.columnFamily); - cfs.metric.repairsStarted.inc(); List allEndpoints = new ArrayList<>(session.state.commonRange.endpoints); allEndpoints.add(ctx.broadcastAddressAndPort()); Future paxosRepair; - if (paxosRepairEnabled() && (((useV2() || isMetadataKeyspace()) && session.repairPaxos) || session.paxosOnly)) + Epoch repairStartingEpoch = ClusterMetadata.current().epoch; + boolean doPaxosRepair = paxosRepairEnabled() && (((useV2() || isMetadataKeyspace()) && session.repairPaxos) || session.paxosOnly); + if (doPaxosRepair) { logger.info("{} {}.{} starting paxos repair", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily); TableMetadata metadata = Schema.instance.getTableMetadata(desc.keyspace, desc.columnFamily); @@ -142,10 +148,10 @@ public class RepairJob extends AsyncFuture implements Runnable { paxosRepair.addCallback(new FutureCallback<>() { - public void onSuccess(Void v) + public void onSuccess(Void ignored) { logger.info("{} {}.{} paxos repair completed", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily); - trySuccess(new RepairResult(desc, Collections.emptyList())); + trySuccess(new RepairResult(desc, Collections.emptyList(), ConsensusMigrationRepairResult.fromCassandraRepair(repairStartingEpoch, false))); } /** @@ -211,7 +217,8 @@ public class RepairJob extends AsyncFuture implements Runnable SystemDistributedKeyspace.successfulRepairJob(session.getId(), desc.keyspace, desc.columnFamily); } cfs.metric.repairsCompleted.inc(); - trySuccess(new RepairResult(desc, stats)); + logger.info("Completing repair with excludedDeadNodes {}", session.excludedDeadNodes); + trySuccess(new RepairResult(desc, stats, ConsensusMigrationRepairResult.fromCassandraRepair(repairStartingEpoch, doPaxosRepair && !session.excludedDeadNodes))); } /** diff --git a/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java b/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java index 347846cf6b..956973c494 100644 --- a/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java +++ b/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java @@ -64,7 +64,7 @@ public class IncrementalRepairTask extends AbstractRepairTask CoordinatorSession coordinatorSession = coordinator.ctx.repair().consistent.coordinated.registerSession(parentSession, allParticipants, neighborsAndRanges.shouldExcludeDeadParticipants); - return coordinatorSession.execute(() -> runRepair(parentSession, true, executor, validationScheduler, allRanges, cfnames)); + return coordinatorSession.execute(() -> runRepair(parentSession, true, executor, validationScheduler, allRanges, neighborsAndRanges.shouldExcludeDeadParticipants, cfnames)); } } diff --git a/src/java/org/apache/cassandra/repair/NormalRepairTask.java b/src/java/org/apache/cassandra/repair/NormalRepairTask.java index e304280c58..05b721d17c 100644 --- a/src/java/org/apache/cassandra/repair/NormalRepairTask.java +++ b/src/java/org/apache/cassandra/repair/NormalRepairTask.java @@ -27,16 +27,19 @@ public class NormalRepairTask extends AbstractRepairTask { private final TimeUUID parentSession; private final List commonRanges; + private final boolean excludedDeadNodes; private final String[] cfnames; protected NormalRepairTask(RepairCoordinator coordinator, TimeUUID parentSession, List commonRanges, + boolean excludedDeadNodes, String[] cfnames) { super(coordinator); this.parentSession = parentSession; this.commonRanges = commonRanges; + this.excludedDeadNodes = excludedDeadNodes; this.cfnames = cfnames; } @@ -49,6 +52,6 @@ public class NormalRepairTask extends AbstractRepairTask @Override public Future performUnsafe(ExecutorPlus executor, Scheduler validationScheduler) { - return runRepair(parentSession, false, executor, validationScheduler, commonRanges, cfnames); + return runRepair(parentSession, false, executor, validationScheduler, commonRanges, excludedDeadNodes, cfnames); } } diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java index 95c7a63f94..872199156e 100644 --- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java +++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java @@ -42,14 +42,16 @@ public class PreviewRepairTask extends AbstractRepairTask { private final TimeUUID parentSession; private final List commonRanges; + private final boolean excludedDeadNodes; private final String[] cfnames; private volatile String successMessage = name() + " completed successfully"; - protected PreviewRepairTask(RepairCoordinator coordinator, TimeUUID parentSession, List commonRanges, String[] cfnames) + protected PreviewRepairTask(RepairCoordinator coordinator, TimeUUID parentSession, List commonRanges, boolean excludedDeadNodes, String[] cfnames) { super(coordinator); this.parentSession = parentSession; this.commonRanges = commonRanges; + this.excludedDeadNodes = excludedDeadNodes; this.cfnames = cfnames; } @@ -68,7 +70,7 @@ public class PreviewRepairTask extends AbstractRepairTask @Override public Future performUnsafe(ExecutorPlus executor, Scheduler validationScheduler) { - Future f = runRepair(parentSession, false, executor, validationScheduler, commonRanges, cfnames); + Future f = runRepair(parentSession, false, executor, validationScheduler, commonRanges, excludedDeadNodes, cfnames); return f.map(result -> { if (result.hasFailed()) return result; diff --git a/src/java/org/apache/cassandra/repair/RepairCoordinator.java b/src/java/org/apache/cassandra/repair/RepairCoordinator.java index c4b7a9fd55..79159c9fe1 100644 --- a/src/java/org/apache/cassandra/repair/RepairCoordinator.java +++ b/src/java/org/apache/cassandra/repair/RepairCoordinator.java @@ -478,7 +478,7 @@ public class RepairCoordinator implements Runnable, ProgressEventNotifier, Repai RepairTask task; if (state.options.isPreview()) { - task = new PreviewRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames); + task = new PreviewRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), neighborsAndRanges.shouldExcludeDeadParticipants, cfnames); } else if (state.options.isIncremental()) { @@ -486,7 +486,7 @@ public class RepairCoordinator implements Runnable, ProgressEventNotifier, Repai } else { - task = new NormalRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames); + task = new NormalRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), neighborsAndRanges.shouldExcludeDeadParticipants, cfnames); } ExecutorPlus executor = createExecutor(); diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java index f777126019..5f69da549d 100644 --- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java +++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java @@ -17,7 +17,8 @@ */ package org.apache.cassandra.repair; -import java.util.*; +import java.util.ArrayList; +import java.util.List; import java.util.function.BiFunction; import java.util.function.Function; @@ -28,7 +29,14 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; -import org.apache.cassandra.repair.messages.*; +import org.apache.cassandra.repair.messages.CleanupMessage; +import org.apache.cassandra.repair.messages.FailSession; +import org.apache.cassandra.repair.messages.PrepareMessage; +import org.apache.cassandra.repair.messages.RepairMessage; +import org.apache.cassandra.repair.messages.StatusRequest; +import org.apache.cassandra.repair.messages.StatusResponse; +import org.apache.cassandra.repair.messages.SyncRequest; +import org.apache.cassandra.repair.messages.ValidationRequest; import org.apache.cassandra.repair.state.AbstractCompletable; import org.apache.cassandra.repair.state.AbstractState; import org.apache.cassandra.repair.state.Completable; @@ -39,6 +47,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.TimeUUID; @@ -86,6 +95,7 @@ public class RepairMessageVerbHandler implements IVerbHandler public void doVerb(final Message message) { + ClusterMetadataService.instance().fetchLogFromCMS(message.epoch()); // TODO add cancel/interrupt message RepairJobDesc desc = message.payload.desc; try diff --git a/src/java/org/apache/cassandra/repair/RepairResult.java b/src/java/org/apache/cassandra/repair/RepairResult.java index 333b48ad33..4899448c71 100644 --- a/src/java/org/apache/cassandra/repair/RepairResult.java +++ b/src/java/org/apache/cassandra/repair/RepairResult.java @@ -19,6 +19,8 @@ package org.apache.cassandra.repair; import java.util.List; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationRepairResult; + /** * RepairJob's result */ @@ -26,10 +28,12 @@ public class RepairResult { public final RepairJobDesc desc; public final List stats; + public final ConsensusMigrationRepairResult consensusMigrationRepairResult; - public RepairResult(RepairJobDesc desc, List stats) + public RepairResult(RepairJobDesc desc, List stats, ConsensusMigrationRepairResult consensusMigrationRepairResult) { this.desc = desc; this.stats = stats; + this.consensusMigrationRepairResult = consensusMigrationRepairResult; } } diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java index 92d56390fe..ae46877a6b 100644 --- a/src/java/org/apache/cassandra/repair/RepairSession.java +++ b/src/java/org/apache/cassandra/repair/RepairSession.java @@ -30,13 +30,11 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; - import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.*; - +import com.google.common.util.concurrent.FutureCallback; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +46,9 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RepairException; -import org.apache.cassandra.gms.*; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.IEndpointStateChangeSubscriber; +import org.apache.cassandra.gms.IFailureDetectionEventListener; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.repair.consistent.ConsistentSession; @@ -59,6 +59,7 @@ import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.repair.state.SessionState; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; @@ -73,7 +74,7 @@ import org.apache.cassandra.utils.concurrent.AsyncFuture; * * A given RepairSession repairs a set of replicas for a given set of ranges on a list * of column families. For each of the column family to repair, RepairSession - * creates a {@link RepairJob} that handles the repair of that CF. + * creates a {@link AbstractRepairJob} that handles the repair of that CF. * * A given RepairJob has the 3 main phases: *
    @@ -122,6 +123,7 @@ public class RepairSession extends AsyncFuture implements I public final boolean repairPaxos; public final boolean paxosOnly; public final boolean dontPurgeTombstones; + public final boolean excludedDeadNodes; private final AtomicBoolean isFailed = new AtomicBoolean(false); @@ -135,7 +137,8 @@ public class RepairSession extends AsyncFuture implements I public final boolean optimiseStreams; public final SharedContext ctx; public final Scheduler validationScheduler; - private volatile List jobs = Collections.emptyList(); + private volatile List jobs = Collections.emptyList(); + private final boolean accordRepair; private volatile boolean terminated = false; @@ -143,6 +146,7 @@ public class RepairSession extends AsyncFuture implements I * Create new repair session. * @param parentRepairSession the parent sessions id * @param commonRange ranges to repair + * @param excludedDeadNodes Was the repair started for --force and were dead nodes excluded as a result * @param keyspace name of keyspace * @param parallelismDegree specifies the degree of parallelism when calculating the merkle trees * @param pullRepair true if the repair should be one way (from remote host to this host and only applicable between two hosts--see RepairOption) @@ -154,6 +158,7 @@ public class RepairSession extends AsyncFuture implements I Scheduler validationScheduler, TimeUUID parentRepairSession, CommonRange commonRange, + boolean excludedDeadNodes, String keyspace, RepairParallelism parallelismDegree, boolean isIncremental, @@ -163,6 +168,7 @@ public class RepairSession extends AsyncFuture implements I boolean repairPaxos, boolean paxosOnly, boolean dontPurgeTombstones, + boolean accordRepair, String... cfnames) { this.ctx = ctx; @@ -178,6 +184,8 @@ public class RepairSession extends AsyncFuture implements I this.optimiseStreams = optimiseStreams; this.dontPurgeTombstones = dontPurgeTombstones; this.taskExecutor = new SafeExecutor(createExecutor(ctx)); + this.accordRepair = accordRepair; + this.excludedDeadNodes = excludedDeadNodes; } @VisibleForTesting @@ -338,10 +346,14 @@ public class RepairSession extends AsyncFuture implements I // Create and submit RepairJob for each ColumnFamily state.phase.jobsSubmitted(); - List jobs = new ArrayList<>(state.cfnames.length); + List jobs = new ArrayList<>(state.cfnames.length); for (String cfname : state.cfnames) { - RepairJob job = new RepairJob(this, cfname); + AbstractRepairJob job = accordRepair ? + new AccordRepairJob(this, cfname) : + new CassandraRepairJob(this, cfname); + // Repairs can drive forward progress for consensus migration so always check + job.addCallback(ConsensusTableMigrationState.completedRepairJobHandler); state.register(job.state); executor.execute(job); jobs.add(job); @@ -381,10 +393,10 @@ public class RepairSession extends AsyncFuture implements I public synchronized void terminate(@Nullable Throwable reason) { terminated = true; - List jobs = this.jobs; + List jobs = this.jobs; if (jobs != null) { - for (RepairJob job : jobs) + for (AbstractRepairJob job : jobs) job.abort(reason); } this.jobs = null; diff --git a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java index e38a930bcc..2eab88ef91 100644 --- a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java +++ b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java @@ -25,7 +25,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Supplier; - import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; @@ -74,7 +73,7 @@ public abstract class RepairMessage } @Override - public void onFailure(InetAddressAndPort from, RequestFailure failureReason) + public void onFailure(InetAddressAndPort from, RequestFailure failure) { } }; diff --git a/src/java/org/apache/cassandra/repair/messages/RepairOption.java b/src/java/org/apache/cassandra/repair/messages/RepairOption.java index bc9231dcc1..bef7acfe16 100644 --- a/src/java/org/apache/cassandra/repair/messages/RepairOption.java +++ b/src/java/org/apache/cassandra/repair/messages/RepairOption.java @@ -17,7 +17,13 @@ */ package org.apache.cassandra.repair.messages; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -57,6 +63,8 @@ public class RepairOption public static final String NO_TOMBSTONE_PURGING = "nopurge"; + public static final String ACCORD_REPAIR_KEY = "accordRepair"; + // we don't want to push nodes too much for repair public static final int MAX_JOB_THREADS = 4; @@ -86,6 +94,7 @@ public class RepairOption } return ranges; } + /** * Construct RepairOptions object from given map of Strings. *

    @@ -167,6 +176,12 @@ public class RepairOption * ranges to the same host multiple times * false * + * + * accordRepair + * "true" if the repair should be of Accord in flight transactions. Will ensure + * that once repair completes all Accord transactions are replicated at quorum + * false + * * * * @@ -188,11 +203,21 @@ public class RepairOption boolean repairPaxos = Boolean.parseBoolean(options.get(REPAIR_PAXOS_KEY)); boolean paxosOnly = Boolean.parseBoolean(options.get(PAXOS_ONLY_KEY)); boolean dontPurgeTombstones = Boolean.parseBoolean(options.get(NO_TOMBSTONE_PURGING)); + boolean accordRepair = Boolean.parseBoolean(options.get(ACCORD_REPAIR_KEY)); if (previewKind != PreviewKind.NONE) { Preconditions.checkArgument(!repairPaxos, "repairPaxos must be set to false for preview repairs"); Preconditions.checkArgument(!paxosOnly, "paxosOnly must be set to false for preview repairs"); + Preconditions.checkArgument(!accordRepair, "accordRepair must be set to false for preview repairs"); + } + + if (accordRepair) + { + Preconditions.checkArgument(!paxosOnly, "paxosOnly must be set to false for Accord repairs"); + Preconditions.checkArgument(previewKind == PreviewKind.NONE, "Can't perform preview repair with an Accord repair"); + Preconditions.checkArgument(!force, "Accord repair only requires a quorum to work so force is not supported"); + incremental = false; } int jobThreads = 1; @@ -212,7 +237,7 @@ public class RepairOption boolean asymmetricSyncing = Boolean.parseBoolean(options.get(OPTIMISE_STREAMS_KEY)); - RepairOption option = new RepairOption(parallelism, primaryRange, incremental, trace, jobThreads, ranges, !ranges.isEmpty(), pullRepair, force, previewKind, asymmetricSyncing, ignoreUnreplicatedKeyspaces, repairPaxos, paxosOnly, dontPurgeTombstones); + RepairOption option = new RepairOption(parallelism, primaryRange, incremental, trace, jobThreads, ranges, pullRepair, force, previewKind, asymmetricSyncing, ignoreUnreplicatedKeyspaces, repairPaxos, paxosOnly, dontPurgeTombstones, accordRepair); // data centers String dataCentersStr = options.get(DATACENTERS_KEY); @@ -286,7 +311,6 @@ public class RepairOption private final boolean incremental; private final boolean trace; private final int jobThreads; - private final boolean isSubrangeRepair; private final boolean pullRepair; private final boolean forceRepair; private final PreviewKind previewKind; @@ -296,12 +320,17 @@ public class RepairOption private final boolean paxosOnly; private final boolean dontPurgeTombstones; + private final boolean accordRepair; + private final Collection columnFamilies = new HashSet<>(); private final Collection dataCenters = new HashSet<>(); private final Collection hosts = new HashSet<>(); private final Collection> ranges = new HashSet<>(); - public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean incremental, boolean trace, int jobThreads, Collection> ranges, boolean isSubrangeRepair, boolean pullRepair, boolean forceRepair, PreviewKind previewKind, boolean optimiseStreams, boolean ignoreUnreplicatedKeyspaces, boolean repairPaxos, boolean paxosOnly, boolean dontPurgeTombstones) + public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean incremental, boolean trace, int jobThreads, + Collection> ranges, boolean pullRepair, boolean forceRepair, + PreviewKind previewKind, boolean optimiseStreams, boolean ignoreUnreplicatedKeyspaces, boolean repairPaxos, + boolean paxosOnly, boolean dontPurgeTombstones, boolean accordRepair) { this.parallelism = parallelism; @@ -310,7 +339,6 @@ public class RepairOption this.trace = trace; this.jobThreads = jobThreads; this.ranges.addAll(ranges); - this.isSubrangeRepair = isSubrangeRepair; this.pullRepair = pullRepair; this.forceRepair = forceRepair; this.previewKind = previewKind; @@ -319,6 +347,7 @@ public class RepairOption this.repairPaxos = repairPaxos; this.paxosOnly = paxosOnly; this.dontPurgeTombstones = dontPurgeTombstones; + this.accordRepair = accordRepair; } public RepairParallelism getParallelism() @@ -381,11 +410,6 @@ public class RepairOption return dataCenters.isEmpty() && hosts.isEmpty(); } - public boolean isSubrangeRepair() - { - return isSubrangeRepair; - } - public PreviewKind getPreviewKind() { return previewKind; @@ -439,6 +463,11 @@ public class RepairOption return dontPurgeTombstones; } + public boolean accordRepair() + { + return accordRepair; + } + @Override public String toString() { @@ -459,6 +488,7 @@ public class RepairOption ", repairPaxos: " + repairPaxos + ", paxosOnly: " + paxosOnly + ", dontPurgeTombstones: " + dontPurgeTombstones + + ", accordRepair: " + accordRepair + ')'; } @@ -472,7 +502,6 @@ public class RepairOption options.put(COLUMNFAMILIES_KEY, Joiner.on(",").join(columnFamilies)); options.put(DATACENTERS_KEY, Joiner.on(",").join(dataCenters)); options.put(HOSTS_KEY, Joiner.on(",").join(hosts)); - options.put(SUB_RANGE_REPAIR_KEY, Boolean.toString(isSubrangeRepair)); options.put(TRACE_KEY, Boolean.toString(trace)); options.put(RANGES_KEY, Joiner.on(",").join(ranges)); options.put(PULL_REPAIR_KEY, Boolean.toString(pullRepair)); @@ -482,6 +511,7 @@ public class RepairOption options.put(REPAIR_PAXOS_KEY, Boolean.toString(repairPaxos)); options.put(PAXOS_ONLY_KEY, Boolean.toString(paxosOnly)); options.put(NO_TOMBSTONE_PURGING, Boolean.toString(dontPurgeTombstones)); + options.put(ACCORD_REPAIR_KEY, Boolean.toString(accordRepair)); return options; } } diff --git a/src/java/org/apache/cassandra/repair/messages/SyncResponse.java b/src/java/org/apache/cassandra/repair/messages/SyncResponse.java index e7e7985fff..0c528a3796 100644 --- a/src/java/org/apache/cassandra/repair/messages/SyncResponse.java +++ b/src/java/org/apache/cassandra/repair/messages/SyncResponse.java @@ -23,12 +23,13 @@ import java.util.List; import java.util.Objects; import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.repair.SyncNodePair; import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.repair.SyncNodePair; import org.apache.cassandra.streaming.SessionSummary; /** @@ -103,7 +104,7 @@ public class SyncResponse extends RepairMessage List summaries = new ArrayList<>(numSummaries); for (int i=0; i { return id.compareTo(o.id); } + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(TableId t, DataOutputPlus out, int version) throws IOException + { + t.serialize(out); + } + + @Override + public TableId deserialize(DataInputPlus in, int version) throws IOException + { + return TableId.deserialize(in); + } + + @Override + public long serializedSize(TableId t, int version) + { + return t.serializedSize(); + } + }; + + public static final MetadataSerializer metadataSerializer = new MetadataSerializer() + { + @Override + public void serialize(TableId t, DataOutputPlus out, Version version) throws IOException + { + t.serialize(out); + } + + @Override + public TableId deserialize(DataInputPlus in, Version version) throws IOException + { + return TableId.deserialize(in); + } + + @Override + public long serializedSize(TableId t, Version version) + { + return t.serializedSize(); + } + }; } diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 1c22d955b7..90c56e47bb 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -70,10 +70,10 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.serialization.UDTAndFunctionsAwareMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; -import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.utils.AbstractIterator; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -320,6 +320,11 @@ public class TableMetadata implements SchemaElement return unbuild().indexes(indexes).build(); } + public TableId id() + { + return id; + } + public boolean isView() { return kind == Kind.VIEW; @@ -344,7 +349,7 @@ public class TableMetadata implements SchemaElement { return false; } - + public boolean isIncrementalBackupsEnabled() { return params.incrementalBackups; diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index 1592718bf1..24b2966bdf 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -448,6 +448,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai */ public RepairSession submitRepairSession(TimeUUID parentRepairSession, CommonRange range, + boolean excludedDeadNodes, String keyspace, RepairParallelism parallelismDegree, boolean isIncremental, @@ -457,6 +458,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai boolean repairPaxos, boolean paxosOnly, boolean dontPurgeTombstones, + boolean accordRepair, ExecutorPlus executor, Scheduler validationScheduler, String... cfnames) @@ -470,9 +472,11 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai if (cfnames.length == 0) return null; - final RepairSession session = new RepairSession(ctx, validationScheduler, parentRepairSession, range, keyspace, + final RepairSession session = new RepairSession(ctx, validationScheduler, parentRepairSession, + range, excludedDeadNodes, keyspace, parallelismDegree, isIncremental, pullRepair, - previewKind, optimiseStreams, repairPaxos, paxosOnly, dontPurgeTombstones, cfnames); + previewKind, optimiseStreams, repairPaxos, paxosOnly, + dontPurgeTombstones, accordRepair, cfnames); repairs.getIfPresent(parentRepairSession).register(session.state); sessions.put(session.getId(), session); diff --git a/src/java/org/apache/cassandra/service/CASRequest.java b/src/java/org/apache/cassandra/service/CASRequest.java index f118dcf847..fb78daa2a5 100644 --- a/src/java/org/apache/cassandra/service/CASRequest.java +++ b/src/java/org/apache/cassandra/service/CASRequest.java @@ -18,15 +18,17 @@ package org.apache.cassandra.service; import accord.primitives.Txn; +import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.db.partitions.PartitionUpdate; -import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.transport.Dispatcher; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult; + /** * Abstract the conditions and updates for a CAS operation. */ @@ -51,7 +53,7 @@ public interface CASRequest */ PartitionUpdate makeUpdates(FilteredPartition current, ClientState clientState, Ballot ballot) throws InvalidRequestException; - Txn toAccordTxn(ClientState clientState, long nowInSecs); + Txn toAccordTxn(ConsistencyLevel consistencyLevel, ConsistencyLevel commitConsistencyLevel, ClientState clientState, long nowInSecs); - RowIterator toCasResult(TxnData data); + ConsensusAttemptResult toCasResult(TxnResult txnResult); } diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index e61c9187a3..a40426c001 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -39,14 +39,16 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; -import com.google.common.base.Preconditions; import com.google.common.cache.CacheLoader; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.Uninterruptibles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import accord.primitives.Keys; import accord.primitives.Txn; import org.apache.cassandra.batchlog.Batch; import org.apache.cassandra.batchlog.BatchlogManager; @@ -54,6 +56,7 @@ import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.Config.NonSerialWriteStrategy; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; @@ -90,8 +93,8 @@ import org.apache.cassandra.exceptions.QueryCancelledException; import org.apache.cassandra.exceptions.ReadAbortException; import org.apache.cassandra.exceptions.ReadFailureException; import org.apache.cassandra.exceptions.ReadTimeoutException; -import org.apache.cassandra.exceptions.RequestFailureException; import org.apache.cassandra.exceptions.RequestFailure; +import org.apache.cassandra.exceptions.RequestFailureException; import org.apache.cassandra.exceptions.RequestTimeoutException; import org.apache.cassandra.exceptions.UnavailableException; import org.apache.cassandra.exceptions.WriteFailureException; @@ -125,9 +128,18 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.txn.AccordUpdate; +import org.apache.cassandra.service.accord.txn.TxnCondition; import org.apache.cassandra.service.accord.txn.TxnData; import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; +import org.apache.cassandra.service.accord.txn.TxnReferenceOperations; +import org.apache.cassandra.service.accord.txn.TxnResult; +import org.apache.cassandra.service.accord.txn.TxnUpdate; +import org.apache.cassandra.service.accord.txn.TxnWrite; +import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.service.paxos.ContentionStrategy; @@ -137,6 +149,7 @@ import org.apache.cassandra.service.paxos.v1.PrepareCallback; import org.apache.cassandra.service.paxos.v1.ProposeCallback; import org.apache.cassandra.service.reads.AbstractReadExecutor; import org.apache.cassandra.service.reads.ReadCallback; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.service.reads.range.RangeCommands; import org.apache.cassandra.service.reads.repair.ReadRepair; import org.apache.cassandra.tcm.ClusterMetadata; @@ -155,10 +168,10 @@ import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.CountDownLatch; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Iterables.concat; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static org.apache.cassandra.config.Config.LegacyPaxosStrategy.accord; import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics; @@ -177,6 +190,11 @@ import static org.apache.cassandra.net.Verb.PAXOS_PROPOSE_REQ; import static org.apache.cassandra.net.Verb.SCHEMA_VERSION_REQ; import static org.apache.cassandra.net.Verb.TRUNCATE_REQ; import static org.apache.cassandra.service.BatchlogResponseHandler.BatchlogCleanup; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.RETRY_NEW_PROTOCOL; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.casResult; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.serialReadResult; +import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.retry_new_protocol; +import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision; import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL; import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL; import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; @@ -322,6 +340,7 @@ public class StorageProxy implements StorageProxyMBean Dispatcher.RequestTime requestTime) throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException, CasWriteUnknownResultException { + TableMetadata metadata = Schema.instance.validateTable(keyspaceName, cfName); if (DatabaseDescriptor.getPartitionDenylistEnabled() && DatabaseDescriptor.getDenylistWritesEnabled() && !partitionDenylist.isKeyPermitted(keyspaceName, cfName, key.getKey())) { denylistMetrics.incrementWritesRejected(); @@ -329,34 +348,61 @@ public class StorageProxy implements StorageProxyMBean key, keyspaceName, cfName)); } - if (DatabaseDescriptor.getLegacyPaxosStrategy() == accord) + ConsensusAttemptResult lastAttemptResult; + do { - TxnData data = AccordService.instance().coordinate(request.toAccordTxn(clientState, nowInSeconds), consistencyForPaxos); - return request.toCasResult(data); - } - else - { - return (Paxos.useV2() || keyspaceName.equals(SchemaConstants.METADATA_KEYSPACE_NAME)) - ? Paxos.cas(key, request, consistencyForPaxos, consistencyForCommit, clientState) - : legacyCas(keyspaceName, cfName, key, request, consistencyForPaxos, consistencyForCommit, clientState, nowInSeconds, requestTime); - } + ConsensusRoutingDecision decision = consensusRouting(metadata, key, consistencyForPaxos, requestTime, true); + switch (decision) + { + case paxosV2: + lastAttemptResult = Paxos.cas(key, + request, + consistencyForPaxos, + consistencyForCommit, + clientState, + requestTime); + break; + case paxosV1: + lastAttemptResult = legacyCas(metadata, + key, + request, + consistencyForPaxos, + consistencyForCommit, + clientState, + nowInSeconds, + requestTime); + break; + case accord: + Txn txn = request.toAccordTxn(consistencyForPaxos, + consistencyForCommit, + clientState, + nowInSeconds); + IAccordService accordService = AccordService.instance(); + accordService.maybeConvertKeyspacesToAccord(txn); + TxnResult txnResult = accordService.coordinate(txn, + consistencyForPaxos, + requestTime); + lastAttemptResult = request.toCasResult(txnResult); + break; + default: + throw new IllegalStateException("Unsupported consensus " + decision); + } + } while (lastAttemptResult.shouldRetryOnNewConsensusProtocol); + return lastAttemptResult.casResult; } - public static RowIterator legacyCas(String keyspaceName, - String cfName, - DecoratedKey key, - CASRequest request, - ConsistencyLevel consistencyForPaxos, - ConsistencyLevel consistencyForCommit, - ClientState clientState, - long nowInSeconds, - Dispatcher.RequestTime requestTime) + private static ConsensusAttemptResult legacyCas(TableMetadata metadata, + DecoratedKey key, + CASRequest request, + ConsistencyLevel consistencyForPaxos, + ConsistencyLevel consistencyForCommit, + ClientState clientState, + long nowInSeconds, + Dispatcher.RequestTime requestTime) throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException { try { - TableMetadata metadata = Schema.instance.validateTable(keyspaceName, cfName); - Function> updateProposer = ballot -> { // read the current values and check they validate the conditions @@ -374,7 +420,7 @@ public class StorageProxy implements StorageProxyMBean { Tracing.trace("CAS precondition does not match current values {}", current); casWriteMetrics.conditionNotMet.inc(); - return Pair.create(PartitionUpdate.emptyUpdate(metadata, key), current.rowIterator()); + return Pair.create(PartitionUpdate.emptyUpdate(metadata, key), current.rowIterator(false)); } // Create the desired updates @@ -399,15 +445,14 @@ public class StorageProxy implements StorageProxyMBean return Pair.create(updates, null); }; - return doPaxos(metadata, - key, - consistencyForPaxos, - consistencyForCommit, - consistencyForCommit, - requestTime, - casWriteMetrics, - updateProposer); - + return casResult(doPaxos(metadata, + key, + consistencyForPaxos, + consistencyForCommit, + consistencyForCommit, + requestTime, + casWriteMetrics, + updateProposer)); } catch (CasWriteUnknownResultException e) { @@ -1165,6 +1210,7 @@ public class StorageProxy implements StorageProxyMBean Collection augmented = TriggerExecutor.instance.execute(mutations); + String keyspaceName = mutations.iterator().next().getKeyspaceName(); boolean updatesView = Keyspace.open(mutations.iterator().next().getKeyspaceName()) .viewManager .updatesAffectView(mutations, true); @@ -1172,8 +1218,10 @@ public class StorageProxy implements StorageProxyMBean long size = IMutation.dataSize(mutations); writeMetrics.mutationSize.update(size); writeMetricsForLevel(consistencyLevel).mutationSize.update(size); - - if (augmented != null) + NonSerialWriteStrategy nonSerialWriteStrategy = DatabaseDescriptor.getNonSerialWriteStrategy(); + if (nonSerialWriteStrategy.writesThroughAccord && !SchemaConstants.getSystemKeyspaces().contains(keyspaceName)) + mutateWithAccord(augmented != null ? augmented : mutations, consistencyLevel, requestTime, nonSerialWriteStrategy); + else if (augmented != null) mutateAtomically(augmented, consistencyLevel, updatesView, requestTime); else { @@ -1184,6 +1232,29 @@ public class StorageProxy implements StorageProxyMBean } } + private static void mutateWithAccord(Collection iMutations, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, Config.NonSerialWriteStrategy nonSerialWriteStrategy) + { + int fragmentIndex = 0; + List fragments = new ArrayList<>(iMutations.size()); + List partitionKeys = new ArrayList<>(iMutations.size()); + for (IMutation mutation : iMutations) + { + for (PartitionUpdate update : mutation.getPartitionUpdates()) + { + PartitionKey pk = PartitionKey.of(update); + partitionKeys.add(pk); + fragments.add(new TxnWrite.Fragment(PartitionKey.of(update), fragmentIndex++, update, TxnReferenceOperations.empty())); + } + } + // Potentially ignore commit consistency level if the strategy specifies accord and not migration + ConsistencyLevel clForCommit = nonSerialWriteStrategy.commitCLForStrategy(consistencyLevel); + AccordUpdate update = new TxnUpdate(fragments, TxnCondition.none(), clForCommit); + Txn.InMemory txn = new Txn.InMemory(Keys.of(partitionKeys), TxnRead.EMPTY, TxnQuery.EMPTY, update); + IAccordService accordService = AccordService.instance(); + accordService.maybeConvertKeyspacesToAccord(txn); + accordService.coordinate(txn, consistencyLevel, requestTime); + } + /** * See mutate. Adds additional steps before and after writing a batch. * Before writing the batch (but after doing availability check against the FD for the row replicas): @@ -1602,7 +1673,7 @@ public class StorageProxy implements StorageProxyMBean if (insertLocal) { - Preconditions.checkNotNull(localReplica); + checkNotNull(localReplica); performLocally(stage, localReplica, mutation::apply, responseHandler, mutation, requestTime); } @@ -1881,43 +1952,68 @@ public class StorageProxy implements StorageProxyMBean return metadata.myNodeState() == NodeState.JOINED; } + private static ConsensusRoutingDecision consensusRouting(TableMetadata metadata, DecoratedKey partitionKey, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, boolean isForWrite) + { + if (metadata.keyspace.equals(SchemaConstants.METADATA_KEYSPACE_NAME)) + return ConsensusRoutingDecision.paxosV2; + return ConsensusRequestRouter.instance.routeAndMaybeMigrate(partitionKey, + metadata.id, + consistencyLevel, + requestTime, + DatabaseDescriptor.getCasContentionTimeout(NANOSECONDS), + isForWrite); + } + private static PartitionIterator readWithConsensus(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException { - // TCM explicitly relies on paxos and doesn't work with accord - if (DatabaseDescriptor.getLegacyPaxosStrategy() == accord && !group.metadata().keyspace.equals(SchemaConstants.METADATA_KEYSPACE_NAME)) + ConsensusAttemptResult lastResult; + do { - return readWithAccord(group, consistencyLevel); - } - else - { - return readWithPaxos(group, consistencyLevel, requestTime); - } + SinglePartitionReadCommand command = group.queries.get(0); + ConsensusRoutingDecision decision = consensusRouting(group.metadata(), command.partitionKey(), consistencyLevel, requestTime, false); + switch (decision) + { + case paxosV2: + lastResult = Paxos.read(group, consistencyLevel, requestTime); + break; + case paxosV1: + lastResult = legacyReadWithPaxos(group, consistencyLevel, requestTime); + break; + case accord: + lastResult = readWithAccord(group, consistencyLevel, requestTime); + break; + default: + throw new IllegalStateException("Unsupported consensus " + decision); + } + } while (lastResult.shouldRetryOnNewConsensusProtocol); + return lastResult.serialReadResult; } - private static PartitionIterator readWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) - throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException - { - return (Paxos.useV2() || group.metadata().keyspace.equals(SchemaConstants.METADATA_KEYSPACE_NAME)) - ? Paxos.read(group, consistencyLevel, requestTime) - : legacyReadWithPaxos(group, consistencyLevel, requestTime); - } - - private static PartitionIterator readWithAccord(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel) + private static ConsensusAttemptResult readWithAccord(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { if (group.queries.size() > 1) throw new InvalidRequestException("SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time"); - TxnRead read = TxnRead.createSerialRead(group.queries.get(0)); + SinglePartitionReadCommand readCommand = group.queries.get(0); + // If the non-SERIAL write strategy is sending all writes through Accord there is no need to use the supplied consistency + // level since Accord will manage reading safely + consistencyLevel = DatabaseDescriptor.getNonSerialWriteStrategy().readCLForStrategy(consistencyLevel); + TxnRead read = TxnRead.createSerialRead(readCommand, consistencyLevel); Txn txn = new Txn.InMemory(read.keys(), read, TxnQuery.ALL); - TxnData data = AccordService.instance().coordinate(txn, consistencyLevel); + IAccordService accordService = AccordService.instance(); + accordService.maybeConvertKeyspacesToAccord(txn); + TxnResult txnResult = accordService.coordinate(txn, consistencyLevel, requestTime); + if (txnResult.kind() == retry_new_protocol) + return RETRY_NEW_PROTOCOL; + TxnData data = (TxnData)txnResult; FilteredPartition partition = data.get(TxnRead.SERIAL_READ); if (partition != null) - return PartitionIterators.singletonIterator(partition.rowIterator()); + return serialReadResult(PartitionIterators.singletonIterator(partition.rowIterator(readCommand.isReversed()))); else - return EmptyIterators.partition(); + return serialReadResult(EmptyIterators.partition()); } - private static PartitionIterator legacyReadWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + private static ConsensusAttemptResult legacyReadWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException { long start = nanoTime(); @@ -1930,7 +2026,6 @@ public class StorageProxy implements StorageProxyMBean // calculate the blockFor before repair any paxos round to avoid RS being altered in between. int blockForRead = consistencyLevel.blockFor(Keyspace.open(metadata.keyspace).getReplicationStrategy()); - PartitionIterator result = null; try { final ConsistencyLevel consistencyForReplayCommitsOrFetch = consistencyLevel == ConsistencyLevel.LOCAL_SERIAL @@ -1967,7 +2062,7 @@ public class StorageProxy implements StorageProxyMBean throw new ReadFailureException(consistencyLevel, e.received, e.blockFor, false, e.failureReasonByEndpoint); } - result = fetchRows(group.queries, consistencyForReplayCommitsOrFetch, requestTime); + return serialReadResult(fetchRows(group.queries, consistencyForReplayCommitsOrFetch, ReadCoordinator.DEFAULT, requestTime)); } catch (UnavailableException e) { @@ -2011,18 +2106,16 @@ public class StorageProxy implements StorageProxyMBean readMetricsForLevel(consistencyLevel).addNano(latency); Keyspace.open(metadata.keyspace).getColumnFamilyStore(metadata.name).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS); } - - return result; } @SuppressWarnings("resource") - private static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + public static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ReadCoordinator coordinator, Dispatcher.RequestTime requestTime) throws UnavailableException, ReadFailureException, ReadTimeoutException { long start = nanoTime(); try { - PartitionIterator result = fetchRows(group.queries, consistencyLevel, requestTime); + PartitionIterator result = fetchRows(group.queries, consistencyLevel, coordinator, requestTime); // Note that the only difference between the command in a group must be the partition key on which // they applied. boolean enforceStrictLiveness = group.queries.get(0).metadata().enforceStrictLiveness(); @@ -2072,6 +2165,11 @@ public class StorageProxy implements StorageProxyMBean } } + public static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + { + return readRegular(group, consistencyLevel, ReadCoordinator.DEFAULT, requestTime); + } + public static void recordReadRegularAbort(ConsistencyLevel consistencyLevel, Throwable cause) { readMetrics.markAbort(cause); @@ -2119,6 +2217,7 @@ public class StorageProxy implements StorageProxyMBean */ private static PartitionIterator fetchRows(List commands, ConsistencyLevel consistencyLevel, + ReadCoordinator coordinator, Dispatcher.RequestTime requestTime) throws UnavailableException, ReadFailureException, ReadTimeoutException { @@ -2131,7 +2230,7 @@ public class StorageProxy implements StorageProxyMBean // for type of speculation we'll use in this read for (int i=0; i anyOutOfRangeOpsRecorded = keyspace -> keyspace.metric.outOfRangeTokenReads.getCount() > 0 || keyspace.metric.outOfRangeTokenWrites.getCount() > 0 @@ -1664,6 +1676,49 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } } + @Override + public void migrateConsensusProtocol(@Nonnull String targetProtocol, + @Nonnull List keyspaceNames, + @Nullable List maybeTableNames, + @Nullable String maybeRangesStr) + { + checkNotNull(targetProtocol, "targetProtocol is null"); + checkArgument(!keyspaceNames.contains(SchemaConstants.METADATA_KEYSPACE_NAME)); + startMigrationToConsensusProtocol(targetProtocol, keyspaceNames, Optional.ofNullable(maybeTableNames), Optional.ofNullable(maybeRangesStr)); + } + + @Override + public List finishConsensusMigration(@Nonnull String keyspace, + @Nullable List maybeTableNames, + @Nullable String maybeRangesStr) + { + checkArgument(!keyspace.equals(SchemaConstants.METADATA_KEYSPACE_NAME)); + return finishMigrationToConsensusProtocol(keyspace, Optional.ofNullable(maybeTableNames), Optional.ofNullable(maybeRangesStr)); + } + + @Override + public void setConsensusMigrationTargetProtocol(@Nonnull String targetProtocol, + @Nullable List keyspaceNames, + @Nullable List maybeTableNames) + { + checkNotNull(targetProtocol, "targetProtocol is null"); + checkNotNull(keyspaceNames, "keyspaceNames is null"); + checkArgument(!keyspaceNames.contains(SchemaConstants.METADATA_KEYSPACE_NAME)); + + ConsensusTableMigrationState.setConsensusMigrationTargetProtocol(targetProtocol, keyspaceNames, Optional.ofNullable(maybeTableNames)); + } + + @Override + public String listConsensusMigrations(@Nullable Set keyspaceNames, + @Nullable Set tableNames, + @Nonnull String format) + { + ClusterMetadata cm = ClusterMetadata.current(); + ConsensusMigrationState snapshot = cm.consensusMigrationState; + Map snapshotAsMap = snapshot.toMap(keyspaceNames, tableNames); + return pojoMapToString(snapshotAsMap, format); + } + public Map> getConcurrency(List stageNames) { Stream stageStream = stageNames.isEmpty() ? stream(Stage.values()) : stageNames.stream().map(Stage::fromPoolName); @@ -3978,11 +4033,21 @@ public class StorageService extends NotificationBroadcasterSupport implements IE // Never ever do this at home. Used by tests. @VisibleForTesting - public IPartitioner setPartitionerUnsafe(IPartitioner newPartitioner) + public void setPartitionerUnsafe(IPartitioner newPartitioner) { - IPartitioner oldPartitioner = DatabaseDescriptor.setPartitionerUnsafe(newPartitioner); + checkNotNull(newPartitioner, "newPartitioner is null"); + checkState(originalPartitioner == null, "Already changed the partitioner without resetting"); + originalPartitioner = DatabaseDescriptor.setPartitionerUnsafe(newPartitioner); valueFactory = new VersionedValue.VersionedValueFactory(newPartitioner); - return oldPartitioner; + } + + @VisibleForTesting + public void resetPartitionerUnsafe() + { + checkState(originalPartitioner != null, "Original partitioner was never changed"); + DatabaseDescriptor.setPartitionerUnsafe(originalPartitioner); + valueFactory = new VersionedValueFactory(originalPartitioner); + originalPartitioner = null; } public void truncate(String keyspace, String table) throws TimeoutException, IOException @@ -4165,6 +4230,16 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return Lists.newArrayList(Schema.instance.distributedKeyspaces().names()); } + @Override + public List getAccordManagedKeyspaces() + { + // TODO (review) These are really just the ones Accord is aware of not necessarily managed + Set keyspaces = Schema.instance.getNonLocalStrategyKeyspaces().names(); + return keyspaces.stream() + .filter(AccordService.instance()::isAccordManagedKeyspace) + .collect(toList()); + } + public Map getViewBuildStatuses(String keyspace, String view, boolean withPort) { Map coreViewStatus = SystemDistributedKeyspace.viewStatus(keyspace, view); @@ -4994,7 +5069,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE archiveCommand = archiveCommand != null ? archiveCommand : fqlOptions.archive_command; maxArchiveRetries = maxArchiveRetries != Integer.MIN_VALUE ? maxArchiveRetries : fqlOptions.max_archive_retries; - Preconditions.checkNotNull(path, "cassandra.yaml did not set log_dir and not set as parameter"); + checkNotNull(path, "cassandra.yaml did not set log_dir and not set as parameter"); FullQueryLogger.instance.enableWithoutClean(File.getPath(path), rollCycle, blocking, maxQueueWeight, maxLogSize, archiveCommand, maxArchiveRetries); } @@ -5359,7 +5434,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public void setRepairRpcTimeout(Long timeoutInMillis) { - Preconditions.checkState(timeoutInMillis > 0); + checkState(timeoutInMillis > 0); DatabaseDescriptor.setRepairRpcTimeout(timeoutInMillis); logger.info("RepairRpcTimeout set to {}ms via JMX", timeoutInMillis); } diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 6e8a2ad449..d11ca1bfb4 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.management.NotificationEmitter; import javax.management.openmbean.CompositeData; @@ -1141,6 +1142,23 @@ public interface StorageServiceMBean extends NotificationEmitter public String getBootstrapState(); void abortBootstrap(String nodeId, String endpoint); + void migrateConsensusProtocol(@Nonnull String targetProtocol, + @Nullable List keyspaceNames, + @Nullable List maybeTableNames, + @Nullable String maybeRangesStr); + + List finishConsensusMigration(@Nonnull String keyspace, + @Nullable List maybeTableNames, + @Nullable String maybeRangesStr); + + void setConsensusMigrationTargetProtocol(@Nonnull String targetProtocol, + @Nullable List keyspaceNames, + @Nullable List maybeTableNames); + + String listConsensusMigrations(@Nullable Set keyspaceNames, @Nullable Set tableNames, @Nonnull String format); + + List getAccordManagedKeyspaces(); + /** Gets the concurrency settings for processing stages*/ static class StageConcurrency implements Serializable { @@ -1188,6 +1206,7 @@ public interface StorageServiceMBean extends NotificationEmitter /** * Start the fully query logger. + * * @param path Path where the full query log will be stored. If null cassandra.yaml value is used. * @param rollCycle How often to create a new file for query data (MINUTELY, DAILY, HOURLY) * @param blocking Whether threads submitting queries to the query log should block if they can't be drained to the filesystem or alternatively drops samples and log diff --git a/src/java/org/apache/cassandra/service/accord/AccordCachingState.java b/src/java/org/apache/cassandra/service/accord/AccordCachingState.java index 994e551f79..d7bce189d2 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCachingState.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCachingState.java @@ -17,8 +17,6 @@ */ package org.apache.cassandra.service.accord; -import java.util.Collections; -import java.util.Set; import java.util.concurrent.Callable; import java.util.function.BiFunction; import java.util.function.Function; @@ -27,7 +25,7 @@ import java.util.function.ToLongFunction; import com.google.common.primitives.Ints; import accord.local.Command.TransientListener; -import accord.utils.DeterministicIdentitySet; +import accord.local.Listeners; import accord.utils.IntrusiveLinkedListNode; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncResults.RunnableResult; @@ -35,14 +33,14 @@ import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.utils.ObjectSizes; import static java.lang.String.format; -import static org.apache.cassandra.service.accord.AccordCachingState.Status.UNINITIALIZED; -import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADING; -import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADED; +import static org.apache.cassandra.service.accord.AccordCachingState.Status.EVICTED; import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_SAVE; +import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADED; +import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADING; import static org.apache.cassandra.service.accord.AccordCachingState.Status.MODIFIED; import static org.apache.cassandra.service.accord.AccordCachingState.Status.SAVING; -import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_SAVE; -import static org.apache.cassandra.service.accord.AccordCachingState.Status.EVICTED; +import static org.apache.cassandra.service.accord.AccordCachingState.Status.UNINITIALIZED; /** * Global (per CommandStore) state of a cached entity (Command or CommandsForKey). @@ -61,7 +59,7 @@ public class AccordCachingState extends IntrusiveLinkedListNode /** * Transient listeners aren't meant to survive process restart, but must survive cache eviction. */ - private Set transientListeners; + private Listeners transientListeners; public AccordCachingState(K key) { @@ -140,7 +138,7 @@ public class AccordCachingState extends IntrusiveLinkedListNode public void addListener(TransientListener listener) { if (transientListeners == null) - transientListeners = new DeterministicIdentitySet<>(); + transientListeners = new Listeners<>(); transientListeners.add(listener); } @@ -149,14 +147,14 @@ public class AccordCachingState extends IntrusiveLinkedListNode return transientListeners != null && transientListeners.remove(listener); } - public void listeners(Set listeners) + public void listeners(Listeners listeners) { transientListeners = listeners; } - public Set listeners() + public Listeners listeners() { - return transientListeners == null ? Collections.emptySet() : transientListeners; + return transientListeners == null ? Listeners.EMPTY : transientListeners; } public boolean hasListeners() diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index c9a3224b9a..696cfe227c 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -30,6 +30,8 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Predicate; + import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; @@ -450,7 +452,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize current = null; } - O mapReduceForRange(Routables keysOrRanges, Ranges slice, BiFunction map, O accumulate, O terminalValue) + O mapReduceForRange(Routables keysOrRanges, Ranges slice, BiFunction map, O accumulate, Predicate terminate) { keysOrRanges = keysOrRanges.slice(slice, Routables.Slice.Minimal); switch (keysOrRanges.domain()) @@ -461,7 +463,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize for (CommandTimeseriesHolder summary : commandsForRanges.search(keys)) { accumulate = map.apply(summary, accumulate); - if (accumulate.equals(terminalValue)) + if (terminate.test(accumulate)) return accumulate; } } @@ -475,7 +477,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize if (summary == null) continue; accumulate = map.apply(summary, accumulate); - if (accumulate.equals(terminalValue)) + if (terminate.test(accumulate)) return accumulate; } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index 4e841a17ad..8b6162f227 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -73,7 +73,7 @@ import static org.apache.cassandra.utils.CollectionSerializers.serializedMapSize public class AccordFetchCoordinator extends AbstractFetchCoordinator implements StreamManager.StreamListener { - private static final Query noopQuery = (txnId, executeAt, data, read, update) -> null; + private static final Query noopQuery = (txnId, executeAt, keys, data, read, update) -> null; public static class StreamData implements Data { @@ -145,7 +145,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements } @Override - public Data merge(Data data) + public StreamData merge(Data data) { StreamData that = (StreamData) data; if (that.streams.keySet().stream().anyMatch(this.streams::containsKey)) @@ -258,7 +258,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements } @Override - public AsyncChain read(Seekable key, Txn.Kind kind, SafeCommandStore commandStore, Timestamp executeAt, DataStore store) + public AsyncChain read(Seekable key, SafeCommandStore commandStore, Timestamp executeAt, DataStore store) { try { diff --git a/src/java/org/apache/cassandra/service/accord/AccordJournal.java b/src/java/org/apache/cassandra/service/accord/AccordJournal.java index a323edaf17..e3e1dd03f9 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordJournal.java +++ b/src/java/org/apache/cassandra/service/accord/AccordJournal.java @@ -20,8 +20,8 @@ package org.apache.cassandra.service.accord; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; -import java.util.EnumMap; -import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Executor; @@ -30,8 +30,12 @@ import java.util.function.Predicate; import java.util.zip.Checksum; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimap; import com.google.common.primitives.Ints; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +69,8 @@ import org.apache.cassandra.journal.KeySupport; import org.apache.cassandra.journal.Params; import org.apache.cassandra.journal.ValueSerializer; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.accord.interop.AccordInteropApply; +import org.apache.cassandra.service.accord.interop.AccordInteropCommit; import org.apache.cassandra.service.accord.serializers.AcceptSerializers; import org.apache.cassandra.service.accord.serializers.ApplySerializers; import org.apache.cassandra.service.accord.serializers.BeginInvalidationSerializers; @@ -78,10 +84,31 @@ import org.apache.cassandra.service.accord.serializers.RecoverySerializers; import org.apache.cassandra.service.accord.serializers.SetDurableSerializers; import org.apache.cassandra.utils.ByteArrayUtil; -import static accord.messages.MessageType.*; +import static accord.messages.MessageType.ACCEPT_INVALIDATE_REQ; +import static accord.messages.MessageType.ACCEPT_REQ; +import static accord.messages.MessageType.APPLY_MAXIMAL_REQ; +import static accord.messages.MessageType.APPLY_MINIMAL_REQ; +import static accord.messages.MessageType.BEGIN_INVALIDATE_REQ; +import static accord.messages.MessageType.BEGIN_RECOVER_REQ; +import static accord.messages.MessageType.COMMIT_INVALIDATE_REQ; +import static accord.messages.MessageType.COMMIT_MAXIMAL_REQ; +import static accord.messages.MessageType.COMMIT_MINIMAL_REQ; +import static accord.messages.MessageType.INFORM_DURABLE_REQ; +import static accord.messages.MessageType.INFORM_OF_TXN_REQ; +import static accord.messages.MessageType.PRE_ACCEPT_REQ; +import static accord.messages.MessageType.PROPAGATE_APPLY_MSG; +import static accord.messages.MessageType.PROPAGATE_COMMIT_MSG; +import static accord.messages.MessageType.PROPAGATE_OTHER_MSG; +import static accord.messages.MessageType.PROPAGATE_PRE_ACCEPT_MSG; +import static accord.messages.MessageType.SET_GLOBALLY_DURABLE_REQ; +import static accord.messages.MessageType.SET_SHARD_DURABLE_REQ; import static org.apache.cassandra.db.TypeSizes.BYTE_SIZE; import static org.apache.cassandra.db.TypeSizes.INT_SIZE; import static org.apache.cassandra.db.TypeSizes.LONG_SIZE; +import static org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_APPLY_MAXIMAL_REQ; +import static org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_APPLY_MINIMAL_REQ; +import static org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_COMMIT_MAXIMAL_REQ; +import static org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_COMMIT_MINIMAL_REQ; public class AccordJournal implements Shutdownable { @@ -487,45 +514,68 @@ public class AccordJournal implements Shutdownable REPLAY (0, ReplayRecord.SERIALIZER), /* Accord protocol requests */ - PRE_ACCEPT (64, PRE_ACCEPT_REQ, PreacceptSerializers.request, TXN ), - ACCEPT (65, ACCEPT_REQ, AcceptSerializers.request, TXN ), - ACCEPT_INVALIDATE (66, ACCEPT_INVALIDATE_REQ, AcceptSerializers.invalidate, EPOCH), - COMMIT_MINIMAL (67, COMMIT_MINIMAL_REQ, CommitSerializers.request, TXN ), - COMMIT_MAXIMAL (68, COMMIT_MAXIMAL_REQ, CommitSerializers.request, TXN ), - COMMIT_INVALIDATE (69, COMMIT_INVALIDATE_REQ, CommitSerializers.invalidate, INVL), - APPLY_MINIMAL (70, APPLY_MINIMAL_REQ, ApplySerializers.request, TXN ), - APPLY_MAXIMAL (71, APPLY_MAXIMAL_REQ, ApplySerializers.request, TXN ), - BEGIN_RECOVER (72, BEGIN_RECOVER_REQ, RecoverySerializers.request, TXN ), - BEGIN_INVALIDATE (73, BEGIN_INVALIDATE_REQ, BeginInvalidationSerializers.request, EPOCH), - INFORM_OF_TXN (74, INFORM_OF_TXN_REQ, InformOfTxnIdSerializers.request, EPOCH), - INFORM_DURABLE (75, INFORM_DURABLE_REQ, InformDurableSerializers.request, TXN ), - SET_SHARD_DURABLE (76, SET_SHARD_DURABLE_REQ, SetDurableSerializers.shardDurable, EPOCH), - SET_GLOBALLY_DURABLE (77, SET_GLOBALLY_DURABLE_REQ, SetDurableSerializers.globallyDurable, EPOCH), + PRE_ACCEPT (64, PRE_ACCEPT_REQ, PreacceptSerializers.request, TXN ), + ACCEPT (65, ACCEPT_REQ, AcceptSerializers.request, TXN ), + ACCEPT_INVALIDATE (66, ACCEPT_INVALIDATE_REQ, AcceptSerializers.invalidate, EPOCH), + COMMIT_MINIMAL (67, COMMIT_MINIMAL_REQ, CommitSerializers.request, TXN ), + COMMIT_MAXIMAL (68, COMMIT_MAXIMAL_REQ, CommitSerializers.request, TXN ), + COMMIT_INVALIDATE (69, COMMIT_INVALIDATE_REQ, CommitSerializers.invalidate, INVL ), + APPLY_MINIMAL (70, APPLY_MINIMAL_REQ, ApplySerializers.request, TXN ), + APPLY_MAXIMAL (71, APPLY_MAXIMAL_REQ, ApplySerializers.request, TXN ), + + INTEROP_COMMIT_MINIMAL (90, INTEROP_COMMIT_MINIMAL_REQ, COMMIT_MINIMAL_REQ, AccordInteropCommit.serializer, TXN), + INTEROP_COMMIT_MAXIMAL (91, INTEROP_COMMIT_MAXIMAL_REQ, COMMIT_MINIMAL_REQ, AccordInteropCommit.serializer, TXN), + INTEROP_APPLY_MINIMAL (92, INTEROP_APPLY_MINIMAL_REQ, COMMIT_MINIMAL_REQ, AccordInteropApply.serializer, TXN), + INTEROP_APPLY_MAXIMAL (93, INTEROP_APPLY_MAXIMAL_REQ, COMMIT_MINIMAL_REQ, AccordInteropApply.serializer, TXN), + + BEGIN_RECOVER (72, BEGIN_RECOVER_REQ, RecoverySerializers.request, TXN ), + BEGIN_INVALIDATE (73, BEGIN_INVALIDATE_REQ, BeginInvalidationSerializers.request, EPOCH), + INFORM_OF_TXN (74, INFORM_OF_TXN_REQ, InformOfTxnIdSerializers.request, EPOCH), + INFORM_DURABLE (75, INFORM_DURABLE_REQ, InformDurableSerializers.request, TXN ), + SET_SHARD_DURABLE (76, SET_SHARD_DURABLE_REQ, SetDurableSerializers.shardDurable, EPOCH), + SET_GLOBALLY_DURABLE (77, SET_GLOBALLY_DURABLE_REQ, SetDurableSerializers.globallyDurable, EPOCH), /* Accord local messages */ - PROPAGATE_PRE_ACCEPT (78, PROPAGATE_PRE_ACCEPT_MSG, FetchSerializers.propagate, LOCAL), - PROPAGATE_COMMIT (79, PROPAGATE_COMMIT_MSG, FetchSerializers.propagate, LOCAL), - PROPAGATE_APPLY (80, PROPAGATE_APPLY_MSG, FetchSerializers.propagate, LOCAL), - PROPAGATE_OTHER (81, PROPAGATE_OTHER_MSG, FetchSerializers.propagate, LOCAL), + PROPAGATE_PRE_ACCEPT (78, PROPAGATE_PRE_ACCEPT_MSG, FetchSerializers.propagate, LOCAL), + PROPAGATE_COMMIT (79, PROPAGATE_COMMIT_MSG, FetchSerializers.propagate, LOCAL), + PROPAGATE_APPLY (80, PROPAGATE_APPLY_MSG, FetchSerializers.propagate, LOCAL), + PROPAGATE_OTHER (81, PROPAGATE_OTHER_MSG, FetchSerializers.propagate, LOCAL), ; final int id; - final MessageType type; + /** + * An incoming message of a given type from Accord's perspective might have multiple + * concrete implementations some of which are supplied by the Cassandra integration. + * The incoming type specifies the handling for writing out a message to the journal. + */ + final MessageType incomingType; + /** + * The outgoing type is the type that will be returned to Accord and it must be a subclass of the incoming type. + * + * This type will always be from accord.messages.MessageType and never from the extended types in the integration. + */ + final MessageType outgoingType; final TxnIdProvider txnIdProvider; final ValueSerializer serializer; Type(int id, ValueSerializer serializer) { - this(id, null, serializer, null); + this(id, null, null, serializer, null); + } + + Type(int id, MessageType incomingType, MessageType outgoingType, IVersionedSerializer serializer, TxnIdProvider txnIdProvider) + { + //noinspection unchecked + this(id, incomingType, outgoingType, MessageSerializer.wrap((IVersionedSerializer) serializer), txnIdProvider); } Type(int id, MessageType type, IVersionedSerializer serializer, TxnIdProvider txnIdProvider) { //noinspection unchecked - this(id, type, MessageSerializer.wrap((IVersionedSerializer) serializer), txnIdProvider); + this(id, type, type, MessageSerializer.wrap((IVersionedSerializer) serializer), txnIdProvider); } - Type(int id, MessageType type, ValueSerializer serializer, TxnIdProvider txnIdProvider) + Type(int id, MessageType incomingType, MessageType outgoingType, ValueSerializer serializer, TxnIdProvider txnIdProvider) { if (id < 0) throw new IllegalArgumentException("Negative Type id " + id); @@ -533,7 +583,8 @@ public class AccordJournal implements Shutdownable throw new IllegalArgumentException("Type id doesn't fit in a single byte: " + id); this.id = id; - this.type = type; + this.incomingType = incomingType; + this.outgoingType = outgoingType; //noinspection unchecked this.serializer = (ValueSerializer) serializer; this.txnIdProvider = txnIdProvider; @@ -542,6 +593,8 @@ public class AccordJournal implements Shutdownable private static final Type[] idToTypeMapping; private static final Map msgTypeToTypeMap; + private static final ListMultimap msgTypeToSynonymousTypesMap; + static { Type[] types = values(); @@ -559,13 +612,26 @@ public class AccordJournal implements Shutdownable } idToTypeMapping = idToType; - EnumMap msgTypeToType = new EnumMap<>(MessageType.class); + Map msgTypeToType = new HashMap<>(); for (Type type : types) { - if (null != type.type && null != msgTypeToType.put(type.type, type)) - throw new IllegalStateException("Duplicate MessageType " + type.type); + if (null != type.incomingType && null != msgTypeToType.put(type.incomingType, type)) + throw new IllegalStateException("Duplicate MessageType " + type.incomingType); } - msgTypeToTypeMap = msgTypeToType; + msgTypeToTypeMap = ImmutableMap.copyOf(msgTypeToType); + + Multimap msgTypeToSynonymousTypes = ArrayListMultimap.create(); + for (Type type : types) + { + if (null != type.outgoingType) + { + Type incomingType = msgTypeToTypeMap.get(type.incomingType); + if (msgTypeToSynonymousTypes.get(type.outgoingType).contains(incomingType)) + throw new IllegalStateException("Duplicate synonymous Type " + type.incomingType); + msgTypeToSynonymousTypes.put(type.outgoingType, incomingType); + } + } + msgTypeToSynonymousTypesMap = ImmutableListMultimap.copyOf(msgTypeToSynonymousTypes); } static Type fromId(int id) @@ -578,6 +644,14 @@ public class AccordJournal implements Shutdownable return type; } + static List synonymousTypesFromMessageType(MessageType msgType) + { + List synonymousTypes = msgTypeToSynonymousTypesMap.get(msgType); + if (null == synonymousTypes) + throw new IllegalArgumentException("Unsupported MessageType " + msgType); + return synonymousTypes; + } + static Type fromMessageType(MessageType msgType) { Type type = msgTypeToTypeMap.get(msgType); @@ -613,7 +687,7 @@ public class AccordJournal implements Shutdownable static { // make noise early if we forget to update our version mappings - Invariants.checkState(MessagingService.current_version == MessagingService.VERSION_50, "Expected current version to be %d but given %d", MessagingService.VERSION_50, MessagingService.current_version); + Invariants.checkState(MessagingService.current_version == MessagingService.VERSION_51, "Expected current version to be %d but given %d", MessagingService.VERSION_51, MessagingService.current_version); } private static int msVersion(int version) @@ -621,7 +695,7 @@ public class AccordJournal implements Shutdownable switch (version) { default: throw new IllegalArgumentException(); - case 1: return MessagingService.VERSION_50; + case 1: return MessagingService.VERSION_51; } } @@ -693,11 +767,12 @@ public class AccordJournal implements Shutdownable { Set keys = new ObjectHashSet<>(messages.size() + 1, 0.9f); for (MessageType message : messages) - keys.add(new Key(txnId, Type.fromMessageType(message))); + for (Type synonymousType : Type.synonymousTypesFromMessageType(message)) + keys.add(new Key(txnId, synonymousType)); Set presentKeys = journal.test(keys); - EnumSet presentMessages = EnumSet.noneOf(MessageType.class); + Set presentMessages = new ObjectHashSet<>(presentKeys.size() + 1, 0.9f); for (Key key : presentKeys) - presentMessages.add(key.type.type); + presentMessages.add(key.type.outgoingType); return presentMessages; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java index 798b529dad..fc63c62049 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java +++ b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java @@ -150,8 +150,8 @@ import org.apache.cassandra.service.accord.serializers.TopologySerializers; import org.apache.cassandra.service.accord.serializers.WaitingOnSerializer; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.Clock; -import org.apache.cassandra.utils.btree.BTree; import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.btree.BTree; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import static accord.utils.Invariants.checkArgument; @@ -1753,7 +1753,7 @@ public class AccordKeyspace break; } - return Iterables.getOnlyElement(statement.getMutations(clientState, options, true, tsMicros, (int) TimeUnit.MICROSECONDS.toSeconds(tsMicros), Dispatcher.RequestTime.forImmediateExecution())); + return Iterables.getOnlyElement(statement.getMutations(clientState, options, true, tsMicros, (int) TimeUnit.MICROSECONDS.toSeconds(tsMicros), Dispatcher.RequestTime.forImmediateExecution(), false)); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java index c51f202d54..6c9622cffd 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java +++ b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java @@ -18,14 +18,18 @@ package org.apache.cassandra.service.accord; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.util.Collections; -import java.util.EnumMap; import java.util.EnumSet; +import java.util.List; import java.util.Map; import java.util.Set; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,61 +46,122 @@ import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessageDelivery; +import org.apache.cassandra.net.MessageFlag; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.Verb; +import static accord.messages.MessageType.Kind.REMOTE; + public class AccordMessageSink implements MessageSink { private static final Logger logger = LoggerFactory.getLogger(AccordMessageSink.class); + public static final class AccordMessageType extends MessageType + { + public static final MessageType INTEROP_READ_REQ = amt(REMOTE, false); + public static final MessageType INTEROP_READ_RSP = amt(REMOTE, false); + public static final MessageType INTEROP_READ_REPAIR_REQ = amt(REMOTE, false); + public static final MessageType INTEROP_READ_REPAIR_RSP = amt(REMOTE, false); + public static final MessageType INTEROP_COMMIT_MINIMAL_REQ = amt(REMOTE, true ); + public static final MessageType INTEROP_COMMIT_MAXIMAL_REQ = amt(REMOTE, true ); + public static final MessageType INTEROP_APPLY_MINIMAL_REQ = amt(REMOTE, true ); + public static final MessageType INTEROP_APPLY_MAXIMAL_REQ = amt(REMOTE, true ); + + + public static final List values; + + static + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (Field f : AccordMessageType.class.getDeclaredFields()) + { + if (f.getType().equals(AccordMessageType.class) && Modifier.isStatic(f.getModifiers())) + { + try + { + builder.add((MessageType) f.get(null)); + } + catch (IllegalAccessException e) + { + throw new RuntimeException(e); + } + } + } + values = builder.build(); + } + + private static MessageType amt(MessageType.Kind kind, boolean hasSideEffects) + { + return new AccordMessageType(kind, hasSideEffects); + } + + private AccordMessageType(MessageType.Kind kind, boolean hasSideEffects) + { + super(kind, hasSideEffects); + } + } + private static class VerbMapping { private static final VerbMapping instance = new VerbMapping(); - private final Map mapping = new EnumMap<>(MessageType.class); + private final Map mapping; private final Map> overrideReplyVerbs = ImmutableMap.>builder() - // read takes Result | Nack - .put(Verb.ACCORD_FETCH_DATA_REQ, EnumSet.of(Verb.ACCORD_FETCH_DATA_RSP, Verb.ACCORD_READ_RSP /* nack */)) - .build(); + // read takes Result | Nack + .put(Verb.ACCORD_FETCH_DATA_REQ, EnumSet.of(Verb.ACCORD_FETCH_DATA_RSP, Verb.ACCORD_READ_RSP /* nack */)) + .put(Verb.ACCORD_INTEROP_COMMIT_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_RSP, Verb.ACCORD_READ_RSP)) + .put(Verb.ACCORD_INTEROP_READ_REPAIR_REQ, EnumSet.of(Verb.ACCORD_INTEROP_READ_REPAIR_RSP, Verb.ACCORD_READ_RSP)) + .build(); private VerbMapping() { - mapping.put(MessageType.PRE_ACCEPT_REQ, Verb.ACCORD_PRE_ACCEPT_REQ); - mapping.put(MessageType.PRE_ACCEPT_RSP, Verb.ACCORD_PRE_ACCEPT_RSP); - mapping.put(MessageType.ACCEPT_REQ, Verb.ACCORD_ACCEPT_REQ); - mapping.put(MessageType.ACCEPT_RSP, Verb.ACCORD_ACCEPT_RSP); - mapping.put(MessageType.ACCEPT_INVALIDATE_REQ, Verb.ACCORD_ACCEPT_INVALIDATE_REQ); - mapping.put(MessageType.COMMIT_MINIMAL_REQ, Verb.ACCORD_COMMIT_REQ); - mapping.put(MessageType.COMMIT_MAXIMAL_REQ, Verb.ACCORD_COMMIT_REQ); - mapping.put(MessageType.COMMIT_INVALIDATE_REQ, Verb.ACCORD_COMMIT_INVALIDATE_REQ); - mapping.put(MessageType.APPLY_MINIMAL_REQ, Verb.ACCORD_APPLY_REQ); - mapping.put(MessageType.APPLY_MAXIMAL_REQ, Verb.ACCORD_APPLY_REQ); - mapping.put(MessageType.APPLY_RSP, Verb.ACCORD_APPLY_RSP); - mapping.put(MessageType.READ_REQ, Verb.ACCORD_READ_REQ); - mapping.put(MessageType.READ_RSP, Verb.ACCORD_READ_RSP); - mapping.put(MessageType.BEGIN_RECOVER_REQ, Verb.ACCORD_BEGIN_RECOVER_REQ); - mapping.put(MessageType.BEGIN_RECOVER_RSP, Verb.ACCORD_BEGIN_RECOVER_RSP); - mapping.put(MessageType.BEGIN_INVALIDATE_REQ, Verb.ACCORD_BEGIN_INVALIDATE_REQ); - mapping.put(MessageType.BEGIN_INVALIDATE_RSP, Verb.ACCORD_BEGIN_INVALIDATE_RSP); - mapping.put(MessageType.WAIT_ON_COMMIT_REQ, Verb.ACCORD_WAIT_ON_COMMIT_REQ); - mapping.put(MessageType.WAIT_ON_COMMIT_RSP, Verb.ACCORD_WAIT_ON_COMMIT_RSP); - mapping.put(MessageType.WAIT_ON_APPLY_REQ, Verb.ACCORD_WAIT_ON_APPLY_REQ); - mapping.put(MessageType.INFORM_OF_TXN_REQ, Verb.ACCORD_INFORM_OF_TXN_REQ); - mapping.put(MessageType.INFORM_DURABLE_REQ, Verb.ACCORD_INFORM_DURABLE_REQ); - mapping.put(MessageType.INFORM_HOME_DURABLE_REQ, Verb.ACCORD_INFORM_HOME_DURABLE_REQ); - mapping.put(MessageType.CHECK_STATUS_REQ, Verb.ACCORD_CHECK_STATUS_REQ); - mapping.put(MessageType.CHECK_STATUS_RSP, Verb.ACCORD_CHECK_STATUS_RSP); - mapping.put(MessageType.GET_DEPS_REQ, Verb.ACCORD_GET_DEPS_REQ); - mapping.put(MessageType.GET_DEPS_RSP, Verb.ACCORD_GET_DEPS_RSP); - mapping.put(MessageType.SIMPLE_RSP, Verb.ACCORD_SIMPLE_RSP); - mapping.put(MessageType.FETCH_DATA_REQ, Verb.ACCORD_FETCH_DATA_REQ); - mapping.put(MessageType.FETCH_DATA_RSP, Verb.ACCORD_FETCH_DATA_RSP); - mapping.put(MessageType.SET_SHARD_DURABLE_REQ, Verb.ACCORD_SET_SHARD_DURABLE_REQ); - mapping.put(MessageType.SET_GLOBALLY_DURABLE_REQ, Verb.ACCORD_SET_GLOBALLY_DURABLE_REQ); - mapping.put(MessageType.QUERY_DURABLE_BEFORE_REQ, Verb.ACCORD_QUERY_DURABLE_BEFORE_REQ); - mapping.put(MessageType.QUERY_DURABLE_BEFORE_RSP, Verb.ACCORD_QUERY_DURABLE_BEFORE_RSP); + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.put(MessageType.SIMPLE_RSP, Verb.ACCORD_SIMPLE_RSP); + builder.put(MessageType.PRE_ACCEPT_REQ, Verb.ACCORD_PRE_ACCEPT_REQ); + builder.put(MessageType.PRE_ACCEPT_RSP, Verb.ACCORD_PRE_ACCEPT_RSP); + builder.put(MessageType.ACCEPT_REQ, Verb.ACCORD_ACCEPT_REQ); + builder.put(MessageType.ACCEPT_RSP, Verb.ACCORD_ACCEPT_RSP); + builder.put(MessageType.ACCEPT_INVALIDATE_REQ, Verb.ACCORD_ACCEPT_INVALIDATE_REQ); + builder.put(MessageType.GET_DEPS_REQ, Verb.ACCORD_GET_DEPS_REQ); + builder.put(MessageType.GET_DEPS_RSP, Verb.ACCORD_GET_DEPS_RSP); + builder.put(MessageType.COMMIT_MINIMAL_REQ, Verb.ACCORD_COMMIT_REQ); + builder.put(MessageType.COMMIT_MAXIMAL_REQ, Verb.ACCORD_COMMIT_REQ); + builder.put(MessageType.COMMIT_INVALIDATE_REQ, Verb.ACCORD_COMMIT_INVALIDATE_REQ); + builder.put(MessageType.APPLY_MINIMAL_REQ, Verb.ACCORD_APPLY_REQ); + builder.put(MessageType.APPLY_MAXIMAL_REQ, Verb.ACCORD_APPLY_REQ); + builder.put(MessageType.APPLY_RSP, Verb.ACCORD_APPLY_RSP); + builder.put(MessageType.READ_REQ, Verb.ACCORD_READ_REQ); + builder.put(MessageType.READ_RSP, Verb.ACCORD_READ_RSP); + builder.put(MessageType.BEGIN_RECOVER_REQ, Verb.ACCORD_BEGIN_RECOVER_REQ); + builder.put(MessageType.BEGIN_RECOVER_RSP, Verb.ACCORD_BEGIN_RECOVER_RSP); + builder.put(MessageType.BEGIN_INVALIDATE_REQ, Verb.ACCORD_BEGIN_INVALIDATE_REQ); + builder.put(MessageType.BEGIN_INVALIDATE_RSP, Verb.ACCORD_BEGIN_INVALIDATE_RSP); + builder.put(MessageType.WAIT_ON_COMMIT_REQ, Verb.ACCORD_WAIT_ON_COMMIT_REQ); + builder.put(MessageType.WAIT_ON_COMMIT_RSP, Verb.ACCORD_WAIT_ON_COMMIT_RSP); + builder.put(MessageType.WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_WAIT_UNTIL_APPLIED_REQ); + builder.put(MessageType.APPLY_AND_WAIT_UNTIL_APPLIED_REQ, Verb.ACCORD_APPLY_AND_WAIT_UNTIL_APPLIED_REQ); + builder.put(MessageType.INFORM_OF_TXN_REQ, Verb.ACCORD_INFORM_OF_TXN_REQ); + builder.put(MessageType.INFORM_DURABLE_REQ, Verb.ACCORD_INFORM_DURABLE_REQ); + builder.put(MessageType.INFORM_HOME_DURABLE_REQ, Verb.ACCORD_INFORM_HOME_DURABLE_REQ); + builder.put(MessageType.CHECK_STATUS_REQ, Verb.ACCORD_CHECK_STATUS_REQ); + builder.put(MessageType.CHECK_STATUS_RSP, Verb.ACCORD_CHECK_STATUS_RSP); + builder.put(MessageType.FETCH_DATA_REQ, Verb.ACCORD_FETCH_DATA_REQ); + builder.put(MessageType.FETCH_DATA_RSP, Verb.ACCORD_FETCH_DATA_RSP); + builder.put(MessageType.SET_SHARD_DURABLE_REQ, Verb.ACCORD_SET_SHARD_DURABLE_REQ); + builder.put(MessageType.SET_GLOBALLY_DURABLE_REQ, Verb.ACCORD_SET_GLOBALLY_DURABLE_REQ); + builder.put(MessageType.QUERY_DURABLE_BEFORE_REQ, Verb.ACCORD_QUERY_DURABLE_BEFORE_REQ); + builder.put(MessageType.QUERY_DURABLE_BEFORE_RSP, Verb.ACCORD_QUERY_DURABLE_BEFORE_RSP); + builder.put(AccordMessageType.INTEROP_READ_REQ, Verb.ACCORD_INTEROP_READ_REQ); + builder.put(AccordMessageType.INTEROP_READ_RSP, Verb.ACCORD_INTEROP_READ_RSP); + builder.put(AccordMessageType.INTEROP_READ_REPAIR_REQ, Verb.ACCORD_INTEROP_READ_REPAIR_REQ); + builder.put(AccordMessageType.INTEROP_READ_REPAIR_RSP, Verb.ACCORD_INTEROP_READ_REPAIR_RSP); + builder.put(AccordMessageType.INTEROP_COMMIT_MINIMAL_REQ, Verb.ACCORD_INTEROP_COMMIT_REQ); + builder.put(AccordMessageType.INTEROP_COMMIT_MAXIMAL_REQ, Verb.ACCORD_INTEROP_COMMIT_REQ); + builder.put(AccordMessageType.INTEROP_APPLY_MINIMAL_REQ, Verb.ACCORD_APPLY_REQ); + builder.put(AccordMessageType.INTEROP_APPLY_MAXIMAL_REQ, Verb.ACCORD_APPLY_REQ); + mapping = builder.build(); - for (MessageType type : MessageType.values()) + for (MessageType type : Iterables.concat(AccordMessageType.values, MessageType.values)) { // Any request can receive a generic failure response if (type == MessageType.FAILURE_RSP) @@ -119,6 +184,15 @@ public class AccordMessageSink implements MessageSink return VerbMapping.instance.mapping.get(type); } + private static Verb getVerb(Request request) + { + MessageType type = request.type(); + if (type != null) + return getVerb(request.type()); + + return null; + } + private final Agent agent; private final MessageDelivery messaging; private final AccordEndpointMapper endpointMapper; @@ -138,7 +212,7 @@ public class AccordMessageSink implements MessageSink @Override public void send(Node.Id to, Request request) { - Verb verb = getVerb(request.type()); + Verb verb = getVerb(request); Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type()); Message message = Message.out(verb, request); InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to); @@ -149,7 +223,7 @@ public class AccordMessageSink implements MessageSink @Override public void send(Node.Id to, Request request, AgentExecutor executor, Callback callback) { - Verb verb = getVerb(request.type()); + Verb verb = getVerb(request); Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type()); Message message = Message.out(verb, request); InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to); @@ -162,6 +236,8 @@ public class AccordMessageSink implements MessageSink { Message replyTo = (Message) replyContext; Message replyMsg = replyTo.responseWith(reply); + if (!reply.isFinal()) + replyMsg = replyMsg.withFlag(MessageFlag.NOT_FINAL); checkReplyType(reply, replyTo); InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(replyingToNode); logger.debug("Replying {} {} to {}", replyMsg.verb(), replyMsg.payload, endpoint); diff --git a/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java b/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java index bff25a9c05..8630377a52 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java +++ b/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java @@ -60,10 +60,10 @@ import org.apache.cassandra.service.accord.api.AccordRoutingKey; import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.serializers.WaitingOnSerializer; -import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.service.accord.txn.AccordUpdate; import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; -import org.apache.cassandra.service.accord.txn.TxnUpdate; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.accord.txn.TxnWrite; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.ObjectSizes; @@ -197,7 +197,7 @@ public class AccordObjectSizes size += seekables(txn.keys()); size += ((TxnRead) txn.read()).estimatedSizeOnHeap(); if (txn.update() != null) - size += ((TxnUpdate) txn.update()).estimatedSizeOnHeap(); + size += ((AccordUpdate) txn.update()).estimatedSizeOnHeap(); if (txn.query() != null) size += ((TxnQuery) txn.query()).estimatedSizeOnHeap(); return size; @@ -250,7 +250,7 @@ public class AccordObjectSizes public static long results(Result result) { - return ((TxnData) result).estimatedSizeOnHeap(); + return ((TxnResult) result).estimatedSizeOnHeap(); } private static final long EMPTY_COMMAND_LISTENER = measure(new Command.ProxyListener(null)); @@ -331,7 +331,7 @@ public class AccordObjectSizes size += sizeNullable(command.accepted(), AccordObjectSizes::timestamp); size += sizeNullable(command.writes(), AccordObjectSizes::writes); - if (command.result() instanceof TxnData) + if (command.result() instanceof TxnResult) size += sizeNullable(command.result(), AccordObjectSizes::results); if (!(command instanceof Command.Committed)) diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java index 516cdead12..6009e52c3a 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java @@ -18,12 +18,13 @@ package org.apache.cassandra.service.accord; -import java.util.Collection; import java.util.Objects; import com.google.common.annotations.VisibleForTesting; import accord.local.Command; +import accord.local.Command.TransientListener; +import accord.local.Listeners; import accord.local.SafeCommand; import accord.primitives.TxnId; @@ -138,7 +139,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState transientListeners() + public Listeners transientListeners() { checkNotInvalidated(); return global.listeners(); diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java index e9bfc0d15d..4fb9bc203c 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java @@ -21,8 +21,11 @@ package org.apache.cassandra.service.accord; import java.util.Map; import java.util.NavigableMap; import java.util.function.BiFunction; +import java.util.function.Predicate; import javax.annotation.Nullable; +import com.google.common.base.Predicates; + import accord.api.Agent; import accord.api.DataStore; import accord.api.Key; @@ -156,7 +159,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore keysOrRanges, Ranges slice) { - Timestamp maxConflict = mapReduce(keysOrRanges, slice, (ts, accum) -> Timestamp.max(ts.max(), accum), Timestamp.NONE, null); + Timestamp maxConflict = mapReduce(keysOrRanges, slice, (ts, accum) -> Timestamp.max(ts.max(), accum), Timestamp.NONE, Predicates.isNull()); return Timestamp.nonNullOrMax(maxConflict, commandStore.commandsForRanges().maxRedundant()); } @@ -198,15 +201,15 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore O mapReduce(Routables keysOrRanges, Ranges slice, BiFunction map, O accumulate, O terminalValue) + private O mapReduce(Routables keysOrRanges, Ranges slice, BiFunction map, O accumulate, Predicate terminate) { - accumulate = commandStore.mapReduceForRange(keysOrRanges, slice, map, accumulate, terminalValue); - if (accumulate.equals(terminalValue)) + accumulate = commandStore.mapReduceForRange(keysOrRanges, slice, map, accumulate, terminate); + if (terminate.test(accumulate)) return accumulate; - return mapReduceForKey(keysOrRanges, slice, map, accumulate, terminalValue); + return mapReduceForKey(keysOrRanges, slice, map, accumulate, terminate); } - private O mapReduceForKey(Routables keysOrRanges, Ranges slice, BiFunction map, O accumulate, O terminalValue) + private O mapReduceForKey(Routables keysOrRanges, Ranges slice, BiFunction map, O accumulate, Predicate terminate) { switch (keysOrRanges.domain()) { @@ -221,7 +224,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore T mapReduce(Seekables keysOrRanges, Ranges slice, TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction map, T accumulate, T terminalValue) { + Predicate terminate = Predicates.equalTo(terminalValue); + return mapReduceWithTerminate(keysOrRanges, slice, testKind, testTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, accumulate, terminate); + } + + @Override + public T mapReduceWithTerminate(Seekables keysOrRanges, Ranges slice, TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction map, T accumulate, Predicate terminate) { accumulate = mapReduce(keysOrRanges, slice, (forKey, prev) -> { CommandTimeseries timeseries; switch (testTimestamp) @@ -276,8 +285,8 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore nullablePartitionUpdateSerializer = NullableSerializer.wrap(partitionUpdateSerializer); + public static final IVersionedSerializer columnMetadataSerializer = new IVersionedSerializer() { @Override @@ -246,4 +250,25 @@ public class AccordSerializers return size; } }; + + public static final IVersionedSerializer consistencyLevelSerializer = new IVersionedSerializer() + { + @Override + public void serialize(ConsistencyLevel t, DataOutputPlus out, int version) throws IOException + { + out.writeByte(t.code); + } + + @Override + public ConsistencyLevel deserialize(DataInputPlus in, int version) throws IOException + { + return ConsistencyLevel.fromCode(in.readByte()); + } + + @Override + public long serializedSize(ConsistencyLevel t, int version) + { + return 1; + } + }; } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 5de4daa233..fd115f8943 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -24,14 +24,17 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nonnull; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import accord.api.BarrierType; import accord.api.Result; import accord.config.LocalConfig; +import accord.coordinate.CoordinationFailed; import accord.coordinate.Preempted; import accord.coordinate.Timeout; import accord.impl.AbstractConfigurationService; @@ -39,12 +42,16 @@ import accord.impl.SimpleProgressLog; import accord.impl.SizeOfIntersectionSorter; import accord.local.DurableBefore; import accord.local.Node; +import accord.local.Node.Id; import accord.local.NodeTimeService; import accord.local.RedundantBefore; import accord.local.ShardDistributor.EvenSplit; import accord.messages.LocalMessage; import accord.messages.Request; +import accord.primitives.Seekables; +import accord.primitives.Timestamp; import accord.primitives.Txn; +import accord.primitives.Txn.Kind; import accord.primitives.TxnId; import accord.topology.TopologyManager; import accord.utils.DefaultRandom; @@ -59,7 +66,9 @@ import org.apache.cassandra.concurrent.Shutdownable; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.exceptions.ExceptionCode; import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.exceptions.RequestTimeoutException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.journal.AsyncWriteCallback; import org.apache.cassandra.metrics.AccordClientRequestMetrics; @@ -75,29 +84,38 @@ import org.apache.cassandra.service.accord.api.AccordTopologySorter; import org.apache.cassandra.service.accord.api.CompositeTopologySorter; import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException; import org.apache.cassandra.service.accord.exceptions.WritePreemptedException; -import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.service.accord.interop.AccordInteropApply; +import org.apache.cassandra.service.accord.interop.AccordInteropExecution; +import org.apache.cassandra.service.accord.interop.AccordInteropPersist; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.transformations.AddAccordKeyspace; +import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.ExecutorUtils; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.ImmediateFuture; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static accord.messages.SimpleReply.Ok; +import static accord.utils.Invariants.checkState; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics; import static org.apache.cassandra.utils.Clock.Global.nanoTime; public class AccordService implements IAccordService, Shutdownable { private static final Logger logger = LoggerFactory.getLogger(AccordService.class); - public static final AccordClientRequestMetrics readMetrics = new AccordClientRequestMetrics("AccordRead"); - public static final AccordClientRequestMetrics writeMetrics = new AccordClientRequestMetrics("AccordWrite"); private static final Future BOOTSTRAP_SUCCESS = ImmediateFuture.success(null); private final Node node; @@ -119,7 +137,13 @@ public class AccordService implements IAccordService, Shutdownable } @Override - public TxnData coordinate(Txn txn, ConsistencyLevel consistencyLevel) + public long barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite) + { + throw new UnsupportedOperationException("No accord barriers should be executed when accord_transactions_enabled = false in cassandra.yaml"); + } + + @Override + public @Nonnull TxnResult coordinate(@Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull Dispatcher.RequestTime requestTime) { throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml"); } @@ -181,6 +205,9 @@ public class AccordService implements IAccordService, Shutdownable { return Pair.create(new Int2ObjectHashMap<>(), DurableBefore.EMPTY); } + + @Override + public void ensureKeyspaceIsAccordManaged(String keyspace) {} }; private static volatile Node.Id localId = null; @@ -257,6 +284,9 @@ public class AccordService implements IAccordService, Shutdownable new AccordTopologySorter.Supplier(configService, DatabaseDescriptor.getNodeProximity())), SimpleProgressLog::new, AccordCommandStores.factory(journal), + new AccordInteropExecution.Factory(agent, configService), + AccordInteropPersist.FACTORY, + AccordInteropApply.FACTORY, configuration); this.nodeShutdown = toShutdownable(node); this.verbHandler = new AccordVerbHandler<>(node, configService, journal); @@ -277,34 +307,21 @@ public class AccordService implements IAccordService, Shutdownable } @Override - public long currentEpoch() + public long barrier(@Nonnull Seekables keysOrRanges, long epoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite) { - return configService.currentEpoch(); - } - - @Override - public TopologyManager topology() - { - return node.topology(); - } - - /** - * Consistency level is just echoed back in timeouts, in the future it may be used for interoperability - * with non-Accord operations. - */ - @Override - public TxnData coordinate(Txn txn, ConsistencyLevel consistencyLevel) - { - AccordClientRequestMetrics metrics = txn.isWrite() ? writeMetrics : readMetrics; + AccordClientRequestMetrics metrics = isForWrite ? accordWriteMetrics : accordReadMetrics; TxnId txnId = null; - final long startNanos = nanoTime(); try { - metrics.keySize.update(txn.keys().size()); - txnId = node.nextTxnId(txn.kind(), txn.keys().domain()); - AsyncResult asyncResult = node.coordinate(txnId, txn); - Result result = AsyncChains.getBlocking(asyncResult, DatabaseDescriptor.getTransactionTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS); - return (TxnData) result; + logger.debug("Starting barrier key: {} epoch: {} barrierType: {} isForWrite {}", keysOrRanges, epoch, barrierType, isForWrite); + txnId = node.nextTxnId(Kind.SyncPoint, keysOrRanges.domain()); + AsyncResult asyncResult = node.barrier(keysOrRanges, epoch, barrierType); + long deadlineNanos = requestTime.startedAtNanos() + timeoutNanos; + Timestamp barrierExecuteAt = AsyncChains.getBlocking(asyncResult, deadlineNanos - nanoTime(), NANOSECONDS); + logger.debug("Completed in {}ms barrier key: {} epoch: {} barrierType: {} isForWrite {}", + NANOSECONDS.toMillis(nanoTime() - requestTime.startedAtNanos()), + keysOrRanges, epoch, barrierType, isForWrite); + return barrierExecuteAt.epoch(); } catch (ExecutionException e) { @@ -312,14 +329,14 @@ public class AccordService implements IAccordService, Shutdownable if (cause instanceof Timeout) { metrics.timeouts.mark(); - throw throwTimeout(txnId, txn, consistencyLevel); + throw newBarrierTimeout(txnId, barrierType.global); } if (cause instanceof Preempted) { //TODO need to improve // Coordinator "could" query the accord state to see whats going on but that doesn't exist yet. // Protocol also doesn't have a way to denote "unknown" outcome, so using a timeout as the closest match - throw throwPreempted(txnId, txn, consistencyLevel); + throw newBarrierPreempted(txnId, barrierType.global); } metrics.failures.mark(); throw new RuntimeException(cause); @@ -332,11 +349,125 @@ public class AccordService implements IAccordService, Shutdownable catch (TimeoutException e) { metrics.timeouts.mark(); - throw throwTimeout(txnId, txn, consistencyLevel); + throw newBarrierTimeout(txnId, barrierType.global); } finally { - metrics.addNano(nanoTime() - startNanos); + // TODO Should barriers have a dedicated latency metric? Should it be a read/write metric? + // What about counts for timeouts/failures/preempts? + metrics.addNano(nanoTime() - requestTime.startedAtNanos()); + } + } + + private static ReadTimeoutException newBarrierTimeout(TxnId txnId, boolean global) + { + return new ReadTimeoutException(global ? ConsistencyLevel.ANY : ConsistencyLevel.QUORUM, 0, 0, false, txnId.toString()); + } + + private static ReadTimeoutException newBarrierPreempted(TxnId txnId, boolean global) + { + return new ReadPreemptedException(global ? ConsistencyLevel.ANY : ConsistencyLevel.QUORUM, 0, 0, false, txnId.toString()); + } + + @Override + public long barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException + { + // Since we could end up having the barrier transaction or the transaction it listens to invalidated + CoordinationFailed existingFailures = null; + Long success = null; + long backoffMillis = 0; + for (int attempt = 0; attempt < DatabaseDescriptor.getAccordBarrierRetryAttempts(); attempt++) + { + try + { + Thread.sleep(backoffMillis); + } + catch (InterruptedException e) + { + if (existingFailures != null) + e.addSuppressed(existingFailures); + throw e; + } + backoffMillis = backoffMillis == 0 ? DatabaseDescriptor.getAccordBarrierRetryInitialBackoffMillis() : Math.min(backoffMillis * 2, DatabaseDescriptor.getAccordBarrierRetryMaxBackoffMillis()); + try + { + success = AccordService.instance().barrier(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeBarrierTimeoutNanos(), barrierType, isForWrite); + break; + } + catch (CoordinationFailed newFailures) + { + existingFailures = Throwables.merge(existingFailures, newFailures); + } + } + if (success == null) + { + checkState(existingFailures != null, "Didn't have success, but also didn't have failures"); + throw existingFailures; + } + return success; + } + + @Override + public long currentEpoch() + { + return configService.currentEpoch(); + } + + @Override + public TopologyManager topology() + { + return node.topology(); + } + + /** + * Consistency level is just echoed back in timeouts, in the future it may be used for interoperability + * with non-Accord operations. + */ + @Override + public @Nonnull TxnResult coordinate(@Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + { + AccordClientRequestMetrics metrics = txn.isWrite() ? accordWriteMetrics : accordReadMetrics; + TxnId txnId = null; + try + { + metrics.keySize.update(txn.keys().size()); + txnId = node.nextTxnId(txn.kind(), txn.keys().domain()); + long deadlineNanos = requestTime.startedAtNanos() + DatabaseDescriptor.getTransactionTimeout(NANOSECONDS); + AsyncResult asyncResult = node.coordinate(txnId, txn); + Result result = AsyncChains.getBlocking(asyncResult, deadlineNanos - nanoTime(), NANOSECONDS); + return (TxnResult) result; + } + catch (ExecutionException e) + { + Throwable cause = e.getCause(); + if (cause instanceof Timeout) + { + metrics.timeouts.mark(); + throw newTimeout(txnId, txn, consistencyLevel); + } + if (cause instanceof Preempted) + { + //TODO need to improve + // Coordinator "could" query the accord state to see whats going on but that doesn't exist yet. + // Protocol also doesn't have a way to denote "unknown" outcome, so using a timeout as the closest match + throw newPreempted(txnId, txn, consistencyLevel); + } + metrics.failures.mark(); + throw new RuntimeException(cause); + } + catch (InterruptedException e) + { + metrics.failures.mark(); + throw new UncheckedInterruptedException(e); + } + catch (TimeoutException e) + { + metrics.timeouts.mark(); + throw newTimeout(txnId, txn, consistencyLevel); + } + finally + { + metrics.addNano(nanoTime() - requestTime.startedAtNanos()); } } @@ -371,13 +502,13 @@ public class AccordService implements IAccordService, Shutdownable }); } - private static RuntimeException throwTimeout(TxnId txnId, Txn txn, ConsistencyLevel consistencyLevel) + private static RequestTimeoutException newTimeout(TxnId txnId, Txn txn, ConsistencyLevel consistencyLevel) { throw txn.isWrite() ? new WriteTimeoutException(WriteType.CAS, consistencyLevel, 0, 0, txnId.toString()) : new ReadTimeoutException(consistencyLevel, 0, 0, false, txnId.toString()); } - private static RuntimeException throwPreempted(TxnId txnId, Txn txn, ConsistencyLevel consistencyLevel) + private static RuntimeException newPreempted(TxnId txnId, Txn txn, ConsistencyLevel consistencyLevel) { throw txn.isWrite() ? new WritePreemptedException(WriteType.CAS, consistencyLevel, 0, 0, txnId.toString()) : new ReadPreemptedException(consistencyLevel, 0, 0, false, txnId.toString()); @@ -438,6 +569,11 @@ public class AccordService implements IAccordService, Shutdownable return scheduler; } + public Id nodeId() + { + return node.id(); + } + @VisibleForTesting public Node node() { @@ -523,6 +659,22 @@ public class AccordService implements IAccordService, Shutdownable return ClusterMetadata.current().accordKeyspaces.contains(keyspace); } + @Override + public void ensureKeyspaceIsAccordManaged(String keyspace) + { + if (isAccordManagedKeyspace(keyspace)) + return; + ClusterMetadataService.instance().commit(new AddAccordKeyspace(keyspace), + metadata -> null, + (code, message) -> { + Invariants.checkState(code == ExceptionCode.ALREADY_EXISTS, + "Expected %s, got %s", ExceptionCode.ALREADY_EXISTS, code); + return null; + }); + // we need to avoid creating a txnId in an epoch when no one has any ranges + FBUtilities.waitOnFuture(AccordService.instance().epochReady(ClusterMetadata.current().epoch)); + } + @Override public Pair, DurableBefore> getRedundantBeforesAndDurableBefore() { diff --git a/src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java b/src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java index 88a0d18b7f..d8b757941a 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java @@ -37,13 +37,11 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey; import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey; import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ownership.DataPlacement; @@ -120,17 +118,17 @@ public class AccordTopologyUtils return shards; } - public static Topology createAccordTopology(Epoch epoch, DistributedSchema schema, DataPlacements placements, Directory directory, Predicate keyspacePredicate) + public static Topology createAccordTopology(ClusterMetadata cm, Predicate keyspacePredicate) { List shards = new ArrayList<>(); - for (KeyspaceMetadata keyspace : schema.getKeyspaces()) + for (KeyspaceMetadata keyspace : cm.schema.getKeyspaces()) { if (!keyspacePredicate.test(keyspace.name)) continue; - shards.addAll(createShards(keyspace, placements, directory)); + shards.addAll(createShards(keyspace, cm.placements, cm.directory)); } shards.sort((a, b) -> a.range.compare(b.range)); - return new Topology(epoch.getEpoch(), shards.toArray(new Shard[0])); + return new Topology(cm.epoch.getEpoch(), shards.toArray(new Shard[0])); } public static EndpointMapping directoryToMapping(EndpointMapping mapping, long epoch, Directory directory) @@ -146,11 +144,6 @@ public class AccordTopologyUtils return builder.build(); } - public static Topology createAccordTopology(ClusterMetadata metadata, Predicate keyspacePredicate) - { - return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, keyspacePredicate); - } - public static Topology createAccordTopology(ClusterMetadata metadata) { return createAccordTopology(metadata, metadata.accordKeyspaces::contains); diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java index 5ca68d10fc..5610ebd9c2 100644 --- a/src/java/org/apache/cassandra/service/accord/IAccordService.java +++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java @@ -18,30 +18,74 @@ package org.apache.cassandra.service.accord; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import com.google.common.collect.ImmutableSet; + +import accord.api.BarrierType; import accord.local.DurableBefore; +import accord.local.Node.Id; import accord.local.RedundantBefore; import accord.messages.Request; +import accord.primitives.Ranges; +import accord.primitives.Seekables; import accord.primitives.Txn; import accord.topology.TopologyManager; import org.agrona.collections.Int2ObjectHashMap; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; +import org.apache.cassandra.service.accord.api.AccordRoutableKey; +import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey; import org.apache.cassandra.service.accord.api.AccordScheduler; -import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Future; public interface IAccordService { + Set SUPPORTED_COMMIT_CONSISTENCY_LEVELS = ImmutableSet.of(ConsistencyLevel.ANY, ConsistencyLevel.ONE, ConsistencyLevel.QUORUM, ConsistencyLevel.SERIAL, ConsistencyLevel.ALL); + Set SUPPORTED_READ_CONSISTENCY_LEVELS = ImmutableSet.of(ConsistencyLevel.ONE, ConsistencyLevel.QUORUM, ConsistencyLevel.SERIAL); + IVerbHandler verbHandler(); - TxnData coordinate(Txn txn, ConsistencyLevel consistencyLevel); + default long barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException + { + throw new UnsupportedOperationException(); + } + + long barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite); + + default void postStreamReceivingBarrier(ColumnFamilyStore cfs, List> ranges) + { + String ks = cfs.keyspace.getName(); + Ranges accordRanges = Ranges.of(ranges + .stream() + .map(r -> new TokenRange(new TokenKey(ks, r.left), new TokenKey(ks, r.right))) + .collect(Collectors.toList()) + .toArray(new accord.primitives.Range[0])); + try + { + barrierWithRetries(accordRanges, Epoch.FIRST.getEpoch(), BarrierType.global_async, true); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + @Nonnull TxnResult coordinate(@Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime); long currentEpoch(); @@ -74,4 +118,26 @@ public interface IAccordService * Fetch the redundnant befores for every command store */ Pair, DurableBefore> getRedundantBeforesAndDurableBefore(); + + default Id nodeId() { throw new UnsupportedOperationException(); } + + default void maybeConvertKeyspacesToAccord(Txn txn) + { + Set allKeyspaces = new HashSet<>(); + txn.keys().forEach(key -> allKeyspaces.add(((AccordRoutableKey) key).keyspace())); + + for (String keyspace : allKeyspaces) + { + + ensureKeyspaceIsAccordManaged(keyspace); + } + + for (String keyspace : allKeyspaces) + { + if (!AccordService.instance().isAccordManagedKeyspace(keyspace)) + throw new IllegalStateException(keyspace + " is not an accord managed keyspace"); + } + } + + void ensureKeyspaceIsAccordManaged(String keyspace); } diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index e01587e61b..f6a5724767 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -23,6 +23,8 @@ import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; + import accord.api.Agent; import accord.api.EventsListener; import accord.api.Result; @@ -32,16 +34,20 @@ import accord.primitives.Ranges; import accord.primitives.Seekables; import accord.primitives.Timestamp; import accord.primitives.Txn; +import accord.primitives.Txn.Kind; import accord.primitives.TxnId; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.metrics.AccordMetrics; import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.utils.JVMStabilityInspector; +import static accord.primitives.Routable.Domain.Key; import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.config.DatabaseDescriptor.getReadRpcTimeout; +import static org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.maybeSaveAccordKeyMigrationLocally; // TODO (expected): merge with AccordService public class AccordAgent implements Agent @@ -78,6 +84,16 @@ public class AccordAgent implements Agent AccordService.instance().scheduler().once(retry, retryBootstrapDelayMicros, MICROSECONDS); } + @Override + public void onLocalBarrier(@Nonnull Seekables keysOrRanges, @Nonnull Timestamp executeAt) + { + if (keysOrRanges.domain() == Key) + { + PartitionKey key = (PartitionKey)keysOrRanges.get(0); + maybeSaveAccordKeyMigrationLocally(key, Epoch.create(executeAt.epoch())); + } + } + @Override public void onUncaughtException(Throwable t) { @@ -99,9 +115,9 @@ public class AccordAgent implements Agent } @Override - public Txn emptyTxn(Txn.Kind kind, Seekables keysOrRanges) + public Txn emptyTxn(Kind kind, Seekables seekables) { - return new Txn.InMemory(kind, keysOrRanges, TxnRead.EMPTY, TxnQuery.ALL, null); + return new Txn.InMemory(kind, seekables, TxnRead.EMPTY, TxnQuery.EMPTY, null); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/api/PartitionKey.java b/src/java/org/apache/cassandra/service/accord/api/PartitionKey.java index 2c4e58ee83..a8dac58234 100644 --- a/src/java/org/apache/cassandra/service/accord/api/PartitionKey.java +++ b/src/java/org/apache/cassandra/service/accord/api/PartitionKey.java @@ -33,6 +33,7 @@ import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.db.marshal.ValueAccessor; import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; @@ -70,6 +71,11 @@ public final class PartitionKey extends AccordRoutableKey implements Key return (PartitionKey) key; } + public static PartitionKey of(PartitionUpdate update) + { + return new PartitionKey(update.metadata().keyspace, update.metadata().id, update.partitionKey()); + } + public static PartitionKey of(Partition partition) { return new PartitionKey(partition.metadata().keyspace, partition.metadata().id, partition.partitionKey()); diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java new file mode 100644 index 0000000000..22821bab39 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.interop; + +import java.util.BitSet; +import javax.annotation.Nullable; + +import accord.api.Result; +import accord.local.Command; +import accord.local.Node.Id; +import accord.local.PreLoadContext; +import accord.local.SafeCommand; +import accord.local.SafeCommandStore; +import accord.local.Status; +import accord.messages.Apply; +import accord.messages.MessageType; +import accord.primitives.Deps; +import accord.primitives.Keys; +import accord.primitives.PartialDeps; +import accord.primitives.PartialRoute; +import accord.primitives.PartialTxn; +import accord.primitives.Route; +import accord.primitives.Seekables; +import accord.primitives.Timestamp; +import accord.primitives.Txn; +import accord.primitives.TxnId; +import accord.primitives.Writes; +import accord.topology.Topologies; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType; +import org.apache.cassandra.service.accord.serializers.ApplySerializers.ApplySerializer; +import org.apache.cassandra.service.accord.txn.AccordUpdate; + +import static accord.utils.Invariants.checkState; +import static accord.utils.MapReduceConsume.forEach; +import static com.google.common.base.Preconditions.checkArgument; + +/** + * Apply that waits until the transaction is actually applied before sending a response + * // TODO (desired): At this point there are a plethora of do X to Command, then wait until state Y before maybe doing Z and returning a response, potentially returning insufficient along the way + * // and these all are a bit copy pasta in terms of managing things like waiting on, obsoletion, cancellation/listeners, insufficient etc. and it would be less fragile + * // in the long run to not duplicate these kind of difficult to get right mechanism and have a single pluggable framework to request each specific behavior + */ +public class AccordInteropApply extends Apply implements Command.TransientListener +{ + public static final Apply.Factory FACTORY = new Apply.Factory() + { + @Override + public Apply create(Kind kind, Id to, Topologies participates, Topologies executes, TxnId txnId, Route route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result) + { + checkArgument(kind != Kind.Maximal, "Shouldn't need to send a maximal commit with interop support"); + ConsistencyLevel commitCL = txn.update() instanceof AccordUpdate ? ((AccordUpdate) txn.update()).cassandraCommitCL() : null; + // Any asynchronous apply option should use the regular Apply that doesn't wait for writes to complete + if (commitCL == null || commitCL == ConsistencyLevel.ANY) + return Apply.FACTORY.create(kind, to, participates, executes, txnId, route, txn, executeAt, deps, writes, result); + return new AccordInteropApply(kind, to, participates, executes, txnId, route, txn, executeAt, deps, writes, result); + } + }; + + public static final IVersionedSerializer serializer = new ApplySerializer() + { + @Override + protected AccordInteropApply deserializeApply(TxnId txnId, PartialRoute scope, long waitForEpoch, Apply.Kind kind, Seekables keys, Timestamp executeAt, PartialDeps deps, PartialTxn txn, Writes writes, Result result) + { + return new AccordInteropApply(kind, txnId, scope, waitForEpoch, keys, executeAt, deps, txn, writes, result); + } + }; + + transient BitSet waitingOn; + transient int waitingOnCount; + + private AccordInteropApply(Kind kind, TxnId txnId, PartialRoute route, long waitForEpoch, Seekables keys, Timestamp executeAt, PartialDeps deps, @Nullable PartialTxn txn, Writes writes, Result result) + { + super(kind, txnId, route, waitForEpoch, keys, executeAt, deps, txn, writes, result); + } + + private AccordInteropApply(Kind kind, Id to, Topologies participates, Topologies executes, TxnId txnId, Route route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result) + { + super(kind, to, participates, executes, txnId, route, txn, executeAt, deps, writes, result); + } + + @Override + public void process() + { + waitingOn = new BitSet(); + super.process(); + } + + + @Override + public ApplyReply apply(SafeCommandStore safeStore) + { + ApplyReply reply = super.apply(safeStore); + checkState(reply == ApplyReply.Redundant || reply == ApplyReply.Applied || reply == ApplyReply.Insufficient, "Unexpected ApplyReply"); + + // Hasn't necessarily finished applying yet so need to check and maybe add a listener + // Redundant means we are competing with a recovery coordinator which is fine + // we don't need to return an error we can wait for the Apply + // Insufficient means it is safe to install the listener and wait for Apply to happen + // once the coordinator sends a maximal commit + // Applied doesn't actually mean the command is in the Applied state so we still need to check and maybe install + // the listener + SafeCommand safeCommand = safeStore.get(txnId, executeAt, scope); + Command current = safeCommand.current(); + // Don't actually think it is possible for this to reach applied while we are stll running, but just to be safe + // check anyways + Status status = current.status(); + switch (status) + { + default: throw new AssertionError(); + case NotDefined: + case PreAccepted: + case Accepted: + case AcceptedInvalidate: + case PreCommitted: + case Committed: + case PreApplied: + case ReadyToExecute: + synchronized (this) + { + waitingOn.set(safeStore.commandStore().id()); + ++waitingOnCount; + } + safeCommand.addListener(this); + break; + + case Applied: + case Invalidated: + case Truncated: + } + + return reply; + } + + private synchronized void ack() + { + // wait for -1 to ensure the setup phase has also completed. Setup calls ack in its callback + // and prevents races where we respond before dispatching all the required reads (if the reads are + // completing faster than the reads can be setup on all required shards) + if (-1 == --waitingOnCount) + { + node.reply(replyTo, replyContext, ApplyReply.Applied, null); + } + } + + @Override + public ApplyReply reduce(ApplyReply r1, ApplyReply r2) + { + return r1 == null || r2 == null + ? r1 == null ? r2 : r1 + : r1.compareTo(r2) >= 0 ? r1 : r2; + } + + @Override + public void accept(ApplyReply reply, Throwable failure) + { + if (reply == ApplyReply.Insufficient) + { + // Respond with insufficient which should make the coordinator send us the commit + // we need to respond + node.reply(replyTo, replyContext, reply, failure); + } + else if (failure != null) + { + node.reply(replyTo, replyContext, null, failure); + node.agent().onUncaughtException(failure); + cancel(); + } + + // Unless failed always ack to indicate setup has completed otherwise the counter never gets to -1 + if (failure == null) + ack(); + } + + private void cancel() + { + node.commandStores().mapReduceConsume(this, waitingOn.stream(), forEach(safeStore -> { + SafeCommand safeCommand = safeStore.ifInitialised(txnId); + if (safeCommand != null) + safeCommand.removeListener(this); + }, node.agent())); + } + + @Override + public TxnId primaryTxnId() + { + return txnId; + } + + @Override + public Seekables keys() + { + if (txn == null) return Keys.EMPTY; + return txn.keys(); + } + + @Override + public MessageType type() + { + switch (kind) + { + case Minimal: return AccordMessageType.INTEROP_APPLY_MINIMAL_REQ; + case Maximal: return AccordMessageType.INTEROP_APPLY_MAXIMAL_REQ; + default: throw new IllegalStateException(); + } + } + + @Override + public String toString() + { + return "AccordInteropApply{" + + "txnId:" + txnId + + ", deps:" + deps + + ", executeAt:" + executeAt + + ", writes:" + writes + + ", result:" + result + + '}'; + } + + @Override + public void onChange(SafeCommandStore safeStore, SafeCommand safeCommand) + { + Command command = safeCommand.current(); + + switch (command.status()) + { + default: throw new AssertionError(); + case NotDefined: + case PreAccepted: + case Accepted: + case AcceptedInvalidate: + case PreCommitted: + case Committed: + case PreApplied: + case ReadyToExecute: + return; + + case Applied: + case Invalidated: + case Truncated: + } + + if (safeCommand.removeListener(this)) + ack(); + } + + @Override + public PreLoadContext listenerPreLoadContext(TxnId caller) + { + return PreLoadContext.contextFor(txnId); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropCommit.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropCommit.java new file mode 100644 index 0000000000..e3051bf644 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropCommit.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.interop; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import accord.local.Node; +import accord.messages.Commit; +import accord.messages.MessageType; +import accord.messages.ReadData; +import accord.primitives.Deps; +import accord.primitives.FullRoute; +import accord.primitives.PartialDeps; +import accord.primitives.PartialRoute; +import accord.primitives.PartialTxn; +import accord.primitives.Timestamp; +import accord.primitives.Txn; +import accord.primitives.TxnId; +import accord.topology.Topologies; +import accord.topology.Topology; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType; +import org.apache.cassandra.service.accord.serializers.CommitSerializers.CommitSerializer; + +public class AccordInteropCommit extends Commit +{ + public static final IVersionedSerializer serializer = new CommitSerializer(AccordInteropRead.class, AccordInteropRead.requestSerializer) + { + @Override + protected AccordInteropCommit deserializeCommit(TxnId txnId, PartialRoute scope, long waitForEpoch, Kind kind, Timestamp executeAt, @Nullable PartialTxn partialTxn, PartialDeps partialDeps, @Nullable FullRoute fullRoute, @Nullable ReadData read) + { + return new AccordInteropCommit(kind, txnId, scope, waitForEpoch, executeAt, partialTxn, partialDeps, fullRoute, read); + } + }; + + public AccordInteropCommit(Kind kind, TxnId txnId, PartialRoute scope, long waitForEpoch, Timestamp executeAt, @Nullable PartialTxn partialTxn, PartialDeps partialDeps, @Nullable FullRoute fullRoute, @Nonnull ReadData readData) + { + super(kind, txnId, scope, waitForEpoch, executeAt, partialTxn, partialDeps, fullRoute, readData); + } + + public AccordInteropCommit(Kind kind, Node.Id to, Topology coordinateTopology, Topologies topologies, TxnId txnId, Txn txn, FullRoute route, Timestamp executeAt, Deps deps, AccordInteropRead read) + { + super(kind, to, coordinateTopology, topologies, txnId, txn, route, executeAt, deps, (t, u, p) -> read); + } + + @Override + public MessageType type() + { + switch (kind) + { + case Minimal: return AccordMessageType.INTEROP_COMMIT_MINIMAL_REQ; + case Maximal: return AccordMessageType.INTEROP_COMMIT_MAXIMAL_REQ; + default: throw new IllegalStateException(); + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java new file mode 100644 index 0000000000..71757ed106 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java @@ -0,0 +1,412 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.interop; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.api.Agent; +import accord.api.Data; +import accord.api.Result; +import accord.coordinate.Execute; +import accord.coordinate.Persist; +import accord.coordinate.TxnExecute; +import accord.local.AgentExecutor; +import accord.local.CommandStore; +import accord.local.Node; +import accord.local.Node.Id; +import accord.messages.Commit; +import accord.messages.Commit.Kind; +import accord.primitives.Deps; +import accord.primitives.FullRoute; +import accord.primitives.Participants; +import accord.primitives.Seekables; +import accord.primitives.Timestamp; +import accord.primitives.Txn; +import accord.primitives.TxnId; +import accord.topology.Shard; +import accord.topology.Topologies; +import accord.topology.Topology; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadResponse; +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.SinglePartitionReadCommand.Group; +import org.apache.cassandra.db.partitions.FilteredPartition; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.metrics.AccordClientRequestMetrics; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.RequestCallback; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.service.StorageProxy; +import org.apache.cassandra.service.accord.AccordEndpointMapper; +import org.apache.cassandra.service.accord.TokenRange; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.service.accord.api.AccordRoutingKey; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.interop.AccordInteropReadCallback.MaximalCommitSender; +import org.apache.cassandra.service.accord.txn.AccordUpdate; +import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.service.accord.txn.TxnRead; +import org.apache.cassandra.service.accord.txn.UnrecoverableRepairUpdate; +import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; +import org.apache.cassandra.service.reads.ReadCoordinator; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.transport.Dispatcher; + +import static accord.utils.Invariants.checkArgument; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics; + +/* + * The core interoperability problem between Accord and C* writes (regular, and read repair) + * is that when the writes don't go through Accord then Accord can read data that is not yet committed + * because Accord replicas can lag behind and multiple coordinators can be attempting to compute the result of a + * transaction and they can compute different results depending on what they consider to be the inputs to the Accord + * transaction. + * + * We generally solve this by forcing non-Accord writes through Accord as well as by having Accord perform read repair + * on its inputs. + * + */ +public class AccordInteropExecution implements Execute, ReadCoordinator, MaximalCommitSender +{ + private static final Logger logger = LoggerFactory.getLogger(AccordInteropExecution.class); + + private static class InteropExecutor implements AgentExecutor + { + private final AccordAgent agent; + + public InteropExecutor(AccordAgent agent) + { + this.agent = agent; + } + + @Override + public Agent agent() + { + return agent; + } + + @Override + public AsyncChain submit(Callable task) + { + try + { + return AsyncChains.success(task.call()); + } + catch (Throwable e) + { + return AsyncChains.failure(e); + } + } + } + + public static class Factory implements Execute.Factory + { + private final InteropExecutor executor; + private final AccordEndpointMapper endpointMapper; + + public Factory(AccordAgent agent, AccordEndpointMapper endpointMapper) + { + this.executor = new InteropExecutor(agent); + this.endpointMapper = endpointMapper; + } + + @Override + public Execute create(Node node, TxnId txnId, Txn txn, FullRoute route, Participants readScope, Timestamp executeAt, Deps deps, BiConsumer callback) + { + // Unrecoverable repair always needs to be run by AccordInteropExecution + AccordUpdate.Kind updateKind = AccordUpdate.kind(txn.update()); + ConsistencyLevel consistencyLevel = txn.read() instanceof TxnRead ? ((TxnRead) txn.read()).cassandraConsistencyLevel() : null; + if (updateKind != AccordUpdate.Kind.UNRECOVERABLE_REPAIR && (consistencyLevel == null || consistencyLevel == ConsistencyLevel.ONE || txn.read().keys().isEmpty())) + return TxnExecute.FACTORY.create(node, txnId, txn, route, readScope, executeAt, deps, callback); + return new AccordInteropExecution(node, txnId, txn, updateKind, route, readScope, executeAt, deps, callback, executor, consistencyLevel, endpointMapper); + } + } + + private final Node node; + private final TxnId txnId; + private final Txn txn; + private final FullRoute route; + private final Participants readScope; + private final Timestamp executeAt; + private final Deps deps; + private final BiConsumer callback; + private final AgentExecutor executor; + private final ConsistencyLevel consistencyLevel; + private final AccordEndpointMapper endpointMapper; + + private final Topologies executes; + private final Topologies allTopologies; + private final Topology executeTopology; + private final Topology coordinateTopology; + + private final AtomicInteger readsCurrentlyUnderConstruction; + + private final Set contacted; + private final AccordUpdate.Kind updateKind; + + public AccordInteropExecution(Node node, TxnId txnId, Txn txn, AccordUpdate.Kind updateKind, FullRoute route, Participants readScope, Timestamp executeAt, Deps deps, BiConsumer callback, + AgentExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper) + { + checkArgument(!txn.read().keys().isEmpty() || updateKind == AccordUpdate.Kind.UNRECOVERABLE_REPAIR); + this.node = node; + this.txnId = txnId; + this.txn = txn; + this.route = route; + this.readScope = readScope; + this.executeAt = executeAt; + this.deps = deps; + this.callback = callback; + this.executor = executor; + + checkArgument(updateKind == AccordUpdate.Kind.UNRECOVERABLE_REPAIR || consistencyLevel == ConsistencyLevel.QUORUM || consistencyLevel == ConsistencyLevel.ALL || consistencyLevel == ConsistencyLevel.SERIAL); + this.consistencyLevel = consistencyLevel; + this.endpointMapper = endpointMapper; + + this.executes = node.topology().forEpoch(route, executeAt.epoch()); + this.allTopologies = txnId.epoch() != executeAt.epoch() + ? node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch()) + : executes; + this.executeTopology = executes.forEpoch(executeAt.epoch()); + this.coordinateTopology = allTopologies.forEpoch(txnId.epoch()); + if (consistencyLevel != ConsistencyLevel.ALL) + { + readsCurrentlyUnderConstruction = new AtomicInteger(txn.read().keys().size()); + contacted = Collections.newSetFromMap(new ConcurrentHashMap<>()); + } + else + { + readsCurrentlyUnderConstruction = null; + contacted = null; + } + this.updateKind = updateKind; + } + + @Override + public boolean localReadSupported() + { + return false; + } + + @Override + public EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata doNotUse, KeyspaceMetadata keyspace, Token token) + { + AccordRoutingKey.TokenKey key = new AccordRoutingKey.TokenKey(keyspace.name, token); + Shard shard = executeTopology.forKey(key); + Range range = ((TokenRange) shard.range).toKeyspaceRange(); + + Replica[] replicas = new Replica[shard.nodes.size()]; + for (int i=0; i message, InetAddressAndPort to, RequestCallback callback) + { + Node.Id id = endpointMapper.mappedId(to); + SinglePartitionReadCommand command = (SinglePartitionReadCommand) message.payload; + AccordInteropRead read = new AccordInteropRead(id, executes, txnId, readScope, executeAt, command); + AccordInteropCommit commit = new AccordInteropCommit(Commit.Kind.Minimal, id, coordinateTopology, allTopologies, + txnId, txn, route, executeAt, deps, read); + node.send(id, commit, executor, new AccordInteropRead.ReadCallback(id, to, message, callback, this)); + } + + @Override + public void sendReadRepairMutation(Message message, InetAddressAndPort to, RequestCallback callback) + { + Node.Id id = endpointMapper.mappedId(to); + Mutation mutation = message.payload; + AccordInteropReadRepair readRepair = new AccordInteropReadRepair(id, executes, txnId, readScope, executeAt, mutation); + node.send(id, readRepair, executor, new AccordInteropReadRepair.ReadRepairCallback(id, to, message, callback, this)); + } + + private AsyncChain readChains() + { + int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(executeAt.hlc()); + // TODO (expected): use normal query nano time + Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution(); + + TxnRead read = (TxnRead) txn.read(); + List> results = new ArrayList<>(); + Seekables keys = txn.read().keys(); + keys.forEach(key -> { + read.forEachWithKey((PartitionKey) key, fragment -> { + SinglePartitionReadCommand command = (SinglePartitionReadCommand) fragment.command(); + + // This should only rarely occur when coordinators start a transaction in a migrating range + // because they haven't yet updated their cluster metadata. + // It would be harmless to do the read, but we can respond faster skipping it + // and getting the transaction on the correct protocol + TableMigrationState tms = ConsensusTableMigrationState.getTableMigrationState(command.metadata().id); + AccordClientRequestMetrics metrics = txn.kind().isWrite() ? accordWriteMetrics : accordReadMetrics; + if (ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(tms, command.partitionKey())) + { + metrics.migrationSkippedReads.mark(); + results.add(AsyncChains.success(TxnData.emptyPartition(fragment.txnDataName(), command))); + return; + } + + Group group = Group.one(command.withNowInSec(nowInSeconds)); + results.add(AsyncChains.ofCallable(Stage.ACCORD_MIGRATION.executor(), () -> { + TxnData result = new TxnData(); + try (PartitionIterator iterator = StorageProxy.readRegular(group, consistencyLevel, this, requestTime)) + { + if (iterator.hasNext()) + { + try (RowIterator partition = iterator.next()) + { + FilteredPartition filtered = FilteredPartition.create(partition); + if (filtered.hasRows() || command.selectsFullPartition()) + result.put(fragment.txnDataName(), filtered); + } + } + } + return result; + })); + }); + }); + + if (results.isEmpty()) + return AsyncChains.success(new TxnData()); + + if (results.size() == 1) + return results.get(0); + + return AsyncChains.reduce(results, Data::merge); + } + + /* + * Any nodes not contacted for read need to be sent commits + */ + @Override + public void notifyOfInitialContacts(EndpointsForToken fullDataRequests, EndpointsForToken transientRequests, EndpointsForToken digestRequests) + { + if (readsCurrentlyUnderConstruction == null) + return; + + for (int i = 0; i < fullDataRequests.size(); i++) + contacted.add(fullDataRequests.endpoint(i)); + for (int i = 0; i < transientRequests.size(); i++) + contacted.add(transientRequests.endpoint(i)); + for (int i = 0; i < digestRequests.size(); i++) + contacted.add(digestRequests.endpoint(i)); + if (readsCurrentlyUnderConstruction.decrementAndGet() == 0) + sendCommitsToUncontacted(); + } + + private void sendCommitsToUncontacted() + { + for (Node.Id to : executeTopology.nodes()) + if (!contacted.contains(endpointMapper.mappedEndpoint(to))) + node.send(to, new Commit(Kind.Minimal, to, coordinateTopology, allTopologies, txnId, txn, route, readScope, executeAt, deps, false)); + } + + @Override + public void start() + { + if (coordinateTopology != executeTopology) + { + for (Node.Id to : allTopologies.nodes()) + { + if (!executeTopology.contains(to)) + node.send(to, new Commit(Commit.Kind.Minimal, to, coordinateTopology, allTopologies, txnId, txn, route, readScope, executeAt, deps, false)); + } + } + AsyncChain result; + if (updateKind == AccordUpdate.Kind.UNRECOVERABLE_REPAIR) + result = executeUnrecoverableRepairUpdate(); + else + result = readChains(); + + CommandStore cs = node.commandStores().select(route.homeKey()); + result.beginAsResult().withExecutor(cs).begin((data, failure) -> { + if (failure == null) + Persist.persist(node, executes, txnId, route, txn, executeAt, deps, txn.execute(txnId, executeAt, data), txn.result(txnId, executeAt, data), callback); + else + callback.accept(null, failure); + }); + } + + private AsyncChain executeUnrecoverableRepairUpdate() + { + return AsyncChains.ofCallable(Stage.ACCORD_MIGRATION.executor(), () -> { + UnrecoverableRepairUpdate repairUpdate = (UnrecoverableRepairUpdate)txn.update(); + // TODO (expected): We should send the read in the same message as the commit. This requires refactor ReadData.Kind so that it doesn't specify the ordinal encoding + // and can be extended similar to MessageType which allows additional types not from Accord to be added + for (Node.Id to : executeTopology.nodes()) + node.send(to, new Commit(Kind.Minimal, to, coordinateTopology, allTopologies, txnId, txn, route, readScope, executeAt, deps, false)); + repairUpdate.runBRR(AccordInteropExecution.this); + return new TxnData(); + }); + } + + @Override + public boolean isEventuallyConsistent() + { + return false; + } + + @Override + public ReadCommand maybeAllowOutOfRangeReads(ReadCommand readCommand) + { + return readCommand.allowOutOfRangeReads(); + } + + @Override + public Mutation maybeAllowOutOfRangeMutations(Mutation m) + { + return m.allowOutOfRangeMutations(); + } + + // Prrovide request callbacks with a way to send maximal commits on Insufficient responses + @Override + public void sendMaximalCommit(Id to) + { + Commit.commitMaximal(node, to, txn, txnId, executeAt, route, deps, readScope); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java new file mode 100644 index 0000000000..7ef1581536 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.interop; + +import java.util.function.BiConsumer; + +import accord.api.Result; +import accord.api.Update; +import accord.coordinate.Persist; +import accord.coordinate.TxnPersist; +import accord.coordinate.tracking.AppliedTracker; +import accord.coordinate.tracking.QuorumTracker; +import accord.coordinate.tracking.RequestStatus; +import accord.coordinate.tracking.ResponseTracker; +import accord.local.Node; +import accord.messages.Apply; +import accord.primitives.Deps; +import accord.primitives.FullRoute; +import accord.primitives.Timestamp; +import accord.primitives.Txn; +import accord.primitives.TxnId; +import accord.primitives.Writes; +import accord.topology.Topologies; +import accord.utils.Invariants; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.service.accord.txn.AccordUpdate; +import org.apache.cassandra.utils.Throwables; + +/** + * Similar to Accord persist, but can wait on a configurable number of responses and sends AccordInteropApply messages + * that only return a response when the Apply has actually occurred. Regular Apply messages only get the transaction + * to PreApplied. + */ +public class AccordInteropPersist extends Persist +{ + public static Persist.Factory FACTORY = new Persist.Factory() + { + @Override + public Persist create(Node node, Topologies topologies, TxnId txnId, FullRoute route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result) + { + Update update = txn.update(); + ConsistencyLevel consistencyLevel = update instanceof AccordUpdate ? ((AccordUpdate) update).cassandraCommitCL() : null; + if (consistencyLevel == null || consistencyLevel == ConsistencyLevel.ANY || writes.isEmpty()) + return TxnPersist.FACTORY.create(node, topologies, txnId, route, txn, executeAt, deps, writes, result); + return new AccordInteropPersist(node, topologies, txnId, route, txn, executeAt, deps, writes, result, consistencyLevel); + } + }; + + private static class CallbackHolder + { + private final ResponseTracker tracker; + private final Result result; + private final BiConsumer clientCallback; + private Throwable failure = null; + + public CallbackHolder(ResponseTracker tracker, Result result, BiConsumer clientCallback) + { + this.tracker = tracker; + this.result = result; + this.clientCallback = clientCallback; + } + + private void handleStatus(RequestStatus status) + { + switch (status) + { + default: throw new IllegalStateException("Unhandled request status " + status); + case Success: + clientCallback.accept(result, null); + return; + case Failed: + clientCallback.accept(null, failure); + return; + case NoChange: + // noop + } + } + + public void recordSuccess(Node.Id node) + { + handleStatus(tracker.recordSuccess(node)); + } + + + public void recordFailure(Node.Id node, Throwable throwable) + { + failure = Throwables.merge(failure, throwable); + handleStatus(tracker.recordFailure(node)); + } + } + + private final ConsistencyLevel consistencyLevel; + private CallbackHolder holder = null; + + public AccordInteropPersist(Node node, Topologies topologies, TxnId txnId, FullRoute route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, ConsistencyLevel consistencyLevel) + { + super(node, topologies, txnId, route, txn, executeAt, deps, writes, result); + Invariants.checkArgument(consistencyLevel == ConsistencyLevel.QUORUM || consistencyLevel == ConsistencyLevel.ALL || consistencyLevel == ConsistencyLevel.SERIAL || consistencyLevel == ConsistencyLevel.ONE); + this.consistencyLevel = consistencyLevel; + } + + @Override + public void registerClientCallback(Writes writes, Result result, BiConsumer clientCallback) + { + + Invariants.checkState(holder == null); + switch (consistencyLevel) + { + case ONE: // Can safely upgrade ONE to QUORUM/SERIAL to get a synchronous commit + case SERIAL: + case QUORUM: + holder = new CallbackHolder(new QuorumTracker(topologies), result, clientCallback); + break; + case ALL: + holder = new CallbackHolder(new AppliedTracker(topologies), result, clientCallback); + break; + default: + throw new IllegalArgumentException("Unhandled consistency level: " + consistencyLevel); + } + } + + @Override + public void onSuccess(Node.Id from, Apply.ApplyReply reply) + { + super.onSuccess(from, reply); + switch (reply) + { + case Redundant: + case Applied: + holder.recordSuccess(from); + return; + case Insufficient: + // On insufficient Persist will send a commit with the missing information + // which will allow a final response to be returned later that could be successful + return; + default: throw new IllegalArgumentException("Unhandled apply response " + reply); + } + } + + @Override + public void onFailure(Node.Id from, Throwable failure) + { + holder.recordFailure(from, failure); + } + + @Override + public void onCallbackFailure(Node.Id from, Throwable failure) + { + holder.recordFailure(from, failure); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropRead.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropRead.java new file mode 100644 index 0000000000..97ace25566 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropRead.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.interop; + +import java.io.IOException; +import javax.annotation.Nullable; + +import accord.api.Data; +import accord.local.Node; +import accord.local.SafeCommandStore; +import accord.messages.AbstractExecute; +import accord.messages.MessageType; +import accord.primitives.PartialTxn; +import accord.primitives.Participants; +import accord.primitives.Ranges; +import accord.primitives.Timestamp; +import accord.primitives.TxnId; +import accord.topology.Topologies; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.db.ReadCommandVerbHandler; +import org.apache.cassandra.db.ReadResponse; +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.RequestCallback; +import org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType; +import org.apache.cassandra.service.accord.serializers.CommandSerializers; +import org.apache.cassandra.service.accord.serializers.KeySerializers; +import org.apache.cassandra.service.accord.serializers.ReadDataSerializers; +import org.apache.cassandra.service.accord.serializers.ReadDataSerializers.ReadDataSerializer; + +public class AccordInteropRead extends AbstractExecute +{ + public static final IVersionedSerializer requestSerializer = new ReadDataSerializer() + { + @Override + public void serialize(AccordInteropRead read, DataOutputPlus out, int version) throws IOException + { + CommandSerializers.txnId.serialize(read.txnId, out, version); + KeySerializers.participants.serialize(read.readScope, out, version); + out.writeUnsignedVInt(read.waitForEpoch()); + out.writeUnsignedVInt(read.executeAtEpoch - read.waitForEpoch()); + SinglePartitionReadCommand.serializer.serialize(read.command, out, version); + } + + @Override + public AccordInteropRead deserialize(DataInputPlus in, int version) throws IOException + { + TxnId txnId = CommandSerializers.txnId.deserialize(in, version); + Participants readScope = KeySerializers.participants.deserialize(in, version); + long waitForEpoch = in.readUnsignedVInt(); + long executeAtEpoch = in.readUnsignedVInt() + waitForEpoch; + SinglePartitionReadCommand command = (SinglePartitionReadCommand) SinglePartitionReadCommand.serializer.deserialize(in, version); + return new AccordInteropRead(txnId, readScope, waitForEpoch, executeAtEpoch, command); + } + + @Override + public long serializedSize(AccordInteropRead read, int version) + { + return CommandSerializers.txnId.serializedSize(read.txnId, version) + + KeySerializers.participants.serializedSize(read.readScope, version) + + TypeSizes.sizeofUnsignedVInt(read.waitForEpoch()) + + TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch - read.waitForEpoch()) + + SinglePartitionReadCommand.serializer.serializedSize(read.command, version); + } + }; + + public static final IVersionedSerializer replySerializer = new ReadDataSerializers.ReplySerializer<>(LocalReadData.serializer); + + private static class LocalReadData implements Data + { + static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(LocalReadData data, DataOutputPlus out, int version) throws IOException + { + ReadResponse.serializer.serialize(data.response, out, version); + } + + @Override + public LocalReadData deserialize(DataInputPlus in, int version) throws IOException + { + return new LocalReadData(ReadResponse.serializer.deserialize(in, version)); + } + + @Override + public long serializedSize(LocalReadData data, int version) + { + return ReadResponse.serializer.serializedSize(data.response, version); + } + }; + + final ReadResponse response; + + public LocalReadData(ReadResponse response) + { + this.response = response; + } + + @Override + public String toString() + { + return "LocalReadData{" + response + '}'; + } + + @Override + public Data merge(Data data) + { + throw new IllegalStateException("Should only ever be a single partition"); + } + } + + static class ReadCallback extends AccordInteropReadCallback + { + public ReadCallback(Node.Id id, InetAddressAndPort endpoint, Message message, RequestCallback wrapped, MaximalCommitSender maximalCommitSender) + { + super(id, endpoint, message, wrapped, maximalCommitSender); + } + + @Override + ReadResponse convertResponse(ReadOk ok) + { + return ((LocalReadData) ok.data).response; + } + } + + private final SinglePartitionReadCommand command; + + public AccordInteropRead(Node.Id to, Topologies topologies, TxnId txnId, Participants readScope, Timestamp executeAt, SinglePartitionReadCommand command) + { + super(to, topologies, txnId, readScope, executeAt); + this.command = command; + } + + public AccordInteropRead(TxnId txnId, Participants readScope, long executeAtEpoch, long waitForEpoch, SinglePartitionReadCommand command) + { + super(txnId, readScope, executeAtEpoch, waitForEpoch); + this.command = command; + } + + @Override + protected AsyncChain execute(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn) + { + return AsyncChains.ofCallable(Stage.READ.executor(), () -> new LocalReadData(ReadCommandVerbHandler.instance.doRead(command, false))); + } + + @Override + protected boolean canExecutePreApplied() + { + return true; + } + + @Override + protected ReadOk constructReadOk(Ranges unavailable, Data data) + { + return new InteropReadOk(unavailable, data); + } + + @Override + public MessageType type() + { + return AccordMessageType.INTEROP_READ_REQ; + } + + @Override + public String toString() + { + return "AccordInteropRead{" + + "txnId=" + txnId + + "command=" + command + + '}'; + } + + private static class InteropReadOk extends ReadOk + { + public InteropReadOk(@Nullable Ranges unavailable, @Nullable Data data) + { + super(unavailable, data); + } + + @Override + public MessageType type() + { + return AccordMessageType.INTEROP_READ_RSP; + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java new file mode 100644 index 0000000000..6bf006b3a8 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.interop; + +import javax.annotation.Nonnull; + +import accord.local.Node; +import accord.messages.Callback; +import accord.messages.ReadData.ReadOk; +import accord.messages.ReadData.ReadReply; +import accord.utils.Invariants; +import org.apache.cassandra.exceptions.RequestFailure; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.RequestCallback; + +import static accord.messages.ReadData.ReadNack.NotCommitted; + +public abstract class AccordInteropReadCallback implements Callback +{ + interface MaximalCommitSender + { + void sendMaximalCommit(@Nonnull Node.Id to); + } + + private final Node.Id id; + private final InetAddressAndPort endpoint; + private final Message message; + private final RequestCallback wrapped; + private final MaximalCommitSender maximalCommitSender; + + public AccordInteropReadCallback(Node.Id id, InetAddressAndPort endpoint, Message message, RequestCallback wrapped, MaximalCommitSender maximalCommitSender) + { + this.id = id; + this.message = message; + this.endpoint = endpoint; + this.wrapped = wrapped; + this.maximalCommitSender = maximalCommitSender; + } + + abstract T convertResponse(ReadOk ok); + + public void onSuccess(Node.Id from, ReadReply reply) + { + Invariants.checkArgument(from.equals(id)); + if (reply.isOk()) + { + wrapped.onResponse(message.responseWith(convertResponse((ReadOk) reply)).withFrom(endpoint)); + } + else if (reply == NotCommitted) + { + // Might still send a response if we send a maximal commit. Accord would tryAlternative and send + // both the commit and an additional repair, but Cassandra doesn't have tryAlternative unless we add + // it and instead opts to trigger additional repair messages based on time. + maximalCommitSender.sendMaximalCommit(id); + } + else + { + wrapped.onFailure(endpoint, RequestFailure.UNKNOWN); + } + } + + public void onFailure(Node.Id from, Throwable failure) + { + wrapped.onFailure(endpoint, RequestFailure.UNKNOWN); + } + + public void onCallbackFailure(Node.Id from, Throwable failure) + { + wrapped.onFailure(endpoint, RequestFailure.UNKNOWN); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadRepair.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadRepair.java new file mode 100644 index 0000000000..c16c99e33e --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadRepair.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.interop; + +import java.io.IOException; +import javax.annotation.Nullable; + +import accord.api.Data; +import accord.local.Node; +import accord.local.SafeCommandStore; +import accord.messages.AbstractExecute; +import accord.messages.MessageType; +import accord.primitives.PartialTxn; +import accord.primitives.Participants; +import accord.primitives.Ranges; +import accord.primitives.Timestamp; +import accord.primitives.TxnId; +import accord.topology.Topologies; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.ReadRepairVerbHandler; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.NoPayload; +import org.apache.cassandra.net.RequestCallback; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType; +import org.apache.cassandra.service.accord.serializers.CommandSerializers; +import org.apache.cassandra.service.accord.serializers.KeySerializers; +import org.apache.cassandra.service.accord.serializers.ReadDataSerializers; +import org.apache.cassandra.service.accord.serializers.ReadDataSerializers.ReadDataSerializer; + +/** + * Applies a read repair mutation from inside the context of a CommandStore via AbstractExecute + * ensuring that the contents of the read repair consist of data that isn't from transactions that + * haven't been committed yet at this command store. + */ +public class AccordInteropReadRepair extends AbstractExecute +{ + public static final IVersionedSerializer requestSerializer = new ReadDataSerializer() + { + @Override + public void serialize(AccordInteropReadRepair repair, DataOutputPlus out, int version) throws IOException + { + CommandSerializers.txnId.serialize(repair.txnId, out, version); + KeySerializers.participants.serialize(repair.readScope, out, version); + out.writeUnsignedVInt(repair.waitForEpoch()); + out.writeUnsignedVInt(repair.executeAtEpoch - repair.waitForEpoch()); + Mutation.serializer.serialize(repair.mutation, out, version); + } + + @Override + public AccordInteropReadRepair deserialize(DataInputPlus in, int version) throws IOException + { + TxnId txnId = CommandSerializers.txnId.deserialize(in, version); + Participants readScope = KeySerializers.participants.deserialize(in, version); + long waitForEpoch = in.readUnsignedVInt(); + long executeAtEpoch = in.readUnsignedVInt() + waitForEpoch; + Mutation mutation = Mutation.serializer.deserialize(in, version); + return new AccordInteropReadRepair(txnId, readScope, waitForEpoch, executeAtEpoch, mutation); + } + + @Override + public long serializedSize(AccordInteropReadRepair repair, int version) + { + return CommandSerializers.txnId.serializedSize(repair.txnId, version) + + KeySerializers.participants.serializedSize(repair.readScope, version) + + TypeSizes.sizeofUnsignedVInt(repair.waitForEpoch()) + + TypeSizes.sizeofUnsignedVInt(repair.executeAtEpoch - repair.waitForEpoch()) + + Mutation.serializer.serializedSize(repair.mutation, version); + } + }; + + static class ReadRepairCallback extends AccordInteropReadCallback + { + public ReadRepairCallback(Node.Id id, InetAddressAndPort endpoint, Message message, RequestCallback wrapped, MaximalCommitSender maximalCommitSender) + { + super(id, endpoint, message, wrapped, maximalCommitSender); + } + + @Override + Object convertResponse(ReadOk ok) + { + return NoPayload.noPayload; + } + } + + private final Mutation mutation; + + private static final IVersionedSerializer noop_data_serializer = new IVersionedSerializer() + { + @Override + public void serialize(Data t, DataOutputPlus out, int version) throws IOException {} + @Override + public Data deserialize(DataInputPlus in, int version) throws IOException { return Data.NOOP_DATA; } + + public long serializedSize(Data t, int version) { return 0; } + }; + + public static final IVersionedSerializer replySerializer = new ReadDataSerializers.ReplySerializer<>(noop_data_serializer); + + public AccordInteropReadRepair(Node.Id to, Topologies topologies, TxnId txnId, Participants readScope, Timestamp executeAt, Mutation mutation) + { + super(to, topologies, txnId, readScope, executeAt); + this.mutation = mutation; + } + + public AccordInteropReadRepair(TxnId txnId, Participants readScope, long executeAtEpoch, long waitForEpoch, Mutation mutation) + { + // TODO (review): remove followup read - Is there anything left to be done for this or can I remove it? + super(txnId, readScope, executeAtEpoch, waitForEpoch); + this.mutation = mutation; + } + + @Override + protected AsyncChain execute(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn) + { + return AsyncChains.ofCallable(Verb.READ_REPAIR_REQ.stage.executor(), () -> { + ReadRepairVerbHandler.instance.applyMutation(mutation); + return Data.NOOP_DATA; + }); + } + + @Override + protected boolean canExecutePreApplied() + { + return true; + } + + @Override + protected boolean executeIfObsoleted() + { + return true; + } + + @Override + protected ReadOk constructReadOk(Ranges unavailable, Data data) + { + return new InteropReadRepairOk(unavailable, data); + } + + @Override + public MessageType type() + { + return AccordMessageType.INTEROP_READ_REPAIR_REQ; + } + + private static class InteropReadRepairOk extends ReadOk + { + public InteropReadRepairOk(@Nullable Ranges unavailable, @Nullable Data data) + { + super(unavailable, data); + } + + @Override + public MessageType type() + { + return AccordMessageType.INTEROP_READ_REPAIR_RSP; + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/serializers/ApplySerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/ApplySerializers.java index e8ccc6258f..4decbdf1f8 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/ApplySerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/ApplySerializers.java @@ -22,19 +22,25 @@ import java.io.IOException; import accord.api.Result; import accord.messages.Apply; +import accord.primitives.PartialDeps; import accord.primitives.PartialRoute; +import accord.primitives.PartialTxn; +import accord.primitives.Seekables; +import accord.primitives.Timestamp; import accord.primitives.TxnId; +import accord.primitives.Writes; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; + public class ApplySerializers { - public static final IVersionedSerializer request = new TxnRequestSerializer() + public abstract static class ApplySerializer extends TxnRequestSerializer { @Override - public void serializeBody(Apply apply, DataOutputPlus out, int version) throws IOException + public void serializeBody(A apply, DataOutputPlus out, int version) throws IOException { out.writeBoolean(apply.kind == Apply.Kind.Maximal); KeySerializers.seekables.serialize(apply.keys(), out, version); @@ -44,21 +50,24 @@ public class ApplySerializers CommandSerializers.writes.serialize(apply.writes, out, version); } + protected abstract A deserializeApply(TxnId txnId, PartialRoute scope, long waitForEpoch, Apply.Kind kind, Seekables keys, + Timestamp executeAt, PartialDeps deps, PartialTxn txn, Writes writes, Result result); + @Override - public Apply deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute scope, long waitForEpoch) throws IOException + public A deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute scope, long waitForEpoch) throws IOException { - return Apply.SerializationSupport.create(txnId, scope, waitForEpoch, - in.readBoolean() ? Apply.Kind.Maximal : Apply.Kind.Minimal, - KeySerializers.seekables.deserialize(in, version), - CommandSerializers.timestamp.deserialize(in, version), - DepsSerializer.partialDeps.deserialize(in, version), - CommandSerializers.nullablePartialTxn.deserialize(in, version), - CommandSerializers.writes.deserialize(in, version), - Result.APPLIED); + return deserializeApply(txnId, scope, waitForEpoch, + in.readBoolean() ? Apply.Kind.Maximal : Apply.Kind.Minimal, + KeySerializers.seekables.deserialize(in, version), + CommandSerializers.timestamp.deserialize(in, version), + DepsSerializer.partialDeps.deserialize(in, version), + CommandSerializers.nullablePartialTxn.deserialize(in, version), + CommandSerializers.writes.deserialize(in, version), + Result.APPLIED); } @Override - public long serializedBodySize(Apply apply, int version) + public long serializedBodySize(A apply, int version) { return TypeSizes.BOOL_SIZE + KeySerializers.seekables.serializedSize(apply.keys(), version) @@ -67,6 +76,16 @@ public class ApplySerializers + CommandSerializers.nullablePartialTxn.serializedSize(apply.txn, version) + CommandSerializers.writes.serializedSize(apply.writes, version); } + } + + public static final IVersionedSerializer request = new ApplySerializer() + { + @Override + protected Apply deserializeApply(TxnId txnId, PartialRoute scope, long waitForEpoch, Apply.Kind kind, Seekables keys, + Timestamp executeAt, PartialDeps deps, PartialTxn txn, Writes writes, Result result) + { + return Apply.SerializationSupport.create(txnId, scope, waitForEpoch, kind, keys, executeAt, deps, txn, writes, result); + } }; public static final IVersionedSerializer reply = new IVersionedSerializer() diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CheckStatusSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CheckStatusSerializers.java index 74ed2eb703..734186ea0b 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/CheckStatusSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/CheckStatusSerializers.java @@ -173,7 +173,7 @@ public class CheckStatusSerializers size += KeySerializers.ranges.serializedSize(ok.truncated, version); size += CommandSerializers.status.serializedSize(ok.invalidIfNotAtLeast, version); size += CommandSerializers.saveStatus.serializedSize(ok.saveStatus, version); - size += CommandSerializers.saveStatus.serializedSize(ok.saveStatus, version); + size += CommandSerializers.saveStatus.serializedSize(ok.maxSaveStatus, version); size += CommandSerializers.ballot.serializedSize(ok.promised, version); size += CommandSerializers.ballot.serializedSize(ok.accepted, version); size += CommandSerializers.nullableTimestamp.serializedSize(ok.executeAt, version); diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java index 77f414e0cd..0c15515206 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java @@ -43,9 +43,9 @@ import org.apache.cassandra.db.marshal.ValueAccessor; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.service.accord.txn.AccordUpdate; import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; -import org.apache.cassandra.service.accord.txn.TxnUpdate; import org.apache.cassandra.service.accord.txn.TxnWrite; import org.apache.cassandra.utils.CastingSerializer; import org.apache.cassandra.utils.NullableSerializer; @@ -181,7 +181,7 @@ public class CommandSerializers private static final IVersionedSerializer read = new CastingSerializer<>(TxnRead.class, TxnRead.serializer); private static final IVersionedSerializer query = new CastingSerializer<>(TxnQuery.class, TxnQuery.serializer); - private static final IVersionedSerializer update = new CastingSerializer<>(TxnUpdate.class, TxnUpdate.serializer); + private static final IVersionedSerializer update = new CastingSerializer<>(AccordUpdate.class, AccordUpdate.serializer); public static final IVersionedSerializer partialTxn = new PartialTxnSerializer(read, query, update); public static final IVersionedSerializer nullablePartialTxn = NullableSerializer.wrap(partialTxn); diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CommitSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CommitSerializers.java index c2ea5e6a24..e67368d00d 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/CommitSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/CommitSerializers.java @@ -19,15 +19,22 @@ package org.apache.cassandra.service.accord.serializers; import java.io.IOException; +import javax.annotation.Nullable; import accord.messages.Commit; +import accord.messages.ReadData; +import accord.primitives.FullRoute; +import accord.primitives.PartialDeps; import accord.primitives.PartialRoute; +import accord.primitives.PartialTxn; +import accord.primitives.Timestamp; import accord.primitives.TxnId; import accord.primitives.Unseekables; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.CastingSerializer; import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable; import static org.apache.cassandra.utils.NullableSerializer.serializeNullable; @@ -35,41 +42,61 @@ import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSi public class CommitSerializers { - public static final IVersionedSerializer request = new TxnRequestSerializer() + public abstract static class CommitSerializer extends TxnRequestSerializer { + private final IVersionedSerializer read; + + public CommitSerializer(Class klass, IVersionedSerializer read) + { + this.read = new CastingSerializer<>(klass, read); + } + @Override - public void serializeBody(Commit msg, DataOutputPlus out, int version) throws IOException + public void serializeBody(C msg, DataOutputPlus out, int version) throws IOException { out.writeBoolean(msg.kind == Commit.Kind.Maximal); CommandSerializers.timestamp.serialize(msg.executeAt, out, version); CommandSerializers.nullablePartialTxn.serialize(msg.partialTxn, out, version); DepsSerializer.partialDeps.serialize(msg.partialDeps, out, version); serializeNullable(msg.route, out, version, KeySerializers.fullRoute); - serializeNullable(msg.read, out, version, ReadDataSerializers.request); + serializeNullable(msg.readData, out, version, read); } + protected abstract C deserializeCommit(TxnId txnId, PartialRoute scope, long waitForEpoch, Commit.Kind kind, Timestamp executeAt, + @Nullable PartialTxn partialTxn, PartialDeps partialDeps, + @Nullable FullRoute fullRoute, @Nullable ReadData read); + @Override - public Commit deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute scope, long waitForEpoch) throws IOException + public C deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute scope, long waitForEpoch) throws IOException { - return Commit.SerializerSupport.create(txnId, scope, waitForEpoch, - in.readBoolean() ? Commit.Kind.Maximal : Commit.Kind.Minimal, - CommandSerializers.timestamp.deserialize(in, version), - CommandSerializers.nullablePartialTxn.deserialize(in, version), - DepsSerializer.partialDeps.deserialize(in, version), - deserializeNullable(in, version, KeySerializers.fullRoute), - deserializeNullable(in, version, ReadDataSerializers.request) + return deserializeCommit(txnId, scope, waitForEpoch, + in.readBoolean() ? Commit.Kind.Maximal : Commit.Kind.Minimal, + CommandSerializers.timestamp.deserialize(in, version), + CommandSerializers.nullablePartialTxn.deserialize(in, version), + DepsSerializer.partialDeps.deserialize(in, version), + deserializeNullable(in, version, KeySerializers.fullRoute), + deserializeNullable(in, version, read) ); } @Override - public long serializedBodySize(Commit msg, int version) + public long serializedBodySize(C msg, int version) { return TypeSizes.BOOL_SIZE + CommandSerializers.timestamp.serializedSize(msg.executeAt, version) + CommandSerializers.nullablePartialTxn.serializedSize(msg.partialTxn, version) + DepsSerializer.partialDeps.serializedSize(msg.partialDeps, version) + serializedNullableSize(msg.route, version, KeySerializers.fullRoute) - + serializedNullableSize(msg.read, version, ReadDataSerializers.request); + + serializedNullableSize(msg.readData, version, read); + } + } + + public static final IVersionedSerializer request = new CommitSerializer(ReadData.class, ReadDataSerializers.readData) + { + @Override + protected Commit deserializeCommit(TxnId txnId, PartialRoute scope, long waitForEpoch, Commit.Kind kind, Timestamp executeAt, @Nullable PartialTxn partialTxn, PartialDeps partialDeps, @Nullable FullRoute fullRoute, @Nullable ReadData read) + { + return Commit.SerializerSupport.create(txnId, scope, waitForEpoch, kind, executeAt, partialTxn, partialDeps, fullRoute, read); } }; diff --git a/src/java/org/apache/cassandra/service/accord/serializers/InformHomeDurableSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/InformHomeDurableSerializers.java index c6a349028b..50f53a04f6 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/InformHomeDurableSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/InformHomeDurableSerializers.java @@ -41,7 +41,6 @@ public class InformHomeDurableSerializers CommandSerializers.timestamp.serialize(inform.executeAt, out, version); CommandSerializers.durability.serialize(inform.durability, out, version); serializeCollection(inform.persistedOn, out, version, TopologySerializers.nodeId); - } @Override diff --git a/src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java index 52e534142a..8b102ae13d 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java @@ -50,8 +50,8 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.accord.TokenRange; -import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.AccordRoutingKey; +import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.utils.NullableSerializer; public class KeySerializers diff --git a/src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java index e9705d3198..db7a4f7bf5 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java @@ -20,9 +20,13 @@ package org.apache.cassandra.service.accord.serializers; import java.io.IOException; +import accord.api.Data; +import accord.messages.ApplyThenWaitUntilApplied; +import accord.messages.ReadData; import accord.messages.ReadData.ReadNack; import accord.messages.ReadData.ReadOk; import accord.messages.ReadData.ReadReply; +import accord.messages.ReadData.ReadType; import accord.messages.ReadTxnData; import accord.messages.WaitUntilApplied; import accord.primitives.Participants; @@ -34,14 +38,80 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.service.accord.txn.TxnResult; +import static org.apache.cassandra.db.TypeSizes.sizeof; import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable; import static org.apache.cassandra.utils.NullableSerializer.serializeNullable; import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSize; public class ReadDataSerializers { - public static final IVersionedSerializer request = new IVersionedSerializer() + public static final IVersionedSerializer readData = new IVersionedSerializer() + { + @Override + public void serialize(ReadData t, DataOutputPlus out, int version) throws IOException + { + out.writeByte(t.kind().val); + serializerFor(t).serialize(t, out, version); + } + + @Override + public ReadData deserialize(DataInputPlus in, int version) throws IOException + { + return serializerFor(ReadType.valueOf(in.readByte())).deserialize(in, version); + } + + @Override + public long serializedSize(ReadData t, int version) + { + return sizeof(t.kind().val) + serializerFor(t).serializedSize(t, version); + } + }; + + private static final ApplyThenWaitUntilAppliedSerializer applyThenWaitUntilApplied = new ApplyThenWaitUntilAppliedSerializer(); + + private static class ApplyThenWaitUntilAppliedSerializer implements ReadDataSerializer + { + @Override + public void serialize(ApplyThenWaitUntilApplied applyThenWaitUntilApplied, DataOutputPlus out, int version) throws IOException + { + CommandSerializers.txnId.serialize(applyThenWaitUntilApplied.txnId, out, version); + KeySerializers.partialRoute.serialize(applyThenWaitUntilApplied.route, out, version); + DepsSerializer.partialDeps.serialize(applyThenWaitUntilApplied.deps, out, version); + KeySerializers.seekables.serialize(applyThenWaitUntilApplied.partialTxnKeys, out, version); + CommandSerializers.writes.serialize(applyThenWaitUntilApplied.writes, out, version); + TxnResult.serializer.serialize((TxnResult) applyThenWaitUntilApplied.txnResult, out, version); + out.writeBoolean(applyThenWaitUntilApplied.notifyAgent); + } + + @Override + public ApplyThenWaitUntilApplied deserialize(DataInputPlus in, int version) throws IOException + { + return ApplyThenWaitUntilApplied.SerializerSupport.create( + CommandSerializers.txnId.deserialize(in, version), + KeySerializers.partialRoute.deserialize(in, version), + DepsSerializer.partialDeps.deserialize(in, version), + KeySerializers.seekables.deserialize(in, version), + CommandSerializers.writes.deserialize(in, version), + TxnResult.serializer.deserialize(in, version), + in.readBoolean()); + } + + @Override + public long serializedSize(ApplyThenWaitUntilApplied applyThenWaitUntilApplied, int version) + { + return CommandSerializers.txnId.serializedSize(applyThenWaitUntilApplied.txnId, version) + + KeySerializers.partialRoute.serializedSize(applyThenWaitUntilApplied.route, version) + + DepsSerializer.partialDeps.serializedSize(applyThenWaitUntilApplied.deps, version) + + KeySerializers.seekables.serializedSize(applyThenWaitUntilApplied.partialTxnKeys, version) + + CommandSerializers.writes.serializedSize(applyThenWaitUntilApplied.writes, version) + + TxnResult.serializer.serializedSize((TxnData)applyThenWaitUntilApplied.txnResult, version) + + sizeof(applyThenWaitUntilApplied.notifyAgent); + } + } + + private static final ReadDataSerializer readTxnData = new ReadDataSerializer() { @Override public void serialize(ReadTxnData read, DataOutputPlus out, int version) throws IOException @@ -72,10 +142,43 @@ public class ReadDataSerializers } }; - public static final IVersionedSerializer reply = new IVersionedSerializer() + public interface ReadDataSerializer extends IVersionedSerializer + { + void serialize(T bound, DataOutputPlus out, int version) throws IOException; + T deserialize(DataInputPlus in, int version) throws IOException; + long serializedSize(T condition, int version); + } + + private static ReadDataSerializer serializerFor(ReadData toSerialize) + { + return serializerFor(toSerialize.kind()); + } + + private static ReadDataSerializer serializerFor(ReadType type) + { + switch (type) + { + case readTxnData: + return readTxnData; + case applyThenWaitUntilApplied: + return applyThenWaitUntilApplied; + case waitUntilApplied: + return waitUntilApplied; + default: + throw new IllegalStateException("Unsupported ExecuteType " + type); + } + } + + public static final class ReplySerializer implements IVersionedSerializer { // TODO (now): use something other than ordinal final ReadNack[] nacks = ReadNack.values(); + private final IVersionedSerializer dataSerializer; + + public ReplySerializer(IVersionedSerializer dataSerializer) + { + this.dataSerializer = dataSerializer; + } @Override public void serialize(ReadReply reply, DataOutputPlus out, int version) throws IOException @@ -89,7 +192,7 @@ public class ReadDataSerializers out.writeByte(0); ReadOk readOk = (ReadOk) reply; serializeNullable(readOk.unavailable, out, version, KeySerializers.ranges); - TxnData.nullableSerializer.serialize((TxnData) readOk.data, out, version); + dataSerializer.serialize((D) readOk.data, out, version); } @Override @@ -100,7 +203,7 @@ public class ReadDataSerializers return nacks[id - 1]; Ranges ranges = deserializeNullable(in, version, KeySerializers.ranges); - TxnData data = TxnData.nullableSerializer.deserialize(in, version); + D data = dataSerializer.deserialize(in, version); return new ReadOk(ranges, data); } @@ -113,20 +216,22 @@ public class ReadDataSerializers ReadOk readOk = (ReadOk) reply; return TypeSizes.BYTE_SIZE + serializedNullableSize(readOk.unavailable, version, KeySerializers.ranges) - + TxnData.nullableSerializer.serializedSize((TxnData) readOk.data, version); + + dataSerializer.serializedSize((D) readOk.data, version); } - }; + } + + public static final IVersionedSerializer reply = new ReplySerializer<>(TxnData.nullableSerializer); // TODO (consider): duplicates ReadTxnData ser/de logic; conside deduplicating if another instance of this is added - public static final IVersionedSerializer waitOnApply = new IVersionedSerializer() + public static final ReadDataSerializer waitUntilApplied = new ReadDataSerializer() { @Override - public void serialize(WaitUntilApplied msg, DataOutputPlus out, int version) throws IOException + public void serialize(WaitUntilApplied waitUntilApplied, DataOutputPlus out, int version) throws IOException { - CommandSerializers.txnId.serialize(msg.txnId, out, version); - KeySerializers.participants.serialize(msg.readScope, out, version); - out.writeUnsignedVInt(msg.waitForEpoch()); - CommandSerializers.timestamp.serialize(msg.executeAt, out , version); + CommandSerializers.txnId.serialize(waitUntilApplied.txnId, out, version); + KeySerializers.participants.serialize(waitUntilApplied.readScope, out, version); + out.writeUnsignedVInt(waitUntilApplied.waitForEpoch()); + CommandSerializers.timestamp.serialize(waitUntilApplied.executeAt, out , version); } @Override @@ -140,12 +245,12 @@ public class ReadDataSerializers } @Override - public long serializedSize(WaitUntilApplied msg, int version) + public long serializedSize(WaitUntilApplied waitUntilApplied, int version) { - return CommandSerializers.txnId.serializedSize(msg.txnId, version) - + KeySerializers.participants.serializedSize(msg.readScope, version) - + TypeSizes.sizeofUnsignedVInt(msg.waitForEpoch()) - + CommandSerializers.timestamp.serializedSize(msg.executeAt, version); + return CommandSerializers.txnId.serializedSize(waitUntilApplied.txnId, version) + + KeySerializers.participants.serializedSize(waitUntilApplied.readScope, version) + + TypeSizes.sizeofUnsignedVInt(waitUntilApplied.waitForEpoch()) + + CommandSerializers.timestamp.serializedSize(waitUntilApplied.executeAt, version); } }; } diff --git a/src/java/org/apache/cassandra/service/accord/serializers/SetDurableSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/SetDurableSerializers.java index 1b55252d24..e9cabbbd15 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/SetDurableSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/SetDurableSerializers.java @@ -23,9 +23,10 @@ import accord.api.RoutingKey; import accord.messages.SetGloballyDurable; import accord.messages.SetShardDurable; import accord.primitives.Deps; -import accord.primitives.Ranges; +import accord.primitives.Seekables; import accord.primitives.SyncPoint; import accord.primitives.TxnId; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -81,8 +82,9 @@ public class SetDurableSerializers { CommandSerializers.txnId.serialize(sp.syncId, out, version); DepsSerializer.deps.serialize(sp.waitFor, out, version); - KeySerializers.ranges.serialize(sp.ranges, out, version); + KeySerializers.seekables.serialize(sp.keysOrRanges, out, version); KeySerializers.routingKey.serialize(sp.homeKey, out, version); + out.writeBoolean(sp.finishedAsync); } @Override @@ -90,18 +92,20 @@ public class SetDurableSerializers { TxnId syncId = CommandSerializers.txnId.deserialize(in, version); Deps waitFor = DepsSerializer.deps.deserialize(in, version); - Ranges ranges = KeySerializers.ranges.deserialize(in, version); + Seekables keysOrRanges = KeySerializers.seekables.deserialize(in, version); RoutingKey homeKey = KeySerializers.routingKey.deserialize(in, version); - return SyncPoint.SerializationSupport.construct(syncId, waitFor, ranges, homeKey); + boolean finishedAsync = in.readBoolean(); + return SyncPoint.SerializationSupport.construct(syncId, waitFor, keysOrRanges, homeKey, finishedAsync); } @Override public long serializedSize(SyncPoint sp, int version) { return CommandSerializers.txnId.serializedSize(sp.syncId, version) - + DepsSerializer.deps.serializedSize(sp.waitFor, version) - + KeySerializers.ranges.serializedSize(sp.ranges, version) - + KeySerializers.routingKey.serializedSize(sp.homeKey, version); + + DepsSerializer.deps.serializedSize(sp.waitFor, version) + + KeySerializers.seekables.serializedSize(sp.keysOrRanges, version) + + KeySerializers.routingKey.serializedSize(sp.homeKey, version) + + TypeSizes.sizeof(sp.finishedAsync); } }; } diff --git a/src/java/org/apache/cassandra/service/accord/txn/AccordUpdate.java b/src/java/org/apache/cassandra/service/accord/txn/AccordUpdate.java new file mode 100644 index 0000000000..ae63f9d26e --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/txn/AccordUpdate.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.txn; + +import java.io.IOException; +import javax.annotation.Nullable; + +import accord.api.Data; +import accord.api.Update; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +public abstract class AccordUpdate implements Update +{ + public enum Kind + { + TXN(0), + UNRECOVERABLE_REPAIR(1), + NONE(2), + ; + + int val; + + Kind(int val) + { + this.val = val; + } + + public static Kind valueOf(int val) + { + switch(val) + { + case 0: + return TXN; + case 1: + return UNRECOVERABLE_REPAIR; + default: + throw new IllegalArgumentException("Unrecognized AccordUpdate.Kind value " + val); + } + } + } + + public static Kind kind(@Nullable Update update) + { + if (update == null) + return Kind.NONE; + return ((AccordUpdate)update).kind(); + } + + public boolean checkCondition(Data data) + { + throw new UnsupportedOperationException(); + } + + public abstract ConsistencyLevel cassandraCommitCL(); + + public abstract Kind kind(); + + public abstract long estimatedSizeOnHeap(); + + public interface AccordUpdateSerializer extends IVersionedSerializer + { + void serialize(T update, DataOutputPlus out, int version) throws IOException; + T deserialize(DataInputPlus in, int version) throws IOException; + long serializedSize(T update, int version); + } + + private static AccordUpdateSerializer serializerFor(AccordUpdate toSerialize) + { + return serializerFor(toSerialize.kind()); + } + + private static AccordUpdateSerializer serializerFor(Kind kind) + { + switch (kind) + { + case TXN: + return TxnUpdate.serializer; + case UNRECOVERABLE_REPAIR: + return UnrecoverableRepairUpdate.serializer; + default: + throw new IllegalStateException("Unsupported AccordUpdate Kind " + kind); + } + } + + public static final AccordUpdateSerializer serializer = new AccordUpdateSerializer() + { + @Override + public void serialize(AccordUpdate update, DataOutputPlus out, int version) throws IOException + { + out.writeByte(update.kind().val); + serializerFor(update).serialize(update, out, version); + } + + @Override + public AccordUpdate deserialize(DataInputPlus in, int version) throws IOException + { + Kind kind = Kind.valueOf(in.readByte()); + return serializerFor(kind).deserialize(in, version); + } + + @Override + public long serializedSize(AccordUpdate update, int version) + { + return 1 + serializerFor(update).serializedSize(update, version); + } + }; +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/service/accord/txn/AccordUpdateParameters.java b/src/java/org/apache/cassandra/service/accord/txn/AccordUpdateParameters.java index afe7e87e0a..e6a977955a 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/AccordUpdateParameters.java +++ b/src/java/org/apache/cassandra/service/accord/txn/AccordUpdateParameters.java @@ -31,6 +31,8 @@ import org.apache.cassandra.db.partitions.Partition; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; +import static com.google.common.base.Preconditions.checkState; + public class AccordUpdateParameters { private final TxnData data; @@ -47,7 +49,7 @@ public class AccordUpdateParameters return data; } - public UpdateParameters updateParameters(TableMetadata metadata, int rowIndex) + public UpdateParameters updateParameters(TableMetadata metadata, DecoratedKey dk, int rowIndex) { // This is currently only used by Guardrails, but this logically have issues with Accord as drifts in config // values could cause unexpected issues in Accord. (ex. some nodes reject writes while others accept) @@ -67,16 +69,24 @@ public class AccordUpdateParameters timestamp, nowInSeconds, ttl, - prefetchRow(metadata, rowIndex)); + prefetchRow(metadata, dk, rowIndex)); } - private Map prefetchRow(TableMetadata metadata, int index) + private Map prefetchRow(TableMetadata metadata, DecoratedKey dk, int index) { for (Map.Entry e : data.entrySet()) { TxnDataName name = e.getKey(); - if (name.isAutoRead() && name.atIndex(index)) - return ImmutableMap.of(name.getDecoratedKey(metadata), e.getValue()); + switch (name.getKind()) + { + case CAS_READ: + checkState(data.entrySet().size() == 1, "CAS read should only have one entry"); + return ImmutableMap.of(dk, e.getValue()); + case AUTO_READ: + if (name.atIndex(index)) + return ImmutableMap.of(name.getDecoratedKey(metadata), e.getValue()); + default: + } } return Collections.emptyMap(); } diff --git a/src/java/org/apache/cassandra/service/accord/txn/RetryWithNewProtocolResult.java b/src/java/org/apache/cassandra/service/accord/txn/RetryWithNewProtocolResult.java new file mode 100644 index 0000000000..d5c02a596b --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/txn/RetryWithNewProtocolResult.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.txn; + +import java.io.IOException; + +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.utils.ObjectSizes; + +/** + * Potentially returned by any transaction that tries to execute in an Epoch + * where the range has migrated away from Accord + */ +public class RetryWithNewProtocolResult extends TxnResult +{ + private static final long SIZE = ObjectSizes.measure(new RetryWithNewProtocolResult(Epoch.FIRST)); + + public final Epoch epoch; + + RetryWithNewProtocolResult(Epoch epoch) + { + this.epoch = epoch; + } + + @Override + public Kind kind() + { + return Kind.retry_new_protocol; + } + + @Override + public long estimatedSizeOnHeap() + { + return SIZE; + } + + public static final TxnResultSerializer serializer = new TxnResultSerializer() + { + @Override + public void serialize(RetryWithNewProtocolResult retry, DataOutputPlus out, int version) throws IOException + { + Epoch.messageSerializer.serialize(retry.epoch, out, version); + } + + @Override + public RetryWithNewProtocolResult deserialize(DataInputPlus in, int version) throws IOException + { + return new RetryWithNewProtocolResult(Epoch.messageSerializer.deserialize(in, version)); + } + + @Override + public long serializedSize(RetryWithNewProtocolResult retry, int version) + { + return Epoch.messageSerializer.serializedSize(retry.epoch, version); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnCondition.java b/src/java/org/apache/cassandra/service/accord/txn/TxnCondition.java index 905667d35c..46f59f0936 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnCondition.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnCondition.java @@ -51,9 +51,8 @@ import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; import static com.google.common.base.Preconditions.checkNotNull; - import static org.apache.cassandra.service.accord.AccordSerializers.clusteringSerializer; -import static org.apache.cassandra.service.accord.txn.TxnRead.SERIAL_READ; +import static org.apache.cassandra.service.accord.txn.TxnRead.CAS_READ; import static org.apache.cassandra.utils.CollectionSerializers.deserializeList; import static org.apache.cassandra.utils.CollectionSerializers.serializeCollection; import static org.apache.cassandra.utils.CollectionSerializers.serializeList; @@ -333,7 +332,7 @@ public abstract class TxnCondition public boolean applies(@Nonnull TxnData data) { checkNotNull(data); - FilteredPartition partition = data.get(SERIAL_READ); + FilteredPartition partition = data.get(CAS_READ); Row row = partition != null ? partition.getRow(clustering) : null; for (Bound bound : bounds) { diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnData.java b/src/java/org/apache/cassandra/service/accord/txn/TxnData.java index c3d8f6e18d..9c2ae88f83 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnData.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnData.java @@ -24,11 +24,15 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; +import com.google.common.collect.Maps; + import accord.api.Data; -import accord.api.Result; +import org.apache.cassandra.db.EmptyIterators; +import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.partitions.FilteredPartition; +import org.apache.cassandra.db.partitions.PartitionIterators; import org.apache.cassandra.db.rows.DeserializationHelper; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.UnfilteredRowIterator; @@ -43,7 +47,9 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.NullableSerializer; import org.apache.cassandra.utils.ObjectSizes; -public class TxnData implements Data, Result, Iterable +import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.txn_data; + +public class TxnData extends TxnResult implements Data, Iterable { private static final long EMPTY_SIZE = ObjectSizes.measure(new TxnData()); @@ -74,8 +80,13 @@ public class TxnData implements Data, Result, Iterable return data.entrySet(); } + public boolean isEmpty() + { + return data.isEmpty(); + } + @Override - public Data merge(Data data) + public TxnData merge(Data data) { TxnData that = (TxnData) data; TxnData merged = new TxnData(); @@ -94,6 +105,7 @@ public class TxnData implements Data, Result, Iterable return left.merge(right); } + @Override public long estimatedSizeOnHeap() { long size = EMPTY_SIZE; @@ -124,6 +136,14 @@ public class TxnData implements Data, Result, Iterable return data.equals(that.data); } + public static TxnData emptyPartition(TxnDataName name, SinglePartitionReadCommand command) + { + TxnData result = new TxnData(); + FilteredPartition empty = FilteredPartition.create(PartitionIterators.getOnlyElement(EmptyIterators.partition(), command)); + result.put(name, empty); + return result; + } + private static final IVersionedSerializer partitionSerializer = new IVersionedSerializer() { @Override @@ -158,7 +178,13 @@ public class TxnData implements Data, Result, Iterable } }; - public static final IVersionedSerializer serializer = new IVersionedSerializer() + @Override + public Kind kind() + { + return txn_data; + } + + public static final TxnResultSerializer serializer = new TxnResultSerializer() { @Override public void serialize(TxnData data, DataOutputPlus out, int version) throws IOException @@ -174,8 +200,8 @@ public class TxnData implements Data, Result, Iterable @Override public TxnData deserialize(DataInputPlus in, int version) throws IOException { - Map data = new HashMap<>(); - long size = in.readUnsignedVInt(); + int size = in.readUnsignedVInt32(); + Map data = Maps.newHashMapWithExpectedSize(size); for (int i=0; i { USER((byte) 1), RETURNING((byte) 2), - AUTO_READ((byte) 3); + AUTO_READ((byte) 3), + CAS_READ((byte) 4); private final byte value; @@ -73,6 +73,8 @@ public class TxnDataName implements Comparable return RETURNING; case 3: return AUTO_READ; + case 4: + return CAS_READ; default: throw new IllegalArgumentException("Unknown kind: " + b); } @@ -142,11 +144,6 @@ public class TxnDataName implements Comparable return Collections.unmodifiableList(Arrays.asList(parts)); } - public boolean isAutoRead() - { - return kind == Kind.AUTO_READ; - } - public DecoratedKey getDecoratedKey(TableMetadata metadata) { checkKind(Kind.AUTO_READ); diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java b/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java index acab7c89f7..df41e78b53 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java @@ -23,8 +23,10 @@ import java.nio.ByteBuffer; import java.util.Objects; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import accord.api.Data; -import accord.local.SafeCommandStore; import accord.primitives.Timestamp; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; @@ -49,6 +51,9 @@ import static org.apache.cassandra.utils.ByteBufferUtil.writeWithVIntLength; public class TxnNamedRead extends AbstractSerialized { + @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(TxnNamedRead.class); + private static final long EMPTY_SIZE = ObjectSizes.measure(new TxnNamedRead(null, null, null)); private final TxnDataName name; @@ -111,7 +116,7 @@ public class TxnNamedRead extends AbstractSerialized return key; } - public AsyncChain read(boolean isForWriteTxn, SafeCommandStore safeStore, Timestamp executeAt) + public AsyncChain read(Timestamp executeAt) { SinglePartitionReadCommand command = (SinglePartitionReadCommand) get(); // TODO (required, safety): before release, double check reasoning that this is safe @@ -121,7 +126,16 @@ public class TxnNamedRead extends AbstractSerialized // this simply looks like the transaction witnessed TTL'd data and the data then expired // immediately after the transaction executed, and this simplifies things a great deal int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(executeAt.hlc()); + return performLocalRead(command, nowInSeconds); + } + public ReadCommand command() + { + return get(); + } + + private AsyncChain performLocalRead(SinglePartitionReadCommand command, int nowInSeconds) + { return AsyncChains.ofCallable(Stage.READ.executor(), () -> { SinglePartitionReadCommand read = command.withNowInSec(nowInSeconds); diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java b/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java index 115864d830..6005673fa6 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java @@ -22,21 +22,33 @@ import java.io.IOException; import javax.annotation.Nullable; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import accord.api.Data; import accord.api.Query; import accord.api.Read; import accord.api.Result; import accord.api.Update; +import accord.primitives.Seekables; import accord.primitives.Timestamp; import accord.primitives.TxnId; +import org.apache.cassandra.config.Config.LWTStrategy; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.EmptyIterators; +import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.metrics.ClientRequestsMetricsHolder; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.utils.ObjectSizes; import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.cassandra.service.accord.txn.TxnRead.CAS_READ; public abstract class TxnQuery implements Query { @@ -49,7 +61,7 @@ public abstract class TxnQuery implements Query } @Override - public Result compute(TxnId txnId, Timestamp executeAt, Data data, @Nullable Read read, @Nullable Update update) + public Result doCompute(TxnId txnId, Timestamp executeAt, Seekables keys, @Nullable Data data, @Nullable Read read, @Nullable Update update) { return data != null ? (TxnData) data : new TxnData(); } @@ -64,7 +76,7 @@ public abstract class TxnQuery implements Query } @Override - public Result compute(TxnId txnId, Timestamp executeAt, Data data, @Nullable Read read, @Nullable Update update) + public Result doCompute(TxnId txnId, Timestamp executeAt, Seekables keys, @Nullable Data data, @Nullable Read read, @Nullable Update update) { return new TxnData(); } @@ -79,19 +91,52 @@ public abstract class TxnQuery implements Query } @Override - public Result compute(TxnId txnId, Timestamp executeAt, Data data, @Nullable Read read, Update update) + public Result doCompute(TxnId txnId, Timestamp executeAt, Seekables keys, @Nullable Data data, @Nullable Read read, Update update) { checkNotNull(txnId, "txnId should not be null"); checkNotNull(data, "data should not be null"); checkNotNull(update, "update should not be null"); - TxnUpdate txnUpdate = (TxnUpdate)update; - boolean conditionCheck = txnUpdate.checkCondition(data); + + AccordUpdate accordUpdate = (AccordUpdate)update; + TxnData txnData = (TxnData)data; + boolean conditionCheck = accordUpdate.checkCondition(data); // If the condition applied an empty result indicates success if (conditionCheck) return new TxnData(); + else if (txnData.isEmpty()) + { + TxnRead txnRead = (TxnRead)read; + SinglePartitionReadCommand command = (SinglePartitionReadCommand)txnRead.iterator().next().get(); + // For CAS must return a non-empty result to indicate error even if there was no partition found + return new TxnData(ImmutableMap.of(CAS_READ, FilteredPartition.create(EmptyIterators.row(command.metadata(), command.partitionKey(), command.isReversed())))); + } else - // If it failed to apply the partition contents (if present) are returned and it indicates failure - return (TxnData)data; + // If it failed to apply the partition contents are returned and it indicates failure + return ((TxnData)data); + } + }; + + public static final TxnQuery EMPTY = new TxnQuery() + { + + @Override + protected byte type() + { + return 4; + } + + @Override + public Result compute(TxnId txnId, Timestamp executeAt, Seekables keys, @Nullable Data data, @Nullable Read read, @Nullable Update update) + { + // Skip the migration checks in the base class for empty transactions, we don't + // want/need the RetryWithNewProtocolResult + return new TxnData(); + } + + @Override + protected Result doCompute(TxnId txnId, Timestamp executeAt, Seekables keys, @Nullable Data data, @Nullable Read read, @Nullable Update update) + { + throw new UnsupportedOperationException(); } }; @@ -101,6 +146,27 @@ public abstract class TxnQuery implements Query abstract protected byte type(); + abstract protected Result doCompute(TxnId txnId, Timestamp executeAt, Seekables keys, @Nullable Data data, @Nullable Read read, @Nullable Update update); + + @Override + public Result compute(TxnId txnId, Timestamp executeAt, Seekables keys, @Nullable Data data, @Nullable Read read, @Nullable Update update) + { + Epoch epoch = Epoch.create(executeAt.epoch()); + if (transactionIsInMigratingOrMigratedRange(epoch, keys)) + { + // Fail fast because we can't be sure where this request should really run or what was intended + if (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord) + throw new IllegalStateException("Mixing a hard coded strategy with migration is unsupported"); + + if (txnId.isWrite()) + ClientRequestsMetricsHolder.accordWriteMetrics.accordMigrationRejects.mark(); + else + ClientRequestsMetricsHolder.accordReadMetrics.accordMigrationRejects.mark(); + return new RetryWithNewProtocolResult(epoch); + } + return doCompute(txnId, executeAt, keys, data, read, update); + } + public long estimatedSizeOnHeap() { return SIZE; @@ -111,7 +177,7 @@ public abstract class TxnQuery implements Query @Override public void serialize(TxnQuery query, DataOutputPlus out, int version) throws IOException { - Preconditions.checkArgument(query == null || query == ALL || query == NONE || query == CONDITION); + Preconditions.checkArgument(query == null | query == ALL | query == NONE | query == CONDITION | query == EMPTY); out.writeByte(query == null ? 0 : query.type()); } @@ -125,14 +191,31 @@ public abstract class TxnQuery implements Query case 1: return ALL; case 2: return NONE; case 3: return CONDITION; + case 4: return EMPTY; } } @Override public long serializedSize(TxnQuery query, int version) { - Preconditions.checkArgument(query == null || query == ALL || query == NONE || query == CONDITION); + Preconditions.checkArgument(query == null | query == ALL | query == NONE | query == CONDITION | query == EMPTY); return TypeSizes.sizeof((byte)2); } }; + + private static boolean transactionIsInMigratingOrMigratedRange(Epoch epoch, Seekables keys) + { + // Whatever this transaction might be it isn't one supported for migration anyways + if (!keys.domain().isKey()) + return false; + + if (keys.size() > 1) + // It has to be a transaction statement and we don't support migration with those + return false; + // Could be a transaction statement, but this check does no additional harm + // and transaction statement will generate an error when it sees + // the RetryOnNewProtocolResult + PartitionKey partitionKey = (PartitionKey)keys.get(0); + return ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(epoch, partitionKey.tableId(), partitionKey.partitionKey()); + } } diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java index de50b96524..122336ad12 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import com.google.common.collect.ImmutableList; @@ -33,49 +35,89 @@ import accord.primitives.Keys; import accord.primitives.Ranges; import accord.primitives.Seekable; import accord.primitives.Timestamp; -import accord.primitives.Txn; import accord.utils.SortedArrays; import org.apache.cassandra.db.SinglePartitionReadCommand; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; +import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.serializers.KeySerializers; +import org.apache.cassandra.service.accord.txn.TxnDataName.Kind; import org.apache.cassandra.utils.ObjectSizes; +import org.apache.cassandra.utils.Simulate; import static accord.utils.SortedArrays.Search.CEIL; +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.cassandra.service.accord.AccordSerializers.consistencyLevelSerializer; +import static org.apache.cassandra.service.accord.IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS; import static org.apache.cassandra.utils.ArraySerializers.deserializeArray; import static org.apache.cassandra.utils.ArraySerializers.serializeArray; import static org.apache.cassandra.utils.ArraySerializers.serializedArraySize; +import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSize; +import static org.apache.cassandra.utils.Simulate.With.MONITORS; public class TxnRead extends AbstractKeySorted implements Read { // There is only potentially one partition in a CAS and SERIAL/LOCAL_SERIAL read public static final String SERIAL_READ_NAME = "SERIAL_READ"; public static final TxnDataName SERIAL_READ = TxnDataName.user(SERIAL_READ_NAME); - private static final long EMPTY_SIZE = ObjectSizes.measure(new TxnRead(new TxnNamedRead[0], null)); - public static final TxnRead EMPTY = new TxnRead(new TxnNamedRead[0], Keys.EMPTY); + public static final TxnRead EMPTY = new TxnRead(new TxnNamedRead[0], Keys.EMPTY, null); + private static final long EMPTY_SIZE = ObjectSizes.measure(EMPTY); + + public static final String CAS_READ_NAME = "CAS_READ"; + public static final TxnDataName CAS_READ = new TxnDataName(Kind.CAS_READ, CAS_READ_NAME); + + @Nonnull private final Keys txnKeys; - - public TxnRead(TxnNamedRead[] items, Keys txnKeys) + + // Cassandra's consistency level used by Accord to safely read data written outside of Accord + @Nullable + private final ConsistencyLevel cassandraConsistencyLevel; + + public TxnRead(@Nonnull TxnNamedRead[] items, @Nonnull Keys txnKeys, @Nullable ConsistencyLevel cassandraConsistencyLevel) { super(items); + checkArgument(cassandraConsistencyLevel == null || SUPPORTED_READ_CONSISTENCY_LEVELS.contains(cassandraConsistencyLevel), "Unsupported consistency level for read"); this.txnKeys = txnKeys; + this.cassandraConsistencyLevel = cassandraConsistencyLevel; } - public TxnRead(List items, Keys txnKeys) + public TxnRead(@Nonnull List items, @Nonnull Keys txnKeys, @Nullable ConsistencyLevel cassandraConsistencyLevel) { super(items); + checkArgument(cassandraConsistencyLevel == null || SUPPORTED_READ_CONSISTENCY_LEVELS.contains(cassandraConsistencyLevel), "Unsupported consistency level for read"); this.txnKeys = txnKeys; + this.cassandraConsistencyLevel = cassandraConsistencyLevel; } - public static TxnRead createSerialRead(SinglePartitionReadCommand readCommand) + public static TxnRead createTxnRead(@Nonnull List items, @Nonnull Keys txnKeys, @Nullable ConsistencyLevel consistencyLevel) + { + return new TxnRead(items, txnKeys, consistencyLevel); + } + + public static TxnRead createSerialRead(SinglePartitionReadCommand readCommand, ConsistencyLevel consistencyLevel) { TxnNamedRead read = new TxnNamedRead(SERIAL_READ, readCommand); - return new TxnRead(ImmutableList.of(read), Keys.of(read.key())); + return new TxnRead(ImmutableList.of(read), Keys.of(read.key()), consistencyLevel); + } + + public static TxnRead createCasRead(SinglePartitionReadCommand readCommand, ConsistencyLevel consistencyLevel) + { + TxnNamedRead read = new TxnNamedRead(CAS_READ, readCommand); + return new TxnRead(ImmutableList.of(read), Keys.of(read.key()), consistencyLevel); + } + + // A read that declares it will read from keys but doesn't actually read any data so dependent transactions will + // still be applied first + public static TxnRead createNoOpRead(Keys keys) + { + return new TxnRead(ImmutableList.of(), keys, null); } public long estimatedSizeOnHeap() @@ -110,9 +152,9 @@ public class TxnRead extends AbstractKeySorted implements Read return txnKeys; } - public Keys readKeys() + public ConsistencyLevel cassandraConsistencyLevel() { - return itemKeys; + return cassandraConsistencyLevel; } @Override @@ -125,7 +167,7 @@ public class TxnRead extends AbstractKeySorted implements Read if (keys.contains(read.key())) reads.add(read); - return new TxnRead(reads, txnKeys.slice(ranges)); + return createTxnRead(reads, txnKeys.slice(ranges), cassandraConsistencyLevel); } @Override @@ -138,7 +180,7 @@ public class TxnRead extends AbstractKeySorted implements Read if (!reads.contains(namedRead)) reads.add(namedRead); - return new TxnRead(reads, txnKeys.with((Keys)read.keys())); + return createTxnRead(reads, txnKeys.with((Keys)read.keys()), cassandraConsistencyLevel); } @Override @@ -158,12 +200,13 @@ public class TxnRead extends AbstractKeySorted implements Read } @Override - public AsyncChain read(Seekable key, Txn.Kind kind, SafeCommandStore safeStore, Timestamp executeAt, DataStore store) + public AsyncChain read(Seekable key, SafeCommandStore safeStore, Timestamp executeAt, DataStore store) { List> results = new ArrayList<>(); - forEachWithKey((PartitionKey) key, read -> results.add(read.read(kind.isWrite(), safeStore, executeAt))); + forEachWithKey((PartitionKey) key, read -> results.add(read.read(executeAt))); if (results.isEmpty()) + // Result type must match everywhere return AsyncChains.success(new TxnData()); if (results.size() == 1) @@ -172,6 +215,7 @@ public class TxnRead extends AbstractKeySorted implements Read return AsyncChains.reduce(results, Data::merge); } + @Simulate(with = MONITORS) public static final IVersionedSerializer serializer = new IVersionedSerializer() { @Override @@ -179,13 +223,16 @@ public class TxnRead extends AbstractKeySorted implements Read { KeySerializers.keys.serialize(read.txnKeys, out, version); serializeArray(read.items, out, version, TxnNamedRead.serializer); + serializeNullable(read.cassandraConsistencyLevel, out, version, consistencyLevelSerializer); } @Override public TxnRead deserialize(DataInputPlus in, int version) throws IOException { Keys keys = KeySerializers.keys.deserialize(in, version); - return new TxnRead(deserializeArray(in, version, TxnNamedRead.serializer, TxnNamedRead[]::new), keys); + TxnNamedRead[] items = deserializeArray(in, version, TxnNamedRead.serializer, TxnNamedRead[]::new); + ConsistencyLevel consistencyLevel = deserializeNullable(in, version, consistencyLevelSerializer); + return new TxnRead(items, keys, consistencyLevel); } @Override @@ -193,6 +240,7 @@ public class TxnRead extends AbstractKeySorted implements Read { long size = KeySerializers.keys.serializedSize(read.txnKeys, version); size += serializedArraySize(read.items, version, TxnNamedRead.serializer); + size += serializedNullableSize(read.cassandraConsistencyLevel, version, consistencyLevelSerializer); return size; } }; diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnResult.java b/src/java/org/apache/cassandra/service/accord/txn/TxnResult.java new file mode 100644 index 0000000000..38eeb88aca --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnResult.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.txn; + +import java.io.IOException; + +import accord.api.Result; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +import static org.apache.cassandra.db.TypeSizes.sizeof; + +public abstract class TxnResult implements Result +{ + public interface TxnResultSerializer extends IVersionedSerializer {} + + public enum Kind + { + txn_data(0), + retry_new_protocol(1); + + int id; + + Kind(int id) + { + this.id = id; + } + + public TxnResultSerializer serializer() + { + switch (this) + { + case txn_data: + return TxnData.serializer; + case retry_new_protocol: + return RetryWithNewProtocolResult.serializer; + default: + throw new IllegalStateException("Unrecognized kind " + this); + } + } + } + + public abstract Kind kind(); + + public abstract long estimatedSizeOnHeap(); + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @SuppressWarnings("unchecked") + @Override + public void serialize(TxnResult txnResult, DataOutputPlus out, int version) throws IOException + { + out.writeByte(txnResult.kind().ordinal()); + txnResult.kind().serializer().serialize(txnResult, out, version); + } + + @Override + public TxnResult deserialize(DataInputPlus in, int version) throws IOException + { + TxnResult.Kind kind = TxnResult.Kind.values()[in.readByte()]; + return (TxnResult)kind.serializer().deserialize(in, version); + } + + @SuppressWarnings("unchecked") + @Override + public long serializedSize(TxnResult txnResult, int version) + { + return sizeof((byte)txnResult.kind().ordinal()) + txnResult.kind().serializer().serializedSize(txnResult, version); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java b/src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java index 01a87a0e9d..cd2aa8a332 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java @@ -26,16 +26,17 @@ import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Function; +import javax.annotation.Nullable; import accord.api.Data; import accord.api.Key; import accord.api.Update; -import accord.api.Write; import accord.primitives.Keys; import accord.primitives.Ranges; import accord.primitives.RoutableKey; import accord.primitives.Timestamp; import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputBuffer; @@ -43,13 +44,17 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.accord.AccordObjectSizes; import org.apache.cassandra.service.accord.AccordSerializers; +import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.serializers.KeySerializers; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.ObjectSizes; +import static accord.utils.Invariants.checkArgument; import static accord.utils.SortedArrays.Search.CEIL; +import static org.apache.cassandra.service.accord.AccordSerializers.consistencyLevelSerializer; import static org.apache.cassandra.service.accord.AccordSerializers.serialize; import static org.apache.cassandra.utils.ArraySerializers.deserializeArray; import static org.apache.cassandra.utils.ArraySerializers.serializeArray; @@ -57,39 +62,50 @@ import static org.apache.cassandra.utils.ArraySerializers.serializedArraySize; import static org.apache.cassandra.utils.ByteBufferUtil.readWithVIntLength; import static org.apache.cassandra.utils.ByteBufferUtil.serializedSizeWithVIntLength; import static org.apache.cassandra.utils.ByteBufferUtil.writeWithVIntLength; +import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSize; -public class TxnUpdate implements Update +public class TxnUpdate extends AccordUpdate { - private static final long EMPTY_SIZE = ObjectSizes.measure(new TxnUpdate(null, new ByteBuffer[0], null)); + private static final long EMPTY_SIZE = ObjectSizes.measure(new TxnUpdate(null, new ByteBuffer[0], null, null)); private final Keys keys; private final ByteBuffer[] fragments; private final ByteBuffer condition; + @Nullable + private final ConsistencyLevel cassandraCommitCL; + // Memoize computation of condition private Boolean conditionResult; - public TxnUpdate(List fragments, TxnCondition condition) + public TxnUpdate(List fragments, TxnCondition condition, @Nullable ConsistencyLevel cassandraCommitCL) { + checkArgument(cassandraCommitCL == null || IAccordService.SUPPORTED_COMMIT_CONSISTENCY_LEVELS.contains(cassandraCommitCL)); // TODO: Figure out a way to shove keys into TxnCondition, and have it implement slice/merge. this.keys = Keys.of(fragments, fragment -> fragment.key); fragments.sort(TxnWrite.Fragment::compareKeys); this.fragments = toSerializedValuesArray(keys, fragments, fragment -> fragment.key, TxnWrite.Fragment.serializer); this.condition = serialize(condition, TxnCondition.serializer); + this.cassandraCommitCL = cassandraCommitCL; } - private TxnUpdate(Keys keys, ByteBuffer[] fragments, ByteBuffer condition) + private TxnUpdate(Keys keys, ByteBuffer[] fragments, ByteBuffer condition, ConsistencyLevel cassandraCommitCL) { this.keys = keys; this.fragments = fragments; this.condition = condition; + this.cassandraCommitCL = cassandraCommitCL; } + @Override public long estimatedSizeOnHeap() { long size = EMPTY_SIZE + ByteBufferUtil.estimatedSizeOnHeap(condition); for (ByteBuffer update : fragments) size += ByteBufferUtil.estimatedSizeOnHeap(update); + size += AccordObjectSizes.keys(keys); return size; } @@ -145,7 +161,7 @@ public class TxnUpdate implements Update { Keys keys = this.keys.slice(ranges); // TODO: Slice the condition. - return new TxnUpdate(keys, select(this.keys, keys, fragments), condition); + return new TxnUpdate(keys, select(this.keys, keys, fragments), condition, cassandraCommitCL); } private static ByteBuffer[] select(Keys in, Keys out, ByteBuffer[] from) @@ -167,7 +183,7 @@ public class TxnUpdate implements Update TxnUpdate that = (TxnUpdate) update; Keys mergedKeys = this.keys.with(that.keys); ByteBuffer[] mergedFragments = merge(this.keys, that.keys, this.fragments, that.fragments, mergedKeys.size()); - return new TxnUpdate(mergedKeys, mergedFragments, condition); + return new TxnUpdate(mergedKeys, mergedFragments, condition, cassandraCommitCL); } private static ByteBuffer[] merge(Keys leftKeys, Keys rightKeys, ByteBuffer[] left, ByteBuffer[] right, int outputSize) @@ -188,7 +204,7 @@ public class TxnUpdate implements Update } @Override - public Write apply(Timestamp executeAt, Data data) + public TxnWrite apply(Timestamp executeAt, Data data) { if (!checkCondition(data)) return TxnWrite.EMPTY_CONDITION_FAILED; @@ -198,6 +214,7 @@ public class TxnUpdate implements Update QueryOptions options = QueryOptions.forProtocolVersion(ProtocolVersion.CURRENT); AccordUpdateParameters parameters = new AccordUpdateParameters((TxnData) data, options); + // First completes all fragments and join them with the repairs pending for those partitions for (TxnWrite.Fragment fragment : fragments) // Filter out fragments that already constitute complete updates to avoid persisting them via TxnWrite: if (!fragment.isComplete()) @@ -218,7 +235,7 @@ public class TxnUpdate implements Update return updates; } - public static final IVersionedSerializer serializer = new IVersionedSerializer() + public static final AccordUpdateSerializer serializer = new AccordUpdateSerializer() { @Override public void serialize(TxnUpdate update, DataOutputPlus out, int version) throws IOException @@ -226,6 +243,7 @@ public class TxnUpdate implements Update KeySerializers.keys.serialize(update.keys, out, version); writeWithVIntLength(update.condition, out); serializeArray(update.fragments, out, version, ByteBufferUtil.byteBufferSerializer); + serializeNullable(update.cassandraCommitCL, out, version, consistencyLevelSerializer); } @Override @@ -234,7 +252,8 @@ public class TxnUpdate implements Update Keys keys = KeySerializers.keys.deserialize(in, version); ByteBuffer condition = readWithVIntLength(in); ByteBuffer[] fragments = deserializeArray(in, version, ByteBufferUtil.byteBufferSerializer, ByteBuffer[]::new); - return new TxnUpdate(keys, fragments, condition); + ConsistencyLevel consistencyLevel = deserializeNullable(in, version, consistencyLevelSerializer); + return new TxnUpdate(keys, fragments, condition, consistencyLevel); } @Override @@ -243,6 +262,7 @@ public class TxnUpdate implements Update long size = KeySerializers.keys.serializedSize(update.keys, version); size += serializedSizeWithVIntLength(update.condition); size += serializedArraySize(update.fragments, version, ByteBufferUtil.byteBufferSerializer); + size += serializedNullableSize(update.cassandraCommitCL, version, consistencyLevelSerializer); assert(ByteBufferUtil.serialized(this, update, version).remaining() == size); return size; } @@ -323,7 +343,7 @@ public class TxnUpdate implements Update return result; } - // maybeCheckCondition? checkConditionMemoized? + @Override public boolean checkCondition(Data data) { // Assert data that was memoized is same as data that is provided? @@ -333,4 +353,16 @@ public class TxnUpdate implements Update conditionResult = condition.applies((TxnData) data); return conditionResult; } + + @Override + public Kind kind() + { + return Kind.TXN; + } + + @Override + public ConsistencyLevel cassandraCommitCL() + { + return cassandraCommitCL; + } } diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java index 89599bc90e..27406036d9 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java @@ -26,14 +26,22 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; +import javax.annotation.Nonnull; -import accord.primitives.*; +import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import accord.api.DataStore; import accord.api.Write; import accord.local.SafeCommandStore; +import accord.primitives.PartialTxn; +import accord.primitives.RoutableKey; +import accord.primitives.Seekable; +import accord.primitives.Timestamp; +import accord.primitives.Writes; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; import org.apache.cassandra.concurrent.Stage; @@ -45,25 +53,31 @@ import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.service.accord.AccordSafeCommandsForKey; import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.AccordSafeCommandsForKey; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.utils.BooleanSerializer; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.ObjectSizes; -import static org.apache.cassandra.utils.ArraySerializers.deserializeArray; +import static org.apache.cassandra.cql3.terms.Lists.accordListPathSupplier; import static org.apache.cassandra.service.accord.AccordSerializers.partitionUpdateSerializer; +import static org.apache.cassandra.utils.ArraySerializers.deserializeArray; import static org.apache.cassandra.utils.ArraySerializers.serializeArray; import static org.apache.cassandra.utils.ArraySerializers.serializedArraySize; public class TxnWrite extends AbstractKeySorted implements Write { + @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(TxnWrite.class); + public static final TxnWrite EMPTY_CONDITION_FAILED = new TxnWrite(Collections.emptyList(), false); private static final long EMPTY_SIZE = ObjectSizes.measure(EMPTY_CONDITION_FAILED); @@ -121,9 +135,9 @@ public class TxnWrite extends AbstractKeySorted implements Writ '}'; } - public AsyncChain write(long timestamp, int nowInSeconds) + public AsyncChain write(@Nonnull Function cellToMaybeNewListPath, long timestamp, int nowInSeconds) { - PartitionUpdate update = new PartitionUpdate.Builder(get(), 0).updateAllTimestampAndLocalDeletionTime(timestamp, nowInSeconds).build(); + PartitionUpdate update = new PartitionUpdate.Builder(get(), 0).updateTimesAndPathsForAccord(cellToMaybeNewListPath, timestamp, nowInSeconds).build(); Mutation mutation = new Mutation(update); return AsyncChains.ofRunnable(Stage.MUTATION.executor(), mutation::apply); } @@ -142,7 +156,6 @@ public class TxnWrite extends AbstractKeySorted implements Writ PartitionKey.serializer.serialize(write.key, out, version); out.writeInt(write.index); ByteBufferUtil.writeWithVIntLength(write.bytes(), out); - } @Override @@ -220,12 +233,12 @@ public class TxnWrite extends AbstractKeySorted implements Writ { return referenceOps.isEmpty(); } - + public Update toUpdate() { return new Update(key, index, baseUpdate); } - + public Update complete(AccordUpdateParameters parameters) { if (isComplete()) @@ -238,7 +251,7 @@ public class TxnWrite extends AbstractKeySorted implements Writ baseUpdate.rowCount(), baseUpdate.canHaveShadowedData()); - UpdateParameters up = parameters.updateParameters(baseUpdate.metadata(), index); + UpdateParameters up = parameters.updateParameters(baseUpdate.metadata(), key, index); TxnData data = parameters.getData(); Row staticRow = applyUpdates(baseUpdate.staticRow(), referenceOps.statics, key, Clustering.STATIC_CLUSTERING, up, data); @@ -323,7 +336,7 @@ public class TxnWrite extends AbstractKeySorted implements Writ } private final boolean isConditionMet; - + private TxnWrite(Update[] items, boolean isConditionMet) { super(items); @@ -370,7 +383,8 @@ public class TxnWrite extends AbstractKeySorted implements Writ // Apply updates not specified fully by the client but built from fragments completed by data from reads. // This occurs, for example, when an UPDATE statement uses a value assigned by a LET statement. - forEachWithKey((PartitionKey) key, write -> results.add(write.write(timestamp, nowInSeconds))); + Function accordListPathSuppler = accordListPathSupplier(timestamp); + forEachWithKey((PartitionKey) key, write -> results.add(write.write(accordListPathSuppler, timestamp, nowInSeconds))); if (isConditionMet) { @@ -380,7 +394,7 @@ public class TxnWrite extends AbstractKeySorted implements Writ TxnUpdate txnUpdate = (TxnUpdate) txn.update(); assert txnUpdate != null : "PartialTxn should contain an update if we're applying a write!"; List updates = txnUpdate.completeUpdatesForKey((RoutableKey) key); - updates.forEach(update -> results.add(update.write(timestamp, nowInSeconds))); + updates.forEach(update -> results.add(update.write(accordListPathSuppler, timestamp, nowInSeconds))); } if (results.isEmpty()) diff --git a/src/java/org/apache/cassandra/service/accord/txn/UnrecoverableRepairUpdate.java b/src/java/org/apache/cassandra/service/accord/txn/UnrecoverableRepairUpdate.java new file mode 100644 index 0000000000..310a3e6a58 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/txn/UnrecoverableRepairUpdate.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.txn; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import accord.api.Data; +import accord.api.Update; +import accord.api.Write; +import accord.local.Node; +import accord.primitives.Ranges; +import accord.primitives.Seekables; +import accord.primitives.Timestamp; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.service.reads.ReadCoordinator; +import org.apache.cassandra.service.reads.repair.BlockingReadRepair; + +/** + * This update is used to support blocking read repair from non-transactional Cassandra reads. Cassandra creates + * a read repair mutation per node and this enables some partitiosn to be readable that would otherwise run into messages + * size limits. + * + * This update is used during the `Execute` phase to apply the repair mutations directly in AccordInteropExecution similar + * to how Accord applies read repair mutations for normal Accord transactions. It will always produce an empty update + * for Accord to use in the Apply phase because Accord doesn't support a per replica Apply and adding it would be redundant + * with the support that exists in AccordInteropExecution. + * + * The state for this update is always kept in memory and is never serialized. Only the Id is propagated so the cache + * can evict the update and then load it back. We don't need to persist it or have it be recoverable because if the original + * coordinator fails to complete the transaction then the dependent Cassandra read that triggered the read repair will + * also fail and it doesn't matter if the read repair is partially applied or not applied at all. + */ +public class UnrecoverableRepairUpdate, P extends ReplicaPlan.ForRead> extends AccordUpdate +{ + private static final ConcurrentHashMap inflightUpdates = new ConcurrentHashMap<>(); + + public static UnrecoverableRepairUpdate removeInflightUpdate(Key updateKey) + { + return inflightUpdates.remove(updateKey); + } + + private static class Key + { + final int nodeId; + final long counter; + + private Key(@Nonnull int nodeId, long counter) + { + this.nodeId = nodeId; + this.counter = counter; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Key key = (Key) o; + + if (nodeId != key.nodeId) return false; + return counter == key.counter; + } + + @Override + public int hashCode() + { + int result = nodeId; + result = 31 * result + (int) (counter ^ (counter >>> 32)); + return result; + } + } + + private static final AtomicLong nextCounter = new AtomicLong(0); + + public final BlockingReadRepair parent; + public final Seekables keys; + public final DecoratedKey dk; + public final Map mutations; + public final ReplicaPlan.ForWrite writePlan; + public final Key updateKey; + + private UnrecoverableRepairUpdate(Node.Id nodeId, BlockingReadRepair parent, + Seekables keys, DecoratedKey dk, Map mutations, ReplicaPlan.ForWrite writePlan) + { + this.parent = parent; + this.keys = keys; + this.dk = dk; + this.mutations = mutations; + this.writePlan = writePlan; + this.updateKey = new Key(nodeId.id, nextCounter.getAndIncrement()); + } + + public static , P extends ReplicaPlan.ForRead> UnrecoverableRepairUpdate create(Node.Id nodeId, BlockingReadRepair parent, + Seekables keys, DecoratedKey dk, Map mutations, + ReplicaPlan.ForWrite writePlan) + { + UnrecoverableRepairUpdate update = new UnrecoverableRepairUpdate<>(nodeId, parent, keys, dk, mutations, writePlan); + inflightUpdates.put(update.updateKey, update); + return update; + } + + @Override + public Seekables keys() + { + return keys; + } + + @Override + public Write apply(Timestamp executeAt, @Nullable Data data) + { + return TxnWrite.EMPTY_CONDITION_FAILED; + } + + @Override + public Update slice(Ranges ranges) + { + return this; + } + + @Override + public Update merge(Update other) + { + return this; + } + + @Override + public ConsistencyLevel cassandraCommitCL() + { + // Leads to standard async persist/commit which is fine since the repair mutations were applied + // as part of execute/read + return null; + } + + @Override + public Kind kind() + { + return Kind.UNRECOVERABLE_REPAIR; + } + + @Override + public long estimatedSizeOnHeap() + { + return 0; + } + + public void runBRR(ReadCoordinator readCoordinator) + { + // This read repair is effectively running as a delegate of the read repair instance that did the reads + // to generate the mutations, but since we already have the mutations we can go ahead and apply them + // now that we are inside a transaction that guarantees that the contents of the mutations consist + // of committed data everywhere we go to apply it + parent.repairPartitionDirectly(readCoordinator, dk, mutations, writePlan); + } + + public static final AccordUpdateSerializer serializer = new AccordUpdateSerializer() + { + @Override + public void serialize(UnrecoverableRepairUpdate update, DataOutputPlus out, int version) throws IOException + { + out.writeUnsignedVInt32(update.updateKey.nodeId); + out.writeUnsignedVInt(update.updateKey.counter); + } + + @Override + public UnrecoverableRepairUpdate deserialize(DataInputPlus in, int version) throws IOException + { + return inflightUpdates.get(new Key(in.readUnsignedVInt32(), in.readUnsignedVInt())); + } + + @Override + public long serializedSize(UnrecoverableRepairUpdate update, int version) + { + return TypeSizes.sizeofUnsignedVInt(update.updateKey.nodeId) + TypeSizes.sizeofUnsignedVInt(update.updateKey.counter); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java new file mode 100644 index 0000000000..512943e463 --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.primitives.Ints; + +import accord.api.BarrierType; +import accord.primitives.Seekables; +import com.github.benmanes.caffeine.cache.CacheLoader; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.Weigher; +import org.apache.cassandra.concurrent.ImmediateExecutor; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.exceptions.CasWriteTimeoutException; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.metrics.ClientRequestsMetricsHolder; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigratedAt; +import org.apache.cassandra.service.paxos.AbstractPaxosRepair.Failure; +import org.apache.cassandra.service.paxos.AbstractPaxosRepair.Result; +import org.apache.cassandra.service.paxos.PaxosRepair; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.ObjectSizes; +import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.UUIDSerializer; + +import static org.apache.cassandra.net.Verb.CONSENSUS_KEY_MIGRATION; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget.paxos; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +/** + * Tracks the migration state of individual keys storing the migration (or not) in system.consensus_migration_state + * with an in-memory cache in front. Only locally replicated keys are tracked here to avoid storing too much + * state when token aware routing is not used. + * + * It is safe to migrate keys multiple times so no effort is made to ensure exactly once behavior and the system table + * expires key migration state after 7 days. + */ +public abstract class ConsensusKeyMigrationState +{ + /* + * Used to notify other replicas when key migration has occurred so they can + * also cache that the key migration was done + */ + public static class ConsensusKeyMigrationFinished + { + @Nonnull + private final UUID tableId; + @Nonnull + private final ByteBuffer partitionKey; + @Nonnull + private final ConsensusMigratedAt consensusMigratedAt; + + private ConsensusKeyMigrationFinished(@Nonnull UUID tableId, + @Nonnull ByteBuffer partitionKey, + @Nonnull ConsensusMigratedAt consensusMigratedAt) + { + this.tableId = tableId; + this.partitionKey = partitionKey; + this.consensusMigratedAt = consensusMigratedAt; + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(ConsensusKeyMigrationFinished t, DataOutputPlus out, int version) throws IOException + { + UUIDSerializer.serializer.serialize(t.tableId, out, version); + ByteBufferUtil.writeWithVIntLength(t.partitionKey, out); + ConsensusMigratedAt.serializer.serialize(t.consensusMigratedAt, out, version); + } + + @Override + public ConsensusKeyMigrationFinished deserialize(DataInputPlus in, int version) throws IOException + { + UUID tableId = UUIDSerializer.serializer.deserialize(in, version); + ByteBuffer partitionKey = ByteBufferUtil.readWithVIntLength(in); + ConsensusMigratedAt consensusMigratedAt = ConsensusMigratedAt.serializer.deserialize(in, version); + return new ConsensusKeyMigrationFinished(tableId, partitionKey, consensusMigratedAt); + } + + @Override + public long serializedSize(ConsensusKeyMigrationFinished t, int version) + { + return UUIDSerializer.serializer.serializedSize(t.tableId, version) + + ByteBufferUtil.serializedSizeWithVIntLength(t.partitionKey) + + ConsensusMigratedAt.serializer.serializedSize(t.consensusMigratedAt, version); + } + }; + } + + /* + * Bundles various aspects of key migration state together to avoid multiple lookups + * and to communicate multiple result values and state + */ + public static class KeyMigrationState + { + static final KeyMigrationState MIGRATION_NOT_NEEDED = new KeyMigrationState(null, null, null, null); + + public final ConsensusMigratedAt consensusMigratedAt; + + public final Epoch currentEpoch; + + public final TableMigrationState tableMigrationState; + + public final DecoratedKey key; + + private KeyMigrationState(ConsensusMigratedAt consensusMigratedAt, Epoch currentEpoch, + TableMigrationState tableMigrationState, DecoratedKey key) + { + this.consensusMigratedAt = consensusMigratedAt; + this.currentEpoch = currentEpoch; + this.tableMigrationState = tableMigrationState; + this.key = key; + } + + /* + * This will trigger a distributed migration for the key, but will only block on local completion + * so Paxos reads can return a result as soon as the local state is ready + */ + public void maybePerformAccordToPaxosKeyMigration(boolean isForWrite) + { + if (paxosReadSatisfiedByKeyMigration()) + return; + + // TODO (desired): Better query start time + TableMigrationState tms = tableMigrationState; + repairKeyAccord(key, tms.keyspaceName, tms.tableId, tms.minMigrationEpoch(key.getToken()).getEpoch(), Dispatcher.RequestTime.forImmediateExecution(), false, isForWrite); + } + + private boolean paxosReadSatisfiedByKeyMigration() + { + // No migration in progress, it's safe + if (tableMigrationState == null) + return true; + + return tableMigrationState.paxosReadSatisfiedByKeyMigrationAtEpoch(key, consensusMigratedAt); + } + } + + private static final int EMPTY_KEY_SIZE = Ints.checkedCast(ObjectSizes.measureDeep(Pair.create(null, UUID.randomUUID()))); + private static final int VALUE_SIZE = Ints.checkedCast(ObjectSizes.measureDeep(new ConsensusMigratedAt(Epoch.EMPTY, ConsensusMigrationTarget.accord))); + + private static final CacheLoader, ConsensusMigratedAt> LOADING_FUNCTION = k -> SystemKeyspace.loadConsensusKeyMigrationState(k.left, k.right); + private static final Weigher, ConsensusMigratedAt> WEIGHER_FUNCTION = (k, v) -> EMPTY_KEY_SIZE + Ints.checkedCast(ByteBufferUtil.estimatedSizeOnHeap(k.left)) + VALUE_SIZE; + private static final LoadingCache, ConsensusMigratedAt> MIGRATION_STATE_CACHE = + Caffeine.newBuilder() + .maximumWeight(DatabaseDescriptor.getConsensusMigrationCacheSizeInMiB() << 20) + .weigher(WEIGHER_FUNCTION) + .executor(ImmediateExecutor.INSTANCE) + .build(LOADING_FUNCTION); + + public static final IVerbHandler consensusKeyMigrationFinishedHandler = message -> { + saveConsensusKeyMigrationLocally(message.payload.partitionKey, message.payload.tableId, message.payload.consensusMigratedAt); + }; + + private ConsensusKeyMigrationState() {} + + @VisibleForTesting + public static void reset() + { + MIGRATION_STATE_CACHE.invalidateAll(); + } + + public static void maybeSaveAccordKeyMigrationLocally(PartitionKey partitionKey, Epoch epoch) + { + TableId tableId = partitionKey.tableId(); + UUID tableUUID = tableId.asUUID(); + DecoratedKey dk = partitionKey.partitionKey(); + ByteBuffer key = dk.getKey(); + + TableMigrationState tms = ClusterMetadata.current().consensusMigrationState.tableStates.get(tableId); + if (tms == null) + return; + + ConsensusMigratedAt migratedAt = new ConsensusMigratedAt(epoch, paxos); + if (!tms.paxosReadSatisfiedByKeyMigrationAtEpoch(dk, migratedAt)) + return; + + saveConsensusKeyMigrationLocally(key, tableUUID, migratedAt); + } + + /* + * Should be called where we know we replicate the key so that the system table contains useful information + * about whether the migration already occurred. + * + * This is a more expensive check that might read from the system table to determine if migration occurred. + */ + public static KeyMigrationState getKeyMigrationState(TableId tableId, DecoratedKey key) + { + ClusterMetadata cm = ClusterMetadata.current(); + TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tableId); + // No state means no migration for this table + if (tms == null) + return KeyMigrationState.MIGRATION_NOT_NEEDED; + + if (Range.isInNormalizedRanges(key.getToken(), tms.migratingRanges)) + { + ConsensusMigratedAt consensusMigratedAt = getConsensusMigratedAt(tableId, key); + if (consensusMigratedAt == null) + return new KeyMigrationState(null, cm.epoch, tms, key); + return new KeyMigrationState(consensusMigratedAt, cm.epoch, tms, key); + } + + return KeyMigrationState.MIGRATION_NOT_NEEDED; + } + + public static @Nullable ConsensusMigratedAt getConsensusMigratedAt(TableId tableId, DecoratedKey key) + { + return MIGRATION_STATE_CACHE.get(Pair.create(key.getKey(), tableId.asUUID())); + } + + /* + * Trigger a distributed repair of Accord state for this key. + */ + static void repairKeyAccord(DecoratedKey key, + String keyspace, + TableId tableId, + long minEpoch, + Dispatcher.RequestTime requestTime, + boolean global, + boolean isForWrite) + { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId); + if (isForWrite) + ClientRequestsMetricsHolder.casWriteMetrics.accordKeyMigrations.mark(); + else + ClientRequestsMetricsHolder.casReadMetrics.accordKeyMigrations.mark(); + long start = nanoTime(); + try + { + // Global will always create a transaction to effect the barrier so all replicas + // will soon be ready to execute, but only waits for the local replica to be ready + // Local will only create a transaction if it can't find an existing one to wait on + BarrierType barrierType = global ? BarrierType.global_async : BarrierType.local; + AccordService.instance().barrier(Seekables.of(new PartitionKey(keyspace, tableId, key)), minEpoch, requestTime, DatabaseDescriptor.getTransactionTimeout(TimeUnit.NANOSECONDS), barrierType, isForWrite); + // We don't save the state to the cache here. Accord will notify the agent every time a barrier happens. + } + finally + { + cfs.metric.keyMigration.addNano(nanoTime() - start); + } + } + + static void repairKeyPaxos(EndpointsForToken naturalReplicas, + Epoch currentEpoch, + DecoratedKey key, + ColumnFamilyStore cfs, + ConsistencyLevel consistencyLevel, + Dispatcher.RequestTime requestTime, + long timeoutNanos, + boolean isLocallyReplicated, + boolean isForWrite) + { + if (isForWrite) + ClientRequestsMetricsHolder.accordWriteMetrics.paxosKeyMigrations.mark(); + else + ClientRequestsMetricsHolder.accordReadMetrics.paxosKeyMigrations.mark(); + TableMetadata tableMetadata = cfs.metadata(); + PaxosRepair repair = PaxosRepair.create(consistencyLevel, key, tableMetadata, timeoutNanos); + long start = nanoTime(); + repair.start(requestTime.startedAtNanos()); + Result result; + try + { + result = repair.await(); + switch (result.outcome) + { + default: + case CANCELLED: + throw new IllegalStateException("Unexpected PaxosRepair outcome " + result.outcome); + case DONE: + // Don't want to repeatedly save this in the non-token aware case + if (isLocallyReplicated) + saveConsensusKeyMigration(naturalReplicas, + new ConsensusKeyMigrationFinished(tableMetadata.id.asUUID(), + key.getKey(), + new ConsensusMigratedAt(currentEpoch, ConsensusMigrationTarget.accord))); + return; + case FAILURE: + Failure failure = (Failure)result; + if (failure.failure == null) + throw new CasWriteTimeoutException(WriteType.CAS, consistencyLevel, 0, 0, 0); + throw new RuntimeException(failure.failure); + } + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + finally + { + cfs.metric.keyMigration.addNano(nanoTime() - start); + } + } + + private static void saveConsensusKeyMigration(EndpointsForToken replicas, ConsensusKeyMigrationFinished finished) + { + Message out = Message.out(CONSENSUS_KEY_MIGRATION, finished); + replicas.endpoints(); + for (Replica replica : replicas) + { + if (replica.isSelf()) + saveConsensusKeyMigrationLocally(finished.partitionKey, finished.tableId, finished.consensusMigratedAt); + else + MessagingService.instance().send(out, replica.endpoint()); + } + } + + private static void saveConsensusKeyMigrationLocally(ByteBuffer partitionKey, UUID tableId, ConsensusMigratedAt consensusMigratedAt) + { + // Order doesn't matter, existing values don't matter, version doesn't matter + // If any of this races or goes backwards the result is that key migration is + // reattempted and it should be very rare + MIGRATION_STATE_CACHE.put(Pair.create(partitionKey, tableId), consensusMigratedAt); + Stage.MUTATION.execute(() -> SystemKeyspace.saveConsensusKeyMigrationState(partitionKey, tableId, consensusMigratedAt)); + } +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java new file mode 100644 index 0000000000..ac63cc4bd9 --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import javax.annotation.Nonnull; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.config.Config.LWTStrategy; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigratedAt; +import org.apache.cassandra.service.paxos.Paxos; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.utils.FBUtilities; + +import static com.google.common.base.Preconditions.checkState; +import static org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.getConsensusMigratedAt; +import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.accord; +import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.paxosV1; +import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.paxosV2; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget.paxos; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; + +/** + * Helper class to decide where to route a request that requires consensus, migrating a key if necessary + * before rerouting. + */ +public class ConsensusRequestRouter +{ + public enum ConsensusRoutingDecision + { + paxosV1, + paxosV2, + accord, + } + + public static volatile ConsensusRequestRouter instance = new ConsensusRequestRouter(); + + @VisibleForTesting + public static void setInstance(ConsensusRequestRouter testInstance) + { + instance = testInstance; + } + + @VisibleForTesting + public static void resetInstance() + { + instance = new ConsensusRequestRouter(); + } + + protected ConsensusRequestRouter() {} + + public ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull TableId tableId, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + { + // In accord mode there might be migration state in CM (unless cleanup gets added), but it doesn't + // matter. All other consensus protocols are not used. + if (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord) + return accord; + + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId); + if (cfs == null) + throw new IllegalStateException("Can't route consensus request for nonexistent table %s".format(tableId.toString())); + return routeAndMaybeMigrate(key, cfs, consistencyLevel, requestTime, timeoutNanos, isForWrite); + } + + protected ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull ColumnFamilyStore cfs, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + { + ClusterMetadata cm = ClusterMetadata.current(); + + TableMigrationState tms = cm.consensusMigrationState.tableStates.get(cfs.getTableId()); + if (tms == null) + return pickPaxos(); + + if (Range.isInNormalizedRanges(key.getToken(), tms.migratedRanges)) + return pickMigrated(tms.targetProtocol); + + if (Range.isInNormalizedRanges(key.getToken(), tms.migratingRanges)) + return pickBasedOnKeyMigrationStatus(cm, tms, key, cfs, consistencyLevel, requestTime, timeoutNanos, isForWrite); + + // It's not migrated so infer the protocol from the target + return pickNotMigrated(tms.targetProtocol); + } + + /** + * If the key was already migrated then we can pick the target protocol otherwise + * we have to run a repair operation on the key to migrate it. + */ + private static ConsensusRoutingDecision pickBasedOnKeyMigrationStatus(ClusterMetadata cm, TableMigrationState tms, DecoratedKey key, ColumnFamilyStore cfs, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + { + checkState(pickPaxos() != paxosV1, "Can't migrate from PaxosV1 to anything"); + + // If it is locally replicated we can check our local migration state to see if it was already migrated + EndpointsForToken naturalReplicas = ReplicaLayout.forNonLocalStrategyTokenRead(cm, cfs.keyspace.getMetadata(), key.getToken()); + boolean isLocallyReplicated = naturalReplicas.lookup(FBUtilities.getBroadcastAddressAndPort()) != null; + if (isLocallyReplicated) + { + ConsensusMigratedAt consensusMigratedAt = getConsensusMigratedAt(tms.tableId, key); + // Check that key migration that was performed satisfies the requirements of the current in flight migration + // for the range + // Be aware that for Accord->Paxos the cache only tells us if the key was repaired locally + // This ends up still being safe because every single Paxos read (in a migrating range) during migration will check + // locally to see if repair is necessary + if (consensusMigratedAt != null && tms.satisfiedByKeyMigrationAtEpoch(key, consensusMigratedAt)) + return pickMigrated(tms.targetProtocol); + + if (tms.targetProtocol == paxos) + { + // Run the Accord barrier txn now so replicas don't start independent + // barrier transactions to accomplish the migration + // They still might need to go through the fast local path for barrier txns + // at each replica, but they won't create their own txn since we created it here + ConsensusKeyMigrationState.repairKeyAccord(key, tms.keyspaceName, tms.tableId, tms.minMigrationEpoch(key.getToken()).getEpoch(), requestTime, true, isForWrite); + return paxosV2; + } + // Fall through for repairKeyPaxos + } + + // If it's not locally replicated then: + // Accord -> Paxos - Paxos will ask Accord to migrate in the read at each replica if necessary + // Paxos -> Accord - Paxos needs to be repaired before Accord runs so do it here + if (tms.targetProtocol == paxos) + return paxosV2; + else + // Should exit exceptionally if the repair is not done + ConsensusKeyMigrationState.repairKeyPaxos(naturalReplicas, cm.epoch, key, cfs, consistencyLevel, requestTime, timeoutNanos, isLocallyReplicated, isForWrite); + + return pickMigrated(tms.targetProtocol); + } + + // Allows tests to inject specific responses + public boolean isKeyInMigratingOrMigratedRangeDuringPaxosBegin(TableId tableId, DecoratedKey key) + { + return isKeyInMigratingOrMigratedRangeFromPaxos(tableId, key); + } + + // Allows tests to inject specific responses + public boolean isKeyInMigratingOrMigratedRangeDuringPaxosAccept(TableId tableId, DecoratedKey key) + { + return isKeyInMigratingOrMigratedRangeFromPaxos(tableId, key); + } + + /* + * A lightweight check against cluster metadata that doesn't check if the key has already been migrated + * using local system table state + */ + public boolean isKeyInMigratingOrMigratedRangeFromPaxos(TableId tableId, DecoratedKey key) + { + TableMigrationState tms = ClusterMetadata.current().consensusMigrationState.tableStates.get(tableId); + // No state means no migration for this table + if (tms == null) + return false; + + if (tms.targetProtocol == ConsensusMigrationTarget.paxos) + return false; + + // The coordinator will need to retry either on Accord if they are trying + // to propose their own value, or by setting the consensus migration epoch to recover an incomplete transaction + if (Range.isInNormalizedRanges(key.getToken(), tms.migratingAndMigratedRanges)) + return true; + + return false; + } + + public boolean isKeyInMigratingOrMigratedRangeFromAccord(Epoch epoch, TableId tableId, DecoratedKey key) + { + ClusterMetadata cm = ClusterMetadataService.instance().fetchLogFromCMS(epoch); + TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tableId); + return isKeyInMigratingOrMigratedRangeFromAccord(tms, key); + } + + /* + * A lightweight check against cluster metadata that doesn't check if the key has already been migrated + * using local system table state. + */ + public boolean isKeyInMigratingOrMigratedRangeFromAccord(TableMigrationState tms, DecoratedKey key) + { + // No state means no migration for this table + if (tms == null) + return false; + + if (tms.targetProtocol == ConsensusMigrationTarget.accord) + return false; + + if (Range.isInNormalizedRanges(key.getToken(), tms.migratingAndMigratedRanges)) + return true; + + return false; + } + + private static ConsensusRoutingDecision pickMigrated(ConsensusMigrationTarget targetProtocol) + { + if (targetProtocol.equals(ConsensusMigrationTarget.accord)) + return accord; + else + return pickPaxos(); + } + + private static ConsensusRoutingDecision pickNotMigrated(ConsensusMigrationTarget targetProtocol) + { + if (targetProtocol.equals(ConsensusMigrationTarget.accord)) + return pickPaxos(); + else + return accord; + } + + private static ConsensusRoutingDecision pickPaxos() + { + return Paxos.useV2() ? paxosV2 : paxosV1; + } +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigrationState.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigrationState.java new file mode 100644 index 0000000000..62b03eefa9 --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigrationState.java @@ -0,0 +1,909 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import java.io.IOException; +import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.primitives.SignedBytes; +import com.google.common.util.concurrent.FutureCallback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.Config.LWTStrategy; +import org.apache.cassandra.config.Config.NonSerialWriteStrategy; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.repair.RepairParallelism; +import org.apache.cassandra.repair.RepairResult; +import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.paxos.Paxos; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.MetadataValue; +import org.apache.cassandra.tcm.serialization.MetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; +import org.apache.cassandra.tcm.transformations.BeginConsensusMigrationForTableAndRange; +import org.apache.cassandra.tcm.transformations.MaybeFinishConsensusMigrationForTableAndRange; +import org.apache.cassandra.tcm.transformations.SetConsensusMigrationTargetProtocol; +import org.apache.cassandra.utils.LocalizeString; +import org.apache.cassandra.utils.NullableSerializer; +import org.apache.cassandra.utils.PojoToString; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Collections.emptyList; +import static org.apache.cassandra.db.TypeSizes.sizeof; +import static org.apache.cassandra.dht.Range.intersectionOfNormalizedRanges; +import static org.apache.cassandra.dht.Range.normalize; +import static org.apache.cassandra.dht.Range.subtract; +import static org.apache.cassandra.dht.Range.subtractNormalizedRanges; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget.reset; +import static org.apache.cassandra.utils.CollectionSerializers.deserializeMap; +import static org.apache.cassandra.utils.CollectionSerializers.deserializeSet; +import static org.apache.cassandra.utils.CollectionSerializers.newHashMap; +import static org.apache.cassandra.utils.CollectionSerializers.newListSerializer; +import static org.apache.cassandra.utils.CollectionSerializers.serializeCollection; +import static org.apache.cassandra.utils.CollectionSerializers.serializeMap; +import static org.apache.cassandra.utils.CollectionSerializers.serializedCollectionSize; +import static org.apache.cassandra.utils.CollectionSerializers.serializedMapSize; + +/** + * Track and update the migration state of individual table and ranges within those tables + */ +public abstract class ConsensusTableMigrationState +{ + private static final Logger logger = LoggerFactory.getLogger(ConsensusTableMigrationState.class); + + public static final MetadataSerializer>> rangesSerializer = newListSerializer(Range.serializer); + + public static final FutureCallback completedRepairJobHandler = new FutureCallback() + { + @Override + public void onSuccess(@Nullable RepairResult repairResult) + { + checkNotNull(repairResult, "repairResult should not be null"); + ConsensusMigrationRepairResult migrationResult = repairResult.consensusMigrationRepairResult; + + // Need to repair both Paxos and base table state + // Could track them separately, but doesn't seem worth the effort + if (migrationResult.type == ConsensusMigrationRepairType.ineligible) + return; + + RepairJobDesc desc = repairResult.desc; + TableMetadata tm = Schema.instance.getTableMetadata(desc.keyspace, desc.columnFamily); + if (tm == null) + return; + TableMigrationState tms = ClusterMetadata.current().consensusMigrationState.tableStates.get(tm.id); + if (tms == null || !Range.intersects(tms.migratingRanges, desc.ranges)) + return; + + if (tms.targetProtocol == ConsensusMigrationTarget.paxos && repairResult.consensusMigrationRepairResult.type != ConsensusMigrationRepairType.accord) + return; + if (tms.targetProtocol == ConsensusMigrationTarget.accord && repairResult.consensusMigrationRepairResult.type != ConsensusMigrationRepairType.paxos) + return; + + logger.info("Repair {} is going to trigger migration completion for ranges {} and epoch {}", desc.sessionId, desc.ranges, migrationResult.minEpoch); + + ClusterMetadataService.instance().commit( + new MaybeFinishConsensusMigrationForTableAndRange( + desc.keyspace, desc.columnFamily, ImmutableList.copyOf(desc.ranges), + migrationResult.minEpoch, migrationResult.type)); + } + + @Override + public void onFailure(Throwable throwable) + { + // Only successes drive forward progress + } + }; + + public static void reset() + { + ClusterMetadata cm = ClusterMetadata.current(); + for (TableMigrationState tms : cm.consensusMigrationState.tableStates.values()) + setConsensusMigrationTargetProtocol("reset", + ImmutableList.of(tms.keyspaceName), + Optional.of(ImmutableList.of(tms.tableName))); + } + + public enum ConsensusMigrationRepairType + { + ineligible(0), + paxos(1), + accord(2); + + public final byte value; + + ConsensusMigrationRepairType(int value) + { + this.value = SignedBytes.checkedCast(value); + } + + public static ConsensusMigrationRepairType fromString(String repairType) + { + return ConsensusMigrationRepairType.valueOf(LocalizeString.toLowerCaseLocalized(repairType)); + } + + public static ConsensusMigrationRepairType fromValue(byte value) + { + switch (value) + { + default: + throw new IllegalArgumentException(value + " is not recognized"); + case 0: + return ConsensusMigrationRepairType.ineligible; + case 1: + return ConsensusMigrationRepairType.paxos; + case 2: + return ConsensusMigrationRepairType.accord; + } + } + } + + public enum ConsensusMigrationTarget + { + paxos(0), + accord(1), + reset(2); + + public final byte value; + + ConsensusMigrationTarget(int value) + { + this.value = SignedBytes.checkedCast(value); + } + + public static ConsensusMigrationTarget fromString(String targetProtocol) + { + return ConsensusMigrationTarget.valueOf(LocalizeString.toLowerCaseLocalized(targetProtocol)); + } + + public static ConsensusMigrationTarget fromValue(byte value) + { + switch (value) + { + default: + throw new IllegalArgumentException(value + " is not recognized"); + case 0: + return paxos; + case 1: + return accord; + case 2: + return reset; + } + } + } + + public static class ConsensusMigrationRepairResult + { + private final ConsensusMigrationRepairType type; + private final Epoch minEpoch; + + private ConsensusMigrationRepairResult(ConsensusMigrationRepairType type, Epoch minEpoch) + { + this.type = type; + this.minEpoch = minEpoch; + } + + public static ConsensusMigrationRepairResult fromCassandraRepair(Epoch minEpoch, boolean migrationEligibleRepair) + { + checkArgument(!migrationEligibleRepair || minEpoch.isAfter(Epoch.EMPTY), "Epoch should not be empty if Paxos and regular repairs were performed"); + if (migrationEligibleRepair) + return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.paxos, minEpoch); + else + return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.ineligible, Epoch.EMPTY); + } + + public static ConsensusMigrationRepairResult fromAccordRepair(Epoch minEpoch) + { + checkArgument(minEpoch.isAfter(Epoch.EMPTY), "Accord repairs should always occur at an Epoch"); + return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.accord, minEpoch); + } + } + + public static class ConsensusMigratedAt + { + public static final IVersionedSerializer serializer = NullableSerializer.wrap(new IVersionedSerializer() + { + @Override + public void serialize(ConsensusMigratedAt t, DataOutputPlus out, int version) throws IOException + { + Epoch.messageSerializer.serialize(t.migratedAtEpoch, out, version); + out.writeByte(t.migratedAtTarget.value); + } + + @Override + public ConsensusMigratedAt deserialize(DataInputPlus in, int version) throws IOException + { + Epoch migratedAtEpoch = Epoch.messageSerializer.deserialize(in, version); + ConsensusMigrationTarget target = ConsensusMigrationTarget.fromValue(in.readByte()); + return new ConsensusMigratedAt(migratedAtEpoch, target); + } + + @Override + public long serializedSize(ConsensusMigratedAt t, int version) + { + return TypeSizes.sizeof(ConsensusMigrationTarget.accord.value) + + Epoch.messageSerializer.serializedSize(t.migratedAtEpoch, version); + } + }); + + // Fields are not nullable when used for messaging + @Nullable + public final Epoch migratedAtEpoch; + + @Nullable + public final ConsensusMigrationTarget migratedAtTarget; + + public ConsensusMigratedAt(Epoch migratedAtEpoch, ConsensusMigrationTarget migratedAtTarget) + { + this.migratedAtEpoch = migratedAtEpoch; + this.migratedAtTarget = migratedAtTarget; + } + } + + // TODO (desired): Move this into the schema for the table once this is based off of TrM + public static class TableMigrationState + { + @Nonnull + public final String keyspaceName; + + @Nonnull + public final String tableName; + + @Nonnull + public final TableId tableId; + + @Nonnull + public final ConsensusMigrationTarget targetProtocol; + + @Nonnull + public final List> migratedRanges; + + /* + * Necessary to track which ranges started migrating at which epoch + * in order to know whether a repair qualifies in terms of finishing + * migration of the range. + */ + @Nonnull + public final NavigableMap>> migratingRangesByEpoch; + + public static final MetadataSerializer serializer = new MetadataSerializer() + { + @Override + public void serialize(TableMigrationState t, DataOutputPlus out, Version version) throws IOException + { + out.write(t.targetProtocol.value); + out.writeUTF(t.keyspaceName); + out.writeUTF(t.tableName); + t.tableId.serialize(out); + serializeCollection(t.migratedRanges, out, version, Range.serializer); + serializeMap(t.migratingRangesByEpoch, out, version, Epoch.serializer, rangesSerializer); + } + + @Override + public TableMigrationState deserialize(DataInputPlus in, Version version) throws IOException + { + ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromValue(in.readByte()); + String keyspaceName = in.readUTF(); + String tableName = in.readUTF(); + TableId tableId = TableId.deserialize(in); + Set> migratedRanges = deserializeSet(in, version, Range.serializer); + Map>> migratingRangesByEpoch = deserializeMap(in, version, Epoch.serializer, rangesSerializer, newHashMap()); + return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, migratingRangesByEpoch); + } + + @Override + public long serializedSize(TableMigrationState t, Version version) + { + return sizeof(t.targetProtocol.value) + + sizeof(t.keyspaceName) + + sizeof(t.tableName) + + t.tableId.serializedSize() + + serializedCollectionSize(t.migratedRanges, version, Range.serializer) + + serializedMapSize(t.migratingRangesByEpoch, version, Epoch.serializer, rangesSerializer); + } + }; + + @Nonnull + public final List> migratingRanges; + + @Nonnull + public final List> migratingAndMigratedRanges; + + public TableMigrationState(@Nonnull String keyspaceName, + @Nonnull String tableName, + @Nonnull TableId tableId, + @Nonnull ConsensusMigrationTarget targetProtocol, + @Nonnull Collection> migratedRanges, + @Nonnull Map>> migratingRangesByEpoch) + { + this.keyspaceName = keyspaceName; + this.tableName = tableName; + this.tableId = tableId; + this.targetProtocol = targetProtocol; + this.migratedRanges = ImmutableList.copyOf(normalize(migratedRanges)); + this.migratingRangesByEpoch = ImmutableSortedMap.copyOf( + migratingRangesByEpoch.entrySet() + .stream() + .map( entry -> new SimpleEntry<>(entry.getKey(), ImmutableList.copyOf(normalize(entry.getValue())))) + .collect(Collectors.toList())); + this.migratingRanges = ImmutableList.copyOf(normalize(migratingRangesByEpoch.values().stream().flatMap(Collection::stream).collect(Collectors.toList()))); + this.migratingAndMigratedRanges = ImmutableList.copyOf(normalize(ImmutableList.>builder().addAll(migratedRanges).addAll(migratingRanges).build())); + } + + public TableMigrationState withRangesMigrating(@Nonnull Collection> ranges, + @Nonnull ConsensusMigrationTarget target) + { + checkState(!migratingRangesByEpoch.containsKey(Epoch.EMPTY), "Shouldn't already have an entry for the empty epoch"); + // Doesn't matter which epoch the range started migrating in for this context so merge them all + Collection> migratingRanges = normalize(migratingRangesByEpoch.values().stream().flatMap(Collection::stream).collect(Collectors.toList())); + checkArgument(target == targetProtocol, "Requested migration to target protocol " + target + " conflicts with in progress migration to protocol " + targetProtocol); + List> normalizedRanges = normalize(ranges); + if (subtract(normalizedRanges, migratingRanges).isEmpty()) + logger.warn("Range " + ranges + " is already being migrated"); + Set> withoutAlreadyMigrated = subtract(normalizedRanges, migratedRanges); + if (withoutAlreadyMigrated.isEmpty()) + logger.warn("Range " + ranges + " is already migrated"); + Set> withoutBoth = subtract(withoutAlreadyMigrated, migratingRanges); + if (withoutBoth.isEmpty()) + logger.warn("Range " + ranges + " is already migrating/migrated"); + + if (!Range.equals(normalizedRanges, withoutBoth)) + logger.warn("Ranges " + normalizedRanges + " to start migrating is already partially migrating/migrated " + withoutBoth); + + Map>> newMigratingRanges = new HashMap<>(migratingRangesByEpoch.size() + 1); + newMigratingRanges.putAll(migratingRangesByEpoch); + newMigratingRanges.put(Epoch.EMPTY, normalizedRanges); + + return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, newMigratingRanges); + } + + public TableMigrationState withReplacementForEmptyEpoch(@Nonnull Epoch replacementEpoch) + { + if (!migratingRangesByEpoch.containsKey(Epoch.EMPTY)) + return this; + Map>> newMigratingRangesByEpoch = new HashMap<>(migratingRangesByEpoch.size()); + migratingRangesByEpoch.forEach((epoch, ranges) -> { + if (epoch.equals(Epoch.EMPTY)) + newMigratingRangesByEpoch.put(replacementEpoch, ranges); + else + newMigratingRangesByEpoch.put(epoch, ranges); + }); + + if (newMigratingRangesByEpoch != null) + return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, newMigratingRangesByEpoch); + else + return this; + } + + public TableMigrationState withRangesRepairedAtEpoch(@Nonnull Collection> ranges, + @Nonnull Epoch epoch) + { + checkState(!migratingRangesByEpoch.containsKey(Epoch.EMPTY), "Shouldn't have an entry for the empty epoch"); + checkArgument(epoch.isAfter(Epoch.EMPTY), "Epoch shouldn't be empty"); + + List> normalizedRepairedRanges = normalize(ranges); + // This should be inclusive because the epoch we store in the map is the epoch in which the range has been marked migrating + // in startMigrationToConsensusProtocol + NavigableMap>> coveredEpochs = migratingRangesByEpoch.headMap(epoch, true); + List> normalizedMigratingRanges = normalize(coveredEpochs.values().stream().flatMap(Collection::stream).collect(Collectors.toList())); + List> normalizedRepairedIntersection = intersectionOfNormalizedRanges(normalizedRepairedRanges, normalizedMigratingRanges); + checkState(!normalizedRepairedIntersection.isEmpty(), "None of Ranges " + ranges + " were being migrated"); + + Map>> newMigratingRangesByEpoch = new HashMap<>(); + + // Everything in this epoch or later can't have been migrated so re-add all of them + newMigratingRangesByEpoch.putAll(migratingRangesByEpoch.tailMap(epoch, false)); + + // Include anything still remaining to be migrated after subtracting what was repaired + for (Map.Entry>> e : coveredEpochs.entrySet()) + { + // Epoch when these ranges started migrating + Epoch rangesEpoch = e.getKey(); + List> epochMigratingRanges = e.getValue(); + List> remainingRanges = subtractNormalizedRanges(epochMigratingRanges, normalizedRepairedIntersection); + if (!remainingRanges.isEmpty()) + newMigratingRangesByEpoch.put(rangesEpoch, remainingRanges); + } + + List> newMigratedRanges = new ArrayList<>(normalizedMigratingRanges.size() + ranges.size()); + newMigratedRanges.addAll(migratedRanges); + newMigratedRanges.addAll(normalizedRepairedIntersection); + return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, newMigratedRanges, newMigratingRangesByEpoch); + } + + public boolean paxosReadSatisfiedByKeyMigrationAtEpoch(DecoratedKey key, ConsensusMigratedAt consensusMigratedAt) + { + // This check is being done from a Paxos read attempt which needs to + // check if Accord needs to resolve any in flight accord transactions + // if the migration target is Accord then nothing needs to be done + if (targetProtocol != ConsensusMigrationTarget.paxos) + return true; + + return satisfiedByKeyMigrationAtEpoch(key, consensusMigratedAt); + } + + public boolean satisfiedByKeyMigrationAtEpoch(@Nonnull DecoratedKey key, @Nullable ConsensusMigratedAt consensusMigratedAt) + { + if (consensusMigratedAt == null) + { + // It hasn't been migrated and needs migration if it is in a migrating range + return Range.isInNormalizedRanges(key.getToken(), migratingRanges); + } + else + { + // It has been migrated and might be from a late enough epoch to satisfy this migration + return consensusMigratedAt.migratedAtTarget == targetProtocol + && migratingRangesByEpoch.headMap(consensusMigratedAt.migratedAtEpoch, true).values() + .stream() + .flatMap(List::stream) + .anyMatch(range -> range.contains(key.getToken())); + } + } + + public Epoch minMigrationEpoch(Token token) + { + for (Map.Entry>> e : migratingRangesByEpoch.entrySet()) + { + if (Range.isInNormalizedRanges(token, e.getValue())) + return e.getKey(); + } + return Epoch.EMPTY; + } + + + public @Nonnull TableId getTableId() + { + return tableId; + } + + public TableMigrationState withMigrationTarget(ConsensusMigrationTarget newTargetProtocol) + { + checkState(!migratingRangesByEpoch.containsKey(Epoch.EMPTY), "Shouldn't have an entry for the empty epoch"); + if (this.targetProtocol == newTargetProtocol) + return this; + + // Migrating ranges remain migrating because individual keys may have already been migrated + // So for correctness we need to perform key migration + // We do need to update the epoch so that a new repair is required to drive the migration + Map>> migratingRangesByEpoch = ImmutableMap.of(Epoch.EMPTY, migratingRanges); + + Token minToken = ColumnFamilyStore.getIfExists(tableId).getPartitioner().getMinimumToken(); + Range fullRange = new Range(minToken, minToken); + // What is migrated already is anything that was never migrated/migrating before (untouched) + List> migratedRanges = ImmutableList.copyOf(normalize(fullRange.subtractAll(migratingAndMigratedRanges))); + + return new TableMigrationState(keyspaceName, tableName, tableId, newTargetProtocol, migratedRanges, migratingRangesByEpoch); + } + + public Map toMap() + { + Builder builder = ImmutableMap.builder(); + builder.put("keyspace", keyspaceName); + builder.put("table", tableName); + builder.put("tableId", tableId.toString()); + builder.put("targetProtocol", targetProtocol.toString()); + builder.put("migratedRanges", migratedRanges.stream().map(Objects::toString).collect(toImmutableList())); + Map> rangesByEpoch = new LinkedHashMap<>(); + for (Map.Entry>> entry : migratingRangesByEpoch.entrySet()) + { + rangesByEpoch.put(entry.getKey().getEpoch(), entry.getValue().stream().map(Objects::toString).collect(toImmutableList())); + } + builder.put("migratingRangesByEpoch", rangesByEpoch); + return builder.build(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TableMigrationState that = (TableMigrationState) o; + return keyspaceName.equals(that.keyspaceName) && tableName.equals(that.tableName) && tableId.equals(that.tableId) && targetProtocol == that.targetProtocol && migratedRanges.equals(that.migratedRanges) && migratingRangesByEpoch.equals(that.migratingRangesByEpoch) && migratingRanges.equals(that.migratingRanges) && migratingAndMigratedRanges.equals(that.migratingAndMigratedRanges); + } + + @Override + public int hashCode() + { + return Objects.hash(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, migratingRangesByEpoch, migratingRanges, migratingAndMigratedRanges); + } + + public List> migratingRanges() + { + return migratingRanges; + } + } + + public static class ConsensusMigrationState implements MetadataValue + { + public static ConsensusMigrationState EMPTY = new ConsensusMigrationState(Epoch.EMPTY, ImmutableMap.of()); + @Nonnull + public final Map tableStates; + + public final Epoch lastModified; + + + public ConsensusMigrationState(@Nonnull Epoch lastModified, @Nonnull Map tableStates) + { + checkNotNull(tableStates, "tableStates is null"); + checkNotNull(lastModified, "lastModified is null"); + this.lastModified = lastModified; + this.tableStates = ImmutableMap.copyOf(tableStates); + } + + public Map toMap(@Nullable Set keyspaceNames, @Nullable Set tableNames) + { + return ImmutableMap.of("lastModifiedEpoch", lastModified.getEpoch(), + "tableStates", tableStatesAsMaps(keyspaceNames, tableNames), + "version", PojoToString.CURRENT_VERSION); + } + + private List> tableStatesAsMaps(@Nullable Set keyspaceNames, + @Nullable Set tableNames) + { + ImmutableList.Builder> builder = ImmutableList.builder(); + for (TableMigrationState tms : tableStates.values()) + { + if (keyspaceNames != null && !keyspaceNames.contains(tms.keyspaceName)) + continue; + if (tableNames != null && !tableNames.contains(tms.tableName)) + continue; + builder.add(tms.toMap()); + } + return builder.build(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConsensusMigrationState that = (ConsensusMigrationState) o; + return tableStates.equals(that.tableStates); + } + + @Override + public int hashCode() + { + return Objects.hash(tableStates); + } + + public static final MetadataSerializer serializer = new MetadataSerializer() + { + @Override + public void serialize(ConsensusMigrationState consensusMigrationState, DataOutputPlus out, Version version) throws IOException + { + Epoch.serializer.serialize(consensusMigrationState.lastModified, out, version); + serializeMap(consensusMigrationState.tableStates, out, version, TableId.metadataSerializer, TableMigrationState.serializer); + } + + @Override + public ConsensusMigrationState deserialize(DataInputPlus in, Version version) throws IOException + { + Epoch lastModified = Epoch.serializer.deserialize(in, version); + Map tableMigrationStates = deserializeMap(in, version, TableId.metadataSerializer, TableMigrationState.serializer, newHashMap()); + return new ConsensusMigrationState(lastModified, tableMigrationStates); + } + + @Override + public long serializedSize(ConsensusMigrationState t, Version version) + { + return Epoch.serializer.serializedSize(t.lastModified, version) + + serializedMapSize(t.tableStates, version, TableId.metadataSerializer, TableMigrationState.serializer); + } + }; + + @Override + public ConsensusMigrationState withLastModified(Epoch epoch) + { + ImmutableMap.Builder newMap = ImmutableMap.builderWithExpectedSize(tableStates.size()); + tableStates.forEach((tableId, tableState) -> { + newMap.put(tableId, tableState.withReplacementForEmptyEpoch(epoch)); + }); + return new ConsensusMigrationState(epoch, newMap.build()); + } + + @Override + public Epoch lastModified() + { + return lastModified; + } + } + + private ConsensusTableMigrationState() {} + + // Used by callers to avoid looking up the TMS multiple times + public static @Nullable TableMigrationState getTableMigrationState(TableId tableId) + { + TableMigrationState tms = ClusterMetadata.current().consensusMigrationState.tableStates.get(tableId); + return tms; + } + + /* + * Set or change the migration target for the keyspaces and tables. Can be used to reverse the direction of a migration + * or instantly migrate a table to a new protocol. + */ + public static void setConsensusMigrationTargetProtocol(@Nonnull String targetProtocolName, + @Nullable List keyspaceNames, + @Nonnull Optional> maybeTables) + { + checkArgument(!maybeTables.isPresent() || (keyspaceNames != null && keyspaceNames.size() == 1), "Must specify one keyspace along with tables"); + checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty"); + keyspaceNames = maybeDefaultKeyspaceNames(keyspaceNames); + ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(targetProtocolName); + + if (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord) + throw new IllegalStateException("Mixing a hard coded strategy with migration is unsupported"); + + if (!Paxos.useV2()) + throw new IllegalStateException("Can't do any consensus migrations from/to PaxosV1, switch to V2 first"); + + List tableIds = keyspacesAndTablesToTableIds(keyspaceNames, maybeTables); + ClusterMetadataService.instance().commit(new SetConsensusMigrationTargetProtocol(targetProtocol, tableIds)); + } + + public static void startMigrationToConsensusProtocol(@Nonnull String targetProtocolName, + @Nullable List keyspaceNames, + @Nonnull Optional> maybeTables, + @Nonnull Optional maybeRangesStr) + { + checkState(keyspaceNames.size() == 1 || !maybeTables.isPresent(), "Must specify one keyspace along with tables"); + checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty"); + ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(targetProtocolName); + checkArgument(targetProtocol != reset, "Can't start migration to reset"); + + + if (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord) + throw new IllegalStateException("Mixing a hard coded strategy with migration is unsupported"); + + NonSerialWriteStrategy nonSerialWriteStrategy = DatabaseDescriptor.getNonSerialWriteStrategy(); + if (!nonSerialWriteStrategy.writesThroughAccord && nonSerialWriteStrategy != NonSerialWriteStrategy.mixed) + throw new IllegalStateException("non-SERIAL writes need to be routed through Accord before attempting migration, or enable mixed mode"); + + if (!Paxos.useV2()) + throw new IllegalStateException("Can't do any consensus migrations to/from PaxosV1, switch to V2 first"); + + keyspaceNames = maybeDefaultKeyspaceNames(keyspaceNames); + List> ranges = maybeRangesToRanges(maybeRangesStr); + List tableIds = keyspacesAndTablesToTableIds(keyspaceNames, maybeTables); + + ClusterMetadataService.instance().commit(new BeginConsensusMigrationForTableAndRange(targetProtocol, ranges, tableIds)); + } + + public static List finishMigrationToConsensusProtocol(@Nonnull String keyspace, + @Nonnull Optional> maybeTables, + @Nonnull Optional maybeRangesStr) + { + checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty"); + + Optional>> localKeyspaceRanges = Optional.of(ImmutableList.copyOf(StorageService.instance.getLocalReplicas(keyspace).onlyFull().ranges())); + List> ranges = maybeRangesToRanges(maybeRangesStr, localKeyspaceRanges); + Map allTableMigrationStates = ClusterMetadata.current().consensusMigrationState.tableStates; + List tableIds = keyspacesAndTablesToTableIds(ImmutableList.of(keyspace), maybeTables, Optional.of(allTableMigrationStates::containsKey)); + + checkState(tableIds.stream().allMatch(allTableMigrationStates::containsKey), "All tables need to be migrating"); + List tableMigrationStates = new ArrayList<>(); + tableIds.forEach(table -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(table); + if (cfs == null) + { + logger.warn("Table {} does not exist or was dropped", cfs); + return; + } + TableMigrationState tms = allTableMigrationStates.get(table); + if (tms == null) + { + logger.warn("Table {} does not have any migration state", cfs.name); + return; + } + if(!Range.intersects(ranges, tms.migratingRanges)) + { + logger.warn("Table {} with migrating ranges {} does not intersect with any requested ranges {}", cfs.name, tms.migratingRanges, ranges); + return; + } + tableMigrationStates.add(tms); + }); + + List migratingToAccord = tableMigrationStates.stream().filter(tms -> tms.targetProtocol == ConsensusMigrationTarget.accord).collect(toImmutableList()); + List migratingToPaxos = tableMigrationStates.stream().filter(tms -> tms.targetProtocol == ConsensusMigrationTarget.paxos).collect(toImmutableList());; + + Integer accordRepairCmd = finishMigrationToAccord(keyspace, migratingToAccord, ranges); + Integer paxosRepairCmd = finishMigrationToPaxos(keyspace, migratingToPaxos, ranges); + List result = new ArrayList<>(); + if (accordRepairCmd != null) + result.add(accordRepairCmd); + if (paxosRepairCmd != null) + result.add(paxosRepairCmd); + return result; + } + + private interface MigrationFinisher + { + Integer finish(Collection tables, List> ranges); + } + + private static Integer finishMigrationTo(String name, List tableMigrationStates, List> requestedRanges, MigrationFinisher migrationFinisher) + { + logger.info("Begin finish migration to {} for ranges {} and tables {}", name, requestedRanges, tableMigrationStates); + List> intersectingRanges = new ArrayList<>(); + tableMigrationStates.stream().map(TableMigrationState::migratingRanges).forEach(intersectingRanges::addAll); + intersectingRanges = Range.normalize(intersectingRanges); + intersectingRanges = Range.intersectionOfNormalizedRanges(intersectingRanges, requestedRanges); + if (intersectingRanges.isEmpty()) + { + logger.warn("No requested ranges {} intersect any migrating ranges in any table in keyspace {}"); + return null; + } + + // Repair requires that the ranges once again be grouped by the ranges provided originally which all + // fall within local range boundaries. This was already checked in maybeRangesToRanges. + List> intersectingRangesGrouped = new ArrayList<>(); + for (Range r : requestedRanges) + { + List> intersectionsForGroup = new ArrayList<>(); + for (Range intersectedRange : intersectingRanges) + intersectionsForGroup.addAll(r.intersectionWith(intersectedRange)); + intersectingRangesGrouped.addAll(normalize(intersectionsForGroup)); + } + return migrationFinisher.finish(tableMigrationStates, intersectingRangesGrouped); + } + + /* + * This is basically just invoking classic Cassandra repair and is pretty redundant with invoking repair + * directly which would also work without issue. It's include so the same interface works for both migrating to/from + * Accord, but it's not great in that repair has a lot of options that might need to be forwarded. + * + * Still maybe more valuable to put this layer of abstraction in so we can change how it works later and it's less + * tightly coupled with the Repair interface which is pretty orthogonal to consensus migration. + */ + private static Integer finishMigrationToAccord(String keyspace, List migratingToAccord, List> requestedRanges) + { + return finishMigrationTo("Accord", migratingToAccord, requestedRanges, (tables, intersectingRanges) -> { + RepairOption repairOption = getRepairOption(tables, intersectingRanges, false); + return StorageService.instance.repair(keyspace, repairOption, emptyList()).left; + }); + } + + private static Integer finishMigrationToPaxos(String keyspace, List migratingToPaxos, List> requestedRanges) + { + return finishMigrationTo("Paxos", migratingToPaxos, requestedRanges, (tables, intersectingRanges) -> { + RepairOption repairOption = getRepairOption(tables, intersectingRanges, true); + return StorageService.instance.repair(keyspace, repairOption, emptyList()).left; + }); + } + + @Nonnull + private static RepairOption getRepairOption(Collection tables, List> intersectingRanges, boolean accordRepair) + { + boolean primaryRange = false; + // TODO (review): Should disabling incremental repair be exposed for the Paxos repair in case someone explicitly does not do incremental repair? + boolean incremental = !accordRepair; + boolean trace = false; + int numJobThreads = 1; + boolean pullRepair = false; + boolean forceRepair = false; + boolean optimiseStreams = false; + boolean ignoreUnreplicatedKeyspaces = true; + boolean repairPaxos = !accordRepair; + boolean paxosOnly = false; + boolean dontPurgeTombstones = false; + RepairOption repairOption = new RepairOption(RepairParallelism.PARALLEL, primaryRange, incremental, trace, numJobThreads, intersectingRanges, pullRepair, forceRepair, PreviewKind.NONE, optimiseStreams, ignoreUnreplicatedKeyspaces, repairPaxos, paxosOnly, dontPurgeTombstones, accordRepair); + tables.forEach(table -> repairOption.getColumnFamilies().add(table.tableName)); + return repairOption; + } + + + // Repair is restricted to local ranges, but manipulating CMS migration state doesn't need to be restricted + private static @Nonnull List> maybeRangesToRanges(@Nonnull Optional maybeRangesStr) + { + return maybeRangesToRanges(maybeRangesStr, Optional.empty()); + } + + private static @Nonnull List> maybeRangesToRanges(@Nonnull Optional maybeRangesStr, Optional>> restrictToRanges) + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + Optional>> maybeParsedRanges = maybeRangesStr.map(rangesStr -> ImmutableList.copyOf(RepairOption.parseRanges(rangesStr, partitioner))); + Token minToken = partitioner.getMinimumToken(); + List> defaultRanges = restrictToRanges.orElse(ImmutableList.of(new Range(minToken, minToken))); + List> ranges = maybeParsedRanges.orElse(defaultRanges); + checkArgument(ranges.stream().allMatch(range -> defaultRanges.stream().anyMatch(defaultRange -> defaultRange.contains(range))), + "If ranges are specified each range must be contained within a local range (" + defaultRanges + ") for this node to allow for precise repairs. Specified " + ranges); + return ranges; + } + + private static List maybeDefaultKeyspaceNames(@Nullable List keyspaceNames) + { + if (keyspaceNames == null || keyspaceNames.isEmpty()) + { + keyspaceNames = ImmutableList.copyOf(StorageService.instance.getNonSystemKeyspaces()); + } + checkState(keyspaceNames.stream().noneMatch(SchemaConstants::isSystemKeyspace), "Migrating system keyspaces is not supported"); + return keyspaceNames; + } + + private static List keyspacesAndTablesToTableIds(@Nonnull List keyspaceNames, @Nonnull Optional> maybeTables) + { + return keyspacesAndTablesToTableIds(keyspaceNames, maybeTables, Optional.empty()); + } + + private static List keyspacesAndTablesToTableIds(@Nonnull List keyspaceNames, @Nonnull Optional> maybeTables, @Nonnull Optional> includeTable) + { + List tableIds = new ArrayList<>(); + for (String keyspaceName : keyspaceNames) + { + Optional> maybeTableIds = maybeTables.map(tableNames -> + tableNames + .stream() + .map(tableName -> { + TableMetadata tm = Schema.instance.getTableMetadata(keyspaceName, tableName); + if (tm == null) + throw new IllegalArgumentException("Unknown table %s.%s".format(keyspaceName, tableName)); + return tm.id; + }) + .collect(toImmutableList())); + tableIds.addAll( + maybeTableIds.orElseGet(() -> + Schema.instance.getKeyspaceInstance(keyspaceName).getColumnFamilyStores() + .stream() + .map(ColumnFamilyStore::getTableId) + .filter(includeTable.orElse(Predicates.alwaysTrue())) // Filter out non-migrating so they don't generate an error + .collect(toImmutableList()))); + } + return tableIds; + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosRepair.java b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosRepair.java index 05c67da3f1..7640d75e89 100644 --- a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosRepair.java +++ b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosRepair.java @@ -24,9 +24,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.function.Consumer; +import javax.annotation.Nullable; import com.google.common.base.Preconditions; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,9 +64,9 @@ public abstract class AbstractPaxosRepair public static class Result extends State { - enum Outcome { DONE, CANCELLED, FAILURE } + public enum Outcome { DONE, CANCELLED, FAILURE } - final Outcome outcome; + public final Outcome outcome; public Result(Outcome outcome) { @@ -127,15 +127,20 @@ public abstract class AbstractPaxosRepair } private final DecoratedKey partitionKey; + @Nullable private final Ballot incompleteBallot; + + protected final long retryTimeoutNanos; + private List listeners = null; private volatile State state; private volatile long startedNanos = Long.MIN_VALUE; - public AbstractPaxosRepair(DecoratedKey partitionKey, Ballot incompleteBallot) + public AbstractPaxosRepair(DecoratedKey partitionKey, Ballot incompleteBallot, long retryTimeoutNanos) { this.partitionKey = partitionKey; this.incompleteBallot = incompleteBallot; + this.retryTimeoutNanos = retryTimeoutNanos; } public State state() @@ -158,7 +163,8 @@ public abstract class AbstractPaxosRepair return isResult(state); } - public Ballot incompleteBallot() + // Shouldn't be null when used by PaxosRepairs, but will be null when used by ConsensusRequestRouter + public @Nullable Ballot incompleteBallot() { return incompleteBallot; } @@ -203,11 +209,19 @@ public abstract class AbstractPaxosRepair public State restart(State state) { return restart(state, Long.MIN_VALUE); } public abstract State restart(State state, long waitUntil); + // Used to start repairs from PaxosTableRepairs public final synchronized AbstractPaxosRepair start() + { + long startedNanos = Math.max(Long.MIN_VALUE + 1, nanoTime()); + return start(startedNanos); + } + + // Used to start repairs from ConsensusRequestRouter + public final synchronized AbstractPaxosRepair start(long queryStartNanos) { updateState(null, null, (state, i2) -> { Preconditions.checkState(!isStarted()); - startedNanos = Math.max(Long.MIN_VALUE + 1, nanoTime()); + startedNanos = queryStartNanos; return restart(state); }); return this; diff --git a/src/java/org/apache/cassandra/service/paxos/Paxos.java b/src/java/org/apache/cassandra/service/paxos/Paxos.java index 75392640a0..4bfce8953b 100644 --- a/src/java/org/apache/cassandra/service/paxos/Paxos.java +++ b/src/java/org/apache/cassandra/service/paxos/Paxos.java @@ -70,8 +70,8 @@ import org.apache.cassandra.exceptions.IsBootstrappingException; import org.apache.cassandra.exceptions.ReadFailureException; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.exceptions.RequestFailureException; import org.apache.cassandra.exceptions.RequestFailure; +import org.apache.cassandra.exceptions.RequestFailureException; import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.exceptions.RequestTimeoutException; import org.apache.cassandra.exceptions.UnavailableException; @@ -87,12 +87,15 @@ import org.apache.cassandra.metrics.ClientRequestMetrics; import org.apache.cassandra.service.CASRequest; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.FailureRecordingCallback.AsMap; +import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; import org.apache.cassandra.service.paxos.Commit.Proposal; import org.apache.cassandra.service.paxos.cleanup.PaxosRepairState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.service.paxos.PaxosPrepare.FoundIncompleteAccepted; import org.apache.cassandra.service.paxos.PaxosPrepare.FoundIncompleteCommitted; +import org.apache.cassandra.service.paxos.PaxosPropose.Superseded; import org.apache.cassandra.service.reads.DataResolver; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.service.reads.repair.NoopReadRepair; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.NodeId; @@ -104,6 +107,7 @@ import org.apache.cassandra.utils.CassandraVersion; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.NoSpamLogger; +import static com.google.common.base.Preconditions.checkState; import static java.util.Collections.emptyMap; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; @@ -125,6 +129,10 @@ import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteM import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetricsMap; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetricsMap; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.RETRY_NEW_PROTOCOL; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.casResult; +import static org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult.serialReadResult; import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL; import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL; import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; @@ -687,7 +695,7 @@ public class Paxos * Any successful prepare phase yielding a read that rejects the condition must be followed by the proposal of * an empty update, to ensure the evaluation of the condition is linearized with respect to other reads and writes. * - * @param key the row key for the row to CAS + * @param partitionKey the row key for the row to CAS * @param request the conditions for the CAS to apply as well as the update to perform if the conditions hold. * @param consistencyForConsensus the consistency for the paxos prepare and propose round. This can only be either SERIAL or LOCAL_SERIAL. * @param consistencyForCommit the consistency for write done during the commit phase. This can be anything, except SERIAL or LOCAL_SERIAL. @@ -695,48 +703,22 @@ public class Paxos * @return null if the operation succeeds in updating the row, or the current values corresponding to conditions. * (since, if the CAS doesn't succeed, it means the current value do not match the conditions). */ - public static RowIterator cas(DecoratedKey key, - CASRequest request, - ConsistencyLevel consistencyForConsensus, - ConsistencyLevel consistencyForCommit, - ClientState clientState) - throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException - { - final long start = nanoTime(); - final long proposeDeadline = start + getCasContentionTimeout(NANOSECONDS); - final long commitDeadline = Math.max(proposeDeadline, start + getWriteRpcTimeout(NANOSECONDS)); - return cas(key, request, consistencyForConsensus, consistencyForCommit, clientState, start, proposeDeadline, commitDeadline); - } - public static RowIterator cas(DecoratedKey key, - CASRequest request, - ConsistencyLevel consistencyForConsensus, - ConsistencyLevel consistencyForCommit, - ClientState clientState, - long proposeDeadline, - long commitDeadline - ) - throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException - { - return cas(key, request, consistencyForConsensus, consistencyForCommit, clientState, nanoTime(), proposeDeadline, commitDeadline); - } - private static RowIterator cas(DecoratedKey partitionKey, - CASRequest request, - ConsistencyLevel consistencyForConsensus, - ConsistencyLevel consistencyForCommit, - ClientState clientState, - long start, - long proposeDeadline, - long commitDeadline - ) + public static ConsensusAttemptResult cas(DecoratedKey partitionKey, + CASRequest request, + ConsistencyLevel consistencyForConsensus, + ConsistencyLevel consistencyForCommit, + ClientState clientState, + Dispatcher.RequestTime requestTime) throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException { + final long proposeDeadline = requestTime.startedAtNanos() + getCasContentionTimeout(NANOSECONDS); + final long commitDeadline = Math.max(proposeDeadline, requestTime.startedAtNanos() + getWriteRpcTimeout(NANOSECONDS)); SinglePartitionReadCommand readCommand = request.readCommand(FBUtilities.nowInSeconds()); TableMetadata metadata = readCommand.metadata(); consistencyForConsensus.validateForCas(); consistencyForCommit.validateForCasCommit(Keyspace.open(metadata.keyspace).getReplicationStrategy()); - Ballot minimumBallot = null; int failedAttemptsDueToContention = 0; try (PaxosOperationLock lock = PaxosState.lock(partitionKey, metadata, proposeDeadline, consistencyForConsensus, true)) { @@ -747,7 +729,14 @@ public class Paxos Tracing.trace("Reading existing values for CAS precondition"); BeginResult begin = begin(proposeDeadline, readCommand, consistencyForConsensus, - true, minimumBallot, failedAttemptsDueToContention, request.requestTime()); + true, null, failedAttemptsDueToContention, request.requestTime()); + + if (begin.retryWithNewConsenusProtocol) + { + casWriteMetrics.beginMigrationRejects.mark(); + return RETRY_NEW_PROTOCOL; + } + Ballot ballot = begin.ballot; Participants participants = begin.participants; failedAttemptsDueToContention = begin.failedAttemptsDueToContention; @@ -766,7 +755,7 @@ public class Paxos { Tracing.trace("CAS precondition rejected", current); casWriteMetrics.conditionNotMet.inc(); - return current.rowIterator(); + return casResult(current.rowIterator(false)); } // If we failed to meet our condition, it does not mean we can do nothing: if we do not propose @@ -782,7 +771,7 @@ public class Paxos if (begin.isLinearizableRead) { Tracing.trace("CAS precondition does not match current values {}; read is already linearizable; aborting", current); - return conditionNotMet(current); + return casResult(conditionNotMet(current)); } Tracing.trace("CAS precondition does not match current values {}; proposing empty update", current); @@ -816,7 +805,7 @@ public class Paxos continue; } - PaxosPropose.Status propose = propose(proposal, participants, conditionMet).awaitUntil(proposeDeadline); + PaxosPropose.Status propose = propose(proposal, participants, conditionMet, false).awaitUntil(proposeDeadline); switch (propose.outcome) { default: throw new IllegalStateException(); @@ -827,7 +816,7 @@ public class Paxos case SUCCESS: { if (!conditionMet) - return conditionNotMet(current); + return casResult(conditionNotMet(current)); // no need to commit a no-op; either it // 1) reached a majority, in which case it was agreed, had no effect and we can do nothing; or @@ -840,7 +829,8 @@ public class Paxos case SUPERSEDED: { - switch (propose.superseded().hadSideEffects) + Superseded superseded = propose.superseded(); + switch (superseded.hadSideEffects) { default: throw new IllegalStateException(); @@ -852,7 +842,12 @@ public class Paxos .markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention); case NO: - minimumBallot = propose.superseded().by; + // Shouldn't retry on this protocol + if (superseded.needsConsensusMigration) + { + casWriteMetrics.acceptMigrationRejects.mark(); + return RETRY_NEW_PROTOCOL; + } // We have been superseded without our proposal being accepted by anyone, so we can safely retry Tracing.trace("Paxos proposal not accepted (pre-empted by a higher ballot)"); if (!waitForContention(proposeDeadline, ++failedAttemptsDueToContention, metadata, partitionKey, consistencyForConsensus, WRITE)) @@ -870,12 +865,12 @@ public class Paxos throw result.maybeFailure().markAndThrowAsTimeoutOrFailure(true, consistencyForCommit, failedAttemptsDueToContention); } Tracing.trace("CAS successful"); - return null; + return casResult((RowIterator)null); } finally { - final long latency = nanoTime() - start; + final long latency = nanoTime() - requestTime.startedAtNanos(); if (failedAttemptsDueToContention > 0) { @@ -893,28 +888,16 @@ public class Paxos { Tracing.trace("CAS precondition rejected", read); casWriteMetrics.conditionNotMet.inc(); - return read.rowIterator(); + return read.rowIterator(false); } - public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, Dispatcher.RequestTime requestTime) - throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException - { - long deadline = requestTime.computeDeadline(DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS)); - return read(group, consistencyForConsensus, requestTime, deadline); - } - - public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, long deadline) - throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException - { - return read(group, consistencyForConsensus, Dispatcher.RequestTime.forImmediateExecution(), deadline); - } - - private static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, Dispatcher.RequestTime requestTime, long deadline) + public static ConsensusAttemptResult read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, Dispatcher.RequestTime requestTime) throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException { long start = nanoTime(); if (group.queries.size() > 1) throw new InvalidRequestException("SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time"); + long deadline = requestTime.computeDeadline(DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS)); int failedAttemptsDueToContention = 0; Ballot minimumBallot = null; @@ -927,6 +910,12 @@ public class Paxos final BeginResult begin = begin(deadline, read, consistencyForConsensus, false, minimumBallot, failedAttemptsDueToContention, requestTime); failedAttemptsDueToContention = begin.failedAttemptsDueToContention; + if (begin.retryWithNewConsenusProtocol) + { + casReadMetrics.beginMigrationRejects.mark(); + return RETRY_NEW_PROTOCOL; + } + switch (PAXOS_VARIANT) { default: @@ -935,16 +924,16 @@ public class Paxos case v2_without_linearizable_reads_or_rejected_writes: case v2_without_linearizable_reads: - return begin.readResponse; + return serialReadResult(begin.readResponse); case v2: // no need to submit an empty proposal, as the promise will be treated as complete for future optimistic reads if (begin.isLinearizableRead) - return begin.readResponse; + return serialReadResult(begin.readResponse); } Proposal proposal = Proposal.empty(begin.ballot, read.partitionKey(), read.metadata()); - PaxosPropose.Status propose = propose(proposal, begin.participants, false).awaitUntil(deadline); + PaxosPropose.Status propose = propose(proposal, begin.participants, true, false).awaitUntil(deadline); switch (propose.outcome) { default: throw new IllegalStateException(); @@ -953,10 +942,21 @@ public class Paxos throw propose.maybeFailure().markAndThrowAsTimeoutOrFailure(false, consistencyForConsensus, failedAttemptsDueToContention); case SUCCESS: - return begin.readResponse; + return serialReadResult(begin.readResponse); case SUPERSEDED: - switch (propose.superseded().hadSideEffects) + Superseded superseded = propose.superseded(); + // For consensus migration we are going to bail out earlier if migration is needed + // otherwise it it will fail every single query that races with migration being started + // during the propose step. Necessary because of CASSANDRA-18276 + // Shouldn't retry again on this protocol + if (superseded.needsConsensusMigration) + { + casReadMetrics.acceptMigrationRejects.mark(); + return RETRY_NEW_PROTOCOL; + } + // TODO https://issues.apache.org/jira/browse/CASSANDRA-18276 side effects shouldn't matter for reads + switch (superseded.hadSideEffects) { default: throw new IllegalStateException(); @@ -974,6 +974,7 @@ public class Paxos if (!waitForContention(deadline, ++failedAttemptsDueToContention, group.metadata(), group.queries.get(0).partitionKey(), consistencyForConsensus, READ)) throw MaybeFailure.noResponses(begin.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention); } + break; } } } @@ -1004,9 +1005,11 @@ public class Paxos final boolean isPromised; final Ballot retryWithAtLeast; - public BeginResult(Ballot ballot, Participants participants, int failedAttemptsDueToContention, PartitionIterator readResponse, boolean isLinearizableRead, boolean isPromised, Ballot retryWithAtLeast) + final boolean retryWithNewConsenusProtocol; + + public BeginResult(Ballot ballot, Participants participants, int failedAttemptsDueToContention, PartitionIterator readResponse, boolean isLinearizableRead, boolean isPromised, Ballot retryWithAtLeast, boolean retryWithNewConsenusProtocol) { - assert isPromised || isLinearizableRead; + assert isPromised || isLinearizableRead || retryWithNewConsenusProtocol; this.ballot = ballot; this.participants = participants; this.failedAttemptsDueToContention = failedAttemptsDueToContention; @@ -1014,6 +1017,12 @@ public class Paxos this.isLinearizableRead = isLinearizableRead; this.isPromised = isPromised; this.retryWithAtLeast = retryWithAtLeast; + this.retryWithNewConsenusProtocol = retryWithNewConsenusProtocol; + } + + static BeginResult retryOnNewProtocol() + { + return new BeginResult(null, null, -1, null, false, false, null, true); } } @@ -1057,6 +1066,14 @@ public class Paxos // prepare PaxosPrepare retry = null; PaxosPrepare.Status prepare = preparing.awaitUntil(deadline); + + // After performing the prepare phase we may discover that we can't propose + // our own transaction on this protocol by discovering a new CM Epoch + if (ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeDuringPaxosBegin(query.metadata().id, query.partitionKey())) + { + return BeginResult.retryOnNewProtocol(); + } + boolean isPromised = false; retry: switch (prepare.outcome) { @@ -1085,7 +1102,7 @@ public class Paxos // and in fact it's possible for a CAS to sometimes determine if side effects occurred by reading // the underlying data and not witnessing the timestamp of its ballot (or any newer for the relevant data). Proposal repropose = new Proposal(inProgress.ballot, inProgress.accepted.update); - PaxosPropose.Status proposeResult = propose(repropose, inProgress.participants, false).awaitUntil(deadline); + PaxosPropose.Status proposeResult = propose(repropose, inProgress.participants, false, true).awaitUntil(deadline); switch (proposeResult.outcome) { default: throw new IllegalStateException(); @@ -1098,6 +1115,7 @@ public class Paxos break retry; case SUPERSEDED: + checkState(!proposeResult.superseded().needsConsensusMigration, "Should not receive needsConsensusMigration rejects from begin"); // since we are proposing a previous value that was maybe superseded by us before completion // we don't need to test the side effects, as we just want to start again, and fall through // to the superseded section below @@ -1122,7 +1140,7 @@ public class Paxos PaxosPrepare.Success success = prepare.success(); Supplier plan = () -> success.participants; - DataResolver resolver = new DataResolver<>(query, plan, NoopReadRepair.instance, requestTime); + DataResolver resolver = new DataResolver<>(ReadCoordinator.DEFAULT, query, plan, NoopReadRepair.instance, requestTime); for (int i = 0 ; i < success.responses.size() ; ++i) resolver.preprocess(success.responses.get(i)); @@ -1140,7 +1158,7 @@ public class Paxos break; } - return new BeginResult(success.ballot, success.participants, failedAttemptsDueToContention, result, !hadShortRead.v && success.isReadSafe, isPromised, success.supersededBy); + return new BeginResult(success.ballot, success.participants, failedAttemptsDueToContention, result, !hadShortRead.v && success.isReadSafe, isPromised, success.supersededBy, false); } case MAYBE_FAILURE: diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosCommitAndPrepare.java b/src/java/org/apache/cassandra/service/paxos/PaxosCommitAndPrepare.java index 7046dfbb37..b81f6b720b 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosCommitAndPrepare.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosCommitAndPrepare.java @@ -45,7 +45,7 @@ public class PaxosCommitAndPrepare static PaxosPrepare commitAndPrepare(Agreed commit, Paxos.Participants participants, SinglePartitionReadCommand readCommand, boolean isWrite, boolean acceptEarlyReadSuccess) { Ballot ballot = newBallot(commit.ballot, participants.consistencyForConsensus); - Request request = new Request(commit, ballot, participants.electorate, readCommand, isWrite); + Request request = new Request(commit, ballot, participants.electorate, readCommand, isWrite, true); PaxosPrepare prepare = new PaxosPrepare(participants, request, acceptEarlyReadSuccess, null); Tracing.trace("Committing {}; Preparing {}", commit.ballot, ballot); @@ -59,21 +59,21 @@ public class PaxosCommitAndPrepare { final Agreed commit; - Request(Agreed commit, Ballot ballot, Paxos.Electorate electorate, SinglePartitionReadCommand read, boolean isWrite) + Request(Agreed commit, Ballot ballot, Paxos.Electorate electorate, SinglePartitionReadCommand read, boolean isWrite, boolean isForRecovery) { - super(ballot, electorate, read, isWrite); + super(ballot, electorate, read, isWrite, isForRecovery); this.commit = commit; } - private Request(Agreed commit, Ballot ballot, Paxos.Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite) + private Request(Agreed commit, Ballot ballot, Paxos.Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite, boolean isForRecovery) { - super(ballot, electorate, partitionKey, table, isWrite); + super(ballot, electorate, partitionKey, table, isWrite, isForRecovery); this.commit = commit; } Request withoutRead() { - return new Request(commit, ballot, electorate, partitionKey, table, isForWrite); + return new Request(commit, ballot, electorate, partitionKey, table, isForWrite, isForRecovery); } public String toString() @@ -84,14 +84,14 @@ public class PaxosCommitAndPrepare public static class RequestSerializer extends PaxosPrepare.AbstractRequestSerializer { - Request construct(Agreed param, Ballot ballot, Paxos.Electorate electorate, SinglePartitionReadCommand read, boolean isWrite) + Request construct(Agreed param, Ballot ballot, Paxos.Electorate electorate, SinglePartitionReadCommand read, boolean isWrite, boolean isForRecovery) { - return new Request(param, ballot, electorate, read, isWrite); + return new Request(param, ballot, electorate, read, isWrite, isForRecovery); } - Request construct(Agreed param, Ballot ballot, Paxos.Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite) + Request construct(Agreed param, Ballot ballot, Paxos.Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite, boolean isForRecovery) { - return new Request(param, ballot, electorate, partitionKey, table, isWrite); + return new Request(param, ballot, electorate, partitionKey, table, isWrite, isForRecovery); } @Override diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java b/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java index 4a3d69eee1..eb11bf84ac 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java @@ -41,6 +41,7 @@ import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.ReadExecutionController; import org.apache.cassandra.db.ReadResponse; import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.exceptions.UnavailableException; @@ -57,6 +58,7 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.KeyMigrationState; import org.apache.cassandra.service.paxos.PaxosPrepare.Status.Outcome; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; @@ -69,6 +71,8 @@ import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer; import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REQ; import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_RSP; +import static org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.getKeyMigrationState; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigratedAt; import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; import static org.apache.cassandra.service.paxos.Commit.Accepted; import static org.apache.cassandra.service.paxos.Commit.Committed; @@ -369,7 +373,7 @@ public class PaxosPrepare extends PaxosRequestCallback im static PaxosPrepare prepareWithBallot(Ballot ballot, Participants participants, SinglePartitionReadCommand readCommand, boolean isWrite, boolean acceptEarlyReadPermission) { Tracing.trace("Preparing {} with read", ballot); - Request request = new Request(ballot, participants.electorate, readCommand, isWrite); + Request request = new Request(ballot, participants.electorate, readCommand, isWrite, true); return prepareWithBallotInternal(participants, request, acceptEarlyReadPermission, null); } @@ -377,7 +381,7 @@ public class PaxosPrepare extends PaxosRequestCallback im static > T prepareWithBallot(Ballot ballot, Participants participants, DecoratedKey partitionKey, TableMetadata table, boolean isWrite, boolean acceptEarlyReadPermission, T onDone) { Tracing.trace("Preparing {}", ballot); - prepareWithBallotInternal(participants, new Request(ballot, participants.electorate, partitionKey, table, isWrite), acceptEarlyReadPermission, onDone); + prepareWithBallotInternal(participants, new Request(ballot, participants.electorate, partitionKey, table, isWrite, true), acceptEarlyReadPermission, onDone); return onDone; } @@ -936,8 +940,9 @@ public class PaxosPrepare extends PaxosRequestCallback im final boolean isForWrite; final DecoratedKey partitionKey; final TableMetadata table; + final boolean isForRecovery; - AbstractRequest(Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isForWrite) + AbstractRequest(Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isForWrite, boolean isForRecovery) { this.ballot = ballot; this.electorate = electorate; @@ -945,9 +950,10 @@ public class PaxosPrepare extends PaxosRequestCallback im this.isForWrite = isForWrite; this.partitionKey = read.partitionKey(); this.table = read.metadata(); + this.isForRecovery = isForRecovery; } - AbstractRequest(Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isForWrite) + AbstractRequest(Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isForWrite, boolean isForRecovery) { this.ballot = ballot; this.electorate = electorate; @@ -955,6 +961,7 @@ public class PaxosPrepare extends PaxosRequestCallback im this.table = table; this.read = null; this.isForWrite = isForWrite; + this.isForRecovery = isForRecovery; } abstract R withoutRead(); @@ -967,19 +974,19 @@ public class PaxosPrepare extends PaxosRequestCallback im static class Request extends AbstractRequest { - Request(Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isWrite) + Request(Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isWrite, boolean isForRecovery) { - super(ballot, electorate, read, isWrite); + super(ballot, electorate, read, isWrite, isForRecovery); } - private Request(Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite) + private Request(Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite, boolean isForRecovery) { - super(ballot, electorate, partitionKey, table, isWrite); + super(ballot, electorate, partitionKey, table, isWrite, isForRecovery); } Request withoutRead() { - return read == null ? this : new Request(ballot, electorate, partitionKey, table, isForWrite); + return read == null ? this : new Request(ballot, electorate, partitionKey, table, isForWrite, isForRecovery); } public String toString() @@ -990,11 +997,16 @@ public class PaxosPrepare extends PaxosRequestCallback im static class Response { + @Nonnull final MaybePromise.Outcome outcome; - Response(MaybePromise.Outcome outcome) + @Nullable + final ConsensusMigratedAt maybeConsenusMigratedAt; + + Response(@Nonnull MaybePromise.Outcome outcome, @Nullable ConsensusMigratedAt maybeConsenusMigratedAt) { this.outcome = outcome; + this.maybeConsenusMigratedAt = maybeConsenusMigratedAt; } Permitted permitted() { return (Permitted) this; } Rejected rejected() { return (Rejected) this; } @@ -1024,9 +1036,9 @@ public class PaxosPrepare extends PaxosRequestCallback im @Nullable final Ballot supersededBy; final Epoch electorateEpoch; - Permitted(MaybePromise.Outcome outcome, long lowBound, @Nullable Accepted latestAcceptedButNotCommitted, Committed latestCommitted, @Nullable ReadResponse readResponse, boolean hadProposalStability, Map gossipInfo, Epoch electorateEpoch, @Nullable Ballot supersededBy) + Permitted(MaybePromise.Outcome outcome, @Nullable ConsensusMigratedAt maybeConsensusMigratedAt, long lowBound, @Nullable Accepted latestAcceptedButNotCommitted, Committed latestCommitted, @Nullable ReadResponse readResponse, boolean hadProposalStability, Map gossipInfo, Epoch electorateEpoch, @Nullable Ballot supersededBy) { - super(outcome); + super(outcome, maybeConsensusMigratedAt); this.lowBound = lowBound; this.latestAcceptedButNotCommitted = latestAcceptedButNotCommitted; this.latestCommitted = latestCommitted; @@ -1048,9 +1060,9 @@ public class PaxosPrepare extends PaxosRequestCallback im { final Ballot supersededBy; - Rejected(Ballot supersededBy) + Rejected(Ballot supersededBy, @Nullable ConsensusMigratedAt maybeConsensusMigratedAt) { - super(REJECT); + super(REJECT, maybeConsensusMigratedAt); this.supersededBy = supersededBy; } @@ -1094,6 +1106,7 @@ public class PaxosPrepare extends PaxosRequestCallback im static Response execute(AbstractRequest request, PaxosState state) { MaybePromise result = state.promiseIfNewer(request.ballot, request.isForWrite); + KeyMigrationState keyMigrationState = getKeyMigrationState(request.table.id, request.partitionKey); switch (result.outcome) { case PROMISE: @@ -1129,6 +1142,8 @@ public class PaxosPrepare extends PaxosRequestCallback im if (request.read != null) { + // Make sure the read is safe and there is no Accord state that needs application + keyMigrationState.maybePerformAccordToPaxosKeyMigration(request.isForWrite); try (ReadExecutionController executionController = request.read.executionController(); UnfilteredPartitionIterator iterator = request.read.executeLocally(executionController)) { @@ -1150,10 +1165,10 @@ public class PaxosPrepare extends PaxosRequestCallback im ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(request.table.id); long lowBound = cfs.getPaxosRepairLowBound(request.partitionKey).uuidTimestamp(); - return new Permitted(result.outcome, lowBound, acceptedButNotCommitted, committed, readResponse, hasProposalStability, gossipInfo, electorateEpoch, supersededBy); + return new Permitted(result.outcome, keyMigrationState.consensusMigratedAt, lowBound, acceptedButNotCommitted, committed, readResponse, hasProposalStability, gossipInfo, electorateEpoch, supersededBy); case REJECT: - return new Rejected(result.supersededBy()); + return new Rejected(result.supersededBy(), keyMigrationState.consensusMigratedAt); default: throw new IllegalStateException(); @@ -1163,8 +1178,8 @@ public class PaxosPrepare extends PaxosRequestCallback im static abstract class AbstractRequestSerializer, T> implements IVersionedSerializer { - abstract R construct(T param, Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isWrite); - abstract R construct(T param, Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite); + abstract R construct(T param, Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isWrite, boolean isForRecovery); + abstract R construct(T param, Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite, boolean isForRecovery); @Override public void serialize(R request, DataOutputPlus out, int version) throws IOException @@ -1182,6 +1197,8 @@ public class PaxosPrepare extends PaxosRequestCallback im request.table.id.serialize(out); DecoratedKey.serializer.serialize(request.partitionKey, out, version); } + if (version >= MessagingService.VERSION_51) + out.writeBoolean(request.isForRecovery); } public R deserialize(T param, DataInputPlus in, int version) throws IOException @@ -1192,38 +1209,47 @@ public class PaxosPrepare extends PaxosRequestCallback im if ((flag & 1) != 0) { SinglePartitionReadCommand readCommand = (SinglePartitionReadCommand) ReadCommand.serializer.deserialize(in, version); - return construct(param, ballot, electorate, readCommand, (flag & 2) == 0); + boolean isForRecovery = false; + if (version >= MessagingService.VERSION_51) + isForRecovery = in.readBoolean(); + return construct(param, ballot, electorate, readCommand, (flag & 2) == 0, isForRecovery); } else { TableMetadata table = Schema.instance.getExistingTableMetadata(TableId.deserialize(in)); DecoratedKey partitionKey = (DecoratedKey) DecoratedKey.serializer.deserialize(in, table.partitioner, version); - return construct(param, ballot, electorate, partitionKey, table, (flag & 2) != 0); + boolean isForRecovery = false; + if (version >= MessagingService.VERSION_51) + isForRecovery = in.readBoolean(); + return construct(param, ballot, electorate, partitionKey, table, (flag & 2) != 0, isForRecovery); } } @Override public long serializedSize(R request, int version) { - return Ballot.sizeInBytes() + long size = Ballot.sizeInBytes() + Electorate.serializer.serializedSize(request.electorate, version) + 1 + (request.read != null ? ReadCommand.serializer.serializedSize(request.read, version) : request.table.id.serializedSize() + DecoratedKey.serializer.serializedSize(request.partitionKey, version)); + if (version >= MessagingService.VERSION_51) + size += TypeSizes.sizeof(request.isForRecovery); + return size; } } public static class RequestSerializer extends AbstractRequestSerializer { - Request construct(Object ignore, Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isWrite) + Request construct(Object ignore, Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isWrite, boolean isForRecovery) { - return new Request(ballot, electorate, read, isWrite); + return new Request(ballot, electorate, read, isWrite, isForRecovery); } - Request construct(Object ignore, Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite) + Request construct(Object ignore, Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite, boolean isForRecovery) { - return new Request(ballot, electorate, partitionKey, table, isWrite); + return new Request(ballot, electorate, partitionKey, table, isWrite, isForRecovery); } public Request deserialize(DataInputPlus in, int version) throws IOException @@ -1232,6 +1258,14 @@ public class PaxosPrepare extends PaxosRequestCallback im } } + private static void serializeRejection(DataOutputPlus out, Ballot supersededBy, ConsensusMigratedAt maybeConsenusMigratedAt, int version) throws IOException + { + out.writeByte(0); + supersededBy.serialize(out); + if (version >= MessagingService.VERSION_51) + ConsensusMigratedAt.serializer.serialize(maybeConsenusMigratedAt, out, version); + } + public static class ResponseSerializer implements IVersionedSerializer { public void serialize(Response response, DataOutputPlus out, int version) throws IOException @@ -1240,7 +1274,7 @@ public class PaxosPrepare extends PaxosRequestCallback im { out.writeByte(0); Rejected rejected = (Rejected) response; - rejected.supersededBy.serialize(out); + serializeRejection(out, rejected.supersededBy, rejected.maybeConsenusMigratedAt, version); } else { @@ -1262,6 +1296,8 @@ public class PaxosPrepare extends PaxosRequestCallback im Epoch.messageSerializer.serialize(promised.electorateEpoch, out, version); if (promised.outcome == PERMIT_READ) promised.supersededBy.serialize(out); + if (version >= MessagingService.VERSION_51) + ConsensusMigratedAt.serializer.serialize(response.maybeConsenusMigratedAt, out, version); } } @@ -1271,7 +1307,10 @@ public class PaxosPrepare extends PaxosRequestCallback im if (flags == 0) { Ballot supersededBy = Ballot.deserialize(in); - return new Rejected(supersededBy); + ConsensusMigratedAt consensusMigratedAt = null; + if (version >= MessagingService.VERSION_51) + consensusMigratedAt = ConsensusMigratedAt.serializer.deserialize(in, version); + return new Rejected(supersededBy, consensusMigratedAt); } else { @@ -1286,15 +1325,20 @@ public class PaxosPrepare extends PaxosRequestCallback im Ballot supersededBy = null; if (outcome == PERMIT_READ) supersededBy = Ballot.deserialize(in); - return new Permitted(outcome, lowBound, acceptedNotCommitted, committed, readResponse, hasProposalStability, gossipInfo, electorateEpoch, supersededBy); + ConsensusMigratedAt consensusMigratedAt = null; + if (version >= MessagingService.VERSION_51) + consensusMigratedAt = ConsensusMigratedAt.serializer.deserialize(in, version); + return new Permitted(outcome, consensusMigratedAt, lowBound, acceptedNotCommitted, committed, readResponse, hasProposalStability, gossipInfo, electorateEpoch, supersededBy); } } public long serializedSize(Response response, int version) { + long size; if (response.isRejected()) { - return 1 + Ballot.sizeInBytes(); + size = 1 + Ballot.sizeInBytes(); + } else { @@ -1308,6 +1352,10 @@ public class PaxosPrepare extends PaxosRequestCallback im + (version >= MessagingService.VERSION_51 ? Epoch.messageSerializer.serializedSize(permitted.electorateEpoch, version) : 0) + (permitted.outcome == PERMIT_READ ? Ballot.sizeInBytes() : 0); } + if (version >= MessagingService.VERSION_51) + size += ConsensusMigratedAt.serializer.serializedSize(response.maybeConsenusMigratedAt, version); + + return size; } } diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosPropose.java b/src/java/org/apache/cassandra/service/paxos/PaxosPropose.java index 77a6fb4971..650bd8818c 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosPropose.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosPropose.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.Consumer; +import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; @@ -38,13 +39,16 @@ import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.paxos.Commit.Proposal; +import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.utils.concurrent.ConditionAsConsumer; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Collections.emptyMap; import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; import static org.apache.cassandra.net.Verb.PAXOS2_PROPOSE_REQ; -import static org.apache.cassandra.service.paxos.PaxosPropose.Superseded.SideEffects.MAYBE; -import static org.apache.cassandra.service.paxos.PaxosPropose.Superseded.SideEffects.NO; +import static org.apache.cassandra.service.paxos.PaxosPropose.Status.SideEffects.MAYBE; +import static org.apache.cassandra.service.paxos.PaxosPropose.Status.SideEffects.NO; +import static org.apache.cassandra.service.paxos.PaxosState.AcceptResult; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.utils.concurrent.ConditionAsConsumer.newConditionAsConsumer; @@ -54,13 +58,13 @@ import static org.apache.cassandra.utils.concurrent.ConditionAsConsumer.newCondi * indicating (respectively) that we have had no side effect, or that we cannot * know if we our proposal produced a side effect. */ -public class PaxosPropose> extends PaxosRequestCallback +public class PaxosPropose> extends PaxosRequestCallback { private static final Logger logger = LoggerFactory.getLogger(PaxosPropose.class); public static final RequestHandler requestHandler = new RequestHandler(); public static final RequestSerializer requestSerializer = new RequestSerializer(); - public static final ResponseSerializer responseSerializer = new ResponseSerializer(); + public static final AcceptResultSerializer ACCEPT_RESULT_SERIALIZER = new AcceptResultSerializer(); /** * Represents the current status of a propose action: it is a status rather than a result, @@ -77,22 +81,30 @@ public class PaxosPropose> } Superseded superseded() { return (Superseded) this; } Paxos.MaybeFailure maybeFailure() { return ((MaybeFailure) this).info; } - public String toString() { return "Success"; } + public String toString() { return outcome.toString(); } + + enum SideEffects { NO, MAYBE } } static class Superseded extends Status { - enum SideEffects { NO, MAYBE } + @Nullable final Ballot by; final SideEffects hadSideEffects; - Superseded(Ballot by, SideEffects hadSideEffects) + // Consensus migration can occur at the same time that we are superseded + // and it's important to preserve returning the uncertainty of the superseded + // at the same time as enforcing the need for consensus migration + final boolean needsConsensusMigration; + Superseded(@Nullable Ballot by, SideEffects hadSideEffects, boolean needsConsensusMigration) { super(Outcome.SUPERSEDED); + checkArgument(needsConsensusMigration == true || by != null, "Must be superseded by ballot if not due to consensus migration"); this.by = by; this.hadSideEffects = hadSideEffects; + this.needsConsensusMigration = needsConsensusMigration; } - public String toString() { return "Superseded(" + by + ',' + hadSideEffects + ')'; } + public String toString() { return "Superseded(" + by + ',' + hadSideEffects + ',' + needsConsensusMigration + ')'; } } private static class MaybeFailure extends Status @@ -107,7 +119,7 @@ public class PaxosPropose> public String toString() { return info.toString(); } } - private static final Status success = new Status(Status.Outcome.SUCCESS); + private static final Status STATUS_SUCCESS = new Status(Status.Outcome.SUCCESS); private static final AtomicLongFieldUpdater responsesUpdater = AtomicLongFieldUpdater.newUpdater(PaxosPropose.class, "responses"); private static final AtomicReferenceFieldUpdater supersededByUpdater = AtomicReferenceFieldUpdater.newUpdater(PaxosPropose.class, Ballot.class, "supersededBy"); @@ -126,6 +138,10 @@ public class PaxosPropose> final int participants; /** Number of accepts required */ final int required; + + /** Repairing an in flight txn not proposing a new one **/ + final boolean isForRecovery; + /** Invoke on reaching a terminal status */ final OnDone onDone; @@ -145,7 +161,9 @@ public class PaxosPropose> /** The newest superseding ballot from a refusal; only returned to the caller if we fail to reach a quorum */ private volatile Ballot supersededBy; - private PaxosPropose(Proposal proposal, int participants, int required, boolean waitForNoSideEffect, OnDone onDone) + private volatile boolean needsConsensusMigration = false; + + private PaxosPropose(Proposal proposal, int participants, int required, boolean waitForNoSideEffect, boolean isForRecovery, OnDone onDone) { this.proposal = proposal; assert required > 0; @@ -153,6 +171,7 @@ public class PaxosPropose> this.participants = participants; this.required = required; this.onDone = onDone; + this.isForRecovery = isForRecovery; } /** @@ -160,8 +179,10 @@ public class PaxosPropose> * or for the present status if the time elapses without a final result being reached. * @param waitForNoSideEffect if true, on failure we will wait until we can say with certainty there are no side effects * or until we know we will never be able to determine this with certainty + * @param isForRecovery if true the value being proposed is not a new value it is a value from an existing in flight proposal + * and will be allowed to proceed even if the key is migrating to a different consensus protocol */ - static Paxos.Async propose(Proposal proposal, Paxos.Participants participants, boolean waitForNoSideEffect) + static Paxos.Async propose(Proposal proposal, Paxos.Participants participants, boolean waitForNoSideEffect, boolean isForRecovery) { if (waitForNoSideEffect && proposal.update.isEmpty()) waitForNoSideEffect = false; // by definition this has no "side effects" (besides linearizing the operation) @@ -169,9 +190,9 @@ public class PaxosPropose> // to avoid unnecessary object allocations we extend PaxosPropose to implements Paxos.Async class Async extends PaxosPropose> implements Paxos.Async { - private Async(Proposal proposal, int participants, int required, boolean waitForNoSideEffect) + private Async(Proposal proposal, int participants, int required, boolean waitForNoSideEffect, boolean isForRecovery) { - super(proposal, participants, required, waitForNoSideEffect, newConditionAsConsumer()); + super(proposal, participants, required, waitForNoSideEffect, isForRecovery, newConditionAsConsumer()); } public Status awaitUntil(long deadline) @@ -190,24 +211,24 @@ public class PaxosPropose> } } - Async propose = new Async(proposal, participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, waitForNoSideEffect); + Async propose = new Async(proposal, participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, waitForNoSideEffect, isForRecovery); propose.start(participants); return propose; } - static > T propose(Proposal proposal, Paxos.Participants participants, boolean waitForNoSideEffect, T onDone) + static > T propose(Proposal proposal, Paxos.Participants participants, boolean waitForNoSideEffect, boolean isForRecovery, T onDone) { if (waitForNoSideEffect && proposal.update.isEmpty()) waitForNoSideEffect = false; // by definition this has no "side effects" (besides linearizing the operation) - PaxosPropose propose = new PaxosPropose<>(proposal, participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, waitForNoSideEffect, onDone); + PaxosPropose propose = new PaxosPropose<>(proposal, participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, waitForNoSideEffect, isForRecovery, onDone); propose.start(participants); return onDone; } void start(Paxos.Participants participants) { - Message message = Message.out(PAXOS2_PROPOSE_REQ, new Request(proposal), participants.isUrgent()); + Message message = Message.out(PAXOS2_PROPOSE_REQ, new Request(proposal, isForRecovery), participants.isUrgent()); boolean executeOnSelf = false; for (int i = 0, size = participants.sizeOfPoll(); i < size ; ++i) @@ -219,7 +240,7 @@ public class PaxosPropose> } if (executeOnSelf) - PAXOS2_PROPOSE_REQ.stage.execute(() -> executeOnSelf(proposal)); + PAXOS2_PROPOSE_REQ.stage.execute(() -> executeOnSelf(proposal, isForRecovery)); } /** @@ -230,35 +251,39 @@ public class PaxosPropose> long responses = this.responses; if (isSuccessful(responses)) - return success; + return STATUS_SUCCESS; - if (!canSucceed(responses) && supersededBy != null) + if (!canSucceed(responses) && (supersededBy != null || needsConsensusMigration)) { Superseded.SideEffects sideEffects = hasNoSideEffects(responses) ? NO : MAYBE; - return new Superseded(supersededBy, sideEffects); + return new Superseded(supersededBy, sideEffects, needsConsensusMigration); } return new MaybeFailure(new Paxos.MaybeFailure(participants, required, accepts(responses), failureReasonsAsMap())); } - private void executeOnSelf(Proposal proposal) + private void executeOnSelf(Proposal proposal, boolean isForRecovery) { - executeOnSelf(proposal, RequestHandler::execute); + executeOnSelf(proposal, isForRecovery, RequestHandler::execute); } - public void onResponse(Response response, InetAddressAndPort from) + public void onResponse(AcceptResult acceptResult, InetAddressAndPort from) { + checkArgument(!isForRecovery || acceptResult.rejectedDueToConsensusMigration == false, "Repair should never be rejected due to consensus migration"); if (logger.isTraceEnabled()) - logger.trace("{} for {} from {}", response, proposal, from); + logger.trace("{} for {} from {}", acceptResult, proposal, from); - Ballot supersededBy = response.supersededBy; + Ballot supersededBy = acceptResult.supersededBy; if (supersededBy != null) supersededByUpdater.accumulateAndGet(this, supersededBy, (a, b) -> a == null ? b : b.uuidTimestamp() > a.uuidTimestamp() ? b : a); - long increment = supersededBy == null + long increment = supersededBy == null && !acceptResult.rejectedDueToConsensusMigration ? ACCEPT_INCREMENT : REFUSAL_INCREMENT; + if (acceptResult.rejectedDueToConsensusMigration) + needsConsensusMigration = true; + update(increment); } @@ -375,31 +400,23 @@ public class PaxosPropose> static class Request { final Proposal proposal; - Request(Proposal proposal) + final boolean isForRecovery; + Request(Proposal proposal, boolean isForRecovery) { this.proposal = proposal; + this.isForRecovery = isForRecovery; } + @Override public String toString() { - return proposal.toString("Propose"); + return "Request{" + + "proposal=" + proposal.toString("Propose") + + ", isForRecovery=" + isForRecovery + + '}'; } } - /** - * The response to a proposal, indicating success (if {@code supersededBy == null}, - * or failure, alongside the ballot that beat us - */ - static class Response - { - final Ballot supersededBy; - Response(Ballot supersededBy) - { - this.supersededBy = supersededBy; - } - public String toString() { return supersededBy == null ? "Accept" : "RejectProposal(supersededBy=" + supersededBy + ')'; } - } - /** * The proposal request handler, i.e. receives a proposal from a peer and responds with either acccept/reject */ @@ -408,14 +425,15 @@ public class PaxosPropose> @Override public void doVerb(Message message) { - Response response = execute(message.payload.proposal, message.from()); - if (response == null) + ClusterMetadataService.instance().fetchLogFromCMS(message.epoch()); + AcceptResult acceptResult = execute(message.payload.proposal, message.payload.isForRecovery, message.from()); + if (acceptResult == null) MessagingService.instance().respondWithFailure(UNKNOWN, message); else - MessagingService.instance().respond(response, message); + MessagingService.instance().respond(acceptResult, message); } - public static Response execute(Proposal proposal, InetAddressAndPort from) + public static AcceptResult execute(Proposal proposal, boolean isForRecovery, InetAddressAndPort from) { if (!Paxos.isInRangeAndShouldProcess(from, proposal.update.partitionKey(), proposal.update.metadata(), false)) return null; @@ -423,7 +441,7 @@ public class PaxosPropose> long start = nanoTime(); try (PaxosState state = PaxosState.get(proposal)) { - return new Response(state.acceptIfLatest(proposal)); + return state.acceptIfLatest(proposal, isForRecovery); } finally { @@ -438,42 +456,61 @@ public class PaxosPropose> public void serialize(Request request, DataOutputPlus out, int version) throws IOException { Proposal.serializer.serialize(request.proposal, out, version); + if (version >= MessagingService.VERSION_51) + out.writeBoolean(request.isForRecovery); } @Override public Request deserialize(DataInputPlus in, int version) throws IOException { Proposal propose = Proposal.serializer.deserialize(in, version); - return new Request(propose); + boolean isForRecovery = false; + if (version >= MessagingService.VERSION_51) + isForRecovery = in.readBoolean(); + return new Request(propose, isForRecovery); } @Override public long serializedSize(Request request, int version) { - return Proposal.serializer.serializedSize(request.proposal, version); + long size = Proposal.serializer.serializedSize(request.proposal, version); + if (version >= MessagingService.VERSION_51) + size += TypeSizes.sizeof(request.isForRecovery); + return size; } } - public static class ResponseSerializer implements IVersionedSerializer + public static class AcceptResultSerializer implements IVersionedSerializer { - public void serialize(Response response, DataOutputPlus out, int version) throws IOException + public void serialize(PaxosState.AcceptResult acceptResult, DataOutputPlus out, int version) throws IOException { - out.writeBoolean(response.supersededBy != null); - if (response.supersededBy != null) - response.supersededBy.serialize(out); + out.writeBoolean(acceptResult.supersededBy != null); + if (acceptResult.supersededBy != null) + acceptResult.supersededBy.serialize(out); + if (version >= MessagingService.VERSION_51) + out.writeBoolean(acceptResult.rejectedDueToConsensusMigration); } - public Response deserialize(DataInputPlus in, int version) throws IOException + public AcceptResult deserialize(DataInputPlus in, int version) throws IOException { boolean isSuperseded = in.readBoolean(); - return isSuperseded ? new Response(Ballot.deserialize(in)) : new Response(null); + Ballot supersededBy = null; + if (isSuperseded) + supersededBy = Ballot.deserialize(in); + boolean rejectedDueToConsensusMigration = false; + if (version >= MessagingService.VERSION_51) + rejectedDueToConsensusMigration = in.readBoolean(); + return new AcceptResult(supersededBy, rejectedDueToConsensusMigration); } - public long serializedSize(Response response, int version) + public long serializedSize(AcceptResult acceptResult, int version) { - return response.supersededBy != null + long size = acceptResult.supersededBy != null ? TypeSizes.sizeof(true) + Ballot.sizeInBytes() : TypeSizes.sizeof(false); + if (version >= MessagingService.VERSION_51) + size += TypeSizes.sizeof(acceptResult.rejectedDueToConsensusMigration); + return size; } } } diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java index ed369539ba..8c9f219a3b 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java @@ -61,6 +61,7 @@ import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.PaxosPropose.Superseded; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.utils.CassandraVersion; @@ -68,6 +69,7 @@ import org.apache.cassandra.utils.ExecutorUtils; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MonotonicClock; +import static com.google.common.base.Preconditions.checkState; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPAIR_RETRY_TIMEOUT_IN_MS; @@ -141,10 +143,11 @@ public class PaxosRepair extends AbstractPaxosRepair public static final RequestSerializer requestSerializer = new RequestSerializer(); public static final ResponseSerializer responseSerializer = new ResponseSerializer(); public static final RequestHandler requestHandler = new RequestHandler(); - private static final long RETRY_TIMEOUT_NANOS = getRetryTimeoutNanos(); private static final ScheduledExecutorPlus RETRIES = executorFactory().scheduled("PaxosRepairRetries"); + private static final long RETRY_TIMEOUT_NANOS = getRetryTimeoutNanos(); + private static long getRetryTimeoutNanos() { long retryMillis = PAXOS_REPAIR_RETRY_TIMEOUT_IN_MS.getLong(); @@ -269,6 +272,7 @@ public class PaxosRepair extends AbstractPaxosRepair { if (logger.isTraceEnabled()) logger.trace("PaxosRepair of {} completing {}", partitionKey(), latestAccepted); + // We need to complete this in-progress accepted proposal, which may not have been seen by a majority // However, since we have not sought any promises, we can simply complete the existing proposal // since this is an idempotent operation - both us and the original proposer (and others) can @@ -277,8 +281,7 @@ public class PaxosRepair extends AbstractPaxosRepair // If ballots with same timestamp have been both accepted and rejected by different nodes, // to avoid a livelock we simply try to poison, knowing we will fail but use a new ballot // (note there are alternative approaches but this is conservative) - - return PaxosPropose.propose(latestAccepted, participants, false, + return PaxosPropose.propose(latestAccepted, participants, false, true, new ProposingRepair(latestAccepted)); } else if (isAcceptedButNotCommitted || isPromisedButNotAccepted || latestWitnessed.compareTo(latestPreviouslyWitnessed) < 0) @@ -336,9 +339,10 @@ public class PaxosRepair extends AbstractPaxosRepair // (else an "earlier" operation can sneak in and invalidate us while we're proposing // with a newer ballot) FoundIncompleteAccepted incomplete = input.incompleteAccepted(); + Proposal propose = new Proposal(incomplete.ballot, incomplete.accepted.update); logger.trace("PaxosRepair of {} found incomplete {}", partitionKey(), incomplete.accepted); - return PaxosPropose.propose(propose, participants, false, + return PaxosPropose.propose(propose, participants, false, true, new ProposingRepair(propose)); // we don't know if we're done, so we must restart } @@ -356,7 +360,7 @@ public class PaxosRepair extends AbstractPaxosRepair // propose the empty ballot logger.trace("PaxosRepair of {} submitting empty proposal", partitionKey()); Proposal proposal = Proposal.empty(input.success().ballot, partitionKey(), table); - return PaxosPropose.propose(proposal, participants, false, + return PaxosPropose.propose(proposal, participants, false, true, new ProposingRepair(proposal)); } @@ -383,7 +387,9 @@ public class PaxosRepair extends AbstractPaxosRepair return retry(this); case SUPERSEDED: - if (isAfter(input.superseded().by, prevSupersededBy)) + Superseded superseded = input.superseded(); + checkState(!superseded.needsConsensusMigration, "Repair should not encounter consensus migration rejection"); + if (isAfter(superseded.by, prevSupersededBy)) prevSupersededBy = input.superseded().by; return retry(this); @@ -423,9 +429,9 @@ public class PaxosRepair extends AbstractPaxosRepair } } - private PaxosRepair(DecoratedKey partitionKey, Ballot incompleteBallot, TableMetadata table, ConsistencyLevel paxosConsistency) + private PaxosRepair(DecoratedKey partitionKey, @Nullable Ballot incompleteBallot, TableMetadata table, ConsistencyLevel paxosConsistency, long retryTimeoutNanos) { - super(partitionKey, incompleteBallot); + super(partitionKey, incompleteBallot, retryTimeoutNanos); // TODO: move precondition into super ctor Preconditions.checkArgument(paxosConsistency.isSerialConsistency()); this.table = table; @@ -435,12 +441,17 @@ public class PaxosRepair extends AbstractPaxosRepair public static PaxosRepair create(ConsistencyLevel consistency, DecoratedKey partitionKey, Ballot incompleteBallot, TableMetadata table) { - return new PaxosRepair(partitionKey, incompleteBallot, table, consistency); + return new PaxosRepair(partitionKey, incompleteBallot, table, consistency, RETRY_TIMEOUT_NANOS); + } + + public static PaxosRepair create(ConsistencyLevel consistency, DecoratedKey partitionKey, TableMetadata table, long retryTimeoutNanos) + { + return new PaxosRepair(partitionKey, null, table, consistency, retryTimeoutNanos); } private State retry(State state) { - Preconditions.checkState(isStarted()); + checkState(isStarted()); if (isResult(state)) return state; @@ -455,7 +466,7 @@ public class PaxosRepair extends AbstractPaxosRepair participants = Participants.get(table, partitionKey(), paxosConsistency); - if (waitUntil > Long.MIN_VALUE && waitUntil - startedNanos() > RETRY_TIMEOUT_NANOS) + if (waitUntil > Long.MIN_VALUE && waitUntil - startedNanos() > retryTimeoutNanos) return new Failure(null); try @@ -477,7 +488,7 @@ public class PaxosRepair extends AbstractPaxosRepair private ConsistencyLevel commitConsistency() { - Preconditions.checkState(paxosConsistency.isSerialConsistency()); + checkState(paxosConsistency.isSerialConsistency()); return paxosConsistency.isDatacenterLocal() ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM; } diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java b/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java index ff5f2d4068..fce825fa3b 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java @@ -29,6 +29,7 @@ import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.service.FailureRecordingCallback; +import org.apache.cassandra.tcm.ClusterMetadataService; import static org.apache.cassandra.exceptions.RequestFailure.TIMEOUT; import static org.apache.cassandra.exceptions.RequestFailure.UNKNOWN; @@ -44,6 +45,7 @@ public abstract class PaxosRequestCallback extends FailureRecordingCallback message) { + ClusterMetadataService.instance().fetchLogFromCMS(message.epoch()); onResponse(message.payload, message.from()); } @@ -69,6 +71,32 @@ public abstract class PaxosRequestCallback extends FailureRecordingCallback { + D apply(A var1, B var2, C var3); + } + + protected void executeOnSelf(I parameter1, J parameter2, TriFunction execute) + { + T response; + try + { + response = execute.apply(parameter1, parameter2, getBroadcastAddressAndPort()); + if (response == null) + return; + } + catch (Exception ex) + { + RequestFailure reason = UNKNOWN; + if (ex instanceof WriteTimeoutException) reason = TIMEOUT; + else logger.error("Failed to apply {}, {} locally", parameter1, parameter2, ex); + + onFailure(getBroadcastAddressAndPort(), reason); + return; + } + + onResponse(response, getBroadcastAddressAndPort()); + } + static boolean shouldExecuteOnSelf(InetAddressAndPort replica) { return USE_SELF_EXECUTION && replica.equals(getBroadcastAddressAndPort()); diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosState.java b/src/java/org/apache/cassandra/service/paxos/PaxosState.java index 4fe03d6ff5..44ab0b6e4a 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosState.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosState.java @@ -30,8 +30,9 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import com.google.common.primitives.Ints; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.cassandra.concurrent.ImmediateExecutor; @@ -43,17 +44,21 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.RequestTimeoutException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.metrics.PaxosMetrics; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; import org.apache.cassandra.service.paxos.uncommitted.PaxosBallotTracker; import org.apache.cassandra.service.paxos.uncommitted.PaxosStateTracker; import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTracker; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.Nemesis; +import static com.google.common.base.Preconditions.checkState; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_DISABLE_COORDINATOR_LOCKING; import static org.apache.cassandra.config.Config.PaxosStatePurging.gc_grace; @@ -69,6 +74,8 @@ import static org.apache.cassandra.service.paxos.Commit.CommittedWithTTL; import static org.apache.cassandra.service.paxos.Commit.Proposal; import static org.apache.cassandra.service.paxos.Commit.isAfter; import static org.apache.cassandra.service.paxos.Commit.latest; +import static org.apache.cassandra.service.paxos.PaxosState.AcceptResult.RETRY_NEW_PROTOCOL; +import static org.apache.cassandra.service.paxos.PaxosState.AcceptResult.SUCCESS; import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.PERMIT_READ; import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.PROMISE; import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.REJECT; @@ -80,6 +87,9 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime; */ public class PaxosState implements PaxosOperationLock { + @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(PaxosState.class.getName()); + private static volatile boolean DISABLE_COORDINATOR_LOCKING = PAXOS_DISABLE_COORDINATOR_LOCKING.getBoolean(); public static final ConcurrentHashMap ACTIVE = new ConcurrentHashMap<>(); public static final Map RECENT = Caffeine.newBuilder() @@ -127,7 +137,7 @@ public class PaxosState implements PaxosOperationLock public static void initializeTrackers() { - Preconditions.checkState(TrackerHandle.tracker != null); + checkState(TrackerHandle.tracker != null); PaxosMetrics.initialize(); } @@ -634,7 +644,7 @@ public class PaxosState implements PaxosOperationLock /** * Record an acceptance of the proposal if there is no newer promise; otherwise inform the caller of the newer ballot */ - public Ballot acceptIfLatest(Proposal proposal) + public AcceptResult acceptIfLatest(Proposal proposal, boolean isForRecovery) { if (paxosStatePurging() == legacy && !(proposal instanceof AcceptedWithTTL)) proposal = AcceptedWithTTL.withDefaultTTL(proposal); @@ -642,20 +652,31 @@ public class PaxosState implements PaxosOperationLock // state.promised can be null, because it is invalidated by committed; // we may also have accepted a newer proposal than we promised, so we confirm that we are the absolute newest // (or that we have the exact same ballot as our promise, which is the typical case) + boolean shouldRejectDueToConsensusMigration; Snapshot before, after; while (true) { Snapshot realBefore = current; before = realBefore.removeExpired((int)proposal.ballot.unix(SECONDS)); Ballot latest = before.latestWitnessedOrLowBound(); + if (isForRecovery) + shouldRejectDueToConsensusMigration = false; + else + shouldRejectDueToConsensusMigration = ConsensusRequestRouter.instance + .isKeyInMigratingOrMigratedRangeDuringPaxosAccept(proposal.update.metadata().id, + proposal.update.partitionKey()); if (!proposal.isSameOrAfter(latest)) { Tracing.trace("Rejecting proposal {}; latest is now {}", proposal.ballot, latest); - return latest; + return new AcceptResult(latest, shouldRejectDueToConsensusMigration); } - if (proposal.hasSameBallot(before.committed)) // TODO: consider not answering - return null; // no need to save anything, or indeed answer at all + if (shouldRejectDueToConsensusMigration) + return RETRY_NEW_PROTOCOL; + + // TODO: Consider not answering in the committed ballot case where there is no need to save anything or answer at all + if (proposal.hasSameBallot(before.committed)) + return null; after = new Snapshot(realBefore.promised, realBefore.promisedWrite, proposal.accepted(), realBefore.committed); if (currentUpdater.compareAndSet(this, realBefore, after)) @@ -672,7 +693,8 @@ public class PaxosState implements PaxosOperationLock // though this Tracing.trace("Accepting proposal {}", proposal); SystemKeyspace.savePaxosProposal(proposal); - return null; + checkState(!shouldRejectDueToConsensusMigration); + return SUCCESS; } public void commit(Agreed commit) @@ -794,6 +816,8 @@ public class PaxosState implements PaxosOperationLock boolean accept = proposal.isSameOrAfter(before.latestWitnessedOrLowBound()); if (accept) { + PartitionUpdate partitionUpdate = proposal.update; + checkState(ConsensusKeyMigrationState.getKeyMigrationState(partitionUpdate.metadata().id, partitionUpdate.partitionKey()).tableMigrationState == null, "Using PaxosV1 while consensus migration is in progress is not supported"); if (proposal.hasSameBallot(before.committed) || currentUpdater.compareAndSet(unsafeState, realBefore, new Snapshot(realBefore.promised, realBefore.promisedWrite, @@ -832,4 +856,35 @@ public class PaxosState implements PaxosOperationLock if (cur != null) return cur.current; return RECENT.get(key); } + + /** + * The response to a proposal, indicating success (if {@code supersededBy == null}, + * or failure, alongside the ballot that beat us + */ + public static class AcceptResult + { + static final AcceptResult SUCCESS = new AcceptResult(false); + + static final AcceptResult RETRY_NEW_PROTOCOL = new AcceptResult(true); + + @Nullable + public final Ballot supersededBy; + + public final boolean rejectedDueToConsensusMigration; + + public AcceptResult(@Nullable Ballot supersededBy, boolean rejectedDueToConsensusMigration) + { + this.supersededBy = supersededBy; + this.rejectedDueToConsensusMigration = rejectedDueToConsensusMigration; + } + + // Success result + private AcceptResult(boolean rejectedDueToConsensusMigration) + { + supersededBy = null; + this.rejectedDueToConsensusMigration = rejectedDueToConsensusMigration; + } + + public String toString() { return supersededBy == null && !rejectedDueToConsensusMigration ? "Accept" : "RejectProposal(supersededBy=" + supersededBy + ", rejectedDueToConsensusMigration=" + rejectedDueToConsensusMigration + ')'; } + } } diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosStartPrepareCleanup.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosStartPrepareCleanup.java index 08cbd9aa09..21fcb90da4 100644 --- a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosStartPrepareCleanup.java +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosStartPrepareCleanup.java @@ -53,6 +53,7 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.service.paxos.PaxosRepairHistory; +import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.utils.concurrent.AsyncFuture; import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_START_PREPARE_REQ; @@ -184,6 +185,7 @@ public class PaxosStartPrepareCleanup extends AsyncFuture i public static IVerbHandler createVerbHandler(SharedContext ctx) { return in -> { + ClusterMetadataService.instance().fetchLogFromCMS(in.epoch()); ColumnFamilyStore table = Schema.instance.getColumnFamilyStoreInstance(in.payload.tableId); // Note: pre-5.1 we would use gossip state included in the request payload to update topology // prior to cleanup. Topology is no longer derived from gossip state, so this has been removed. diff --git a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java index 88b3ba49fa..286a5b85d7 100644 --- a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java +++ b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java @@ -66,6 +66,7 @@ public abstract class AbstractReadExecutor { private static final Logger logger = LoggerFactory.getLogger(AbstractReadExecutor.class); + protected final ReadCoordinator coordinator; protected final ReadCommand command; private final ReplicaPlan.SharedForTokenRead replicaPlan; protected final ReadRepair readRepair; @@ -78,14 +79,15 @@ public abstract class AbstractReadExecutor private final int initialDataRequestCount; protected volatile PartitionIterator result = null; - AbstractReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, int initialDataRequestCount, Dispatcher.RequestTime requestTime) + AbstractReadExecutor(ReadCoordinator coordinator, ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, int initialDataRequestCount, Dispatcher.RequestTime requestTime) { + this.coordinator = coordinator; this.command = command; this.replicaPlan = ReplicaPlan.shared(replicaPlan); this.initialDataRequestCount = initialDataRequestCount; // the ReadRepair and DigestResolver both need to see our updated - this.readRepair = ReadRepair.create(command, this.replicaPlan, requestTime); - this.digestResolver = new DigestResolver<>(command, this.replicaPlan, requestTime); + this.readRepair = ReadRepair.create(coordinator, command, this.replicaPlan, requestTime); + this.digestResolver = new DigestResolver<>(coordinator, command, this.replicaPlan, requestTime); this.handler = new ReadCallback<>(digestResolver, command, this.replicaPlan, requestTime); this.cfs = cfs; this.traceState = Tracing.instance.get(); @@ -136,13 +138,14 @@ public abstract class AbstractReadExecutor { boolean hasLocalEndpoint = false; Message message = null; + readCommand = coordinator.maybeAllowOutOfRangeReads(readCommand); for (Replica replica: replicas) { assert replica.isFull() || readCommand.acceptsTransient(); InetAddressAndPort endpoint = replica.endpoint(); - if (replica.isSelf()) + if (replica.isSelf() && coordinator.localReadSupported()) { hasLocalEndpoint = true; continue; @@ -154,7 +157,7 @@ public abstract class AbstractReadExecutor if (null == message) message = readCommand.createMessage(false, requestTime).withEpoch(ClusterMetadata.current().epoch); - MessagingService.instance().sendWithCallback(message, endpoint, handler); + coordinator.sendReadCommand(message, endpoint, handler); } // We delay the local (potentially blocking) read till the end to avoid stalling remote requests. @@ -179,8 +182,11 @@ public abstract class AbstractReadExecutor EndpointsForToken selected = replicaPlan().contacts(); EndpointsForToken fullDataRequests = selected.filter(Replica::isFull, initialDataRequestCount); makeFullDataRequests(fullDataRequests); - makeTransientDataRequests(selected.filterLazily(Replica::isTransient)); - makeDigestRequests(selected.filterLazily(r -> r.isFull() && !fullDataRequests.contains(r))); + EndpointsForToken transientRequests = selected.filter(Replica::isTransient); + makeTransientDataRequests(transientRequests); + EndpointsForToken digestRequests = selected.filter(r -> r.isFull() && !fullDataRequests.contains(r)); + makeDigestRequests(digestRequests); + coordinator.notifyOfInitialContacts(fullDataRequests, transientRequests, digestRequests); } /** @@ -189,6 +195,7 @@ public abstract class AbstractReadExecutor public static AbstractReadExecutor getReadExecutor(ClusterMetadata metadata, SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, + ReadCoordinator coordinator, Dispatcher.RequestTime requestTime) throws UnavailableException { Keyspace keyspace = Keyspace.open(command.metadata().keyspace); @@ -200,25 +207,26 @@ public abstract class AbstractReadExecutor command.partitionKey().getToken(), command.indexQueryPlan(), consistencyLevel, - retry); + retry, + coordinator); // Speculative retry is disabled *OR* // 11980: Disable speculative retry if using EACH_QUORUM in order to prevent miscounting DC responses if (retry.equals(NeverSpeculativeRetryPolicy.INSTANCE) || consistencyLevel == ConsistencyLevel.EACH_QUORUM) - return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, false); + return new NeverSpeculatingReadExecutor(coordinator, cfs, command, replicaPlan, requestTime, false); // There are simply no extra replicas to speculate. // Handle this separately so it can record failed attempts to speculate due to lack of replicas if (replicaPlan.contacts().size() == replicaPlan.readCandidates().size()) { boolean recordFailedSpeculation = consistencyLevel != ConsistencyLevel.ALL; - return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, recordFailedSpeculation); + return new NeverSpeculatingReadExecutor(coordinator, cfs, command, replicaPlan, requestTime, recordFailedSpeculation); } if (retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE)) - return new AlwaysSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime); + return new AlwaysSpeculatingReadExecutor(coordinator, cfs, command, replicaPlan, requestTime); else // PERCENTILE or CUSTOM. - return new SpeculatingReadExecutor(cfs, command, replicaPlan, requestTime); + return new SpeculatingReadExecutor(coordinator, cfs, command, replicaPlan, requestTime); } public boolean hasLocalRead() @@ -272,13 +280,14 @@ public abstract class AbstractReadExecutor */ private final boolean logFailedSpeculation; - public NeverSpeculatingReadExecutor(ColumnFamilyStore cfs, + public NeverSpeculatingReadExecutor(ReadCoordinator coordinator, + ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, Dispatcher.RequestTime requestTime, boolean logFailedSpeculation) { - super(cfs, command, replicaPlan, 1, requestTime); + super(coordinator, cfs, command, replicaPlan, 1, requestTime); this.logFailedSpeculation = logFailedSpeculation; } @@ -295,7 +304,8 @@ public abstract class AbstractReadExecutor { private volatile boolean speculated = false; - public SpeculatingReadExecutor(ColumnFamilyStore cfs, + public SpeculatingReadExecutor(ReadCoordinator coordinator, + ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, Dispatcher.RequestTime requestTime) @@ -303,7 +313,7 @@ public abstract class AbstractReadExecutor // We're hitting additional targets for read repair (??). Since our "extra" replica is the least- // preferred by the snitch, we do an extra data read to start with against a replica more // likely to respond; better to let RR fail than the entire query. - super(cfs, command, replicaPlan, replicaPlan.readQuorum() < replicaPlan.contacts().size() ? 2 : 1, requestTime); + super(coordinator, cfs, command, replicaPlan, replicaPlan.readQuorum() < replicaPlan.contacts().size() ? 2 : 1, requestTime); } public void maybeTryAdditionalReplicas() @@ -366,14 +376,15 @@ public abstract class AbstractReadExecutor private static class AlwaysSpeculatingReadExecutor extends AbstractReadExecutor { - public AlwaysSpeculatingReadExecutor(ColumnFamilyStore cfs, + public AlwaysSpeculatingReadExecutor(ReadCoordinator coordinator, + ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, Dispatcher.RequestTime requestTime) { // presumably, we speculate an extra data request here in case it is our data request that fails to respond, // and there are no more nodes to consult - super(cfs, command, replicaPlan, replicaPlan.contacts().size() > 1 ? 2 : 1, requestTime); + super(coordinator, cfs, command, replicaPlan, replicaPlan.contacts().size() > 1 ? 2 : 1, requestTime); } public void maybeTryAdditionalReplicas() diff --git a/src/java/org/apache/cassandra/service/reads/DataResolver.java b/src/java/org/apache/cassandra/service/reads/DataResolver.java index 332a785708..03eee1c2c5 100644 --- a/src/java/org/apache/cassandra/service/reads/DataResolver.java +++ b/src/java/org/apache/cassandra/service/reads/DataResolver.java @@ -23,7 +23,6 @@ import java.util.Collection; import java.util.List; import java.util.function.Supplier; import java.util.function.UnaryOperator; - import javax.annotation.Nullable; import com.google.common.base.Joiner; @@ -57,22 +56,23 @@ import org.apache.cassandra.service.reads.repair.RepairedDataTracker; import org.apache.cassandra.service.reads.repair.RepairedDataVerifier; import org.apache.cassandra.transport.Dispatcher; -import static com.google.common.collect.Iterables.*; +import static com.google.common.collect.Iterables.any; +import static com.google.common.collect.Iterables.transform; public class DataResolver, P extends ReplicaPlan.ForRead> extends ResponseResolver { private final boolean enforceStrictLiveness; - private final ReadRepair readRepair; + public final ReadRepair readRepair; private final boolean trackRepairedStatus; - public DataResolver(ReadCommand command, Supplier replicaPlan, ReadRepair readRepair, Dispatcher.RequestTime requestTime) + public DataResolver(ReadCoordinator coordinator, ReadCommand command, Supplier replicaPlan, ReadRepair readRepair, Dispatcher.RequestTime requestTime) { - this(command, replicaPlan, readRepair, requestTime, false); + this(coordinator, command, replicaPlan, readRepair, requestTime, false); } - public DataResolver(ReadCommand command, Supplier replicaPlan, ReadRepair readRepair, Dispatcher.RequestTime requestTime, boolean trackRepairedStatus) + public DataResolver(ReadCoordinator coordinator, ReadCommand command, Supplier replicaPlan, ReadRepair readRepair, Dispatcher.RequestTime requestTime, boolean trackRepairedStatus) { - super(command, replicaPlan, requestTime); + super(coordinator, command, replicaPlan, requestTime); this.enforceStrictLiveness = command.metadata().enforceStrictLiveness(); this.readRepair = readRepair; this.trackRepairedStatus = trackRepairedStatus; @@ -209,6 +209,7 @@ public class DataResolver, P extends ReplicaPlan.ForRead< originalResponse, command, context.mergedResultCounter, + coordinator, requestTime, enforceStrictLiveness) : originalResponse; @@ -249,7 +250,7 @@ public class DataResolver, P extends ReplicaPlan.ForRead< // before it counts against the limit. If this "pre-count" filter causes a short read, additional rows // will be fetched from the first-phase iterator. - ReplicaFilteringProtection rfp = new ReplicaFilteringProtection<>(replicaPlan().keyspace(), + ReplicaFilteringProtection rfp = new ReplicaFilteringProtection<>(coordinator, replicaPlan().keyspace(), command, replicaPlan().consistencyLevel(), requestTime, @@ -257,6 +258,7 @@ public class DataResolver, P extends ReplicaPlan.ForRead< DatabaseDescriptor.getCachedReplicaRowsWarnThreshold(), DatabaseDescriptor.getCachedReplicaRowsFailThreshold()); + // We need separate contexts, as each context has his own counter ResolveContext firstPhaseContext = new ResolveContext(replicas, false); PartitionIterator firstPhasePartitions = resolveInternal(firstPhaseContext, rfp.mergeController(), diff --git a/src/java/org/apache/cassandra/service/reads/DigestResolver.java b/src/java/org/apache/cassandra/service/reads/DigestResolver.java index cc248422c0..59c3df383d 100644 --- a/src/java/org/apache/cassandra/service/reads/DigestResolver.java +++ b/src/java/org/apache/cassandra/service/reads/DigestResolver.java @@ -30,8 +30,8 @@ import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.locator.Endpoints; -import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.net.Message; import org.apache.cassandra.service.reads.repair.NoopReadRepair; @@ -45,9 +45,9 @@ public class DigestResolver, P extends ReplicaPlan.ForRea { private volatile Message dataResponse; - public DigestResolver(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + public DigestResolver(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) { - super(command, replicaPlan, requestTime); + super(coordinator, command, replicaPlan, requestTime); Preconditions.checkArgument(command instanceof SinglePartitionReadCommand, "DigestResolver can only be used with SinglePartitionReadCommand commands"); } @@ -87,7 +87,7 @@ public class DigestResolver, P extends ReplicaPlan.ForRea // This path can be triggered only if we've got responses from full replicas and they match, but // transient replica response still contains data, which needs to be reconciled. DataResolver dataResolver - = new DataResolver<>(command, replicaPlan, NoopReadRepair.instance, requestTime); + = new DataResolver<>(coordinator, command, replicaPlan, NoopReadRepair.instance, requestTime); dataResolver.preprocess(dataResponse); // Reconcile with transient replicas diff --git a/src/java/org/apache/cassandra/service/reads/ReadCoordinator.java b/src/java/org/apache/cassandra/service/reads/ReadCoordinator.java new file mode 100644 index 0000000000..8f41464779 --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/ReadCoordinator.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads; + +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadResponse; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.RequestCallback; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.tcm.ClusterMetadata; + +public interface ReadCoordinator +{ + ReadCoordinator DEFAULT = new ReadCoordinator() + { + public boolean localReadSupported() + { + return true; + } + + public EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token) + { + return ReplicaLayout.forNonLocalStrategyTokenRead(metadata, keyspace, token); + } + + public void sendReadCommand(Message message, InetAddressAndPort to, RequestCallback callback) + { + MessagingService.instance().sendWithCallback(message, to, callback); + } + + public void sendReadRepairMutation(Message message, InetAddressAndPort to, RequestCallback callback) + { + MessagingService.instance().sendWithCallback(message, to, callback); + } + + public boolean isEventuallyConsistent() + { + return true; + } + }; + + boolean localReadSupported(); + EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token); + default ReadCommand maybeAllowOutOfRangeReads(ReadCommand command) + { + return command; + } + void sendReadCommand(Message message, InetAddressAndPort to, RequestCallback callback); + default void notifyOfInitialContacts(EndpointsForToken fullDataRequests, EndpointsForToken transientRequests, EndpointsForToken digestRequests) {} + void sendReadRepairMutation(Message message, InetAddressAndPort to, RequestCallback callback); + default Mutation maybeAllowOutOfRangeMutations(Mutation m) + { + return m; + } + boolean isEventuallyConsistent(); +} diff --git a/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java index 056f3b55df..2744e8d4be 100644 --- a/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java +++ b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java @@ -23,8 +23,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.NavigableSet; -import java.util.concurrent.TimeUnit; import java.util.Queue; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import org.slf4j.Logger; @@ -98,6 +98,7 @@ public class ReplicaFilteringProtection> private static final Function NULL_TO_NO_STATS = rowIterator -> rowIterator == null ? EncodingStats.NO_STATS : rowIterator.stats(); + private final ReadCoordinator coordinator; private final Keyspace keyspace; private final ReadCommand command; private final ConsistencyLevel consistency; @@ -119,7 +120,8 @@ public class ReplicaFilteringProtection> */ private final List> originalPartitions; - ReplicaFilteringProtection(Keyspace keyspace, + ReplicaFilteringProtection(ReadCoordinator coordinator, + Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, Dispatcher.RequestTime requestTime, @@ -127,6 +129,7 @@ public class ReplicaFilteringProtection> int cachedRowsWarnThreshold, int cachedRowsFailThreshold) { + this.coordinator = coordinator; this.keyspace = keyspace; this.command = command; this.consistency = consistency; @@ -149,7 +152,7 @@ public class ReplicaFilteringProtection> { @SuppressWarnings("unchecked") DataResolver resolver = - new DataResolver<>(cmd, replicaPlan, (NoopReadRepair) NoopReadRepair.instance, requestTime); + new DataResolver<>(coordinator, cmd, replicaPlan, (NoopReadRepair) NoopReadRepair.instance, requestTime); ReadCallback handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime); diff --git a/src/java/org/apache/cassandra/service/reads/ResponseResolver.java b/src/java/org/apache/cassandra/service/reads/ResponseResolver.java index 5dd81eb7bc..61956322d8 100644 --- a/src/java/org/apache/cassandra/service/reads/ResponseResolver.java +++ b/src/java/org/apache/cassandra/service/reads/ResponseResolver.java @@ -34,6 +34,7 @@ public abstract class ResponseResolver, P extends Replica { protected static final Logger logger = LoggerFactory.getLogger(ResponseResolver.class); + protected final ReadCoordinator coordinator; protected final ReadCommand command; protected final Supplier replicaPlan; @@ -41,8 +42,9 @@ public abstract class ResponseResolver, P extends Replica protected final Accumulator> responses; protected final Dispatcher.RequestTime requestTime; - public ResponseResolver(ReadCommand command, Supplier replicaPlan, Dispatcher.RequestTime requestTime) + public ResponseResolver(ReadCoordinator coordinator, ReadCommand command, Supplier replicaPlan, Dispatcher.RequestTime requestTime) { + this.coordinator = coordinator; this.command = command; this.replicaPlan = replicaPlan; this.responses = new Accumulator<>(replicaPlan.get().readCandidates().size()); diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java index e9870f1f1d..102042758e 100644 --- a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java +++ b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java @@ -18,9 +18,6 @@ package org.apache.cassandra.service.reads; -import org.apache.cassandra.locator.Endpoints; -import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.ReplicaPlans; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,16 +38,20 @@ import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.ExcludingBounds; import org.apache.cassandra.dht.Range; +import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.service.reads.repair.NoopReadRepair; +import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.service.StorageProxy; +import org.apache.cassandra.service.reads.repair.NoopReadRepair; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; public class ShortReadPartitionsProtection extends Transformation implements MorePartitions { private static final Logger logger = LoggerFactory.getLogger(ShortReadPartitionsProtection.class); + + private final ReadCoordinator coordinator; private final ReadCommand command; private final Replica source; @@ -65,13 +66,16 @@ public class ShortReadPartitionsProtection extends Transformation, P extends ReplicaPlan.ForRead> UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, ReplicaPlan.Shared replicaPlan) { - DataResolver resolver = new DataResolver<>(cmd, replicaPlan, (NoopReadRepair)NoopReadRepair.instance, requestTime); + cmd = coordinator.maybeAllowOutOfRangeReads(cmd); + DataResolver resolver = new DataResolver<>(coordinator, cmd, replicaPlan, (NoopReadRepair)NoopReadRepair.instance, requestTime); ReadCallback handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime); - if (source.isSelf()) + if (source.isSelf() && coordinator.localReadSupported()) { Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler, requestTime)); } @@ -189,7 +194,7 @@ public class ShortReadPartitionsProtection extends Transformation implemen ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan); ReadRepair readRepair = - ReadRepair.create(command, sharedReplicaPlan, requestTime); + ReadRepair.create(ReadCoordinator.DEFAULT, command, sharedReplicaPlan, requestTime); DataResolver resolver = - new DataResolver<>(rangeCommand, sharedReplicaPlan, readRepair, requestTime, trackRepairedStatus); + new DataResolver<>(ReadCoordinator.DEFAULT, rangeCommand, sharedReplicaPlan, readRepair, requestTime, trackRepairedStatus); ReadCallback handler = new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, requestTime); diff --git a/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java index 53f55f8938..c866911e1a 100644 --- a/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java +++ b/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java @@ -38,6 +38,7 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.reads.DataResolver; import org.apache.cassandra.service.reads.ReadCallback; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.service.reads.repair.NoopReadRepair; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; @@ -92,7 +93,7 @@ public class ScanAllRangesCommandIterator extends RangeCommandIterator ReplicaPlan.ForRangeRead plan = ReplicaPlans.forFullRangeRead(keyspace, consistencyLevel, command.dataRange().keyRange(), replicasToQuery, totalRangeCount); ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(plan); - DataResolver resolver = new DataResolver<>(command, sharedReplicaPlan, NoopReadRepair.instance, requestTime, false); + DataResolver resolver = new DataResolver<>(ReadCoordinator.DEFAULT, command, sharedReplicaPlan, NoopReadRepair.instance, requestTime, false); ReadCallback handler = new ReadCallback<>(resolver, command, sharedReplicaPlan, requestTime); int nodes = 0; diff --git a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java index 8343b83b07..7ef6d2fe04 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java @@ -39,11 +39,11 @@ import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.metrics.ReadRepairMetrics; import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.reads.DataResolver; import org.apache.cassandra.service.reads.DigestResolver; import org.apache.cassandra.service.reads.ReadCallback; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; @@ -54,6 +54,7 @@ public abstract class AbstractReadRepair, P extends Repli { protected static final Logger logger = LoggerFactory.getLogger(AbstractReadRepair.class); + protected final ReadCoordinator coordinator; protected final ReadCommand command; protected final Dispatcher.RequestTime requestTime; protected final ReplicaPlan.Shared replicaPlan; @@ -75,10 +76,11 @@ public abstract class AbstractReadRepair, P extends Repli } } - public AbstractReadRepair(ReadCommand command, + public AbstractReadRepair(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) { + this.coordinator = coordinator; this.command = command; this.requestTime = requestTime; this.replicaPlan = replicaPlan; @@ -92,9 +94,9 @@ public abstract class AbstractReadRepair, P extends Repli void sendReadCommand(Replica to, ReadCallback readCallback, boolean speculative, boolean trackRepairedStatus) { - ReadCommand command = this.command; - - if (to.isSelf()) + ReadCommand command = coordinator.maybeAllowOutOfRangeReads(this.command); + + if (to.isSelf() && coordinator.localReadSupported()) { Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(command, readCallback, requestTime, trackRepairedStatus)); return; @@ -117,7 +119,7 @@ public abstract class AbstractReadRepair, P extends Repli } Message message = command.createMessage(trackRepairedStatus && to.isFull(), requestTime); - MessagingService.instance().sendWithCallback(message, to.endpoint(), readCallback); + coordinator.sendReadCommand(message, to.endpoint(), readCallback); } abstract Meter getRepairMeter(); @@ -139,7 +141,7 @@ public abstract class AbstractReadRepair, P extends Repli boolean trackRepairedStatus = DatabaseDescriptor.getRepairedDataTrackingForPartitionReadsEnabled(); // Do a full data read to resolve the correct response (and repair node that need be) - DataResolver resolver = new DataResolver<>(command, replicaPlan, this, requestTime, trackRepairedStatus); + DataResolver resolver = new DataResolver<>(coordinator, command, replicaPlan, this, requestTime, trackRepairedStatus); ReadCallback readCallback = new ReadCallback<>(resolver, command, replicaPlan, requestTime); digestRepair = new DigestRepair<>(resolver, readCallback, resultConsumer); diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java index 61b529ca00..973fbcb45d 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java @@ -21,9 +21,6 @@ package org.apache.cassandra.service.reads.repair; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; - -import org.apache.cassandra.utils.concurrent.AsyncFuture; -import org.apache.cassandra.utils.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; @@ -38,27 +35,32 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.InOurDc; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.locator.Replicas; -import org.apache.cassandra.locator.InOurDc; import org.apache.cassandra.metrics.ReadRepairMetrics; -import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.reads.ReadCoordinator; +import org.apache.cassandra.service.reads.repair.BlockingReadRepair.PendingPartitionRepair; import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.concurrent.AsyncFuture; +import org.apache.cassandra.utils.concurrent.CountDownLatch; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; -import static org.apache.cassandra.net.Verb.*; +import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch; import static com.google.common.collect.Iterables.all; public class BlockingPartitionRepair - extends AsyncFuture implements RequestCallback + extends AsyncFuture implements RequestCallback, PendingPartitionRepair { + private final ReadCoordinator coordinator; private final DecoratedKey key; private final ReplicaPlan.ForWrite repairPlan; private final Map pendingRepairs; @@ -66,8 +68,10 @@ public class BlockingPartitionRepair private final int blockFor; private volatile long mutationsSentTime; - public BlockingPartitionRepair(DecoratedKey key, Map repairs, ReplicaPlan.ForWrite repairPlan) + @VisibleForTesting + public BlockingPartitionRepair(ReadCoordinator coordinator, DecoratedKey key, Map repairs, ReplicaPlan.ForWrite repairPlan) { + this.coordinator = coordinator; this.key = key; this.pendingRepairs = new ConcurrentHashMap<>(repairs); this.repairPlan = repairPlan; @@ -99,18 +103,21 @@ public class BlockingPartitionRepair latch = newCountDownLatch(Math.max(blockFor, 0)); } + @Override public ReplicaPlan.ForWrite repairPlan() { return repairPlan; } - int blockFor() + @Override + public int blockFor() { return blockFor; } @VisibleForTesting - int waitingOn() + @Override + public int waitingOn() { return latch.count(); } @@ -147,7 +154,7 @@ public class BlockingPartitionRepair @VisibleForTesting protected void sendRR(Message message, InetAddressAndPort endpoint) { - MessagingService.instance().sendWithCallback(message, endpoint, this); + coordinator.sendReadRepairMutation(message, endpoint, this); } public void sendInitialRepairs() @@ -159,7 +166,7 @@ public class BlockingPartitionRepair { Replica destination = entry.getKey(); Preconditions.checkArgument(destination.isFull(), "Can't send repairs to transient replicas: %s", destination); - Mutation mutation = entry.getValue(); + Mutation mutation = coordinator.maybeAllowOutOfRangeMutations(entry.getValue()); TableId tableId = extractUpdate(mutation).metadata().id; Tracing.trace("Sending read-repair-mutation to {}", destination); @@ -177,6 +184,7 @@ public class BlockingPartitionRepair * @param timeUnit the time unit of the future time * @return true if repair is done; otherwise, false. */ + @Override public boolean awaitRepairsUntil(long timeoutAt, TimeUnit timeUnit) { long timeoutAtNanos = timeUnit.toNanos(timeoutAt); @@ -191,18 +199,18 @@ public class BlockingPartitionRepair } } + @Override + public boolean awaitRepairs(long remaining, TimeUnit timeUnit) throws InterruptedException + { + return latch.await(remaining, timeUnit); + } + private static int msgVersionIdx(int version) { return version - MessagingService.minimum_version; } - /** - * If it looks like we might not receive acks for all the repair mutations we sent out, combine all - * the unacked mutations and send them to the minority of nodes not involved in the read repair data - * read / write cycle. We will accept acks from them in lieu of acks from the initial mutations sent - * out, so long as we receive the same number of acks as repair mutations transmitted. This prevents - * misbehaving nodes from killing a quorum read, while continuing to guarantee monotonic quorum reads - */ + @Override public void maybeSendAdditionalWrites(long timeout, TimeUnit timeoutUnit) { if (awaitRepairsUntil(timeout + timeoutUnit.convert(mutationsSentTime, TimeUnit.NANOSECONDS), timeoutUnit)) @@ -242,6 +250,7 @@ public class BlockingPartitionRepair continue; } + mutation = coordinator.maybeAllowOutOfRangeMutations(mutation); Tracing.trace("Sending speculative read-repair-mutation to {}", replica); sendRR(Message.out(READ_REPAIR_REQ, mutation), replica.endpoint()); ReadRepairDiagnostics.speculatedWrite(this, replica.endpoint(), mutation); diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java index 4a56e6fe18..a1e90fd965 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java @@ -18,30 +18,55 @@ package org.apache.cassandra.service.reads.repair; +import java.util.Collection; import java.util.Map; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; -import org.apache.cassandra.db.DecoratedKey; +import com.google.common.util.concurrent.UncheckedExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import accord.primitives.Keys; +import accord.primitives.Txn; import com.codahale.metrics.Meter; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.config.Config.LWTStrategy; +import org.apache.cassandra.config.Config.NonSerialWriteStrategy; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.ReplicaPlan.ForWrite; import org.apache.cassandra.metrics.ReadRepairMetrics; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.txn.TxnQuery; +import org.apache.cassandra.service.accord.txn.TxnRead; +import org.apache.cassandra.service.accord.txn.TxnResult; +import org.apache.cassandra.service.accord.txn.UnrecoverableRepairUpdate; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; /** * 'Classic' read repair. Doesn't allow the client read to return until @@ -53,13 +78,66 @@ public class BlockingReadRepair, P extends ReplicaPlan.Fo { private static final Logger logger = LoggerFactory.getLogger(BlockingReadRepair.class); - protected final Queue repairs = new ConcurrentLinkedQueue<>(); + protected final Queue repairs = new ConcurrentLinkedQueue<>(); - BlockingReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + interface PendingPartitionRepair { - super(command, replicaPlan, requestTime); + + /** + * Wait for the repair to complete util a future time + * If the {@param timeoutAt} is a past time, the method returns immediately with the repair result. + * @param timeoutAt future time + * @param timeUnit the time unit of the future time + * @return true if repair is done; otherwise, false. + */ + default boolean awaitRepairsUntil(long timeoutAt, TimeUnit timeUnit) + { + long timeoutAtNanos = timeUnit.toNanos(timeoutAt); + long remaining = timeoutAtNanos - nanoTime(); + try + { + return awaitRepairs(remaining, timeUnit); + } + catch (InterruptedException e) + { + throw new UncheckedInterruptedException(e); + } + catch (ExecutionException e) + { + throw new UncheckedExecutionException(e); + } + } + + boolean awaitRepairs(long remaining, TimeUnit timeUnit) throws InterruptedException, ExecutionException; + + /** + * If it looks like we might not receive acks for all the repair mutations we sent out, combine all + * the unacked mutations and send them to the minority of nodes not involved in the read repair data + * read / write cycle. We will accept acks from them in lieu of acks from the initial mutations sent + * out, so long as we receive the same number of acks as repair mutations transmitted. This prevents + * misbehaving nodes from killing a quorum read, while continuing to guarantee monotonic quorum reads + */ + default void maybeSendAdditionalWrites(long timeout, TimeUnit timeoutUnit) {} + + default int blockFor() + { + return -1; + } + + default int waitingOn() + { + return -1; + } + + ForWrite repairPlan(); } + BlockingReadRepair(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + { + super(coordinator, command, replicaPlan, requestTime); + } + + @Override public UnfilteredPartitionIterators.MergeListener getMergeListener(P replicaPlan) { return new PartitionIteratorMergeListener<>(replicaPlan, command, this); @@ -74,7 +152,7 @@ public class BlockingReadRepair, P extends ReplicaPlan.Fo @Override public void maybeSendAdditionalWrites() { - for (BlockingPartitionRepair repair: repairs) + for (PendingPartitionRepair repair: repairs) { repair.maybeSendAdditionalWrites(cfs.additionalWriteLatencyMicros, MICROSECONDS); } @@ -83,10 +161,10 @@ public class BlockingReadRepair, P extends ReplicaPlan.Fo @Override public void awaitWrites() { - BlockingPartitionRepair timedOut = null; + PendingPartitionRepair timedOut = null; ReplicaPlan.ForWrite repairPlan = null; - for (BlockingPartitionRepair repair : repairs) + for (PendingPartitionRepair repair : repairs) { long deadline = requestTime.computeDeadline(DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS)); @@ -116,10 +194,80 @@ public class BlockingReadRepair, P extends ReplicaPlan.Fo } @Override - public void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan) + public void repairPartition(DecoratedKey dk, Map mutations, ReplicaPlan.ForWrite writePlan) { - BlockingPartitionRepair blockingRepair = new BlockingPartitionRepair(partitionKey, mutations, writePlan); - blockingRepair.sendInitialRepairs(); - repairs.add(blockingRepair); + NonSerialWriteStrategy nonSerialWriteStrategy = DatabaseDescriptor.getNonSerialWriteStrategy(); + if (coordinator.isEventuallyConsistent() && (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord + || nonSerialWriteStrategy.blockingReadRepairThroughAccord)) + { + Collection partitionUpdates = Mutation.merge(mutations.values()).getPartitionUpdates(); + checkState(partitionUpdates.size() == 1, "Expect only one PartitionUpdate"); + PartitionUpdate update = partitionUpdates.iterator().next(); + PartitionKey partitionKey = PartitionKey.of(update); + Keys key = Keys.of(partitionKey); + // This is going create a new BlockingReadRepair inside an Accord transaction which will go down + // the !isEventuallyConsistent path and apply the repairs through Accord command stores using AccordInteropExecution + UnrecoverableRepairUpdate repairUpdate = UnrecoverableRepairUpdate.create(AccordService.instance().nodeId(), this, key, dk, mutations, writePlan); + Future repairFuture; + try + { + Txn txn = new Txn.InMemory(key, TxnRead.createNoOpRead(key), TxnQuery.NONE, repairUpdate); + repairFuture = Stage.ACCORD_MIGRATION.submit(() -> { + try + { + return AccordService.instance().coordinate(txn, ConsistencyLevel.ANY, requestTime); + } + finally + { + // If we successfully ran the repair txn then the update should definitely + // be there for us to clear which means we are sure it was there to be sent + checkNotNull(UnrecoverableRepairUpdate.removeInflightUpdate(repairUpdate.updateKey)); + } + }); + } + catch (Throwable t) + { + UnrecoverableRepairUpdate.removeInflightUpdate(repairUpdate.updateKey); + throw t; + } + + repairs.add(new PendingPartitionRepair() + { + @Override + public boolean awaitRepairs(long remaining, TimeUnit timeUnit) throws InterruptedException, ExecutionException + { + try + { + repairFuture.get(remaining, timeUnit); + return true; + } + catch (TimeoutException e) + { + + return false; + } + } + + @Override + public ForWrite repairPlan() + { + return writePlan; + } + }); + } + else + { + BlockingPartitionRepair blockingRepair = new BlockingPartitionRepair(coordinator, dk, mutations, writePlan); + blockingRepair.sendInitialRepairs(); + repairs.add(blockingRepair); + } + } + + public void repairPartitionDirectly(ReadCoordinator readCoordinator, DecoratedKey dk, Map mutations, ForWrite writePlan) + { + ReadRepair delegateRR = ReadRepairStrategy.BLOCKING.create(readCoordinator, command, replicaPlan, requestTime); + delegateRR.repairPartition(dk, mutations, writePlan); + delegateRR.maybeSendAdditionalWrites(); + delegateRR.awaitWrites(); } } diff --git a/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java index 5cf72b33cf..2aad00bc9d 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java @@ -30,6 +30,7 @@ import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.service.reads.DigestResolver; +import org.apache.cassandra.service.reads.ReadCoordinator; /** * Bypasses the read repair path for short read protection and testing @@ -79,4 +80,11 @@ public class NoopReadRepair, P extends ReplicaPlan.ForRea { } + + @Override + public void repairPartitionDirectly(ReadCoordinator coordinator, DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan) + { + // Shouldn't be possible to invoke this since repairPartition is a no op + throw new UnsupportedOperationException(); + } } diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java index 46b30a9279..3a91cf67fc 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java @@ -28,8 +28,10 @@ import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.ReplicaPlan.ForWrite; import org.apache.cassandra.metrics.ReadRepairMetrics; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.service.reads.ReadCoordinator; /** * Only performs the collection of data responses and reconciliation of them, doesn't send repair mutations @@ -38,9 +40,9 @@ import org.apache.cassandra.transport.Dispatcher; public class ReadOnlyReadRepair, P extends ReplicaPlan.ForRead> extends AbstractReadRepair { - ReadOnlyReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + ReadOnlyReadRepair(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) { - super(command, replicaPlan, requestTime); + super(coordinator, command, replicaPlan, requestTime); } @Override @@ -67,6 +69,12 @@ public class ReadOnlyReadRepair, P extends ReplicaPlan.Fo throw new UnsupportedOperationException("ReadOnlyReadRepair shouldn't be trying to repair partitions"); } + @Override + public void repairPartitionDirectly(ReadCoordinator coordinator, DecoratedKey partitionKey, Map mutations, ForWrite writePlan) + { + throw new UnsupportedOperationException("ReadOnlyReadRepair shouldn't be trying to repair partitions"); + } + @Override public void awaitWrites() { diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java index a63cc7f6bf..bff068a6d7 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java @@ -19,32 +19,33 @@ package org.apache.cassandra.service.reads.repair; import java.util.Map; import java.util.function.Consumer; +import javax.annotation.Nullable; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.locator.Endpoints; - import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.service.reads.DigestResolver; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.service.reads.ReadCoordinator; public interface ReadRepair, P extends ReplicaPlan.ForRead> { public interface Factory { , P extends ReplicaPlan.ForRead> - ReadRepair create(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime); + ReadRepair create(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime); } static , P extends ReplicaPlan.ForRead> - ReadRepair create(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + ReadRepair create(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) { - return command.metadata().params.readRepair.create(command, replicaPlan, requestTime); + return command.metadata().params.readRepair.create(coordinator, command, replicaPlan, requestTime); } /** @@ -58,7 +59,7 @@ public interface ReadRepair, P extends ReplicaPlan.ForRea * @param digestResolver supplied so we can get the original data response * @param resultConsumer hook for the repair to set it's result on completion */ - public void startRepair(DigestResolver digestResolver, Consumer resultConsumer); + public void startRepair(DigestResolver digestResolver, @Nullable Consumer resultConsumer); /** * Block on the reads (or timeout) sent out in {@link ReadRepair#startRepair} @@ -94,4 +95,9 @@ public interface ReadRepair, P extends ReplicaPlan.ForRea * we will block repair only on the replicas that have responded. */ void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan); + + /** + * Repairs a partition using the provided read coordinator + */ + void repairPartitionDirectly(ReadCoordinator coordinator, DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan); } diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java index 22615494a7..7f8d861888 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java @@ -22,6 +22,7 @@ import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.service.reads.ReadCoordinator; import static org.apache.cassandra.utils.LocalizeString.toUpperCaseLocalized; @@ -30,18 +31,18 @@ public enum ReadRepairStrategy implements ReadRepair.Factory NONE { public , P extends ReplicaPlan.ForRead> - ReadRepair create(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + ReadRepair create(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) { - return new ReadOnlyReadRepair<>(command, replicaPlan, requestTime); + return new ReadOnlyReadRepair<>(coordinator, command, replicaPlan, requestTime); } }, BLOCKING { public , P extends ReplicaPlan.ForRead> - ReadRepair create(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + ReadRepair create(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) { - return new BlockingReadRepair<>(command, replicaPlan, requestTime); + return new BlockingReadRepair<>(coordinator, command, replicaPlan, requestTime); } }; diff --git a/src/java/org/apache/cassandra/streaming/OutgoingStream.java b/src/java/org/apache/cassandra/streaming/OutgoingStream.java index cc42ab6b82..77386a6225 100644 --- a/src/java/org/apache/cassandra/streaming/OutgoingStream.java +++ b/src/java/org/apache/cassandra/streaming/OutgoingStream.java @@ -19,7 +19,10 @@ package org.apache.cassandra.streaming; import java.io.IOException; +import java.util.List; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.utils.TimeUUID; @@ -54,4 +57,5 @@ public interface OutgoingStream long getEstimatedSize(); TableId getTableId(); int getNumFiles(); + List> ranges(); } diff --git a/src/java/org/apache/cassandra/streaming/SessionSummary.java b/src/java/org/apache/cassandra/streaming/SessionSummary.java index 9588e4918f..f5bcfa31be 100644 --- a/src/java/org/apache/cassandra/streaming/SessionSummary.java +++ b/src/java/org/apache/cassandra/streaming/SessionSummary.java @@ -25,7 +25,8 @@ import java.util.Collection; import java.util.List; import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.IPartitionerDependentSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; @@ -78,7 +79,7 @@ public class SessionSummary return result; } - public static IVersionedSerializer serializer = new IVersionedSerializer() + public static IPartitionerDependentSerializer serializer = new IPartitionerDependentSerializer() { public void serialize(SessionSummary summary, DataOutputPlus out, int version) throws IOException { @@ -98,7 +99,7 @@ public class SessionSummary } } - public SessionSummary deserialize(DataInputPlus in, int version) throws IOException + public SessionSummary deserialize(DataInputPlus in, IPartitioner partitioner, int version) throws IOException { InetAddressAndPort coordinator = inetAddressAndPortSerializer.deserialize(in, version); InetAddressAndPort peer = inetAddressAndPortSerializer.deserialize(in, version); @@ -107,14 +108,14 @@ public class SessionSummary List receivingSummaries = new ArrayList<>(numRcvd); for (int i=0; i sendingSummaries = new ArrayList<>(numRcvd); for (int i=0; i> ranges; + + public StreamReceiveTask(StreamSession session, TableId tableId, List> ranges, int totalStreams, long totalSize) { super(session, tableId); - this.receiver = ColumnFamilyStore.getIfExists(tableId).getStreamManager().createStreamReceiver(session, totalStreams); + Range.assertNormalized(ranges); + this.receiver = ColumnFamilyStore.getIfExists(tableId).getStreamManager().createStreamReceiver(session, ranges, totalStreams); this.totalStreams = totalStreams; this.totalSize = totalSize; + this.ranges = ranges; } /** @@ -164,6 +172,12 @@ public class StreamReceiveTask extends StreamTask receiver.abort(); } + @Override + protected List> ranges() + { + return ranges; + } + @VisibleForTesting public static void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java index 217c70f058..050e37c749 100644 --- a/src/java/org/apache/cassandra/streaming/StreamSession.java +++ b/src/java/org/apache/cassandra/streaming/StreamSession.java @@ -36,7 +36,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; - import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; @@ -45,13 +44,12 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import io.netty.channel.Channel; import io.netty.util.concurrent.Future; //checkstyle: permit this import -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; @@ -72,7 +70,17 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.async.StreamingMultiplexedChannel; -import org.apache.cassandra.streaming.messages.*; +import org.apache.cassandra.streaming.messages.CompleteMessage; +import org.apache.cassandra.streaming.messages.IncomingStreamMessage; +import org.apache.cassandra.streaming.messages.OutgoingStreamMessage; +import org.apache.cassandra.streaming.messages.PrepareAckMessage; +import org.apache.cassandra.streaming.messages.PrepareSynAckMessage; +import org.apache.cassandra.streaming.messages.PrepareSynMessage; +import org.apache.cassandra.streaming.messages.ReceivedMessage; +import org.apache.cassandra.streaming.messages.SessionFailedMessage; +import org.apache.cassandra.streaming.messages.StreamInitMessage; +import org.apache.cassandra.streaming.messages.StreamMessage; +import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.NoSpamLogger; @@ -454,6 +462,7 @@ public class StreamSession //Range and if it's transient RangesAtEndpoint unwrappedRanges = replicas.unwrap(); List streams = getOutgoingStreamsForRanges(unwrappedRanges, stores, pendingRepair, previewKind); + addTransferStreams(streams); Set> toBeUpdated = transferredRangesPerKeyspace.get(keyspace); if (toBeUpdated == null) @@ -735,7 +744,7 @@ public class StreamSession if (channel.connected()) { - state(State.FAILED); // make sure subsequent error handling sees the session in a final state + state(State.FAILED); // make sure subsequent error handling sees the session in a final state sendControlMessage(new SessionFailedMessage()).awaitUninterruptibly(); } StringBuilder failureReason = new StringBuilder("Failed because of an unknown exception\n"); @@ -1260,7 +1269,7 @@ public class StreamSession { failIfFinished(); if (summary.files > 0) - receivers.put(summary.tableId, new StreamReceiveTask(this, summary.tableId, summary.files, summary.totalSize)); + receivers.put(summary.tableId, new StreamReceiveTask(this, summary.tableId, summary.ranges, summary.files, summary.totalSize)); } private void startStreamingFiles(@Nullable PrepareDirection prepareDirection) diff --git a/src/java/org/apache/cassandra/streaming/StreamSummary.java b/src/java/org/apache/cassandra/streaming/StreamSummary.java index 3f957c69a7..34aa56c66d 100644 --- a/src/java/org/apache/cassandra/streaming/StreamSummary.java +++ b/src/java/org/apache/cassandra/streaming/StreamSummary.java @@ -19,23 +19,31 @@ package org.apache.cassandra.streaming; import java.io.IOException; import java.io.Serializable; +import java.util.List; import com.google.common.base.Objects; +import com.google.common.collect.ImmutableList; import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.IPartitionerDependentSerializer; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.utils.CollectionSerializers; /** * Summary of streaming. */ public class StreamSummary implements Serializable { - public static final IVersionedSerializer serializer = new StreamSummarySerializer(); + public static final IPartitionerDependentSerializer serializer = new StreamSummarySerializer(); public final TableId tableId; + public final List> ranges; /** * Number of files to transfer. Can be 0 if nothing to transfer for some streaming request. @@ -43,9 +51,10 @@ public class StreamSummary implements Serializable public final int files; public final long totalSize; - public StreamSummary(TableId tableId, int files, long totalSize) + public StreamSummary(TableId tableId, List> ranges, int files, long totalSize) { this.tableId = tableId; + this.ranges = ranges; this.files = files; this.totalSize = totalSize; } @@ -56,13 +65,13 @@ public class StreamSummary implements Serializable if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; StreamSummary summary = (StreamSummary) o; - return files == summary.files && totalSize == summary.totalSize && tableId.equals(summary.tableId); + return files == summary.files && totalSize == summary.totalSize && tableId.equals(summary.tableId) && ranges.equals(summary.ranges); } @Override public int hashCode() { - return Objects.hashCode(tableId, files, totalSize); + return Objects.hashCode(tableId, ranges, files, totalSize); } @Override @@ -70,27 +79,33 @@ public class StreamSummary implements Serializable { final StringBuilder sb = new StringBuilder("StreamSummary{"); sb.append("path=").append(tableId); + sb.append(", ranges=").append(ranges); sb.append(", files=").append(files); sb.append(", totalSize=").append(totalSize); sb.append('}'); return sb.toString(); } - public static class StreamSummarySerializer implements IVersionedSerializer + public static class StreamSummarySerializer implements IPartitionerDependentSerializer { public void serialize(StreamSummary summary, DataOutputPlus out, int version) throws IOException { summary.tableId.serialize(out); out.writeInt(summary.files); out.writeLong(summary.totalSize); + if (version >= MessagingService.VERSION_51) + CollectionSerializers.serializeCollection(summary.ranges, out, version, Range.rangeSerializer); } - public StreamSummary deserialize(DataInputPlus in, int version) throws IOException + public StreamSummary deserialize(DataInputPlus in, IPartitioner p, int version) throws IOException { TableId tableId = TableId.deserialize(in); int files = in.readInt(); long totalSize = in.readLong(); - return new StreamSummary(tableId, files, totalSize); + List> ranges = ImmutableList.of(); + if (version >= MessagingService.VERSION_51) + ranges = CollectionSerializers.deserializeList(in, p, version, Range.rangeSerializer); + return new StreamSummary(tableId, ranges, files, totalSize); } public long serializedSize(StreamSummary summary, int version) @@ -98,6 +113,8 @@ public class StreamSummary implements Serializable long size = summary.tableId.serializedSize(); size += TypeSizes.sizeof(summary.files); size += TypeSizes.sizeof(summary.totalSize); + if (version >= MessagingService.VERSION_51) + size += CollectionSerializers.serializedCollectionSize(summary.ranges, version, Range.rangeSerializer); return size; } } diff --git a/src/java/org/apache/cassandra/streaming/StreamTask.java b/src/java/org/apache/cassandra/streaming/StreamTask.java index 1e22c34ce9..886257fda7 100644 --- a/src/java/org/apache/cassandra/streaming/StreamTask.java +++ b/src/java/org/apache/cassandra/streaming/StreamTask.java @@ -17,6 +17,10 @@ */ package org.apache.cassandra.streaming; +import java.util.List; + +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.TableId; /** @@ -51,11 +55,13 @@ public abstract class StreamTask */ public abstract void abort(); + protected abstract List> ranges(); + /** * @return StreamSummary that describes this task */ public StreamSummary getSummary() { - return new StreamSummary(tableId, getTotalNumberOfFiles(), getTotalSize()); + return new StreamSummary(tableId, ranges(), getTotalNumberOfFiles(), getTotalSize()); } } diff --git a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java index 0721316ccb..8b4fe1f1bd 100644 --- a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java +++ b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java @@ -20,7 +20,10 @@ package org.apache.cassandra.streaming; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -33,6 +36,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.messages.OutgoingStreamMessage; import org.apache.cassandra.utils.ExecutorUtils; @@ -57,6 +62,8 @@ public class StreamTransferTask extends StreamTask private long totalSize = 0; private int totalFiles = 0; + private final Set> ranges = new HashSet<>(); + public StreamTransferTask(StreamSession session, TableId tableId) { super(session, tableId); @@ -70,6 +77,7 @@ public class StreamTransferTask extends StreamTask streams.put(message.header.sequenceNumber, message); totalSize += message.stream.getEstimatedSize(); totalFiles += message.stream.getNumFiles(); + ranges.addAll(stream.ranges()); } /** @@ -149,6 +157,12 @@ public class StreamTransferTask extends StreamTask } } + @Override + protected List> ranges() + { + return Range.normalize(ranges); + } + public synchronized int getTotalNumberOfFiles() { return totalFiles; diff --git a/src/java/org/apache/cassandra/streaming/TableStreamManager.java b/src/java/org/apache/cassandra/streaming/TableStreamManager.java index 208dc344a9..d19064c957 100644 --- a/src/java/org/apache/cassandra/streaming/TableStreamManager.java +++ b/src/java/org/apache/cassandra/streaming/TableStreamManager.java @@ -19,7 +19,10 @@ package org.apache.cassandra.streaming; import java.util.Collection; +import java.util.List; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.TimeUUID; @@ -36,7 +39,7 @@ public interface TableStreamManager /** * Creates a {@link StreamReceiver} for the given session, expecting the given number of streams */ - StreamReceiver createStreamReceiver(StreamSession session, int totalStreams); + StreamReceiver createStreamReceiver(StreamSession session, List> ranges, int totalStreams); /** * Creates an {@link IncomingStream} for the given header diff --git a/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java b/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java index 05a0afcfd4..c79c1a9a1f 100644 --- a/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java +++ b/src/java/org/apache/cassandra/streaming/management/StreamSummaryCompositeData.java @@ -19,7 +19,14 @@ package org.apache.cassandra.streaming.management; import java.util.HashMap; import java.util.Map; -import javax.management.openmbean.*; +import javax.management.openmbean.CompositeData; +import javax.management.openmbean.CompositeDataSupport; +import javax.management.openmbean.CompositeType; +import javax.management.openmbean.OpenDataException; +import javax.management.openmbean.OpenType; +import javax.management.openmbean.SimpleType; + +import com.google.common.collect.ImmutableList; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.StreamSummary; @@ -75,6 +82,7 @@ public class StreamSummaryCompositeData { Object[] values = cd.getAll(ITEM_NAMES); return new StreamSummary(TableId.fromString((String) values[0]), + ImmutableList.of(), (int) values[1], (long) values[2]); } diff --git a/src/java/org/apache/cassandra/streaming/messages/CompleteMessage.java b/src/java/org/apache/cassandra/streaming/messages/CompleteMessage.java index bf3526663c..86620c3859 100644 --- a/src/java/org/apache/cassandra/streaming/messages/CompleteMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/CompleteMessage.java @@ -17,15 +17,16 @@ */ package org.apache.cassandra.streaming.messages; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; public class CompleteMessage extends StreamMessage { public static Serializer serializer = new Serializer() { - public CompleteMessage deserialize(DataInputPlus in, int version) + public CompleteMessage deserialize(DataInputPlus in, IPartitioner partitioner, int version) { return new CompleteMessage(); } diff --git a/src/java/org/apache/cassandra/streaming/messages/IncomingStreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/IncomingStreamMessage.java index ff1e61fd59..4ee726ee83 100644 --- a/src/java/org/apache/cassandra/streaming/messages/IncomingStreamMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/IncomingStreamMessage.java @@ -21,20 +21,20 @@ import java.io.IOException; import java.util.Objects; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; - import org.apache.cassandra.streaming.IncomingStream; -import org.apache.cassandra.streaming.StreamingChannel; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamManager; import org.apache.cassandra.streaming.StreamReceiveException; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingChannel; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; public class IncomingStreamMessage extends StreamMessage { public static Serializer serializer = new Serializer() { - public IncomingStreamMessage deserialize(DataInputPlus input, int version) throws IOException + public IncomingStreamMessage deserialize(DataInputPlus input, IPartitioner partitioner, int version) throws IOException { StreamMessageHeader header = StreamMessageHeader.serializer.deserialize(input, version); StreamSession session = StreamManager.instance.findSession(header.sender, header.planId, header.sessionIndex, header.sendByFollower); diff --git a/src/java/org/apache/cassandra/streaming/messages/KeepAliveMessage.java b/src/java/org/apache/cassandra/streaming/messages/KeepAliveMessage.java index a09cfcae82..928783f401 100644 --- a/src/java/org/apache/cassandra/streaming/messages/KeepAliveMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/KeepAliveMessage.java @@ -17,9 +17,10 @@ */ package org.apache.cassandra.streaming.messages; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; public class KeepAliveMessage extends StreamMessage { @@ -37,7 +38,7 @@ public class KeepAliveMessage extends StreamMessage public static Serializer serializer = new Serializer() { - public KeepAliveMessage deserialize(DataInputPlus in, int version) + public KeepAliveMessage deserialize(DataInputPlus in, IPartitioner partitioner, int version) { return new KeepAliveMessage(); } diff --git a/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java index 4128ddb4b0..dcd3b755e8 100644 --- a/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java @@ -21,18 +21,19 @@ import java.io.IOException; import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.OutgoingStream; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.utils.FBUtilities; public class OutgoingStreamMessage extends StreamMessage { public static Serializer serializer = new Serializer() { - public OutgoingStreamMessage deserialize(DataInputPlus in, int version) + public OutgoingStreamMessage deserialize(DataInputPlus in, IPartitioner partitioner, int version) { throw new UnsupportedOperationException("Not allowed to call deserialize on an outgoing stream"); } diff --git a/src/java/org/apache/cassandra/streaming/messages/PrepareAckMessage.java b/src/java/org/apache/cassandra/streaming/messages/PrepareAckMessage.java index 479ef3424d..72d61d29cb 100644 --- a/src/java/org/apache/cassandra/streaming/messages/PrepareAckMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/PrepareAckMessage.java @@ -20,9 +20,10 @@ package org.apache.cassandra.streaming.messages; import java.io.IOException; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; public class PrepareAckMessage extends StreamMessage { @@ -33,7 +34,7 @@ public class PrepareAckMessage extends StreamMessage //nop } - public PrepareAckMessage deserialize(DataInputPlus in, int version) throws IOException + public PrepareAckMessage deserialize(DataInputPlus in, IPartitioner partitioner, int version) throws IOException { return new PrepareAckMessage(); } diff --git a/src/java/org/apache/cassandra/streaming/messages/PrepareSynAckMessage.java b/src/java/org/apache/cassandra/streaming/messages/PrepareSynAckMessage.java index 9d97de69fa..e29e651824 100644 --- a/src/java/org/apache/cassandra/streaming/messages/PrepareSynAckMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/PrepareSynAckMessage.java @@ -22,10 +22,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.streaming.StreamSummary; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; public class PrepareSynAckMessage extends StreamMessage { @@ -38,12 +39,12 @@ public class PrepareSynAckMessage extends StreamMessage StreamSummary.serializer.serialize(summary, out, version); } - public PrepareSynAckMessage deserialize(DataInputPlus input, int version) throws IOException + public PrepareSynAckMessage deserialize(DataInputPlus input, IPartitioner partitioner, int version) throws IOException { PrepareSynAckMessage message = new PrepareSynAckMessage(); int numSummaries = input.readInt(); for (int i = 0; i < numSummaries; i++) - message.summaries.add(StreamSummary.serializer.deserialize(input, version)); + message.summaries.add(StreamSummary.serializer.deserialize(input, partitioner, version)); return message; } diff --git a/src/java/org/apache/cassandra/streaming/messages/PrepareSynMessage.java b/src/java/org/apache/cassandra/streaming/messages/PrepareSynMessage.java index 1160033bd3..e901365e5e 100644 --- a/src/java/org/apache/cassandra/streaming/messages/PrepareSynMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/PrepareSynMessage.java @@ -17,21 +17,22 @@ */ package org.apache.cassandra.streaming.messages; -import java.io.*; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamRequest; import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.streaming.StreamSummary; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; public class PrepareSynMessage extends StreamMessage { public static Serializer serializer = new Serializer() { - public PrepareSynMessage deserialize(DataInputPlus input, int version) throws IOException + public PrepareSynMessage deserialize(DataInputPlus input, IPartitioner partitioner, int version) throws IOException { PrepareSynMessage message = new PrepareSynMessage(); // requests @@ -41,7 +42,7 @@ public class PrepareSynMessage extends StreamMessage // summaries int numSummaries = input.readInt(); for (int i = 0; i < numSummaries; i++) - message.summaries.add(StreamSummary.serializer.deserialize(input, version)); + message.summaries.add(StreamSummary.serializer.deserialize(input, partitioner, version)); return message; } diff --git a/src/java/org/apache/cassandra/streaming/messages/ReceivedMessage.java b/src/java/org/apache/cassandra/streaming/messages/ReceivedMessage.java index 378f72f896..c6b7a0f638 100644 --- a/src/java/org/apache/cassandra/streaming/messages/ReceivedMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/ReceivedMessage.java @@ -17,18 +17,19 @@ */ package org.apache.cassandra.streaming.messages; -import java.io.*; +import java.io.IOException; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; public class ReceivedMessage extends StreamMessage { public static Serializer serializer = new Serializer() { - public ReceivedMessage deserialize(DataInputPlus input, int version) throws IOException + public ReceivedMessage deserialize(DataInputPlus input, IPartitioner partitioner, int version) throws IOException { return new ReceivedMessage(TableId.deserialize(input), input.readInt()); } diff --git a/src/java/org/apache/cassandra/streaming/messages/SessionFailedMessage.java b/src/java/org/apache/cassandra/streaming/messages/SessionFailedMessage.java index f09b64327e..f05be58aa6 100644 --- a/src/java/org/apache/cassandra/streaming/messages/SessionFailedMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/SessionFailedMessage.java @@ -17,15 +17,16 @@ */ package org.apache.cassandra.streaming.messages; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; public class SessionFailedMessage extends StreamMessage { public static Serializer serializer = new Serializer() { - public SessionFailedMessage deserialize(DataInputPlus in, int version) + public SessionFailedMessage deserialize(DataInputPlus in, IPartitioner partitioner, int version) { return new SessionFailedMessage(); } diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamInitMessage.java b/src/java/org/apache/cassandra/streaming/messages/StreamInitMessage.java index 889c732f0f..2fd65d7dff 100644 --- a/src/java/org/apache/cassandra/streaming/messages/StreamInitMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/StreamInitMessage.java @@ -20,14 +20,15 @@ package org.apache.cassandra.streaming.messages; import java.io.IOException; import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.streaming.StreamingChannel; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; -import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingChannel; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.utils.TimeUUID; import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer; @@ -93,7 +94,7 @@ public class StreamInitMessage extends StreamMessage out.writeInt(message.previewKind.getSerializationVal()); } - public StreamInitMessage deserialize(DataInputPlus in, int version) throws IOException + public StreamInitMessage deserialize(DataInputPlus in, IPartitioner partitioner, int version) throws IOException { InetAddressAndPort from = inetAddressAndPortSerializer.deserialize(in, version); int sessionIndex = in.readInt(); diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java index db393a5434..6e5dc08f88 100644 --- a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java @@ -21,10 +21,11 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.streaming.StreamingChannel; import org.apache.cassandra.streaming.StreamingDataOutputPlus; -import org.apache.cassandra.streaming.StreamSession; /** * StreamMessage is an abstract base class that every messages in streaming protocol inherit. @@ -44,16 +45,16 @@ public abstract class StreamMessage return 1 + message.type.outSerializer.serializedSize(message, version); } - public static StreamMessage deserialize(DataInputPlus in, int version) throws IOException + public static StreamMessage deserialize(DataInputPlus in, IPartitioner partitioner, int version) throws IOException { Type type = Type.lookupById(in.readByte()); - return type.inSerializer.deserialize(in, version); + return type.inSerializer.deserialize(in, partitioner, version); } /** StreamMessage serializer */ public static interface Serializer { - V deserialize(DataInputPlus in, int version) throws IOException; + V deserialize(DataInputPlus in, IPartitioner partitioner, int version) throws IOException; void serialize(V message, StreamingDataOutputPlus out, int version, StreamSession session) throws IOException; long serializedSize(V message, int version) throws IOException; } diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java index 5d51cfd3e3..2b9dd0a04d 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java @@ -54,6 +54,9 @@ import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; import org.apache.cassandra.tcm.extensions.ExtensionKey; import org.apache.cassandra.tcm.extensions.ExtensionValue; import org.apache.cassandra.tcm.membership.Directory; @@ -97,6 +100,7 @@ public class ClusterMetadata public final AccordKeyspaces accordKeyspaces; public final LockedRanges lockedRanges; public final InProgressSequences inProgressSequences; + public final ConsensusMigrationState consensusMigrationState; public final ImmutableMap, ExtensionValue> extensions; // This isn't serialized as part of ClusterMetadata it's really just a view over the Directory. @@ -132,6 +136,7 @@ public class ClusterMetadata AccordKeyspaces.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, + ConsensusMigrationState.EMPTY, ImmutableMap.of()); } @@ -144,6 +149,7 @@ public class ClusterMetadata AccordKeyspaces accordKeyspaces, LockedRanges lockedRanges, InProgressSequences inProgressSequences, + ConsensusMigrationState consensusMigrationState, Map, ExtensionValue> extensions) { this(EMPTY_METADATA_IDENTIFIER, @@ -156,6 +162,7 @@ public class ClusterMetadata accordKeyspaces, lockedRanges, inProgressSequences, + consensusMigrationState, extensions); } @@ -169,6 +176,7 @@ public class ClusterMetadata AccordKeyspaces accordKeyspaces, LockedRanges lockedRanges, InProgressSequences inProgressSequences, + ConsensusMigrationState consensusMigrationState, Map, ExtensionValue> extensions) { // TODO: token map is a feature of the specific placement strategy, and so may not be a relevant component of @@ -185,6 +193,7 @@ public class ClusterMetadata this.accordKeyspaces = accordKeyspaces; this.lockedRanges = lockedRanges; this.inProgressSequences = inProgressSequences; + this.consensusMigrationState = consensusMigrationState; this.extensions = ImmutableMap.copyOf(extensions); this.locator = Locator.usingDirectory(directory); } @@ -241,6 +250,7 @@ public class ClusterMetadata capLastModified(accordKeyspaces, epoch), capLastModified(lockedRanges, epoch), capLastModified(inProgressSequences, epoch), + capLastModified(consensusMigrationState, epoch), capLastModified(extensions, epoch)); } @@ -262,6 +272,7 @@ public class ClusterMetadata accordKeyspaces, lockedRanges, inProgressSequences, + consensusMigrationState, extensions); } @@ -388,6 +399,7 @@ public class ClusterMetadata private AccordKeyspaces accordKeyspaces; private LockedRanges lockedRanges; private InProgressSequences inProgressSequences; + private ConsensusMigrationState consensusMigrationState; private final Map, ExtensionValue> extensions; private final Set modifiedKeys; @@ -403,6 +415,7 @@ public class ClusterMetadata this.accordKeyspaces = metadata.accordKeyspaces; this.lockedRanges = metadata.lockedRanges; this.inProgressSequences = metadata.inProgressSequences; + this.consensusMigrationState = metadata.consensusMigrationState; extensions = new HashMap<>(metadata.extensions); modifiedKeys = new HashSet<>(); } @@ -538,6 +551,31 @@ public class ClusterMetadata return this; } + public Transformer with(Map newTableMigrationStates) + { + return with(newTableMigrationStates, true); + } + + public Transformer with(Map newTableMigrationStates, + boolean addRemaining) + { + if (addRemaining) + { + ImmutableMap.Builder tableMigrationStatesBuilder = ImmutableMap.builder(); + consensusMigrationState.tableStates.entrySet() + .stream() + .filter(existingTMS -> !newTableMigrationStates.containsKey(existingTMS.getKey())) + .forEach(tableMigrationStatesBuilder::put); + tableMigrationStatesBuilder.putAll(newTableMigrationStates.entrySet()); + consensusMigrationState = new ConsensusMigrationState(Epoch.EMPTY, tableMigrationStatesBuilder.build()); + } + else + { + consensusMigrationState = new ConsensusMigrationState(Epoch.EMPTY, newTableMigrationStates); + } + return this; + } + public Transformer with(ExtensionKey key, ExtensionValue obj) { if (MetadataKeys.CORE_METADATA.contains(key)) @@ -630,6 +668,12 @@ public class ClusterMetadata inProgressSequences = inProgressSequences.withLastModified(epoch); } + if (consensusMigrationState != base.consensusMigrationState) + { + modifiedKeys.add(MetadataKeys.CONSENSUS_MIGRATION_STATE); + consensusMigrationState = consensusMigrationState.withLastModified(epoch); + } + return new Transformed(new ClusterMetadata(base.metadataIdentifier, epoch, partitioner, @@ -640,6 +684,7 @@ public class ClusterMetadata accordKeyspaces, lockedRanges, inProgressSequences, + consensusMigrationState, extensions), ImmutableSet.copyOf(modifiedKeys)); } @@ -656,6 +701,7 @@ public class ClusterMetadata accordKeyspaces, lockedRanges, inProgressSequences, + consensusMigrationState, extensions); } @@ -672,6 +718,7 @@ public class ClusterMetadata ", placement=" + placements + ", lockedRanges=" + lockedRanges + ", inProgressSequences=" + inProgressSequences + + ", consensusMigrationState=" + consensusMigrationState + ", extensions=" + extensions + ", modifiedKeys=" + modifiedKeys + '}'; @@ -759,6 +806,7 @@ public class ClusterMetadata @Override public String toString() { + // TODO is this supposed to be missing fields? return "ClusterMetadata{" + "epoch=" + epoch + ", schema=" + schema + @@ -766,6 +814,7 @@ public class ClusterMetadata ", tokenMap=" + tokenMap + ", placements=" + placements + ", lockedRanges=" + lockedRanges + + ", consensusMigrationState=" + lockedRanges + '}'; } @@ -780,8 +829,10 @@ public class ClusterMetadata directory.equals(that.directory) && tokenMap.equals(that.tokenMap) && placements.equals(that.placements) && + accordKeyspaces.equals(that.accordKeyspaces) && lockedRanges.equals(that.lockedRanges) && inProgressSequences.equals(that.inProgressSequences) && + consensusMigrationState.equals(that.consensusMigrationState) && extensions.equals(that.extensions); } @@ -830,7 +881,7 @@ public class ClusterMetadata @Override public int hashCode() { - return Objects.hash(epoch, schema, directory, tokenMap, placements, lockedRanges, inProgressSequences, extensions); + return Objects.hash(epoch, schema, directory, tokenMap, placements, accordKeyspaces, lockedRanges, inProgressSequences, consensusMigrationState, extensions); } public static ClusterMetadata current() @@ -910,6 +961,7 @@ public class ClusterMetadata AccordKeyspaces.serializer.serialize(metadata.accordKeyspaces, out, version); LockedRanges.serializer.serialize(metadata.lockedRanges, out, version); InProgressSequences.serializer.serialize(metadata.inProgressSequences, out, version); + ConsensusMigrationState.serializer.serialize(metadata.consensusMigrationState, out, version); out.writeInt(metadata.extensions.size()); for (Map.Entry, ExtensionValue> entry : metadata.extensions.entrySet()) { @@ -947,6 +999,7 @@ public class ClusterMetadata AccordKeyspaces accordKeyspaces = AccordKeyspaces.serializer.deserialize(in, version); LockedRanges lockedRanges = LockedRanges.serializer.deserialize(in, version); InProgressSequences ips = InProgressSequences.serializer.deserialize(in, version); + ConsensusMigrationState consensusMigrationState = ConsensusMigrationState.serializer.deserialize(in, version); int items = in.readInt(); Map, ExtensionValue> extensions = new HashMap<>(items); for (int i = 0; i < items; i++) @@ -966,6 +1019,7 @@ public class ClusterMetadata accordKeyspaces, lockedRanges, ips, + consensusMigrationState, extensions); } @@ -988,7 +1042,8 @@ public class ClusterMetadata DataPlacements.serializer.serializedSize(metadata.placements, version) + AccordKeyspaces.serializer.serializedSize(metadata.accordKeyspaces, version) + LockedRanges.serializer.serializedSize(metadata.lockedRanges, version) + - InProgressSequences.serializer.serializedSize(metadata.inProgressSequences, version); + InProgressSequences.serializer.serializedSize(metadata.inProgressSequences, version) + + ConsensusMigrationState.serializer.serializedSize(metadata.consensusMigrationState, version); return size; } diff --git a/src/java/org/apache/cassandra/tcm/Epoch.java b/src/java/org/apache/cassandra/tcm/Epoch.java index 0d070b4a5b..d15030e3ec 100644 --- a/src/java/org/apache/cassandra/tcm/Epoch.java +++ b/src/java/org/apache/cassandra/tcm/Epoch.java @@ -87,6 +87,11 @@ public class Epoch implements Comparable, Serializable return l.compareTo(r) > 0 ? l : r; } + public static Epoch min(Epoch l, Epoch r) + { + return l.compareTo(r) < 0 ? l : r; + } + public boolean isDirectlyBefore(Epoch epoch) { if (epoch.equals(Epoch.FIRST)) diff --git a/src/java/org/apache/cassandra/tcm/MetadataKeys.java b/src/java/org/apache/cassandra/tcm/MetadataKeys.java index 18a9e6d023..df65474a53 100644 --- a/src/java/org/apache/cassandra/tcm/MetadataKeys.java +++ b/src/java/org/apache/cassandra/tcm/MetadataKeys.java @@ -42,6 +42,7 @@ public class MetadataKeys public static final MetadataKey ACCORD_KEYSPACES = make(CORE_NS, "ownership", "accord_keyspaces"); public static final MetadataKey LOCKED_RANGES = make(CORE_NS, "sequences", "locked_ranges"); public static final MetadataKey IN_PROGRESS_SEQUENCES = make(CORE_NS, "sequences", "in_progress"); + public static final MetadataKey CONSENSUS_MIGRATION_STATE = make(CORE_NS, "consensus", "migration_state"); public static final ImmutableSet CORE_METADATA = ImmutableSet.of(SCHEMA, NODE_DIRECTORY, @@ -49,7 +50,8 @@ public class MetadataKeys DATA_PLACEMENTS, ACCORD_KEYSPACES, LOCKED_RANGES, - IN_PROGRESS_SEQUENCES); + IN_PROGRESS_SEQUENCES, + CONSENSUS_MIGRATION_STATE); public static MetadataKey make(String...parts) { diff --git a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java index 1e115a5379..34799e9b56 100644 --- a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java @@ -28,6 +28,7 @@ import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Keyspaces; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; import org.apache.cassandra.tcm.Commit.Replicator; import org.apache.cassandra.tcm.log.Entry; import org.apache.cassandra.tcm.log.LocalLog; @@ -176,6 +177,7 @@ public class StubClusterMetadataService extends ClusterMetadataService AccordKeyspaces.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, + ConsensusTableMigrationState.ConsensusMigrationState.EMPTY, ImmutableMap.of()); } return new StubClusterMetadataService(new UniformRangePlacement(), diff --git a/src/java/org/apache/cassandra/tcm/Transformation.java b/src/java/org/apache/cassandra/tcm/Transformation.java index f90a0da634..cdbf44fcb6 100644 --- a/src/java/org/apache/cassandra/tcm/Transformation.java +++ b/src/java/org/apache/cassandra/tcm/Transformation.java @@ -38,8 +38,25 @@ import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; -import org.apache.cassandra.tcm.transformations.*; +import org.apache.cassandra.tcm.transformations.AddAccordKeyspace; +import org.apache.cassandra.tcm.transformations.AlterSchema; +import org.apache.cassandra.tcm.transformations.AlterTopology; +import org.apache.cassandra.tcm.transformations.Assassinate; +import org.apache.cassandra.tcm.transformations.BeginConsensusMigrationForTableAndRange; +import org.apache.cassandra.tcm.transformations.CancelInProgressSequence; +import org.apache.cassandra.tcm.transformations.CustomTransformation; +import org.apache.cassandra.tcm.transformations.ForceSnapshot; +import org.apache.cassandra.tcm.transformations.MaybeFinishConsensusMigrationForTableAndRange; +import org.apache.cassandra.tcm.transformations.PrepareJoin; +import org.apache.cassandra.tcm.transformations.PrepareLeave; +import org.apache.cassandra.tcm.transformations.PrepareMove; +import org.apache.cassandra.tcm.transformations.PrepareReplace; +import org.apache.cassandra.tcm.transformations.Register; +import org.apache.cassandra.tcm.transformations.SetConsensusMigrationTargetProtocol; import org.apache.cassandra.tcm.transformations.Startup; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; +import org.apache.cassandra.tcm.transformations.Unregister; +import org.apache.cassandra.tcm.transformations.UnsafeJoin; import org.apache.cassandra.tcm.transformations.cms.AdvanceCMSReconfiguration; import org.apache.cassandra.tcm.transformations.cms.FinishAddToCMS; import org.apache.cassandra.tcm.transformations.cms.Initialize; @@ -221,7 +238,11 @@ public interface Transformation ADVANCE_CMS_RECONFIGURATION(33, () -> AdvanceCMSReconfiguration.serializer), CANCEL_CMS_RECONFIGURATION(34, () -> CancelCMSReconfiguration.serializer), ALTER_TOPOLOGY(35, () -> AlterTopology.serializer), - ADD_ACCORD_KEYSPACE(36, () -> AddAccordKeyspace.serializer) + + ADD_ACCORD_KEYSPACE(36, () -> AddAccordKeyspace.serializer), + BEGIN_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(37, () -> BeginConsensusMigrationForTableAndRange.serializer), + MAYBE_FINISH_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(38, () -> MaybeFinishConsensusMigrationForTableAndRange.serializer), + SET_CONSENSUS_MIGRATION_TARGET_PROTOCOL(39, () -> SetConsensusMigrationTargetProtocol.serializer) ; private final Supplier> serializer; diff --git a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java index a5530e6faa..f87278d4b3 100644 --- a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java +++ b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java @@ -34,7 +34,6 @@ import java.util.UUID; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +54,7 @@ import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaKeyspace; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MultiStepOperation; @@ -299,6 +299,7 @@ public class GossipHelper AccordKeyspaces.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, + ConsensusMigrationState.EMPTY, Collections.emptyMap()); } @@ -387,6 +388,7 @@ public class GossipHelper AccordKeyspaces.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, + ConsensusMigrationState.EMPTY, extensions); DataPlacements placements = new UniformRangePlacement().calculatePlacements(Epoch.UPGRADE_GOSSIP, forPlacementCalculation, @@ -400,6 +402,7 @@ public class GossipHelper AccordKeyspaces.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, + ConsensusMigrationState.EMPTY, extensions); } diff --git a/src/java/org/apache/cassandra/tcm/ownership/TokenMap.java b/src/java/org/apache/cassandra/tcm/ownership/TokenMap.java index c17d91ae48..ca0a3eb930 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/TokenMap.java +++ b/src/java/org/apache/cassandra/tcm/ownership/TokenMap.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Objects; import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +42,6 @@ import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; -import org.apache.cassandra.utils.BiMultiValMap; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.SortedBiMultiValMap; @@ -112,7 +112,7 @@ public class TokenMap implements MetadataValue return new TokenMap(lastModified, partitioner, finalisedCopy); } - public BiMultiValMap asMap() + public SortedBiMultiValMap asMap() { return SortedBiMultiValMap.create(map); } @@ -164,7 +164,14 @@ public class TokenMap implements MetadataValue ranges.add(r); } - public Token nextToken(List tokens, Token token) + public Token getPredecessor(Token token) + { + int index = Collections.binarySearch(tokens, token); + assert index >= 0 : token + " not found in " + StringUtils.join(map.keySet(), ", "); + return index == 0 ? tokens.get(tokens.size() - 1) : tokens.get(index - 1); + } + + public static Token nextToken(List tokens, Token token) { return tokens.get(nextTokenIndex(tokens, token)); } diff --git a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java index faa86e357b..5cdd1b3016 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java @@ -26,6 +26,9 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Streams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,14 +41,19 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff; import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.SchemaTransformation; +import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Tables; import org.apache.cassandra.schema.ViewMetadata; import org.apache.cassandra.schema.Views; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadata.Transformer; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Transformation; @@ -58,11 +66,13 @@ import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.vint.VIntCoding; import static org.apache.cassandra.cql3.statements.schema.AlterSchemaStatement.NO_EXECUTION_TIMESTAMP; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static org.apache.cassandra.exceptions.ExceptionCode.ALREADY_EXISTS; import static org.apache.cassandra.exceptions.ExceptionCode.CONFIG_ERROR; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; import static org.apache.cassandra.exceptions.ExceptionCode.SERVER_ERROR; import static org.apache.cassandra.exceptions.ExceptionCode.SYNTAX_ERROR; +import static org.apache.cassandra.utils.Collectors3.toImmutableMap; public class AlterSchema implements Transformation { @@ -231,7 +241,7 @@ public class AlterSchema implements Transformation }); next = next.with(newPlacementsBuilder.build()); } - + next = maybeUpdateConsensusTableMigrationStateForDroppedTables(prev.consensusMigrationState, next, diff.altered, diff.dropped); return Transformation.success(next, LockedRanges.AffectedRanges.EMPTY); } @@ -247,6 +257,24 @@ public class AlterSchema implements Transformation return byReplication; } + private Transformer maybeUpdateConsensusTableMigrationStateForDroppedTables(ConsensusMigrationState prev, Transformer next, ImmutableList altered, Keyspaces dropped) + { + Set tableIds = Streams.concat( + altered.stream().flatMap(diff -> diff.tables.dropped.stream().map(TableMetadata::id)), + dropped.stream().flatMap(ks -> ks.tables.stream().map(TableMetadata::id))) + .collect(toImmutableSet()); + if (tableIds.stream().anyMatch(prev.tableStates.keySet()::contains)) + { + ImmutableMap newTableStates = + prev.tableStates.entrySet().stream().filter(e -> !tableIds.contains(e.getKey())).collect(toImmutableMap()); + return next.with(newTableStates); + } + else + { + return next; + } + } + private static Iterable normaliseTableEpochs(Epoch nextEpoch, Stream tables) { return tables.map(tm -> tm.epoch.is(nextEpoch) diff --git a/src/java/org/apache/cassandra/tcm/transformations/BeginConsensusMigrationForTableAndRange.java b/src/java/org/apache/cassandra/tcm/transformations/BeginConsensusMigrationForTableAndRange.java new file mode 100644 index 0000000000..a00db104af --- /dev/null +++ b/src/java/org/apache/cassandra/tcm/transformations/BeginConsensusMigrationForTableAndRange.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm.transformations; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import javax.annotation.Nonnull; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.Transformation; +import org.apache.cassandra.tcm.sequences.LockedRanges; +import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; +import static org.apache.cassandra.tcm.ClusterMetadata.Transformer; +import static org.apache.cassandra.utils.CollectionSerializers.deserializeList; +import static org.apache.cassandra.utils.CollectionSerializers.serializeCollection; +import static org.apache.cassandra.utils.CollectionSerializers.serializedCollectionSize; +import static org.apache.cassandra.utils.Collectors3.toImmutableMap; + +public class BeginConsensusMigrationForTableAndRange implements Transformation +{ + public static Serializer serializer = new Serializer(); + + @Nonnull + public final ConsensusMigrationTarget targetProtocol; + + @Nonnull + public final List> ranges; + + @Nonnull + public final List tables; + + public BeginConsensusMigrationForTableAndRange(@Nonnull ConsensusMigrationTarget targetProtocol, + @Nonnull List> ranges, + @Nonnull List tables) + { + checkNotNull(targetProtocol, "targetProtocol should not be null"); + checkNotNull(ranges, "ranges should not be null"); + checkArgument(!ranges.isEmpty(), "ranges should not be empty"); + checkNotNull(tables, "tables should not be null"); + checkArgument(!tables.isEmpty(), "tables should not be empty"); + this.targetProtocol = targetProtocol; + this.ranges = ranges; + this.tables = tables; + } + + public Kind kind() + { + return Kind.BEGIN_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE; + } + + public Result execute(ClusterMetadata prev) + { + Map tableStates = prev.consensusMigrationState.tableStates; + List columnFamilyStores = tables.stream().map(Schema.instance::getColumnFamilyStoreInstance).collect(toImmutableList()); + + Transformer transformer = prev.transformer(); + + Map newStates = columnFamilyStores + .stream() + .map(cfs -> + tableStates.containsKey(cfs.getTableId()) ? + tableStates.get(cfs.getTableId()).withRangesMigrating(ranges, targetProtocol) : + new TableMigrationState(cfs.keyspace.getName(), cfs.name, cfs.getTableId(), targetProtocol, ImmutableSet.of(), ImmutableMap.of(Epoch.EMPTY, ranges))) + .collect(toImmutableMap(TableMigrationState::getTableId, Function.identity())); + + return Transformation.success(transformer.with(newStates), LockedRanges.AffectedRanges.EMPTY); + } + + static class Serializer implements AsymmetricMetadataSerializer + { + + public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException + { + BeginConsensusMigrationForTableAndRange v = (BeginConsensusMigrationForTableAndRange)t; + out.writeUTF(v.targetProtocol.toString()); + ConsensusTableMigrationState.rangesSerializer.serialize(v.ranges, out, version); + serializeCollection(v.tables, out, version, TableId.metadataSerializer); + } + + public BeginConsensusMigrationForTableAndRange deserialize(DataInputPlus in, Version version) throws IOException + { + ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(in.readUTF()); + List> ranges = ConsensusTableMigrationState.rangesSerializer.deserialize(in, version); + List tables = deserializeList(in, version, TableId.metadataSerializer); + return new BeginConsensusMigrationForTableAndRange(targetProtocol, ranges, tables); + } + + public long serializedSize(Transformation t, Version version) + { + BeginConsensusMigrationForTableAndRange v = (BeginConsensusMigrationForTableAndRange) t; + return TypeSizes.sizeof(v.targetProtocol.toString()) + + ConsensusTableMigrationState.rangesSerializer.serializedSize(v.ranges, version) + + serializedCollectionSize(v.tables, version, TableId.metadataSerializer); + } + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/transformations/MaybeFinishConsensusMigrationForTableAndRange.java b/src/java/org/apache/cassandra/tcm/transformations/MaybeFinishConsensusMigrationForTableAndRange.java new file mode 100644 index 0000000000..64e6248c81 --- /dev/null +++ b/src/java/org/apache/cassandra/tcm/transformations/MaybeFinishConsensusMigrationForTableAndRange.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm.transformations; + +import java.io.IOException; +import java.util.List; +import javax.annotation.Nonnull; + +import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationRepairType; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.Transformation; +import org.apache.cassandra.tcm.sequences.LockedRanges; +import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static java.lang.String.format; +import static org.apache.cassandra.dht.Range.intersects; +import static org.apache.cassandra.dht.Range.normalize; +import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; + + +public class MaybeFinishConsensusMigrationForTableAndRange implements Transformation +{ + public static Serializer serializer = new Serializer(); + + @Nonnull + public final String keyspace; + + @Nonnull + public final String cf; + + @Nonnull + public final List> repairedRanges; + + @Nonnull + public final Epoch minEpoch; + + @Nonnull + public final ConsensusMigrationRepairType repairType; + + public MaybeFinishConsensusMigrationForTableAndRange(@Nonnull String keyspace, + @Nonnull String cf, + @Nonnull List> repairedRanges, + @Nonnull Epoch minEpoch, + @Nonnull ConsensusMigrationRepairType repairType) + { + checkNotNull(keyspace, "keyspace should not be null"); + checkNotNull(cf, "cf should not be null"); + checkNotNull(repairedRanges, "repairedRanges should not be null"); + checkArgument(!repairedRanges.isEmpty(), "repairedRanges should not be empty"); + checkNotNull(minEpoch, "minEpoch should not be null"); + checkArgument(minEpoch.isAfter(Epoch.EMPTY), "minEpoch should not be empty"); + checkNotNull(repairType, "repairType is null"); + checkArgument(repairType != ConsensusMigrationRepairType.ineligible, "Shouldn't attempt to finish migration with ineligible repair"); + this.keyspace = keyspace; + this.cf = cf; + this.repairedRanges = repairedRanges; + this.minEpoch = minEpoch; + this.repairType = repairType; + } + + public Kind kind() + { + return Kind.MAYBE_FINISH_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE; + } + + public Result execute(@Nonnull ClusterMetadata metadata) + { + System.out.println("Completed repair " + repairType + " ranges " + repairedRanges); + checkNotNull(metadata, "clusterMetadata should not be null"); + String ksAndCF = keyspace + "." + cf; + TableMetadata tbm = Schema.instance.getTableMetadata(keyspace, cf); + if (tbm == null) + return new Rejected(INVALID, format("Table %s is not currently performing consensus migration", ksAndCF)); + + ConsensusMigrationState consensusMigrationState = metadata.consensusMigrationState; + ConsensusTableMigrationState.TableMigrationState tms = consensusMigrationState.tableStates.get(tbm.id); + if (tms == null) + return new Rejected(INVALID, format("Table %s is not currently performing consensus migration", ksAndCF)); + + if (tms.targetProtocol == ConsensusMigrationTarget.accord && repairType != ConsensusMigrationRepairType.paxos) + return new Rejected(INVALID, format("Table %s is not currently performing consensus migration to Accord and the repair was a Paxos repair", ksAndCF)); + + if (tms.targetProtocol == ConsensusMigrationTarget.paxos && repairType != ConsensusMigrationRepairType.accord) + return new Rejected(INVALID, format("Table %s is not currently performing consensus migration to Paxos and the repair was an Accord repair", ksAndCF)); + + List> normalizedRepairedRanges = normalize(repairedRanges); + + // Bail out if repair doesn't actually intersect with any migrating ranges + if (!intersects(tms.migratingRanges, normalizedRepairedRanges)) + return new Rejected(INVALID, format("Table %s is migrating ranges %s, which doesn't include repaired ranges %s", ksAndCF, tms.migratingRanges, normalizedRepairedRanges)); + + TableMigrationState newTableMigrationState = tms.withRangesRepairedAtEpoch(normalizedRepairedRanges, minEpoch); + + return Transformation.success(metadata.transformer().with(ImmutableMap.of(newTableMigrationState.tableId, newTableMigrationState)), LockedRanges.AffectedRanges.EMPTY); + } + + static class Serializer implements AsymmetricMetadataSerializer + { + public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException + { + MaybeFinishConsensusMigrationForTableAndRange v = (MaybeFinishConsensusMigrationForTableAndRange)t; + out.writeUTF(v.keyspace); + out.writeUTF(v.cf); + ConsensusTableMigrationState.rangesSerializer.serialize(v.repairedRanges, out, version); + Epoch.serializer.serialize(v.minEpoch, out, version); + out.write(v.repairType.value); + } + + public MaybeFinishConsensusMigrationForTableAndRange deserialize(DataInputPlus in, Version version) throws IOException + { + String keyspace = in.readUTF(); + String cf = in.readUTF(); + List> repairedRanges = ConsensusTableMigrationState.rangesSerializer.deserialize(in, version); + Epoch minEpoch = Epoch.serializer.deserialize(in, version); + ConsensusMigrationRepairType repairType = ConsensusMigrationRepairType.fromValue(in.readByte()); + return new MaybeFinishConsensusMigrationForTableAndRange(keyspace, cf, repairedRanges, minEpoch, repairType); + } + + public long serializedSize(Transformation t, Version version) + { + MaybeFinishConsensusMigrationForTableAndRange v = (MaybeFinishConsensusMigrationForTableAndRange)t; + return TypeSizes.sizeof(v.keyspace) + + TypeSizes.sizeof(v.cf) + + ConsensusTableMigrationState.rangesSerializer.serializedSize(v.repairedRanges, version) + + Epoch.serializer.serializedSize(v.minEpoch) + + TypeSizes.sizeof(v.repairType.value); + } + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/transformations/SetConsensusMigrationTargetProtocol.java b/src/java/org/apache/cassandra/tcm/transformations/SetConsensusMigrationTargetProtocol.java new file mode 100644 index 0000000000..c0fd662d67 --- /dev/null +++ b/src/java/org/apache/cassandra/tcm/transformations/SetConsensusMigrationTargetProtocol.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm.transformations; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import javax.annotation.Nonnull; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadata.Transformer; +import org.apache.cassandra.tcm.Transformation; +import org.apache.cassandra.tcm.sequences.LockedRanges; +import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget.reset; +import static org.apache.cassandra.tcm.Transformation.Kind.SET_CONSENSUS_MIGRATION_TARGET_PROTOCOL; +import static org.apache.cassandra.utils.CollectionSerializers.deserializeList; +import static org.apache.cassandra.utils.CollectionSerializers.serializeCollection; +import static org.apache.cassandra.utils.CollectionSerializers.serializedCollectionSize; +import static org.apache.cassandra.utils.Collectors3.toImmutableMap; + +/* + * Narrowly focused on setting or changing the consensus migration protocol. The real use case + * is when a migration is already in progress or done and you want to change the target. + */ +public class SetConsensusMigrationTargetProtocol implements Transformation +{ + public static Serializer serializer = new Serializer(); + + @Nonnull + public final ConsensusMigrationTarget targetProtocol; + + @Nonnull + public final List tables; + + public SetConsensusMigrationTargetProtocol(@Nonnull ConsensusMigrationTarget targetProtocol, + @Nonnull List tables) + { + this.targetProtocol = targetProtocol; + this.tables = tables; + } + + @Override + public Kind kind() + { + return SET_CONSENSUS_MIGRATION_TARGET_PROTOCOL; + } + + @Override + public Result execute(ClusterMetadata metadata) + { + Map tableStates = metadata.consensusMigrationState.tableStates; + List columnFamilyStores = tables.stream().map(Schema.instance::getColumnFamilyStoreInstance).collect(toImmutableList()); + + Transformer transformer = metadata.transformer(); + + Map newStates; + + if (targetProtocol == reset) + { + newStates = tableStates.entrySet().stream().filter(entry -> !tables.contains(entry.getKey())).collect(toImmutableMap()); + } + else + { + newStates = columnFamilyStores + .stream() + .map(cfs -> + tableStates.containsKey(cfs.getTableId()) ? + tableStates.get(cfs.getTableId()).withMigrationTarget(targetProtocol) : + new TableMigrationState(cfs.keyspace.getName(), cfs.name, cfs.getTableId(), targetProtocol, ImmutableSet.of(), ImmutableMap.of())) + .collect(toImmutableMap(TableMigrationState::getTableId, Function.identity())); + } + + return Transformation.success(transformer.with(newStates, targetProtocol == reset ? false : true), LockedRanges.AffectedRanges.EMPTY); + } + + static class Serializer implements AsymmetricMetadataSerializer + { + public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException + { + SetConsensusMigrationTargetProtocol v = (SetConsensusMigrationTargetProtocol)t; + out.writeUTF(v.targetProtocol.toString()); + serializeCollection(v.tables, out, version, TableId.metadataSerializer); + } + + public SetConsensusMigrationTargetProtocol deserialize(DataInputPlus in, Version version) throws IOException + { + ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(in.readUTF()); + List tables = deserializeList(in, version, TableId.metadataSerializer); + return new SetConsensusMigrationTargetProtocol(targetProtocol, tables); + } + + public long serializedSize(Transformation t, Version version) + { + SetConsensusMigrationTargetProtocol v = (SetConsensusMigrationTargetProtocol) t; + return TypeSizes.sizeof(v.targetProtocol.toString()) + + serializedCollectionSize(v.tables, version, TableId.metadataSerializer); + } + } +} diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 747da83485..b90be2cf1b 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -41,7 +41,6 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; - import javax.annotation.Nullable; import javax.management.JMX; import javax.management.MBeanServerConnection; @@ -55,12 +54,20 @@ import javax.management.remote.JMXConnectorFactory; import javax.management.remote.JMXServiceURL; import javax.rmi.ssl.SslRMIClientSocketFactory; +import com.google.common.base.Function; +import com.google.common.base.Strings; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Uninterruptibles; + import org.apache.cassandra.audit.AuditLogManager; import org.apache.cassandra.audit.AuditLogManagerMBean; import org.apache.cassandra.audit.AuditLogOptions; import org.apache.cassandra.audit.AuditLogOptionsCompositeData; - -import com.google.common.collect.ImmutableMap; import org.apache.cassandra.auth.AuthCache; import org.apache.cassandra.auth.AuthCacheMBean; import org.apache.cassandra.auth.CIDRGroupsMappingManager; @@ -116,19 +123,9 @@ import org.apache.cassandra.service.StorageServiceMBean; import org.apache.cassandra.streaming.StreamManagerMBean; import org.apache.cassandra.streaming.StreamState; import org.apache.cassandra.streaming.management.StreamStateCompositeData; -import org.apache.cassandra.tools.nodetool.formatter.TableBuilder; - -import com.google.common.base.Function; -import com.google.common.base.Strings; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.Uninterruptibles; - import org.apache.cassandra.tcm.CMSOperations; import org.apache.cassandra.tools.nodetool.GetTimeout; +import org.apache.cassandra.tools.nodetool.formatter.TableBuilder; import org.apache.cassandra.utils.NativeLibrary; import static org.apache.cassandra.config.CassandraRelevantProperties.NODETOOL_JMX_NOTIFICATION_POLL_INTERVAL_SECONDS; @@ -523,7 +520,12 @@ public class NodeProbe implements AutoCloseable public void repairAsync(final PrintStream out, final String keyspace, Map options) throws IOException { - RepairRunner runner = new RepairRunner(out, ssProxy, keyspace, options); + blockOnAsyncRepair(out, keyspace, ssProxy.repairAsync(keyspace, options)); + } + + public void blockOnAsyncRepair(final PrintStream out, final String keyspace, Integer cmd) throws IOException + { + RepairRunner runner = new RepairRunner(out, ssProxy, keyspace, cmd); try { if (jmxc != null) @@ -1330,6 +1332,12 @@ public class NodeProbe implements AutoCloseable return ssProxy.getNonLocalStrategyKeyspaces(); } + + public List getAccordManagedKeyspace() + { + return ssProxy.getAccordManagedKeyspaces(); + } + public String getClusterName() { return ssProxy.getClusterName(); diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index 5b149acd9a..9f72c59eab 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -17,20 +17,7 @@ */ package org.apache.cassandra.tools; -import static com.google.common.base.Throwables.getStackTraceAsString; -import static com.google.common.collect.Iterables.toArray; -import static com.google.common.collect.Lists.newArrayList; -import static java.lang.Integer.parseInt; -import static java.lang.String.format; -import static org.apache.cassandra.io.util.File.WriteMode.APPEND; -import static org.apache.commons.lang3.ArrayUtils.EMPTY_STRING_ARRAY; -import static org.apache.commons.lang3.StringUtils.EMPTY; -import static org.apache.commons.lang3.StringUtils.isEmpty; -import static org.apache.commons.lang3.StringUtils.isNotEmpty; - import java.io.Console; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.io.util.FileWriter; import java.io.FileNotFoundException; import java.io.IOError; import java.io.IOException; @@ -44,16 +31,10 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Scanner; import java.util.SortedMap; - import javax.management.InstanceNotFoundException; import com.google.common.base.Joiner; import com.google.common.base.Throwables; - -import org.apache.cassandra.locator.EndpointSnitchInfoMBean; -import org.apache.cassandra.tools.nodetool.*; -import org.apache.cassandra.utils.FBUtilities; - import com.google.common.collect.Maps; import io.airlift.airline.Cli; @@ -67,6 +48,22 @@ import io.airlift.airline.ParseCommandUnrecognizedException; import io.airlift.airline.ParseOptionConversionException; import io.airlift.airline.ParseOptionMissingException; import io.airlift.airline.ParseOptionMissingValueException; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileWriter; +import org.apache.cassandra.locator.EndpointSnitchInfoMBean; +import org.apache.cassandra.tools.nodetool.*; +import org.apache.cassandra.utils.FBUtilities; + +import static com.google.common.base.Throwables.getStackTraceAsString; +import static com.google.common.collect.Iterables.toArray; +import static com.google.common.collect.Lists.newArrayList; +import static java.lang.Integer.parseInt; +import static java.lang.String.format; +import static org.apache.cassandra.io.util.File.WriteMode.APPEND; +import static org.apache.commons.lang3.ArrayUtils.EMPTY_STRING_ARRAY; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.apache.commons.lang3.StringUtils.isEmpty; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; public class NodeTool { @@ -274,6 +271,14 @@ public class NodeTool .withCommand(CMSAdmin.DumpDirectory.class) .withCommand(CMSAdmin.DumpLog.class); + builder.withGroup("consensus_admin") + .withDescription("List and mark ranges as migrating between consensus protocols") + .withDefaultCommand(CassHelp.class) + .withCommand(ConsensusMigrationAdmin.BeginMigration.class) + .withCommands(ConsensusMigrationAdmin.SetTargetProtocol.class) + .withCommands(ConsensusMigrationAdmin.ListCmd.class) + .withCommands(ConsensusMigrationAdmin.FinishMigration.class); + Cli parser = builder.build(); int status = 0; @@ -473,7 +478,7 @@ public class NodeTool protected enum KeyspaceSet { - ALL, NON_SYSTEM, NON_LOCAL_STRATEGY + ALL, NON_SYSTEM, NON_LOCAL_STRATEGY, ACCORD_MANAGED } protected List parseOptionalKeyspace(List cmdArgs, NodeProbe nodeProbe) @@ -492,6 +497,8 @@ public class NodeTool keyspaces.addAll(keyspaces = nodeProbe.getNonLocalStrategyKeyspaces()); else if (defaultKeyspaceSet == KeyspaceSet.NON_SYSTEM) keyspaces.addAll(keyspaces = nodeProbe.getNonSystemKeyspaces()); + else if (defaultKeyspaceSet == KeyspaceSet.ACCORD_MANAGED) + keyspaces.addAll(nodeProbe.getAccordManagedKeyspace()); else keyspaces.addAll(nodeProbe.getKeyspaces()); } diff --git a/src/java/org/apache/cassandra/tools/RepairRunner.java b/src/java/org/apache/cassandra/tools/RepairRunner.java index 01aa520185..3a4f77ccd2 100644 --- a/src/java/org/apache/cassandra/tools/RepairRunner.java +++ b/src/java/org/apache/cassandra/tools/RepairRunner.java @@ -21,24 +21,24 @@ import java.io.IOException; import java.io.PrintStream; import java.text.SimpleDateFormat; import java.util.List; -import java.util.Map; import org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus; import org.apache.cassandra.service.StorageServiceMBean; import org.apache.cassandra.utils.concurrent.Condition; - import org.apache.cassandra.utils.progress.ProgressEvent; import org.apache.cassandra.utils.progress.ProgressEventType; import org.apache.cassandra.utils.progress.jmx.JMXNotificationProgressListener; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus.FAILED; import static org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus.valueOf; import static org.apache.cassandra.tools.NodeProbe.JMX_NOTIFICATION_POLL_INTERVAL_SECONDS; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized; import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; -import static org.apache.cassandra.utils.progress.ProgressEventType.*; +import static org.apache.cassandra.utils.progress.ProgressEventType.COMPLETE; +import static org.apache.cassandra.utils.progress.ProgressEventType.ERROR; +import static org.apache.cassandra.utils.progress.ProgressEventType.PROGRESS; public class RepairRunner extends JMXNotificationProgressListener { @@ -47,23 +47,21 @@ public class RepairRunner extends JMXNotificationProgressListener private final PrintStream out; private final StorageServiceMBean ssProxy; private final String keyspace; - private final Map options; private final Condition condition = newOneTimeCondition(); - private int cmd; + private Integer cmd; private volatile Exception error; - public RepairRunner(PrintStream out, StorageServiceMBean ssProxy, String keyspace, Map options) + public RepairRunner(PrintStream out, StorageServiceMBean ssProxy, String keyspace, Integer cmd) { this.out = out; this.ssProxy = ssProxy; this.keyspace = keyspace; - this.options = options; + this.cmd = cmd; } public void run() throws Exception { - cmd = ssProxy.repairAsync(keyspace, options); if (cmd <= 0) { // repairAsync can only return 0 for replication factor 1. diff --git a/src/java/org/apache/cassandra/tools/nodetool/ConsensusMigrationAdmin.java b/src/java/org/apache/cassandra/tools/nodetool/ConsensusMigrationAdmin.java new file mode 100644 index 0000000000..cd24bf91fa --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/ConsensusMigrationAdmin.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools.nodetool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import io.airlift.airline.Arguments; +import io.airlift.airline.Command; +import io.airlift.airline.Option; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Collections.emptyList; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonList; + +/** + * For managing migration from one consensus protocol to another. + * + * Mark ranges as migrating, and list the migrating ranges. + */ +public abstract class ConsensusMigrationAdmin extends NodeTool.NodeToolCmd +{ + @Command(name = "list", description = "List migrating tables and ranges") + public static class ListCmd extends ConsensusMigrationAdmin + { + @Arguments(usage = "[ ...]", description = "The keyspace followed by one or many tables") + private List schemaArgs = new ArrayList<>(); + + @Option(title = "format", name = {"-f", "--format"}, description = "Output format, YAML and JSON are the only supported formats, default YAML, prefix with `minified-` to turn off pretty printing") + private String format = "yaml"; + + protected void execute(NodeProbe probe) + { + Set keyspaceNames = schemaArgs.size() > 0 ? singleton(schemaArgs.get(0)) : null; + Set tableNames = schemaArgs.size() > 1 ? new HashSet<>(schemaArgs.subList(1, schemaArgs.size())) : null; + String output = probe.getStorageService().listConsensusMigrations(keyspaceNames, tableNames, format); + probe.output().out.println(output); + } + } + + @Command(name = "begin-migration", description = "Mark the range as migrating for the specified token range and tables") + public static class BeginMigration extends ConsensusMigrationAdmin + { + @Option(title = "start_token", name = {"-st", "--start-token"}, description = "Use -st to specify a token at which the repair range starts") + private String startToken = null; + + @Option(title = "end_token", name = {"-et", "--end-token"}, description = "Use -et to specify a token at which repair range ends") + private String endToken = null; + + @Option(title = "target_protocol", name = {"-tp", "--target-protocol"}, description = "Use -tp to specify what consensus protocol should be migrated to", required=true) + private String targetProtocol = null; + + @Arguments(usage = "[ ...]", description = "The keyspace followed by one or many tables") + private List schemaArgs = new ArrayList<>(); + + protected void execute(NodeProbe probe) + { + checkArgument((endToken != null && startToken != null) || (endToken == null && startToken == null), "Must specify start and end token together"); + String maybeRangesStr = startToken != null ? startToken + ":" + endToken : null; + List keyspaceNames = parseOptionalKeyspace(schemaArgs, probe, KeyspaceSet.ACCORD_MANAGED); + List maybeTableNames = schemaArgs.size() > 1 ? schemaArgs.subList(1, schemaArgs.size()) : null; + probe.getStorageService().migrateConsensusProtocol(targetProtocol, keyspaceNames, maybeTableNames, maybeRangesStr); + probe.output().out.println("Marked requested ranges as migrating. Repair needs to be run in order to complete the migration"); + } + } + + @Command(name = "finish-migration", description = "Complete the migration for a range that has already begun migration") + public static class FinishMigration extends ConsensusMigrationAdmin + { + @Option(title = "start_token", name = {"-st", "--start-token"}, description = "Use -st to specify a token at which the repair range starts (exclusive)") + private String startToken = null; + + @Option(title = "end_token", name = {"-et", "--end-token"}, description = "Use -et to specify a token at which repair range ends (inclusive)") + private String endToken = null; + + @Arguments(usage = "[ ...]", description = "The keyspace followed by one or many tables") + private List schemaArgs = new ArrayList<>(); + + protected void execute(NodeProbe probe) + { + checkArgument((endToken != null) == (startToken != null), "Start and end token must be specified together"); + String maybeRangesStr = startToken != null ? startToken + ":" + endToken : null; + List keyspaceNames = parseOptionalKeyspace(schemaArgs, probe, KeyspaceSet.ACCORD_MANAGED); + List maybeTableNames = schemaArgs.size() > 1 ? schemaArgs.subList(1, schemaArgs.size()) : null; + for (String keyspace : keyspaceNames) + { + List commands = probe.getStorageService().finishConsensusMigration(keyspace, maybeTableNames, maybeRangesStr); + for (Integer command : commands) + { + try + { + probe.blockOnAsyncRepair(probe.output().out, keyspace, command); + } + catch (IOException e) + { + throw new RuntimeException("Error occurred attempting to finish migration for keyspace " + keyspace + " tables " + maybeTableNames + " and ranges " + maybeRangesStr, e); + } + } + } + probe.output().out.printf("Finished consensus migration range (%s) of keyspaces %s and tables %s%n", maybeRangesStr, keyspaceNames, maybeTableNames); + } + } + + @Command(name = "set-target-protocol", description = "Set or change the target consensus protocol of the specified tables. If a migration is in progress then the migration will be reversed with migrating ranges still migrating, unmigrated ranges marked as migrated, and migrating ranges will need migration. Be aware that if no migration was in progress for a table it will immediately cause the table to run on the target protocol because the ranges requiring migration are derived from the migrated ranges that don't exist.") + public static class SetTargetProtocol extends ConsensusMigrationAdmin + { + @Arguments(usage = "[ ...]", description = "The keyspace followed by one or many tables") + private List schemaArgs = new ArrayList<>(); + + @Option(title = "target_protocol", name = {"-tp", "--target-protocol"}, description = "Use -tp to specify what consensus protocol should be migrated to", required=true) + private String targetProtocol = null; + + @Option(title = "force_completion", name = {"-f", "--force-completion"}, description = "Forces migration state for all ranges of the specified table regardless of whether migration completed successfully or not. Should only be used if table is empty or has had no writes since last repair.") + private boolean forceCompletion = false; + + protected void execute(NodeProbe probe) + { + checkArgument(schemaArgs.size() >= 2, "Must specify a keyspace and at least one table"); + List keyspaceNames = schemaArgs.size() > 0 ? singletonList(schemaArgs.get(0)) : emptyList(); + List maybeTableNames = schemaArgs.size() > 1 ? schemaArgs.subList(1, schemaArgs.size()) : null; + probe.getStorageService().setConsensusMigrationTargetProtocol(targetProtocol, keyspaceNames, maybeTableNames); + } + } +} diff --git a/src/java/org/apache/cassandra/tools/nodetool/Repair.java b/src/java/org/apache/cassandra/tools/nodetool/Repair.java index c66992acc9..8d5b0607d4 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/Repair.java +++ b/src/java/org/apache/cassandra/tools/nodetool/Repair.java @@ -32,14 +32,14 @@ import java.util.Set; import java.util.function.Supplier; import com.google.common.collect.Sets; +import org.apache.commons.lang3.StringUtils; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.repair.RepairParallelism; import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool.NodeToolCmd; -import org.apache.commons.lang3.StringUtils; @Command(name = "repair", description = "Repair one or more tables") public class Repair extends NodeToolCmd @@ -134,7 +134,8 @@ public class Repair extends NodeToolCmd @Override public void execute(NodeProbe probe) { - List keyspaces = parseOptionalKeyspace(args, probe, KeyspaceSet.NON_LOCAL_STRATEGY); + KeyspaceSet keyspaceSet = KeyspaceSet.NON_LOCAL_STRATEGY; + List keyspaces = parseOptionalKeyspace(args, probe, keyspaceSet); String[] cfnames = parseOptionalTables(args); if (primaryRange && (!specificDataCenters.isEmpty() || !specificHosts.isEmpty())) diff --git a/src/java/org/apache/cassandra/utils/AbstractBiMultiValMap.java b/src/java/org/apache/cassandra/utils/AbstractBiMultiValMap.java new file mode 100644 index 0000000000..e29427d386 --- /dev/null +++ b/src/java/org/apache/cassandra/utils/AbstractBiMultiValMap.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.util.Collection; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; + +public abstract class AbstractBiMultiValMap implements Map +{ + protected abstract Map forwardDelegate(); + protected abstract Multimap reverseDelegate(); + + public Multimap inverse() + { + return Multimaps.unmodifiableMultimap(reverseDelegate()); + } + + public void clear() + { + forwardDelegate().clear(); + reverseDelegate().clear(); + } + + public boolean containsKey(Object key) + { + return forwardDelegate().containsKey(key); + } + + public boolean containsValue(Object value) + { + return reverseDelegate().containsKey(value); + } + + public Set> entrySet() + { + return forwardDelegate().entrySet(); + } + + public V get(Object key) + { + return forwardDelegate().get(key); + } + + public boolean isEmpty() + { + return forwardDelegate().isEmpty(); + } + + public Set keySet() + { + return forwardDelegate().keySet(); + } + + public V put(K key, V value) + { + V oldVal = forwardDelegate().put(key, value); + if (oldVal != null) + reverseDelegate().remove(oldVal, key); + reverseDelegate().put(value, key); + return oldVal; + } + + public void putAll(Map m) + { + for (Map.Entry entry : m.entrySet()) + put(entry.getKey(), entry.getValue()); + } + + public V remove(Object key) + { + V oldVal = forwardDelegate().remove(key); + reverseDelegate().remove(oldVal, key); + return oldVal; + } + + public Collection removeValue(V value) + { + Collection keys = reverseDelegate().removeAll(value); + for (K key : keys) + forwardDelegate().remove(key); + return keys; + } + + public int size() + { + return forwardDelegate().size(); + } + + public Collection values() + { + return reverseDelegate().keys(); + } + + public Collection valueSet() + { + return reverseDelegate().keySet(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (!(o instanceof AbstractBiMultiValMap)) return false; + AbstractBiMultiValMap that = (AbstractBiMultiValMap) o; + return forwardDelegate().equals(that.forwardDelegate()) && reverseDelegate().equals(that.reverseDelegate()); + } + + @Override + public int hashCode() + { + return Objects.hash(forwardDelegate(), reverseDelegate()); + } +} diff --git a/src/java/org/apache/cassandra/utils/BiMultiValMap.java b/src/java/org/apache/cassandra/utils/BiMultiValMap.java index f439c5c496..2859e6964b 100644 --- a/src/java/org/apache/cassandra/utils/BiMultiValMap.java +++ b/src/java/org/apache/cassandra/utils/BiMultiValMap.java @@ -17,15 +17,11 @@ */ package org.apache.cassandra.utils; -import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.Objects; -import java.util.Set; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; /** * @@ -35,7 +31,7 @@ import com.google.common.collect.Multimaps; * @param * @param */ -public class BiMultiValMap implements Map +public class BiMultiValMap extends AbstractBiMultiValMap { protected final Map forwardMap; protected final Multimap reverseMap; @@ -59,104 +55,15 @@ public class BiMultiValMap implements Map reverseMap.putAll(map.inverse()); } - public Multimap inverse() + @Override + protected Map forwardDelegate() { - return Multimaps.unmodifiableMultimap(reverseMap); - } - - public void clear() - { - forwardMap.clear(); - reverseMap.clear(); - } - - public boolean containsKey(Object key) - { - return forwardMap.containsKey(key); - } - - public boolean containsValue(Object value) - { - return reverseMap.containsKey(value); - } - - public Set> entrySet() - { - return forwardMap.entrySet(); - } - - public V get(Object key) - { - return forwardMap.get(key); - } - - public boolean isEmpty() - { - return forwardMap.isEmpty(); - } - - public Set keySet() - { - return forwardMap.keySet(); - } - - public V put(K key, V value) - { - V oldVal = forwardMap.put(key, value); - if (oldVal != null) - reverseMap.remove(oldVal, key); - reverseMap.put(value, key); - return oldVal; - } - - public void putAll(Map m) - { - for (Map.Entry entry : m.entrySet()) - put(entry.getKey(), entry.getValue()); - } - - public V remove(Object key) - { - V oldVal = forwardMap.remove(key); - reverseMap.remove(oldVal, key); - return oldVal; - } - - public Collection removeValue(V value) - { - Collection keys = reverseMap.removeAll(value); - for (K key : keys) - forwardMap.remove(key); - return keys; - } - - public int size() - { - return forwardMap.size(); - } - - public Collection values() - { - return reverseMap.keys(); - } - - public Collection valueSet() - { - return reverseMap.keySet(); + return forwardMap; } @Override - public boolean equals(Object o) + protected Multimap reverseDelegate() { - if (this == o) return true; - if (!(o instanceof BiMultiValMap)) return false; - BiMultiValMap that = (BiMultiValMap) o; - return forwardMap.equals(that.forwardMap) && reverseMap.equals(that.reverseMap); - } - - @Override - public int hashCode() - { - return Objects.hash(forwardMap, reverseMap); + return reverseMap; } } diff --git a/src/java/org/apache/cassandra/utils/CollectionSerializers.java b/src/java/org/apache/cassandra/utils/CollectionSerializers.java index 0cdd5685ab..1fcb7cc2f3 100644 --- a/src/java/org/apache/cassandra/utils/CollectionSerializers.java +++ b/src/java/org/apache/cassandra/utils/CollectionSerializers.java @@ -26,15 +26,19 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.IntFunction; +import javax.annotation.Nonnull; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.IPartitionerDependentSerializer; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.tcm.serialization.MetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; -import static com.google.common.primitives.Ints.checkedCast; import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; public class CollectionSerializers @@ -46,6 +50,20 @@ public class CollectionSerializers valueSerializer.serialize(value, out, version); } + public static void serializeCollection(Collection values, DataOutputPlus out, Version version, MetadataSerializer valueSerializer) throws IOException + { + out.writeUnsignedVInt32(values.size()); + for (V value : values) + valueSerializer.serialize(value, out, version); + } + + public static void serializeCollection(Collection values, DataOutputPlus out, int version, IPartitionerDependentSerializer valueSerializer) throws IOException + { + out.writeUnsignedVInt32(values.size()); + for (V value : values) + valueSerializer.serialize(value, out, version); + } + public static > void serializeList(L values, DataOutputPlus out, int version, IVersionedSerializer valueSerializer) throws IOException { int size = values.size(); @@ -64,19 +82,59 @@ public class CollectionSerializers } } + public static void serializeMap(Map map, DataOutputPlus out, Version version, MetadataSerializer keySerializer, MetadataSerializer valueSerializer) throws IOException + { + out.writeUnsignedVInt32(map.size()); + for (Map.Entry e : map.entrySet()) + { + keySerializer.serialize(e.getKey(), out, version); + valueSerializer.serialize(e.getValue(), out, version); + } + } + + public static void serializeMap(Map map, DataOutputPlus out, int version, IVersionedSerializer keySerializer, IPartitionerDependentSerializer valueSerializer) throws IOException + { + out.writeUnsignedVInt32(map.size()); + for (Map.Entry e : map.entrySet()) + { + keySerializer.serialize(e.getKey(), out, version); + valueSerializer.serialize(e.getValue(), out, version); + } + } + public static List deserializeList(DataInputPlus in, int version, IVersionedSerializer serializer) throws IOException { return deserializeCollection(in, version, serializer, newArrayList()); } + public static List deserializeList(DataInputPlus in, Version version, MetadataSerializer serializer) throws IOException + { + return deserializeCollection(in, version, serializer, newArrayList()); + } + + public static List deserializeList(DataInputPlus in, IPartitioner partitioner, int version, IPartitionerDependentSerializer serializer) throws IOException + { + return deserializeCollection(in, partitioner, version, serializer, newArrayList()); + } + public static Set deserializeSet(DataInputPlus in, int version, IVersionedSerializer serializer) throws IOException { return deserializeCollection(in, version, serializer, newHashSet()); } + public static Set deserializeSet(DataInputPlus in, IPartitioner partitioner, int version, IPartitionerDependentSerializer serializer) throws IOException + { + return deserializeCollection(in, partitioner, version, serializer, newHashSet()); + } + + public static Set deserializeSet(DataInputPlus in, Version version, MetadataSerializer serializer) throws IOException + { + return deserializeCollection(in, version, serializer, newHashSet()); + } + public static > M deserializeMap(DataInputPlus in, int version, IVersionedSerializer keySerializer, IVersionedSerializer valueSerializer, IntFunction factory) throws IOException { - int size = checkedCast(in.readUnsignedVInt32()); + int size = in.readUnsignedVInt32(); M result = factory.apply(size); while (size-- > 0) { @@ -87,6 +145,32 @@ public class CollectionSerializers return result; } + public static Map deserializeMap(DataInputPlus in, Version version, MetadataSerializer keySerializer, MetadataSerializer valueSerializer, IntFunction> factory) throws IOException + { + int size = in.readUnsignedVInt32(); + Map result = factory.apply(size); + while (size-- > 0) + { + K key = keySerializer.deserialize(in, version); + V value = valueSerializer.deserialize(in, version); + result.put(key, value); + } + return result; + } + + public static Map deserializeMap(DataInputPlus in, IPartitioner partitioner, int version, IVersionedSerializer keySerializer, IPartitionerDependentSerializer valueSerializer, IntFunction> factory) throws IOException + { + int size = in.readUnsignedVInt32(); + Map result = factory.apply(size); + while (size-- > 0) + { + K key = keySerializer.deserialize(in, version); + V value = valueSerializer.deserialize(in, partitioner, version); + result.put(key, value); + } + return result; + } + public static Map deserializeMap(DataInputPlus in, int version, IVersionedSerializer keySerializer, IVersionedSerializer valueSerializer) throws IOException { return deserializeMap(in, version, keySerializer, valueSerializer, newHashMap()); @@ -100,6 +184,22 @@ public class CollectionSerializers return size; } + public static long serializedCollectionSize(Collection values, Version version, MetadataSerializer valueSerializer) + { + long size = sizeofUnsignedVInt(values.size()); + for (V value : values) + size += valueSerializer.serializedSize(value, version); + return size; + } + + public static long serializedCollectionSize(Collection values, int version, IPartitionerDependentSerializer valueSerializer) + { + long size = sizeofUnsignedVInt(values.size()); + for (V value : values) + size += valueSerializer.serializedSize(value, version); + return size; + } + public static > long serializedListSize(L values, int version, IVersionedSerializer valueSerializer) { int items = values.size(); @@ -118,6 +218,24 @@ public class CollectionSerializers return size; } + public static long serializedMapSize(Map map, Version version, MetadataSerializer keySerializer, MetadataSerializer valueSerializer) + { + long size = sizeofUnsignedVInt(map.size()); + for (Map.Entry e : map.entrySet()) + size += keySerializer.serializedSize(e.getKey(), version) + + valueSerializer.serializedSize(e.getValue(), version); + return size; + } + + public static long serializedMapSize(Map map, int version, IVersionedSerializer keySerializer, IPartitionerDependentSerializer valueSerializer) + { + long size = sizeofUnsignedVInt(map.size()); + for (Map.Entry e : map.entrySet()) + size += keySerializer.serializedSize(e.getKey(), version) + + valueSerializer.serializedSize(e.getValue(), version); + return size; + } + public static IntFunction> newHashSet() { return i -> i == 0 ? Collections.emptySet() : Sets.newHashSetWithExpectedSize(i); @@ -135,7 +253,7 @@ public class CollectionSerializers public static int readCollectionSize(DataInputPlus in, int version) throws IOException { - return checkedCast(in.readUnsignedVInt()); + return in.readUnsignedVInt32(); } /* @@ -144,7 +262,7 @@ public class CollectionSerializers */ private static > C deserializeCollection(DataInputPlus in, int version, IVersionedSerializer serializer, IntFunction factory) throws IOException { - int size = checkedCast(in.readUnsignedVInt32()); + int size = in.readUnsignedVInt32(); C result = factory.apply(size); while (size-- > 0) result.add(serializer.deserialize(in, version)); @@ -174,4 +292,70 @@ public class CollectionSerializers } }; } + + private static > C deserializeCollection(DataInputPlus in, IPartitioner partitioner, int version, IPartitionerDependentSerializer serializer, IntFunction factory) throws IOException + { + int size = in.readUnsignedVInt32(); + C result = factory.apply(size); + while (size-- > 0) + result.add(serializer.deserialize(in, partitioner, version)); + return result; + } + + private static > C deserializeCollection(DataInputPlus in, Version version, MetadataSerializer serializer, IntFunction factory) throws IOException + { + int size = in.readUnsignedVInt32(); + C result = factory.apply(size); + while (size-- > 0) + result.add(serializer.deserialize(in, version)); + return result; + } + + public static IPartitionerDependentSerializer> newCollectionSerializer(@Nonnull final IPartitionerDependentSerializer serializer) + { + return new IPartitionerDependentSerializer>() + { + @Override + public void serialize(Collection t, DataOutputPlus out, int version) throws IOException + { + serializeCollection(t, out, version, serializer); + } + + @Override + public Collection deserialize(DataInputPlus in, IPartitioner p, int version) throws IOException + { + return deserializeCollection(in, p, version, serializer, newArrayList()); + } + + @Override + public long serializedSize(Collection t, int version) + { + return serializedCollectionSize(t, version, serializer); + } + }; + } + + public static MetadataSerializer> newListSerializer(@Nonnull final MetadataSerializer serializer) + { + return new MetadataSerializer>() + { + @Override + public void serialize(List t, DataOutputPlus out, Version version) throws IOException + { + serializeCollection(t, out, version, serializer); + } + + @Override + public List deserialize(DataInputPlus in, Version version) throws IOException + { + return deserializeList(in, version, serializer); + } + + @Override + public long serializedSize(List t, Version version) + { + return serializedCollectionSize(t, version, serializer); + } + }; + } } diff --git a/src/java/org/apache/cassandra/utils/PojoToString.java b/src/java/org/apache/cassandra/utils/PojoToString.java new file mode 100644 index 0000000000..4e6c8a95ab --- /dev/null +++ b/src/java/org/apache/cassandra/utils/PojoToString.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.DumperOptions.FlowStyle; +import org.yaml.snakeyaml.Yaml; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.cassandra.utils.JsonUtils.JSON_OBJECT_MAPPER; + +/** + * Helper to format POJOs that are easy to convert (primitives, nonnull, and built in collections) + * in various human + machine readable formats. Useful for JMX and nodetool. + */ +public class PojoToString +{ + public static final Integer VERSION_50 = 0; + + public static final Integer CURRENT_VERSION = VERSION_50; + + enum Format + { + YAML, + MINIFIED_YAML(true), + JSON, + MINIFIED_JSON(true); + + boolean minified; + + Format() + { + this(false); + } + + Format(boolean minified) + { + this.minified = minified; + } + + public boolean isYaml() + { + return this == YAML || this == MINIFIED_YAML; + } + + public static Format fromString(String formatString) + { + formatString = LocalizeString.toUpperCaseLocalized(formatString); + switch (formatString) + { + case "YAML": + return YAML; + case "MINIFIED-YAML": + return MINIFIED_YAML; + case "JSON": + return JSON; + case "MINIFIED-JSON": + return MINIFIED_JSON; + default: throw new IllegalArgumentException("Unsupported format " + formatString + + " supported formats are YAML, MINIFIED-YAML, JSON, MINIFIED-JSON"); + } + } + } + private static final Set> ALLOWED_PRIMITIVES = ImmutableSet.of( + String.class, + Double.class, + Float.class, + Long.class, + Integer.class, + Short.class, + Byte.class + ); + + private static final List> ALLOWED_COLLECTIONS = ImmutableList.of( + List.class, + Set.class + ); + + /** + * Helper to convert POJOs from a restricted set (primitive Java types and collections) to a human/machine readable + * format that is specified by the format parameter. + * + * This doesn't enforce what objects are serialized so you can get error or messy output if you try and serialize + * things that aren't primitive or collections. + * + * The map must contain a 'version' key set to CURRENT_VERSION + * @param map Map POJO that must be restricted to easily representable types (map, set , list, primitives), and contains the 'version' key set to CURRENT_VERSION + * @param formatString Human/machine readable format name, can be YAML or JSON, prefix with MINIFIED- to get a minified version + * @return The map formatted in the requested format + * @throws IllegalArgumentException If the 'version' key is not present and set to CURRENT_VERSION + */ + public static String pojoMapToString(Map map, String formatString) + { + checkArgument(CURRENT_VERSION.equals(map.get("version"))); + return pojoToString(map, formatString); + } + + private static String pojoToString(Object obj, String formatString) + { + validateAllowedTypes(obj); + Format format = Format.fromString(formatString); + if (format.isYaml()) + { + DumperOptions dumperOptions = new DumperOptions(); + if (format.minified) + { + dumperOptions.setDefaultFlowStyle(FlowStyle.FLOW); + dumperOptions.setIndent(1); + dumperOptions.setWidth(Integer.MAX_VALUE); + dumperOptions.setSplitLines(false); + } + // TODO How do you get snake yaml to produce minified output? + return new Yaml(dumperOptions).dump(obj); + } + else + { + try + { + if (format.minified) + return JSON_OBJECT_MAPPER.writeValueAsString(obj); + else + return JSON_OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(obj); + } + catch (JsonProcessingException e) + { + throw new RuntimeException(e); + } + } + } + + private static void validateAllowedTypes(Object o) + { + if (o == null) + throw new NullPointerException("Null objects are unsupported"); + if (o instanceof Map) + { + for (Map.Entry entry : ((Map)o).entrySet()) + { + Object key = entry.getKey(); + if (!(key instanceof String | key instanceof Long | key instanceof Integer)) + throw new IllegalArgumentException("Map has entry with key " + entry.getKey() + " of " + + key.getClass() + " which is unsupported, only String is supported for map keys"); + validateAllowedTypes(entry.getValue()); + } + } + else if (o instanceof Collection) + { + if (!(o instanceof Set | o instanceof List)) + throw new IllegalArgumentException("Collection " + o + " with " + o.getClass() + " is not in allow list " + ALLOWED_COLLECTIONS); + for (Object element : ((Collection)o)) + validateAllowedTypes(element); + } + else if (!ALLOWED_PRIMITIVES.contains(o.getClass())) + throw new IllegalArgumentException("Scalar " + o + " with " + o.getClass() + " is not in allow list " + ALLOWED_PRIMITIVES); + + } +} diff --git a/src/java/org/apache/cassandra/utils/SortedBiMultiValMap.java b/src/java/org/apache/cassandra/utils/SortedBiMultiValMap.java index 44ac0a01c0..1b0d8541ee 100644 --- a/src/java/org/apache/cassandra/utils/SortedBiMultiValMap.java +++ b/src/java/org/apache/cassandra/utils/SortedBiMultiValMap.java @@ -18,17 +18,21 @@ package org.apache.cassandra.utils; import java.util.Collection; -import java.util.SortedMap; +import java.util.NavigableMap; import java.util.TreeMap; import com.google.common.collect.SortedSetMultimap; import com.google.common.collect.TreeMultimap; -public class SortedBiMultiValMap extends BiMultiValMap +public class SortedBiMultiValMap extends AbstractBiMultiValMap { - protected SortedBiMultiValMap(SortedMap forwardMap, SortedSetMultimap reverseMap) + protected final NavigableMap forwardMap; + protected final SortedSetMultimap reverseMap; + + protected SortedBiMultiValMap(NavigableMap forwardMap, SortedSetMultimap reverseMap) { - super(forwardMap, reverseMap); + this.forwardMap = forwardMap; + this.reverseMap = reverseMap; } public static , V extends Comparable> SortedBiMultiValMap create() @@ -36,10 +40,10 @@ public class SortedBiMultiValMap extends BiMultiValMap return new SortedBiMultiValMap(new TreeMap(), TreeMultimap.create()); } - public static , V extends Comparable> SortedBiMultiValMap create(BiMultiValMap map) + public static , V extends Comparable, M extends AbstractBiMultiValMap> SortedBiMultiValMap create(M map) { SortedBiMultiValMap newMap = SortedBiMultiValMap.create(); - newMap.forwardMap.putAll(map.forwardMap); + newMap.forwardMap.putAll(map.forwardDelegate()); // Put each individual TreeSet instead of Multimap#putAll(Multimap) to get linear complexity // See CASSANDRA-14660 for (Entry> entry : map.inverse().asMap().entrySet()) @@ -47,4 +51,15 @@ public class SortedBiMultiValMap extends BiMultiValMap return newMap; } + @Override + protected NavigableMap forwardDelegate() + { + return forwardMap; + } + + @Override + protected SortedSetMultimap reverseDelegate() + { + return reverseMap; + } } diff --git a/src/java/org/apache/cassandra/utils/TimeUUID.java b/src/java/org/apache/cassandra/utils/TimeUUID.java index b7930ed582..49c31478e1 100644 --- a/src/java/org/apache/cassandra/utils/TimeUUID.java +++ b/src/java/org/apache/cassandra/utils/TimeUUID.java @@ -242,6 +242,11 @@ public class TimeUUID implements Serializable, Comparable return unixMicros * 10 - (UUID_EPOCH_UNIX_MILLIS * 10000); } + public static long unixMicrosToMsb(long unixMicros) + { + return TimeUUID.rawTimestampToMsb(TimeUUID.unixMicrosToRawTimestamp(unixMicros)); + } + public static long msbToRawTimestamp(long msb) { assert (UUID_VERSION_BITS_IN_MSB & msb) == TIMESTAMP_UUID_VERSION_IN_MSB; diff --git a/test/data/serialization/5.0/service.SyncComplete.bin b/test/data/serialization/5.0/service.SyncComplete.bin index 7c775cef66..d8465dbf61 100644 Binary files a/test/data/serialization/5.0/service.SyncComplete.bin and b/test/data/serialization/5.0/service.SyncComplete.bin differ diff --git a/test/data/serialization/5.0/service.ValidationComplete.bin b/test/data/serialization/5.0/service.ValidationComplete.bin index f0c502458a..c595fbcc0f 100644 Binary files a/test/data/serialization/5.0/service.ValidationComplete.bin and b/test/data/serialization/5.0/service.ValidationComplete.bin differ diff --git a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java index 7f98f9003d..a36c26c4f1 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java @@ -1183,7 +1183,7 @@ public abstract class AbstractCluster implements ICluster createOptimizedSyncCount.get()) > 0); } } @@ -104,7 +107,7 @@ public class OptimiseStreamsRepairTest extends TestBaseImpl { public static void install(ClassLoader cl, int id) { - new ByteBuddy().rebase(RepairJob.class) + new ByteBuddy().rebase(CassandraRepairJob.class) .method(named("createOptimisedSyncingSyncTasks").and(takesArguments(1))) .intercept(MethodDelegation.to(BBHelper.class)) .make() @@ -114,6 +117,7 @@ public class OptimiseStreamsRepairTest extends TestBaseImpl public static List createOptimisedSyncingSyncTasks(List trees, @SuperCall Callable> zuperCall) { + createOptimizedSyncCount.incrementAndGet(); List tasks = null; try { diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java index 6470a4d0c5..6364401de5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.google.common.util.concurrent.FutureCallback; import org.junit.Assert; @@ -47,10 +49,12 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.ICoordinator; import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IMessageFilters.Filter; import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.reads.repair.BlockingReadRepair; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; import org.apache.cassandra.utils.concurrent.Condition; @@ -71,6 +75,7 @@ import static org.apache.cassandra.net.Verb.READ_REPAIR_RSP; import static org.apache.cassandra.net.Verb.READ_REQ; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class ReadRepairTest extends TestBaseImpl @@ -81,7 +86,8 @@ public class ReadRepairTest extends TestBaseImpl @Test public void testBlockingReadRepair() throws Throwable { - testReadRepair(ReadRepairStrategy.BLOCKING); + testReadRepair(ReadRepairStrategy.BLOCKING, false); + testReadRepair(ReadRepairStrategy.BLOCKING, true); } /** * @@ -95,8 +101,14 @@ public class ReadRepairTest extends TestBaseImpl private void testReadRepair(ReadRepairStrategy strategy) throws Throwable { - try (Cluster cluster = init(Cluster.create(3))) + testReadRepair(strategy, false); + } + + private void testReadRepair(ReadRepairStrategy strategy, boolean brrThroughAccord) throws Throwable + { + try (Cluster cluster = init(Cluster.create(3, config -> config.set("non_serial_write_strategy", brrThroughAccord ? "migration" : "normal")))) { + cluster.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE)); cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c int, v int, PRIMARY KEY (k, c)) " + String.format("WITH read_repair='%s'", strategy))); @@ -111,8 +123,11 @@ public class ReadRepairTest extends TestBaseImpl // verify that the third node doesn't have the row assertRows(cluster.get(3).executeInternal(selectQuery)); - // read with CL=QUORUM to trigger read repair + // read with CL=QUORUM to trigger read repair, force 3 to be involved in the read so that read repair + // will occur + Filter blockReadFromOne = cluster.filters().inbound().from(3).to(1).verbs(READ_REQ.id).drop(); assertRows(cluster.coordinator(3).execute(selectQuery, QUORUM), row); + blockReadFromOne.off(); // verify whether the coordinator has the repaired row depending on the read repair strategy if (strategy == ReadRepairStrategy.NONE) @@ -143,13 +158,13 @@ public class ReadRepairTest extends TestBaseImpl catch (Exception ex) { // the containing exception class was loaded by another class loader. Comparing the message as a workaround to assert the exception - Assert.assertTrue(ex.getClass().toString().contains("ReadTimeoutException")); + assertTrue(ex.getClass().toString().contains("ReadTimeoutException")); long actualTimeTaken = currentTimeMillis() - start; long magicDelayAmount = 100L; // it might not be the best way to check if the time taken is around the timeout value. // Due to the delays, the actual time taken from client perspective is slighly more than the timeout value - Assert.assertTrue(actualTimeTaken > reducedReadTimeout); + assertTrue(actualTimeTaken > reducedReadTimeout); // But it should not exceed too much - Assert.assertTrue(actualTimeTaken < reducedReadTimeout + magicDelayAmount); + assertTrue(actualTimeTaken < reducedReadTimeout + magicDelayAmount); assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"), row(1, 1, 1)); // the partition happened when the repaired node sending back ack. The mutation should be in fact applied. } @@ -423,8 +438,11 @@ public class ReadRepairTest extends TestBaseImpl catch (ExecutionException e) { Throwable cause = e.getCause(); - Assert.assertTrue("Expected a different error message, but got " + cause.getMessage(), - cause.getMessage().contains("INVALID_ROUTING from /127.0.0.2:7012")); + Matcher matcher = Pattern.compile("Operation failed - received (\\d+) responses and 1 failures: INVALID_ROUTING from /127.0.0.2:7012").matcher(cause.getMessage()); + assertTrue("Expected a different error message, but got " + cause.getMessage(), + matcher.matches()); + int responses = Integer.valueOf(matcher.group(1)); + assertTrue(responses >= 1 && responses <= 3); } catch (InterruptedException e) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadSpeculationTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadSpeculationTest.java index 445315f343..f6f5e2a812 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadSpeculationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadSpeculationTest.java @@ -25,6 +25,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.junit.Assert; import org.junit.Test; @@ -86,7 +87,8 @@ public class ReadSpeculationTest extends TestBaseImpl ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(TABLE); DecoratedKey dk = cfs.decorateKey(bytes(PK_VALUE)); ReplicaPlan.ForTokenRead plan = ReplicaPlans.forRead(keyspace, dk.getToken(), null, - QUORUM, cfs.metadata().params.speculativeRetry); + QUORUM, cfs.metadata().params.speculativeRetry, + ReadCoordinator.DEFAULT); return plan.contacts().endpointList().stream().map(InetSocketAddress::getAddress).collect(Collectors.toList()); }, null); logger.info("Replicas provided in a read plan contacts: {}", readPlanEndpoints); diff --git a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java index c45108eea3..75b0806899 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java @@ -36,6 +36,7 @@ import org.junit.Test; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.compaction.AbstractCompactionStrategy; import org.apache.cassandra.db.compaction.LeveledCompactionStrategy; @@ -144,8 +145,14 @@ public class SSTableIdGenerationTest extends TestBaseImpl .withConfig(config -> config.set(ENABLE_UUID_FIELD_NAME, true)) .start())) { - cluster.disableAutoCompaction(KEYSPACE); cluster.schemaChange(createTableStmt(KEYSPACE, "tbl", null)); + for (IInvokableInstance instance : cluster) + { + instance.runOnInstance(() -> { + for (ColumnFamilyStore cs : Keyspace.open(KEYSPACE).getColumnFamilyStores()) + cs.disableAutoCompaction(); + }); + } createSSTables(cluster.get(1), KEYSPACE, "tbl", 1, 2); assertSSTablesCount(cluster.get(1), 0, 2, KEYSPACE, "tbl"); verfiySSTableActivity(cluster, false); diff --git a/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java b/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java index 2e26659243..c6c9413285 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java @@ -36,17 +36,23 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.cassandra.config.Config.LWTStrategy; +import org.apache.cassandra.config.Config.NonSerialWriteStrategy; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Pair; import static com.google.common.collect.Iterators.toArray; import static java.lang.String.format; import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL; import static org.apache.cassandra.distributed.shared.AssertUtils.row; /** @@ -81,24 +87,31 @@ public class ShortReadProtectionTest extends TestBaseImpl @Parameterized.Parameter(2) public boolean paging; - @Parameterized.Parameters(name = "{index}: read_cl={0} flush={1} paging={2}") + @Parameterized.Parameter(3) + public Pair transactionStrategies; + + @Parameterized.Parameters(name = "{index}: read_cl={0} flush={1} paging={2}, transactionStrategies={3}") public static Collection data() { List result = new ArrayList<>(); - for (ConsistencyLevel readConsistencyLevel : Arrays.asList(ALL, QUORUM)) - for (boolean flush : BOOLEANS) - for (boolean paging : BOOLEANS) - result.add(new Object[]{ readConsistencyLevel, flush, paging }); + for (Pair transactionStrategies : Arrays.asList(Pair.create(LWTStrategy.accord, NonSerialWriteStrategy.migration), Pair.create(LWTStrategy.migration, NonSerialWriteStrategy.normal))) + for (ConsistencyLevel readConsistencyLevel : Arrays.asList(ALL, QUORUM, SERIAL)) + for (boolean flush : BOOLEANS) + for (boolean paging : BOOLEANS) + result.add(new Object[]{ readConsistencyLevel, flush, paging, transactionStrategies}); return result; } @BeforeClass public static void setupCluster() throws IOException { + // TODO this blocks some of the original testing of SRP invoking BRR since it is BRRing through Accord + // but maybe that is out of scope and is covered by the dedicated BRR tests? cluster = init(Cluster.build() .withNodes(NUM_NODES) .withConfig(config -> config.set("hinted_handoff_enabled", false)) .start()); + cluster.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE)); } @AfterClass @@ -111,6 +124,14 @@ public class ShortReadProtectionTest extends TestBaseImpl @Before public void setupTester() { + String lwtStrategy = transactionStrategies.left.toString(); + String nonSerialWriteStrategy = transactionStrategies.right.toString(); + cluster.forEach(node -> { + node.runOnInstance(() -> { + DatabaseDescriptor.setLWTStrategy(LWTStrategy.valueOf(lwtStrategy)); + DatabaseDescriptor.setNonSerialWriteStrategy(NonSerialWriteStrategy.valueOf(nonSerialWriteStrategy)); + }); + }); tester = new Tester(readConsistencyLevel, flush, paging); } @@ -427,7 +448,7 @@ public class ShortReadProtectionTest extends TestBaseImpl this.paging = paging; qualifiedTableName = KEYSPACE + ".t_" + seqNumber.getAndIncrement(); - assert readConsistencyLevel == ALL || readConsistencyLevel == QUORUM + assert readConsistencyLevel == ALL || readConsistencyLevel == QUORUM || readConsistencyLevel == SERIAL : "Only ALL and QUORUM consistency levels are supported"; } @@ -485,12 +506,12 @@ public class ShortReadProtectionTest extends TestBaseImpl /** * Internally runs the specified write queries in the specified node. If the {@link #readConsistencyLevel} is - * QUORUM the write will also be internally done in the next replica in the ring, to simulate a QUORUM write. + * QUORUM/SERIAL the write will also be internally done in the next replica in the ring, to simulate a QUORUM/SERIAL write. */ private Tester toNode(int node, String... queries) { IInvokableInstance replica = cluster.get(node); - IInvokableInstance nextReplica = readConsistencyLevel == QUORUM + IInvokableInstance nextReplica = (readConsistencyLevel == QUORUM || readConsistencyLevel == SERIAL) ? cluster.get(node == NUM_NODES ? 1 : node + 1) : null; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java index 8a515bb614..db8f1c21d9 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java @@ -20,28 +20,34 @@ package org.apache.cassandra.distributed.test.accord; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.stream.Collectors; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; - -import org.apache.cassandra.distributed.Cluster; -import org.assertj.core.api.Assertions; - +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.primitives.Unseekables; import accord.topology.Topologies; +import org.apache.cassandra.config.Config.NonSerialWriteStrategy; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.functions.types.utils.Bytes; import org.apache.cassandra.db.marshal.Int32Type; @@ -49,19 +55,25 @@ import org.apache.cassandra.db.marshal.ListType; import org.apache.cassandra.db.marshal.MapType; import org.apache.cassandra.db.marshal.SetType; import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.ICoordinator; import org.apache.cassandra.distributed.api.QueryResults; import org.apache.cassandra.distributed.api.SimpleQueryResult; +import org.apache.cassandra.distributed.shared.AssertUtils; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTestUtils; import org.apache.cassandra.utils.ByteBufferUtil; +import org.assertj.core.api.Assertions; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static java.util.Collections.singletonList; import static org.apache.cassandra.cql3.CQLTester.row; import static org.apache.cassandra.distributed.util.QueryResultUtil.assertThat; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +@RunWith(Parameterized.class) public class AccordCQLTest extends AccordTestBase { private static final Logger logger = LoggerFactory.getLogger(AccordCQLTest.class); @@ -72,11 +84,36 @@ public class AccordCQLTest extends AccordTestBase return logger; } + @Parameterized.Parameter + public String nonSerialWriteStrategyName; + + NonSerialWriteStrategy nonSerialWriteStrategy; + + @Parameterized.Parameters(name = "nonSerialWriteStrategy={0}") + public static Collection data() + { + return ImmutableList.of(new Object[] {NonSerialWriteStrategy.accord.toString()}, new Object[] {NonSerialWriteStrategy.migration.toString()}); + } + + @Before + public void setNonSerialWriteStrategy() + { + nonSerialWriteStrategy = NonSerialWriteStrategy.valueOf(nonSerialWriteStrategyName); + String nonSerialWriteStrategyName = this.nonSerialWriteStrategyName; + SHARED_CLUSTER.forEach(node -> { + node.runOnInstance(() -> { + DatabaseDescriptor.setNonSerialWriteStrategy(NonSerialWriteStrategy.valueOf(nonSerialWriteStrategyName)); + }); + }); + } + @BeforeClass public static void setupClass() throws IOException { - AccordTestBase.setupClass(); + AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("lwt_strategy", "accord") + .set("non_serial_write_strategy", "migration")), 2); SHARED_CLUSTER.schemaChange("CREATE TYPE " + KEYSPACE + ".person (height int, age int)"); + SHARED_CLUSTER.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE)); } @Test @@ -138,7 +175,8 @@ public class AccordCQLTest extends AccordTestBase { String keyspace = "multipleShards"; String currentTable = keyspace + ".tbl"; - List ddls = Arrays.asList("CREATE KEYSPACE " + keyspace + " WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}", + List ddls = Arrays.asList("DROP KEYSPACE IF EXISTS " + keyspace + ";", + "CREATE KEYSPACE " + keyspace + " WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}", "CREATE TABLE " + currentTable + " (k blob, c int, v int, primary key (k, c))"); List tokens = tokens(); List keys = tokensToKeys(tokens); @@ -354,7 +392,12 @@ public class AccordCQLTest extends AccordTestBase private void assertResultsFromAccordMatches(Cluster cluster, String accordRead, String simpleRead, int key) { - Object[][] simpleReadResult = cluster.get(1).executeInternal(simpleRead, key); + Object[][] simpleReadResult; + if (nonSerialWriteStrategy.ignoresSuppliedConsistencyLevel) + // With accord non-SERIAL write strategy the commit CL is effectively ANY so we need to read at SERIAL + simpleReadResult = cluster.coordinator(1).execute(simpleRead, ConsistencyLevel.SERIAL, key); + else + simpleReadResult = cluster.get(1).executeInternal(simpleRead, key); Object[][] accordReadResult = executeWithRetry(cluster, accordRead, key).toObjectArrays(); Assertions.assertThat(withRemovedNullOnlyRows(accordReadResult)).isEqualTo(withRemovedNullOnlyRows(simpleReadResult)); @@ -587,10 +630,56 @@ public class AccordCQLTest extends AccordTestBase String check = "BEGIN TRANSACTION\n" + " SELECT v FROM " + currentTable + " WHERE k = 1;\n" + "COMMIT TRANSACTION"; - assertRowEqualsWithPreemptedRetry(cluster, new Object[] { endingvalue }, check); + assertRowEqualsWithPreemptedRetry(cluster, new Object[] { 2 }, check); }); } + @Test + public void testConstantNonStaticRowReadBeforeUpdate() throws Exception + { + test("CREATE TABLE " + currentTable + " (k int, c int, v int, PRIMARY KEY (k, c))", + cluster -> + { + cluster.coordinator(1).execute("INSERT INTO " + currentTable + " (k, c, v) VALUES (1, 2, ?)", ConsistencyLevel.ALL, 3); + + String update = "BEGIN TRANSACTION\n" + + " LET row1 = (SELECT * FROM " + currentTable + " WHERE k = 1 AND c = 2);\n" + + " SELECT row1.v;\n" + + " UPDATE " + currentTable + " SET v += 1 WHERE k = 1 AND c = 2;\n" + + "COMMIT TRANSACTION"; + assertRowEquals(cluster, new Object[] { 3 }, update); + + String check = "BEGIN TRANSACTION\n" + + " SELECT v FROM " + currentTable + " WHERE k = 1 AND c = 2;\n" + + "COMMIT TRANSACTION"; + assertRowEqualsWithPreemptedRetry(cluster, new Object[] { 4 }, check); + }); + } + + @Test + public void testRangeDeletion() throws Exception + { + test("CREATE TABLE " + currentTable + " (k int, c int, v int, PRIMARY KEY (k, c))", + cluster -> + { + cluster.coordinator(1).execute("INSERT INTO " + currentTable + " (k, c, v) VALUES (1, 2, ?)", ConsistencyLevel.ALL, 3); + cluster.coordinator(1).execute("INSERT INTO " + currentTable + " (k, c, v) VALUES (1, 3, ?)", ConsistencyLevel.ALL, 4); + cluster.coordinator(1).execute("INSERT INTO " + currentTable + " (k, c, v) VALUES (1, 4, ?)", ConsistencyLevel.ALL, 5); + + String update = "BEGIN TRANSACTION\n" + + " LET row1 = (SELECT * FROM " + currentTable + " WHERE k = 1 AND c = 2);\n" + + " SELECT row1.v;\n" + + " DELETE FROM " + currentTable + " WHERE k = 1 AND c >=3 AND c <= 4;\n" + + "COMMIT TRANSACTION"; + assertRowEquals(cluster, new Object[] { 3 }, update); + + Object[][] check = cluster.coordinator(1).execute("SELECT * FROM " + currentTable + " WHERE k = 1;", ConsistencyLevel.SERIAL); + assertArrayEquals(new Object[] { 1, 2, 3 }, check[0]); + assertEquals(1, check.length); + }); + } + + @Test public void testPartitionKeyReferenceCondition() throws Exception { @@ -2434,7 +2523,9 @@ public class AccordCQLTest extends AccordTestBase @Test public void demoTest() throws Throwable { + SHARED_CLUSTER.schemaChange("DROP KEYSPACE IF EXISTS demo_ks;"); SHARED_CLUSTER.schemaChange("CREATE KEYSPACE demo_ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2};"); + SHARED_CLUSTER.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged("demo_ks")); SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.org_docs ( org_name text, doc_id int, contents_version int static, title text, permissions int, PRIMARY KEY (org_name, doc_id) );"); SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.org_users ( org_name text, user text, members_version int static, permissions int, PRIMARY KEY (org_name, user) );"); SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.user_docs ( user text, doc_id int, title text, org_name text, permissions int, PRIMARY KEY (user, doc_id) );"); @@ -2519,6 +2610,8 @@ public class AccordCQLTest extends AccordTestBase cluster -> { ICoordinator coordinator = cluster.coordinator(1); int startingAccordCoordinateCount = getAccordCoordinateCount(); + assertRowEquals(cluster, new Object[]{false}, "UPDATE " + currentTable + " SET v = 4 WHERE id = 1 AND c = 2 IF EXISTS"); + assertRowEquals(cluster, new Object[]{false}, "UPDATE " + currentTable + " SET v = 4 WHERE id = 1 AND c = 2 IF v = 3"); coordinator.execute("INSERT INTO " + currentTable + " (id, c, v, s) VALUES (1, 2, 3, 5);", ConsistencyLevel.ALL); assertRowSerial(cluster, "SELECT id, c, v, s FROM " + currentTable + " WHERE id = 1 AND c = 2", 1, 2, 3, 5); assertRowEquals(cluster, new Object[]{true}, "UPDATE " + currentTable + " SET v = 4 WHERE id = 1 AND c = 2 IF v = 3"); @@ -2533,8 +2626,118 @@ public class AccordCQLTest extends AccordTestBase assertRowSerial(cluster, "SELECT id, c, v, s FROM " + currentTable + " WHERE id = 1 AND c = 2", 1, 2, 5, 5); assertRowEquals(cluster, new Object[]{true}, "UPDATE " + currentTable + " SET s = 6 WHERE id = 1 IF s = 5"); assertRowSerial(cluster, "SELECT id, c, v, s FROM " + currentTable + " WHERE id = 1 AND c = 2", 1, 2, 5, 6); + + // Test that read before write works with CAS + assertRowEquals(cluster, new Object[]{true}, "UPDATE " + currentTable + " SET s +=1, v += 1 WHERE id = 1 AND c = 2 IF EXISTS"); + assertRowSerial(cluster, "SELECT id, c, v, s FROM " + currentTable + " WHERE id = 1 AND c = 2", 1, 2, 6, 7); + + // Check range deletion works + coordinator.execute("INSERT INTO " + currentTable + " (id, c, v, s) VALUES (1, 2, 6, 7);", ConsistencyLevel.ALL); + coordinator.execute("INSERT INTO " + currentTable + " (id, c, v) VALUES (1, 3, 3);", ConsistencyLevel.ALL); + assertRowEquals(cluster, new Object[]{true}, "BEGIN BATCH \n" + + "UPDATE " + currentTable + " SET s +=1, v += 1 WHERE id = 1 AND c = 2 IF EXISTS; \n" + + "DELETE FROM " + currentTable + " WHERE id = 1 AND c > 0 AND c < 10; \n" + + "APPLY BATCH;"); + Object[][] rangeDeletionCheck = coordinator.execute("SELECT id, c, v, s FROM " + currentTable + " WHERE id = 1", ConsistencyLevel.SERIAL); + assertArrayEquals(new Object[] { 1, 2, 7, 8 }, rangeDeletionCheck[0]); + assertEquals(1, rangeDeletionCheck.length); + // Make sure all the consensus using queries actually were run on Accord - assertEquals( 11, getAccordCoordinateCount() - startingAccordCoordinateCount); - }); + if (nonSerialWriteStrategy.writesThroughAccord) + assertEquals( 20, getAccordCoordinateCount() - startingAccordCoordinateCount); + else + // Non-serial writes don't go through Accord in these modes + assertEquals( 17, getAccordCoordinateCount() - startingAccordCoordinateCount); + }); + } + + // Reproduces some bugs that simulator finds + @Test + public void testCASSimulatorLite() throws Exception + { + test("CREATE TABLE " + currentTable + " (pk int, count int, seq1 text, seq2 list, PRIMARY KEY (pk))", + cluster -> { + ICoordinator coordinator = cluster.coordinator(1); + coordinator.execute("INSERT INTO " + currentTable + " (pk, count, seq1, seq2) VALUES (1, 0, '', []) USING TIMESTAMP 0", ConsistencyLevel.ALL); + + ListType LIST_TYPE = ListType.getInstance(Int32Type.instance, true); + ExecutorService es = Executors.newCachedThreadPool(); + List> futures = new ArrayList<>(); + for (int ii = 0; ii < 10; ii++) + { + int id = ii; + futures.add(es.submit(() -> coordinator.execute("UPDATE " + currentTable + " SET count = count + 1, seq1 = seq1 + ?, seq2 = seq2 + ? WHERE pk = ? IF EXISTS", ConsistencyLevel.ALL, id + ",", ByteBufferUtil.getArray(LIST_TYPE.decompose(singletonList(id))), 1))); + } + for (Future f : futures) + f.get(); + + Object[][] result = coordinator.execute("SELECT pk, count, seq1, seq2 FROM " + currentTable + " WHERE pk = 1", ConsistencyLevel.SERIAL); + + int[] seq1 = Arrays.stream(((String) result[0][2]).split(",")) + .filter(s -> !s.isEmpty()) + .mapToInt(Integer::parseInt) + .toArray(); + int[] seq2 = ((ArrayList) result[0][3]).stream().mapToInt(x -> x).toArray(); + logger.info("String append of ids executed {}", Arrays.toString(seq1)); + logger.info("List append of ids executed {}", Arrays.toString(seq2)); + assertArrayEquals("History doesn't match between the two columns", seq1, seq2); + }); + } + + @Test + public void testTransactionCasSimulatorLite() throws Exception + { + test("CREATE TABLE " + currentTable + " (pk int, count int, seq1 text, seq2 list, PRIMARY KEY (pk))", + cluster -> + { + ICoordinator coordinator = cluster.coordinator(1); + coordinator.execute("INSERT INTO " + currentTable + " (pk, count, seq1, seq2) VALUES (1, 0, '', []) USING TIMESTAMP 0", ConsistencyLevel.ALL); + + ListType LIST_TYPE = ListType.getInstance(Int32Type.instance, true); + ExecutorService es = Executors.newCachedThreadPool(); + List> futures = new ArrayList<>(); + for (int ii = 0; ii < 10; ii++) + { + int id = ii; + String update = "BEGIN TRANSACTION\n" + + " LET row1 = (SELECT * FROM " + currentTable + " WHERE pk = 1);\n" + + " UPDATE " + currentTable + " SET count += 1, seq1 = seq1 + ?, seq2 = seq2 + ? WHERE pk=1;\n" + + "COMMIT TRANSACTION"; + futures.add(es.submit(() -> coordinator.executeWithResult(update, ConsistencyLevel.ANY, id + ",", ByteBufferUtil.getArray(LIST_TYPE.decompose(singletonList(id)))))); + } + for (Future f : futures) + f.get(); + + String check = "BEGIN TRANSACTION\n" + + " SELECT * FROM " + currentTable + " WHERE pk = 1;\n" + + "COMMIT TRANSACTION"; + Object[][] result = coordinator.execute(check, ConsistencyLevel.ALL); + + int[] seq1 = Arrays.stream(((String) result[0][2]).split(",")) + .filter(s -> !s.isEmpty()) + .mapToInt(Integer::parseInt) + .toArray(); + int[] seq2 = ((ArrayList) result[0][3]).stream().mapToInt(x -> x).toArray(); + logger.info("String append of ids executed {}", Arrays.toString(seq1)); + logger.info("List append of ids executed {}", Arrays.toString(seq2)); + assertArrayEquals("History doesn't match between the two columns", seq1, seq2); + } + ); + } + + @Test + public void testSerialReadDescending() throws Throwable + { + test("CREATE TABLE " + currentTable + " (k int, c int, v int, PRIMARY KEY(k, c))", + cluster -> { + ICoordinator coordinator = cluster.coordinator(1); + for (int i = 1; i <= 10; i++) + coordinator.execute("INSERT INTO " + currentTable + " (k, c, v) VALUES (0, ?, ?) USING TIMESTAMP 0;", ConsistencyLevel.ALL, i, i * 10); + assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 1", AssertUtils.row(10, 100)); + assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 2", AssertUtils.row(10, 100), AssertUtils.row(9, 90)); + assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 3", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80)); + assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 4", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80), AssertUtils.row(7, 70)); + } + ); } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java index 06b44805b3..827eebf6f9 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java @@ -39,6 +39,7 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.AssertionUtils; import org.assertj.core.api.Assertions; @@ -78,7 +79,7 @@ public class AccordFeatureFlagTest extends TestBaseImpl assertEquals("No Accord virtual tables should exist", Collections.emptyList(), tables); // Make sure we throw if someone tries to coordinate a transaction against the no-op service: - Assertions.assertThatThrownBy(() -> cluster.get(1).callOnInstance(() -> AccordService.instance().coordinate(null, null))) + Assertions.assertThatThrownBy(() -> cluster.get(1).callOnInstance(() -> AccordService.instance().coordinate(null, null, Dispatcher.RequestTime.forImmediateExecution()))) .isInstanceOf(UnsupportedOperationException.class); } } @@ -91,7 +92,7 @@ public class AccordFeatureFlagTest extends TestBaseImpl .withoutVNodes() .withConfig(c -> c.with(Feature.NETWORK) .set("accord.enabled", "false") - .set("legacy_paxos_strategy", "accord")).createWithoutStarting()) + .set("lwt_strategy", "accord")).createWithoutStarting()) { Assertions.assertThatThrownBy(() -> cluster.startup()) diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java index ba9e1b801c..7315df858a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java @@ -18,11 +18,17 @@ package org.apache.cassandra.distributed.test.accord; +import java.io.IOException; +import java.util.function.Function; + +import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import accord.impl.SimpleProgressLog; import accord.messages.Commit; +import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.IMessageFilters; import org.apache.cassandra.distributed.impl.Instance; import org.apache.cassandra.net.Message; @@ -38,9 +44,16 @@ public class AccordIntegrationTest extends AccordTestBase return logger; } + @BeforeClass + public static void setUp() throws IOException + { + AccordTestBase.setupCluster(Function.identity(), 2); + } + @Test public void testRecovery() throws Exception { + pauseSimpleProgressLog(); test(cluster -> { IMessageFilters.Filter lostApply = cluster.filters().verbs(Verb.ACCORD_APPLY_REQ.id).drop(); IMessageFilters.Filter lostCommit = cluster.filters().verbs(Verb.ACCORD_COMMIT_REQ.id).to(2).drop(); @@ -89,12 +102,13 @@ public class AccordIntegrationTest extends AccordTestBase @Test public void testLostCommitReadTriggersFallbackRead() throws Exception { + pauseSimpleProgressLog(); test(cluster -> { // It's expected that the required Read will happen regardless of whether this fails to return a read cluster.filters().verbs(Verb.ACCORD_COMMIT_REQ.id).messagesMatching((from, to, iMessage) -> cluster.get(from).callOnInstance(() -> { Message msg = Instance.deserializeMessage(iMessage); if (msg.payload instanceof Commit) - return ((Commit) msg.payload).read != null; + return ((Commit) msg.payload).readData != null; return false; })).drop(); @@ -113,4 +127,10 @@ public class AccordIntegrationTest extends AccordTestBase assertRowEqualsWithPreemptedRetry(cluster, new Object[] { 0, 0, 1 }, check, 0, 0); }); } + + private void pauseSimpleProgressLog() + { + for (IInvokableInstance instance : SHARED_CLUSTER) + instance.runOnInstance(() -> SimpleProgressLog.PAUSE_FOR_TEST = true); + } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteropReadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteropReadTest.java new file mode 100644 index 0000000000..6022fdda55 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteropReadTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord; + +import org.junit.Test; + +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.statements.ModificationStatement; +import org.apache.cassandra.cql3.statements.SelectStatement; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.SimpleQueryResult; +import org.apache.cassandra.distributed.impl.RowUtil; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.distributed.util.QueryResultUtil; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.QueryState; +import org.apache.cassandra.service.accord.AccordService; + +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; + +public class AccordInteropReadTest extends TestBaseImpl +{ + + private static void localWrite(String s) + { + ModificationStatement stmt = (ModificationStatement) QueryProcessor.parseStatement(s).prepare(ClientState.forInternalCalls()); + stmt.executeLocally(QueryState.forInternalCalls(), QueryOptions.DEFAULT); + } + + private static SimpleQueryResult localRead(String s) + { + SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(s).prepare(ClientState.forInternalCalls()); + return RowUtil.toQueryResult(stmt.executeLocally(QueryState.forInternalCalls(), QueryOptions.DEFAULT)); + } + + private static Object[] obj(Object... values) + { + return values; + } + + @Test + public void serialReadTest() throws Throwable + { + try (Cluster cluster = builder().withNodes(3) + .withConfig(config -> config.with(GOSSIP).with(NETWORK) + .set("non_serial_write_strategy", "mixed") + .set("lwt_strategy", "accord")) + .start()) + { + cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':3}"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, PRIMARY KEY (k, c))"); + cluster.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged("ks")); + + cluster.get(1).runOnInstance(() -> localWrite("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 1)")); + cluster.get(2).runOnInstance(() -> localWrite("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)")); + cluster.get(3).shutdown(); + cluster.get(1).runOnInstance(() -> QueryResultUtil.assertThat(localRead("SELECT * FROM ks.tbl WHERE k=1")).isEqualTo(obj(obj(1, 1, 1)))); + cluster.get(2).runOnInstance(() -> QueryResultUtil.assertThat(localRead("SELECT * FROM ks.tbl WHERE k=1")).isEqualTo(obj(obj(1, 1, 2)))); + + + String query = "BEGIN TRANSACTION\n" + + " LET row1 = (SELECT v FROM ks.tbl WHERE k=0 AND c=0);\n" + + " SELECT row1.v;\n" + + " IF row1 IS NULL THEN\n" + + " INSERT INTO ks.tbl (k, c, v) VALUES (0, 0, 1);\n" + + " END IF\n" + + "COMMIT TRANSACTION"; + + SimpleQueryResult result = cluster.coordinator(1).executeWithResult("SELECT * FROM ks.tbl WHERE k=1", ConsistencyLevel.SERIAL); + QueryResultUtil.assertThat(result).isEqualTo(obj(obj(1, 1, 2))); + cluster.get(1).runOnInstance(() -> QueryResultUtil.assertThat(localRead("SELECT * FROM ks.tbl WHERE k=1")).isEqualTo(obj(obj(1, 1, 2)))); + cluster.get(2).runOnInstance(() -> QueryResultUtil.assertThat(localRead("SELECT * FROM ks.tbl WHERE k=1")).isEqualTo(obj(obj(1, 1, 2)))); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteroperabilityTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteroperabilityTest.java new file mode 100644 index 0000000000..74c64c3b13 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteroperabilityTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord; + +import java.io.IOException; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.service.accord.AccordService; + +public class AccordInteroperabilityTest extends AccordTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(AccordInteroperabilityTest.class); + + @Override + protected Logger logger() + { + return logger; + } + + @BeforeClass + public static void setupClass() throws IOException + { + AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("lwt_strategy", "accord") + .set("non_serial_write_strategy", "accord")), 3); + SHARED_CLUSTER.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE)); + } + + @Test + public void testSerialReadDescending() throws Throwable + { + test("CREATE TABLE " + currentTable + " (k int, c int, v int, PRIMARY KEY(k, c))", + cluster -> { + ICoordinator coordinator = cluster.coordinator(1); + for (int i = 1; i <= 10; i++) + coordinator.execute("INSERT INTO " + currentTable + " (k, c, v) VALUES (0, ?, ?) USING TIMESTAMP 0;", ConsistencyLevel.ALL, i, i * 10); + assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 1", AssertUtils.row(10, 100)); + assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 2", AssertUtils.row(10, 100), AssertUtils.row(9, 90)); + assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 3", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80)); + assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 4", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80), AssertUtils.row(7, 70)); + } + ); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java index 4db878da47..496e8d08e4 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java @@ -60,7 +60,7 @@ public class AccordMetricsTest extends AccordTestBase @BeforeClass public static void setupClass() throws IOException { - AccordTestBase.setupClass(); + AccordTestBase.setupCluster(Function.identity(), 2); SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0))); for (int i = 0; i < SHARED_CLUSTER.size(); i++) // initialize metrics logger.trace(SHARED_CLUSTER.get(i + 1).callOnInstance(() -> AccordMetrics.readMetrics.toString() + AccordMetrics.writeMetrics.toString())); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java new file mode 100644 index 0000000000..69c550f633 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java @@ -0,0 +1,655 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import javax.annotation.Nonnull; + +import com.google.common.collect.ImmutableList; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.Config.PaxosVariant; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.SimpleBuilders.PartitionUpdateBuilder; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.NodeToolResult; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.Ballot.Flag; +import org.apache.cassandra.service.paxos.BallotGenerator; +import org.apache.cassandra.service.paxos.Commit.Agreed; +import org.apache.cassandra.service.paxos.Commit.Proposal; +import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.utils.ByteArrayUtil; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.JsonUtils; +import org.apache.cassandra.utils.PojoToString; +import org.yaml.snakeyaml.Yaml; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static org.apache.cassandra.Util.spinUntilSuccess; +import static org.apache.cassandra.db.SystemKeyspace.CONSENSUS_MIGRATION_STATE; +import static org.apache.cassandra.db.SystemKeyspace.PAXOS; +import static org.apache.cassandra.dht.Range.normalize; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ANY; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL; +import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME; +import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.paxosV2; +import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.PROMISE; +import static org.assertj.core.api.Fail.fail; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/* + * This test suite is intended to serve as an integration test with some pretty good visibility into actual execution + * that can run quickly, and make sure all the right steps are running during migration. + * + * For correctness related to wrong/right answers we rely on simulator to validate. + */ +public class AccordMigrationTest extends AccordTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(AccordMigrationTest.class); + + private static final int CLUSTERING_VALUE = 2; + + private static final String TABLE_FMT = "CREATE TABLE %s (id int, c int, v int, s int static, PRIMARY KEY ((id), c));"; + + private static final String CAS_FMT = "UPDATE %s SET v = 4 WHERE id = ? AND c = %d IF v = 42"; + + private static IPartitioner partitioner; + + private static Token minToken; + + private static Token maxToken; + + private static Token midToken; + + private static Token upperMidToken; + + private static Token lowerMidToken; + + private static ICoordinator coordinator; + + // To create a precise repair where the repaired range is fully contained in a locally replicated range + // we need to align with this token. The local ranges are (9223372036854775805,-1] and (-1,9223372036854775805] + // No idea why the partitioner creates such an + private Token maxAlignedWithLocalRanges = new LongToken(9223372036854775805L); + + @Override + protected Logger logger() + { + return logger; + } + + @BeforeClass + public static void setupClass() throws IOException + { + ServerTestUtils.daemonInitialization(); + // Otherwise repair complains if you don't specify a keyspace + CassandraRelevantProperties.SYSTEM_TRACES_DEFAULT_RF.setInt(3); + AccordTestBase.setupCluster(builder -> + builder.appendConfig(config -> + config.set("paxos_variant", PaxosVariant.v2.name()) + .set("non_serial_write_strategy", "migration")), + 3); + partitioner = FBUtilities.newPartitioner(SHARED_CLUSTER.get(1).callsOnInstance(() -> DatabaseDescriptor.getPartitioner().getClass().getSimpleName()).call()); + StorageService.instance.setPartitionerUnsafe(partitioner); + ServerTestUtils.prepareServerNoRegister(); + minToken = partitioner.getMinimumToken(); + maxToken = partitioner.getMaximumToken(); + midToken = partitioner.midpoint(minToken, maxToken); + upperMidToken = partitioner.midpoint(midToken, maxToken); + lowerMidToken = partitioner.midpoint(minToken, midToken); + coordinator = SHARED_CLUSTER.coordinator(1); + SHARED_CLUSTER.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE)); + } + + @AfterClass + public static void tearDownClass() + { + StorageService.instance.resetPartitionerUnsafe(); + } + + @After + public void tearDown() throws Exception + { + super.tearDown(); + // Reset migration state + forEach(() -> { + ConsensusRequestRouter.resetInstance(); + ConsensusKeyMigrationState.reset(); + }); + SHARED_CLUSTER.get(1).runOnInstance(() -> { + ConsensusTableMigrationState.reset(); + }); + SHARED_CLUSTER.coordinators().forEach(coordinator -> coordinator.execute(format("TRUNCATE TABLE %s.%s", SYSTEM_KEYSPACE_NAME, CONSENSUS_MIGRATION_STATE), ALL)); + SHARED_CLUSTER.coordinators().forEach(coordinator -> coordinator.execute(format("TRUNCATE TABLE %s.%s", SYSTEM_KEYSPACE_NAME, PAXOS), ALL)); + } + + private static String nodetool(ICoordinator coordinator, String... commandAndArgs) + { + NodeToolResult nodetoolResult = coordinator.instance().nodetoolResult(commandAndArgs); + if (!nodetoolResult.getStdout().isEmpty()) + System.out.println(nodetoolResult.getStdout()); + if (!nodetoolResult.getStderr().isEmpty()) + System.err.println(nodetoolResult.getStderr()); + if (nodetoolResult.getError() != null) + fail("Failed nodetool " + Arrays.asList(commandAndArgs), nodetoolResult.getError()); + // TODO why does standard out end up in stderr in nodetool? + return nodetoolResult.getStdout(); + } + + private static int getKeyBetweenTokens(Token left, Token right) + { + return getKeysBetweenTokens(left, right).next(); + } + + private static Iterator getKeysBetweenTokens(Token left, Token right) + { + return new Iterator() + { + int candidate = 0; + @Override + public boolean hasNext() + { + return true; + } + + @Override + public Integer next() + { + for (int i = 0; i < 1_000_000; i++) + { + int value = candidate; + candidate++; + if (partitioner.getToken(ByteBufferUtil.bytes(value)).compareTo(right) < 0 && partitioner.getToken(ByteBufferUtil.bytes(value)).compareTo(left) > 0) + return value; + } + throw new IllegalStateException("Gave up after 1 million attempts"); + } + }; + } + + /* + * Force routing a request to Paxos even after a range has been marked migrating to simulate + * a race between updating cluster metadata and making a routing decision to a specific consensus + * protocol. Paxos should still detect the routing change at two points. After running the promise phase + * (round of messaging might discover a new epoch) and during the accept phase (might not get a majority due + * to rejects caused by acceptors refusing due to migration). + * + * This is used directly to test that begin rejects after discovering a migration, and indirectly in + * PaxosToAccordMigrationNotHappeningUpToAccept. + */ + public static class RoutesToPaxosOnce extends ConsensusRequestRouter + { + boolean routed; + + @Override + protected ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull ColumnFamilyStore cfs, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + { + if (routed) + return super.routeAndMaybeMigrate(key, cfs, consistencyLevel, requestTime, timeoutNanos, isForWrite); + routed = true; + return paxosV2; + } + } + + /* + * To allow for testing of Paxos we want to force begin to succeed, but accept to fail + * with a retry on new protocol reject. + */ + public static class PaxosToAccordMigrationNotHappeningUpToBegin extends RoutesToPaxosOnce + { + @Override + public boolean isKeyInMigratingOrMigratedRangeDuringPaxosBegin(TableId tableId, DecoratedKey key) + { + return false; + } + } + + public static class PaxosToAccordMigrationNotHappeningUpToAccept extends PaxosToAccordMigrationNotHappeningUpToBegin + { + @Override + public boolean isKeyInMigratingOrMigratedRangeDuringPaxosAccept(TableId tableId, DecoratedKey key) + { + return false; + } + } + + public static class RoutesToAccordOnce extends ConsensusRequestRouter + { + boolean routed; + + @Override + protected ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull ColumnFamilyStore cfs, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + { + if (routed) + return super.routeAndMaybeMigrate(key, cfs, consistencyLevel, requestTime, timeoutNanos, isForWrite); + routed = true; + return ConsensusRoutingDecision.accord; + } + } + + /* + * Helper to invoke a query and assert that the right metrics change indicating the correct + * paths were taken to execute the query during migration + */ + private static void assertTargetAccordWrite(Consumer query, int coordinatorIndex, int key, int expectedAccordWriteCount, int expectedCasWriteCount, int expectedKeyMigrationCount, int expectedCasBeginRejects, int expectedCasAcceptRejects) + { + int startingWriteCount = getAccordWriteCount(coordinatorIndex); + int startingCasWriteCount = getCasWriteCount(coordinatorIndex); + int startingKeyMigrationCount = getKeyMigrationCount(coordinatorIndex); + int startingCasWriteBeginRejects = getCasWriteBeginRejects(coordinatorIndex); + int startingCasWriteAcceptRejects = getCasWriteAcceptRejects(coordinatorIndex); + query.accept(key); + assertEquals("Accord writes", expectedAccordWriteCount, getAccordWriteCount(coordinatorIndex) - startingWriteCount); + assertEquals("CAS writes", expectedCasWriteCount, getCasWriteCount(coordinatorIndex) - startingCasWriteCount); + assertEquals("Key Migrations", expectedKeyMigrationCount, getKeyMigrationCount(coordinatorIndex) - startingKeyMigrationCount); + assertEquals("CAS Begin rejects", expectedCasBeginRejects, getCasWriteBeginRejects(coordinatorIndex) - startingCasWriteBeginRejects); + assertEquals("CAS Accept rejects", expectedCasAcceptRejects, getCasWriteAcceptRejects(coordinatorIndex) - startingCasWriteAcceptRejects); + } + + private static Object[][] assertTargetAccordRead(Function query, int coordinatorIndex, int key, int expectedAccordReadCount, int expectedCasPrepareCount, int expectedKeyMigrationCount, int expectedCasReadBeginRejects, int expectedCasReadAcceptRejects) + { + int startingReadCount = getAccordReadCount(coordinatorIndex); + int startingCasPrepareCount = getCasPrepareCount(coordinatorIndex); + int startingKeyMigrationCount = getKeyMigrationCount(coordinatorIndex); + int startingCasReadBeginRejects = getCasReadBeginRejects(coordinatorIndex); + int startingCasReadAcceptRejects = getCasReadAcceptRejects(coordinatorIndex); + Object[][] result = query.apply(key); + assertEquals("Accord reads", expectedAccordReadCount, getAccordReadCount(coordinatorIndex) - startingReadCount); + assertEquals("CAS prepares", expectedCasPrepareCount, getCasPrepareCount(coordinatorIndex) - startingCasPrepareCount); + assertEquals("Key Migrations", expectedKeyMigrationCount, getKeyMigrationCount(coordinatorIndex) - startingKeyMigrationCount); + assertEquals("CAS Begin rejects", expectedCasReadBeginRejects, getCasReadBeginRejects(coordinatorIndex) - startingCasReadBeginRejects); + assertEquals("CAS Accept rejects", expectedCasReadAcceptRejects, getCasReadAcceptRejects(coordinatorIndex) - startingCasReadAcceptRejects); + return result; + } + + private static void assertTargetPaxosWrite(Consumer query, int coordinatorIndex, int key, int expectedAccordWriteCount, int expectedCasWriteCount, int expectedKeyMigrationCount, int expectedMigrationRejects, int expectedSkippedReads) + { + int startingWriteCount = getAccordWriteCount(coordinatorIndex); + int startingCasWriteCount = getCasWriteCount(coordinatorIndex); + int startingKeyMigrationCount = getKeyMigrationCount(coordinatorIndex); + int startingMigrationRejectsCount = getAccordMigrationRejects(coordinatorIndex); + int startingSkippedReadsCount = getAccordMigrationSkippedReads(); + query.accept(key); + assertEquals("Accord writes", expectedAccordWriteCount, getAccordWriteCount(coordinatorIndex) - startingWriteCount); + assertEquals("CAS writes", expectedCasWriteCount, getCasWriteCount(coordinatorIndex) - startingCasWriteCount); + assertEquals("Key Migrations", expectedKeyMigrationCount, getKeyMigrationCount(coordinatorIndex) - startingKeyMigrationCount); + assertEquals("Accord migration rejects", expectedMigrationRejects, getAccordMigrationRejects(coordinatorIndex) - startingMigrationRejectsCount); + assertEquals("Accord skipped reads", expectedSkippedReads, getAccordMigrationSkippedReads() - startingSkippedReadsCount); + } + + @Test + public void testPaxosToAccordCAS() throws Exception + { + test(format(TABLE_FMT, currentTable), + cluster -> { + String casCQL = format(CAS_FMT, currentTable, CLUSTERING_VALUE); + Consumer runCasNoApply = key -> assertRowEquals(cluster, new Object[]{false}, casCQL, key); + Consumer runCasApplies = key -> assertRowEquals(cluster, new Object[]{true}, casCQL, key); + Consumer runCasOnSecondNode = key -> assertEquals( "[applied]", cluster.coordinator(2).executeWithResult(casCQL, ANY, key).names().get(0)); + String tableName = currentTable.split("\\.")[1]; + int migratingKey = getKeyBetweenTokens(midToken, maxToken); + int notMigratingKey = getKeyBetweenTokens(minToken, midToken); + Range migratingRange = new Range(midToken, maxToken); + List> migratingRanges = ImmutableList.of(migratingRange); + + // Not actually migrating yet so should do nothing special + assertTargetAccordWrite(runCasNoApply, 1, migratingKey, 0, 1, 0, 0, 0); + + // Mark ranges migrating and check migration state is correct + nodetool(coordinator, "consensus_admin", "begin-migration", "-st", midToken.toString(), "-et", maxToken.toString(), "-tp", "accord", KEYSPACE, tableName); + assertMigrationState(tableName, ConsensusMigrationTarget.accord, emptyList(), migratingRanges, 1); + + // Should be routed directly to Accord, and perform key migration, as well as key migration read in Accord + assertTargetAccordWrite(runCasNoApply, 1, migratingKey, 1, 0, 1, 0, 0); + + // Should not repeat key migration, and should still do a migration read in Accord + assertTargetAccordWrite(runCasNoApply, 1, migratingKey, 1, 0, 0, 0, 0); + + // Should run on Paxos since it is not in the migrating range + assertTargetAccordWrite(runCasNoApply, 1, notMigratingKey, 0, 1, 0, 0, 0); + + // Check that the coordinator on the other node also has saved that the key migration was performed + // and runs the query on Accord immediately without key migration + assertTargetAccordWrite(runCasOnSecondNode, 2, migratingKey, 1, 0, 0, 0, 0); + + // Forced repair while a node is down shouldn't work, use repair instead of finish-migration because repair exposes --force + // and regular Cassandra repairs are eligible to drive migration so it's important they check --force and down nodes + InetAddressAndPort secondNodeBroadcastAddress = InetAddressAndPort.getByAddress(cluster.get(2).broadcastAddress()); + cluster.get(1).runOnInstance(() -> { + EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(secondNodeBroadcastAddress); + Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.markDead(secondNodeBroadcastAddress, endpointState)); + }); + nodetool(coordinator, "repair", "--force"); + assertMigrationState(tableName, ConsensusMigrationTarget.accord, emptyList(), migratingRanges, 1); + cluster.get(1).runOnInstance(() -> { + EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(secondNodeBroadcastAddress); + Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.realMarkAlive(secondNodeBroadcastAddress, endpointState)); + }); + + // Full repair should complete the migration and update the metadata, adding --force when nodes are up should be fine + nodetool(coordinator, "repair", "--force"); + assertMigrationState(tableName, ConsensusMigrationTarget.accord, migratingRanges, emptyList(), 0); + + // Should run on Accord, and not perform key migration nor should it need to perform a migration read in Accord now that it is repaired + assertTargetAccordWrite(runCasNoApply, 1, migratingKey, 1, 0, 0, 0, 0); + + // Should run on Paxos, and not perform key migration + assertTargetAccordWrite(runCasNoApply, 1, notMigratingKey, 0, 1, 0, 0, 0); + + // Pivot to testing repair with a subrange of the migrating range as well as key migration + // Will use the unmigrated range between lowerMidToken and midToken + nodetool(coordinator, "consensus_admin", "begin-migration", "-st", lowerMidToken.toString(), "-et", midToken.toString(), "-tp", "accord", KEYSPACE, tableName); + + // Generate several keys to test with instead of resetting key state + Iterator testingKeys = getKeysBetweenTokens(lowerMidToken, midToken); + migratingKey = testingKeys.next(); + + // Check that Paxos repair is run and actually repairs a transaction that was accepted, but not committed + String ballotString = BallotGenerator.Global.nextBallot(Flag.GLOBAL).toString(); + saveAcceptedPaxosProposal(tableName, ballotString, migratingKey); + // PaxosRepair will have inserted a condition matching row, so it can apply, demonstrating repair and + // key migration occurred + assertTargetAccordWrite(runCasApplies, 1, migratingKey, 1, 0, 1, 0, 0); + + // This will force the request to run on Paxos up to Accept + // and the accept will be rejected at both nodes and we are certain we need to retry the transaction + cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new PaxosToAccordMigrationNotHappeningUpToBegin())); + // Update inserted row so the condition can apply, if the condition check doesn't apply + // then it won't get to propose/accept + migratingKey = testingKeys.next(); + Consumer makeCASApply = key -> cluster.coordinator(1).execute("UPDATE " + currentTable + " SET v = 42 WHERE id = ? AND c = ?", ALL, key, CLUSTERING_VALUE); + makeCASApply.accept(migratingKey); + assertTargetAccordWrite(runCasApplies, 1, migratingKey, 1, 1, 1, 0, 1); + + // One node will now accept the other will reject and we are uncertain if we should retry the transaction + // and should surface that as a timeout exception + migratingKey = testingKeys.next(); + makeCASApply.accept(migratingKey); + cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new PaxosToAccordMigrationNotHappeningUpToAccept())); + try + { + runCasNoApply.accept(migratingKey); + fail("Should have thrown timeout exception"); + } + catch (Throwable t) + { + if (!t.getClass().getName().equals("org.apache.cassandra.exceptions.CasWriteTimeoutException")) + throw new RuntimeException(t); + } + + // Test that if we find out about a migration from the prepare phase Paxos.begin we + // retry it on Accord + cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new RoutesToPaxosOnce())); + // Should exit Paxos from begin, key migration should occur because it's a new key, and Accord will need to do a migration read + assertTargetAccordWrite(runCasNoApply, 1, testingKeys.next(), 1, 1, 1, 1, 0); + + // Now do two repairs to complete the migration repair, and we are done with black box integration testing + // First repair is a range smack dab in the middle + Token startTokenForRepair = partitioner.midpoint(lowerMidToken, midToken); + Token endTokenForRepair = partitioner.midpoint(startTokenForRepair, midToken); + nodetool(coordinator, "consensus_admin", "finish-migration", "-st", startTokenForRepair.toString(), "-et", endTokenForRepair.toString()); + List> migratedRanges = ImmutableList.of(new Range<>(startTokenForRepair, endTokenForRepair), migratingRange); + List> midMigratingRanges = ImmutableList.of(new Range<>(lowerMidToken, startTokenForRepair), new Range<>(endTokenForRepair, midToken)); + List> migratingAndMigratedRanges = ImmutableList.of(new Range<>(lowerMidToken, maxToken)); + assertMigrationState(tableName, ConsensusMigrationTarget.accord, migratedRanges, midMigratingRanges, 1); + + nodetool(coordinator, "consensus_admin", "finish-migration"); + assertMigrationState(tableName, ConsensusMigrationTarget.accord, migratingAndMigratedRanges, emptyList(), 0); + }); + } + + /* + * Read has a few code paths that are separate from CAS that need to be tested + * such as switching consensus protocol, rejecting read during accept, and throwing + * timeout exception if uncertain about side effects + */ + @Test + public void testPaxosToAccordSerialRead() throws Exception + { + test(format(TABLE_FMT, currentTable), + cluster -> { + String tableName = currentTable.split("\\.")[1]; + String readCQL = format("SELECT * FROM %s WHERE id = ? and c = %s", currentTable, CLUSTERING_VALUE); + Function runRead = key -> cluster.coordinator(1).execute(readCQL, SERIAL, key); + Range migratingRange = new Range<>(new LongToken(Long.MIN_VALUE + 1), new LongToken(Long.MIN_VALUE)); + List> migratingRanges = ImmutableList.of(migratingRange); + int key = 0; + + assertTargetAccordRead(runRead, 1, 0, 0, 1, 0, 0, 0); + // Mark wrap around range as migrating + nodetool(coordinator, "consensus_admin", "begin-migration", "-st", String.valueOf(Long.MIN_VALUE + 1), "-et", String.valueOf(Long.MIN_VALUE), "-tp", "accord", KEYSPACE, tableName); + assertMigrationState(tableName, ConsensusMigrationTarget.accord, emptyList(), migratingRanges, 1); + // Should run directly on accord, migrate the key, and perform a quorum read from Accord, Paxos repair will run prepare once + assertTargetAccordRead(runRead, 1, key++, 1, 1, 1, 0, 0); + + // Should run up to accept with both nodes refusing to accept + savePromisedAndCommittedPaxosProposal(tableName, key); + cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new PaxosToAccordMigrationNotHappeningUpToBegin())); + assertTargetAccordRead(runRead, 1, key++, 1, 2, 1, 0, 1); + }); + } + + @Test + public void testAccordToPaxos() throws Exception + { + test(format(TABLE_FMT, currentTable), + cluster -> { + String casCQL = format(CAS_FMT, currentTable, CLUSTERING_VALUE); + Consumer runCasNoApply = key -> assertRowEquals(cluster, new Object[]{false}, casCQL, key); + String tableName = currentTable.split("\\.")[1]; + + // Mark a subrange as migrating and finish migrating half of it + nodetool(coordinator, "consensus_admin", "begin-migration", "-st", midToken.toString(), "-et", maxToken.toString(), "-tp", "accord", KEYSPACE, tableName); + nodetool(coordinator, "consensus_admin", "finish-migration", "-st", midToken.toString(), "-et", "3074457345618258601"); + nodetool(coordinator, "consensus_admin", "finish-migration", "-st", "3074457345618258601", "-et", upperMidToken.toString()); + Range accordMigratedRange = new Range(midToken, upperMidToken); + Range accordMigratingRange = new Range(upperMidToken, maxToken); + assertMigrationState(tableName, ConsensusMigrationTarget.accord, ImmutableList.of(accordMigratedRange), ImmutableList.of(accordMigratingRange), 1); + + // Test that we can reverse the migration and go back to Paxos + nodetool(coordinator, "consensus_admin", "set-target-protocol", "-tp", "paxos", KEYSPACE, tableName); + assertMigrationState(tableName, ConsensusMigrationTarget.paxos, ImmutableList.of(new Range(minToken, midToken), new Range(maxToken, minToken)), ImmutableList.of(accordMigratingRange), 1); + Iterator paxosNonMigratingKeys = getKeysBetweenTokens(minToken, midToken); + Iterator paxosMigratingKeys = getKeysBetweenTokens(upperMidToken, maxToken); + Iterator accordKeys = getKeysBetweenTokens(midToken, upperMidToken); + + // Paxos non-migrating keys should run on Paxos as per normal + assertTargetPaxosWrite(runCasNoApply, 1, paxosNonMigratingKeys.next(), 0, 1, 0, 0, 0); + + // Paxos migrating keys should be key migrated which means a local barrier is run by Paxos during read at each replica, the key migration barrier is also counted as a write + assertTargetPaxosWrite(runCasNoApply, 1, paxosMigratingKeys.next(), 1, 1, 1, 0, 0); + + // A key from a range migrated to Accord is now not migrating/migrated and should be accessed through Accord + assertTargetPaxosWrite(runCasNoApply, 1, accordKeys.next(), 1, 0, 0, 0, 0); + + // If an Accord transaction races with cluster metadata updates it should be rejected if the epoch it runs in contains the migration + cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new RoutesToAccordOnce())); + assertTargetPaxosWrite(runCasNoApply, 1, paxosMigratingKeys.next(), 2, 1, 1, 1, 1); + + // Repair the currently migrating range from when targets were switched, but it's not an Accord repair, this is to make sure the wrong repair type doesn't trigger progress + nodetool(coordinator, "repair", "-st", upperMidToken.toString(), "-et", maxAlignedWithLocalRanges.toString()); + assertMigrationState(tableName, ConsensusMigrationTarget.paxos, ImmutableList.of(new Range(minToken, midToken), new Range(maxToken, minToken)), ImmutableList.of(accordMigratingRange), 1); + + // Paxos migrating keys should still need key migration after non-Accord repair + assertTargetPaxosWrite(runCasNoApply, 1, paxosMigratingKeys.next(), 1, 1, 1, 0, 0); + + // Now do it with an Accord repair so key migration shouldn't be necessary + nodetool(coordinator, "consensus_admin", "finish-migration", "-st", upperMidToken.toString(), "-et", maxAlignedWithLocalRanges.toString()); + Range repairedRange = new Range(upperMidToken, maxAlignedWithLocalRanges); + // Sliver remaining because of precise repairs + // TODO This precision isn't needed for Accord repair? Worth lifting that restriction or keep it consistent? + Range remainingRange = new Range(maxAlignedWithLocalRanges, maxToken); + assertMigrationState(tableName, ConsensusMigrationTarget.paxos, ImmutableList.of(new Range(minToken, midToken), repairedRange, new Range(maxToken, minToken)), ImmutableList.of(remainingRange), 1); + + // Paxos migrating keys shouldn't need key migration after Accord repair + assertTargetPaxosWrite(runCasNoApply, 1, paxosMigratingKeys.next(), 0, 1, 0, 0, 0); + }); + } + + private static void assertMigrationState(String tableName, ConsensusMigrationTarget target, List> migratedRanges, List> migratingRanges, int numMigratingEpochs) throws Throwable + { + // Validate nodetool consensus admin list output + String yamlResultString = nodetool(SHARED_CLUSTER.coordinator(1), "consensus_admin", "list"); + Map yamlStateMap = new Yaml().load(yamlResultString); + String minifiedYamlResultString = nodetool(SHARED_CLUSTER.coordinator(1), "consensus_admin", "list", "-f", "minified-yaml"); + Map minifiedYamlStateMap = new Yaml().load(minifiedYamlResultString); + String jsonResultString = nodetool(SHARED_CLUSTER.coordinator(1), "consensus_admin", "list", "-f", "json"); + Map jsonStateMap = JsonUtils.JSON_OBJECT_MAPPER.readValue(jsonResultString, new TypeReference>(){}); + String minifiedJsonResultString = nodetool(SHARED_CLUSTER.coordinator(1), "consensus_admin", "list", "-f", "minified-json"); + Map minifiedJsonStateMap = JsonUtils.JSON_OBJECT_MAPPER.readValue(minifiedJsonResultString, new TypeReference>(){}); + + List tableIds = new ArrayList<>(); + for (Map migrationStateMap : ImmutableList.of(yamlStateMap, jsonStateMap, minifiedYamlStateMap, minifiedJsonStateMap)) + { + assertEquals(PojoToString.CURRENT_VERSION, migrationStateMap.get("version")); + assertTrue(Epoch.EMPTY.getEpoch() < ((Number) migrationStateMap.get("lastModifiedEpoch")).longValue()); + List> tableStates = (List>) migrationStateMap.get("tableStates"); + assertEquals(tableStates.size(), 1); + Map tableStateMap = tableStates.get(0); + assertEquals(tableName, tableStateMap.get("table")); + assertEquals(KEYSPACE, tableStateMap.get("keyspace")); + tableIds.add((String) tableStateMap.get("tableId")); + List> migratedRangesFromStateMap = ((List) tableStateMap.get("migratedRanges")).stream().map(Range::fromString).collect(toImmutableList()); + assertEquals(migratedRanges, migratedRangesFromStateMap); + Map>> migratingRangesByEpochFromStateMap = new LinkedHashMap<>(); + for (Map.Entry> entry : ((Map>) tableStateMap.get("migratingRangesByEpoch")).entrySet()) + { + long epoch = entry.getKey() instanceof Number ? ((Number)entry.getKey()).longValue() : Long.valueOf((String)entry.getKey()); + migratingRangesByEpochFromStateMap.put(epoch, entry.getValue().stream().map(Range::fromString).collect(toImmutableList())); + } + if (migratingRanges.isEmpty()) + assertEquals(0, migratingRangesByEpochFromStateMap.size()); + else + assertEquals(migratingRanges, migratingRangesByEpochFromStateMap.values().iterator().next()); + } + + // Also check JSON format at least loads without error + // Validate in memory state at each node + List> migratingAndMigratedRanges = normalize(ImmutableList.>builder().addAll(migratedRanges).addAll(migratingRanges).build()); + spinUntilSuccess(() -> { + for (IInvokableInstance instance : SHARED_CLUSTER) + { + ConsensusMigrationState snapshot = getMigrationStateSnapshot(instance); + assertEquals(1, snapshot.tableStates.size()); + TableMigrationState state = snapshot.tableStates.values().iterator().next(); + assertEquals(KEYSPACE, state.keyspaceName); + assertEquals(tableName, state.tableName); + for (String tableId : tableIds) + assertEquals(tableId, state.tableId.toString()); + assertEquals(target, state.targetProtocol); + assertEquals("Migrated ranges:", migratedRanges, state.migratedRanges); + assertEquals("Migrating ranges:", migratingRanges, state.migratingRanges); + assertEquals("Migrating and migrated ranges:", migratingAndMigratedRanges, state.migratingAndMigratedRanges); + assertEquals(numMigratingEpochs, state.migratingRangesByEpoch.size()); + if (migratingRanges.isEmpty()) + assertEquals(0, state.migratingRangesByEpoch.size()); + else + assertEquals(migratingRanges, state.migratingRangesByEpoch.values().iterator().next()); + } + }); + } + + /** + * Save a promise that is after the committed one to make a subsequent read not linearizable + */ + private static void savePromisedAndCommittedPaxosProposal(String tableName, int key) + { + String committedBallotString = BallotGenerator.Global.nextBallot(Flag.GLOBAL).toString(); + String promisedBallotString = BallotGenerator.Global.nextBallot(Flag.GLOBAL).toString(); + forEach(() -> { + TableMetadata metadata = ColumnFamilyStore.getIfExists(KEYSPACE, tableName).metadata(); + ByteBuffer lowMidMigratingKeyBuffer = ByteBuffer.wrap(ByteArrayUtil.bytes(key)); + DecoratedKey dk = new BufferDecoratedKey(DatabaseDescriptor.getPartitioner().getToken(lowMidMigratingKeyBuffer), lowMidMigratingKeyBuffer); + try (PaxosState state = PaxosState.get(dk, metadata)) + { + Ballot ballot = Ballot.fromString(committedBallotString); + PartitionUpdateBuilder updateBuilder = new PartitionUpdateBuilder(metadata, key); + updateBuilder.row(CLUSTERING_VALUE).add("v", 42); + + state.commit(new Agreed(ballot, updateBuilder.build())); + state.promiseIfNewer(Ballot.fromString(promisedBallotString), true); + } + }); + } + + private static void saveAcceptedPaxosProposal(String tableName, String ballotString, int key) + { + forEach(() -> { + TableMetadata metadata = ColumnFamilyStore.getIfExists(KEYSPACE, tableName).metadata(); + ByteBuffer lowMidMigratingKeyBuffer = ByteBuffer.wrap(ByteArrayUtil.bytes(key)); + DecoratedKey dk = new BufferDecoratedKey(DatabaseDescriptor.getPartitioner().getToken(lowMidMigratingKeyBuffer), lowMidMigratingKeyBuffer); + try (PaxosState state = PaxosState.get(dk, metadata)) + { + Ballot ballot = Ballot.fromString(ballotString); + assertEquals( PROMISE, state.promiseIfNewer(ballot, true).outcome()); + PartitionUpdateBuilder updateBuilder = new PartitionUpdateBuilder(metadata, key); + updateBuilder.row(CLUSTERING_VALUE).add("v", 42); + // Set isForRepair to true to force accepting the proposal for testing purposes + assertEquals( null, state.acceptIfLatest(new Proposal(ballot, updateBuilder.build()), true).supersededBy); + } + }); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java index 9146192ba2..8010a7e581 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java @@ -26,17 +26,19 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import com.google.common.base.Splitter; +import com.google.common.primitives.Ints; +import org.junit.After; import org.junit.AfterClass; import org.junit.Before; -import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import accord.primitives.Txn; +import accord.impl.SimpleProgressLog; import net.bytebuddy.ByteBuddy; import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; import net.bytebuddy.implementation.MethodDelegation; @@ -47,23 +49,31 @@ import org.apache.cassandra.cql3.statements.TransactionStatement; import org.apache.cassandra.cql3.transactions.ReferenceValue; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.Cluster.Builder; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableRunnable; import org.apache.cassandra.distributed.api.QueryResults; import org.apache.cassandra.distributed.api.SimpleQueryResult; +import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.distributed.shared.Metrics; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.distributed.util.QueryResultUtil; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTestUtils; import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException; import org.apache.cassandra.service.accord.exceptions.WritePreemptedException; -import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.FailingConsumer; -import org.apache.cassandra.utils.Shared; import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import static org.junit.Assert.assertArrayEquals; public abstract class AccordTestBase extends TestBaseImpl @@ -71,22 +81,15 @@ public abstract class AccordTestBase extends TestBaseImpl private static final Logger logger = LoggerFactory.getLogger(AccordTestBase.class); private static final int MAX_RETRIES = 10; - @Shared - public static class State - { - public static AtomicInteger coordinateCounts = new AtomicInteger(); - } - protected static final AtomicInteger COUNTER = new AtomicInteger(0); protected static Cluster SHARED_CLUSTER; protected String currentTable; - @BeforeClass - public static void setupClass() throws IOException + public static void setupCluster(Function options, int nodes) throws IOException { - SHARED_CLUSTER = createCluster(); + SHARED_CLUSTER = createCluster(nodes, options); } @AfterClass @@ -102,12 +105,25 @@ public abstract class AccordTestBase extends TestBaseImpl currentTable = KEYSPACE + ".tbl" + COUNTER.getAndIncrement(); } + @After + public void tearDown() throws Exception + { + for (IInvokableInstance instance : SHARED_CLUSTER) + instance.runOnInstance(() -> SimpleProgressLog.PAUSE_FOR_TEST = false); + } + protected static void assertRowSerial(Cluster cluster, String query, int k, int c, int v, int s) { Object[][] result = cluster.coordinator(1).execute(query, ConsistencyLevel.SERIAL); assertArrayEquals(new Object[]{new Object[] {k, c, v, s}}, result); } + protected static void assertRowSerial(Cluster cluster, String query, Object[]... expected) + { + Object[][] result = cluster.coordinator(1).execute(query, ConsistencyLevel.SERIAL); + AssertUtils.assertRows(result, expected); + } + protected void test(String tableDDL, FailingConsumer fn) throws Exception { test(Collections.singletonList(tableDDL), fn); @@ -136,23 +152,123 @@ public abstract class AccordTestBase extends TestBaseImpl test("CREATE TABLE " + currentTable + " (k int, c int, v int, primary key (k, c))", fn); } - protected int getAccordCoordinateCount() + protected static ConsensusMigrationState getMigrationStateSnapshot(IInvokableInstance instance) throws IOException { - return State.coordinateCounts.get(); + byte[] serializedBytes = instance.callOnInstance(() -> { + DataOutputBuffer output = new DataOutputBuffer(); + try + { + ConsensusMigrationState.serializer.serialize( + ClusterMetadata.current().consensusMigrationState, + output, Version.V0); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + return output.toByteArray(); + }); + DataInputPlus input = new DataInputBuffer(serializedBytes); + return ConsensusMigrationState.serializer.deserialize(input, Version.V0); } - private static Cluster createCluster() throws IOException + protected static int getAccordCoordinateCount() + { + return getAccordWriteCount() + getAccordReadCount(); + } + + protected static int getCasWriteCount(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.Latency.CASWrite")); + } + + protected static int getCasPrepareCount(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.keyspace.CasPrepareLatency.distributed_test_keyspace")); + } + + protected static int getAccordWriteCount() + { + return getAccordWriteCount(1); + } + + protected static int getAccordWriteCount(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.Latency.AccordWrite")); + } + + protected static int getAccordReadCount() + { + return getAccordReadCount(1); + } + + protected static int getAccordReadCount(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.Latency.AccordRead")); + } + + protected static int getAccordMigrationRejects(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.AccordMigrationRejects.AccordWrite")); + } + + protected static int getAccordMigrationSkippedReads() + { + // Skipped reads can occur at any node so sum them + long sum = 0; + for (IInvokableInstance instance : SHARED_CLUSTER) + sum += instance.metrics().getCounter("org.apache.cassandra.metrics.ClientRequest.MigrationSkippedReads.AccordWrite"); + return Ints.checkedCast(sum); + } + + protected static int getKeyMigrationCount(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.Table.KeyMigrationLatency.all")); + } + + protected static int getCasWriteBeginRejects(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.PaxosBeginMigrationRejects.CASWrite")); + } + + protected static int getCasReadBeginRejects(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.PaxosBeginMigrationRejects.CASRead")); + } + + protected static int getCasWriteAcceptRejects(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.PaxosAcceptMigrationRejects.CASWrite")); + } + + protected static int getCasReadAcceptRejects(int coordinatorIndex) + { + return Ints.checkedCast(getMetrics(coordinatorIndex).getCounter("org.apache.cassandra.metrics.ClientRequest.PaxosAcceptMigrationRejects.CASRead")); + } + + protected static Metrics getMetrics(int coordinatorIndex) + { + return SHARED_CLUSTER.get(coordinatorIndex).metrics(); + } + + protected static void forEach(SerializableRunnable runnable) + { + for (IInvokableInstance instance : SHARED_CLUSTER) + instance.runOnInstance(runnable); + } + + private static Cluster createCluster(int nodes, Function options) throws IOException { // need to up the timeout else tests get flaky // disable vnode for now, but should enable before trunk - return init(Cluster.build(2) + Cluster.Builder builder = Cluster.build(nodes) .withoutVNodes() .withConfig(c -> c.with(Feature.NETWORK, Feature.GOSSIP).set("write_request_timeout", "10s") .set("transaction_timeout", "15s") - .set("legacy_paxos_strategy", "migration")) // TODO: switch back to "accord" when TrM integration works - .withInstanceInitializer(EnforceUpdateDoesNotPerformRead::install) - .withInstanceInitializer(BBAccordCoordinateCountHelper::install) - .start()); + .set("transaction_timeout", "15s")) + .withInstanceInitializer(EnforceUpdateDoesNotPerformRead::install); + builder = options.apply(builder); + return init(builder.start()); } protected static SimpleQueryResult executeAsTxn(Cluster cluster, String check, Object... boundValues) @@ -168,6 +284,18 @@ public abstract class AccordTestBase extends TestBaseImpl return cluster.coordinator(1).executeWithResult(check, ConsistencyLevel.ANY, boundValues); } + private static SimpleQueryResult execute(Cluster cluster, String check, ConsistencyLevel cl, Object... boundValues) + { + return cluster.coordinator(1).executeWithResult(check, cl, boundValues); + } + + protected static SimpleQueryResult assertRowEquals(Cluster cluster, SimpleQueryResult expected, String check, ConsistencyLevel cl, Object... boundValues) + { + SimpleQueryResult result = execute(cluster, check, cl, boundValues); + QueryResultUtil.assertThat(result).isEqualTo(expected); + return result; + } + protected static SimpleQueryResult assertRowEquals(Cluster cluster, SimpleQueryResult expected, String check, Object... boundValues) { SimpleQueryResult result = execute(cluster, check, boundValues); @@ -175,6 +303,11 @@ public abstract class AccordTestBase extends TestBaseImpl return result; } + protected static SimpleQueryResult assertRowEquals(Cluster cluster, Object[] row, String check, ConsistencyLevel cl, Object... boundValues) + { + return assertRowEquals(cluster, QueryResults.builder().row(row).build(), check, cl, boundValues); + } + protected static SimpleQueryResult assertRowEquals(Cluster cluster, Object[] row, String check, Object... boundValues) { return assertRowEquals(cluster, QueryResults.builder().row(row).build(), check, boundValues); @@ -308,25 +441,5 @@ public abstract class AccordTestBase extends TestBaseImpl } } - public static class BBAccordCoordinateCountHelper - { - static void install(ClassLoader cl, int nodeNumber) - { - if (nodeNumber != 1) - return; - new ByteBuddy().rebase(AccordService.class) - .method(named("coordinate").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBAccordCoordinateCountHelper.class)) - .make() - .load(cl, ClassLoadingStrategy.Default.INJECTION); - } - - public static TxnData coordinate(Txn txn, @SuperCall Callable actual) throws Exception - { - State.coordinateCounts.incrementAndGet(); - return actual.call(); - } - } - protected abstract Logger logger(); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java index 0b1cee3e0a..084709e496 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java @@ -18,9 +18,11 @@ package org.apache.cassandra.distributed.test.accord; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; +import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,6 +30,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.distributed.api.SimpleQueryResult; import org.apache.cassandra.service.accord.AccordService; +import static java.util.function.UnaryOperator.identity; + public class NewSchemaTest extends AccordTestBase { private static final Logger logger = LoggerFactory.getLogger(NewSchemaTest.class); @@ -38,6 +42,12 @@ public class NewSchemaTest extends AccordTestBase return logger; } + @BeforeClass + public static void setupClass() throws IOException + { + AccordTestBase.setupCluster(identity(), 2); + } + @Test public void test() { diff --git a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailCollectionSizeOnSSTableWriteTest.java b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailCollectionSizeOnSSTableWriteTest.java index 6b7daef26f..3405185d45 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailCollectionSizeOnSSTableWriteTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailCollectionSizeOnSSTableWriteTest.java @@ -26,9 +26,12 @@ import org.junit.Test; import com.datastax.driver.core.Session; import com.datastax.driver.core.SimpleStatement; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.guardrails.Guardrails; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.IInvokableInstance; import static java.nio.ByteBuffer.allocate; @@ -57,7 +60,6 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester .set("collection_size_warn_threshold", WARN_THRESHOLD + "B") .set("collection_size_fail_threshold", FAIL_THRESHOLD + "B")) .start()); - cluster.disableAutoCompaction(KEYSPACE); driverCluster = com.datastax.driver.core.Cluster.builder().addContactPoint("127.0.0.1").build(); driverSession = driverCluster.connect(); } @@ -84,7 +86,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testSetSize() throws Throwable { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v set)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v set)"); execute("INSERT INTO %s (k, v) VALUES (0, null)"); execute("INSERT INTO %s (k, v) VALUES (1, ?)", set()); @@ -108,7 +110,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testSetSizeFrozen() { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v frozen>)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen>)"); execute("INSERT INTO %s (k, v) VALUES (0, null)"); execute("INSERT INTO %s (k, v) VALUES (1, ?)", set()); @@ -123,7 +125,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testSetSizeWithUpdates() { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v set)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v set)"); execute("INSERT INTO %s (k, v) VALUES (0, ?)", set(allocate(1))); execute("UPDATE %s SET v = v + ? WHERE k = 0", set(allocate(1))); @@ -145,7 +147,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testSetSizeAfterCompaction() throws Throwable { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v set)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v set)"); execute("INSERT INTO %s (k, v) VALUES (0, ?)", set(allocate(1))); assertNotWarnedOnFlush(); @@ -175,7 +177,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testListSize() throws Throwable { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v list)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v list)"); execute("INSERT INTO %s (k, v) VALUES (0, null)"); execute("INSERT INTO %s (k, v) VALUES (1, ?)", list()); @@ -199,7 +201,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testListSizeFrozen() { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v frozen>)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen>)"); execute("INSERT INTO %s (k, v) VALUES (0, null)"); execute("INSERT INTO %s (k, v) VALUES (1, ?)", list()); @@ -214,7 +216,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testListSizeWithUpdates() { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v list)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v list)"); execute("INSERT INTO %s (k, v) VALUES (0, ?)", list(allocate(1))); execute("UPDATE %s SET v = v + ? WHERE k = 0", list(allocate(1))); @@ -236,7 +238,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testListSizeAfterCompaction() throws Throwable { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v list)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v list)"); execute("INSERT INTO %s (k, v) VALUES (0, ?)", list(allocate(1))); assertNotWarnedOnFlush(); @@ -266,7 +268,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testMapSize() throws Throwable { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v map)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v map)"); execute("INSERT INTO %s (k, v) VALUES (0, null)"); execute("INSERT INTO %s (k, v) VALUES (1, ?)", map()); @@ -297,7 +299,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testMapSizeFrozen() { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v frozen>)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen>)"); execute("INSERT INTO %s (k, v) VALUES (0, null)"); execute("INSERT INTO %s (k, v) VALUES (1, ?)", map()); @@ -316,7 +318,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testMapSizeWithUpdates() { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v map)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v map)"); execute("INSERT INTO %s (k, v) VALUES (0, ?)", map(allocate(1), allocate(1))); execute("UPDATE %s SET v = v + ? WHERE k = 0", map(allocate(1), allocate(1))); @@ -350,7 +352,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testMapSizeAfterCompaction() { - schemaChange("CREATE TABLE %s (k int PRIMARY KEY, v map)"); + createTable("CREATE TABLE %s (k int PRIMARY KEY, v map)"); execute("INSERT INTO %s (k, v) VALUES (0, ?)", map(allocate(1), allocate(1))); execute("UPDATE %s SET v = v + ? WHERE k = 0", map(allocate(1), allocate(1))); @@ -397,7 +399,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testCompositePartitionKey() { - schemaChange("CREATE TABLE %s (k1 int, k2 text, v set, PRIMARY KEY((k1, k2)))"); + createTable("CREATE TABLE %s (k1 int, k2 text, v set, PRIMARY KEY((k1, k2)))"); execute("INSERT INTO %s (k1, k2, v) VALUES (0, 'a', ?)", set(allocate(WARN_THRESHOLD))); assertWarnedOnFlush(warnMessage("(0, 'a')")); @@ -409,7 +411,7 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester @Test public void testCompositeClusteringKey() { - schemaChange("CREATE TABLE %s (k int, c1 int, c2 text, v set, PRIMARY KEY(k, c1, c2))"); + createTable("CREATE TABLE %s (k int, c1 int, c2 text, v set, PRIMARY KEY(k, c1, c2))"); execute("INSERT INTO %s (k, c1, c2, v) VALUES (1, 10, 'a', ?)", set(allocate(WARN_THRESHOLD))); assertWarnedOnFlush(warnMessage("(1, 10, 'a')")); @@ -434,4 +436,16 @@ public class GuardrailCollectionSizeOnSSTableWriteTest extends GuardrailTester { return String.format("Detected collection v in row %s in table %s of size", key, qualifiedTableName); } + + private void createTable(String cql) + { + schemaChange(cql); + for (IInvokableInstance instance : cluster) + { + instance.runOnInstance(() -> { + for (ColumnFamilyStore cs : Keyspace.open(KEYSPACE).getColumnFamilyStores()) + cs.disableAutoCompaction(); + }); + } + } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java index 33402af732..de4b6541ed 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java @@ -152,6 +152,7 @@ public class ClusterMetadataTestHelper AccordKeyspaces.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, + null, ImmutableMap.of()); } @@ -166,6 +167,7 @@ public class ClusterMetadataTestHelper AccordKeyspaces.EMPTY, null, null, + null, ImmutableMap.of()); } @@ -180,6 +182,7 @@ public class ClusterMetadataTestHelper AccordKeyspaces.EMPTY, null, null, + null, ImmutableMap.of()); } diff --git a/test/long/org/apache/cassandra/dht/tokenallocator/Murmur3ReplicationAwareTokenAllocatorTest.java b/test/long/org/apache/cassandra/dht/tokenallocator/Murmur3ReplicationAwareTokenAllocatorTest.java index e28ecfa453..83665f1c9b 100644 --- a/test/long/org/apache/cassandra/dht/tokenallocator/Murmur3ReplicationAwareTokenAllocatorTest.java +++ b/test/long/org/apache/cassandra/dht/tokenallocator/Murmur3ReplicationAwareTokenAllocatorTest.java @@ -30,7 +30,7 @@ public class Murmur3ReplicationAwareTokenAllocatorTest extends AbstractReplicati @Test public void testExistingCluster() { - super.testExistingCluster(new Murmur3Partitioner(), MAX_VNODE_COUNT); + super.testExistingCluster(Murmur3Partitioner.instance, MAX_VNODE_COUNT); } @Test @@ -43,6 +43,6 @@ public class Murmur3ReplicationAwareTokenAllocatorTest extends AbstractReplicati private void flakyTestNewCluster() { - testNewCluster(new Murmur3Partitioner(), MAX_VNODE_COUNT); + testNewCluster(Murmur3Partitioner.instance, MAX_VNODE_COUNT); } } diff --git a/test/long/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorTest.java b/test/long/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorTest.java index 5e13519fcd..6835f2c5b0 100644 --- a/test/long/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorTest.java +++ b/test/long/org/apache/cassandra/dht/tokenallocator/NoReplicationTokenAllocatorTest.java @@ -26,9 +26,9 @@ import java.util.Random; import com.google.common.collect.Maps; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; +import org.junit.Assert; import org.junit.Test; -import org.junit.Assert; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.RandomPartitioner; @@ -42,13 +42,13 @@ public class NoReplicationTokenAllocatorTest extends TokenAllocatorTestBase @Test public void testNewClusterWithMurmur3Partitioner() { - testNewCluster(new Murmur3Partitioner()); + testNewCluster(Murmur3Partitioner.instance); } @Test public void testNewClusterWithRandomPartitioner() { - testNewCluster(new RandomPartitioner()); + testNewCluster(RandomPartitioner.instance); } private void testNewCluster(IPartitioner partitioner) @@ -75,13 +75,13 @@ public class NoReplicationTokenAllocatorTest extends TokenAllocatorTestBase @Test public void testExistingClusterWithMurmur3Partitioner() { - testExistingCluster(new Murmur3Partitioner()); + testExistingCluster(Murmur3Partitioner.instance); } @Test public void testExistingClusterWithRandomPartitioner() { - testExistingCluster(new RandomPartitioner()); + testExistingCluster(RandomPartitioner.instance); } private void testExistingCluster(IPartitioner partitioner) diff --git a/test/long/org/apache/cassandra/dht/tokenallocator/RandomReplicationAwareTokenAllocatorTest.java b/test/long/org/apache/cassandra/dht/tokenallocator/RandomReplicationAwareTokenAllocatorTest.java index bb1a2c8f3c..4e7982e0c7 100644 --- a/test/long/org/apache/cassandra/dht/tokenallocator/RandomReplicationAwareTokenAllocatorTest.java +++ b/test/long/org/apache/cassandra/dht/tokenallocator/RandomReplicationAwareTokenAllocatorTest.java @@ -34,13 +34,13 @@ public class RandomReplicationAwareTokenAllocatorTest extends AbstractReplicatio @Test public void testExistingCluster() { - testExistingCluster(new RandomPartitioner(), MAX_VNODE_COUNT); + testExistingCluster(RandomPartitioner.instance, MAX_VNODE_COUNT); } @Test public void testNewClusterr() { - testNewCluster(new RandomPartitioner(), MAX_VNODE_COUNT); + testNewCluster(RandomPartitioner.instance, MAX_VNODE_COUNT); } } diff --git a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBench.java b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBench.java index e10b91d9b2..07ff3edce9 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBench.java @@ -78,6 +78,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; +import static java.util.Collections.emptyList; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; /** @@ -130,14 +131,14 @@ public class ZeroCopyStreamingBench serializedBlockStream = blockStreamCaptureChannel.getSerializedStream(); out.close(); - session.prepareReceiving(new StreamSummary(sstable.metadata().id, 1, serializedBlockStream.readableBytes())); + session.prepareReceiving(new StreamSummary(sstable.metadata().id, emptyList(), 1, serializedBlockStream.readableBytes())); CassandraStreamHeader entireSSTableStreamHeader = CassandraStreamHeader.builder() .withSSTableVersion(sstable.descriptor.version) .withSSTableLevel(0) .withEstimatedKeys(sstable.estimatedKeys()) - .withSections(Collections.emptyList()) + .withSections(emptyList()) .withSerializationHeader(sstable.header.toComponent()) .withComponentManifest(context.manifest()) .isEntireSSTable(true) @@ -219,7 +220,7 @@ public class ZeroCopyStreamingBench StreamResultFuture future = StreamResultFuture.createInitiator(nextTimeUUID(), StreamOperation.BOOTSTRAP, Collections.emptyList(), streamCoordinator); InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort(); - streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED, null)); + streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, emptyList(), emptyList(), StreamSession.State.INITIALIZED, null)); StreamSession session = streamCoordinator.getOrCreateOutboundSession(peer); session.init(future); diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java b/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java index 7083832c01..a2a86a9f65 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/btree/AtomicBTreePartitionUpdateBench.java @@ -68,11 +68,11 @@ import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.BulkIterator; import org.apache.cassandra.utils.btree.BTree; import org.apache.cassandra.utils.btree.UpdateFunction; import org.apache.cassandra.utils.concurrent.ImmediateFuture; import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.BulkIterator; import org.apache.cassandra.utils.memory.ByteBufferCloner; import org.apache.cassandra.utils.memory.Cloner; import org.apache.cassandra.utils.memory.HeapPool; @@ -107,7 +107,7 @@ public class AtomicBTreePartitionUpdateBench private static final MutableDeletionInfo NO_DELETION_INFO = new MutableDeletionInfo(DeletionTime.LIVE); private static final HeapPool POOL = new HeapPool(Long.MAX_VALUE, 1.0f, () -> ImmediateFuture.success(Boolean.TRUE)); private static final ByteBuffer zero = Int32Type.instance.decompose(0); - private static final DecoratedKey decoratedKey = new BufferDecoratedKey(new ByteOrderedPartitioner().getToken(zero), zero); + private static final DecoratedKey decoratedKey = new BufferDecoratedKey(ByteOrderedPartitioner.instance.getToken(zero), zero); static { diff --git a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java index fac4977076..4b71603565 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java +++ b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java @@ -67,6 +67,7 @@ import org.apache.cassandra.simulator.asm.DeterministicChanceSupplier; import org.apache.cassandra.simulator.asm.InterceptAsClassTransformer; import org.apache.cassandra.simulator.asm.NemesisFieldSelectors; import org.apache.cassandra.simulator.cluster.ClusterActions; +import org.apache.cassandra.simulator.cluster.ClusterActions.ConsensusChange; import org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange; import org.apache.cassandra.simulator.systems.Failures; import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites.Capture; @@ -151,6 +152,9 @@ public class ClusterSimulation implements AutoCloseable protected TopologyChange[] topologyChanges = TopologyChange.values(); protected int topologyChangeLimit = -1; + protected ConsensusChange[] consensusChanges = ConsensusChange.values(); + protected int consensusChangeLimit = -1; + protected int primaryKeyCount; protected int secondsToSimulate; @@ -176,7 +180,8 @@ public class ClusterSimulation implements AutoCloseable schedulerLongDelayNanos = new LongRange(50, 5000, MICROSECONDS, NANOSECONDS), clockDriftNanos = new LongRange(1, 5000, MILLISECONDS, NANOSECONDS), clockDiscontinuitIntervalNanos = new LongRange(10, 60, SECONDS, NANOSECONDS), - topologyChangeIntervalNanos = new LongRange(5, 15, SECONDS, NANOSECONDS); + topologyChangeIntervalNanos = new LongRange(5, 15, SECONDS, NANOSECONDS), + consensusChangeIntervalNanos = new LongRange(1, 5, SECONDS, NANOSECONDS); @@ -193,6 +198,7 @@ public class ClusterSimulation implements AutoCloseable protected HeapPool.Logged.Listener memoryListener; protected SimulatedTime.Listener timeListener = (i1, i2) -> {}; protected LongConsumer onThreadLocalRandomCheck; + protected String lwtStrategy = "migration"; public Builder failures(Failures failures) { @@ -312,6 +318,24 @@ public class ClusterSimulation implements AutoCloseable return this; } + public Builder consensusChanges(ConsensusChange[] consensusChanges) + { + this.consensusChanges = consensusChanges; + return this; + } + + public Builder consensusChangeIntervalNanos(LongRange consensusChangeIntervalNanos) + { + this.consensusChangeIntervalNanos = consensusChangeIntervalNanos; + return this; + } + + public Builder consensusChangeLimit(int consensusChangeLimit) + { + this.consensusChangeLimit = consensusChangeLimit; + return this; + } + public int primaryKeyCount() { return primaryKeyCount; @@ -551,6 +575,12 @@ public class ClusterSimulation implements AutoCloseable return this; } + public Builder lwtStrategy(String strategy) + { + this.lwtStrategy = strategy; + return this; + } + public abstract ClusterSimulation create(long seed) throws IOException; } @@ -840,6 +870,8 @@ public class ClusterSimulation implements AutoCloseable scheduler = builder.schedulerFactory.create(random); options = new ClusterActions.Options(builder.topologyChangeLimit, Choices.uniform(KindOfSequence.values()).choose(random).period(builder.topologyChangeIntervalNanos, random), Choices.random(random, builder.topologyChanges), + builder.consensusChangeLimit, Choices.uniform(KindOfSequence.values()).choose(random).period(builder.consensusChangeIntervalNanos, random), + Choices.random(random, builder.consensusChanges), minRf, initialRf, maxRf, null); this.factory = factory; } diff --git a/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java b/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java index 61b15dd40e..511f443ceb 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java +++ b/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java @@ -41,6 +41,7 @@ import io.netty.util.concurrent.FastThreadLocal; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.simulator.Debug.Info; import org.apache.cassandra.simulator.Debug.Levels; +import org.apache.cassandra.simulator.cluster.ClusterActions.ConsensusChange; import org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange; import org.apache.cassandra.simulator.debug.SelfReconcile; import org.apache.cassandra.simulator.logging.SeedDefiner; @@ -61,9 +62,10 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_GLOB import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_APPROX; import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_MONOTONIC_PRECISE; import static org.apache.cassandra.config.CassandraRelevantProperties.CONSISTENT_DIRECTORY_LISTINGS; -import static org.apache.cassandra.config.CassandraRelevantProperties.DETERMINISM_UNSAFE_UUID_NODE; -import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_SSTABLE_ACTIVITY_TRACKING; import static org.apache.cassandra.config.CassandraRelevantProperties.DETERMINISM_SSTABLE_COMPRESSION_DEFAULT; +import static org.apache.cassandra.config.CassandraRelevantProperties.DETERMINISM_UNSAFE_UUID_NODE; +import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_GOSSIP_ENDPOINT_REMOVAL; +import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_SSTABLE_ACTIVITY_TRACKING; import static org.apache.cassandra.config.CassandraRelevantProperties.DTEST_API_LOG_TOPOLOGY; import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIPER_SKIP_WAITING_TO_SETTLE; import static org.apache.cassandra.config.CassandraRelevantProperties.IGNORE_MISSING_NATIVE_FILE_HINTS; @@ -74,7 +76,6 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPA import static org.apache.cassandra.config.CassandraRelevantProperties.RING_DELAY; import static org.apache.cassandra.config.CassandraRelevantProperties.SHUTDOWN_ANNOUNCE_DELAY_IN_MS; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_AUTH_DEFAULT_RF; -import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_GOSSIP_ENDPOINT_REMOVAL; import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_JVM_DTEST_DISABLE_SSL; import static org.apache.cassandra.simulator.debug.Reconcile.reconcileWith; import static org.apache.cassandra.simulator.debug.Record.record; @@ -178,9 +179,16 @@ public class SimulationRunner protected String topologyChanges = stream(TopologyChange.values()).map(Object::toString).collect(Collectors.joining(",")); @Option(name = { "--cluster-action-interval" }, title = "int...int(s|ms|us|ns)", description = "The period of time between two cluster actions (default 5..15s)") protected String topologyChangeInterval = "5..15s"; - @Option(name = { "--cluster-action-limit" }, title = "int", description = "The maximum number of topology change events to perform (default 0)") + @Option(name = { "--cluster-action-limit" }, title = "int", description = "The maximum number of topology change events to perform (default 0 disabled, -1 unlimited)") protected String topologyChangeLimit = "0"; + @Option(name = { "--consensus-actions" }, title = "ACCORD_MIGRATE", description = "Consensus migration actions to select from, comma delimited (ACCORD_MIGRATE)") + protected String consensusChanges = stream(ConsensusChange.values()).map(Object::toString).collect(Collectors.joining(",")); + @Option(name = { "--consensus-action-interval" }, title = "int...int(s|ms|us|ns)", description = "The period of time between two consensus actions (default 5..15s)") + protected String consensusChangeInterval = "1..5s"; + @Option(name = { "--consensus-action-limit" }, title = "int", description = "The maximum number of consensus change events to perform (default 0 disabled, -1 unlimited)") + protected String consensusChangeLimit = "0"; + @Option(name = { "-s", "--run-time" }, title = "int", description = "Length of simulated time to run in seconds (default -1)") protected int secondsToSimulate = -1; @@ -256,6 +264,9 @@ public class SimulationRunner @Option(name = { "--capture" }, title = "wait,wake,now", description = "Capture thread stack traces alongside events, choose from (wait,wake,now)") protected String capture; + @Option(name = { "--lwt-strategy" }, title = "migration|accord]", description = "What execution strategy to use for CAS and serial read") + protected String lwtStrategy; + protected void propagate(B builder) { builder.threadCount(threadCount); @@ -294,13 +305,14 @@ public class SimulationRunner }); parseNanosRange(Optional.ofNullable(topologyChangeInterval)).ifPresent(builder::topologyChangeIntervalNanos); builder.topologyChangeLimit(Integer.parseInt(topologyChangeLimit)); - Optional.ofNullable(priority).ifPresent(kinds -> { - builder.scheduler(stream(kinds.split(",")) - .filter(v -> !v.isEmpty()) - .map(v -> RunnableActionScheduler.Kind.valueOf(toUpperCaseLocalized(v))) - .toArray(RunnableActionScheduler.Kind[]::new)); + Optional.ofNullable(consensusChanges).ifPresent(consensusChanges -> { + builder.consensusChanges(stream(consensusChanges.split(",")) + .filter(v -> !v.isEmpty()) + .map(v -> ConsensusChange.valueOf(toUpperCaseLocalized(v))) + .toArray(ConsensusChange[]::new)); }); - + parseNanosRange(Optional.ofNullable(consensusChangeInterval)).ifPresent(builder::consensusChangeIntervalNanos); + builder.consensusChangeLimit(Integer.parseInt(consensusChangeLimit)); Optional.ofNullable(this.capture) .map(s -> s.split(",")) .map(s -> new Capture( @@ -324,6 +336,8 @@ public class SimulationRunner .orElse(new int[0]); builder.debug(debugLevels, debugPrimaryKeys); } + + Optional.ofNullable(lwtStrategy).ifPresent(builder::lwtStrategy); } public void run(B builder) throws IOException diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActions.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActions.java index 60b6a62e57..ea7256247f 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActions.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActions.java @@ -85,6 +85,11 @@ public class ClusterActions extends SimulatedSystems JOIN, LEAVE, REPLACE, CHANGE_RF } + public enum ConsensusChange + { + ACCORD_MIGRATE + } + public static class Options { public final int topologyChangeLimit; @@ -92,6 +97,9 @@ public class ClusterActions extends SimulatedSystems public final Choices allChoices; public final Choices choicesNoLeave; public final Choices choicesNoJoin; + public final int consensusChangeLimit; + public final KindOfSequence.Period consensusChangeInterval; + public final Choices consensusChoices; public final int[] minRf, initialRf, maxRf; public final PaxosVariant changePaxosVariantTo; @@ -108,32 +116,45 @@ public class ClusterActions extends SimulatedSystems this.allChoices = copy.allChoices; this.choicesNoLeave = copy.choicesNoLeave; this.choicesNoJoin = copy.choicesNoJoin; + this.consensusChangeLimit = copy.consensusChangeLimit; + this.consensusChangeInterval = copy.consensusChangeInterval; + this.consensusChoices = copy.consensusChoices; this.minRf = copy.minRf; this.initialRf = copy.initialRf; this.maxRf = copy.maxRf; this.changePaxosVariantTo = changePaxosVariantTo; } - public Options(int topologyChangeLimit, KindOfSequence.Period topologyChangeInterval, Choices choices, int[] minRf, int[] initialRf, int[] maxRf, PaxosVariant changePaxosVariantTo) + public Options(int topologyChangeLimit, + KindOfSequence.Period topologyChangeInterval, + Choices topologyChangeChoices, + int consensusChangeLimit, + KindOfSequence.Period consensusChangeInterval, + Choices consensusChangeChoices, + int[] minRf, int[] initialRf, int[] maxRf, + PaxosVariant changePaxosVariantTo) { if (Arrays.equals(minRf, maxRf)) - choices = choices.without(TopologyChange.CHANGE_RF); + topologyChangeChoices = topologyChangeChoices.without(TopologyChange.CHANGE_RF); this.topologyChangeInterval = topologyChangeInterval; this.topologyChangeLimit = topologyChangeLimit; + this.consensusChangeInterval = consensusChangeInterval; + this.consensusChangeLimit = consensusChangeLimit; this.minRf = minRf; this.initialRf = initialRf; this.maxRf = maxRf; - this.allChoices = choices; + this.allChoices = topologyChangeChoices; this.choicesNoJoin = allChoices.without(JOIN).without(REPLACE); this.choicesNoLeave = allChoices.without(LEAVE); + this.consensusChoices = consensusChangeChoices; this.changePaxosVariantTo = changePaxosVariantTo; } public static Options noActions(int clusterSize) { int[] rf = new int[]{clusterSize}; - return new Options(0, UNIFORM.period(null, null), Choices.uniform(), rf, rf, rf, null); + return new Options(0, UNIFORM.period(null, null), Choices.uniform(), 0, UNIFORM.period(null, null), Choices.uniform(), rf, rf, rf, null); } public Options changePaxosVariantTo(PaxosVariant newVariant) diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java index 1e8656dd2a..6ddf61109d 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java @@ -44,6 +44,7 @@ import org.apache.cassandra.simulator.OrderOn.StrictSequential; import org.apache.cassandra.simulator.systems.InterceptedExecution; import org.apache.cassandra.simulator.systems.InterceptingExecutor; import org.apache.cassandra.simulator.systems.SimulatedSystems; +import org.apache.cassandra.simulator.utils.KindOfSequence; import org.apache.cassandra.tcm.ClusterMetadataService; import static java.util.Collections.singletonList; @@ -62,7 +63,8 @@ public class KeyspaceActions extends ClusterActions final ConsistencyLevel serialConsistency; final int[] primaryKeys; - final EnumSet ops = EnumSet.noneOf(TopologyChange.class); + final EnumSet topologyOps = EnumSet.noneOf(TopologyChange.class); + final EnumSet consensusOps = EnumSet.noneOf(ConsensusChange.class); final NodeLookup nodeLookup; final TokenPlacementModel.NodeFactory factory; final int[] minRf, initialRf, maxRf; @@ -77,7 +79,9 @@ public class KeyspaceActions extends ClusterActions final int[] currentRf; Topology topology; boolean haveChangedVariant; + boolean haveConsensusMigrated; int topologyChangeCount = 0; + int consensusChangeCount = 0; public KeyspaceActions(SimulatedSystems simulated, String keyspace, String table, String createTableCql, @@ -118,7 +122,8 @@ public class KeyspaceActions extends ClusterActions maxRf = options.maxRf; currentRf = initialRf.clone(); membersOfQuorumDcs = serialConsistency == LOCAL_SERIAL ? all.dcs[0] : all.toArray(); - ops.addAll(Arrays.asList(options.allChoices.options)); + topologyOps.addAll(Arrays.asList(options.allChoices.options)); + consensusOps.addAll(Arrays.asList(options.consensusChoices.options)); } public ActionPlan plan(boolean joinAll) @@ -214,7 +219,7 @@ public class KeyspaceActions extends ClusterActions for (int i = 0 ; i < primaryKeys.length ; ++i) { int primaryKey = primaryKeys[i]; - LongToken token = new Murmur3Partitioner().getToken(Int32Type.instance.decompose(primaryKey)); + LongToken token = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(primaryKey)); List readReplicas = readPlacements.replicasFor(token.token); List writeReplicas = writePlacements.replicasFor(token.token); @@ -234,15 +239,53 @@ public class KeyspaceActions extends ClusterActions private Action next() { - if (options.topologyChangeLimit >= 0 && topologyChangeCount++ > options.topologyChangeLimit) + Action nextTopologyChangeAction = nextTopologyChangeAction(); + if (nextTopologyChangeAction != null) + return nextTopologyChangeAction; + + Action nextConsensusChangeAction = nextConsensusChangeAction(); + if (nextConsensusChangeAction != null) + return nextConsensusChangeAction; + + if (options.changePaxosVariantTo != null && !haveChangedVariant) + { + haveChangedVariant = true; + return schedule(new OnClusterSetPaxosVariant(KeyspaceActions.this, options.changePaxosVariantTo), options.topologyChangeInterval); + } + + return null; + } + + private Action nextConsensusChangeAction() + { + if (options.consensusChangeLimit >= 0 && ++consensusChangeCount > options.consensusChangeLimit) return null; - while (!ops.isEmpty() && (!registered.isEmpty() || joined.size() > sum(minRf))) + while (!consensusOps.isEmpty() && !haveConsensusMigrated) + { + ConsensusChange nextChange = options.consensusChoices.choose(random); + switch (nextChange) + { + case ACCORD_MIGRATE: + haveConsensusMigrated = true; + return schedule(new OnClusterMigrateConsensus(this), options.topologyChangeInterval); + } + } + + return null; + } + + private Action nextTopologyChangeAction() + { + if (options.topologyChangeLimit >= 0 && ++topologyChangeCount > options.topologyChangeLimit) + return null; + + while (!topologyOps.isEmpty() && (!registered.isEmpty() || joined.size() > sum(minRf))) { if (options.changePaxosVariantTo != null && !haveChangedVariant && random.decide(1f / (1 + registered.size()))) { haveChangedVariant = true; - return schedule(new OnClusterSetPaxosVariant(KeyspaceActions.this, options.changePaxosVariantTo)); + return schedule(new OnClusterSetPaxosVariant(KeyspaceActions.this, options.changePaxosVariantTo), options.topologyChangeInterval); } // pick a dc @@ -251,8 +294,8 @@ public class KeyspaceActions extends ClusterActions // try to pick an action (and simply loop again if we cannot for this dc) TopologyChange next; if (registered.size(dc) > 0 && joined.size(dc) > currentRf[dc]) next = options.allChoices.choose(random); - else if (registered.size(dc) > 0 && ops.contains(JOIN)) next = options.choicesNoLeave.choose(random); - else if (joined.size(dc) > currentRf[dc] && ops.contains(LEAVE)) next = options.choicesNoJoin.choose(random); + else if (registered.size(dc) > 0 && topologyOps.contains(JOIN)) next = options.choicesNoLeave.choose(random); + else if (joined.size(dc) > currentRf[dc] && topologyOps.contains(LEAVE)) next = options.choicesNoJoin.choose(random); else if (joined.size(dc) > minRf[dc]) next = CHANGE_RF; else continue; @@ -330,19 +373,12 @@ public class KeyspaceActions extends ClusterActions } } } - - if (options.changePaxosVariantTo != null && !haveChangedVariant) - { - haveChangedVariant = true; - return schedule(new OnClusterSetPaxosVariant(KeyspaceActions.this, options.changePaxosVariantTo)); - } - return null; } - private Action schedule(Action action) + private Action schedule(Action action, KindOfSequence.Period period) { - action.setDeadline(time, time.nanoTime() + options.topologyChangeInterval.get(random)); + action.setDeadline(time, time.nanoTime() + period.get(random)); return action; } @@ -364,7 +400,7 @@ public class KeyspaceActions extends ClusterActions time.permitDiscontinuities(); } }); - return schedule(action); + return schedule(action, options.topologyChangeInterval); } void updateTopology(Topology newTopology) diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterMigrateConsensus.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterMigrateConsensus.java new file mode 100644 index 0000000000..d4c4cb87d5 --- /dev/null +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterMigrateConsensus.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.simulator.cluster; + +import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.harry.model.TokenPlacementModel; +import org.apache.cassandra.simulator.Action; +import org.apache.cassandra.simulator.ActionList; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.ownership.TokenMap; +import org.apache.cassandra.utils.Pair; + +import static com.google.common.base.Preconditions.checkState; +import static org.apache.cassandra.simulator.Action.Modifiers.NONE; + +class OnClusterMigrateConsensus extends Action +{ + private final KeyspaceActions actions; + + OnClusterMigrateConsensus(KeyspaceActions actions) + { + super("Performing consensus migration", NONE, NONE); + this.actions = actions; + } + + public ActionList performSimple() + { + List result = new ArrayList<>(); + List>> ranges = new ArrayList<>(); + ClusterMetadata cm = ClusterMetadata.current(); + TokenMap tm = cm.tokenMap; + IPartitioner partitioner = tm.partitioner(); + TokenPlacementModel.Lookup lookup = actions.factory.lookup(); + Map idToNodeId = new HashMap<>(); + for (int id : actions.all.toArray()) + idToNodeId.put(id, lookup.nodeId(id)); + + for (int ii = 0; ii < actions.all.size(); ii++) + { + int nodeIdx = ii + 1; + List tokens = tm.tokens(idToNodeId.get(nodeIdx)); + checkState(tokens.size() == 1, "Expect only 1, not handling vnodes tokenRanges " + tokens); + Token token = tokens.get(0); + Range tokenRange = new Range(tm.getPredecessor(token), token); + Range firstRange = new Range<>(tokenRange.left, partitioner.split(tokenRange.left, tokenRange.right, 0.33)); + Range secondRange = new Range<>(firstRange.right, partitioner.split(tokenRange.left, tokenRange.right, 0.66)); + Range thirdRange = new Range<>(secondRange.right, tokenRange.right); + ranges.add(Pair.create(nodeIdx, new SimpleEntry<>(firstRange.left.toString(), firstRange.right.toString()))); + ranges.add(Pair.create(nodeIdx, new SimpleEntry<>(secondRange.left.toString(), secondRange.right.toString()))); + ranges.add(Pair.create(nodeIdx, new SimpleEntry<>(thirdRange.left.toString(), thirdRange.right.toString()))); + } + + Collections.shuffle(ranges); + + System.out.println("Ranges to migrate " + ranges); + + ranges.stream().forEach(p -> result.add(new OnClusterMigrateConsensusOneRange(actions, p.left(), p.right()))); + return ActionList.of(result); + } +} diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterMigrateConsensusOneRange.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterMigrateConsensusOneRange.java new file mode 100644 index 0000000000..84a1d43c50 --- /dev/null +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterMigrateConsensusOneRange.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.simulator.cluster; + +import java.util.Map; + +import com.google.common.collect.ImmutableList; + +import org.apache.cassandra.simulator.Action; +import org.apache.cassandra.simulator.ActionList; + +import static org.apache.cassandra.simulator.Action.Modifiers.NONE; +import static org.apache.cassandra.simulator.Action.Modifiers.STRICT; + +class OnClusterMigrateConsensusOneRange extends Action +{ + private final KeyspaceActions actions; + private final int repairOn; + Map.Entry startMigrationRange; + + OnClusterMigrateConsensusOneRange(KeyspaceActions actions, int repairOn, Map.Entry startMigrationRange) + { + super("Performing consensus migration one range " + startMigrationRange, STRICT, NONE); + this.actions = actions; + this.repairOn = repairOn; + this.startMigrationRange = startMigrationRange; + } + + public ActionList performSimple() + { + return ActionList.of(new OnInstanceStartConsensusMigration(actions, 1, startMigrationRange ), + new OnClusterRepairRanges(actions, new int[] { repairOn }, true, false, ImmutableList.of(startMigrationRange))); + } +} diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java index 46edfb3926..bf75f920ed 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java @@ -97,7 +97,7 @@ class OnInstanceRepair extends ClusterAction { Collection> ranges = rangesSupplier.call(); // no need to wait for completion, as we track all task submissions and message exchanges, and ensure they finish before continuing to next action - StorageService.instance.repair(keyspaceName, new RepairOption(RepairParallelism.SEQUENTIAL, isPrimaryRangeOnly, false, false, 1, ranges, false, false, force, PreviewKind.NONE, false, true, repairPaxos, repairOnlyPaxos, false), singletonList((tag, event) -> { + StorageService.instance.repair(keyspaceName, new RepairOption(RepairParallelism.SEQUENTIAL, isPrimaryRangeOnly, false, false, 1, ranges, false, force, PreviewKind.NONE, false, true, repairPaxos, repairOnlyPaxos, false, false), singletonList((tag, event) -> { if (event.getType() == ProgressEventType.COMPLETE) listener.run(); })); diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceStartConsensusMigration.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceStartConsensusMigration.java new file mode 100644 index 0000000000..ba23f98587 --- /dev/null +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceStartConsensusMigration.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.simulator.cluster; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.service.StorageService; + +import static org.apache.cassandra.simulator.Action.Modifiers.RELIABLE_NO_TIMEOUTS; + +class OnInstanceStartConsensusMigration extends ClusterAction +{ + + public OnInstanceStartConsensusMigration(KeyspaceActions actions, int on, Map.Entry startMigrationRange) + { + this(actions, on, RELIABLE_NO_TIMEOUTS, RELIABLE_NO_TIMEOUTS, startMigrationRange); + } + + public OnInstanceStartConsensusMigration(KeyspaceActions actions, int on, Modifiers self, Modifiers transitive, Map.Entry startMigrationRange) + { + super("Start consensus migration on " + on, self, transitive, actions, on, invokableBlockingStartConsensusMigration(actions.keyspace, actions.table, startMigrationRange)); + } + + private static IIsolatedExecutor.SerializableRunnable invokableBlockingStartConsensusMigration(String keyspaceName, String cfName, Map.Entry range) + { + return () -> { + List keyspaces = new ArrayList<>(); + keyspaces.add(keyspaceName); + List tables = new ArrayList<>(); + tables.add(cfName); + StorageService.instance.migrateConsensusProtocol("accord", keyspaces, tables, range.getKey() + ":" + range.getValue()); + }; + } +} diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java b/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java index 5face389ac..cbbf85fc32 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java +++ b/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java @@ -61,7 +61,7 @@ public class Reconcile private static final Pattern STRIP_TRACES = Pattern.compile("(Wakeup|Continue|Timeout|Waiting)\\[(((([a-zA-Z]\\.)*[a-zA-Z0-9_$]+\\.[a-zA-Z0-9_<>$]+:[\\-0-9]+; )*(([a-zA-Z]\\.)*[a-zA-Z0-9_$]+\\.[a-zA-Z0-9_<>$]+:[\\-0-9]+))( #\\[.*?]#)?) ?(by\\[.*?])?]"); private static final Pattern STRIP_NOW_TRACES = Pattern.compile("( #\\[.*?]#)"); private static final Pattern NORMALISE_THREAD_RECORDING_IN = Pattern.compile("(Thread\\[[^]]+:[0-9]+),?[0-9]+(,node[0-9]+)]"); - static final Pattern NORMALISE_LAMBDA = Pattern.compile("((\\$\\$Lambda\\$[0-9]+/[0-9]+)?(@[0-9a-f]+)?)"); + static final Pattern NORMALISE_LAMBDA = Pattern.compile("((\\$\\$Lambda\\$[0-9]+/(0x)?[a-f0-9]+)?(@[0-9a-f]+)?)"); static final Pattern NORMALISE_THREAD = Pattern.compile("(Thread\\[[^]]+:[0-9]+),[0-9](,node[0-9]+)(_[0-9]+)?]"); public static class AbstractReconciler diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcile.java b/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcile.java index 78ee783bd7..72bc99fba7 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcile.java +++ b/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcile.java @@ -320,5 +320,4 @@ public class SelfReconcile ).replaceAll("$1$2]") ).replaceAll("$1]"); } - } diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java index 8ed4556194..5bfb218c7e 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java @@ -42,6 +42,7 @@ import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.distributed.api.LogResult; import org.apache.cassandra.distributed.impl.FileLogAction; import org.apache.cassandra.distributed.impl.Instance; +import org.apache.cassandra.distributed.shared.Metrics; import org.apache.cassandra.io.util.File; import org.apache.cassandra.simulator.Action; import org.apache.cassandra.simulator.ActionList; @@ -99,7 +100,7 @@ abstract class AbstractPairOfSequencesPaxosSimulation extends PaxosSimulation long seed, int[] primaryKeys, long runForNanos, LongSupplier jitter) { - super(runForNanos < 0 ? STREAM_LIMITED : clusterOptions.topologyChangeLimit < 0 ? TIME_LIMITED : TIME_AND_STREAM_LIMITED, + super(runForNanos < 0 ? STREAM_LIMITED : (clusterOptions.topologyChangeLimit <= 0 && clusterOptions.consensusChangeLimit <= 0) ? TIME_LIMITED : TIME_AND_STREAM_LIMITED, simulated, cluster, scheduler, runForNanos, jitter); this.readRatio = readRatio; this.concurrency = concurrency; @@ -183,6 +184,11 @@ abstract class AbstractPairOfSequencesPaxosSimulation extends PaxosSimulation }; } + protected Metrics getMetrics(int coordinatorIndex) + { + return cluster.get(coordinatorIndex).metrics(); + } + public ActionPlan plan() { ActionPlan plan = new KeyspaceActions(simulated, KEYSPACE, TABLE, createTableStmt(), cluster, diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesPaxosSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesPaxosSimulation.java index b07b4a86cd..de3e5f15b7 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesPaxosSimulation.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesPaxosSimulation.java @@ -286,6 +286,10 @@ public class PairOfSequencesPaxosSimulation extends AbstractPairOfSequencesPaxos @Override boolean joinAll() { + // Consensus migration means Accord is running and Accord doesn't yet support joining nodes + if ((clusterOptions.consensusChangeLimit == -1 || clusterOptions.consensusChangeLimit > 0) + && clusterOptions.consensusChoices.options.length > 0) + return true; return false; } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosClusterSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosClusterSimulation.java index a0c6682211..6e7d058bea 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosClusterSimulation.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosClusterSimulation.java @@ -22,8 +22,8 @@ import java.io.IOException; import org.apache.cassandra.config.Config.PaxosVariant; import org.apache.cassandra.distributed.api.ConsistencyLevel; -import org.apache.cassandra.simulator.RandomSource; import org.apache.cassandra.simulator.ClusterSimulation; +import org.apache.cassandra.simulator.RandomSource; import org.apache.cassandra.simulator.utils.KindOfSequence; import static java.util.concurrent.TimeUnit.SECONDS; @@ -69,6 +69,11 @@ class PaxosClusterSimulation extends ClusterSimulation implemen random.reset(seed); return new PaxosClusterSimulation(random, seed, uniqueNum, this); } + + public String lwtStrategy() + { + return lwtStrategy; + } } PaxosClusterSimulation(RandomSource random, long seed, int uniqueNum, Builder builder) throws IOException diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulationRunner.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulationRunner.java index 373a376714..6c9f683c61 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulationRunner.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulationRunner.java @@ -19,6 +19,7 @@ package org.apache.cassandra.simulator.paxos; import java.io.IOException; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; @@ -31,6 +32,7 @@ import io.airlift.airline.Option; import org.apache.cassandra.config.Config; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.simulator.SimulationRunner; +import org.apache.cassandra.simulator.utils.IntRange; import org.apache.cassandra.simulator.SimulatorUtils; public class PaxosSimulationRunner extends SimulationRunner @@ -63,6 +65,18 @@ public class PaxosSimulationRunner extends SimulationRunner super.propagate(builder); propagateTo(consistency, withStateCache, withoutStateCache, variant, toVariant, builder); } + + @Override + protected void run( long seed, PaxosClusterSimulation.Builder builder) throws IOException + { + if (Objects.equals(builder.lwtStrategy(), "accord")) + { + // Apply handicaps + builder.dcs(new IntRange(1, 1)); + builder.nodes(new IntRange(3, 3)); + } + super.run(seed, builder); + } } @Command(name = "record") @@ -94,7 +108,8 @@ public class PaxosSimulationRunner extends SimulationRunner } @Command(name = "reconcile") - public static class Reconcile extends SimulationRunner.Reconcile + public static class + Reconcile extends SimulationRunner.Reconcile { @Option(name = "--consistency") String consistency; diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordJournalSimulationTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordJournalSimulationTest.java index 5cb392d88a..c73eead744 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordJournalSimulationTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordJournalSimulationTest.java @@ -210,7 +210,7 @@ public class AccordJournalSimulationTest extends SimulationTestBase Ranges ranges = Ranges.of(new TokenRange(AccordRoutingKey.SentinelKey.min("system"), AccordRoutingKey.SentinelKey.max("system"))); Topologies topologies = Utils.topologies(TopologyUtils.initialTopology(new Node.Id[] {node}, ranges, 3)); Keys keys = Keys.of(toKey(0)); - Txn txn = new Txn.InMemory(keys, new TxnRead(new TxnNamedRead[0], keys), TxnQuery.ALL, new NoopUpdate()); + Txn txn = new Txn.InMemory(keys, new TxnRead(new TxnNamedRead[0], keys, null), TxnQuery.ALL, new NoopUpdate()); FullRoute route = route(); return new PreAccept(node, topologies, id, txn, route); } diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java index 474f168615..431eb7e454 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java @@ -101,6 +101,20 @@ public class ShortPaxosSimulationTest PaxosSimulationRunner.main(new String[] { "run", "--variant", "v2", "-n", "3..6", "-t", "1000", "-c", "2", "--cluster-action-limit", "2", "-s", "30" }); } + @Test + public void casOnAccordSimulationTest() throws IOException + { + PaxosSimulationRunner.main(new String[] { "run", + "--lwt-strategy", "migration", + "-n", "3...6", + "-t", "1000", + "--cluster-action-limit", "0", + "--consensus-action-limit", "-1", + "--consensus-actions", "ACCORD_MIGRATE", + "-c", "10", + "-s", "30"}); + } + @Test @Ignore("fails due to OOM DirectMemory - unclear why") public void selfReconcileTest() throws IOException diff --git a/test/unit/org/apache/cassandra/CassandraTestBase.java b/test/unit/org/apache/cassandra/CassandraTestBase.java new file mode 100644 index 0000000000..bfb7bc8c41 --- /dev/null +++ b/test/unit/org/apache/cassandra/CassandraTestBase.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra; + +import java.lang.annotation.Annotation; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.reflect.Method; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.dht.LengthPartitioner; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.OrderPreservingPartitioner; +import org.apache.cassandra.dht.RandomPartitioner; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.tcm.ClusterMetadataService; + +import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION; +import static org.junit.Assert.assertTrue; + +/* + * Many tests declare their own test base and duplicate functionality + * Hopefully this can serve as a place to put common initialization patterns and annotations + * So people have fewer problems to solve when authoring tests. + */ +public class CassandraTestBase +{ + @Retention(RetentionPolicy.RUNTIME) + public @interface UseMurmur3Partitioner {} + + @Retention(RetentionPolicy.RUNTIME) + public @interface UseRandomPartitioner {} + + @Retention(RetentionPolicy.RUNTIME) + public @interface UseOrderPreservingPartitioner {} + + @Retention(RetentionPolicy.RUNTIME) + public @interface UseLengthPartitioner {} + + @Retention(RetentionPolicy.RUNTIME) + public @interface UseByteOrderedPartitioner {} + + @Retention(RetentionPolicy.RUNTIME) + public @interface DDDaemonInitialization {} + + @Retention(RetentionPolicy.RUNTIME) + public @interface SchemaLoaderPrepareServer {} + + @Retention(RetentionPolicy.RUNTIME) + public @interface SchemaLoaderLoadSchema {} + + @Retention(RetentionPolicy.RUNTIME) + public @interface PrepareServerNoRegister {} + + @Retention(RetentionPolicy.RUNTIME) + public @interface DisableMBeanRegistration {} + + private static boolean classResetStorageServicePartitioner; + + private static Boolean oldMBeanRegistrationValue; + + @BeforeClass + public static void cassandraTestBaseBeforeClass() + { + if (hasClassAnnotation(DisableMBeanRegistration.class)) + { + oldMBeanRegistrationValue = ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.getBoolean(); + ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(true); + } + + if (hasClassAnnotation(DDDaemonInitialization.class)) + DatabaseDescriptor.daemonInitialization(); + else if (hasClassAnnotation(SchemaLoaderPrepareServer.class)) + SchemaLoader.prepareServer(); + else if (hasClassAnnotation(SchemaLoaderLoadSchema.class)) + SchemaLoader.loadSchema(); + else if (hasClassAnnotation(PrepareServerNoRegister.class)) + ServerTestUtils.daemonInitialization(); + + int partitionerAnnotationCount = 0; + if (hasClassAnnotation(UseMurmur3Partitioner.class)) + { + partitionerAnnotationCount++; + classResetStorageServicePartitioner = true; + StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); + } + if (hasClassAnnotation(UseRandomPartitioner.class)) + { + partitionerAnnotationCount++; + classResetStorageServicePartitioner = true; + StorageService.instance.setPartitionerUnsafe(RandomPartitioner.instance); + } + if (hasClassAnnotation(UseOrderPreservingPartitioner.class)) + { + partitionerAnnotationCount++; + classResetStorageServicePartitioner = true; + StorageService.instance.setPartitionerUnsafe(OrderPreservingPartitioner.instance); + } + if (hasClassAnnotation(UseLengthPartitioner.class)) + { + partitionerAnnotationCount++; + classResetStorageServicePartitioner = true; + StorageService.instance.setPartitionerUnsafe(LengthPartitioner.instance); + } + if (hasClassAnnotation(UseByteOrderedPartitioner.class)) + { + partitionerAnnotationCount++; + classResetStorageServicePartitioner = true; + StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance); + } + assertTrue("At most one partitioner should be annotated", partitionerAnnotationCount <= 1); + + if (hasClassAnnotation(PrepareServerNoRegister.class)) + ServerTestUtils.prepareServerNoRegister(); + } + + @AfterClass + public static void cassandraTestBaseAfterClass() + { + if (oldMBeanRegistrationValue != null) + { + ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(oldMBeanRegistrationValue); + oldMBeanRegistrationValue = null; + } + + if (classResetStorageServicePartitioner) + { + StorageService.instance.resetPartitionerUnsafe(); + classResetStorageServicePartitioner = false; + } + } + + public static boolean hasClassAnnotation(Class annotation) + { + return hasClassAnnotation(testClass, annotation); + } + + public static boolean hasClassAnnotation(Class clazz, Class annotation) + { + if (clazz == null) + return false; + if (clazz.getAnnotation(annotation) != null) + return true; + return hasClassAnnotation(clazz.getSuperclass(), annotation); + } + + private static Class testClass; + + @ClassRule + public static TestWatcher classWatcher = new TestWatcher() + { + @Override + public void starting(Description description) + { + testClass = description.getTestClass(); + } + }; + + @Rule + public TestName testMethodName = new TestName(); + public Method testMethod; + + private boolean testResetPartitioner; + + ClusterMetadataService toRestore; + + @Before + public void cassandraTestBaseSetUp() throws Exception + { + testMethod = testClass.getMethod(testMethodName.getMethodName()); + int partitionerAnnotationCount = 0; + if (hasMethodAnnotation(UseMurmur3Partitioner.class)) + { + partitionerAnnotationCount++; + testResetPartitioner = true; + StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); + } + if (hasMethodAnnotation(UseRandomPartitioner.class)) + { + partitionerAnnotationCount++; + testResetPartitioner = true; + StorageService.instance.setPartitionerUnsafe(RandomPartitioner.instance); + } + if (hasMethodAnnotation(UseOrderPreservingPartitioner.class)) + { + partitionerAnnotationCount++; + testResetPartitioner = true; + StorageService.instance.setPartitionerUnsafe(OrderPreservingPartitioner.instance); + } + if (hasMethodAnnotation(UseLengthPartitioner.class)) + { + partitionerAnnotationCount++; + testResetPartitioner = true; + StorageService.instance.setPartitionerUnsafe(LengthPartitioner.instance); + } + if (hasMethodAnnotation(UseByteOrderedPartitioner.class)) + { + partitionerAnnotationCount++; + testResetPartitioner = true; + StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance); + } + + if (testResetPartitioner) + { + toRestore = ClusterMetadataService.unsetInstance(); + ClusterMetadataService withNewPartitioner = ClusterMetadataTestHelper.instanceForTest(); + ClusterMetadataService.setInstance(withNewPartitioner); + } + assertTrue("At most one partitioner should be annotated", partitionerAnnotationCount <= 1); + } + + private boolean hasMethodAnnotation(Class annotation) + { + return testMethod.getAnnotation(annotation) != null; + } + + @After + public void cassandraTestBaseTearDown() + { + if (testResetPartitioner) + { + StorageService.instance.resetPartitionerUnsafe(); + testResetPartitioner = false; + ClusterMetadataService.unsetInstance(); + + if (toRestore != null) + { + ClusterMetadataService.setInstance(toRestore); + toRestore = null; + } + } + } +} diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java index 22b0eb9999..74cf51de92 100644 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@ -155,7 +155,9 @@ import org.awaitility.Awaitility; import org.hamcrest.Matcher; import org.mockito.Mockito; import org.mockito.internal.stubbing.defaultanswers.ForwardsInvocations; +import org.awaitility.core.ThrowingRunnable; +import static com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; @@ -230,7 +232,7 @@ public class Util private AtomicBoolean exhausted = new AtomicBoolean(); public Iterator iterator() { - Preconditions.checkState(!exhausted.getAndSet(true)); + checkState(!exhausted.getAndSet(true)); return source; } }; @@ -699,19 +701,21 @@ public class Util public static class PartitionerSwitcher implements AutoCloseable { - final IPartitioner oldP; final IPartitioner newP; + boolean closed; + public PartitionerSwitcher(IPartitioner partitioner) { newP = partitioner; - oldP = StorageService.instance.setPartitionerUnsafe(partitioner); + StorageService.instance.setPartitionerUnsafe(partitioner); } public void close() { - IPartitioner p = StorageService.instance.setPartitionerUnsafe(oldP); - assert p == newP; + checkState(!closed, "Already reset"); + closed = true; + StorageService.instance.resetPartitionerUnsafe(); } } @@ -748,6 +752,21 @@ public class Util .untilAsserted(() -> assertThat(message, call.call(), equalTo(expected))); } + public static void spinUntilSuccess(ThrowingRunnable runnable) + { + spinUntilSuccess(runnable, 10); + } + + public static void spinUntilSuccess(ThrowingRunnable runnable, int timeoutInSeconds) + { + Awaitility.await() + .pollInterval(Duration.ofMillis(100)) + .pollDelay(0, TimeUnit.MILLISECONDS) + .atMost(timeoutInSeconds, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(runnable); + } + public static void joinThread(Thread thread) throws InterruptedException { thread.join(10000); diff --git a/test/unit/org/apache/cassandra/auth/CassandraAuthorizerTest.java b/test/unit/org/apache/cassandra/auth/CassandraAuthorizerTest.java index 7ead4e4118..97e8fb4b08 100644 --- a/test/unit/org/apache/cassandra/auth/CassandraAuthorizerTest.java +++ b/test/unit/org/apache/cassandra/auth/CassandraAuthorizerTest.java @@ -36,6 +36,7 @@ public class CassandraAuthorizerTest extends CQLTester @BeforeClass public static void setupAuth() { + // This runs after the base class sets up Cassandra and might not even work CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(true); requireAuthentication(); requireNetwork(); diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java index d267a4b96a..f8599f4c6e 100644 --- a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java +++ b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java @@ -23,15 +23,16 @@ import java.util.List; import java.util.concurrent.ExecutionException; import com.google.common.collect.Lists; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.SchemaLoaderPrepareServer; +import org.apache.cassandra.CassandraTestBase.UseByteOrderedPartitioner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.Util.PartitionerSwitcher; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; @@ -45,7 +46,6 @@ import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.partitions.ImmutableBTreePartition; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; -import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.KeyspaceParams; @@ -66,7 +66,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -public class BatchlogManagerTest +@UseByteOrderedPartitioner +@SchemaLoaderPrepareServer +public class BatchlogManagerTest extends CassandraTestBase { private static final String KEYSPACE1 = "BatchlogManagerTest1"; private static final String CF_STANDARD1 = "Standard1"; @@ -75,14 +77,9 @@ public class BatchlogManagerTest private static final String CF_STANDARD4 = "Standard4"; private static final String CF_STANDARD5 = "Standard5"; - static PartitionerSwitcher sw; - @BeforeClass public static void defineSchema() throws ConfigurationException { - DatabaseDescriptor.daemonInitialization(); - sw = Util.switchPartitioner(Murmur3Partitioner.instance); - SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1, 1, BytesType.instance), @@ -92,12 +89,6 @@ public class BatchlogManagerTest SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD5, 1, BytesType.instance)); } - @AfterClass - public static void cleanup() - { - sw.close(); - } - @Before public void setUp() throws Exception { diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index 9ab3dab6bf..eb10476ace 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -91,7 +91,8 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.Config$DiskOptimizationStrategy", "org.apache.cassandra.config.Config$FlushCompression", "org.apache.cassandra.config.Config$InternodeCompression", - "org.apache.cassandra.config.Config$LegacyPaxosStrategy", + "org.apache.cassandra.config.Config$LWTStrategy", + "org.apache.cassandra.config.Config$NonSerialWriteStrategy", "org.apache.cassandra.config.Config$MemtableAllocationType", "org.apache.cassandra.config.Config$PaxosOnLinearizabilityViolation", "org.apache.cassandra.config.Config$PaxosStatePurging", diff --git a/test/unit/org/apache/cassandra/cql3/statements/TxnDataNameTest.java b/test/unit/org/apache/cassandra/cql3/statements/TxnDataNameTest.java index 8c1214a26b..d084082a08 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/TxnDataNameTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/TxnDataNameTest.java @@ -18,11 +18,12 @@ package org.apache.cassandra.cql3.statements; -import org.apache.cassandra.service.accord.txn.TxnDataName; import org.junit.Test; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.service.accord.txn.TxnDataName; +import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.utils.Generators; import org.assertj.core.api.Assertions; import org.quicktheories.core.Gen; @@ -62,6 +63,7 @@ public class TxnDataNameTest case USER: return TxnDataName.user(symbolGen.generate(rnd)); case RETURNING: return TxnDataName.returning(); case AUTO_READ: return new TxnDataName(kind, symbolGen.generate(rnd), symbolGen.generate(rnd), symbolGen.generate(rnd)); + case CAS_READ: return TxnRead.CAS_READ; default: throw new IllegalArgumentException("Unknown kind: " + kind); } }; diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java index 2c6ce589da..6b034af523 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java @@ -203,7 +203,7 @@ public class CQLVectorTest extends CQLTester.InMemory execute("INSERT INTO %s (pk, value) VALUES (0, {z: [{y:1}, {y:2}]})"); assertRows(execute("SELECT * FROM %s"), - row(0, userType("z", vector(userType("y", 1), userType("y", 2))))); + row(0, userType("z", vector((Object)userType("y", 1), (Object)userType("y", 2))))); } @Test diff --git a/test/unit/org/apache/cassandra/db/CleanupTransientTest.java b/test/unit/org/apache/cassandra/db/CleanupTransientTest.java index d1a275c657..33877c9b8e 100644 --- a/test/unit/org/apache/cassandra/db/CleanupTransientTest.java +++ b/test/unit/org/apache/cassandra/db/CleanupTransientTest.java @@ -26,8 +26,10 @@ import java.util.UUID; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.PrepareServerNoRegister; +import org.apache.cassandra.CassandraTestBase.UseRandomPartitioner; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.compaction.CompactionManager; @@ -45,10 +47,11 @@ import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; -public class CleanupTransientTest +@PrepareServerNoRegister +@UseRandomPartitioner +public class CleanupTransientTest extends CassandraTestBase { private static final IPartitioner partitioner = RandomPartitioner.instance; - private static IPartitioner oldPartitioner; public static final int LOOPS = 200; public static final String KEYSPACE1 = "CleanupTest1"; @@ -70,10 +73,7 @@ public class CleanupTransientTest @BeforeClass public static void setup() throws Exception { - DatabaseDescriptor.daemonInitialization(); DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); - oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner); - ServerTestUtils.prepareServerNoRegister(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple("2/1"), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java index 419772d304..0462c6d61b 100644 --- a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java +++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java @@ -219,6 +219,7 @@ public class ReadCommandVerbHandlerOutOfRangeTest false, 0, false, + false, tmd, FBUtilities.nowInSeconds(), ColumnFilter.all(tmd), @@ -254,6 +255,7 @@ public class ReadCommandVerbHandlerOutOfRangeTest false, 0, false, + false, tmd, FBUtilities.nowInSeconds(), ColumnFilter.all(tmd), diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java index f71b1ff7ca..bc2b285d98 100644 --- a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java +++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java @@ -173,6 +173,7 @@ public class ReadCommandVerbHandlerTest false, 0, false, + false, metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(metadata), diff --git a/test/unit/org/apache/cassandra/db/ReadResponseTest.java b/test/unit/org/apache/cassandra/db/ReadResponseTest.java index 988677a83b..e594369037 100644 --- a/test/unit/org/apache/cassandra/db/ReadResponseTest.java +++ b/test/unit/org/apache/cassandra/db/ReadResponseTest.java @@ -255,6 +255,7 @@ public class ReadResponseTest isDigest, 0, false, + false, metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(metadata), diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java index 19b758f73e..20420aeef3 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java @@ -42,6 +42,9 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.SchemaLoaderPrepareServer; +import org.apache.cassandra.CassandraTestBase.UseByteOrderedPartitioner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; @@ -53,15 +56,12 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.compaction.AbstractStrategyHolder.GroupedSSTableContainer; -import org.apache.cassandra.dht.ByteOrderedPartitioner; -import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; import org.apache.cassandra.notifications.SSTableAddedNotification; import org.apache.cassandra.notifications.SSTableDeletingNotification; import org.apache.cassandra.schema.CompactionParams; import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.utils.ByteBufferUtil; @@ -73,27 +73,27 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class CompactionStrategyManagerTest +/** + * We use byte ordered partitioner in this test to be able to easily infer an SSTable + * disk assignment based on its generation - See {@link this#getSSTableIndex(Integer[], SSTableReader)} + */ +@SchemaLoaderPrepareServer +@UseByteOrderedPartitioner +public class CompactionStrategyManagerTest extends CassandraTestBase { private static final Logger logger = LoggerFactory.getLogger(CompactionStrategyManagerTest.class); private static final String KS_PREFIX = "Keyspace1"; private static final String TABLE_PREFIX = "CF_STANDARD"; - private static IPartitioner originalPartitioner; private static boolean backups; @BeforeClass public static void beforeClass() { - SchemaLoader.prepareServer(); backups = DatabaseDescriptor.isIncrementalBackupsEnabled(); DatabaseDescriptor.setIncrementalBackupsEnabled(false); - /** - * We use byte ordered partitioner in this test to be able to easily infer an SSTable - * disk assignment based on its generation - See {@link this#getSSTableIndex(Integer[], SSTableReader)} - */ - originalPartitioner = StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance); + SchemaLoader.createKeyspace(KS_PREFIX, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KS_PREFIX, TABLE_PREFIX) @@ -110,7 +110,6 @@ public class CompactionStrategyManagerTest @AfterClass public static void afterClass() { - DatabaseDescriptor.setPartitionerUnsafe(originalPartitioner); DatabaseDescriptor.setIncrementalBackupsEnabled(backups); } diff --git a/test/unit/org/apache/cassandra/db/compaction/PartialCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/PartialCompactionsTest.java index 1d877904a2..92c479ffb5 100644 --- a/test/unit/org/apache/cassandra/db/compaction/PartialCompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/PartialCompactionsTest.java @@ -41,8 +41,8 @@ import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.CloseableIterator; import org.apache.cassandra.utils.FBUtilities; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; public class PartialCompactionsTest extends SchemaLoader @@ -120,7 +120,7 @@ public class PartialCompactionsTest extends SchemaLoader private static int liveRows(ColumnFamilyStore cfs) { return Util.getAll(Util.cmd(cfs, "key1").build()).stream() - .map(partition -> count(partition.rowIterator())) + .map(partition -> count(partition.rowIterator(false))) .reduce(Integer::sum) .orElse(0); } diff --git a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java index f2b47cbf6e..cbd38ba8e7 100644 --- a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java +++ b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java @@ -90,7 +90,7 @@ public class ColumnFilterTest SchemaLoader.prepareServer(); DatabaseDescriptor.setSeedProvider(Arrays::asList); DatabaseDescriptor.setDefaultFailureDetector(); - DatabaseDescriptor.setPartitionerUnsafe(new Murmur3Partitioner()); + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); } // Select all diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java index c29f059000..abb6c2dc1b 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java @@ -62,6 +62,7 @@ import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import static java.util.Collections.emptyList; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -145,14 +146,14 @@ public class CassandraEntireSSTableStreamWriterTest CassandraEntireSSTableStreamWriter writer = new CassandraEntireSSTableStreamWriter(sstable, session, context); writer.write(out); - session.prepareReceiving(new StreamSummary(sstable.metadata().id, 1, 5104)); + session.prepareReceiving(new StreamSummary(sstable.metadata().id, emptyList(), 1, 5104)); CassandraStreamHeader header = CassandraStreamHeader.builder() .withSSTableVersion(sstable.descriptor.version) .withSSTableLevel(0) .withEstimatedKeys(sstable.estimatedKeys()) - .withSections(Collections.emptyList()) + .withSections(emptyList()) .withSerializationHeader(sstable.header.toComponent()) .withComponentManifest(context.manifest()) .isEntireSSTable(true) @@ -208,7 +209,7 @@ public class CassandraEntireSSTableStreamWriterTest StreamResultFuture future = StreamResultFuture.createInitiator(nextTimeUUID(), StreamOperation.BOOTSTRAP, Collections.emptyList(), streamCoordinator); InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort(); - streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED, null)); + streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, emptyList(), emptyList(), StreamSession.State.INITIALIZED, null)); StreamSession session = streamCoordinator.getOrCreateOutboundSession(peer); session.init(future); diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java index 41dcdcb66b..7085ddf18a 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java @@ -33,15 +33,13 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Uninterruptibles; - -import org.apache.cassandra.Util; -import org.apache.cassandra.locator.RangesAtEndpoint; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; @@ -53,6 +51,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.CompactionParams; import org.apache.cassandra.schema.KeyspaceParams; @@ -69,6 +68,7 @@ import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Ref; +import static java.util.Collections.emptyList; import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -272,7 +272,7 @@ public class CassandraStreamManagerTest Collection summaries = new ArrayList<>(); for (int i = 0; i < 10; i++) { - StreamSummary summary = new StreamSummary(tbm.id, i, (i + 1) * 10); + StreamSummary summary = new StreamSummary(tbm.id, emptyList(), i, (i + 1) * 10); summaries.add(summary); } return summaries; diff --git a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java index 7f0fe2c9c6..55793c9cac 100644 --- a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java @@ -87,6 +87,7 @@ import org.apache.cassandra.utils.Throwables; import org.jboss.byteman.contrib.bmunit.BMRule; import org.jboss.byteman.contrib.bmunit.BMUnitRunner; +import static java.util.Collections.emptyList; import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; import static org.junit.Assert.assertTrue; @@ -228,7 +229,7 @@ public class EntireSSTableStreamConcurrentComponentMutationTest streaming.get(3, TimeUnit.MINUTES); concurrentMutations.get(3, TimeUnit.MINUTES); - session.prepareReceiving(new StreamSummary(sstable.metadata().id, 1, 5104)); + session.prepareReceiving(new StreamSummary(sstable.metadata().id, emptyList(), 1, 5104)); StreamMessageHeader messageHeader = new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null); try (DataInputBuffer in = new DataInputBuffer(serializedFile.nioBuffer(), false)) @@ -321,10 +322,10 @@ public class EntireSSTableStreamConcurrentComponentMutationTest private StreamSession setupStreamingSessionForTest() { StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.BOOTSTRAP, 1, new NettyStreamingConnectionFactory(), false, false, null, PreviewKind.NONE); - StreamResultFuture future = StreamResultFuture.createInitiator(nextTimeUUID(), StreamOperation.BOOTSTRAP, Collections.emptyList(), streamCoordinator); + StreamResultFuture future = StreamResultFuture.createInitiator(nextTimeUUID(), StreamOperation.BOOTSTRAP, emptyList(), streamCoordinator); InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort(); - streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED, null)); + streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, emptyList(), emptyList(), StreamSession.State.INITIALIZED, null)); StreamSession session = streamCoordinator.getOrCreateOutboundSession(peer); session.init(future); diff --git a/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java b/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java index 2e6870e6a6..5a292bc195 100644 --- a/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java +++ b/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java @@ -23,35 +23,39 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; -import org.junit.*; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DDDaemonInitialization; +import org.apache.cassandra.CassandraTestBase.UseOrderPreservingPartitioner; import org.apache.cassandra.ServerTestUtils; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.dht.OrderPreservingPartitioner; import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.StubClusterMetadataService; -public class ViewUtilsTest +@DDDaemonInitialization +@UseOrderPreservingPartitioner +public class ViewUtilsTest extends CassandraTestBase { private final String KS = "Keyspace1"; @BeforeClass public static void setUp() throws ConfigurationException, IOException { - DatabaseDescriptor.daemonInitialization(); - DatabaseDescriptor.setPartitionerUnsafe(OrderPreservingPartitioner.instance); ServerTestUtils.cleanupAndLeaveDirs(); Keyspace.setInitialized(); } diff --git a/test/unit/org/apache/cassandra/db/virtual/StreamingVirtualTableTest.java b/test/unit/org/apache/cassandra/db/virtual/StreamingVirtualTableTest.java index 30a70338f9..b008151978 100644 --- a/test/unit/org/apache/cassandra/db/virtual/StreamingVirtualTableTest.java +++ b/test/unit/org/apache/cassandra/db/virtual/StreamingVirtualTableTest.java @@ -54,6 +54,7 @@ import org.apache.cassandra.streaming.StreamingState; import org.apache.cassandra.utils.FBUtilities; import org.assertj.core.util.Throwables; +import static java.util.Collections.emptyList; import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -90,15 +91,15 @@ public class StreamingVirtualTableTest extends CQLTester { StreamingState state = stream(true); assertRows(execute(t("select id, follower, operation, peers, status, progress_percentage, last_updated_at, failure_cause, success_message from %s")), - new Object[] { state.id(), true, "Repair", Collections.emptyList(), "init", 0F, new Date(state.lastUpdatedAtMillis()), null, null }); + new Object[] { state.id(), true, "Repair", emptyList(), "init", 0F, new Date(state.lastUpdatedAtMillis()), null, null }); state.phase.start(); assertRows(execute(t("select id, follower, operation, peers, status, progress_percentage, last_updated_at, failure_cause, success_message from %s")), - new Object[] { state.id(), true, "Repair", Collections.emptyList(), "start", 0F, new Date(state.lastUpdatedAtMillis()), null, null }); + new Object[] { state.id(), true, "Repair", emptyList(), "start", 0F, new Date(state.lastUpdatedAtMillis()), null, null }); - state.handleStreamEvent(new StreamEvent.SessionPreparedEvent(state.id(), new SessionInfo(PEER2, 1, PEER1, Collections.emptyList(), Collections.emptyList(), StreamSession.State.PREPARING, null), StreamSession.PrepareDirection.ACK)); + state.handleStreamEvent(new StreamEvent.SessionPreparedEvent(state.id(), new SessionInfo(PEER2, 1, PEER1, emptyList(), emptyList(), StreamSession.State.PREPARING, null), StreamSession.PrepareDirection.ACK)); - state.onSuccess(new StreamState(state.id(), StreamOperation.REPAIR, ImmutableSet.of(new SessionInfo(PEER2, 1, PEER1, Collections.emptyList(), Collections.emptyList(), StreamSession.State.COMPLETE, null)))); + state.onSuccess(new StreamState(state.id(), StreamOperation.REPAIR, ImmutableSet.of(new SessionInfo(PEER2, 1, PEER1, emptyList(), emptyList(), StreamSession.State.COMPLETE, null)))); assertRows(execute(t("select id, follower, operation, peers, status, progress_percentage, last_updated_at, failure_cause, success_message from %s")), new Object[] { state.id(), true, "Repair", Arrays.asList(address(127, 0, 0, 2).toString()), "success", 100F, new Date(state.lastUpdatedAtMillis()), null, null }); } @@ -222,7 +223,7 @@ public class StreamingVirtualTableTest extends CQLTester private static StreamSummary streamSummary() { int files = ThreadLocalRandom.current().nextInt(2, 10); - return new StreamSummary(TableId.fromUUID(UUID.randomUUID()), files, files * 1024); + return new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), files, files * 1024); } @Test @@ -232,7 +233,7 @@ public class StreamingVirtualTableTest extends CQLTester RuntimeException t = new RuntimeException("You failed!"); state.onFailure(t); assertRows(execute(t("select id, follower, peers, status, progress_percentage, last_updated_at, failure_cause, success_message from %s")), - new Object[] { state.id(), true, Collections.emptyList(), "failure", 100F, new Date(state.lastUpdatedAtMillis()), Throwables.getStackTrace(t), null }); + new Object[] { state.id(), true, emptyList(), "failure", 100F, new Date(state.lastUpdatedAtMillis()), Throwables.getStackTrace(t), null }); } private static String t(String query) diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java index ad5829c7c0..00885d8c1f 100644 --- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java +++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java @@ -33,8 +33,10 @@ import com.google.common.collect.Multimap; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.runner.RunWith; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.PrepareServerNoRegister; +import org.apache.cassandra.CassandraTestBase.UseMurmur3Partitioner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.CassandraRelevantProperties; @@ -48,7 +50,6 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.NodeId; @@ -57,16 +58,15 @@ import org.apache.cassandra.tcm.sequences.BootstrapAndJoin; import org.apache.cassandra.utils.Pair; import org.jboss.byteman.contrib.bmunit.BMRule; import org.jboss.byteman.contrib.bmunit.BMRules; -import org.jboss.byteman.contrib.bmunit.BMUnitRunner; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -@RunWith(BMUnitRunner.class) -public class BootStrapperTest +@UseMurmur3Partitioner +@PrepareServerNoRegister +public class BootStrapperTest extends CassandraTestBase { - static IPartitioner oldPartitioner; static Predicate originalAlivePredicate = RangeStreamer.ALIVE_PREDICATE; public static AtomicBoolean nonOptimizationHit = new AtomicBoolean(false); public static AtomicBoolean optimizationHit = new AtomicBoolean(false); @@ -88,9 +88,6 @@ public class BootStrapperTest @BeforeClass public static void setup() throws ConfigurationException { - DatabaseDescriptor.daemonInitialization(); - oldPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); - ServerTestUtils.prepareServerNoRegister(); SchemaLoader.startGossiper(); SchemaLoader.schemaDefinition("BootStrapperTest"); RangeStreamer.ALIVE_PREDICATE = Predicates.alwaysTrue(); @@ -100,7 +97,6 @@ public class BootStrapperTest @AfterClass public static void tearDown() { - DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner); RangeStreamer.ALIVE_PREDICATE = originalAlivePredicate; } diff --git a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java index 6cd4a1331d..abc6f023c1 100644 --- a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java +++ b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java @@ -20,26 +20,26 @@ package org.apache.cassandra.dht; import java.math.BigInteger; import java.util.List; -import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.SchemaLoaderPrepareServer; +import org.apache.cassandra.CassandraTestBase.UseLengthPartitioner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.marshal.IntegerType; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; -import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; -import org.apache.cassandra.utils.FBUtilities; /** * Test cases where multiple keys collides, ie have the same token. @@ -48,29 +48,21 @@ import org.apache.cassandra.utils.FBUtilities; * length partitioner that takes the length of the key as token, making * collision easy and predictable. */ -public class KeyCollisionTest +@UseLengthPartitioner +@SchemaLoaderPrepareServer +public class KeyCollisionTest extends CassandraTestBase { - static IPartitioner oldPartitioner; private static final String KEYSPACE1 = "KeyCollisionTest1"; private static final String CF = "Standard1"; @BeforeClass public static void defineSchema() throws ConfigurationException { - DatabaseDescriptor.daemonInitialization(); - oldPartitioner = StorageService.instance.setPartitionerUnsafe(LengthPartitioner.instance); - SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE1, CF)); } - @AfterClass - public static void tearDown() - { - DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner); - } - @Test public void testGetSliceWithCollision() throws Exception { diff --git a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java index 01b41d4b3b..e57a714e7b 100644 --- a/test/unit/org/apache/cassandra/dht/LengthPartitioner.java +++ b/test/unit/org/apache/cassandra/dht/LengthPartitioner.java @@ -122,6 +122,8 @@ public class LengthPartitioner extends AccordSplitter implements IPartitioner public void validate(String token) {} }; + private LengthPartitioner() {} + public Token.TokenFactory getTokenFactory() { return tokenFactory; diff --git a/test/unit/org/apache/cassandra/dht/RangeTest.java b/test/unit/org/apache/cassandra/dht/RangeTest.java index 84ca1246a3..ff23a5910f 100644 --- a/test/unit/org/apache/cassandra/dht/RangeTest.java +++ b/test/unit/org/apache/cassandra/dht/RangeTest.java @@ -18,8 +18,8 @@ package org.apache.cassandra.dht; import java.nio.ByteBuffer; -import java.util.Collection; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -27,27 +27,48 @@ import java.util.Random; import java.util.Set; import com.google.common.base.Joiner; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DDDaemonInitialization; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken; +import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.Util.range; -import static org.junit.Assert.*; +import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_RANGE_EXPENSIVE_CHECKS; +import static org.apache.cassandra.dht.Range.fromString; +import static org.apache.cassandra.dht.Range.intersectionOfNormalizedRanges; +import static org.apache.cassandra.dht.Range.invertNormalizedRanges; +import static org.apache.cassandra.dht.Range.isInNormalizedRanges; +import static org.apache.cassandra.dht.Range.normalize; +import static org.apache.cassandra.dht.Range.subtractNormalizedRanges; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -public class RangeTest +@DDDaemonInitialization +public class RangeTest extends CassandraTestBase { @BeforeClass - public static void setupDD() + public static void enableExpensiveRangeChecks() { - DatabaseDescriptor.daemonInitialization(); + assertFalse(TEST_RANGE_EXPENSIVE_CHECKS.getBoolean()); // Expect off by default + CassandraRelevantProperties.TEST_RANGE_EXPENSIVE_CHECKS.setBoolean(true); + assertTrue(TEST_RANGE_EXPENSIVE_CHECKS.getBoolean()); } @Test @@ -578,7 +599,7 @@ public class RangeTest private > void assertNormalize(List> input, List> expected) { - List> result = Range.normalize(input); + List> result = normalize(input); assert result.equals(expected) : "Expecting " + expected + " but got " + result; } @@ -736,4 +757,132 @@ public class RangeTest assertEquals(ranges, Range.subtract(ranges, asList(r(6, 7), r(20, 25)))); assertEquals(Sets.newHashSet(r(1, 4), r(11, 15)), Range.subtract(ranges, asList(r(4, 7), r(8, 11)))); } + + @Test + public void testIntersectsBounds() + { + Range r = r(0, 100); + assertTrue(r.intersects(bounds(5, 10))); + assertTrue(r.intersects(bounds(100, 110))); + assertTrue(r.intersects(bounds(-100, 200))); + assertTrue(r.intersects(bounds(10, 15))); + assertTrue(r.intersects(bounds(20,20))); + + assertFalse(r.intersects(bounds(-5, 0))); + assertFalse(r.intersects(bounds(-5, -1))); + assertFalse(r.intersects(bounds(110, 114))); + } + + private static Bounds bounds(long left, long right) + { + return new Bounds<>(t(left), t(right)); + } + + @Test + @UseMurmur3Partitioner + public void testIsInNormalizedRanges() + { + List> ranges = ImmutableList.of(fromString("(1,10]"), fromString("(10,20]"), fromString("(30,40]"), fromString("(50,60]"), fromString("(60,70]"), fromString("(80,90]"), fromString("(" + Long.MAX_VALUE + ",-9223372036854775808]")); + for (int ii = 0; ii < 100; ii++) + { + boolean isIn = isInNormalizedRanges(new LongToken(ii), ranges); + if (ii > 1 && ii <= 20) + assertTrue("Index " + ii, isIn); + else if (ii > 30 && ii <= 40) + assertTrue("Index " + ii, isIn); + else if (ii > 50 && ii <= 70) + assertTrue("Index " + ii, isIn); + else if (ii > 80 && ii <= 90) + assertTrue("Index " + ii, isIn); + else + assertFalse("Index " + ii, isIn); + } + assertFalse(isInNormalizedRanges(new LongToken(Long.MAX_VALUE), ranges)); + assertTrue(isInNormalizedRanges(new LongToken(Long.MIN_VALUE), ranges)); + ranges = ImmutableList.of(fromString("(-9223372036854775808,-9223372036854775807]")); + assertFalse(isInNormalizedRanges(new LongToken(Long.MIN_VALUE), ranges)); + assertTrue(isInNormalizedRanges(new LongToken(Long.MIN_VALUE + 1), ranges)); + ranges = ImmutableList.of(fromString("(" + (Long.MAX_VALUE - 1) + ",-9223372036854775808]")); + assertFalse(isInNormalizedRanges(new LongToken(Long.MAX_VALUE - 1), ranges)); + assertTrue(isInNormalizedRanges(new LongToken(Long.MAX_VALUE), ranges)); + assertTrue(isInNormalizedRanges(new LongToken(Long.MIN_VALUE), ranges)); + assertFalse(isInNormalizedRanges(new LongToken(Long.MAX_VALUE - 1), normalize(ranges))); + assertTrue(isInNormalizedRanges(new LongToken(Long.MAX_VALUE), normalize(ranges))); + assertTrue(isInNormalizedRanges(new LongToken(Long.MIN_VALUE), normalize(ranges))); + } + + @Test + @UseMurmur3Partitioner + public void testSubtractNormalizedRanges() + { + List> ranges = ImmutableList.of(fromString("(1,10]"), fromString("(10,20]"), fromString("(30,40]"), fromString("(50,60]"), fromString("(60,70]"), fromString("(80,90]"), fromString("(" + Long.MAX_VALUE + ",-9223372036854775808]")); + for (int ii = 0; ii < 100; ii++) + { + boolean isIn = isInNormalizedRanges(new LongToken(ii), ranges); + if (ii > 1 && ii <= 20) + assertTrue("Index " + ii, isIn); + else if (ii > 30 && ii <= 40) + assertTrue("Index " + ii, isIn); + else if (ii > 50 && ii <= 70) + assertTrue("Index " + ii, isIn); + else if (ii > 80 && ii <= 90) + assertTrue("Index " + ii, isIn); + else + assertFalse("Index " + ii, isIn); + } + List> rightMostRange = ImmutableList.of(r(Long.MAX_VALUE, Long.MIN_VALUE)); + List> maxLongRange = ImmutableList.of(r(Long.MAX_VALUE - 1, Long.MAX_VALUE)); + + assertEquals(emptyList(), subtractNormalizedRanges(ranges, ranges)); + assertEquals(emptyList(), subtractNormalizedRanges(rightMostRange, ranges)); + assertEquals(maxLongRange, subtractNormalizedRanges(maxLongRange, ranges)); + ranges = maxLongRange; + assertEquals(emptyList(), subtractNormalizedRanges(ranges, ranges)); + assertEquals(rightMostRange, subtractNormalizedRanges(rightMostRange, ranges)); + assertEquals(emptyList(), subtractNormalizedRanges(maxLongRange, ranges)); + ranges = ImmutableList.of(fromString("(" + (Long.MAX_VALUE - 1) + ",-9223372036854775808]")); + assertEquals(emptyList(), subtractNormalizedRanges(ranges, ranges)); + assertEquals(emptyList(), subtractNormalizedRanges(rightMostRange, ranges)); + assertEquals(emptyList(), subtractNormalizedRanges(maxLongRange, ranges)); + } + + @Test + public void testExpensiveChecksBurn() + { + long seed = System.nanoTime(); + System.out.println(seed); + Random r = new java.util.Random(seed); + + Stopwatch elapsed = Stopwatch.createStarted(); + while (elapsed.elapsed(SECONDS) != 10) + { + int numRanges = 3; + List> a = new ArrayList(); + for (int ii = 0; ii < numRanges; ii++) + { + a.add(new Range<>(new LongToken(r.nextLong()), new LongToken(r.nextLong()))); + } + a = ImmutableList.copyOf(normalize(a)); + List> b = new ArrayList(); + for (int ii = 0; ii < numRanges; ii++) + { + b.add(new Range<>(new LongToken(r.nextLong()), new LongToken(r.nextLong()))); + } + b = ImmutableList.copyOf(normalize(b)); + + for (int ii = 0; ii < 1000; ii++) + { + Token t = new LongToken(r.nextLong()); + isInNormalizedRanges(t, a); + isInNormalizedRanges(t, b); + } + + intersectionOfNormalizedRanges(a, b); + intersectionOfNormalizedRanges(b, a); + subtractNormalizedRanges(a, b); + subtractNormalizedRanges(b, a); + invertNormalizedRanges(a); + invertNormalizedRanges(b); + } + } } diff --git a/test/unit/org/apache/cassandra/dht/SplitterTest.java b/test/unit/org/apache/cassandra/dht/SplitterTest.java index 1de22ff8fc..707d294a80 100644 --- a/test/unit/org/apache/cassandra/dht/SplitterTest.java +++ b/test/unit/org/apache/cassandra/dht/SplitterTest.java @@ -46,25 +46,25 @@ public class SplitterTest @Test public void randomSplitTestNoVNodesRandomPartitioner() { - randomSplitTestNoVNodes(new RandomPartitioner()); + randomSplitTestNoVNodes(RandomPartitioner.instance); } @Test public void randomSplitTestNoVNodesMurmur3Partitioner() { - randomSplitTestNoVNodes(new Murmur3Partitioner()); + randomSplitTestNoVNodes(Murmur3Partitioner.instance); } @Test public void randomSplitTestVNodesRandomPartitioner() { - randomSplitTestVNodes(new RandomPartitioner()); + randomSplitTestVNodes(RandomPartitioner.instance); } @Test public void randomSplitTestVNodesMurmur3Partitioner() { - randomSplitTestVNodes(new Murmur3Partitioner()); + randomSplitTestVNodes(Murmur3Partitioner.instance); } // CASSANDRA-18013 @@ -235,13 +235,13 @@ public class SplitterTest @Test public void testSplitMurmur3Partitioner() { - testSplit(new Murmur3Partitioner()); + testSplit(Murmur3Partitioner.instance); } @Test public void testSplitRandomPartitioner() { - testSplit(new RandomPartitioner()); + testSplit(RandomPartitioner.instance); } @SuppressWarnings("unchecked") @@ -359,13 +359,13 @@ public class SplitterTest @Test public void testTokensInRangeRandomPartitioner() { - testTokensInRange(new RandomPartitioner()); + testTokensInRange(RandomPartitioner.instance); } @Test public void testTokensInRangeMurmur3Partitioner() { - testTokensInRange(new Murmur3Partitioner()); + testTokensInRange(Murmur3Partitioner.instance); } private static void testTokensInRange(IPartitioner partitioner) @@ -391,13 +391,13 @@ public class SplitterTest @Test public void testElapsedTokensRandomPartitioner() { - testElapsedMultiRange(new RandomPartitioner()); + testElapsedMultiRange(RandomPartitioner.instance); } @Test public void testElapsedTokensMurmur3Partitioner() { - testElapsedMultiRange(new Murmur3Partitioner()); + testElapsedMultiRange(Murmur3Partitioner.instance); } private static void testElapsedMultiRange(IPartitioner partitioner) @@ -457,13 +457,13 @@ public class SplitterTest @Test public void testPositionInRangeRandomPartitioner() { - testPositionInRangeMultiRange(new RandomPartitioner()); + testPositionInRangeMultiRange(RandomPartitioner.instance); } @Test public void testPositionInRangeMurmur3Partitioner() { - testPositionInRangeMultiRange(new Murmur3Partitioner()); + testPositionInRangeMultiRange(Murmur3Partitioner.instance); } private static void testPositionInRangeMultiRange(IPartitioner partitioner) diff --git a/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java b/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java index d731385fd3..816366e26f 100644 --- a/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java +++ b/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java @@ -19,18 +19,18 @@ package org.apache.cassandra.dht; import java.util.Collections; -import org.apache.cassandra.db.commitlog.CommitLog; -import org.apache.cassandra.locator.RangesAtEndpoint; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.StreamEvent; import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.net.MessagingService.current_version; @@ -51,7 +51,7 @@ public class StreamStateStoreTest public void testUpdateAndQueryAvailableRanges() { // let range (0, 100] of keyspace1 be bootstrapped. - IPartitioner p = new Murmur3Partitioner(); + IPartitioner p = Murmur3Partitioner.instance; Token.TokenFactory factory = p.getTokenFactory(); Range range = new Range<>(factory.fromString("0"), factory.fromString("100")); diff --git a/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java b/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java index 6b4ef40465..7099565e3e 100644 --- a/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java +++ b/test/unit/org/apache/cassandra/dht/tokenallocator/TokenAllocationTest.java @@ -28,16 +28,15 @@ import java.util.Random; import java.util.Set; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.ServerTestUtils; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DDDaemonInitialization; +import org.apache.cassandra.CassandraTestBase.UseMurmur3Partitioner; import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; @@ -56,18 +55,12 @@ import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -public class TokenAllocationTest +@DDDaemonInitialization +@UseMurmur3Partitioner +public class TokenAllocationTest extends CassandraTestBase { - static IPartitioner oldPartitioner; static Random rand = new Random(1); - @BeforeClass - public static void beforeClass() throws ConfigurationException - { - DatabaseDescriptor.daemonInitialization(); - oldPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); - } - @Before public void before() throws ConfigurationException { @@ -81,12 +74,6 @@ public class TokenAllocationTest ClusterMetadataService.unsetInstance(); } - @AfterClass - public static void afterClass() - { - DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner); - } - private static TokenAllocation createForTest(ClusterMetadata metadata, int replicas, int numTokens) { return TokenAllocation.create(metadata.locator.local().datacenter, metadata, replicas, numTokens); diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java index b7b67ba9bd..c61a1b50a2 100644 --- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java @@ -49,7 +49,6 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import com.datastax.driver.core.utils.UUIDs; -import org.apache.cassandra.Util; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.cql3.functions.types.DataType; @@ -59,7 +58,6 @@ import org.apache.cassandra.cql3.functions.types.UDTValue; import org.apache.cassandra.cql3.functions.types.UserType; import org.apache.cassandra.db.marshal.FloatType; import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -118,57 +116,54 @@ public abstract class CQLSSTableWriterTest @Test public void testUnsortedWriter() throws Exception { - try (AutoCloseable ignored = Util.switchPartitioner(ByteOrderedPartitioner.instance)) + String schema = "CREATE TABLE " + qualifiedTable + " (" + + " k int PRIMARY KEY," + + " v1 text," + + " v2 int" + + ")"; + String insert = "INSERT INTO " + qualifiedTable + " (k, v1, v2) VALUES (?, ?, ?)"; + CQLSSTableWriter writer = CQLSSTableWriter.builder() + .inDirectory(dataDir) + .forTable(schema) + .using(insert).build(); + + writer.addRow(0, "test1", 24); + writer.addRow(1, "test2", 44); + writer.addRow(2, "test3", 42); + writer.addRow(ImmutableMap.of("k", 3, "v2", 12)); + + writer.close(); + + loadSSTables(dataDir, keyspace, table); + + if (verifyDataAfterLoading) { - String schema = "CREATE TABLE " + qualifiedTable + " (" - + " k int PRIMARY KEY," - + " v1 text," - + " v2 int" - + ")"; - String insert = "INSERT INTO " + qualifiedTable + " (k, v1, v2) VALUES (?, ?, ?)"; - CQLSSTableWriter writer = CQLSSTableWriter.builder() - .inDirectory(dataDir) - .forTable(schema) - .using(insert).build(); + UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable); + assertEquals(4, rs.size()); - writer.addRow(0, "test1", 24); - writer.addRow(1, "test2", 44); - writer.addRow(2, "test3", 42); - writer.addRow(ImmutableMap.of("k", 3, "v2", 12)); + Iterator iter = rs.iterator(); + UntypedResultSet.Row row; - writer.close(); + row = iter.next(); + assertEquals(0, row.getInt("k")); + assertEquals("test1", row.getString("v1")); + assertEquals(24, row.getInt("v2")); - loadSSTables(dataDir, keyspace, table); + row = iter.next(); + assertEquals(1, row.getInt("k")); + assertEquals("test2", row.getString("v1")); + //assertFalse(row.has("v2")); + assertEquals(44, row.getInt("v2")); - if (verifyDataAfterLoading) - { - UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable); - assertEquals(4, rs.size()); + row = iter.next(); + assertEquals(2, row.getInt("k")); + assertEquals("test3", row.getString("v1")); + assertEquals(42, row.getInt("v2")); - Iterator iter = rs.iterator(); - UntypedResultSet.Row row; - - row = iter.next(); - assertEquals(0, row.getInt("k")); - assertEquals("test1", row.getString("v1")); - assertEquals(24, row.getInt("v2")); - - row = iter.next(); - assertEquals(1, row.getInt("k")); - assertEquals("test2", row.getString("v1")); - //assertFalse(row.has("v2")); - assertEquals(44, row.getInt("v2")); - - row = iter.next(); - assertEquals(2, row.getInt("k")); - assertEquals("test3", row.getString("v1")); - assertEquals(42, row.getInt("v2")); - - row = iter.next(); - assertEquals(3, row.getInt("k")); - assertEquals(null, row.getBytes("v1")); // Using getBytes because we know it won't NPE - assertEquals(12, row.getInt("v2")); - } + row = iter.next(); + assertEquals(3, row.getInt("k")); + assertEquals(null, row.getBytes("v1")); // Using getBytes because we know it won't NPE + assertEquals(12, row.getInt("v2")); } } diff --git a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java index e795254903..323cd01fbb 100644 --- a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java @@ -478,7 +478,7 @@ public class ScrubTest // This test assumes ByteOrderPartitioner to create out-of-order SSTable IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner(); - DatabaseDescriptor.setPartitionerUnsafe(new ByteOrderedPartitioner()); + DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance); // Create out-of-order SSTable File tempDir = FileUtils.createTempFile("ScrubTest.testScrubOutOfOrder", "").parent(); diff --git a/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java b/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java index 6ceee331f6..36e9f8fb06 100644 --- a/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java @@ -291,7 +291,7 @@ public class PartitionIndexTest @Test public void testAddEmptyKey() throws Exception { - IPartitioner p = new RandomPartitioner(); + IPartitioner p = RandomPartitioner.instance; File file = FileUtils.createTempFile("ColumnTrieReaderTest", ""); FileHandle.Builder fhBuilder = makeHandle(file); diff --git a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java index aea166aa3e..079437c6ff 100644 --- a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java @@ -248,7 +248,7 @@ public class IndexSummaryTest @Test public void testAddEmptyKey() throws Exception { - IPartitioner p = new RandomPartitioner(); + IPartitioner p = RandomPartitioner.instance; try (IndexSummaryBuilder builder = new IndexSummaryBuilder(1, 1, BASE_SAMPLING_LEVEL)) { builder.maybeAddEntry(p.decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER), 0); diff --git a/test/unit/org/apache/cassandra/locator/AssureSufficientLiveNodesTest.java b/test/unit/org/apache/cassandra/locator/AssureSufficientLiveNodesTest.java index 9e9ff605df..20da42dd12 100644 --- a/test/unit/org/apache/cassandra/locator/AssureSufficientLiveNodesTest.java +++ b/test/unit/org/apache/cassandra/locator/AssureSufficientLiveNodesTest.java @@ -36,9 +36,10 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.PrepareServerNoRegister; +import org.apache.cassandra.CassandraTestBase.UseMurmur3Partitioner; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.ServerTestUtils; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; @@ -49,6 +50,7 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.Tables; import org.apache.cassandra.service.reads.NeverSpeculativeRetryPolicy; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.utils.FBUtilities; import org.jboss.byteman.contrib.bmunit.BMRule; import org.jboss.byteman.contrib.bmunit.BMUnitRunner; @@ -69,7 +71,9 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; targetClass = "FailureDetector", targetMethod = "isAlive", action = "return true;") -public class AssureSufficientLiveNodesTest +@PrepareServerNoRegister +@UseMurmur3Partitioner +public class AssureSufficientLiveNodesTest extends CassandraTestBase { private static final AtomicInteger testIdGen = new AtomicInteger(0); private static final Supplier keyspaceNameGen = () -> "race_" + testIdGen.getAndIncrement(); @@ -82,9 +86,6 @@ public class AssureSufficientLiveNodesTest @BeforeClass public static void setUpClass() throws Throwable { - ServerTestUtils.daemonInitialization(); - DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); - ServerTestUtils.prepareServerNoRegister(); // Register peers with expected DC for NetworkTopologyStrategy. List instances = ImmutableList.of( // datacenter 1 @@ -139,7 +140,7 @@ public class AssureSufficientLiveNodesTest // alter to KeyspaceParams.nts(DC1, 3, DC2, 3), // test - keyspace -> ReplicaPlans.forRead(keyspace, tk, null, EACH_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE) + keyspace -> ReplicaPlans.forRead(keyspace, tk, null, EACH_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT) ); } @@ -172,7 +173,7 @@ public class AssureSufficientLiveNodesTest // alter to KeyspaceParams.nts(DC1, 3, DC2, 3), // test - keyspace -> ReplicaPlans.forRead(keyspace, tk, null, QUORUM, NeverSpeculativeRetryPolicy.INSTANCE) + keyspace -> ReplicaPlans.forRead(keyspace, tk, null, QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT) ); raceOfReplicationStrategyTest( // init. The # of live endpoints is 3 = 2 + 1 @@ -180,7 +181,7 @@ public class AssureSufficientLiveNodesTest // alter to. (3 + 3) / 2 + 1 > 3 KeyspaceParams.nts(DC1, 2, DC2, 1, DC3, 3), // test - keyspace -> ReplicaPlans.forRead(keyspace, tk, null, QUORUM, NeverSpeculativeRetryPolicy.INSTANCE) + keyspace -> ReplicaPlans.forRead(keyspace, tk, null, QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT) ); } @@ -204,7 +205,7 @@ public class AssureSufficientLiveNodesTest // alter to KeyspaceParams.nts(DC1, 3), // test - keyspace -> ReplicaPlans.forRead(keyspace, tk, null, EACH_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE) + keyspace -> ReplicaPlans.forRead(keyspace, tk, null, EACH_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT) ); } @@ -228,7 +229,7 @@ public class AssureSufficientLiveNodesTest // alter to KeyspaceParams.nts(DC1, 3), // test - keyspace -> ReplicaPlans.forRead(keyspace, tk, null, LOCAL_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE) + keyspace -> ReplicaPlans.forRead(keyspace, tk, null, LOCAL_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT) ); } diff --git a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java index 8fc3498bfb..babb7d659b 100644 --- a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java @@ -32,6 +32,7 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.schema.DistributedSchema; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.Directory; @@ -91,6 +92,7 @@ public class MetaStrategyTest AccordKeyspaces.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, + ConsensusMigrationState.EMPTY, ImmutableMap.of()); } diff --git a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java index fc235526fc..e59c847f77 100644 --- a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java @@ -20,29 +20,42 @@ package org.apache.cassandra.locator; import java.io.IOException; import java.net.UnknownHostException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; import java.util.stream.Collectors; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; - -import org.junit.*; +import org.junit.After; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; import org.junit.rules.ExpectedException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DisableMBeanRegistration; +import org.apache.cassandra.CassandraTestBase.PrepareServerNoRegister; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.Util; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.*; +import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.RegistrationStatus; @@ -59,20 +72,13 @@ import static org.apache.cassandra.locator.Replica.transientReplica; import static org.apache.cassandra.locator.SimpleLocationProvider.LOCATION; import static org.junit.Assert.assertTrue; -public class NetworkTopologyStrategyTest +@PrepareServerNoRegister +@DisableMBeanRegistration +public class NetworkTopologyStrategyTest extends CassandraTestBase { private static final String KEYSPACE = "ks1"; private static final Logger logger = LoggerFactory.getLogger(NetworkTopologyStrategyTest.class); - @BeforeClass - public static void setupDD() - { - DatabaseDescriptor.daemonInitialization(); - DatabaseDescriptor.setPartitionerUnsafe(OrderPreservingPartitioner.instance); - DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); - ClusterMetadataService.setInstance(ClusterMetadataTestHelper.instanceForTest()); - } - @After public void teardown() { @@ -80,6 +86,7 @@ public class NetworkTopologyStrategyTest } @Test + @UseOrderPreservingPartitioner public void testProperties() throws IOException, ConfigurationException { createDummyTokens(true); @@ -104,6 +111,7 @@ public class NetworkTopologyStrategyTest } @Test + @UseOrderPreservingPartitioner public void testPropertiesWithEmptyDC() throws IOException, ConfigurationException { createDummyTokens(false); @@ -126,6 +134,7 @@ public class NetworkTopologyStrategyTest } @Test + @UseOrderPreservingPartitioner public void testLargeCluster() throws UnknownHostException, ConfigurationException { int[] dcRacks = new int[]{2, 4, 8}; @@ -201,47 +210,45 @@ public class NetworkTopologyStrategyTest } @Test + @UseMurmur3Partitioner public void testCalculateEndpoints() throws UnknownHostException { final int NODES = 100; final int VNODES = 64; final int RUNS = 10; - try (WithPartitioner m3p = new WithPartitioner(Murmur3Partitioner.instance)) + Map datacenters = ImmutableMap.of("rf1", 1, "rf3", 3, "rf5_1", 5, "rf5_2", 5, "rf5_3", 5); + List nodes = new ArrayList<>(NODES); + for (byte i = 0; i < NODES; ++i) + nodes.add(InetAddressAndPort.getByAddress(new byte[]{ 127, 0, 0, i })); + for (int run = 0; run < RUNS; ++run) { - Map datacenters = ImmutableMap.of("rf1", 1, "rf3", 3, "rf5_1", 5, "rf5_2", 5, "rf5_3", 5); - List nodes = new ArrayList<>(NODES); - for (byte i = 0; i < NODES; ++i) - nodes.add(InetAddressAndPort.getByAddress(new byte[]{ 127, 0, 0, i })); - for (int run = 0; run < RUNS; ++run) - { - ServerTestUtils.resetCMS(); - Random rand = new Random(run); - Locator locator = generateLocator(datacenters, nodes, rand); + ServerTestUtils.resetCMS(); + Random rand = new Random(run); + Locator locator = generateLocator(datacenters, nodes, rand); - for (int i = 0; i < NODES; ++i) // Nodes + for (int i = 0; i < NODES; ++i) // Nodes + { + Set tokens = new HashSet<>(); + while (tokens.size() < VNODES) // tokens/vnodes per node { - Set tokens = new HashSet<>(); - while (tokens.size() < VNODES) // tokens/vnodes per node - { - tokens.add(Murmur3Partitioner.instance.getRandomToken(rand)); - } - // Here we fake the registration status because we want all the nodes to be registered in cluster - // metadata using the locations we setup in generateLocator. This registration occurs as a part of - // the addEndpoint call here and behaves as expected for all nodes _except_ the one with the address - // which matches the local broadcast address (i.e. 127.0.0.1, which is #2 in the list of nodes). - // The location we want this to be registered with is {DC: rf5_1, rack: 3}, but while - // RegistrationStatus.instance indicates that the node is yet to be registered, the Locator will - // correctly return the initialization location obtained from - // DatabaseDescriptor::getInitialLocationProvider, which ultimately resolves to - // SimpleLocationProvider (because test/conf/cassandra.yaml specifies use of SimpleSnitch) and so - // we register that one node with the location {DC: datacenter1, rack: rack1}. - // This is purely an artefact of the contrived testing setup and in more realistic scenarios, - // including the majority of tests, isn't an issue. - RegistrationStatus.instance.onRegistration(); - ClusterMetadataTestHelper.addEndpoint(nodes.get(i), tokens, locator.location(nodes.get(i))); + tokens.add(Murmur3Partitioner.instance.getRandomToken(rand)); } - testEquivalence(ClusterMetadata.current(), locator, datacenters, rand); + // Here we fake the registration status because we want all the nodes to be registered in cluster + // metadata using the locations we setup in generateLocator. This registration occurs as a part of + // the addEndpoint call here and behaves as expected for all nodes _except_ the one with the address + // which matches the local broadcast address (i.e. 127.0.0.1, which is #2 in the list of nodes). + // The location we want this to be registered with is {DC: rf5_1, rack: 3}, but while + // RegistrationStatus.instance indicates that the node is yet to be registered, the Locator will + // correctly return the initialization location obtained from + // DatabaseDescriptor::getInitialLocationProvider, which ultimately resolves to + // SimpleLocationProvider (because test/conf/cassandra.yaml specifies use of SimpleSnitch) and so + // we register that one node with the location {DC: datacenter1, rack: rack1}. + // This is purely an artefact of the contrived testing setup and in more realistic scenarios, + // including the majority of tests, isn't an issue. + RegistrationStatus.instance.onRegistration(); + ClusterMetadataTestHelper.addEndpoint(nodes.get(i), tokens, locator.location(nodes.get(i))); } + testEquivalence(ClusterMetadata.current(), locator, datacenters, rand); } } @@ -438,35 +445,32 @@ public class NetworkTopologyStrategyTest } @Test + @UseMurmur3Partitioner public void testTransientReplica() throws Exception { - try (WithPartitioner m3p = new WithPartitioner(Murmur3Partitioner.instance)) - { - List endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"), - InetAddressAndPort.getByName("127.0.0.2"), - InetAddressAndPort.getByName("127.0.0.3"), - InetAddressAndPort.getByName("127.0.0.4")); + List endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"), + InetAddressAndPort.getByName("127.0.0.2"), + InetAddressAndPort.getByName("127.0.0.3"), + InetAddressAndPort.getByName("127.0.0.4")); - ClusterMetadataTestHelper.addEndpoint(endpoints.get(0), tk(100), LOCATION); - ClusterMetadataTestHelper.addEndpoint(endpoints.get(1), tk(200), LOCATION); - ClusterMetadataTestHelper.addEndpoint(endpoints.get(2), tk(300), LOCATION); - ClusterMetadataTestHelper.addEndpoint(endpoints.get(3), tk(400), LOCATION); + ClusterMetadataTestHelper.addEndpoint(endpoints.get(0), tk(100), LOCATION); + ClusterMetadataTestHelper.addEndpoint(endpoints.get(1), tk(200), LOCATION); + ClusterMetadataTestHelper.addEndpoint(endpoints.get(2), tk(300), LOCATION); + ClusterMetadataTestHelper.addEndpoint(endpoints.get(3), tk(400), LOCATION); - Map configOptions = new HashMap<>(); - configOptions.put(LOCATION.datacenter, "3/1"); - NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(KEYSPACE, configOptions); - - Util.assertRCEquals(EndpointsForRange.of(fullReplica(endpoints.get(0), range(400, 100)), - fullReplica(endpoints.get(1), range(400, 100)), - transientReplica(endpoints.get(2), range(400, 100))), - strategy.calculateNaturalReplicas(tk(99), ClusterMetadata.current())); + Map configOptions = new HashMap<>(); + configOptions.put(LOCATION.datacenter, "3/1"); + NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(KEYSPACE, configOptions); + Util.assertRCEquals(EndpointsForRange.of(fullReplica(endpoints.get(0), range(400, 100)), + fullReplica(endpoints.get(1), range(400, 100)), + transientReplica(endpoints.get(2), range(400, 100))), + strategy.calculateNaturalReplicas(tk(99), ClusterMetadata.current())); - Util.assertRCEquals(EndpointsForRange.of(fullReplica(endpoints.get(1), range(100, 200)), - fullReplica(endpoints.get(2), range(100, 200)), - transientReplica(endpoints.get(3), range(100, 200))), - strategy.calculateNaturalReplicas(tk(101), ClusterMetadata.current())); - } + Util.assertRCEquals(EndpointsForRange.of(fullReplica(endpoints.get(1), range(100, 200)), + fullReplica(endpoints.get(2), range(100, 200)), + transientReplica(endpoints.get(3), range(100, 200))), + strategy.calculateNaturalReplicas(tk(101), ClusterMetadata.current())); } @Rule @@ -486,6 +490,7 @@ public class NetworkTopologyStrategyTest } @Test + @UseOrderPreservingPartitioner public void shouldWarnOnHigherReplicationFactorThanNodesInDC() { HashMap configOptions = new HashMap<>(); diff --git a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java index 35bdb98d8d..d2bbd3a080 100644 --- a/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java @@ -31,10 +31,11 @@ import java.util.stream.Collectors; import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DDDaemonInitialization; +import org.apache.cassandra.CassandraTestBase.UseRandomPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.StubClusterMetadataService; @@ -48,18 +49,14 @@ import static org.junit.Assert.fail; /** * Unit tests for {@link PropertyFileSnitch}. */ -public class PropertyFileSnitchTest +@DDDaemonInitialization +@UseRandomPartitioner +public class PropertyFileSnitchTest extends CassandraTestBase { private Path effectiveFile; private Path backupFile; private InetAddressAndPort localAddress; - @BeforeClass - public static void setupDD() - { - DatabaseDescriptor.daemonInitialization(); - } - @Before public void setup() throws ConfigurationException, IOException { diff --git a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java index 874e50d17a..625e2e65ef 100644 --- a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java @@ -28,38 +28,47 @@ import java.util.concurrent.ExecutionException; import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; -import org.junit.*; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; import org.junit.rules.ExpectedException; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DisableMBeanRegistration; +import org.apache.cassandra.CassandraTestBase.PrepareServerNoRegister; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.dht.*; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.OrderPreservingPartitioner; import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.schema.ReplicationParams; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.tcm.transformations.Register; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.NodeVersion; +import org.apache.cassandra.tcm.transformations.Register; import org.apache.cassandra.utils.ByteBufferUtil; -import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION; import static org.apache.cassandra.ServerTestUtils.recreateCMS; +import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class SimpleStrategyTest +@PrepareServerNoRegister +@DisableMBeanRegistration +public class SimpleStrategyTest extends CassandraTestBase { public static final String KEYSPACE1 = "SimpleStrategyTest"; public static final String MULTIDC = "MultiDCSimpleStrategyTest"; @@ -69,16 +78,9 @@ public class SimpleStrategyTest ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(true); } - @BeforeClass - public static void defineSchema() + @Before + public void defineSchema() { - DatabaseDescriptor.daemonInitialization(); - } - - public static void withPartitioner(IPartitioner partitioner) - { - DatabaseDescriptor.setPartitionerUnsafe(partitioner); - ServerTestUtils.prepareServerNoRegister(); recreateCMS(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1)); SchemaLoader.createKeyspace(MULTIDC, KeyspaceParams.simple(3)); @@ -92,9 +94,10 @@ public class SimpleStrategyTest } @Test + @UseRandomPartitioner public void testBigIntegerEndpoints() throws UnknownHostException { - withPartitioner(RandomPartitioner.instance); + defineSchema(); List endpointTokens = new ArrayList<>(); List keyTokens = new ArrayList<>(); for (int i = 0; i < 5; i++) { @@ -105,24 +108,23 @@ public class SimpleStrategyTest } @Test + @UseOrderPreservingPartitioner public void testStringEndpoints() throws UnknownHostException { - IPartitioner partitioner = OrderPreservingPartitioner.instance; - withPartitioner(partitioner); + defineSchema(); List endpointTokens = new ArrayList(); List keyTokens = new ArrayList(); for (int i = 0; i < 5; i++) { endpointTokens.add(new StringToken(String.valueOf((char)('a' + i * 2)))); - keyTokens.add(partitioner.getToken(ByteBufferUtil.bytes(String.valueOf((char) ('a' + i * 2 + 1))))); + keyTokens.add(OrderPreservingPartitioner.instance.getToken(ByteBufferUtil.bytes(String.valueOf((char) ('a' + i * 2 + 1))))); } verifyGetNaturalEndpoints(endpointTokens.toArray(new Token[0]), keyTokens.toArray(new Token[0])); } @Test + @UseMurmur3Partitioner public void testMultiDCSimpleStrategyEndpoints() throws UnknownHostException { - withPartitioner(Murmur3Partitioner.instance); - // Topology taken directly from the topology_test.test_size_estimates_multidc dtest that regressed Multimap dc1 = HashMultimap.create(); dc1.put(InetAddressAndPort.getByName("127.0.0.1"), new Murmur3Partitioner.LongToken(-6639341390736545756L)); @@ -186,9 +188,10 @@ public class SimpleStrategyTest } @Test + @UseRandomPartitioner public void testGetEndpointsDuringBootstrap() throws UnknownHostException, ExecutionException, InterruptedException { - withPartitioner(RandomPartitioner.instance); + defineSchema(); // the token difference will be RING_SIZE * 2. final int RING_SIZE = 10; @@ -264,10 +267,9 @@ public class SimpleStrategyTest } @Test + @UseMurmur3Partitioner public void transientReplica() throws Exception { - withPartitioner(Murmur3Partitioner.instance); - List endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"), InetAddressAndPort.getByName("127.0.0.2"), InetAddressAndPort.getByName("127.0.0.3"), @@ -305,9 +307,10 @@ public class SimpleStrategyTest public ExpectedException expectedEx = ExpectedException.none(); @Test + @UseMurmur3Partitioner public void testSimpleStrategyThrowsConfigurationException() throws ConfigurationException, UnknownHostException { - withPartitioner(Murmur3Partitioner.instance); + defineSchema(); expectedEx.expect(ConfigurationException.class); expectedEx.expectMessage("SimpleStrategy requires a replication_factor strategy option."); @@ -327,9 +330,10 @@ public class SimpleStrategyTest } @Test + @UseMurmur3Partitioner public void shouldReturnNoEndpointsForEmptyRing() { - withPartitioner(Murmur3Partitioner.instance); + defineSchema(); HashMap configOptions = new HashMap<>(); configOptions.put("replication_factor", "1"); @@ -341,9 +345,10 @@ public class SimpleStrategyTest } @Test + @UseMurmur3Partitioner public void shouldWarnOnHigherReplicationFactorThanNodes() { - withPartitioner(Murmur3Partitioner.instance); + defineSchema(); HashMap configOptions = new HashMap<>(); configOptions.put("replication_factor", "2"); diff --git a/test/unit/org/apache/cassandra/net/MessageTest.java b/test/unit/org/apache/cassandra/net/MessageTest.java index ddc5f6b9c6..15d062e1a7 100644 --- a/test/unit/org/apache/cassandra/net/MessageTest.java +++ b/test/unit/org/apache/cassandra/net/MessageTest.java @@ -50,7 +50,7 @@ import static com.google.common.base.Throwables.getStackTraceAsString; import static org.apache.cassandra.exceptions.RemoteExceptionTest.normalizeThrowable; import static org.apache.cassandra.net.Message.serializer; import static org.apache.cassandra.net.MessagingService.VERSION_40; -import static org.apache.cassandra.net.MessagingService.VERSION_50; +import static org.apache.cassandra.net.MessagingService.VERSION_51; import static org.apache.cassandra.net.NoPayload.noPayload; import static org.apache.cassandra.net.ParamType.RESPOND_TO; import static org.apache.cassandra.net.ParamType.TRACE_SESSION; @@ -342,7 +342,7 @@ public class MessageTest RequestFailure reason1 = (RequestFailure)msg1.payload; RequestFailure reason2 = (RequestFailure)msg2.payload; assertEquals(reason1.reason, reason2.reason); - if (version >= VERSION_50) + if (version >= VERSION_51) { if (reason1.failure == null) assertNull(reason2.failure); diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java index ea32bd750b..2a589c028a 100644 --- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java @@ -37,12 +37,6 @@ import java.util.stream.Collectors; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; - -import org.apache.cassandra.repair.messages.SyncResponse; -import org.apache.cassandra.repair.messages.ValidationResponse; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.service.paxos.cleanup.PaxosRepairState; -import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -50,6 +44,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.ByteOrderedPartitioner; @@ -64,11 +59,14 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.SyncRequest; +import org.apache.cassandra.repair.messages.SyncResponse; +import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.paxos.Paxos; import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupRequest; import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupResponse; +import org.apache.cassandra.service.paxos.cleanup.PaxosRepairState; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -78,8 +76,11 @@ import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.asserts.SyncTaskListAssert; +import org.assertj.core.api.Assertions; import static java.util.Collections.emptySet; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_START_PREPARE_REQ; import static org.apache.cassandra.repair.RepairParallelism.SEQUENTIAL; import static org.apache.cassandra.streaming.PreviewKind.NONE; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -87,8 +88,6 @@ import static org.apache.cassandra.utils.asserts.SyncTaskAssert.assertThat; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_START_PREPARE_REQ; -import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_REQ; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -114,7 +113,7 @@ public class RepairJobTest private static InetAddressAndPort addr4; private static InetAddressAndPort addr5; private MeasureableRepairSession session; - private RepairJob job; + private CassandraRepairJob job; private RepairJobDesc sessionJobDesc; // So that threads actually get recycled and we can have accurate memory accounting while testing @@ -123,14 +122,14 @@ public class RepairJobTest { private final List> syncCompleteCallbacks = new ArrayList<>(); - public MeasureableRepairSession(TimeUUID parentRepairSession, CommonRange commonRange, String keyspace, + public MeasureableRepairSession(TimeUUID parentRepairSession, CommonRange commonRange, boolean excludedDeadNodes, String keyspace, RepairParallelism parallelismDegree, boolean isIncremental, boolean pullRepair, PreviewKind previewKind, boolean optimiseStreams, boolean repairPaxos, boolean paxosOnly, - boolean dontPurgeTombstones, String... cfnames) + boolean dontPurgeTombstones, boolean accordRepair, String... cfnames) { super(SharedContext.Global.instance, new Scheduler.NoopScheduler(), - parentRepairSession, commonRange, keyspace, parallelismDegree, isIncremental, pullRepair, - previewKind, optimiseStreams, repairPaxos, paxosOnly, dontPurgeTombstones, cfnames); + parentRepairSession, commonRange, excludedDeadNodes, keyspace, parallelismDegree, isIncremental, pullRepair, + previewKind, optimiseStreams, repairPaxos, paxosOnly, dontPurgeTombstones, accordRepair, cfnames); } @Override @@ -194,11 +193,11 @@ public class RepairJobTest ActiveRepairService.UNREPAIRED_SSTABLE, false, PreviewKind.NONE); this.session = new MeasureableRepairSession(parentRepairSession, - new CommonRange(neighbors, emptySet(), FULL_RANGE), + new CommonRange(neighbors, emptySet(), FULL_RANGE), false, KEYSPACE, SEQUENTIAL, false, false, - NONE, false, true, false, false, CF); + NONE, false, true, false, false, false, CF); - this.job = new RepairJob(session, CF); + this.job = new CassandraRepairJob(session, CF); this.sessionJobDesc = new RepairJobDesc(session.state.parentRepairSession, session.getId(), session.state.keyspace, CF, session.ranges()); @@ -268,7 +267,7 @@ public class RepairJobTest // Use addr4 instead of one of the provided trees to force everything to be remote sync tasks as // LocalSyncTasks try to reach over the network. - List syncTasks = RepairJob.createStandardSyncTasks(SharedContext.Global.instance, sessionJobDesc, mockTreeResponses, + List syncTasks = CassandraRepairJob.createStandardSyncTasks(SharedContext.Global.instance, sessionJobDesc, mockTreeResponses, addr4, // local noTransient(), session.isIncremental, @@ -330,7 +329,7 @@ public class RepairJobTest interceptRepairMessages(mockTrees, new ArrayList<>()); - try + try { job.run(); job.get(TEST_TIMEOUT_S, TimeUnit.SECONDS); @@ -368,7 +367,7 @@ public class RepairJobTest treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different"), treeResponse(addr3, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local noTransient(), // transient @@ -404,7 +403,7 @@ public class RepairJobTest List treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"), treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local transientPredicate(addr2), @@ -434,7 +433,7 @@ public class RepairJobTest List treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"), treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local transientPredicate(addr1), @@ -494,7 +493,7 @@ public class RepairJobTest List treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"), treeResponse(addr2, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, local, // local isTransient, @@ -512,13 +511,13 @@ public class RepairJobTest treeResponse(addr2, RANGE_1, "two", RANGE_2, "two", RANGE_3, "two"), treeResponse(addr3, RANGE_1, "three", RANGE_2, "three", RANGE_3, "three")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local ep -> ep.equals(addr3), // transient - false, - true, - PreviewKind.ALL)); + false, + true, + PreviewKind.ALL)); assertThat(tasks).hasSize(3); @@ -543,7 +542,7 @@ public class RepairJobTest treeResponse(addr5, RANGE_1, "five", RANGE_2, "five", RANGE_3, "five")); Predicate isTransient = ep -> ep.equals(addr4) || ep.equals(addr5); - Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local isTransient, // transient @@ -610,7 +609,7 @@ public class RepairJobTest treeResponse(addr5, RANGE_1, "five", RANGE_2, "five", RANGE_3, "five")); Predicate isTransient = ep -> ep.equals(addr4) || ep.equals(addr5); - Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, local, // local isTransient, // transient @@ -659,13 +658,13 @@ public class RepairJobTest treeResponse(addr4, RANGE_1, "four", RANGE_2, "four", RANGE_3, "four"), treeResponse(addr5, RANGE_1, "five", RANGE_2, "five", RANGE_3, "five")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr4, // local ep -> ep.equals(addr4) || ep.equals(addr5), // transient - false, - pullRepair, - PreviewKind.ALL)); + false, + pullRepair, + PreviewKind.ALL)); assertThat(tasks.get(pair(addr4, addr5))).isNull(); } @@ -677,13 +676,13 @@ public class RepairJobTest treeResponse(addr2, RANGE_1, "two", RANGE_2, "two", RANGE_3, "two"), treeResponse(addr3, RANGE_1, "three", RANGE_2, "three", RANGE_3, "three")); - Map tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createOptimisedSyncingSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local noTransient(), addr -> "DC1", - false, - PreviewKind.ALL)); + false, + PreviewKind.ALL)); for (SyncNodePair pair : new SyncNodePair[]{ pair(addr1, addr2), pair(addr1, addr3), @@ -712,13 +711,13 @@ public class RepairJobTest treeResponse(addr2, RANGE_1, "one", RANGE_2, "two"), treeResponse(addr3, RANGE_1, "three", RANGE_2, "two")); - Map tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(SharedContext.Global.instance, JOB_DESC, + Map tasks = toMap(CassandraRepairJob.createOptimisedSyncingSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr4, // local noTransient(), addr -> "DC1", - false, - PreviewKind.ALL)); + false, + PreviewKind.ALL)); SyncTaskListAssert.assertThat(tasks.values()).areAllInstanceOf(AsymmetricRemoteSyncTask.class); @@ -745,13 +744,13 @@ public class RepairJobTest treeResponse(addr3, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same")); RepairJobDesc desc = new RepairJobDesc(nextTimeUUID(), nextTimeUUID(), "ks", "cf", Collections.emptyList()); - Map tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(SharedContext.Global.instance, desc, + Map tasks = toMap(CassandraRepairJob.createOptimisedSyncingSyncTasks(SharedContext.Global.instance, desc, treeResponses, addr1, // local ep -> ep.equals(addr3), addr -> "DC1", - false, - PreviewKind.ALL)); + false, + PreviewKind.ALL)); SyncTask task = tasks.get(pair(addr1, addr2)); diff --git a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java index 470a2efc53..a5db870369 100644 --- a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java @@ -65,10 +65,9 @@ public class RepairSessionTest Set endpoints = Sets.newHashSet(remote); RepairSession session = new RepairSession(SharedContext.Global.instance, new Scheduler.NoopScheduler(), parentSessionId, new CommonRange(endpoints, Collections.emptySet(), Arrays.asList(repairRange)), - "Keyspace1", RepairParallelism.SEQUENTIAL, + false, "Keyspace1", RepairParallelism.SEQUENTIAL, false, false, - PreviewKind.NONE, false, false, false, false, - "Standard1"); + PreviewKind.NONE, false, false, false, false, false, "Standard1"); // perform convict session.convict(remote, Double.MAX_VALUE); diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java index 1657ceff48..9e8080f903 100644 --- a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java +++ b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java @@ -25,13 +25,13 @@ import java.util.List; import java.util.UUID; import com.google.common.collect.Lists; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DDDaemonInitialization; +import org.apache.cassandra.CassandraTestBase.UseMurmur3Partitioner; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; import org.apache.cassandra.dht.Range; @@ -52,37 +52,30 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.streaming.StreamSummary; import org.apache.cassandra.utils.MerkleTrees; +import static java.util.Collections.emptyList; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; -public class RepairMessageSerializationsTest +@UseMurmur3Partitioner +@DDDaemonInitialization +public class RepairMessageSerializationsTest extends CassandraTestBase { private static final int PROTOCOL_VERSION = MessagingService.current_version; private static final int GC_BEFORE = 1000000; - private static IPartitioner originalPartitioner; @BeforeClass public static void before() { - DatabaseDescriptor.daemonInitialization(); - originalPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); ClusterMetadataTestHelper.setInstanceForTest(); SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create("serializationsTestKeyspace", KeyspaceParams.simple(3))); SchemaTestUtil.announceNewTable(TableMetadata.minimal("serializationsTestKeyspace", "repairMessages")); } - @AfterClass - public static void after() - { - DatabaseDescriptor.setPartitionerUnsafe(originalPartitioner); - } - @Test public void validationRequestMessage() throws IOException { @@ -175,8 +168,8 @@ public class RepairMessageSerializationsTest InetAddressAndPort dst = InetAddressAndPort.getByName("127.0.0.3"); List summaries = new ArrayList<>(); summaries.add(new SessionSummary(src, dst, - Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), 5, 100)), - Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), 500, 10)) + Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 5, 100)), + Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 500, 10)) )); SyncResponse msg = new SyncResponse(buildRepairJobDesc(), new SyncNodePair(src, dst), true, summaries); serializeRoundTrip(msg, SyncResponse.serializer); diff --git a/test/unit/org/apache/cassandra/schema/ValidationTest.java b/test/unit/org/apache/cassandra/schema/ValidationTest.java index 75727f218f..e9edbf729d 100644 --- a/test/unit/org/apache/cassandra/schema/ValidationTest.java +++ b/test/unit/org/apache/cassandra/schema/ValidationTest.java @@ -27,14 +27,19 @@ import java.util.Set; import org.apache.cassandra.config.DatabaseDescriptor; import org.junit.BeforeClass; + import org.junit.Test; import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DDDaemonInitialization; + import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class ValidationTest +@DDDaemonInitialization +public class ValidationTest extends CassandraTestBase { @BeforeClass public static void beforeClass() diff --git a/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java b/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java index d6f4aa9a09..b0207565c0 100644 --- a/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java +++ b/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java @@ -30,6 +30,9 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DDDaemonInitialization; +import org.apache.cassandra.CassandraTestBase.UseOrderPreservingPartitioner; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.OrderPreservingPartitioner; import org.apache.cassandra.dht.Range; @@ -63,7 +66,9 @@ import static org.apache.cassandra.service.StorageServiceTest.assertMultimapEqua * is used to calculate the endpoints to fetch from and check they are alive for both RangeRelocator (move) and * bootstrap (RangeRelocator). */ -public class BootstrapTransientTest +@DDDaemonInitialization +@UseOrderPreservingPartitioner +public class BootstrapTransientTest extends CassandraTestBase { static final String KEYSPACE = "TestKeyspace"; static InetAddressAndPort address02; diff --git a/test/unit/org/apache/cassandra/service/RemoveTest.java b/test/unit/org/apache/cassandra/service/RemoveTest.java index b2c664ebf2..9e85c6a656 100644 --- a/test/unit/org/apache/cassandra/service/RemoveTest.java +++ b/test/unit/org/apache/cassandra/service/RemoveTest.java @@ -24,15 +24,16 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.PrepareServerNoRegister; +import org.apache.cassandra.CassandraTestBase.UseRandomPartitioner; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.dht.Token; @@ -43,19 +44,14 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.tcm.membership.NodeId; -import static org.apache.cassandra.tcm.membership.MembershipUtils.*; +import static org.apache.cassandra.tcm.membership.MembershipUtils.endpoint; -public class RemoveTest +@UseRandomPartitioner +@PrepareServerNoRegister +public class RemoveTest extends CassandraTestBase { - static - { - DatabaseDescriptor.daemonInitialization(); - CommitLog.instance.start(); - } - static final IPartitioner partitioner = RandomPartitioner.instance; StorageService ss = StorageService.instance; - static IPartitioner oldPartitioner; ArrayList endpointTokens = new ArrayList(); ArrayList keyTokens = new ArrayList(); List hosts = new ArrayList<>(); @@ -66,17 +62,9 @@ public class RemoveTest @BeforeClass public static void setupClass() throws ConfigurationException { - oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner); - ServerTestUtils.prepareServerNoRegister(); MessagingService.instance().listen(); } - @AfterClass - public static void tearDownClass() - { - StorageService.instance.setPartitionerUnsafe(oldPartitioner); - } - @Before public void setup() throws IOException, ConfigurationException { diff --git a/test/unit/org/apache/cassandra/service/SerializationsTest.java b/test/unit/org/apache/cassandra/service/SerializationsTest.java index 20431fc335..9251b590bb 100644 --- a/test/unit/org/apache/cassandra/service/SerializationsTest.java +++ b/test/unit/org/apache/cassandra/service/SerializationsTest.java @@ -26,9 +26,6 @@ import java.util.List; import java.util.UUID; import com.google.common.collect.Lists; - -import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; -import org.apache.cassandra.io.util.FileInputStreamPlus; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -41,13 +38,19 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputStreamPlus; +import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.repair.SyncNodePair; import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.repair.SyncNodePair; import org.apache.cassandra.repair.Validator; -import org.apache.cassandra.repair.messages.*; +import org.apache.cassandra.repair.messages.RepairMessage; +import org.apache.cassandra.repair.messages.SyncRequest; +import org.apache.cassandra.repair.messages.SyncResponse; +import org.apache.cassandra.repair.messages.ValidationRequest; +import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.repair.state.ValidationState; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; @@ -62,6 +65,8 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTrees; import org.apache.cassandra.utils.TimeUUID; +import static java.util.Collections.emptyList; + public class SerializationsTest extends AbstractSerializationsTester { private static PartitionerSwitcher partitionerSwitcher; @@ -218,12 +223,12 @@ public class SerializationsTest extends AbstractSerializationsTester // sync success List summaries = new ArrayList<>(); summaries.add(new SessionSummary(src, dest, - Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), 5, 100)), - Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), 500, 10)) + Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 5, 100)), + Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 500, 10)) )); SyncResponse success = new SyncResponse(DESC, src, dest, true, summaries); // sync fail - SyncResponse fail = new SyncResponse(DESC, src, dest, false, Collections.emptyList()); + SyncResponse fail = new SyncResponse(DESC, src, dest, false, emptyList()); testRepairMessageWrite("service.SyncComplete.bin", SyncResponse.serializer, success, fail); } diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java index 7814b82ba0..3405dc9bbf 100644 --- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java +++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java @@ -36,11 +36,11 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.CassandraTestBase; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.audit.AuditLogManager; import org.apache.cassandra.audit.AuditLogOptions; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; import org.apache.cassandra.dht.OrderPreservingPartitioner; import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; @@ -49,7 +49,6 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.WithPartitioner; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.ReplicationParams; @@ -69,7 +68,7 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.GOSSIP_DIS import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class StorageServiceServerTest +public class StorageServiceServerTest extends CassandraTestBase { static final String DC1 = "DC1"; static final String DC2 = "DC2"; @@ -94,7 +93,6 @@ public class StorageServiceServerTest id4 = InetAddressAndPort.getByName("127.0.0.4"); id5 = InetAddressAndPort.getByName("127.0.0.5"); registerNodes(); - ServerTestUtils.markCMS(); } private static void registerNodes() @@ -159,6 +157,7 @@ public class StorageServiceServerTest } @Test + @UseOrderPreservingPartitioner public void testLocalPrimaryRangeForEndpointWithNetworkTopologyStrategy() throws Exception { setupDefaultPlacements(); @@ -190,6 +189,7 @@ public class StorageServiceServerTest } @Test + @UseOrderPreservingPartitioner public void testPrimaryRangeForEndpointWithinDCWithNetworkTopologyStrategy() throws Exception { setupDefaultPlacements(); @@ -225,6 +225,7 @@ public class StorageServiceServerTest } @Test + @UseOrderPreservingPartitioner public void testPrimaryRangesWithNetworkTopologyStrategy() throws Exception { setupDefaultPlacements(); @@ -255,6 +256,7 @@ public class StorageServiceServerTest } @Test + @UseOrderPreservingPartitioner public void testPrimaryRangesWithNetworkTopologyStrategyOneDCOnly() throws Exception { setupDefaultPlacements(); @@ -286,6 +288,7 @@ public class StorageServiceServerTest } @Test + @UseOrderPreservingPartitioner public void testPrimaryRangeForEndpointWithinDCWithNetworkTopologyStrategyOneDCOnly() throws Exception { setupDefaultPlacements(); @@ -317,6 +320,7 @@ public class StorageServiceServerTest } @Test + @UseOrderPreservingPartitioner public void testPrimaryRangesWithVnodes() throws Exception { // DC1 @@ -367,6 +371,7 @@ public class StorageServiceServerTest } @Test + @UseOrderPreservingPartitioner public void testPrimaryRangeForEndpointWithinDCWithVnodes() throws Exception { // DC1 @@ -431,6 +436,7 @@ public class StorageServiceServerTest } @Test + @UseOrderPreservingPartitioner public void testPrimaryRangesWithSimpleStrategy() throws Exception { ClusterMetadataTestHelper.join(id1, new StringToken("A")); @@ -456,6 +462,7 @@ public class StorageServiceServerTest /* Does not make much sense to use -local and -pr with simplestrategy, but just to prevent human errors */ @Test + @UseOrderPreservingPartitioner public void testPrimaryRangeForEndpointWithinDCWithSimpleStrategy() throws Exception { ClusterMetadataTestHelper.join(id1, new StringToken("A")); @@ -483,46 +490,43 @@ public class StorageServiceServerTest } @Test + @UseMurmur3Partitioner public void testCreateRepairRangeFrom() throws Exception { - try (WithPartitioner m3p = new WithPartitioner(Murmur3Partitioner.instance)) - { - registerNodes(); - ClusterMetadataTestHelper.join(id1, new LongToken(1000L)); - ClusterMetadataTestHelper.join(id2, new LongToken(2000L)); - ClusterMetadataTestHelper.join(id3, new LongToken(3000L)); - ClusterMetadataTestHelper.join(id4, new LongToken(4000L)); + ClusterMetadataTestHelper.join(id1, new LongToken(1000L)); + ClusterMetadataTestHelper.join(id2, new LongToken(2000L)); + ClusterMetadataTestHelper.join(id3, new LongToken(3000L)); + ClusterMetadataTestHelper.join(id4, new LongToken(4000L)); - Collection> repairRangeFrom = StorageService.instance.createRepairRangeFrom("1500", "3700"); - Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(3); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(1500L), new LongToken(2000L))); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(2000L), new LongToken(3000L))); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(3000L), new LongToken(3700L))); + Collection> repairRangeFrom = StorageService.instance.createRepairRangeFrom("1500", "3700"); + Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(3); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(1500L), new LongToken(2000L))); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(2000L), new LongToken(3000L))); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(3000L), new LongToken(3700L))); - repairRangeFrom = StorageService.instance.createRepairRangeFrom("500", "700"); - Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(1); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(500L), new LongToken(700L))); + repairRangeFrom = StorageService.instance.createRepairRangeFrom("500", "700"); + Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(1); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(500L), new LongToken(700L))); - repairRangeFrom = StorageService.instance.createRepairRangeFrom("500", "1700"); - Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(2); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(500L), new LongToken(1000L))); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(1000L), new LongToken(1700L))); + repairRangeFrom = StorageService.instance.createRepairRangeFrom("500", "1700"); + Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(2); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(500L), new LongToken(1000L))); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(1000L), new LongToken(1700L))); - repairRangeFrom = StorageService.instance.createRepairRangeFrom("2500", "2300"); - Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(5); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(2500L), new LongToken(3000L))); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(3000L), new LongToken(4000L))); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(4000L), new LongToken(1000L))); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(1000L), new LongToken(2000L))); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(2000L), new LongToken(2300L))); + repairRangeFrom = StorageService.instance.createRepairRangeFrom("2500", "2300"); + Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(5); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(2500L), new LongToken(3000L))); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(3000L), new LongToken(4000L))); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(4000L), new LongToken(1000L))); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(1000L), new LongToken(2000L))); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(2000L), new LongToken(2300L))); - repairRangeFrom = StorageService.instance.createRepairRangeFrom("2000", "3000"); - Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(1); - Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(2000L), new LongToken(3000L))); + repairRangeFrom = StorageService.instance.createRepairRangeFrom("2000", "3000"); + Assertions.assertThat(repairRangeFrom.size()).as(repairRangeFrom.toString()).isEqualTo(1); + Assertions.assertThat(repairRangeFrom).contains(new Range<>(new LongToken(2000L), new LongToken(3000L))); - repairRangeFrom = StorageService.instance.createRepairRangeFrom("2000", "2000"); - Assertions.assertThat(repairRangeFrom).isEmpty(); - } + repairRangeFrom = StorageService.instance.createRepairRangeFrom("2000", "2000"); + Assertions.assertThat(repairRangeFrom).isEmpty(); } /** diff --git a/test/unit/org/apache/cassandra/service/accord/AccordReadRepairTest.java b/test/unit/org/apache/cassandra/service/accord/AccordReadRepairTest.java new file mode 100644 index 0000000000..8d8fb0fc46 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/accord/AccordReadRepairTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord; + +import java.io.IOException; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.IMessageFilters.Filter; +import org.apache.cassandra.distributed.test.accord.AccordTestBase; +import org.apache.cassandra.net.Verb; + +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.util.QueryResultUtil.assertThat; + +public class AccordReadRepairTest extends AccordTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(org.apache.cassandra.distributed.test.accord.AccordCQLTest.class); + + @Override + protected Logger logger() + { + return logger; + } + + @BeforeClass + public static void setupClass() throws IOException + { + AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("lwt_strategy", "accord").set("non_serial_write_strategy", "mixed")), 2); + SHARED_CLUSTER.schemaChange("CREATE TYPE " + KEYSPACE + ".person (height int, age int)"); + SHARED_CLUSTER.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE)); + } + + /* + * SERIAL read and CAS create Accord transactions which will then invoke Cassandra coordination to perform the read + * and proxy any read repairs that are generated. + */ + @Test + public void testSerialReadRepair() throws Exception + { + testReadRepair(cluster -> cluster.coordinator(1).execute("SELECT * FROM " + currentTable + " WHERE k = 1 AND c = 1;", ConsistencyLevel.SERIAL), + new Object[][] {{1, 1, 1, 1}}); + } + + @Test + public void testCASFailedConditionReadRepair() throws Exception + { + // Even if the condition fails to apply the data checked when applying the condition should be repaired + testReadRepair(cluster -> cluster.coordinator(1).execute("INSERT INTO " + currentTable + " (k, c, v1) VALUES (1, 1, 99) IF NOT EXISTS;", ConsistencyLevel.SERIAL), + new Object[][] {{false, 1, 1, 1, 1}}); + } + + @Test + public void testCASReadRepair() throws Exception + { + // If the condition applies the read repair should preserve the existing timestamp + testReadRepair(cluster -> cluster.coordinator(1).execute("UPDATE " + currentTable + " SET v2 = 99 WHERE k = 1 and c = 1 IF EXISTS;", ConsistencyLevel.SERIAL), + new Object[][] {{Boolean.TRUE}}); + } + + /* + * non-SERIAL consistency levels are coordinated by C* and then if a partition needs to be repaired an Accord transaction + * is created for each partition repair to proxy the repair mutations safely. + */ + @Test + public void testNonSerialReadRepair() throws Exception + { + for (ConsistencyLevel cl : ImmutableList.of(ConsistencyLevel.QUORUM)) + testReadRepair(cluster -> cluster.coordinator(1).execute("SELECT * FROM " + currentTable + " WHERE k = 1 AND c = 1;", cl), + new Object[][] {{1, 1, 1, 1}}); + } + + void testReadRepair(Function accordTxn, Object[][] expected) throws Exception + { + test("CREATE TABLE " + currentTable + " (k int, c int, v1 int, v2 int, PRIMARY KEY ((k), c));", + cluster -> { + Filter mutationFilter = cluster.filters().verbs(Verb.MUTATION_REQ.id).drop().on(); + cluster.filters().verbs(Verb.HINT_REQ.id, Verb.HINT_RSP.id).drop().on(); + cluster.coordinator(1).execute("INSERT INTO " + currentTable + " (k, c, v1, v2) VALUES (1, 1, 1, 1) USING TIMESTAMP 42;", ConsistencyLevel.ONE); + mutationFilter.off(); + Filter blockNodeOneReads = cluster.filters().verbs(Verb.READ_REQ.id).to(1).drop().on(); + assertThat(cluster.coordinator(2).executeWithResult("SELECT * FROM " + currentTable + " WHERE k = 1 AND c = 1;", ConsistencyLevel.ONE)) + .isEmpty(); + blockNodeOneReads.off(); + // Should perform read repair + Object[][] result = accordTxn.apply(cluster); + assertRows(result, expected); + blockNodeOneReads.on(); + // Side effect of the read repair should be visible now + assertThat(cluster.coordinator(2).executeWithResult("SELECT k, c, v1, WRITETIME(v1) FROM " + currentTable + " WHERE k = 1 AND c = 1;", ConsistencyLevel.ONE)) + .isEqualTo(1, 1, 1, 42L); + }); + } +} diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index 4e21b5096c..8a64409cca 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -214,7 +214,7 @@ public class AccordTestUtils Data readData = read.keys().stream().map(key -> { try { - return AsyncChains.getBlocking(read.read(key, txn.kind(), safeStore, executeAt, null)); + return AsyncChains.getBlocking(read.read(key, safeStore, executeAt, null)); } catch (InterruptedException e) { @@ -227,7 +227,7 @@ public class AccordTestUtils }) .reduce(null, TxnData::merge); return Pair.create(txn.execute(txnId, executeAt, readData), - txn.query().compute(txnId, executeAt, readData, txn.read(), txn.update())); + txn.query().compute(txnId, executeAt, txn.keys(), readData, txn.read(), txn.update())); } diff --git a/test/unit/org/apache/cassandra/service/accord/txn/TxnUpdateTest.java b/test/unit/org/apache/cassandra/service/accord/txn/AccordUpdateTest.java similarity index 91% rename from test/unit/org/apache/cassandra/service/accord/txn/TxnUpdateTest.java rename to test/unit/org/apache/cassandra/service/accord/txn/AccordUpdateTest.java index 203e60ab16..fbfd1190cc 100644 --- a/test/unit/org/apache/cassandra/service/accord/txn/TxnUpdateTest.java +++ b/test/unit/org/apache/cassandra/service/accord/txn/AccordUpdateTest.java @@ -18,18 +18,18 @@ package org.apache.cassandra.service.accord.txn; -import org.apache.cassandra.service.accord.AccordTestUtils; import org.junit.BeforeClass; import org.junit.Test; import accord.primitives.Txn; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.service.accord.AccordTestUtils; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; import static org.apache.cassandra.utils.SerializerTestUtils.assertSerializerIOEquality; -public class TxnUpdateTest +public class AccordUpdateTest { @BeforeClass public static void setupClass() @@ -44,7 +44,7 @@ public class TxnUpdateTest public void predicateSerializer() { Txn txn = AccordTestUtils.createTxn(0, 0); - TxnUpdate update = (TxnUpdate) txn.update(); - assertSerializerIOEquality(update, TxnUpdate.serializer); + AccordUpdate update = (AccordUpdate) txn.update(); + assertSerializerIOEquality(update, AccordUpdate.serializer); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/service/paxos/AbstractPaxosRepairTest.java b/test/unit/org/apache/cassandra/service/paxos/AbstractPaxosRepairTest.java index 9721879d56..b91604792d 100644 --- a/test/unit/org/apache/cassandra/service/paxos/AbstractPaxosRepairTest.java +++ b/test/unit/org/apache/cassandra/service/paxos/AbstractPaxosRepairTest.java @@ -51,7 +51,7 @@ public class AbstractPaxosRepairTest { public PaxosTestRepair() { - super(Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1)), null); + super(Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1)), null, -1); } public State restart(State state, long waitUntil) diff --git a/test/unit/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairsTest.java b/test/unit/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairsTest.java index 22441fec42..fe21b820c5 100644 --- a/test/unit/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairsTest.java +++ b/test/unit/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairsTest.java @@ -51,7 +51,7 @@ public class PaxosTableRepairsTest public MockRepair(DecoratedKey key) { - super(key, null); + super(key, null, -1); } public State restart(State state, long waitUntil) diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTrackerTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTrackerTest.java index 1f9db0851f..c9ec812f10 100644 --- a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTrackerTest.java +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTrackerTest.java @@ -20,23 +20,26 @@ package org.apache.cassandra.service.paxos.uncommitted; import java.io.IOException; -import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.paxos.Ballot; -import org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome; -import org.junit.*; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.service.paxos.Paxos; import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome; import org.apache.cassandra.utils.ByteBufferUtil; import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.REJECT; @@ -126,7 +129,7 @@ public class PaxosBallotTrackerTest case PROPOSE: try (PaxosState state = PaxosState.get(commit)) { - state.acceptIfLatest(commit); + state.acceptIfLatest(commit, false); } break; case COMMIT: @@ -220,7 +223,7 @@ public class PaxosBallotTrackerTest DecoratedKey key = dk(1); try (PaxosState state = PaxosState.get(key, cfm)) { - Ballot result = state.acceptIfLatest(new Commit.Proposal(ballot2, PartitionUpdate.emptyUpdate(cfm, key))); + Ballot result = state.acceptIfLatest(new Commit.Proposal(ballot2, PartitionUpdate.emptyUpdate(cfm, key)), false).supersededBy; Assert.assertNull(result); } @@ -228,7 +231,7 @@ public class PaxosBallotTrackerTest ballotTracker.updateLowBound(ballot4); try (PaxosState state = PaxosState.get(key, cfm)) { - Ballot result = state.acceptIfLatest(new Commit.Proposal(ballot3, PartitionUpdate.emptyUpdate(cfm, key))); + Ballot result = state.acceptIfLatest(new Commit.Proposal(ballot3, PartitionUpdate.emptyUpdate(cfm, key)), false).supersededBy; Assert.assertEquals(ballot4, result); } } diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTests.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTests.java index 2804508e64..654c10b6d0 100644 --- a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTests.java +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTests.java @@ -18,14 +18,23 @@ package org.apache.cassandra.service.paxos.uncommitted; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; import com.google.common.collect.Lists; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.commitlog.CommitLog; -import org.apache.cassandra.dht.*; +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.utils.ByteBufferUtil; @@ -42,7 +51,7 @@ class PaxosUncommittedTests CommitLog.instance.start(); } - static final IPartitioner PARTITIONER = new ByteOrderedPartitioner(); + static final IPartitioner PARTITIONER = ByteOrderedPartitioner.instance; static final Token MIN_TOKEN = PARTITIONER.getMinimumToken(); static final Range FULL_RANGE = new Range<>(MIN_TOKEN, MIN_TOKEN); static final Collection> ALL_RANGES = Collections.singleton(FULL_RANGE); diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java index 8c3dd250f5..dc8fafa966 100644 --- a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java @@ -20,7 +20,10 @@ package org.apache.cassandra.service.paxos.uncommitted; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import org.junit.*; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; @@ -36,7 +39,7 @@ import org.apache.cassandra.utils.CloseableIterator; import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; -import static org.apache.cassandra.service.paxos.Commit.*; +import static org.apache.cassandra.service.paxos.Commit.Proposal; import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.ALL_RANGES; import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.PAXOS_CFS; @@ -97,7 +100,7 @@ public class PaxosUncommittedTrackerIntegrationTest try (PaxosState state = PaxosState.get(key, cfm)) { - state.acceptIfLatest(proposal); + state.acceptIfLatest(proposal, false); } try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfm.id, ALL_RANGES)) @@ -124,7 +127,7 @@ public class PaxosUncommittedTrackerIntegrationTest try (PaxosState state = PaxosState.get(key, cfm)) { state.promiseIfNewer(proposal.ballot, true); - state.acceptIfLatest(proposal); + state.acceptIfLatest(proposal, false); } try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfm.id, ALL_RANGES)) { diff --git a/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java b/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java index 7d22439d8d..3c0450d2e8 100644 --- a/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java +++ b/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java @@ -30,10 +30,12 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; +import org.apache.cassandra.CassandraTestBase; +import org.apache.cassandra.CassandraTestBase.DDDaemonInitialization; +import org.apache.cassandra.CassandraTestBase.UseMurmur3Partitioner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.Util; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.db.BufferClusteringBound; import org.apache.cassandra.db.BufferClusteringBoundary; @@ -66,7 +68,6 @@ import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.rows.Rows; import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.db.rows.UnfilteredRowIterator; -import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; @@ -83,7 +84,9 @@ import static org.apache.cassandra.net.Verb.READ_REQ; * Base class for testing various components which deal with read responses */ @Ignore -public abstract class AbstractReadResponseTest +@DDDaemonInitialization +@UseMurmur3Partitioner +public abstract class AbstractReadResponseTest extends CassandraTestBase { public static final String KEYSPACE1 = "DataResolverTest"; public static final String KEYSPACE3 = "DataResolverTest3"; @@ -124,9 +127,6 @@ public abstract class AbstractReadResponseTest @BeforeClass public static void setupClass() throws Throwable { - DatabaseDescriptor.daemonInitialization(); - DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); - TableMetadata.Builder builder1 = TableMetadata.builder(KEYSPACE1, CF_STANDARD) .addPartitionKeyColumn("key", BytesType.instance) diff --git a/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java index d281025666..19e7724807 100644 --- a/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java +++ b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java @@ -25,17 +25,18 @@ import java.util.function.BiFunction; import com.google.common.collect.Iterators; import com.google.common.collect.Sets; - import org.junit.Assert; import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DeletionInfo; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.EmptyIterators; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.MutableDeletionInfo; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.RangeTombstone; @@ -54,8 +55,6 @@ import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker; import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.RowIterator; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.EndpointsForRange; @@ -66,7 +65,7 @@ import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.locator.ReplicaUtils; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.net.*; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.reads.repair.ReadRepair; import org.apache.cassandra.service.reads.repair.RepairedDataTracker; import org.apache.cassandra.service.reads.repair.RepairedDataVerifier; @@ -137,7 +136,7 @@ public class DataResolverTest extends AbstractReadResponseTest public void testResolveNewerSingleRow() { EndpointsForRange replicas = makeReplicas(2); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); InetAddressAndPort peer1 = replicas.get(0).endpoint(); resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") .add("c1", "v1") @@ -169,7 +168,7 @@ public class DataResolverTest extends AbstractReadResponseTest public void testResolveDisjointSingleRow() { EndpointsForRange replicas = makeReplicas(2); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); InetAddressAndPort peer1 = replicas.get(0).endpoint(); resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") .add("c1", "v1") @@ -206,7 +205,7 @@ public class DataResolverTest extends AbstractReadResponseTest public void testResolveDisjointMultipleRows() throws UnknownHostException { EndpointsForRange replicas = makeReplicas(2); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); InetAddressAndPort peer1 = replicas.get(0).endpoint(); resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") .add("c1", "v1") @@ -253,7 +252,7 @@ public class DataResolverTest extends AbstractReadResponseTest public void testResolveDisjointMultipleRowsWithRangeTombstones() { EndpointsForRange replicas = makeReplicas(4); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); RangeTombstone tombstone1 = tombstone("1", "11", 1, nowInSec); RangeTombstone tombstone2 = tombstone("3", "31", 1, nowInSec); @@ -334,7 +333,7 @@ public class DataResolverTest extends AbstractReadResponseTest public void testResolveWithOneEmpty() { EndpointsForRange replicas = makeReplicas(2); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); InetAddressAndPort peer1 = replicas.get(0).endpoint(); resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") .add("c2", "v2") @@ -365,7 +364,7 @@ public class DataResolverTest extends AbstractReadResponseTest { EndpointsForRange replicas = makeReplicas(2); TestableReadRepair readRepair = new TestableReadRepair(command); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); resolver.preprocess(response(command, replicas.get(0).endpoint(), EmptyIterators.unfilteredPartition(cfm))); resolver.preprocess(response(command, replicas.get(1).endpoint(), EmptyIterators.unfilteredPartition(cfm))); @@ -381,7 +380,7 @@ public class DataResolverTest extends AbstractReadResponseTest public void testResolveDeleted() { EndpointsForRange replicas = makeReplicas(2); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); // one response with columns timestamped before a delete in another response InetAddressAndPort peer1 = replicas.get(0).endpoint(); resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") @@ -407,7 +406,7 @@ public class DataResolverTest extends AbstractReadResponseTest public void testResolveMultipleDeleted() { EndpointsForRange replicas = makeReplicas(4); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); // deletes and columns with interleaved timestamp, with out of order return sequence InetAddressAndPort peer1 = replicas.get(0).endpoint(); resolver.preprocess(response(command, peer1, fullPartitionDelete(cfm, dk, 0, nowInSec))); @@ -492,7 +491,7 @@ public class DataResolverTest extends AbstractReadResponseTest private void resolveRangeTombstonesOnBoundary(long timestamp1, long timestamp2) { EndpointsForRange replicas = makeReplicas(2); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); InetAddressAndPort peer1 = replicas.get(0).endpoint(); InetAddressAndPort peer2 = replicas.get(1).endpoint(); @@ -566,7 +565,7 @@ public class DataResolverTest extends AbstractReadResponseTest */ private void testRepairRangeTombstoneBoundary(EndpointsForRange replicas, int timestamp1, int timestamp2, int timestamp3) throws UnknownHostException { - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); InetAddressAndPort peer1 = replicas.get(0).endpoint(); InetAddressAndPort peer2 = replicas.get(1).endpoint(); @@ -619,7 +618,7 @@ public class DataResolverTest extends AbstractReadResponseTest { EndpointsForRange replicas = makeReplicas(2); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); InetAddressAndPort peer1 = replicas.get(0).endpoint(); InetAddressAndPort peer2 = replicas.get(1).endpoint(); @@ -658,7 +657,7 @@ public class DataResolverTest extends AbstractReadResponseTest public void testRepairRangeTombstoneWithPartitionDeletion2() { EndpointsForRange replicas = makeReplicas(2); - DataResolver resolver = new DataResolver(command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, command, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); InetAddressAndPort peer1 = replicas.get(0).endpoint(); InetAddressAndPort peer2 = replicas.get(1).endpoint(); @@ -742,7 +741,7 @@ public class DataResolverTest extends AbstractReadResponseTest EndpointsForRange replicas = makeReplicas(2); ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build(); TestableReadRepair readRepair = new TestableReadRepair(cmd); - DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); long[] ts = {100, 200}; @@ -794,7 +793,7 @@ public class DataResolverTest extends AbstractReadResponseTest EndpointsForRange replicas = makeReplicas(2); ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build(); TestableReadRepair readRepair = new TestableReadRepair(cmd); - DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); long[] ts = {100, 200}; @@ -838,7 +837,7 @@ public class DataResolverTest extends AbstractReadResponseTest EndpointsForRange replicas = makeReplicas(2); ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build(); TestableReadRepair readRepair = new TestableReadRepair(cmd); - DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); long[] ts = {100, 200}; @@ -888,7 +887,7 @@ public class DataResolverTest extends AbstractReadResponseTest EndpointsForRange replicas = makeReplicas(2); ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build(); TestableReadRepair readRepair = new TestableReadRepair(cmd); - DataResolver resolver = new DataResolver(cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); + DataResolver resolver = new DataResolver(ReadCoordinator.DEFAULT, cmd, plan(replicas, ALL), readRepair, Dispatcher.RequestTime.forImmediateExecution()); long[] ts = {100, 200}; @@ -1260,7 +1259,7 @@ public class DataResolverTest extends AbstractReadResponseTest public TestableDataResolver(ReadCommand command, ReplicaPlan.SharedForRangeRead plan, ReadRepair readRepair, Dispatcher.RequestTime requestTime) { - super(command, plan, readRepair, requestTime, true); + super(ReadCoordinator.DEFAULT, command, plan, readRepair, requestTime, true); } protected RepairedDataVerifier getRepairedDataVerifier(ReadCommand command) @@ -1326,7 +1325,7 @@ public class DataResolverTest extends AbstractReadResponseTest private ReplicaPlan.SharedForRangeRead plan(EndpointsForRange replicas, ConsistencyLevel consistencyLevel) { - BiFunction, Token, ReplicaPlan.ForWrite> repairPlan = (self, t) -> ReplicaPlans.forReadRepair(self, ClusterMetadata.current(), ks, consistencyLevel, t, (i) -> true); + BiFunction, Token, ReplicaPlan.ForWrite> repairPlan = (self, t) -> ReplicaPlans.forReadRepair(self, ClusterMetadata.current(), ks, consistencyLevel, t, (i) -> true, ReadCoordinator.DEFAULT); return ReplicaPlan.shared(new ReplicaPlan.ForRangeRead(ks, ks.getReplicationStrategy(), consistencyLevel, diff --git a/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java index 17baa4fa55..bde9763baf 100644 --- a/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java +++ b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java @@ -69,7 +69,7 @@ public class DigestResolverTest extends AbstractReadResponseTest { SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2)); - DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); + DigestResolver resolver = new DigestResolver(ReadCoordinator.DEFAULT, command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); PartitionUpdate response = update(row(1000, 4, 4), row(1000, 5, 5)).build(); @@ -102,7 +102,7 @@ public class DigestResolverTest extends AbstractReadResponseTest { final long startNanos = System.nanoTime(); final Dispatcher.RequestTime requestTime = new Dispatcher.RequestTime(startNanos, startNanos); - final DigestResolver resolver = new DigestResolver<>(command, plan, requestTime); + final DigestResolver resolver = new DigestResolver<>(ReadCoordinator.DEFAULT, command, plan, requestTime); final ReadCallback callback = new ReadCallback<>(resolver, command, plan, requestTime); final CountDownLatch startlatch = new CountDownLatch(2); @@ -137,7 +137,7 @@ public class DigestResolverTest extends AbstractReadResponseTest { SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2)); - DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); + DigestResolver resolver = new DigestResolver(ReadCoordinator.DEFAULT, command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); PartitionUpdate response1 = update(row(1000, 4, 4), row(1000, 5, 5)).build(); PartitionUpdate response2 = update(row(2000, 4, 5)).build(); @@ -158,7 +158,7 @@ public class DigestResolverTest extends AbstractReadResponseTest { SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), trans(EP2)); - DigestResolver resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); + DigestResolver resolver = new DigestResolver<>(ReadCoordinator.DEFAULT, command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); PartitionUpdate response1 = update(row(1000, 4, 4), row(1000, 5, 5)).build(); PartitionUpdate response2 = update(row(1000, 5, 5)).build(); @@ -179,7 +179,7 @@ public class DigestResolverTest extends AbstractReadResponseTest { SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), trans(EP2)); - DigestResolver resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); + DigestResolver resolver = new DigestResolver<>(ReadCoordinator.DEFAULT, command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); PartitionUpdate response2 = update(row(1000, 5, 5)).build(); Assert.assertFalse(resolver.isDataPresent()); @@ -194,7 +194,7 @@ public class DigestResolverTest extends AbstractReadResponseTest { SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2), trans(EP3)); - DigestResolver resolver = new DigestResolver<>(command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); + DigestResolver resolver = new DigestResolver<>(ReadCoordinator.DEFAULT, command, plan(ConsistencyLevel.QUORUM, targetReplicas), new Dispatcher.RequestTime(0L, 0L)); PartitionUpdate fullResponse = update(row(1000, 1, 1)).build(); PartitionUpdate digestResponse = update(row(1000, 1, 1)).build(); diff --git a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java index 989130adb4..832fbcbe23 100644 --- a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java +++ b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java @@ -101,7 +101,7 @@ public class ReadExecutorTest { assertEquals(0, cfs.metric.speculativeInsufficientReplicas.getCount()); assertEquals(0, ks.metric.speculativeInsufficientReplicas.getCount()); - AbstractReadExecutor executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, LOCAL_QUORUM), Dispatcher.RequestTime.forImmediateExecution(), true); + AbstractReadExecutor executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(ReadCoordinator.DEFAULT, cfs, new MockSinglePartitionReadCommand(), plan(targets, LOCAL_QUORUM), Dispatcher.RequestTime.forImmediateExecution(), true); executor.maybeTryAdditionalReplicas(); try { @@ -116,7 +116,7 @@ public class ReadExecutorTest assertEquals(1, ks.metric.speculativeInsufficientReplicas.getCount()); //Shouldn't increment - executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, LOCAL_QUORUM), Dispatcher.RequestTime.forImmediateExecution(), false); + executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(ReadCoordinator.DEFAULT, cfs, new MockSinglePartitionReadCommand(), plan(targets, LOCAL_QUORUM), Dispatcher.RequestTime.forImmediateExecution(), false); executor.maybeTryAdditionalReplicas(); try { @@ -142,7 +142,7 @@ public class ReadExecutorTest assertEquals(0, cfs.metric.speculativeFailedRetries.getCount()); assertEquals(0, ks.metric.speculativeRetries.getCount()); assertEquals(0, ks.metric.speculativeFailedRetries.getCount()); - AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(DAYS.toMillis(365)), plan(LOCAL_QUORUM, targets, targets.subList(0, 2)), Dispatcher.RequestTime.forImmediateExecution()); + AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(ReadCoordinator.DEFAULT, cfs, new MockSinglePartitionReadCommand(DAYS.toMillis(365)), plan(LOCAL_QUORUM, targets, targets.subList(0, 2)), Dispatcher.RequestTime.forImmediateExecution()); executor.maybeTryAdditionalReplicas(); new Thread() { @@ -183,7 +183,7 @@ public class ReadExecutorTest assertEquals(0, cfs.metric.speculativeFailedRetries.getCount()); assertEquals(0, ks.metric.speculativeRetries.getCount()); assertEquals(0, ks.metric.speculativeFailedRetries.getCount()); - AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(LOCAL_QUORUM, targets, targets.subList(0, 2)), Dispatcher.RequestTime.forImmediateExecution()); + AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(ReadCoordinator.DEFAULT, cfs, new MockSinglePartitionReadCommand(), plan(LOCAL_QUORUM, targets, targets.subList(0, 2)), Dispatcher.RequestTime.forImmediateExecution()); executor.maybeTryAdditionalReplicas(); try { @@ -209,7 +209,7 @@ public class ReadExecutorTest { MockSinglePartitionReadCommand command = new MockSinglePartitionReadCommand(TimeUnit.DAYS.toMillis(365)); ReplicaPlan.ForTokenRead plan = plan(ConsistencyLevel.LOCAL_ONE, targets, targets.subList(0, 1)); - AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, command, plan, Dispatcher.RequestTime.forImmediateExecution()); + AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(ReadCoordinator.DEFAULT, cfs, command, plan, Dispatcher.RequestTime.forImmediateExecution()); // Issue an initial request against the first endpoint... executor.executeAsync(); @@ -255,7 +255,7 @@ public class ReadExecutorTest MockSinglePartitionReadCommand(long timeout) { - super(cfs.metadata().epoch, false, 0, false, cfs.metadata(), 0, null, null, null, Util.dk("ry@n_luvs_teh_y@nk33z"), null, null, false, null); + super(cfs.metadata().epoch, false, 0, false, false, cfs.metadata(), 0, null, null, null, Util.dk("ry@n_luvs_teh_y@nk33z"), null, null, false, null); this.timeout = timeout; } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java index 1689069cf9..d4a5b7a05f 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java @@ -66,14 +66,15 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.locator.ReplicaUtils; +import org.apache.cassandra.service.reads.ReadCoordinator; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.net.Message; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Tables; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.ByteBufferUtil; @@ -360,7 +361,7 @@ public abstract class AbstractReadRepairTest replicas, 1, null, - (self, token) -> forReadRepair(self, ClusterMetadata.current(), keyspace, consistencyLevel, token, (r) -> true), + (self, token) -> forReadRepair(self, ClusterMetadata.current(), keyspace, consistencyLevel, token, (r) -> true, ReadCoordinator.DEFAULT), Epoch.EMPTY); } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java index 6806172402..a0320c92c7 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java @@ -42,6 +42,7 @@ import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.net.Message; import org.apache.cassandra.service.reads.ReadCallback; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.service.reads.ReadCoordinator; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -53,7 +54,7 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest { public InstrumentedReadRepairHandler(Map repairs, ReplicaPlan.ForWrite writePlan) { - super(Util.dk("not a real usable value"), repairs, writePlan); + super(ReadCoordinator.DEFAULT, Util.dk("not a real usable value"), repairs, writePlan); } Map mutationsSent = new HashMap<>(); @@ -86,7 +87,7 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest { public InstrumentedBlockingReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) { - super(command, replicaPlan, requestTime); + super(ReadCoordinator.DEFAULT, command, replicaPlan, requestTime); } Set readCommandRecipients = new HashSet<>(); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java index 9258922ff8..1a330402d8 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java @@ -47,6 +47,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.net.Message; import org.apache.cassandra.service.reads.ReadCallback; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.service.reads.repair.ReadRepairEvent.ReadRepairEventType; import org.apache.cassandra.transport.Dispatcher; @@ -135,7 +136,7 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest DiagnosticBlockingRepairHandler(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) { - super(command, replicaPlan, requestTime); + super(ReadCoordinator.DEFAULT, command, replicaPlan, requestTime); DiagnosticEventService.instance().subscribe(ReadRepairEvent.class, this::onRepairEvent); } @@ -183,7 +184,7 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest DiagnosticPartitionReadRepairHandler(DecoratedKey key, Map repairs, ReplicaPlan.ForWrite forReadRepair) { - super(key, repairs, forReadRepair); + super(ReadCoordinator.DEFAULT, key, repairs, forReadRepair); DiagnosticEventService.instance().subscribe(PartitionRepairEvent.class, this::onRepairEvent); } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java index 749e444425..a094ebfa6b 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java @@ -34,6 +34,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.service.reads.ReadCallback; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.service.reads.ReadCoordinator; public class ReadOnlyReadRepairTest extends AbstractReadRepairTest { @@ -42,7 +43,7 @@ public class ReadOnlyReadRepairTest extends AbstractReadRepairTest { public InstrumentedReadOnlyReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) { - super(command, replicaPlan, requestTime); + super(ReadCoordinator.DEFAULT, command, replicaPlan, requestTime); } Set readCommandRecipients = new HashSet<>(); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java index 5138de0300..d0f0682fbf 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java @@ -57,6 +57,7 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.utils.ByteBufferUtil; @@ -79,7 +80,7 @@ public class ReadRepairTest { public InstrumentedReadRepairHandler(Map repairs, ReplicaPlan.ForWrite writePlan) { - super(Util.dk("not a valid key"), repairs, writePlan); + super(ReadCoordinator.DEFAULT, Util.dk("not a valid key"), repairs, writePlan); } Map mutationsSent = new HashMap<>(); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java b/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java index 682dc740ff..7bdd08349a 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java @@ -281,6 +281,7 @@ public class RepairedDataVerifierTest isDigest, 0, false, + false, metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(metadata), diff --git a/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java b/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java index eecd106e06..650fa73eb2 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java @@ -35,7 +35,9 @@ import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.ReplicaPlan.ForWrite; import org.apache.cassandra.service.reads.DigestResolver; +import org.apache.cassandra.service.reads.ReadCoordinator; public class TestableReadRepair, P extends ReplicaPlan.ForRead> implements ReadRepair @@ -89,7 +91,6 @@ public class TestableReadRepair, P extends ReplicaPlan.Fo @Override public void awaitReads() throws ReadTimeoutException { - } @Override @@ -117,6 +118,12 @@ public class TestableReadRepair, P extends ReplicaPlan.Fo sent.put(entry.getKey().endpoint(), entry.getValue()); } + @Override + public void repairPartitionDirectly(ReadCoordinator coordinator, DecoratedKey partitionKey, Map mutations, ForWrite writePlan) + { + throw new UnsupportedOperationException(); + } + public Mutation getForEndpoint(InetAddressAndPort endpoint) { return sent.get(endpoint); diff --git a/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java b/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java index 778b6b2d71..b44d6b42c9 100644 --- a/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java +++ b/test/unit/org/apache/cassandra/streaming/SessionInfoTest.java @@ -27,6 +27,8 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.utils.FBUtilities; +import static java.util.Collections.emptyList; + public class SessionInfoTest { /** @@ -41,11 +43,11 @@ public class SessionInfoTest Collection summaries = new ArrayList<>(); for (int i = 0; i < 10; i++) { - StreamSummary summary = new StreamSummary(tableId, i, (i + 1) * 10); + StreamSummary summary = new StreamSummary(tableId, emptyList(), i, (i + 1) * 10); summaries.add(summary); } - StreamSummary sending = new StreamSummary(tableId, 10, 100); + StreamSummary sending = new StreamSummary(tableId, emptyList(), 10, 100); SessionInfo info = new SessionInfo(local, 0, local, summaries, Collections.singleton(sending), StreamSession.State.PREPARING, null); assert info.getTotalFilesToReceive() == 45; diff --git a/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java b/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java index 79856ee153..0a854c0744 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java @@ -70,6 +70,7 @@ import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; +import static java.util.Collections.emptyList; import static org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper.*; import static org.apache.cassandra.tcm.ownership.OwnershipUtils.beginJoin; import static org.apache.cassandra.tcm.ownership.OwnershipUtils.beginMove; @@ -366,7 +367,7 @@ public class StreamReaderTest StreamResultFuture future = StreamResultFuture.createInitiator(nextTimeUUID(), StreamOperation.REPAIR, Collections.emptyList(), streamCoordinator); InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort(); - streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED, "")); + streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, emptyList(), emptyList(), StreamSession.State.INITIALIZED, "")); StreamSession session = streamCoordinator.getOrCreateOutboundSession(peer); session.init(future); @@ -380,7 +381,7 @@ public class StreamReaderTest CassandraStreamHeader streamHeader = streamMessageHeader(tokens); long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); IStreamReader reader = streamReader(header, streamHeader, session); - StreamSummary streamSummary = new StreamSummary(streamHeader.tableId, 1, 0); + StreamSummary streamSummary = new StreamSummary(streamHeader.tableId, emptyList(), 1, 0); session.prepareReceiving(streamSummary); reader.read(incomingStream(tokens)); assertEquals(StorageMetrics.totalOpsForInvalidToken.getCount(), startMetricCount); @@ -392,7 +393,7 @@ public class StreamReaderTest StreamMessageHeader header = streamHeader(); CassandraStreamHeader streamHeader = streamMessageHeader(tokens); long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); - StreamSummary streamSummary = new StreamSummary(streamHeader.tableId, 1, 0); + StreamSummary streamSummary = new StreamSummary(streamHeader.tableId, emptyList(), 1, 0); session.prepareReceiving(streamSummary); try { diff --git a/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java b/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java index 03d6aa7fd3..069d0fb58d 100644 --- a/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java +++ b/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java @@ -30,6 +30,7 @@ import org.junit.Test; import io.netty.buffer.ByteBuf; import io.netty.channel.embedded.EmbeddedChannel; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputBuffer; @@ -50,9 +51,8 @@ import org.apache.cassandra.streaming.messages.StreamInitMessage; import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.TimeUUID; -import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; - import static org.apache.cassandra.net.TestChannel.REMOTE_ADDR; +import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; public class StreamingInboundHandlerTest { @@ -125,7 +125,7 @@ public class StreamingInboundHandlerTest temp.flip(); DataInputPlus in = new DataInputBuffer(temp, false); // session not found - IncomingStreamMessage.serializer.deserialize(in, MessagingService.current_version); + IncomingStreamMessage.serializer.deserialize(in, IPartitioner.global(), MessagingService.current_version); } @Test diff --git a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java index 318c0c6e3e..b9df8471a2 100644 --- a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java +++ b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java @@ -51,6 +51,7 @@ import org.apache.cassandra.tcm.sequences.LockedRanges; import org.mockito.Mockito; import static org.apache.cassandra.tcm.MetadataKeys.ACCORD_KEYSPACES; +import static org.apache.cassandra.tcm.MetadataKeys.CONSENSUS_MIGRATION_STATE; import static org.apache.cassandra.tcm.MetadataKeys.DATA_PLACEMENTS; import static org.apache.cassandra.tcm.MetadataKeys.IN_PROGRESS_SEQUENCES; import static org.apache.cassandra.tcm.MetadataKeys.LOCKED_RANGES; @@ -305,6 +306,8 @@ public class ClusterMetadataTransformationTest return metadata.inProgressSequences; else if (key == ACCORD_KEYSPACES) return metadata.accordKeyspaces; + else if (key == CONSENSUS_MIGRATION_STATE) + return metadata.consensusMigrationState; throw new IllegalArgumentException("Unknown metadata key " + key); } diff --git a/test/unit/org/apache/cassandra/tools/nodetool/NetStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/NetStatsTest.java index 7bddc9b23a..9664c90714 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/NetStatsTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/NetStatsTest.java @@ -40,6 +40,7 @@ import org.apache.cassandra.streaming.StreamSummary; import org.apache.cassandra.tools.ToolRunner; import org.apache.cassandra.utils.FBUtilities; +import static java.util.Collections.emptyList; import static org.apache.cassandra.net.Verb.ECHO_REQ; import static org.assertj.core.api.Assertions.assertThat; @@ -111,7 +112,7 @@ public class NetStatsTest extends CQLTester @Test public void testHumanReadable() throws IOException { - List streamSummaries = Collections.singletonList(new StreamSummary(TableId.generate(), 1, 1024)); + List streamSummaries = Collections.singletonList(new StreamSummary(TableId.generate(), emptyList(), 1, 1024)); SessionInfo info = new SessionInfo(InetAddressAndPort.getLocalHost(), 1, InetAddressAndPort.getLocalHost(), diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java index 7f08d6ccf2..51e6c9fb4c 100644 --- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java +++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java @@ -233,7 +233,7 @@ public class BloomFilterTest @Test public void testMurmur3FilterHash() { - IPartitioner partitioner = new Murmur3Partitioner(); + IPartitioner partitioner = Murmur3Partitioner.instance; Iterator gen = new KeyGenerator.RandomStringGenerator(new Random().nextInt(), FilterTestHelper.ELEMENTS); long[] expected = new long[2]; long[] actual = new long[2]; diff --git a/test/unit/org/apache/cassandra/utils/SerializationsTest.java b/test/unit/org/apache/cassandra/utils/SerializationsTest.java index e23bb38832..0fc38209e5 100644 --- a/test/unit/org/apache/cassandra/utils/SerializationsTest.java +++ b/test/unit/org/apache/cassandra/utils/SerializationsTest.java @@ -118,7 +118,7 @@ public class SerializationsTest extends AbstractSerializationsTester private void testBloomFilterTable(String file, boolean oldBfFormat) throws Exception { - Murmur3Partitioner partitioner = new Murmur3Partitioner(); + Murmur3Partitioner partitioner = Murmur3Partitioner.instance; try (FileInputStreamPlus in = new File(file).newInputStream(); IFilter filter = BloomFilterSerializer.forVersion(oldBfFormat).deserialize(in))