diff --git a/src/java/org/apache/cassandra/config/AccordSpec.java b/src/java/org/apache/cassandra/config/AccordSpec.java index 9eb8a1d0be..697d7edc1e 100644 --- a/src/java/org/apache/cassandra/config/AccordSpec.java +++ b/src/java/org/apache/cassandra/config/AccordSpec.java @@ -18,6 +18,8 @@ package org.apache.cassandra.config; +import org.apache.cassandra.service.consensus.TransactionalMode; + public class AccordSpec { public volatile boolean enabled = false; @@ -49,4 +51,22 @@ public class AccordSpec public volatile DurationSpec durability_txnid_lag = new DurationSpec.IntSecondsBound(5); public volatile DurationSpec shard_durability_cycle = new DurationSpec.IntMinutesBound(2); public volatile DurationSpec global_durability_cycle = new DurationSpec.IntMinutesBound(10); + + public enum TransactionalRangeMigration + { + auto, explicit + } + + /** + * Defines the behavior of range migration opt-in when changing transactional settings on a table. In auto, + * all ranges are marked as migrating and no additional user action is needed aside from running repairs. In + * explicit, no ranges are marked as migrating, and the user needs to explicitly mark ranges as migrating to + * the target transactional mode via nodetool. + */ + public volatile TransactionalRangeMigration range_migration = TransactionalRangeMigration.auto; + + /** + * default transactional mode for tables created by this node when no transactional mode has been specified in the DDL + */ + public TransactionalMode default_transactional_mode = TransactionalMode.off; } diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 3352ce9fed..45a8a61ce9 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -44,7 +44,6 @@ import org.apache.cassandra.io.compress.BufferType; import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.service.StartupChecks.StartupCheckType; import org.apache.cassandra.utils.StorageCompatibilityMode; -import org.apache.cassandra.service.accord.IAccordService; import static org.apache.cassandra.config.CassandraRelevantProperties.AUTOCOMPACTION_ON_STARTUP_ENABLED; import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_AVAILABLE_PROCESSORS; @@ -1167,8 +1166,6 @@ public class Config public volatile boolean client_request_size_metrics_enabled = true; - public LWTStrategy lwt_strategy = LWTStrategy.migration; - public NonSerialWriteStrategy non_serial_write_strategy = NonSerialWriteStrategy.normal; public volatile int max_top_size_partition_count = 10; public volatile int max_top_tombstone_partition_count = 10; @@ -1387,118 +1384,6 @@ public class Config cell } - /** - * How to pick a consensus protocol for CAS - * and serial read operations. Transaction statements - * will always run on Accord. Legacy in this context includes PaxosV2. - */ - public enum LWTStrategy - { - /** - * Allow both Accord and PaxosV1/V2 to run on the same cluster - * Some keys and ranges might be running on Accord if they - * have been migrated and the rest will run on Paxos until - * they are migrated. - */ - migration, - - /** - * Everything will be run on Accord. Useful for new deployments - * that don't want to accidentally start using legacy Paxos - * requiring migration to Accord. - */ - accord - } - - /* - * Configure how non-serial writes should be executed. For Accord transactions to function correctly - * when mixed with non-SERIAL writes it's necessary for the writes to occur through Accord. - * - * Accord will also use this configuration to determine what consistency level to perform its reads - * at since it will need to be able to read data written at non-SERIAL consistency levels. - * - * BlockingReadRepair will also use this configuration to determine how BRR mutations are applied. For migration - * and accord the BRR mutations will be applied as Accord transactions so that BRR doesn't expose Accord to - * uncommitted Accord data that is being RRed. This can occur when Accord has applied a transaction at some, but not - * all replica since Accord defaults to asynchronous commit. - * - * By routing repairs through Accord it is guaranteed that the Accord derived contents of the repair have already been applied at any - * replica where Accord applies the transaction. This also prevents BRR from breaking atomicity of Accord writes. - * - * If they are not written through Accord then reads through Accord will be required to occur at - * consistency level compatible with the non-serial writes preventing single replica reads from being performed - * by Accord. It will also require Accord to perform read repair of non-serial writes. - * - * Even then there is the potential for Accord to inconsistently execute transactions at different replicas - * because different coordinators for an Accord transaction may encounter different non-SERIAL write state and - * race to commit different outcomes for the transaction. - * - * This is different from Paxos because Paxos performs consensus on the actual values to be applied so recovery - * coordinators will always produce a consistent state when applying a transaction. Accord performs consensus on - * the execution order of transaction and different coordinators witnessing different states not managed by Accord - * can produce multiple outcomes for a transaction. - * - * // TODO (maybe): To safely migrate you would have to route all writes through Accord with the current implementation - * // We could do it by range instead in the migration version, but then we need to know when all in flight writes - * // are done before marking a range as migrated. Would waiting out the timeout be enough (timeout bugs!)? - */ - public enum NonSerialWriteStrategy - { - /* - * Execute writes through Cassandra via StorageProxy's normal write path. This can lead Accord to compute - * multiple outcomes for a transaction that depends on data written by non-SERIAL writes. - */ - normal(false, false, false), - /* - * Allow mixing of non-SERIAL writes and Accord, but still force BRR through Accord - */ - mixed(false, false, true), - /* - * Execute writes through Accord skipping StorageProxy's normal write path, but commit - * writes at the provided consistency level so they can be read via non-SERIAL consistency levels. - */ - migration(false, true, true), - /* - * Execute writes through Accord skipping StorageProxy's normal write path. Ignores the provided consistency level - * which makes Accord commit writes at ANY similar to Paxos with commit consistency level ANY. - */ - accord(true, true, true); - - public final boolean ignoresSuppliedConsistencyLevel; - public final boolean writesThroughAccord; - - public final boolean blockingReadRepairThroughAccord; - - NonSerialWriteStrategy(boolean ignoresSuppliedConsistencyLevel, boolean writesThroughAccord, boolean blockingReadRepairThroughAccord) - { - this.ignoresSuppliedConsistencyLevel = ignoresSuppliedConsistencyLevel; - this.writesThroughAccord = writesThroughAccord; - this.blockingReadRepairThroughAccord = blockingReadRepairThroughAccord; - } - - public ConsistencyLevel commitCLForStrategy(ConsistencyLevel consistencyLevel) - { - if (ignoresSuppliedConsistencyLevel) - return null; - - if (!IAccordService.SUPPORTED_COMMIT_CONSISTENCY_LEVELS.contains(consistencyLevel)) - throw new UnsupportedOperationException("Consistency level " + consistencyLevel + " is unsupported with Accord for write/commit, supported are ANY, ONE, QUORUM, and ALL"); - - return consistencyLevel; - } - - public ConsistencyLevel readCLForStrategy(ConsistencyLevel consistencyLevel) - { - if (ignoresSuppliedConsistencyLevel) - return null; - - if (!IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS.contains(consistencyLevel)) - throw new UnsupportedOperationException("Consistency level " + consistencyLevel + " is unsupported with Accord for read, supported are ONE, QUORUM, and SERIAL"); - - return consistencyLevel; - } - } - private static final Set SENSITIVE_KEYS = new HashSet() {{ add("client_encryption_options"); add("server_encryption_options"); diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 246df534e8..9498e666c4 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -78,8 +78,6 @@ import org.apache.cassandra.auth.INetworkAuthorizer; import org.apache.cassandra.auth.IRoleManager; import org.apache.cassandra.config.Config.CommitLogSync; import org.apache.cassandra.config.Config.DiskAccessMode; -import org.apache.cassandra.config.Config.LWTStrategy; -import org.apache.cassandra.config.Config.NonSerialWriteStrategy; import org.apache.cassandra.config.Config.PaxosOnLinearizabilityViolation; import org.apache.cassandra.config.Config.PaxosStatePurging; import org.apache.cassandra.db.ConsistencyLevel; @@ -120,6 +118,7 @@ import org.apache.cassandra.security.JREProvider; import org.apache.cassandra.security.SSLFactory; import org.apache.cassandra.service.CacheService.CacheType; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.paxos.Paxos; import org.apache.cassandra.tcm.RegistrationStatus; import org.apache.cassandra.utils.FBUtilities; @@ -168,8 +167,8 @@ import static org.apache.cassandra.utils.Clock.Global.logInitializationOutcome; public class DatabaseDescriptor { - public static final String NO_ACCORD_PAXOS_STRATEGY_WITH_ACCORD_DISABLED_MESSAGE = - "Cannot use lwt_strategy \"accord\" while Accord transactions are disabled."; + public static final String NO_ACCORD_PAXOS_STRATEGY_WITH_ACCORD_DISABLED_MESSAGE = + "Cannot use lwt_strategy \"accord\" while Accord transactions are disabled."; static { @@ -968,8 +967,8 @@ public class DatabaseDescriptor { // if consensusMigrationCacheSizeInMiB option was set to "auto" then size of the cache should be "min(1% of Heap (in MB), 50MB) consensusMigrationCacheSizeInMiB = (conf.consensus_migration_cache_size == null) - ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.01 / 1024 / 1024)), 50) - : conf.consensus_migration_cache_size.toMebibytes(); + ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.01 / 1024 / 1024)), 50) + : conf.consensus_migration_cache_size.toMebibytes(); if (consensusMigrationCacheSizeInMiB < 0) throw new NumberFormatException(); // to escape duplicating error message @@ -1166,14 +1165,6 @@ public class DatabaseDescriptor // run audit logging options through sanitation and validation if (conf.audit_logging_options != null) setAuditLoggingOptions(conf.audit_logging_options); - - if (conf.lwt_strategy == LWTStrategy.accord) - { - if (!conf.accord.enabled) - throw new ConfigurationException(NO_ACCORD_PAXOS_STRATEGY_WITH_ACCORD_DISABLED_MESSAGE); - if (conf.non_serial_write_strategy == Config.NonSerialWriteStrategy.normal) - throw new ConfigurationException("If Accord is used for LWTs then regular writes needs to be routed through Accord for interoperability by setting non_serial_write_strategy to \"accord\" or \"migration\""); - } } @VisibleForTesting @@ -3653,25 +3644,14 @@ public class DatabaseDescriptor return conf.paxos_topology_repair_strict_each_quorum; } - // TODO (desired): This configuration should come out of TrM to force the cluster to agree on it - public static LWTStrategy getLWTStrategy() + public static AccordSpec.TransactionalRangeMigration getTransactionalRangeMigration() { - return conf.lwt_strategy; + return conf.accord.range_migration; } - public static void setLWTStrategy(LWTStrategy lwtStrategy) + public static void setTransactionalRangeMigration(AccordSpec.TransactionalRangeMigration val) { - conf.lwt_strategy = lwtStrategy; - } - - public static Config.NonSerialWriteStrategy getNonSerialWriteStrategy() - { - return conf.non_serial_write_strategy; - } - - public static void setNonSerialWriteStrategy(NonSerialWriteStrategy nonSerialWriteStrategy) - { - conf.non_serial_write_strategy = nonSerialWriteStrategy; + conf.accord.range_migration = Preconditions.checkNotNull(val); } public static int getAccordBarrierRetryAttempts() @@ -3694,6 +3674,11 @@ public class DatabaseDescriptor return conf.accord.range_barrier_timeout.to(TimeUnit.NANOSECONDS); } + public static TransactionalMode defaultTransactionalMode() + { + return conf.accord.default_transactional_mode; + } + public static void setNativeTransportMaxRequestDataInFlightPerIpInBytes(long maxRequestDataInFlightInBytes) { if (maxRequestDataInFlightInBytes == -1) diff --git a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java index b2edb6cc33..bbfc333ca6 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java +++ b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java @@ -36,7 +36,6 @@ import org.slf4j.LoggerFactory; import accord.api.Update; import accord.primitives.Txn; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.UpdateParameters; import org.apache.cassandra.cql3.conditions.ColumnCondition; @@ -502,7 +501,7 @@ public class CQL3CasRequest implements CASRequest Update update = createUpdate(clientState, commitConsistencyLevel); // If the write strategy is sending all writes through Accord there is no need to use the supplied consistency // level since Accord will manage reading safely - consistencyLevel = DatabaseDescriptor.getNonSerialWriteStrategy().readCLForStrategy(consistencyLevel); + consistencyLevel = metadata.params.transactionalMode.readCLForStrategy(consistencyLevel); TxnRead read = TxnRead.createCasRead(readCommand, consistencyLevel); // In a CAS requesting only one key is supported and writes // can't be dependent on any data that is read (only conditions) @@ -514,7 +513,7 @@ public class CQL3CasRequest implements CASRequest { // Potentially ignore commit consistency level if non-SERIAL write strategy is Accord // since it is safe to match what non-SERIAL writes do - commitConsistencyLevel = DatabaseDescriptor.getNonSerialWriteStrategy().commitCLForStrategy(commitConsistencyLevel); + commitConsistencyLevel = metadata.params.transactionalMode.commitCLForStrategy(commitConsistencyLevel); return new TxnUpdate(createWriteFragments(clientState), createCondition(), commitConsistencyLevel); } diff --git a/src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java b/src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java index f1a8872f07..208b9590f4 100644 --- a/src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java @@ -63,9 +63,11 @@ import org.apache.cassandra.db.SinglePartitionReadQuery; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.api.AccordRoutableKey; import org.apache.cassandra.service.accord.txn.AccordUpdate; import org.apache.cassandra.service.accord.txn.TxnCondition; import org.apache.cassandra.service.accord.txn.TxnData; @@ -78,13 +80,12 @@ import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.accord.txn.TxnUpdate; import org.apache.cassandra.service.accord.txn.TxnWrite; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.utils.FBUtilities; import static accord.primitives.Txn.Kind.EphemeralRead; import static accord.primitives.Txn.Kind.Read; -import static org.apache.cassandra.config.Config.NonSerialWriteStrategy.accord; -import static org.apache.cassandra.config.DatabaseDescriptor.getNonSerialWriteStrategy; import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse; import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull; import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue; @@ -314,6 +315,11 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement, return new Keys(keySet); } + private static TransactionalMode transactionalModeForSingleKey(Keys keys) + { + return Schema.instance.getTableMetadata(((AccordRoutableKey) keys.get(0)).table()).params.transactionalMode; + } + @VisibleForTesting public Txn createTxn(ClientState state, QueryOptions options) { @@ -326,7 +332,8 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement, List reads = createNamedReads(options, state, ImmutableMap.of(), keySet::add); Keys txnKeys = toKeys(keySet); TxnRead read = createTxnRead(reads, txnKeys, null); - Txn.Kind kind = txnKeys.size() == 1 && getNonSerialWriteStrategy() == accord ? EphemeralRead : Read; + Txn.Kind kind = txnKeys.size() == 1 && transactionalModeForSingleKey(txnKeys) == TransactionalMode.full + ? EphemeralRead : Read; return new Txn.InMemory(kind, txnKeys, read, TxnQuery.ALL, null); } else @@ -376,8 +383,6 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement, Txn txn = createTxn(state.getClientState(), options); - AccordService.instance().maybeConvertTablesToAccord(txn); - TxnResult txnResult = AccordService.instance().coordinate(txn, options.getConsistency(), requestTime); if (txnResult.kind() == retry_new_protocol) throw new IllegalStateException("Transaction statement should never be required to switch consensus protocols"); diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java index 0282cbd409..115a6a3374 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java @@ -40,6 +40,7 @@ import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.service.QueryState; +import org.apache.cassandra.service.accord.AccordTopology; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.transport.Event.SchemaChange; @@ -199,6 +200,9 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac if (null != user && !user.isAnonymous()) createdResources(diff).forEach(r -> grantPermissionsOnResource(r, user)); + // if the changes affected accord, wait for accord to apply them + AccordTopology.awaitTopologyReadiness(diff, result.epoch); + return new ResultMessage.SchemaChange(schemaChangeEvent(diff)); } diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java index 94120ac63c..2a8cca87fc 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java @@ -59,12 +59,14 @@ import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; import org.apache.cassandra.schema.MemtableParams; +import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableParams; import org.apache.cassandra.schema.UserFunctions; import org.apache.cassandra.schema.ViewMetadata; import org.apache.cassandra.schema.Views; import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; @@ -83,6 +85,8 @@ import static org.apache.cassandra.schema.TableMetadata.Flag; public abstract class AlterTableStatement extends AlterSchemaStatement { + private static final Logger logger = LoggerFactory.getLogger(AlterTableStatement.class); + protected final String tableName; private final boolean ifExists; protected ClientState state; @@ -583,6 +587,42 @@ public abstract class AlterTableStatement extends AlterSchemaStatement validateDefaultTimeToLive(attrs.asNewTableParams()); } + private TableParams validateAndUpdateTransactionalMigration(TableParams prev, TableParams next) + { + if (next.transactionalMode.accordIsEnabled && SchemaConstants.isSystemKeyspace(keyspaceName)) + throw ire("Cannot enable accord on system tables (%s.%s)", keyspaceName, tableName); + + boolean modeChange = prev.transactionalMode != next.transactionalMode; + boolean wasMigrating = prev.transactionalMigrationFrom.isMigrating(); + boolean forceMigrationChange = prev.transactionalMigrationFrom != next.transactionalMigrationFrom; + + if (modeChange && next.transactionalMode.accordIsEnabled && !DatabaseDescriptor.getAccordTransactionsEnabled()) + throw ire(format("Cannot change transactional mode to %s for %s.%s with accord_transactions_enabled set to false", + next.transactionalMode, keyspaceName, tableName)); + + // user is manually updating migration mode, don't interfere + if (forceMigrationChange) + { + logger.warn("Forcing unsafe migration change from {} to {} with transaction mode {}", prev.transactionalMigrationFrom, next.transactionalMigrationFrom, next.transactionalMode); + return next; + } + + if (!modeChange) + return next; + + // if the user is trying to revert to the mode being migrated from, allow it. The migration states will be inverted when + // the transformation is applied. Otherwise throw + if (wasMigrating && next.transactionalMode != prev.transactionalMigrationFrom.from) + throw ire(format("Cannot change transactional mode from %s to %s for %s.%s before transactional migration has completed", + prev.transactionalMode, next.transactionalMode, + keyspaceName, tableName)); + + // set table to migrating + TransactionalMigrationFromMode migrateFrom = TransactionalMigrationFromMode.fromMode(prev.transactionalMode, next.transactionalMode); + return next.unbuild().transactionalMigrationFrom(migrateFrom).build(); + } + + public KeyspaceMetadata apply(Epoch epoch, KeyspaceMetadata keyspace, TableMetadata table, ClusterMetadata metadata) { attrs.validate(); @@ -610,6 +650,8 @@ public abstract class AlterTableStatement extends AlterSchemaStatement if (!params.compression.isEnabled()) Guardrails.uncompressedTablesEnabled.ensureEnabled(state); + params = validateAndUpdateTransactionalMigration(table.params, params); + return keyspace.withSwapped(keyspace.tables.withSwapped(table.withSwapped(params))); } } diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java index b8e51d1286..711af0c7ee 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java @@ -31,6 +31,7 @@ import org.apache.cassandra.audit.AuditLogEntryType; import org.apache.cassandra.auth.DataResource; import org.apache.cassandra.auth.IResource; import org.apache.cassandra.auth.Permission; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.*; import org.apache.cassandra.cql3.constraints.ColumnConstraints; import org.apache.cassandra.cql3.functions.masking.ColumnMask; @@ -46,6 +47,7 @@ import org.apache.cassandra.transport.Event.SchemaChange; import org.apache.cassandra.transport.Event.SchemaChange.Change; import org.apache.cassandra.transport.Event.SchemaChange.Target; +import static java.lang.String.format; import static java.util.Comparator.comparing; import static com.google.common.collect.Iterables.concat; @@ -145,6 +147,16 @@ public final class CreateTableStatement extends AlterSchemaStatement if (!table.params.compression.isEnabled()) Guardrails.uncompressedTablesEnabled.ensureEnabled(state); + if (table.params.transactionalMode.accordIsEnabled && SchemaConstants.isSystemKeyspace(keyspaceName)) + throw ire("Cannot enable accord on system tables (%s.%s)", keyspaceName, tableName); + + if (table.params.transactionalMode.accordIsEnabled && !DatabaseDescriptor.getAccordTransactionsEnabled()) + throw ire(format("Cannot create table %s.%s with transactional mode %s with accord.enabled set to false", + keyspaceName, tableName, table.params.transactionalMode)); + + if (table.params.transactionalMigrationFrom.isMigrating()) + throw ire("Cannot set transactional migration on new tables (%s.%s), %s", keyspaceName, tableName, table.params.transactionalMigrationFrom); + return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.tables.with(table))); } diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index eb18918628..eb9c492798 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -23,6 +23,7 @@ import java.util.Set; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.statements.PropertyDefinitions; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; @@ -34,6 +35,8 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableParams; import org.apache.cassandra.schema.TableParams.Option; import org.apache.cassandra.service.accord.fastpath.FastPathStrategy; +import org.apache.cassandra.service.consensus.TransactionalMode; +import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; @@ -64,7 +67,10 @@ public final class TableAttributes extends PropertyDefinitions TableParams asNewTableParams() { - return build(TableParams.builder()); + TableParams.Builder builder = TableParams.builder(); + if (!hasOption(TRANSACTIONAL_MODE)) + builder.transactionalMode(DatabaseDescriptor.defaultTransactionalMode()); + return build(builder); } TableParams asAlteredTableParams(TableParams previous) @@ -165,6 +171,12 @@ public final class TableAttributes extends PropertyDefinitions } } + if (hasOption(Option.TRANSACTIONAL_MODE)) + builder.transactionalMode(TransactionalMode.fromString(getString(Option.TRANSACTIONAL_MODE))); + + if (hasOption(Option.TRANSACTIONAL_MIGRATION_FROM)) + builder.transactionalMigrationFrom(TransactionalMigrationFromMode.fromString(getString(Option.TRANSACTIONAL_MIGRATION_FROM))); + return builder.build(); } diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index 8f155773c8..64015fec48 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -136,8 +136,6 @@ import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithNowInSec; import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigratedAt; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; import static org.apache.cassandra.gms.ApplicationState.DC; import static org.apache.cassandra.gms.ApplicationState.HOST_ID; import static org.apache.cassandra.gms.ApplicationState.INTERNAL_ADDRESS_AND_PORT; @@ -146,6 +144,8 @@ import static org.apache.cassandra.gms.ApplicationState.RACK; import static org.apache.cassandra.gms.ApplicationState.RELEASE_VERSION; import static org.apache.cassandra.gms.ApplicationState.STATUS_WITH_PORT; import static org.apache.cassandra.gms.ApplicationState.TOKENS; +import org.apache.cassandra.service.consensus.migration.ConsensusMigratedAt; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget; import static org.apache.cassandra.service.paxos.Commit.latest; import static org.apache.cassandra.service.snapshot.SnapshotOptions.systemSnapshot; import static org.apache.cassandra.utils.CassandraVersion.NULL_VERSION; diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 99ef8e96b3..e75b6be269 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -246,7 +246,7 @@ public class CassandraStreamReceiver implements StreamReceiver checkNotNull(minVersion, "Unable to determine minimum cluster version"); IAccordService accordService = AccordService.instance(); if (session.streamOperation().requiresBarrierTransaction() - && accordService.isAccordManagedTable(cfs.getTableId()) + && cfs.metadata().isAccordEnabled() && CassandraVersion.CASSANDRA_5_0.compareTo(minVersion) >= 0) accordService.postStreamReceivingBarrier(cfs, ranges); diff --git a/src/java/org/apache/cassandra/repair/AccordRepairJob.java b/src/java/org/apache/cassandra/repair/AccordRepairJob.java index 8db43b46b3..d82e4407fa 100644 --- a/src/java/org/apache/cassandra/repair/AccordRepairJob.java +++ b/src/java/org/apache/cassandra/repair/AccordRepairJob.java @@ -33,7 +33,7 @@ import org.apache.cassandra.dht.AccordSplitter; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.TokenRange; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationRepairResult; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationRepairResult; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; diff --git a/src/java/org/apache/cassandra/repair/CassandraRepairJob.java b/src/java/org/apache/cassandra/repair/CassandraRepairJob.java index e2d373c587..7662907ad9 100644 --- a/src/java/org/apache/cassandra/repair/CassandraRepairJob.java +++ b/src/java/org/apache/cassandra/repair/CassandraRepairJob.java @@ -50,7 +50,7 @@ import org.apache.cassandra.repair.asymmetric.ReduceHelper; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationRepairResult; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationRepairResult; import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tcm.ClusterMetadata; diff --git a/src/java/org/apache/cassandra/repair/RepairResult.java b/src/java/org/apache/cassandra/repair/RepairResult.java index 4899448c71..6c04f6be07 100644 --- a/src/java/org/apache/cassandra/repair/RepairResult.java +++ b/src/java/org/apache/cassandra/repair/RepairResult.java @@ -19,7 +19,7 @@ package org.apache.cassandra.repair; import java.util.List; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationRepairResult; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationRepairResult; /** * RepairJob's result diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java index ae46877a6b..d98cc6141e 100644 --- a/src/java/org/apache/cassandra/repair/RepairSession.java +++ b/src/java/org/apache/cassandra/repair/RepairSession.java @@ -59,7 +59,7 @@ import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.repair.state.SessionState; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigration; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; @@ -353,7 +353,7 @@ public class RepairSession extends AsyncFuture implements I new AccordRepairJob(this, cfname) : new CassandraRepairJob(this, cfname); // Repairs can drive forward progress for consensus migration so always check - job.addCallback(ConsensusTableMigrationState.completedRepairJobHandler); + job.addCallback(ConsensusTableMigration.completedRepairJobHandler); state.register(job.state); executor.execute(job); jobs.add(job); diff --git a/src/java/org/apache/cassandra/schema/DistributedSchema.java b/src/java/org/apache/cassandra/schema/DistributedSchema.java index e4eead15b5..17f4d33ccb 100644 --- a/src/java/org/apache/cassandra/schema/DistributedSchema.java +++ b/src/java/org/apache/cassandra/schema/DistributedSchema.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.UUID; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import org.apache.cassandra.auth.AuthKeyspace; import org.apache.cassandra.config.DatabaseDescriptor; @@ -79,10 +80,18 @@ public class DistributedSchema implements MetadataValue return new DistributedSchema(Keyspaces.of(DistributedMetadataLogKeyspace.initialMetadata(knownDatacenters)), Epoch.FIRST); } + private static ImmutableMap keyspacesToTableMap(Keyspaces keyspaces) + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + keyspaces.forEach(ksm -> ksm.tablesAndViews().forEach(tbl -> builder.put(tbl.id, tbl))); + return builder.build(); + } + private final Keyspaces keyspaces; private final Epoch epoch; private final UUID version; private final Map keyspaceInstances = new HashMap<>(); + private final transient ImmutableMap tables; public DistributedSchema(Keyspaces keyspaces) { @@ -95,6 +104,7 @@ public class DistributedSchema implements MetadataValue this.keyspaces = keyspaces; this.epoch = epoch; this.version = new UUID(0, epoch.getEpoch()); + this.tables = keyspacesToTableMap(keyspaces); validate(); } @@ -120,6 +130,11 @@ public class DistributedSchema implements MetadataValue return keyspaces.get(keyspace).get(); } + public TableMetadata getTableMetadata(TableId id) + { + return tables.get(id); + } + public static DistributedSchema fromSystemTables(Keyspaces keyspaces, Set knownDatacenters) { if (!keyspaces.containsKeyspace(SchemaConstants.METADATA_KEYSPACE_NAME)) diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 88a3f80d0b..e78e159995 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -361,6 +361,21 @@ public class TableMetadata implements SchemaElement return false; } + public boolean isAccordEnabled() + { + return params.transactionalMode.accordIsEnabled; + } + + public boolean migratingFromAccord() + { + return params.transactionalMigrationFrom.migratingFromAccord(); + } + + public boolean requiresAccordSupport() + { + return isAccordEnabled() || migratingFromAccord(); + } + public ImmutableCollection columns() { return columns.values(); diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 40614f65ea..8cdc41f687 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -33,6 +33,8 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.accord.fastpath.FastPathStrategy; +import org.apache.cassandra.service.consensus.TransactionalMode; +import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.service.reads.PercentileSpeculativeRetryPolicy; @@ -71,7 +73,9 @@ public final class TableParams CRC_CHECK_CHANCE, CDC, READ_REPAIR, - FAST_PATH; + FAST_PATH, + TRANSACTIONAL_MODE, + TRANSACTIONAL_MIGRATION_FROM; @Override public String toString() @@ -100,6 +104,8 @@ public final class TableParams public final boolean cdc; public final ReadRepairStrategy readRepair; public final FastPathStrategy fastPath; + public final TransactionalMode transactionalMode; + public final TransactionalMigrationFromMode transactionalMigrationFrom; private TableParams(Builder builder) { @@ -125,6 +131,8 @@ public final class TableParams cdc = builder.cdc; readRepair = builder.readRepair; fastPath = builder.fastPath; + transactionalMode = builder.transactionalMode != null ? builder.transactionalMode : TransactionalMode.off; + transactionalMigrationFrom = builder.transactionalMigrationFrom; } public static Builder builder() @@ -153,7 +161,9 @@ public final class TableParams .extensions(params.extensions) .cdc(params.cdc) .readRepair(params.readRepair) - .fastPath(params.fastPath); + .fastPath(params.fastPath) + .transactionalMode(params.transactionalMode) + .transactionalMigrationFrom(params.transactionalMigrationFrom); } public Builder unbuild() @@ -245,7 +255,9 @@ public final class TableParams && extensions.equals(p.extensions) && cdc == p.cdc && readRepair == p.readRepair - && fastPath.equals(fastPath); + && fastPath.equals(fastPath) + && transactionalMode == p.transactionalMode + && transactionalMigrationFrom == p.transactionalMigrationFrom; } @Override @@ -270,7 +282,9 @@ public final class TableParams extensions, cdc, readRepair, - fastPath); + fastPath, + transactionalMode, + transactionalMigrationFrom); } @Override @@ -298,6 +312,8 @@ public final class TableParams .add(CDC.toString(), cdc) .add(READ_REPAIR.toString(), readRepair) .add(Option.FAST_PATH.toString(), fastPath) + .add(Option.TRANSACTIONAL_MODE.toString(), transactionalMode) + .add(Option.TRANSACTIONAL_MIGRATION_FROM.toString(), transactionalMigrationFrom) .toString(); } @@ -348,8 +364,17 @@ public final class TableParams .append("AND min_index_interval = ").append(minIndexInterval) .newLine() .append("AND read_repair = ").appendWithSingleQuotes(readRepair.toString()) - .newLine() - .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()); + .newLine(); + + if (!isView) + { + builder.append("AND transactional_mode = ").appendWithSingleQuotes(transactionalMode.toString()) + .newLine() + .append("AND transactional_migration_from = ").appendWithSingleQuotes(transactionalMigrationFrom.toString()) + .newLine(); + } + + builder.append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()); } public static final class Builder @@ -374,6 +399,8 @@ public final class TableParams private boolean cdc; private ReadRepairStrategy readRepair = ReadRepairStrategy.BLOCKING; private FastPathStrategy fastPath = FastPathStrategy.inheritKeyspace(); + private TransactionalMode transactionalMode = TransactionalMode.off; + public TransactionalMigrationFromMode transactionalMigrationFrom = TransactionalMigrationFromMode.none; public Builder() { @@ -498,6 +525,18 @@ public final class TableParams return this; } + public Builder transactionalMode(TransactionalMode val) + { + transactionalMode = val; + return this; + } + + public Builder transactionalMigrationFrom(TransactionalMigrationFromMode val) + { + transactionalMigrationFrom = val; + return this; + } + public Builder extensions(Map val) { extensions = ImmutableMap.copyOf(val); @@ -534,6 +573,8 @@ public final class TableParams { out.writeBoolean(t.allowAutoSnapshot); out.writeBoolean(t.incrementalBackups); + out.writeInt(t.transactionalMode.ordinal()); + out.writeInt(t.transactionalMigrationFrom.ordinal()); } } @@ -559,7 +600,9 @@ public final class TableParams .cdc(in.readBoolean()) .readRepair(ReadRepairStrategy.fromString(in.readUTF())) .allowAutoSnapshot(!version.isAtLeast(Version.V4) || in.readBoolean()) - .incrementalBackups(!version.isAtLeast(Version.V4) || in.readBoolean()); + .incrementalBackups(!version.isAtLeast(Version.V4) || in.readBoolean()) + .transactionalMode(version.isAtLeast(Version.V4) ? TransactionalMode.fromOrdinal(in.readInt()) : TransactionalMode.off) + .transactionalMigrationFrom(version.isAtLeast(Version.V4) ? TransactionalMigrationFromMode.fromOrdinal(in.readInt()) : TransactionalMigrationFromMode.off); return builder.build(); } @@ -584,7 +627,9 @@ public final class TableParams sizeof(t.cdc) + sizeof(t.readRepair.name()) + (version.isAtLeast(Version.V4) ? sizeof(t.allowAutoSnapshot) : 0) + - (version.isAtLeast(Version.V4) ? sizeof(t.incrementalBackups) : 0); + (version.isAtLeast(Version.V4) ? sizeof(t.incrementalBackups) : 0) + + (version.isAtLeast(Version.V4) ? sizeof(t.transactionalMode.ordinal()) : 0) + + (version.isAtLeast(Version.V4) ? sizeof(t.transactionalMigrationFrom.ordinal()) : 0); } private void serializeMap(Map map, DataOutputPlus out) throws IOException diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index 4b5fad2d22..dc78f99531 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -42,6 +42,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import com.google.common.base.Preconditions; import com.google.common.cache.CacheLoader; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.Uninterruptibles; @@ -57,7 +58,7 @@ import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.Config; -import org.apache.cassandra.config.Config.NonSerialWriteStrategy; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; @@ -131,7 +132,6 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.api.PartitionKey; -import org.apache.cassandra.service.accord.txn.AccordUpdate; import org.apache.cassandra.service.accord.txn.TxnCondition; import org.apache.cassandra.service.accord.txn.TxnData; import org.apache.cassandra.service.accord.txn.TxnQuery; @@ -175,8 +175,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Iterables.concat; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static org.apache.cassandra.config.Config.NonSerialWriteStrategy.accord; -import static org.apache.cassandra.config.DatabaseDescriptor.getNonSerialWriteStrategy; import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics; @@ -383,7 +381,6 @@ public class StorageProxy implements StorageProxyMBean clientState, nowInSeconds); IAccordService accordService = AccordService.instance(); - accordService.maybeConvertTablesToAccord(txn); TxnResult txnResult = accordService.coordinate(txn, consistencyForPaxos, requestTime); @@ -1187,6 +1184,54 @@ public class StorageProxy implements StorageProxyMBean } } + private static ConsistencyLevel consistencyLevelForCommit(Collection mutations, ConsistencyLevel consistencyLevel) + { + ConsistencyLevel result = null; + for (IMutation mutation : mutations) + { + for (TableId tableId : mutation.getTableIds()) + { + TransactionalMode mode = Schema.instance.getTableMetadata(tableId).params.transactionalMode; + ConsistencyLevel commitCL = mode.commitCLForStrategy(consistencyLevel); + if (result == null || commitCL.compareTo(result) > 0) + result = commitCL; + } + } + return result; + } + + private static boolean writesThroughAccord(List mutations, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + { + boolean accordWrite = false; + boolean normalWrite = false; + for (int i=0,mi=mutations.size(); i mutations, ConsistencyLevel consistencyLevel, @@ -1220,12 +1265,15 @@ public class StorageProxy implements StorageProxyMBean .viewManager .updatesAffectView(mutations, true); + long size = IMutation.dataSize(mutations); writeMetrics.mutationSize.update(size); writeMetricsForLevel(consistencyLevel).mutationSize.update(size); - NonSerialWriteStrategy nonSerialWriteStrategy = getNonSerialWriteStrategy(); - if (nonSerialWriteStrategy.writesThroughAccord && !SchemaConstants.getSystemKeyspaces().contains(keyspaceName)) - mutateWithAccord(augmented != null ? augmented : mutations, consistencyLevel, requestTime, nonSerialWriteStrategy); + if (writesThroughAccord(mutations, consistencyLevel, requestTime)) + { + Preconditions.checkState(!SchemaConstants.getSystemKeyspaces().contains(keyspaceName)); + mutateWithAccord(augmented != null ? augmented : mutations, consistencyLevel, requestTime); + } else if (augmented != null) mutateAtomically(augmented, consistencyLevel, updatesView, requestTime); else @@ -1237,12 +1285,12 @@ public class StorageProxy implements StorageProxyMBean } } - private static void mutateWithAccord(Collection iMutations, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, Config.NonSerialWriteStrategy nonSerialWriteStrategy) + private static void mutateWithAccord(Collection mutations, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { int fragmentIndex = 0; - List fragments = new ArrayList<>(iMutations.size()); - List partitionKeys = new ArrayList<>(iMutations.size()); - for (IMutation mutation : iMutations) + List fragments = new ArrayList<>(mutations.size()); + List partitionKeys = new ArrayList<>(mutations.size()); + for (IMutation mutation : mutations) { for (PartitionUpdate update : mutation.getPartitionUpdates()) { @@ -1252,11 +1300,10 @@ public class StorageProxy implements StorageProxyMBean } } // Potentially ignore commit consistency level if the strategy specifies accord and not migration - ConsistencyLevel clForCommit = nonSerialWriteStrategy.commitCLForStrategy(consistencyLevel); - AccordUpdate update = new TxnUpdate(fragments, TxnCondition.none(), clForCommit); + ConsistencyLevel clForCommit = consistencyLevelForCommit(mutations, consistencyLevel); + TxnUpdate update = new TxnUpdate(fragments, TxnCondition.none(), clForCommit); Txn.InMemory txn = new Txn.InMemory(Keys.of(partitionKeys), TxnRead.EMPTY, TxnQuery.EMPTY, update); IAccordService accordService = AccordService.instance(); - accordService.maybeConvertTablesToAccord(txn); accordService.coordinate(txn, consistencyLevel, requestTime); } @@ -2002,13 +2049,12 @@ public class StorageProxy implements StorageProxyMBean SinglePartitionReadCommand readCommand = group.queries.get(0); // If the non-SERIAL write strategy is sending all writes through Accord there is no need to use the supplied consistency // level since Accord will manage reading safely - NonSerialWriteStrategy nonSerialWriteStrategy = getNonSerialWriteStrategy(); - consistencyLevel = nonSerialWriteStrategy.readCLForStrategy(consistencyLevel); + TransactionalMode transactionalMode = group.metadata().params.transactionalMode; + consistencyLevel = transactionalMode.readCLForStrategy(consistencyLevel); TxnRead read = TxnRead.createSerialRead(readCommand, consistencyLevel); Invariants.checkState(read.keys().size() == 1, "Ephemeral reads are only strict-serializable for single partition reads"); - Txn txn = new Txn.InMemory(nonSerialWriteStrategy == accord ? EphemeralRead : Read, read.keys(), read, TxnQuery.ALL, null); + Txn txn = new Txn.InMemory(transactionalMode == TransactionalMode.full ? EphemeralRead : Read, read.keys(), read, TxnQuery.ALL, null); IAccordService accordService = AccordService.instance(); - accordService.maybeConvertTablesToAccord(txn); TxnResult txnResult = accordService.coordinate(txn, consistencyLevel, requestTime); if (txnResult.kind() == retry_new_protocol) return RETRY_NEW_PROTOCOL; diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index d473ae5f09..ef4068e992 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -168,9 +168,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.schema.ViewMetadata; -import org.apache.cassandra.service.accord.AccordService; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.service.disk.usage.DiskUsageBroadcaster; import org.apache.cassandra.service.paxos.Paxos; import org.apache.cassandra.service.paxos.PaxosCommit; @@ -259,8 +257,8 @@ import static org.apache.cassandra.service.StorageService.Mode.JOINING_FAILED; import static org.apache.cassandra.service.StorageService.Mode.LEAVING; import static org.apache.cassandra.service.StorageService.Mode.MOVE_FAILED; import static org.apache.cassandra.service.StorageService.Mode.NORMAL; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.finishMigrationToConsensusProtocol; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.startMigrationToConsensusProtocol; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigration.finishMigrationToConsensusProtocol; +import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigration.startMigrationToConsensusProtocol; import static org.apache.cassandra.tcm.membership.NodeState.BOOTSTRAPPING; import static org.apache.cassandra.tcm.membership.NodeState.BOOT_REPLACING; import static org.apache.cassandra.tcm.membership.NodeState.JOINED; @@ -1696,18 +1694,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return finishMigrationToConsensusProtocol(keyspace, Optional.ofNullable(maybeTableNames), Optional.ofNullable(maybeRangesStr)); } - @Override - public void setConsensusMigrationTargetProtocol(@Nonnull String targetProtocol, - @Nullable List keyspaceNames, - @Nullable List maybeTableNames) - { - checkNotNull(targetProtocol, "targetProtocol is null"); - checkNotNull(keyspaceNames, "keyspaceNames is null"); - checkArgument(!keyspaceNames.contains(SchemaConstants.METADATA_KEYSPACE_NAME)); - - ConsensusTableMigrationState.setConsensusMigrationTargetProtocol(targetProtocol, keyspaceNames, Optional.ofNullable(maybeTableNames)); - } - @Override public String listConsensusMigrations(@Nullable Set keyspaceNames, @Nullable Set tableNames, @@ -4235,7 +4221,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces(); return keyspaces.stream().flatMap(ks -> ks.tables.stream()) - .filter(tbm -> AccordService.instance().isAccordManagedTable(tbm.id)) + .filter(TableMetadata::requiresAccordSupport) .map(tbm -> tbm.keyspace) .distinct() .sorted() @@ -4245,10 +4231,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE @Override public List getAccordManagedTables() { - // TODO (review) These are really just the ones Accord is aware of not necessarily managed Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces(); return keyspaces.stream().flatMap(ks -> ks.tables.stream()) - .filter(tbm -> AccordService.instance().isAccordManagedTable(tbm.id)) + .filter(TableMetadata::requiresAccordSupport) .map(tbm -> tbm.keyspace + '.' + tbm.name) .collect(toList()); } diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index b4e28c3e9b..c58205898d 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -1151,10 +1151,6 @@ public interface StorageServiceMBean extends NotificationEmitter @Nullable List maybeTableNames, @Nullable String maybeRangesStr); - void setConsensusMigrationTargetProtocol(@Nonnull String targetProtocol, - @Nullable List keyspaceNames, - @Nullable List maybeTableNames); - String listConsensusMigrations(@Nullable Set keyspaceNames, @Nullable Set tableNames, @Nonnull String format); List getAccordManagedKeyspaces(); diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index a730b62813..e831aa1fbd 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -37,7 +37,7 @@ import accord.impl.CoordinateDurabilityScheduling; import org.apache.cassandra.cql3.statements.RequestValidations; import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory; import org.apache.cassandra.tcm.ClusterMetadataService; -import org.apache.cassandra.tcm.transformations.AddAccordTable; +import org.apache.cassandra.service.accord.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,7 +74,6 @@ import org.apache.cassandra.concurrent.Shutdownable; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.WriteType; -import org.apache.cassandra.exceptions.ExceptionCode; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.RequestTimeoutException; import org.apache.cassandra.exceptions.WriteTimeoutException; @@ -83,23 +82,16 @@ import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessageDelivery; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordSyncPropagator.Notification; -import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.AccordRoutingKey.KeyspaceSplitter; -import org.apache.cassandra.service.accord.api.AccordScheduler; -import org.apache.cassandra.service.accord.api.AccordTopologySorter; -import org.apache.cassandra.service.accord.api.CompositeTopologySorter; import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException; import org.apache.cassandra.service.accord.exceptions.WritePreemptedException; import org.apache.cassandra.service.accord.txn.TxnResult; -import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.ExecutorUtils; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.AsyncPromise; @@ -206,20 +198,11 @@ public class AccordService implements IAccordService, Shutdownable @Override public void receive(Message> message) {} - @Override - public boolean isAccordManagedTable(TableId keyspace) - { - return false; - } - @Override public Pair, DurableBefore> getRedundantBeforesAndDurableBefore() { return Pair.create(new Int2ObjectHashMap<>(), DurableBefore.EMPTY); } - - @Override - public void ensureTableIsAccordManaged(TableId tableId) {} }; private static volatile IAccordService instance = null; @@ -676,28 +659,6 @@ public class AccordService implements IAccordService, Shutdownable return configService; } - @Override - public boolean isAccordManagedTable(TableId tableId) - { - return ClusterMetadata.current().accordTables.contains(tableId); - } - - @Override - public void ensureTableIsAccordManaged(TableId tableId) - { - if (isAccordManagedTable(tableId)) - return; - ClusterMetadataService.instance().commit(new AddAccordTable(tableId), - metadata -> null, - (code, message) -> { - Invariants.checkState(code == ExceptionCode.ALREADY_EXISTS, - "Expected %s, got %s", ExceptionCode.ALREADY_EXISTS, code); - return null; - }); - // we need to avoid creating a txnId in an epoch when no one has any ranges - FBUtilities.waitOnFuture(AccordService.instance().epochReady(ClusterMetadata.current().epoch)); - } - @Override public Pair, DurableBefore> getRedundantBeforesAndDurableBefore() { diff --git a/src/java/org/apache/cassandra/service/accord/AccordTopology.java b/src/java/org/apache/cassandra/service/accord/AccordTopology.java index 46b4d026bf..0814c322d5 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTopology.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTopology.java @@ -19,17 +19,20 @@ package org.apache.cassandra.service.accord; import java.util.*; -import java.util.function.Predicate; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import accord.primitives.Ranges; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import accord.local.Node; import accord.topology.Shard; import accord.topology.Topology; import accord.utils.Invariants; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -45,6 +48,9 @@ import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.VersionedEndpoints; +import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; /** * Deterministically computes accord topology from a ClusterMetadata instance @@ -212,7 +218,7 @@ public class AccordTopology return builder.build(); } - public static Topology createAccordTopology(Epoch epoch, DistributedSchema schema, DataPlacements placements, Directory directory, AccordFastPath accordFastPath, Predicate tablePredicate, ShardLookup lookup) + public static Topology createAccordTopology(Epoch epoch, DistributedSchema schema, DataPlacements placements, Directory directory, AccordFastPath accordFastPath, ShardLookup lookup) { List shards = new ArrayList<>(); Set unavailable = accordFastPath.unavailableIds(); @@ -220,7 +226,7 @@ public class AccordTopology for (KeyspaceMetadata keyspace : schema.getKeyspaces()) { - List tables = keyspace.tables.stream().filter(tbl -> tablePredicate.test(tbl.id)).collect(Collectors.toList()); + List tables = keyspace.tables.stream().filter(TableMetadata::requiresAccordSupport).collect(Collectors.toList()); if (tables.isEmpty()) continue; List ksShards = KeyspaceShard.forKeyspace(keyspace, placements, directory, lookup); @@ -231,19 +237,14 @@ public class AccordTopology return new Topology(epoch.getEpoch(), shards.toArray(new Shard[0])); } - public static Topology createAccordTopology(ClusterMetadata metadata, Predicate tablePredicate, ShardLookup lookup) + public static Topology createAccordTopology(ClusterMetadata metadata, ShardLookup lookup) { - return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, metadata.accordFastPath, tablePredicate, lookup); - } - - public static Topology createAccordTopology(ClusterMetadata metadata, Predicate tablePredicate) - { - return createAccordTopology(metadata, tablePredicate, new ShardLookup()); + return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, metadata.accordFastPath, lookup); } public static Topology createAccordTopology(ClusterMetadata metadata, Topology current) { - return createAccordTopology(metadata, metadata.accordTables::contains, createShardLookup(current)); + return createAccordTopology(metadata, createShardLookup(current)); } public static Topology createAccordTopology(ClusterMetadata metadata) @@ -274,4 +275,59 @@ public class AccordTopology topology.forEach(shard -> map.put(shard.range, shard)); return map; } + private static boolean hasAccordSchemaChange(TableMetadata before, TableMetadata after) + { + return after.requiresAccordSupport() && (before == null || !before.requiresAccordSupport()); + } + + private static boolean hasAccordSchemaChange(TableMetadata created) + { + return hasAccordSchemaChange(null, created); + } + + private static boolean hasAccordSchemaChange(Diff.Altered diff) + { + return hasAccordSchemaChange(diff.before, diff.after); + } + + private static boolean hasAccordSchemaChange(Keyspaces.KeyspacesDiff keyspacesDiff) + { + for (KeyspaceMetadata.KeyspaceDiff keyspaceDiff : keyspacesDiff.altered) + { + if (Iterables.any(keyspaceDiff.tables.created, AccordTopology::hasAccordSchemaChange)) + return true; + + if (Iterables.any(keyspaceDiff.tables.altered, AccordTopology::hasAccordSchemaChange)) + return true; + } + + return false; + } + + /** + * If an accord related schema change occurs, we need to wait until accord has processed them + * before unblocking the change + */ + public static void awaitTopologyReadiness(Keyspaces.KeyspacesDiff keyspacesDiff, Epoch epoch) + { + if (!AccordService.isSetup()) + return; + + if (!hasAccordSchemaChange(keyspacesDiff)) + return; + + try + { + AccordService.instance().epochReady(epoch).get(DatabaseDescriptor.getTransactionTimeout(MILLISECONDS), MILLISECONDS); + } + catch (InterruptedException e) + { + throw new UncheckedInterruptedException(e); + } + catch (ExecutionException | TimeoutException e) + { + throw new RuntimeException(e); + } + } + } diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java index 9422df4916..b037acc7c5 100644 --- a/src/java/org/apache/cassandra/service/accord/IAccordService.java +++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java @@ -18,16 +18,6 @@ package org.apache.cassandra.service.accord; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; - -import com.google.common.collect.ImmutableSet; - import accord.api.BarrierType; import accord.local.DurableBefore; import accord.local.Node.Id; @@ -37,30 +27,32 @@ import accord.primitives.Ranges; import accord.primitives.Seekables; import accord.primitives.Txn; import accord.topology.TopologyManager; +import com.google.common.collect.ImmutableSet; import org.agrona.collections.Int2ObjectHashMap; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; -import org.apache.cassandra.service.accord.api.AccordRoutableKey; import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey; -import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.api.AccordScheduler; import org.apache.cassandra.service.accord.txn.TxnResult; -import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.transport.Dispatcher; -import org.apache.cassandra.tcm.transformations.AddAccordTable; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Future; +import javax.annotation.Nonnull; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + public interface IAccordService { - Set SUPPORTED_COMMIT_CONSISTENCY_LEVELS = ImmutableSet.of(ConsistencyLevel.ANY, ConsistencyLevel.ONE, ConsistencyLevel.QUORUM, ConsistencyLevel.SERIAL, ConsistencyLevel.ALL); + Set SUPPORTED_COMMIT_CONSISTENCY_LEVELS = ImmutableSet.of(ConsistencyLevel.ANY, ConsistencyLevel.ONE, ConsistencyLevel.LOCAL_ONE, ConsistencyLevel.QUORUM, ConsistencyLevel.SERIAL, ConsistencyLevel.ALL); Set SUPPORTED_READ_CONSISTENCY_LEVELS = ImmutableSet.of(ConsistencyLevel.ONE, ConsistencyLevel.QUORUM, ConsistencyLevel.SERIAL); IVerbHandler verbHandler(); @@ -112,52 +104,10 @@ public interface IAccordService void receive(Message> message); - /** - * Temporary method to avoid double-streaming keyspaces - * @param tableId - * @return - */ - boolean isAccordManagedTable(TableId tableId); - /** * Fetch the redundnant befores for every command store */ Pair, DurableBefore> getRedundantBeforesAndDurableBefore(); default Id nodeId() { throw new UnsupportedOperationException(); } - - default void maybeConvertTablesToAccord(Txn txn) - { - Set allTables = new HashSet<>(); - Set newTables = new HashSet<>(); - txn.keys().forEach(key -> { - TableId table = key instanceof AccordRoutableKey ? ((AccordRoutableKey) key).table() : ((TokenRange) key).table(); - if (allTables.add(table) && !isAccordManagedTable(table)) - newTables.add(table); - }); - - if (newTables.isEmpty()) - return; - - for (TableId table : newTables) - AddAccordTable.addTable(table); - - // we need to avoid creating a txnId in an epoch when no one has any ranges - FBUtilities.waitOnFuture(epochReady(ClusterMetadata.current().epoch)); - - for (TableId table : allTables) - { - if (!isAccordManagedTable(table)) - throw new IllegalStateException(table + " is not an accord managed table"); - } - } - - void ensureTableIsAccordManaged(TableId tableId); - - default void ensureKeyspaceIsAccordManaged(String keyspace) - { - // TODO: remove when accord enabled is handled via schema - Keyspace ks = Keyspace.open(keyspace); - ks.getMetadata().tables.forEach(metadata -> ensureTableIsAccordManaged(metadata.id)); - } } diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java index eb8ddc5c39..bafb96b4db 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java @@ -86,8 +86,8 @@ import org.apache.cassandra.service.accord.txn.TxnData; import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.service.accord.txn.UnrecoverableRepairUpdate; import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigration; +import org.apache.cassandra.service.consensus.migration.TableMigrationState; import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.transport.Dispatcher; @@ -262,9 +262,9 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen // because they haven't yet updated their cluster metadata. // It would be harmless to do the read, but we can respond faster skipping it // and getting the transaction on the correct protocol - TableMigrationState tms = ConsensusTableMigrationState.getTableMigrationState(command.metadata().id); + TableMigrationState tms = ConsensusTableMigration.getTableMigrationState(command.metadata().id); AccordClientRequestMetrics metrics = txn.kind().isWrite() ? accordWriteMetrics : accordReadMetrics; - if (ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(tms, command.partitionKey())) + if (ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(command.metadata(), tms, command.partitionKey())) { metrics.migrationSkippedReads.mark(); results.add(AsyncChains.success(TxnData.emptyPartition(fragment.txnDataName(), command))); diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java b/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java index 7afa75de16..defa96b554 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java @@ -32,8 +32,6 @@ import accord.api.Update; import accord.primitives.Seekables; import accord.primitives.Timestamp; import accord.primitives.TxnId; -import org.apache.cassandra.config.Config.LWTStrategy; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.EmptyIterators; import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.TypeSizes; @@ -154,10 +152,6 @@ public abstract class TxnQuery implements Query Epoch epoch = Epoch.create(executeAt.epoch()); if (transactionIsInMigratingOrMigratedRange(epoch, keys)) { - // Fail fast because we can't be sure where this request should really run or what was intended - if (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord) - throw new IllegalStateException("Mixing a hard coded strategy with migration is unsupported"); - if (txnId.isWrite()) ClientRequestsMetricsHolder.accordWriteMetrics.accordMigrationRejects.mark(); else diff --git a/src/java/org/apache/cassandra/service/consensus/TransactionalMode.java b/src/java/org/apache/cassandra/service/consensus/TransactionalMode.java new file mode 100644 index 0000000000..2bcaee3ce2 --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/TransactionalMode.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus; + +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.utils.LocalizeString; + +/* + * Configure the transactional behavior of a table. Enables accord on a table and defines how it mixes with non-serial writes + * + * For Accord transactions to function correctly when mixed with non-SERIAL writes it's necessary for the writes to occur through Accord. + * + * Accord will also use this configuration to determine what consistency level to perform its reads + * at since it will need to be able to read data written at non-SERIAL consistency levels. + * + * BlockingReadRepair will also use this configuration to determine how BRR mutations are applied. For migration + * and accord the BRR mutations will be applied as Accord transactions so that BRR doesn't expose Accord to + * uncommitted Accord data that is being RRed. This can occur when Accord has applied a transaction at some, but not + * all replica since Accord defaults to asynchronous commit. + * + * By routing repairs through Accord it is guaranteed that the Accord derived contents of the repair have already been applied at any + * replica where Accord applies the transaction. This also prevents BRR from breaking atomicity of Accord writes. + * + * If they are not written through Accord then reads through Accord will be required to occur at + * consistency level compatible with the non-serial writes preventing single replica reads from being performed + * by Accord. It will also require Accord to perform read repair of non-serial writes. + * + * Even then there is the potential for Accord to inconsistently execute transactions at different replicas + * because different coordinators for an Accord transaction may encounter different non-SERIAL write state and + * race to commit different outcomes for the transaction. + * + * This is different from Paxos because Paxos performs consensus on the actual values to be applied so recovery + * coordinators will always produce a consistent state when applying a transaction. Accord performs consensus on + * the execution order of transaction and different coordinators witnessing different states not managed by Accord + * can produce multiple outcomes for a transaction. + * + * // TODO to safely migrate you would have to route all writes through Accord with the current implementation + * // We could do it by range instead in the migration version, but then we need to know when all in flight writes + * // are done before marking a range as migrated. Would waiting out the timeout be enough (timeout bugs!)? + */ +public enum TransactionalMode +{ + // Running on Paxos V1 or V2 with Accord disabled + off(false, false, false, false), + + /* + * Execute writes through Cassandra via StorageProxy's normal write path. This can lead Accord to compute + * multiple outcomes for a transaction that depends on data written by non-SERIAL writes. + */ + unsafe(true, false, false, false), + + /* + * Allow mixing of non-SERIAL writes and Accord, but still force BRR through Accord. + * This mode makes it safe to perform non-SERIAL or SERIAL reads of Accord data, but unsafe + * to write data that Accord may attempt to read. + */ + unsafe_writes(true, false, false, true), + + /* + * Execute writes through Accord skipping StorageProxy's normal write path, but commit + * writes at the provided consistency level so they can be read via non-SERIAL consistency levels. + * This mode makes it safe to read/write data that Accord will read/write. + */ + mixed_reads(true, false, true, true), + + /* + * Execute writes through Accord skipping StorageProxy's normal write path. Ignores the provided consistency level + * which makes Accord commit writes at ANY similar to Paxos with commit consistency level ANY. + */ + full(true, true, true, true); + + public final boolean accordIsEnabled; + public final boolean ignoresSuppliedConsistencyLevel; + public final boolean writesThroughAccord; + + public final boolean blockingReadRepairThroughAccord; + private final String cqlParam; + + TransactionalMode(boolean accordIsEnabled, boolean ignoresSuppliedConsistencyLevel, boolean writesThroughAccord, boolean blockingReadRepairThroughAccord) + { + this.accordIsEnabled = accordIsEnabled; + this.ignoresSuppliedConsistencyLevel = ignoresSuppliedConsistencyLevel; + this.writesThroughAccord = writesThroughAccord; + this.blockingReadRepairThroughAccord = blockingReadRepairThroughAccord; + this.cqlParam = String.format("transactional_mode = '%s'", LocalizeString.toLowerCaseLocalized(this.name())); + } + + public ConsistencyLevel commitCLForStrategy(ConsistencyLevel consistencyLevel) + { + if (ignoresSuppliedConsistencyLevel) + return null; + + if (!IAccordService.SUPPORTED_COMMIT_CONSISTENCY_LEVELS.contains(consistencyLevel)) + throw new UnsupportedOperationException("Consistency level " + consistencyLevel + " is unsupported with Accord for write/commit, supported are ANY, ONE, QUORUM, and ALL"); + + return consistencyLevel; + } + + public ConsistencyLevel readCLForStrategy(ConsistencyLevel consistencyLevel) + { + if (ignoresSuppliedConsistencyLevel) + return null; + + if (!IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS.contains(consistencyLevel)) + throw new UnsupportedOperationException("Consistency level " + consistencyLevel + " is unsupported with Accord for read, supported are ONE, QUORUM, and SERIAL"); + + return consistencyLevel; + } + + public String asCqlParam() + { + return cqlParam; + } + + public static TransactionalMode fromOrdinal(int ordinal) + { + return values()[ordinal]; + } + + public static TransactionalMode fromString(String name) + { + return valueOf(LocalizeString.toLowerCaseLocalized(name)); + } +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java index d3651015c2..7a0bbaa1ff 100644 --- a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java @@ -57,7 +57,6 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.api.PartitionKey; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigratedAt; import org.apache.cassandra.service.paxos.AbstractPaxosRepair.Failure; import org.apache.cassandra.service.paxos.AbstractPaxosRepair.Result; import org.apache.cassandra.service.paxos.PaxosRepair; @@ -70,9 +69,9 @@ import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.UUIDSerializer; import static org.apache.cassandra.net.Verb.CONSENSUS_KEY_MIGRATION; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget.paxos; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; + +import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget.paxos; + import static org.apache.cassandra.utils.Clock.Global.nanoTime; /** diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigratedAt.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigratedAt.java new file mode 100644 index 0000000000..2b995bdcc6 --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigratedAt.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import java.io.IOException; +import javax.annotation.Nullable; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.utils.NullableSerializer; + +public class ConsensusMigratedAt +{ + public static final IVersionedSerializer serializer = NullableSerializer.wrap(new IVersionedSerializer() + { + @Override + public void serialize(ConsensusMigratedAt t, DataOutputPlus out, int version) throws IOException + { + Epoch.messageSerializer.serialize(t.migratedAtEpoch, out, version); + out.writeByte(t.migratedAtTarget.value); + } + + @Override + public ConsensusMigratedAt deserialize(DataInputPlus in, int version) throws IOException + { + Epoch migratedAtEpoch = Epoch.messageSerializer.deserialize(in, version); + ConsensusMigrationTarget target = ConsensusMigrationTarget.fromValue(in.readByte()); + return new ConsensusMigratedAt(migratedAtEpoch, target); + } + + @Override + public long serializedSize(ConsensusMigratedAt t, int version) + { + return TypeSizes.sizeof(ConsensusMigrationTarget.accord.value) + + Epoch.messageSerializer.serializedSize(t.migratedAtEpoch, version); + } + }); + + // Fields are not nullable when used for messaging + @Nullable + public final Epoch migratedAtEpoch; + + @Nullable + public final ConsensusMigrationTarget migratedAtTarget; + + public ConsensusMigratedAt(Epoch migratedAtEpoch, ConsensusMigrationTarget migratedAtTarget) + { + this.migratedAtEpoch = migratedAtEpoch; + this.migratedAtTarget = migratedAtTarget; + } +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationRepairResult.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationRepairResult.java new file mode 100644 index 0000000000..9233667b5a --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationRepairResult.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import org.apache.cassandra.tcm.Epoch; + +import static com.google.common.base.Preconditions.checkArgument; + +public class ConsensusMigrationRepairResult +{ + public final ConsensusMigrationRepairType type; + public final Epoch minEpoch; + + private ConsensusMigrationRepairResult(ConsensusMigrationRepairType type, Epoch minEpoch) + { + this.type = type; + this.minEpoch = minEpoch; + } + + public static ConsensusMigrationRepairResult fromCassandraRepair(Epoch minEpoch, boolean migrationEligibleRepair) + { + checkArgument(!migrationEligibleRepair || minEpoch.isAfter(Epoch.EMPTY), "Epoch should not be empty if Paxos and regular repairs were performed"); + if (migrationEligibleRepair) + return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.paxos, minEpoch); + else + return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.ineligible, Epoch.EMPTY); + } + + public static ConsensusMigrationRepairResult fromAccordRepair(Epoch minEpoch) + { + checkArgument(minEpoch.isAfter(Epoch.EMPTY), "Accord repairs should always occur at an Epoch"); + return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.accord, minEpoch); + } +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationRepairType.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationRepairType.java new file mode 100644 index 0000000000..233682d07f --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationRepairType.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import com.google.common.primitives.SignedBytes; + +import org.apache.cassandra.utils.LocalizeString; + +public enum ConsensusMigrationRepairType +{ + ineligible(0), + paxos(1), + accord(2); + + public final byte value; + + ConsensusMigrationRepairType(int value) + { + this.value = SignedBytes.checkedCast(value); + } + + public static ConsensusMigrationRepairType fromString(String repairType) + { + return ConsensusMigrationRepairType.valueOf(LocalizeString.toLowerCaseLocalized(repairType)); + } + + public static ConsensusMigrationRepairType fromValue(byte value) + { + switch (value) + { + default: + throw new IllegalArgumentException(value + " is not recognized"); + case 0: + return ConsensusMigrationRepairType.ineligible; + case 1: + return ConsensusMigrationRepairType.paxos; + case 2: + return ConsensusMigrationRepairType.accord; + } + } +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationState.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationState.java new file mode 100644 index 0000000000..7364db38c0 --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationState.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.DistributedSchema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.MetadataValue; +import org.apache.cassandra.tcm.serialization.MetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; +import org.apache.cassandra.utils.PojoToString; + +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.cassandra.utils.CollectionSerializers.deserializeMap; +import static org.apache.cassandra.utils.CollectionSerializers.newHashMap; +import static org.apache.cassandra.utils.CollectionSerializers.serializeMap; +import static org.apache.cassandra.utils.CollectionSerializers.serializedMapSize; + +// TODO this will mostly go away once we can move TableMigrationState into the table schema +public class ConsensusMigrationState implements MetadataValue +{ + public static ConsensusMigrationState EMPTY = new ConsensusMigrationState(Epoch.EMPTY, ImmutableMap.of()); + @Nonnull + public final Map tableStates; + + public final Epoch lastModified; + + public ConsensusMigrationState(@Nonnull Epoch lastModified, @Nonnull Map tableStates) + { + checkNotNull(tableStates, "tableStates is null"); + checkNotNull(lastModified, "lastModified is null"); + this.lastModified = lastModified; + this.tableStates = ImmutableMap.copyOf(tableStates); + } + + public Map toMap(@Nullable Set keyspaceNames, @Nullable Set tableNames) + { + return ImmutableMap.of("lastModifiedEpoch", lastModified.getEpoch(), + "tableStates", tableStatesAsMaps(keyspaceNames, tableNames), + "version", PojoToString.CURRENT_VERSION); + } + + private List> tableStatesAsMaps(@Nullable Set keyspaceNames, + @Nullable Set tableNames) + { + ImmutableList.Builder> builder = ImmutableList.builder(); + for (TableMigrationState tms : tableStates.values()) + { + if (keyspaceNames != null && !keyspaceNames.contains(tms.keyspaceName)) + continue; + if (tableNames != null && !tableNames.contains(tms.tableName)) + continue; + builder.add(tms.toMap()); + } + return builder.build(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConsensusMigrationState that = (ConsensusMigrationState) o; + return tableStates.equals(that.tableStates); + } + + public ConsensusMigrationState withReversedMigrations(Map tables, Epoch epoch) + { + if (tables.isEmpty()) + return this; + + ImmutableMap.Builder updated = ImmutableMap.builder(); + + tableStates.forEach((id, state) -> { + if (!tables.containsKey(id)) + updated.put(id, state); + }); + + tables.values().forEach(metadata -> { + TableMigrationState state = tableStates.get(metadata.id); + if (state != null) + updated.put(metadata.id, state.reverseMigration(ConsensusMigrationTarget.fromTransactionalMode(metadata.params.transactionalMode), epoch)); + }); + + return new ConsensusMigrationState(lastModified, updated.build()); + } + + private static void withRangesMigrating(Map current, ImmutableMap.Builder next, TableMetadata metadata, List> ranges, boolean overwrite) + { + TableMigrationState tableState; + ConsensusMigrationTarget target = ConsensusMigrationTarget.fromTransactionalMode(metadata.params.transactionalMode); + if (!overwrite && current.containsKey(metadata.id)) + { + tableState = current.get(metadata.id).withRangesMigrating(ranges, target); + } + else + { + tableState = new TableMigrationState(metadata.keyspace, metadata.name, metadata.id, target, ImmutableSet.of(), ImmutableMap.of(Epoch.EMPTY, ranges)); + } + next.put(metadata.id, tableState); + } + + private static void putUnchanged(Map current, ImmutableMap.Builder next, Set changed) + { + current.forEach((id, migrationState) -> { + if (!changed.contains(id)) + next.put(id, migrationState); + }); + } + + private static void putUnchanged(Map current, ImmutableMap.Builder next, Collection changed) + { + Set changedIds = changed.stream().map(TableMetadata::id).collect(Collectors.toSet()); + putUnchanged(current, next, changedIds); + } + + public ConsensusMigrationState withRangesMigrating(Collection tables, List> ranges, boolean overwrite) + { + ImmutableMap.Builder updated = ImmutableMap.builder(); + putUnchanged(tableStates, updated, tables); + tables.forEach(metadata -> withRangesMigrating(tableStates, updated, metadata, ranges, overwrite)); + return new ConsensusMigrationState(lastModified, updated.build()); + } + + public ConsensusMigrationState withMigrationsCompletedFor(Collection completed) + { + ImmutableMap.Builder updated = ImmutableMap.builder(); + putUnchanged(tableStates, updated, new HashSet<>(completed)); + for (Map.Entry entry : tableStates.entrySet()) + { + if (completed.contains(entry.getKey())) + continue; + updated.put(entry); + } + return new ConsensusMigrationState(lastModified, updated.build()); + } + + public ConsensusMigrationState withRangesRepairedAtEpoch(TableMetadata metadata, List> ranges, Epoch minEpoch) + { + TableMigrationState state = Preconditions.checkNotNull(tableStates.get(metadata.id)); + state = state.withRangesRepairedAtEpoch(ranges, minEpoch); + + if (state.hasMigratedFullTokenRange(metadata.partitioner)) + { + return withMigrationsCompletedFor(Collections.singleton(metadata.id)); + } + else + { + ImmutableMap.Builder updated = ImmutableMap.builder(); + putUnchanged(tableStates, updated, Collections.singleton(metadata.id)); + updated.put(metadata.id, state); + return new ConsensusMigrationState(lastModified, updated.build()); + } + + } + + public ConsensusMigrationState withMigrationsRemovedFor(Set removed) + { + ImmutableMap.Builder updated = ImmutableMap.builder(); + putUnchanged(tableStates, updated, removed); + return new ConsensusMigrationState(lastModified, updated.build()); + } + + @Override + public int hashCode() + { + return Objects.hash(tableStates); + } + + @Override + public ConsensusMigrationState withLastModified(Epoch epoch) + { + ImmutableMap.Builder newMap = ImmutableMap.builderWithExpectedSize(tableStates.size()); + tableStates.forEach((tableId, tableState) -> { + newMap.put(tableId, tableState.withReplacementForEmptyEpoch(epoch)); + }); + return new ConsensusMigrationState(epoch, newMap.build()); + } + + @Override + public Epoch lastModified() + { + return lastModified; + } + + public void validateAgainstSchema(DistributedSchema schema) + { + tableStates.forEach((id, migrationState) -> { + TableMetadata metadata = schema.getTableMetadata(id); + Preconditions.checkState(ConsensusMigrationTarget.fromTransactionalMode(metadata.params.transactionalMode).equals(migrationState.targetProtocol)); + }); + } + + public static final MetadataSerializer serializer = new MetadataSerializer() + { + @Override + public void serialize(ConsensusMigrationState consensusMigrationState, DataOutputPlus out, Version version) throws IOException + { + Epoch.serializer.serialize(consensusMigrationState.lastModified, out, version); + serializeMap(consensusMigrationState.tableStates, out, version, TableId.metadataSerializer, TableMigrationState.serializer); + } + + @Override + public ConsensusMigrationState deserialize(DataInputPlus in, Version version) throws IOException + { + Epoch lastModified = Epoch.serializer.deserialize(in, version); + Map tableMigrationStates = deserializeMap(in, version, TableId.metadataSerializer, TableMigrationState.serializer, newHashMap()); + return new ConsensusMigrationState(lastModified, tableMigrationStates); + } + + @Override + public long serializedSize(ConsensusMigrationState t, Version version) + { + return Epoch.serializer.serializedSize(t.lastModified, version) + + serializedMapSize(t.tableStates, version, TableId.metadataSerializer, TableMigrationState.serializer); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationTarget.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationTarget.java new file mode 100644 index 0000000000..1e170f02f9 --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationTarget.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import com.google.common.primitives.SignedBytes; + +import org.apache.cassandra.service.consensus.TransactionalMode; +import org.apache.cassandra.utils.LocalizeString; + +public enum ConsensusMigrationTarget +{ + paxos(0), + accord(1); + + public final byte value; + + ConsensusMigrationTarget(int value) + { + this.value = SignedBytes.checkedCast(value); + } + + public static ConsensusMigrationTarget fromString(String targetProtocol) + { + return ConsensusMigrationTarget.valueOf(LocalizeString.toLowerCaseLocalized(targetProtocol)); + } + + public static ConsensusMigrationTarget fromValue(byte value) + { + switch (value) + { + default: + throw new IllegalArgumentException(value + " is not recognized"); + case 0: + return paxos; + case 1: + return accord; + } + } + + public static ConsensusMigrationTarget fromTransactionalMode(TransactionalMode mode) + { + return mode.accordIsEnabled ? accord : paxos; + } +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java index ac63cc4bd9..1188076c90 100644 --- a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java @@ -22,16 +22,16 @@ import javax.annotation.Nonnull; import com.google.common.annotations.VisibleForTesting; -import org.apache.cassandra.config.Config.LWTStrategy; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.dht.Range; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigratedAt; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.paxos.Paxos; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; @@ -44,9 +44,8 @@ import static org.apache.cassandra.service.consensus.migration.ConsensusKeyMigra import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.accord; import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.paxosV1; import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.paxosV2; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget.paxos; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; + +import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget.paxos; /** * Helper class to decide where to route a request that requires consensus, migrating a key if necessary @@ -77,32 +76,85 @@ public class ConsensusRequestRouter protected ConsensusRequestRouter() {} - public ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull TableId tableId, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + ConsensusRoutingDecision decisionFor(TransactionalMode transactionalMode) { - // In accord mode there might be migration state in CM (unless cleanup gets added), but it doesn't - // matter. All other consensus protocols are not used. - if (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord) + if (transactionalMode.accordIsEnabled) return accord; - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId); - if (cfs == null) - throw new IllegalStateException("Can't route consensus request for nonexistent table %s".format(tableId.toString())); - return routeAndMaybeMigrate(key, cfs, consistencyLevel, requestTime, timeoutNanos, isForWrite); + return pickPaxos(); } - protected ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull ColumnFamilyStore cfs, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + private static TableMetadata metadata(ClusterMetadata cm, String keyspace, String table) + { + KeyspaceMetadata ksm = cm.schema.getKeyspaceMetadata(keyspace); + TableMetadata tbm = ksm != null ? ksm.getTableOrViewNullable(table) : null; + + if (tbm == null) + throw new IllegalStateException("Can't route consensus request to nonexistent CFS %s.%s".format(keyspace, table)); + + return tbm; + } + + public ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull String keyspace, @Nonnull String table, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) { ClusterMetadata cm = ClusterMetadata.current(); + TableMetadata metadata = metadata(cm, keyspace, table); + return routeAndMaybeMigrate(cm, metadata, key, consistencyLevel, requestTime, timeoutNanos, isForWrite); + } - TableMigrationState tms = cm.consensusMigrationState.tableStates.get(cfs.getTableId()); + public ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull TableId tableId, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + { + ClusterMetadata cm = ClusterMetadata.current(); + TableMetadata metadata = cm.schema.getTableMetadata(tableId); + if (metadata == null) + throw new IllegalStateException("Can't route consensus request for nonexistent table %s".format(tableId.toString())); + return routeAndMaybeMigrate(cm, metadata, key, consistencyLevel, requestTime, timeoutNanos, isForWrite); + } + + protected static boolean mayWriteThroughAccord(TableMetadata metadata) + { + return metadata.params.transactionalMode.writesThroughAccord || metadata.params.transactionalMigrationFrom.writesThroughAccord(); + } + + public boolean shouldWriteThroughAccordAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull TableId tableId, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + { + ClusterMetadata cm = ClusterMetadata.current(); + TableMetadata metadata = cm.schema.getTableMetadata(tableId); + if (metadata == null) + throw new IllegalStateException("Can't route consensus request for nonexistent table %s".format(tableId.toString())); + + if (!mayWriteThroughAccord(metadata)) + return false; + + consistencyLevel = consistencyLevel.isDatacenterLocal() ? ConsistencyLevel.LOCAL_SERIAL : ConsistencyLevel.SERIAL; + ConsensusRoutingDecision decision = routeAndMaybeMigrate(cm, metadata, key, consistencyLevel, requestTime, timeoutNanos, isForWrite); + switch (decision) + { + case paxosV1: + case paxosV2: + return false; + case accord: + return true; + default: + throw new IllegalStateException("Unsupported consensus " + decision); + } + } + + protected ConsensusRoutingDecision routeAndMaybeMigrate(ClusterMetadata cm, @Nonnull TableMetadata tmd, @Nonnull DecoratedKey key, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + { + + if (!tmd.params.transactionalMigrationFrom.isMigrating()) + return decisionFor(tmd.params.transactionalMode); + + TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tmd.id); if (tms == null) - return pickPaxos(); + return decisionFor(tmd.params.transactionalMigrationFrom.from); if (Range.isInNormalizedRanges(key.getToken(), tms.migratedRanges)) return pickMigrated(tms.targetProtocol); if (Range.isInNormalizedRanges(key.getToken(), tms.migratingRanges)) - return pickBasedOnKeyMigrationStatus(cm, tms, key, cfs, consistencyLevel, requestTime, timeoutNanos, isForWrite); + return pickBasedOnKeyMigrationStatus(cm, tmd, tms, key, consistencyLevel, requestTime, timeoutNanos, isForWrite); // It's not migrated so infer the protocol from the target return pickNotMigrated(tms.targetProtocol); @@ -112,10 +164,13 @@ public class ConsensusRequestRouter * If the key was already migrated then we can pick the target protocol otherwise * we have to run a repair operation on the key to migrate it. */ - private static ConsensusRoutingDecision pickBasedOnKeyMigrationStatus(ClusterMetadata cm, TableMigrationState tms, DecoratedKey key, ColumnFamilyStore cfs, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + private static ConsensusRoutingDecision pickBasedOnKeyMigrationStatus(ClusterMetadata cm, TableMetadata tmd, TableMigrationState tms, DecoratedKey key, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) { checkState(pickPaxos() != paxosV1, "Can't migrate from PaxosV1 to anything"); + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tmd.id); + if (cfs == null) + throw new IllegalStateException("Can't route consensus request to nonexistent CFS %s.%s".format(tmd.keyspace, tmd.name)); // If it is locally replicated we can check our local migration state to see if it was already migrated EndpointsForToken naturalReplicas = ReplicaLayout.forNonLocalStrategyTokenRead(cm, cfs.keyspace.getMetadata(), key.getToken()); boolean isLocallyReplicated = naturalReplicas.lookup(FBUtilities.getBroadcastAddressAndPort()) != null; @@ -192,15 +247,18 @@ public class ConsensusRequestRouter { ClusterMetadata cm = ClusterMetadataService.instance().fetchLogFromCMS(epoch); TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tableId); - return isKeyInMigratingOrMigratedRangeFromAccord(tms, key); + return isKeyInMigratingOrMigratedRangeFromAccord(cm.schema.getTableMetadata(tableId), tms, key); } /* * A lightweight check against cluster metadata that doesn't check if the key has already been migrated * using local system table state. */ - public boolean isKeyInMigratingOrMigratedRangeFromAccord(TableMigrationState tms, DecoratedKey key) + public boolean isKeyInMigratingOrMigratedRangeFromAccord(TableMetadata metadata, TableMigrationState tms, DecoratedKey key) { + if (!metadata.params.transactionalMigrationFrom.isMigrating()) + return false; + // No state means no migration for this table if (tms == null) return false; diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigration.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigration.java new file mode 100644 index 0000000000..62727a0133 --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigration.java @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import java.util.*; +import java.util.function.Predicate; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.FutureCallback; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.repair.RepairParallelism; +import org.apache.cassandra.repair.RepairResult; +import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.consensus.TransactionalMode; +import org.apache.cassandra.service.paxos.Paxos; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.serialization.MetadataSerializer; +import org.apache.cassandra.tcm.transformations.BeginConsensusMigrationForTableAndRange; +import org.apache.cassandra.tcm.transformations.MaybeFinishConsensusMigrationForTableAndRange; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Collections.emptyList; +import static org.apache.cassandra.dht.Range.normalize; +import static org.apache.cassandra.utils.CollectionSerializers.newListSerializer; + +/** + * Track and update the migration state of individual table and ranges within those tables + */ +public abstract class ConsensusTableMigration +{ + private static final Logger logger = LoggerFactory.getLogger(ConsensusTableMigration.class); + + public static final MetadataSerializer>> rangesSerializer = newListSerializer(Range.serializer); + + public static final FutureCallback completedRepairJobHandler = new FutureCallback() + { + @Override + public void onSuccess(@Nullable RepairResult repairResult) + { + checkNotNull(repairResult, "repairResult should not be null"); + ConsensusMigrationRepairResult migrationResult = repairResult.consensusMigrationRepairResult; + + // Need to repair both Paxos and base table state + // Could track them separately, but doesn't seem worth the effort + if (migrationResult.type == ConsensusMigrationRepairType.ineligible) + return; + + RepairJobDesc desc = repairResult.desc; + TableMetadata tm = Schema.instance.getTableMetadata(desc.keyspace, desc.columnFamily); + if (tm == null) + return; + TableMigrationState tms = ClusterMetadata.current().consensusMigrationState.tableStates.get(tm.id); + if (tms == null || !Range.intersects(tms.migratingRanges, desc.ranges)) + return; + + if (tms.targetProtocol == ConsensusMigrationTarget.paxos && repairResult.consensusMigrationRepairResult.type != ConsensusMigrationRepairType.accord) + return; + if (tms.targetProtocol == ConsensusMigrationTarget.accord && repairResult.consensusMigrationRepairResult.type != ConsensusMigrationRepairType.paxos) + return; + + ClusterMetadataService.instance().commit( + new MaybeFinishConsensusMigrationForTableAndRange( + desc.keyspace, desc.columnFamily, ImmutableList.copyOf(desc.ranges), + migrationResult.minEpoch, migrationResult.type)); + } + + @Override + public void onFailure(Throwable throwable) + { + // Only successes drive forward progress + } + }; + + private ConsensusTableMigration() {} + + public static @Nullable TableMigrationState getTableMigrationState(TableId tableId) + { + ClusterMetadata cm = ClusterMetadata.current(); + return cm.consensusMigrationState.tableStates.get(tableId); + } + // Used by callers to avoid looking up the TMS multiple times + public static @Nullable TableMigrationState getTableMigrationState(long epoch, TableId tableId) + { + ClusterMetadata cm = ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(epoch)); + return cm.consensusMigrationState.tableStates.get(tableId); + } + + public static void startMigrationToConsensusProtocol(@Nonnull String targetProtocolName, + @Nullable List keyspaceNames, + @Nonnull Optional> maybeTables, + @Nonnull Optional maybeRangesStr) + { + checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty"); + ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(targetProtocolName); + + if (keyspaceNames == null || keyspaceNames.isEmpty()) + { + keyspaceNames = ImmutableList.copyOf(StorageService.instance.getNonLocalStrategyKeyspaces()); + } + checkState(keyspaceNames.size() == 1 || !maybeTables.isPresent(), "Can't specify tables with multiple keyspaces"); + List ids = keyspacesAndTablesToTableIds(keyspaceNames, maybeTables); + + // TODO (review): should this perform the schema change to make these tables accord tables? + List tableIds = new ArrayList<>(); + for (TableId tableId : ids) + { + TableMetadata metadata = Schema.instance.getTableMetadata(tableId); + if (metadata == null || !metadata.params.transactionalMigrationFrom.isMigrating()) + continue; + TransactionalMode transactionalMode = metadata.params.transactionalMode; + if (!transactionalMode.writesThroughAccord && transactionalMode != TransactionalMode.unsafe_writes) + throw new IllegalStateException("non-SERIAL writes need to be routed through Accord before attempting migration, or enable mixed mode"); + tableIds.add(tableId); + } + + if (!Paxos.useV2()) + throw new IllegalStateException("Can't do any consensus migrations to/from PaxosV1, switch to V2 first"); + + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + Optional>> maybeParsedRanges = maybeRangesStr.map(rangesStr -> ImmutableList.copyOf(RepairOption.parseRanges(rangesStr, partitioner))); + Token minToken = partitioner.getMinimumToken(); + List> ranges = maybeParsedRanges.orElse(ImmutableList.of(new Range(minToken, minToken))); + + + ClusterMetadataService.instance().commit(new BeginConsensusMigrationForTableAndRange(targetProtocol, ranges, tableIds)); + } + + public static List finishMigrationToConsensusProtocol(@Nonnull String keyspace, + @Nonnull Optional> maybeTables, + @Nonnull Optional maybeRangesStr) + { + checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty"); + + Optional>> localKeyspaceRanges = Optional.of(ImmutableList.copyOf(StorageService.instance.getLocalReplicas(keyspace).onlyFull().ranges())); + List> ranges = maybeRangesToRanges(maybeRangesStr, localKeyspaceRanges); + Map allTableMigrationStates = ClusterMetadata.current().consensusMigrationState.tableStates; + List tableIds = keyspacesAndTablesToTableIds(ImmutableList.of(keyspace), maybeTables, Optional.of(allTableMigrationStates::containsKey)); + + checkState(tableIds.stream().allMatch(allTableMigrationStates::containsKey), "All tables need to be migrating"); + List tableMigrationStates = new ArrayList<>(); + tableIds.forEach(table -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(table); + if (cfs == null) + { + logger.warn("Table {} does not exist or was dropped", cfs); + return; + } + TableMigrationState tms = allTableMigrationStates.get(table); + if (tms == null) + { + logger.warn("Table {} does not have any migration state", cfs.name); + return; + } + if(!Range.intersects(ranges, tms.migratingRanges)) + { + logger.warn("Table {} with migrating ranges {} does not intersect with any requested ranges {}", cfs.name, tms.migratingRanges, ranges); + return; + } + tableMigrationStates.add(tms); + }); + + List migratingToAccord = tableMigrationStates.stream().filter(tms -> tms.targetProtocol == ConsensusMigrationTarget.accord).collect(toImmutableList()); + List migratingToPaxos = tableMigrationStates.stream().filter(tms -> tms.targetProtocol == ConsensusMigrationTarget.paxos).collect(toImmutableList());; + + Integer accordRepairCmd = finishMigrationToAccord(keyspace, migratingToAccord, ranges); + Integer paxosRepairCmd = finishMigrationToPaxos(keyspace, migratingToPaxos, ranges); + List result = new ArrayList<>(); + if (accordRepairCmd != null) + result.add(accordRepairCmd); + if (paxosRepairCmd != null) + result.add(paxosRepairCmd); + return result; + } + + private interface MigrationFinisher + { + Integer finish(Collection tables, List> ranges); + } + + private static Integer finishMigrationTo(String name, List tableMigrationStates, List> requestedRanges, MigrationFinisher migrationFinisher) + { + logger.info("Begin finish migration to {} for ranges {} and tables {}", name, requestedRanges, tableMigrationStates); + List> intersectingRanges = new ArrayList<>(); + tableMigrationStates.stream().map(TableMigrationState::migratingRanges).forEach(intersectingRanges::addAll); + intersectingRanges = Range.normalize(intersectingRanges); + intersectingRanges = Range.intersectionOfNormalizedRanges(intersectingRanges, requestedRanges); + if (intersectingRanges.isEmpty()) + { + logger.warn("No requested ranges {} intersect any migrating ranges in any table in keyspace {}"); + return null; + } + + // Repair requires that the ranges once again be grouped by the ranges provided originally which all + // fall within local range boundaries. This was already checked in maybeRangesToRanges. + List> intersectingRangesGrouped = new ArrayList<>(); + for (Range r : requestedRanges) + { + List> intersectionsForGroup = new ArrayList<>(); + for (Range intersectedRange : intersectingRanges) + intersectionsForGroup.addAll(r.intersectionWith(intersectedRange)); + intersectingRangesGrouped.addAll(normalize(intersectionsForGroup)); + } + return migrationFinisher.finish(tableMigrationStates, intersectingRangesGrouped); + } + + /* + * This is basically just invoking classic Cassandra repair and is pretty redundant with invoking repair + * directly which would also work without issue. It's include so the same interface works for both migrating to/from + * Accord, but it's not great in that repair has a lot of options that might need to be forwarded. + * + * Still maybe more valuable to put this layer of abstraction in so we can change how it works later and it's less + * tightly coupled with the Repair interface which is pretty orthogonal to consensus migration. + */ + private static Integer finishMigrationToAccord(String keyspace, List migratingToAccord, List> requestedRanges) + { + return finishMigrationTo("Accord", migratingToAccord, requestedRanges, (tables, intersectingRanges) -> { + RepairOption repairOption = getRepairOption(tables, intersectingRanges, false); + return StorageService.instance.repair(keyspace, repairOption, emptyList()).left; + }); + } + + private static Integer finishMigrationToPaxos(String keyspace, List migratingToPaxos, List> requestedRanges) + { + return finishMigrationTo("Paxos", migratingToPaxos, requestedRanges, (tables, intersectingRanges) -> { + RepairOption repairOption = getRepairOption(tables, intersectingRanges, true); + return StorageService.instance.repair(keyspace, repairOption, emptyList()).left; + }); + } + + + private static List keyspacesAndTablesToTableIds(@Nonnull List keyspaceNames, @Nonnull Optional> maybeTables) + { + return keyspacesAndTablesToTableIds(keyspaceNames, maybeTables, Optional.empty()); + } + + private static List keyspacesAndTablesToTableIds(@Nonnull List keyspaceNames, @Nonnull Optional> maybeTables, @Nonnull Optional> includeTable) + { + List tableIds = new ArrayList<>(); + for (String keyspaceName : keyspaceNames) + { + Optional> maybeTableIds = maybeTables.map(tableNames -> + tableNames + .stream() + .map(tableName -> { + TableMetadata tm = Schema.instance.getTableMetadata(keyspaceName, tableName); + if (tm == null) + throw new IllegalArgumentException("Unknown table %s.%s".format(keyspaceName, tableName)); + return tm.id; + }) + .collect(toImmutableList())); + tableIds.addAll( + maybeTableIds.orElseGet(() -> + Schema.instance.getKeyspaceInstance(keyspaceName).getColumnFamilyStores() + .stream() + .map(ColumnFamilyStore::getTableId) + .filter(includeTable.orElse(Predicates.alwaysTrue())) // Filter out non-migrating so they don't generate an error + .collect(toImmutableList()))); + } + return tableIds; + } + + @Nonnull + private static RepairOption getRepairOption(Collection tables, List> intersectingRanges, boolean accordRepair) + { + boolean primaryRange = false; + // TODO (review): Should disabling incremental repair be exposed for the Paxos repair in case someone explicitly does not do incremental repair? + boolean incremental = !accordRepair; + boolean trace = false; + int numJobThreads = 1; + boolean pullRepair = false; + boolean forceRepair = false; + boolean optimiseStreams = false; + boolean ignoreUnreplicatedKeyspaces = true; + boolean repairPaxos = !accordRepair; + boolean paxosOnly = false; + boolean dontPurgeTombstones = false; + RepairOption repairOption = new RepairOption(RepairParallelism.PARALLEL, primaryRange, incremental, trace, numJobThreads, intersectingRanges, pullRepair, forceRepair, PreviewKind.NONE, optimiseStreams, ignoreUnreplicatedKeyspaces, repairPaxos, paxosOnly, dontPurgeTombstones, accordRepair); + tables.forEach(table -> repairOption.getColumnFamilies().add(table.tableName)); + return repairOption; + } + + + // Repair is restricted to local ranges, but manipulating CMS migration state doesn't need to be restricted + private static @Nonnull List> maybeRangesToRanges(@Nonnull Optional maybeRangesStr) + { + return maybeRangesToRanges(maybeRangesStr, Optional.empty()); + } + + private static @Nonnull List> maybeRangesToRanges(@Nonnull Optional maybeRangesStr, Optional>> restrictToRanges) + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + Optional>> maybeParsedRanges = maybeRangesStr.map(rangesStr -> ImmutableList.copyOf(RepairOption.parseRanges(rangesStr, partitioner))); + Token minToken = partitioner.getMinimumToken(); + List> defaultRanges = restrictToRanges.orElse(ImmutableList.of(new Range(minToken, minToken))); + List> ranges = maybeParsedRanges.orElse(defaultRanges); + checkArgument(ranges.stream().allMatch(range -> defaultRanges.stream().anyMatch(defaultRange -> defaultRange.contains(range))), + "If ranges are specified each range must be contained within a local range (" + defaultRanges + ") for this node to allow for precise repairs. Specified " + ranges); + return ranges; + } +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigrationState.java b/src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigrationState.java deleted file mode 100644 index 62b03eefa9..0000000000 --- a/src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigrationState.java +++ /dev/null @@ -1,909 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.service.consensus.migration; - -import java.io.IOException; -import java.util.AbstractMap.SimpleEntry; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMap.Builder; -import com.google.common.collect.ImmutableSortedMap; -import com.google.common.primitives.SignedBytes; -import com.google.common.util.concurrent.FutureCallback; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.config.Config.LWTStrategy; -import org.apache.cassandra.config.Config.NonSerialWriteStrategy; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.repair.RepairJobDesc; -import org.apache.cassandra.repair.RepairParallelism; -import org.apache.cassandra.repair.RepairResult; -import org.apache.cassandra.repair.messages.RepairOption; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.paxos.Paxos; -import org.apache.cassandra.streaming.PreviewKind; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.ClusterMetadataService; -import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.tcm.MetadataValue; -import org.apache.cassandra.tcm.serialization.MetadataSerializer; -import org.apache.cassandra.tcm.serialization.Version; -import org.apache.cassandra.tcm.transformations.BeginConsensusMigrationForTableAndRange; -import org.apache.cassandra.tcm.transformations.MaybeFinishConsensusMigrationForTableAndRange; -import org.apache.cassandra.tcm.transformations.SetConsensusMigrationTargetProtocol; -import org.apache.cassandra.utils.LocalizeString; -import org.apache.cassandra.utils.NullableSerializer; -import org.apache.cassandra.utils.PojoToString; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.util.Collections.emptyList; -import static org.apache.cassandra.db.TypeSizes.sizeof; -import static org.apache.cassandra.dht.Range.intersectionOfNormalizedRanges; -import static org.apache.cassandra.dht.Range.normalize; -import static org.apache.cassandra.dht.Range.subtract; -import static org.apache.cassandra.dht.Range.subtractNormalizedRanges; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget.reset; -import static org.apache.cassandra.utils.CollectionSerializers.deserializeMap; -import static org.apache.cassandra.utils.CollectionSerializers.deserializeSet; -import static org.apache.cassandra.utils.CollectionSerializers.newHashMap; -import static org.apache.cassandra.utils.CollectionSerializers.newListSerializer; -import static org.apache.cassandra.utils.CollectionSerializers.serializeCollection; -import static org.apache.cassandra.utils.CollectionSerializers.serializeMap; -import static org.apache.cassandra.utils.CollectionSerializers.serializedCollectionSize; -import static org.apache.cassandra.utils.CollectionSerializers.serializedMapSize; - -/** - * Track and update the migration state of individual table and ranges within those tables - */ -public abstract class ConsensusTableMigrationState -{ - private static final Logger logger = LoggerFactory.getLogger(ConsensusTableMigrationState.class); - - public static final MetadataSerializer>> rangesSerializer = newListSerializer(Range.serializer); - - public static final FutureCallback completedRepairJobHandler = new FutureCallback() - { - @Override - public void onSuccess(@Nullable RepairResult repairResult) - { - checkNotNull(repairResult, "repairResult should not be null"); - ConsensusMigrationRepairResult migrationResult = repairResult.consensusMigrationRepairResult; - - // Need to repair both Paxos and base table state - // Could track them separately, but doesn't seem worth the effort - if (migrationResult.type == ConsensusMigrationRepairType.ineligible) - return; - - RepairJobDesc desc = repairResult.desc; - TableMetadata tm = Schema.instance.getTableMetadata(desc.keyspace, desc.columnFamily); - if (tm == null) - return; - TableMigrationState tms = ClusterMetadata.current().consensusMigrationState.tableStates.get(tm.id); - if (tms == null || !Range.intersects(tms.migratingRanges, desc.ranges)) - return; - - if (tms.targetProtocol == ConsensusMigrationTarget.paxos && repairResult.consensusMigrationRepairResult.type != ConsensusMigrationRepairType.accord) - return; - if (tms.targetProtocol == ConsensusMigrationTarget.accord && repairResult.consensusMigrationRepairResult.type != ConsensusMigrationRepairType.paxos) - return; - - logger.info("Repair {} is going to trigger migration completion for ranges {} and epoch {}", desc.sessionId, desc.ranges, migrationResult.minEpoch); - - ClusterMetadataService.instance().commit( - new MaybeFinishConsensusMigrationForTableAndRange( - desc.keyspace, desc.columnFamily, ImmutableList.copyOf(desc.ranges), - migrationResult.minEpoch, migrationResult.type)); - } - - @Override - public void onFailure(Throwable throwable) - { - // Only successes drive forward progress - } - }; - - public static void reset() - { - ClusterMetadata cm = ClusterMetadata.current(); - for (TableMigrationState tms : cm.consensusMigrationState.tableStates.values()) - setConsensusMigrationTargetProtocol("reset", - ImmutableList.of(tms.keyspaceName), - Optional.of(ImmutableList.of(tms.tableName))); - } - - public enum ConsensusMigrationRepairType - { - ineligible(0), - paxos(1), - accord(2); - - public final byte value; - - ConsensusMigrationRepairType(int value) - { - this.value = SignedBytes.checkedCast(value); - } - - public static ConsensusMigrationRepairType fromString(String repairType) - { - return ConsensusMigrationRepairType.valueOf(LocalizeString.toLowerCaseLocalized(repairType)); - } - - public static ConsensusMigrationRepairType fromValue(byte value) - { - switch (value) - { - default: - throw new IllegalArgumentException(value + " is not recognized"); - case 0: - return ConsensusMigrationRepairType.ineligible; - case 1: - return ConsensusMigrationRepairType.paxos; - case 2: - return ConsensusMigrationRepairType.accord; - } - } - } - - public enum ConsensusMigrationTarget - { - paxos(0), - accord(1), - reset(2); - - public final byte value; - - ConsensusMigrationTarget(int value) - { - this.value = SignedBytes.checkedCast(value); - } - - public static ConsensusMigrationTarget fromString(String targetProtocol) - { - return ConsensusMigrationTarget.valueOf(LocalizeString.toLowerCaseLocalized(targetProtocol)); - } - - public static ConsensusMigrationTarget fromValue(byte value) - { - switch (value) - { - default: - throw new IllegalArgumentException(value + " is not recognized"); - case 0: - return paxos; - case 1: - return accord; - case 2: - return reset; - } - } - } - - public static class ConsensusMigrationRepairResult - { - private final ConsensusMigrationRepairType type; - private final Epoch minEpoch; - - private ConsensusMigrationRepairResult(ConsensusMigrationRepairType type, Epoch minEpoch) - { - this.type = type; - this.minEpoch = minEpoch; - } - - public static ConsensusMigrationRepairResult fromCassandraRepair(Epoch minEpoch, boolean migrationEligibleRepair) - { - checkArgument(!migrationEligibleRepair || minEpoch.isAfter(Epoch.EMPTY), "Epoch should not be empty if Paxos and regular repairs were performed"); - if (migrationEligibleRepair) - return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.paxos, minEpoch); - else - return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.ineligible, Epoch.EMPTY); - } - - public static ConsensusMigrationRepairResult fromAccordRepair(Epoch minEpoch) - { - checkArgument(minEpoch.isAfter(Epoch.EMPTY), "Accord repairs should always occur at an Epoch"); - return new ConsensusMigrationRepairResult(ConsensusMigrationRepairType.accord, minEpoch); - } - } - - public static class ConsensusMigratedAt - { - public static final IVersionedSerializer serializer = NullableSerializer.wrap(new IVersionedSerializer() - { - @Override - public void serialize(ConsensusMigratedAt t, DataOutputPlus out, int version) throws IOException - { - Epoch.messageSerializer.serialize(t.migratedAtEpoch, out, version); - out.writeByte(t.migratedAtTarget.value); - } - - @Override - public ConsensusMigratedAt deserialize(DataInputPlus in, int version) throws IOException - { - Epoch migratedAtEpoch = Epoch.messageSerializer.deserialize(in, version); - ConsensusMigrationTarget target = ConsensusMigrationTarget.fromValue(in.readByte()); - return new ConsensusMigratedAt(migratedAtEpoch, target); - } - - @Override - public long serializedSize(ConsensusMigratedAt t, int version) - { - return TypeSizes.sizeof(ConsensusMigrationTarget.accord.value) - + Epoch.messageSerializer.serializedSize(t.migratedAtEpoch, version); - } - }); - - // Fields are not nullable when used for messaging - @Nullable - public final Epoch migratedAtEpoch; - - @Nullable - public final ConsensusMigrationTarget migratedAtTarget; - - public ConsensusMigratedAt(Epoch migratedAtEpoch, ConsensusMigrationTarget migratedAtTarget) - { - this.migratedAtEpoch = migratedAtEpoch; - this.migratedAtTarget = migratedAtTarget; - } - } - - // TODO (desired): Move this into the schema for the table once this is based off of TrM - public static class TableMigrationState - { - @Nonnull - public final String keyspaceName; - - @Nonnull - public final String tableName; - - @Nonnull - public final TableId tableId; - - @Nonnull - public final ConsensusMigrationTarget targetProtocol; - - @Nonnull - public final List> migratedRanges; - - /* - * Necessary to track which ranges started migrating at which epoch - * in order to know whether a repair qualifies in terms of finishing - * migration of the range. - */ - @Nonnull - public final NavigableMap>> migratingRangesByEpoch; - - public static final MetadataSerializer serializer = new MetadataSerializer() - { - @Override - public void serialize(TableMigrationState t, DataOutputPlus out, Version version) throws IOException - { - out.write(t.targetProtocol.value); - out.writeUTF(t.keyspaceName); - out.writeUTF(t.tableName); - t.tableId.serialize(out); - serializeCollection(t.migratedRanges, out, version, Range.serializer); - serializeMap(t.migratingRangesByEpoch, out, version, Epoch.serializer, rangesSerializer); - } - - @Override - public TableMigrationState deserialize(DataInputPlus in, Version version) throws IOException - { - ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromValue(in.readByte()); - String keyspaceName = in.readUTF(); - String tableName = in.readUTF(); - TableId tableId = TableId.deserialize(in); - Set> migratedRanges = deserializeSet(in, version, Range.serializer); - Map>> migratingRangesByEpoch = deserializeMap(in, version, Epoch.serializer, rangesSerializer, newHashMap()); - return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, migratingRangesByEpoch); - } - - @Override - public long serializedSize(TableMigrationState t, Version version) - { - return sizeof(t.targetProtocol.value) - + sizeof(t.keyspaceName) - + sizeof(t.tableName) - + t.tableId.serializedSize() - + serializedCollectionSize(t.migratedRanges, version, Range.serializer) - + serializedMapSize(t.migratingRangesByEpoch, version, Epoch.serializer, rangesSerializer); - } - }; - - @Nonnull - public final List> migratingRanges; - - @Nonnull - public final List> migratingAndMigratedRanges; - - public TableMigrationState(@Nonnull String keyspaceName, - @Nonnull String tableName, - @Nonnull TableId tableId, - @Nonnull ConsensusMigrationTarget targetProtocol, - @Nonnull Collection> migratedRanges, - @Nonnull Map>> migratingRangesByEpoch) - { - this.keyspaceName = keyspaceName; - this.tableName = tableName; - this.tableId = tableId; - this.targetProtocol = targetProtocol; - this.migratedRanges = ImmutableList.copyOf(normalize(migratedRanges)); - this.migratingRangesByEpoch = ImmutableSortedMap.copyOf( - migratingRangesByEpoch.entrySet() - .stream() - .map( entry -> new SimpleEntry<>(entry.getKey(), ImmutableList.copyOf(normalize(entry.getValue())))) - .collect(Collectors.toList())); - this.migratingRanges = ImmutableList.copyOf(normalize(migratingRangesByEpoch.values().stream().flatMap(Collection::stream).collect(Collectors.toList()))); - this.migratingAndMigratedRanges = ImmutableList.copyOf(normalize(ImmutableList.>builder().addAll(migratedRanges).addAll(migratingRanges).build())); - } - - public TableMigrationState withRangesMigrating(@Nonnull Collection> ranges, - @Nonnull ConsensusMigrationTarget target) - { - checkState(!migratingRangesByEpoch.containsKey(Epoch.EMPTY), "Shouldn't already have an entry for the empty epoch"); - // Doesn't matter which epoch the range started migrating in for this context so merge them all - Collection> migratingRanges = normalize(migratingRangesByEpoch.values().stream().flatMap(Collection::stream).collect(Collectors.toList())); - checkArgument(target == targetProtocol, "Requested migration to target protocol " + target + " conflicts with in progress migration to protocol " + targetProtocol); - List> normalizedRanges = normalize(ranges); - if (subtract(normalizedRanges, migratingRanges).isEmpty()) - logger.warn("Range " + ranges + " is already being migrated"); - Set> withoutAlreadyMigrated = subtract(normalizedRanges, migratedRanges); - if (withoutAlreadyMigrated.isEmpty()) - logger.warn("Range " + ranges + " is already migrated"); - Set> withoutBoth = subtract(withoutAlreadyMigrated, migratingRanges); - if (withoutBoth.isEmpty()) - logger.warn("Range " + ranges + " is already migrating/migrated"); - - if (!Range.equals(normalizedRanges, withoutBoth)) - logger.warn("Ranges " + normalizedRanges + " to start migrating is already partially migrating/migrated " + withoutBoth); - - Map>> newMigratingRanges = new HashMap<>(migratingRangesByEpoch.size() + 1); - newMigratingRanges.putAll(migratingRangesByEpoch); - newMigratingRanges.put(Epoch.EMPTY, normalizedRanges); - - return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, newMigratingRanges); - } - - public TableMigrationState withReplacementForEmptyEpoch(@Nonnull Epoch replacementEpoch) - { - if (!migratingRangesByEpoch.containsKey(Epoch.EMPTY)) - return this; - Map>> newMigratingRangesByEpoch = new HashMap<>(migratingRangesByEpoch.size()); - migratingRangesByEpoch.forEach((epoch, ranges) -> { - if (epoch.equals(Epoch.EMPTY)) - newMigratingRangesByEpoch.put(replacementEpoch, ranges); - else - newMigratingRangesByEpoch.put(epoch, ranges); - }); - - if (newMigratingRangesByEpoch != null) - return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, newMigratingRangesByEpoch); - else - return this; - } - - public TableMigrationState withRangesRepairedAtEpoch(@Nonnull Collection> ranges, - @Nonnull Epoch epoch) - { - checkState(!migratingRangesByEpoch.containsKey(Epoch.EMPTY), "Shouldn't have an entry for the empty epoch"); - checkArgument(epoch.isAfter(Epoch.EMPTY), "Epoch shouldn't be empty"); - - List> normalizedRepairedRanges = normalize(ranges); - // This should be inclusive because the epoch we store in the map is the epoch in which the range has been marked migrating - // in startMigrationToConsensusProtocol - NavigableMap>> coveredEpochs = migratingRangesByEpoch.headMap(epoch, true); - List> normalizedMigratingRanges = normalize(coveredEpochs.values().stream().flatMap(Collection::stream).collect(Collectors.toList())); - List> normalizedRepairedIntersection = intersectionOfNormalizedRanges(normalizedRepairedRanges, normalizedMigratingRanges); - checkState(!normalizedRepairedIntersection.isEmpty(), "None of Ranges " + ranges + " were being migrated"); - - Map>> newMigratingRangesByEpoch = new HashMap<>(); - - // Everything in this epoch or later can't have been migrated so re-add all of them - newMigratingRangesByEpoch.putAll(migratingRangesByEpoch.tailMap(epoch, false)); - - // Include anything still remaining to be migrated after subtracting what was repaired - for (Map.Entry>> e : coveredEpochs.entrySet()) - { - // Epoch when these ranges started migrating - Epoch rangesEpoch = e.getKey(); - List> epochMigratingRanges = e.getValue(); - List> remainingRanges = subtractNormalizedRanges(epochMigratingRanges, normalizedRepairedIntersection); - if (!remainingRanges.isEmpty()) - newMigratingRangesByEpoch.put(rangesEpoch, remainingRanges); - } - - List> newMigratedRanges = new ArrayList<>(normalizedMigratingRanges.size() + ranges.size()); - newMigratedRanges.addAll(migratedRanges); - newMigratedRanges.addAll(normalizedRepairedIntersection); - return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, newMigratedRanges, newMigratingRangesByEpoch); - } - - public boolean paxosReadSatisfiedByKeyMigrationAtEpoch(DecoratedKey key, ConsensusMigratedAt consensusMigratedAt) - { - // This check is being done from a Paxos read attempt which needs to - // check if Accord needs to resolve any in flight accord transactions - // if the migration target is Accord then nothing needs to be done - if (targetProtocol != ConsensusMigrationTarget.paxos) - return true; - - return satisfiedByKeyMigrationAtEpoch(key, consensusMigratedAt); - } - - public boolean satisfiedByKeyMigrationAtEpoch(@Nonnull DecoratedKey key, @Nullable ConsensusMigratedAt consensusMigratedAt) - { - if (consensusMigratedAt == null) - { - // It hasn't been migrated and needs migration if it is in a migrating range - return Range.isInNormalizedRanges(key.getToken(), migratingRanges); - } - else - { - // It has been migrated and might be from a late enough epoch to satisfy this migration - return consensusMigratedAt.migratedAtTarget == targetProtocol - && migratingRangesByEpoch.headMap(consensusMigratedAt.migratedAtEpoch, true).values() - .stream() - .flatMap(List::stream) - .anyMatch(range -> range.contains(key.getToken())); - } - } - - public Epoch minMigrationEpoch(Token token) - { - for (Map.Entry>> e : migratingRangesByEpoch.entrySet()) - { - if (Range.isInNormalizedRanges(token, e.getValue())) - return e.getKey(); - } - return Epoch.EMPTY; - } - - - public @Nonnull TableId getTableId() - { - return tableId; - } - - public TableMigrationState withMigrationTarget(ConsensusMigrationTarget newTargetProtocol) - { - checkState(!migratingRangesByEpoch.containsKey(Epoch.EMPTY), "Shouldn't have an entry for the empty epoch"); - if (this.targetProtocol == newTargetProtocol) - return this; - - // Migrating ranges remain migrating because individual keys may have already been migrated - // So for correctness we need to perform key migration - // We do need to update the epoch so that a new repair is required to drive the migration - Map>> migratingRangesByEpoch = ImmutableMap.of(Epoch.EMPTY, migratingRanges); - - Token minToken = ColumnFamilyStore.getIfExists(tableId).getPartitioner().getMinimumToken(); - Range fullRange = new Range(minToken, minToken); - // What is migrated already is anything that was never migrated/migrating before (untouched) - List> migratedRanges = ImmutableList.copyOf(normalize(fullRange.subtractAll(migratingAndMigratedRanges))); - - return new TableMigrationState(keyspaceName, tableName, tableId, newTargetProtocol, migratedRanges, migratingRangesByEpoch); - } - - public Map toMap() - { - Builder builder = ImmutableMap.builder(); - builder.put("keyspace", keyspaceName); - builder.put("table", tableName); - builder.put("tableId", tableId.toString()); - builder.put("targetProtocol", targetProtocol.toString()); - builder.put("migratedRanges", migratedRanges.stream().map(Objects::toString).collect(toImmutableList())); - Map> rangesByEpoch = new LinkedHashMap<>(); - for (Map.Entry>> entry : migratingRangesByEpoch.entrySet()) - { - rangesByEpoch.put(entry.getKey().getEpoch(), entry.getValue().stream().map(Objects::toString).collect(toImmutableList())); - } - builder.put("migratingRangesByEpoch", rangesByEpoch); - return builder.build(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TableMigrationState that = (TableMigrationState) o; - return keyspaceName.equals(that.keyspaceName) && tableName.equals(that.tableName) && tableId.equals(that.tableId) && targetProtocol == that.targetProtocol && migratedRanges.equals(that.migratedRanges) && migratingRangesByEpoch.equals(that.migratingRangesByEpoch) && migratingRanges.equals(that.migratingRanges) && migratingAndMigratedRanges.equals(that.migratingAndMigratedRanges); - } - - @Override - public int hashCode() - { - return Objects.hash(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, migratingRangesByEpoch, migratingRanges, migratingAndMigratedRanges); - } - - public List> migratingRanges() - { - return migratingRanges; - } - } - - public static class ConsensusMigrationState implements MetadataValue - { - public static ConsensusMigrationState EMPTY = new ConsensusMigrationState(Epoch.EMPTY, ImmutableMap.of()); - @Nonnull - public final Map tableStates; - - public final Epoch lastModified; - - - public ConsensusMigrationState(@Nonnull Epoch lastModified, @Nonnull Map tableStates) - { - checkNotNull(tableStates, "tableStates is null"); - checkNotNull(lastModified, "lastModified is null"); - this.lastModified = lastModified; - this.tableStates = ImmutableMap.copyOf(tableStates); - } - - public Map toMap(@Nullable Set keyspaceNames, @Nullable Set tableNames) - { - return ImmutableMap.of("lastModifiedEpoch", lastModified.getEpoch(), - "tableStates", tableStatesAsMaps(keyspaceNames, tableNames), - "version", PojoToString.CURRENT_VERSION); - } - - private List> tableStatesAsMaps(@Nullable Set keyspaceNames, - @Nullable Set tableNames) - { - ImmutableList.Builder> builder = ImmutableList.builder(); - for (TableMigrationState tms : tableStates.values()) - { - if (keyspaceNames != null && !keyspaceNames.contains(tms.keyspaceName)) - continue; - if (tableNames != null && !tableNames.contains(tms.tableName)) - continue; - builder.add(tms.toMap()); - } - return builder.build(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ConsensusMigrationState that = (ConsensusMigrationState) o; - return tableStates.equals(that.tableStates); - } - - @Override - public int hashCode() - { - return Objects.hash(tableStates); - } - - public static final MetadataSerializer serializer = new MetadataSerializer() - { - @Override - public void serialize(ConsensusMigrationState consensusMigrationState, DataOutputPlus out, Version version) throws IOException - { - Epoch.serializer.serialize(consensusMigrationState.lastModified, out, version); - serializeMap(consensusMigrationState.tableStates, out, version, TableId.metadataSerializer, TableMigrationState.serializer); - } - - @Override - public ConsensusMigrationState deserialize(DataInputPlus in, Version version) throws IOException - { - Epoch lastModified = Epoch.serializer.deserialize(in, version); - Map tableMigrationStates = deserializeMap(in, version, TableId.metadataSerializer, TableMigrationState.serializer, newHashMap()); - return new ConsensusMigrationState(lastModified, tableMigrationStates); - } - - @Override - public long serializedSize(ConsensusMigrationState t, Version version) - { - return Epoch.serializer.serializedSize(t.lastModified, version) - + serializedMapSize(t.tableStates, version, TableId.metadataSerializer, TableMigrationState.serializer); - } - }; - - @Override - public ConsensusMigrationState withLastModified(Epoch epoch) - { - ImmutableMap.Builder newMap = ImmutableMap.builderWithExpectedSize(tableStates.size()); - tableStates.forEach((tableId, tableState) -> { - newMap.put(tableId, tableState.withReplacementForEmptyEpoch(epoch)); - }); - return new ConsensusMigrationState(epoch, newMap.build()); - } - - @Override - public Epoch lastModified() - { - return lastModified; - } - } - - private ConsensusTableMigrationState() {} - - // Used by callers to avoid looking up the TMS multiple times - public static @Nullable TableMigrationState getTableMigrationState(TableId tableId) - { - TableMigrationState tms = ClusterMetadata.current().consensusMigrationState.tableStates.get(tableId); - return tms; - } - - /* - * Set or change the migration target for the keyspaces and tables. Can be used to reverse the direction of a migration - * or instantly migrate a table to a new protocol. - */ - public static void setConsensusMigrationTargetProtocol(@Nonnull String targetProtocolName, - @Nullable List keyspaceNames, - @Nonnull Optional> maybeTables) - { - checkArgument(!maybeTables.isPresent() || (keyspaceNames != null && keyspaceNames.size() == 1), "Must specify one keyspace along with tables"); - checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty"); - keyspaceNames = maybeDefaultKeyspaceNames(keyspaceNames); - ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(targetProtocolName); - - if (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord) - throw new IllegalStateException("Mixing a hard coded strategy with migration is unsupported"); - - if (!Paxos.useV2()) - throw new IllegalStateException("Can't do any consensus migrations from/to PaxosV1, switch to V2 first"); - - List tableIds = keyspacesAndTablesToTableIds(keyspaceNames, maybeTables); - ClusterMetadataService.instance().commit(new SetConsensusMigrationTargetProtocol(targetProtocol, tableIds)); - } - - public static void startMigrationToConsensusProtocol(@Nonnull String targetProtocolName, - @Nullable List keyspaceNames, - @Nonnull Optional> maybeTables, - @Nonnull Optional maybeRangesStr) - { - checkState(keyspaceNames.size() == 1 || !maybeTables.isPresent(), "Must specify one keyspace along with tables"); - checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty"); - ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(targetProtocolName); - checkArgument(targetProtocol != reset, "Can't start migration to reset"); - - - if (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord) - throw new IllegalStateException("Mixing a hard coded strategy with migration is unsupported"); - - NonSerialWriteStrategy nonSerialWriteStrategy = DatabaseDescriptor.getNonSerialWriteStrategy(); - if (!nonSerialWriteStrategy.writesThroughAccord && nonSerialWriteStrategy != NonSerialWriteStrategy.mixed) - throw new IllegalStateException("non-SERIAL writes need to be routed through Accord before attempting migration, or enable mixed mode"); - - if (!Paxos.useV2()) - throw new IllegalStateException("Can't do any consensus migrations to/from PaxosV1, switch to V2 first"); - - keyspaceNames = maybeDefaultKeyspaceNames(keyspaceNames); - List> ranges = maybeRangesToRanges(maybeRangesStr); - List tableIds = keyspacesAndTablesToTableIds(keyspaceNames, maybeTables); - - ClusterMetadataService.instance().commit(new BeginConsensusMigrationForTableAndRange(targetProtocol, ranges, tableIds)); - } - - public static List finishMigrationToConsensusProtocol(@Nonnull String keyspace, - @Nonnull Optional> maybeTables, - @Nonnull Optional maybeRangesStr) - { - checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty"); - - Optional>> localKeyspaceRanges = Optional.of(ImmutableList.copyOf(StorageService.instance.getLocalReplicas(keyspace).onlyFull().ranges())); - List> ranges = maybeRangesToRanges(maybeRangesStr, localKeyspaceRanges); - Map allTableMigrationStates = ClusterMetadata.current().consensusMigrationState.tableStates; - List tableIds = keyspacesAndTablesToTableIds(ImmutableList.of(keyspace), maybeTables, Optional.of(allTableMigrationStates::containsKey)); - - checkState(tableIds.stream().allMatch(allTableMigrationStates::containsKey), "All tables need to be migrating"); - List tableMigrationStates = new ArrayList<>(); - tableIds.forEach(table -> { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(table); - if (cfs == null) - { - logger.warn("Table {} does not exist or was dropped", cfs); - return; - } - TableMigrationState tms = allTableMigrationStates.get(table); - if (tms == null) - { - logger.warn("Table {} does not have any migration state", cfs.name); - return; - } - if(!Range.intersects(ranges, tms.migratingRanges)) - { - logger.warn("Table {} with migrating ranges {} does not intersect with any requested ranges {}", cfs.name, tms.migratingRanges, ranges); - return; - } - tableMigrationStates.add(tms); - }); - - List migratingToAccord = tableMigrationStates.stream().filter(tms -> tms.targetProtocol == ConsensusMigrationTarget.accord).collect(toImmutableList()); - List migratingToPaxos = tableMigrationStates.stream().filter(tms -> tms.targetProtocol == ConsensusMigrationTarget.paxos).collect(toImmutableList());; - - Integer accordRepairCmd = finishMigrationToAccord(keyspace, migratingToAccord, ranges); - Integer paxosRepairCmd = finishMigrationToPaxos(keyspace, migratingToPaxos, ranges); - List result = new ArrayList<>(); - if (accordRepairCmd != null) - result.add(accordRepairCmd); - if (paxosRepairCmd != null) - result.add(paxosRepairCmd); - return result; - } - - private interface MigrationFinisher - { - Integer finish(Collection tables, List> ranges); - } - - private static Integer finishMigrationTo(String name, List tableMigrationStates, List> requestedRanges, MigrationFinisher migrationFinisher) - { - logger.info("Begin finish migration to {} for ranges {} and tables {}", name, requestedRanges, tableMigrationStates); - List> intersectingRanges = new ArrayList<>(); - tableMigrationStates.stream().map(TableMigrationState::migratingRanges).forEach(intersectingRanges::addAll); - intersectingRanges = Range.normalize(intersectingRanges); - intersectingRanges = Range.intersectionOfNormalizedRanges(intersectingRanges, requestedRanges); - if (intersectingRanges.isEmpty()) - { - logger.warn("No requested ranges {} intersect any migrating ranges in any table in keyspace {}"); - return null; - } - - // Repair requires that the ranges once again be grouped by the ranges provided originally which all - // fall within local range boundaries. This was already checked in maybeRangesToRanges. - List> intersectingRangesGrouped = new ArrayList<>(); - for (Range r : requestedRanges) - { - List> intersectionsForGroup = new ArrayList<>(); - for (Range intersectedRange : intersectingRanges) - intersectionsForGroup.addAll(r.intersectionWith(intersectedRange)); - intersectingRangesGrouped.addAll(normalize(intersectionsForGroup)); - } - return migrationFinisher.finish(tableMigrationStates, intersectingRangesGrouped); - } - - /* - * This is basically just invoking classic Cassandra repair and is pretty redundant with invoking repair - * directly which would also work without issue. It's include so the same interface works for both migrating to/from - * Accord, but it's not great in that repair has a lot of options that might need to be forwarded. - * - * Still maybe more valuable to put this layer of abstraction in so we can change how it works later and it's less - * tightly coupled with the Repair interface which is pretty orthogonal to consensus migration. - */ - private static Integer finishMigrationToAccord(String keyspace, List migratingToAccord, List> requestedRanges) - { - return finishMigrationTo("Accord", migratingToAccord, requestedRanges, (tables, intersectingRanges) -> { - RepairOption repairOption = getRepairOption(tables, intersectingRanges, false); - return StorageService.instance.repair(keyspace, repairOption, emptyList()).left; - }); - } - - private static Integer finishMigrationToPaxos(String keyspace, List migratingToPaxos, List> requestedRanges) - { - return finishMigrationTo("Paxos", migratingToPaxos, requestedRanges, (tables, intersectingRanges) -> { - RepairOption repairOption = getRepairOption(tables, intersectingRanges, true); - return StorageService.instance.repair(keyspace, repairOption, emptyList()).left; - }); - } - - @Nonnull - private static RepairOption getRepairOption(Collection tables, List> intersectingRanges, boolean accordRepair) - { - boolean primaryRange = false; - // TODO (review): Should disabling incremental repair be exposed for the Paxos repair in case someone explicitly does not do incremental repair? - boolean incremental = !accordRepair; - boolean trace = false; - int numJobThreads = 1; - boolean pullRepair = false; - boolean forceRepair = false; - boolean optimiseStreams = false; - boolean ignoreUnreplicatedKeyspaces = true; - boolean repairPaxos = !accordRepair; - boolean paxosOnly = false; - boolean dontPurgeTombstones = false; - RepairOption repairOption = new RepairOption(RepairParallelism.PARALLEL, primaryRange, incremental, trace, numJobThreads, intersectingRanges, pullRepair, forceRepair, PreviewKind.NONE, optimiseStreams, ignoreUnreplicatedKeyspaces, repairPaxos, paxosOnly, dontPurgeTombstones, accordRepair); - tables.forEach(table -> repairOption.getColumnFamilies().add(table.tableName)); - return repairOption; - } - - - // Repair is restricted to local ranges, but manipulating CMS migration state doesn't need to be restricted - private static @Nonnull List> maybeRangesToRanges(@Nonnull Optional maybeRangesStr) - { - return maybeRangesToRanges(maybeRangesStr, Optional.empty()); - } - - private static @Nonnull List> maybeRangesToRanges(@Nonnull Optional maybeRangesStr, Optional>> restrictToRanges) - { - IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); - Optional>> maybeParsedRanges = maybeRangesStr.map(rangesStr -> ImmutableList.copyOf(RepairOption.parseRanges(rangesStr, partitioner))); - Token minToken = partitioner.getMinimumToken(); - List> defaultRanges = restrictToRanges.orElse(ImmutableList.of(new Range(minToken, minToken))); - List> ranges = maybeParsedRanges.orElse(defaultRanges); - checkArgument(ranges.stream().allMatch(range -> defaultRanges.stream().anyMatch(defaultRange -> defaultRange.contains(range))), - "If ranges are specified each range must be contained within a local range (" + defaultRanges + ") for this node to allow for precise repairs. Specified " + ranges); - return ranges; - } - - private static List maybeDefaultKeyspaceNames(@Nullable List keyspaceNames) - { - if (keyspaceNames == null || keyspaceNames.isEmpty()) - { - keyspaceNames = ImmutableList.copyOf(StorageService.instance.getNonSystemKeyspaces()); - } - checkState(keyspaceNames.stream().noneMatch(SchemaConstants::isSystemKeyspace), "Migrating system keyspaces is not supported"); - return keyspaceNames; - } - - private static List keyspacesAndTablesToTableIds(@Nonnull List keyspaceNames, @Nonnull Optional> maybeTables) - { - return keyspacesAndTablesToTableIds(keyspaceNames, maybeTables, Optional.empty()); - } - - private static List keyspacesAndTablesToTableIds(@Nonnull List keyspaceNames, @Nonnull Optional> maybeTables, @Nonnull Optional> includeTable) - { - List tableIds = new ArrayList<>(); - for (String keyspaceName : keyspaceNames) - { - Optional> maybeTableIds = maybeTables.map(tableNames -> - tableNames - .stream() - .map(tableName -> { - TableMetadata tm = Schema.instance.getTableMetadata(keyspaceName, tableName); - if (tm == null) - throw new IllegalArgumentException("Unknown table %s.%s".format(keyspaceName, tableName)); - return tm.id; - }) - .collect(toImmutableList())); - tableIds.addAll( - maybeTableIds.orElseGet(() -> - Schema.instance.getKeyspaceInstance(keyspaceName).getColumnFamilyStores() - .stream() - .map(ColumnFamilyStore::getTableId) - .filter(includeTable.orElse(Predicates.alwaysTrue())) // Filter out non-migrating so they don't generate an error - .collect(toImmutableList()))); - } - return tableIds; - } -} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/TableMigrationState.java b/src/java/org/apache/cassandra/service/consensus/migration/TableMigrationState.java new file mode 100644 index 0000000000..a5b0f7db5e --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/TableMigrationState.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.IPartitioner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.serialization.MetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.cassandra.db.TypeSizes.sizeof; +import static org.apache.cassandra.dht.Range.intersectionOfNormalizedRanges; +import static org.apache.cassandra.dht.Range.normalize; +import static org.apache.cassandra.dht.Range.subtract; +import static org.apache.cassandra.dht.Range.subtractNormalizedRanges; +import static org.apache.cassandra.utils.CollectionSerializers.deserializeMap; +import static org.apache.cassandra.utils.CollectionSerializers.deserializeSet; +import static org.apache.cassandra.utils.CollectionSerializers.newHashMap; +import static org.apache.cassandra.utils.CollectionSerializers.serializeCollection; +import static org.apache.cassandra.utils.CollectionSerializers.serializeMap; +import static org.apache.cassandra.utils.CollectionSerializers.serializedCollectionSize; +import static org.apache.cassandra.utils.CollectionSerializers.serializedMapSize; + +// TODO Move this into the schema for the table once this is based off of TrM +public class TableMigrationState +{ + private static final Logger logger = LoggerFactory.getLogger(TableMigrationState.class); + + @Nonnull + public final String keyspaceName; + + @Nonnull + public final String tableName; + + @Nonnull + public final TableId tableId; + + @Nonnull + public final ConsensusMigrationTarget targetProtocol; + + @Nonnull + public final List> migratedRanges; + + /* + * Necessary to track which ranges started migrating at which epoch + * in order to know whether a repair qualifies in terms of finishing + * migration of the range. + */ + @Nonnull + public final NavigableMap>> migratingRangesByEpoch; + + @Nonnull + public final List> migratingRanges; + + @Nonnull + public final List> migratingAndMigratedRanges; + + public TableMigrationState(@Nonnull String keyspaceName, + @Nonnull String tableName, + @Nonnull TableId tableId, + @Nonnull ConsensusMigrationTarget targetProtocol, + @Nonnull Collection> migratedRanges, + @Nonnull Map>> migratingRangesByEpoch) + { + this.keyspaceName = keyspaceName; + this.tableName = tableName; + this.tableId = tableId; + this.targetProtocol = targetProtocol; + this.migratedRanges = ImmutableList.copyOf(normalize(migratedRanges)); + this.migratingRangesByEpoch = ImmutableSortedMap.copyOf( + migratingRangesByEpoch.entrySet() + .stream() + .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), ImmutableList.copyOf(normalize(entry.getValue())))) + .collect(Collectors.toList())); + this.migratingRanges = ImmutableList.copyOf(normalize(migratingRangesByEpoch.values().stream().flatMap(Collection::stream).collect(Collectors.toList()))); + this.migratingAndMigratedRanges = ImmutableList.copyOf(normalize(ImmutableList.>builder().addAll(migratedRanges).addAll(migratingRanges).build())); + } + + public TableMigrationState reverseMigration(ConsensusMigrationTarget target, Epoch epoch) + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + Range fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken()); + List> allTouched = new ArrayList<>(migratedRanges); + allTouched.addAll(migratingRanges); + allTouched = Range.deoverlap(allTouched); + return new TableMigrationState(keyspaceName, tableName, tableId, target, + Range.normalize(fullRange.subtractAll(allTouched)), + Collections.singletonMap(epoch, migratingRanges)); + } + + public boolean hasMigratedFullTokenRange(IPartitioner partitioner) + { + // migrated ranges are normalized + if (!migratingRanges.isEmpty() || migratedRanges.size() > 1) + return false; + + Range fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken()); + return migratedRanges.get(0).contains(fullRange); + } + + @Nonnull + public List> migratingRanges() { + + return migratingRanges; + } + + public TableMigrationState withRangesMigrating(@Nonnull Collection> ranges, + @Nonnull ConsensusMigrationTarget target) + { + checkState(!migratingRangesByEpoch.containsKey(Epoch.EMPTY), "Shouldn't already have an entry for the empty epoch"); + // Doesn't matter which epoch the range started migrating in for this context so merge them all + Collection> migratingRanges = normalize(migratingRangesByEpoch.values().stream().flatMap(Collection::stream).collect(Collectors.toList())); + checkArgument(target == targetProtocol, "Requested migration to target protocol " + target + " conflicts with in progress migration to protocol " + targetProtocol); + List> normalizedRanges = normalize(ranges); + if (subtract(normalizedRanges, migratingRanges).isEmpty()) + logger.warn("Range " + ranges + " is already being migrated"); + Set> withoutAlreadyMigrated = subtract(normalizedRanges, migratedRanges); + if (withoutAlreadyMigrated.isEmpty()) + logger.warn("Range " + ranges + " is already migrated"); + Set> withoutBoth = subtract(withoutAlreadyMigrated, migratingRanges); + if (withoutBoth.isEmpty()) + logger.warn("Range " + ranges + " is already migrating/migrated"); + + if (!Range.equals(normalizedRanges, withoutBoth)) + logger.warn("Ranges " + normalizedRanges + " to start migrating is already partially migrating/migrated " + withoutBoth); + + Map>> newMigratingRanges = new HashMap<>(migratingRangesByEpoch.size() + 1); + newMigratingRanges.putAll(migratingRangesByEpoch); + newMigratingRanges.put(Epoch.EMPTY, normalizedRanges); + + return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, newMigratingRanges); + } + + public TableMigrationState withReplacementForEmptyEpoch(@Nonnull Epoch replacementEpoch) + { + if (!migratingRangesByEpoch.containsKey(Epoch.EMPTY)) + return this; + Map>> newMigratingRangesByEpoch = new HashMap<>(migratingRangesByEpoch.size()); + migratingRangesByEpoch.forEach((epoch, ranges) -> { + if (epoch.equals(Epoch.EMPTY)) + newMigratingRangesByEpoch.put(replacementEpoch, ranges); + else + newMigratingRangesByEpoch.put(epoch, ranges); + }); + + if (newMigratingRangesByEpoch != null) + return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, newMigratingRangesByEpoch); + else + return this; + } + + public TableMigrationState withRangesRepairedAtEpoch(@Nonnull Collection> ranges, + @Nonnull Epoch epoch) + { + checkState(!migratingRangesByEpoch.containsKey(Epoch.EMPTY), "Shouldn't have an entry for the empty epoch"); + checkArgument(epoch.isAfter(Epoch.EMPTY), "Epoch shouldn't be empty"); + + List> normalizedRepairedRanges = normalize(ranges); + // This should be inclusive because the epoch we store in the map is the epoch in which the range has been marked migrating + // in startMigrationToConsensusProtocol + NavigableMap>> coveredEpochs = migratingRangesByEpoch.headMap(epoch, true); + List> normalizedMigratingRanges = normalize(coveredEpochs.values().stream().flatMap(Collection::stream).collect(Collectors.toList())); + List> normalizedRepairedIntersection = intersectionOfNormalizedRanges(normalizedRepairedRanges, normalizedMigratingRanges); + checkState(!normalizedRepairedIntersection.isEmpty(), "None of Ranges " + ranges + " were being migrated"); + + Map>> newMigratingRangesByEpoch = new HashMap<>(); + + // Everything in this epoch or later can't have been migrated so re-add all of them + newMigratingRangesByEpoch.putAll(migratingRangesByEpoch.tailMap(epoch, false)); + + // Include anything still remaining to be migrated after subtracting what was repaired + for (Map.Entry>> e : coveredEpochs.entrySet()) + { + // Epoch when these ranges started migrating + Epoch rangesEpoch = e.getKey(); + List> epochMigratingRanges = e.getValue(); + List> remainingRanges = subtractNormalizedRanges(epochMigratingRanges, normalizedRepairedIntersection); + if (!remainingRanges.isEmpty()) + newMigratingRangesByEpoch.put(rangesEpoch, remainingRanges); + } + + List> newMigratedRanges = new ArrayList<>(normalizedMigratingRanges.size() + ranges.size()); + newMigratedRanges.addAll(migratedRanges); + newMigratedRanges.addAll(normalizedRepairedIntersection); + return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, newMigratedRanges, newMigratingRangesByEpoch); + } + + public boolean paxosReadSatisfiedByKeyMigrationAtEpoch(DecoratedKey key, ConsensusMigratedAt consensusMigratedAt) + { + // This check is being done from a Paxos read attempt which needs to + // check if Accord needs to resolve any in flight accord transactions + // if the migration target is Accord then nothing needs to be done + if (targetProtocol != ConsensusMigrationTarget.paxos) + return true; + + return satisfiedByKeyMigrationAtEpoch(key, consensusMigratedAt); + } + + public boolean satisfiedByKeyMigrationAtEpoch(@Nonnull DecoratedKey key, @Nullable ConsensusMigratedAt consensusMigratedAt) + { + if (consensusMigratedAt == null) + { + // It hasn't been migrated and needs migration if it is in a migrating range + return Range.isInNormalizedRanges(key.getToken(), migratingRanges); + } + else + { + // It has been migrated and might be from a late enough epoch to satisfy this migration + return consensusMigratedAt.migratedAtTarget == targetProtocol + && migratingRangesByEpoch.headMap(consensusMigratedAt.migratedAtEpoch, true).values() + .stream() + .flatMap(List::stream) + .anyMatch(range -> range.contains(key.getToken())); + } + } + + public Epoch minMigrationEpoch(Token token) + { + // TODO should there be an index to make this more efficient? + for (Map.Entry>> e : migratingRangesByEpoch.entrySet()) + { + if (Range.isInNormalizedRanges(token, e.getValue())) + return e.getKey(); + } + return Epoch.EMPTY; + } + + + public @Nonnull TableId getTableId() + { + return tableId; + } + + public TableMigrationState withMigrationTarget(ConsensusMigrationTarget newTargetProtocol) + { + checkState(!migratingRangesByEpoch.containsKey(Epoch.EMPTY), "Shouldn't have an entry for the empty epoch"); + if (targetProtocol == newTargetProtocol) + return this; + + // Migrating ranges remain migrating because individual keys may have already been migrated + // So for correctness we need to perform key migration + // We do need to update the epoch so that a new repair is required to drive the migration + Map>> migratingRangesByEpoch = ImmutableMap.of(Epoch.EMPTY, migratingRanges); + + Token minToken = ColumnFamilyStore.getIfExists(tableId).getPartitioner().getMinimumToken(); + Range fullRange = new Range(minToken, minToken); + // What is migrated already is anything that was never migrated/migrating before (untouched) + List> migratedRanges = ImmutableList.copyOf(normalize(fullRange.subtractAll(migratingAndMigratedRanges))); + + return new TableMigrationState(keyspaceName, tableName, tableId, newTargetProtocol, migratedRanges, migratingRangesByEpoch); + } + + public Map toMap() + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.put("keyspace", keyspaceName); + builder.put("table", tableName); + builder.put("tableId", tableId.toString()); + builder.put("targetProtocol", targetProtocol.toString()); + builder.put("migratedRanges", migratedRanges.stream().map(Objects::toString).collect(toImmutableList())); + Map> rangesByEpoch = new LinkedHashMap<>(); + for (Map.Entry>> entry : migratingRangesByEpoch.entrySet()) + { + rangesByEpoch.put(entry.getKey().getEpoch(), entry.getValue().stream().map(Objects::toString).collect(toImmutableList())); + } + builder.put("migratingRangesByEpoch", rangesByEpoch); + return builder.build(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TableMigrationState that = (TableMigrationState) o; + return keyspaceName.equals(that.keyspaceName) && tableName.equals(that.tableName) && tableId.equals(that.tableId) && targetProtocol == that.targetProtocol && migratedRanges.equals(that.migratedRanges) && migratingRangesByEpoch.equals(that.migratingRangesByEpoch) && migratingRanges.equals(that.migratingRanges) && migratingAndMigratedRanges.equals(that.migratingAndMigratedRanges); + } + + @Override + public int hashCode() + { + return Objects.hash(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, migratingRangesByEpoch, migratingRanges, migratingAndMigratedRanges); + } + + public static final MetadataSerializer serializer = new MetadataSerializer() + { + @Override + public void serialize(TableMigrationState t, DataOutputPlus out, Version version) throws IOException + { + out.write(t.targetProtocol.value); + out.writeUTF(t.keyspaceName); + out.writeUTF(t.tableName); + t.tableId.serialize(out); + serializeCollection(t.migratedRanges, out, version, Range.serializer); + serializeMap(t.migratingRangesByEpoch, out, version, Epoch.serializer, ConsensusTableMigration.rangesSerializer); + } + + @Override + public TableMigrationState deserialize(DataInputPlus in, Version version) throws IOException + { + ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromValue(in.readByte()); + String keyspaceName = in.readUTF(); + String tableName = in.readUTF(); + TableId tableId = TableId.deserialize(in); + Set> migratedRanges = deserializeSet(in, version, Range.serializer); + Map>> migratingRangesByEpoch = deserializeMap(in, version, Epoch.serializer, ConsensusTableMigration.rangesSerializer, newHashMap()); + return new TableMigrationState(keyspaceName, tableName, tableId, targetProtocol, migratedRanges, migratingRangesByEpoch); + } + + @Override + public long serializedSize(TableMigrationState t, Version version) + { + return sizeof(t.targetProtocol.value) + + sizeof(t.keyspaceName) + + sizeof(t.tableName) + + t.tableId.serializedSize() + + serializedCollectionSize(t.migratedRanges, version, Range.serializer) + + serializedMapSize(t.migratingRangesByEpoch, version, Epoch.serializer, ConsensusTableMigration.rangesSerializer); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/consensus/migration/TransactionalMigrationFromMode.java b/src/java/org/apache/cassandra/service/consensus/migration/TransactionalMigrationFromMode.java new file mode 100644 index 0000000000..8cbef514d8 --- /dev/null +++ b/src/java/org/apache/cassandra/service/consensus/migration/TransactionalMigrationFromMode.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.consensus.migration; + +import org.apache.cassandra.service.consensus.TransactionalMode; +import org.apache.cassandra.utils.LocalizeString; + +/** + * This tracks the state of a migration either from Paxos -> Accord, Accord [interop mode a] -> Accord [interop mode b] or Accord -> Paxos. + * The `TransactionalMode` associated with each transition from a system is how interoperability should be achieved during the migration with various performance/safety tradeoffs. + */ +public enum TransactionalMigrationFromMode +{ + none(null), // No migration is in progress. The currently active transaction system could be either Accord or Paxos. + off(TransactionalMode.off), + unsafe(TransactionalMode.unsafe), + unsafe_writes(TransactionalMode.unsafe_writes), + mixed_reads(TransactionalMode.mixed_reads), + full(TransactionalMode.full); + + public final TransactionalMode from; + + TransactionalMigrationFromMode(TransactionalMode from) + { + this.from = from; + } + + public static TransactionalMigrationFromMode fromMode(TransactionalMode prev, TransactionalMode next) + { + if (next.accordIsEnabled == prev.accordIsEnabled) + return none; + + switch (prev) + { + default: throw new IllegalArgumentException(); + case off: return off; + case unsafe: return unsafe; + case unsafe_writes: return unsafe_writes; + case mixed_reads: return mixed_reads; + case full: return full; + } + } + + public static TransactionalMigrationFromMode fromOrdinal(int ordinal) + { + return values()[ordinal]; + } + + public static TransactionalMigrationFromMode fromString(String name) + { + return valueOf(LocalizeString.toLowerCaseLocalized(name)); + } + + public boolean migratingFromAccord() + { + return from != null && from.accordIsEnabled; + } + + public boolean writesThroughAccord() + { + return from != null && from.writesThroughAccord; + } + + public boolean isMigrating() + { + return this != none; + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java b/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java index 1cd7da413c..02efebbeb2 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java @@ -72,7 +72,8 @@ import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAdd import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REQ; import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_RSP; import static org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.getKeyMigrationState; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigratedAt; + +import org.apache.cassandra.service.consensus.migration.ConsensusMigratedAt; import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; import static org.apache.cassandra.service.paxos.Commit.Accepted; import static org.apache.cassandra.service.paxos.Commit.Committed; diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java index e7d43b19ca..4f02b1a060 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java @@ -34,8 +34,7 @@ import accord.primitives.Keys; import accord.primitives.Txn; import com.codahale.metrics.Meter; import org.apache.cassandra.concurrent.Stage; -import org.apache.cassandra.config.Config.LWTStrategy; -import org.apache.cassandra.config.Config.NonSerialWriteStrategy; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; @@ -196,9 +195,8 @@ public class BlockingReadRepair, P extends ReplicaPlan.Fo @Override public void repairPartition(DecoratedKey dk, Map mutations, ReplicaPlan.ForWrite writePlan) { - NonSerialWriteStrategy nonSerialWriteStrategy = DatabaseDescriptor.getNonSerialWriteStrategy(); - if (coordinator.isEventuallyConsistent() && (DatabaseDescriptor.getLWTStrategy() == LWTStrategy.accord - || nonSerialWriteStrategy.blockingReadRepairThroughAccord)) + TransactionalMode transactionalMode = command.metadata().params.transactionalMode; + if (coordinator.isEventuallyConsistent() && transactionalMode.blockingReadRepairThroughAccord) { Collection partitionUpdates = Mutation.merge(mutations.values()).getPartitionUpdates(); checkState(partitionUpdates.size() == 1, "Expect only one PartitionUpdate"); diff --git a/src/java/org/apache/cassandra/streaming/StreamPlan.java b/src/java/org/apache/cassandra/streaming/StreamPlan.java index 88f77e8867..93b864d79b 100644 --- a/src/java/org/apache/cassandra/streaming/StreamPlan.java +++ b/src/java/org/apache/cassandra/streaming/StreamPlan.java @@ -25,7 +25,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.KeyspaceMetadata; -import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.TimeUUID; import static com.google.common.collect.Iterables.all; @@ -238,7 +238,7 @@ public class StreamPlan public static String[] nonAccordTablesForKeyspace(KeyspaceMetadata ksm) { String[] result = ksm.tables.stream() - .filter(tbl -> !AccordService.instance().isAccordManagedTable(tbl.id)) + .filter(tbl -> !tbl.isAccordEnabled()) .map(tbl -> tbl.name) .toArray(String[]::new); @@ -247,11 +247,11 @@ public class StreamPlan public static boolean hasNonAccordTables(KeyspaceMetadata ksm) { - return ksm.tables.stream().anyMatch(tbl -> !AccordService.instance().isAccordManagedTable(tbl.id)); + return ksm.tables.stream().anyMatch(tbl -> !tbl.isAccordEnabled()); } public static boolean hasAccordTables(KeyspaceMetadata ksm) { - return ksm.tables.stream().anyMatch(tbl -> AccordService.instance().isAccordManagedTable(tbl.id)); + return ksm.tables.stream().anyMatch(TableMetadata::isAccordEnabled); } } diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java index 2c2e3b7d30..95c3e1ba2c 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java @@ -57,13 +57,12 @@ import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.service.consensus.migration.TableMigrationState; import org.apache.cassandra.tcm.extensions.ExtensionKey; import org.apache.cassandra.tcm.extensions.ExtensionValue; import org.apache.cassandra.service.accord.AccordFastPath; import org.apache.cassandra.tcm.membership.*; -import org.apache.cassandra.tcm.ownership.AccordTables; import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.PrimaryRangeComparator; @@ -96,7 +95,6 @@ public class ClusterMetadata public final Directory directory; public final TokenMap tokenMap; public final DataPlacements placements; - public final AccordTables accordTables; public final AccordFastPath accordFastPath; public final LockedRanges lockedRanges; public final InProgressSequences inProgressSequences; @@ -133,7 +131,6 @@ public class ClusterMetadata directory, new TokenMap(partitioner), DataPlacements.EMPTY, - AccordTables.EMPTY, AccordFastPath.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, @@ -147,7 +144,6 @@ public class ClusterMetadata Directory directory, TokenMap tokenMap, DataPlacements placements, - AccordTables accordTables, AccordFastPath accordFastPath, LockedRanges lockedRanges, InProgressSequences inProgressSequences, @@ -161,7 +157,6 @@ public class ClusterMetadata directory, tokenMap, placements, - accordTables, accordFastPath, lockedRanges, inProgressSequences, @@ -176,7 +171,6 @@ public class ClusterMetadata Directory directory, TokenMap tokenMap, DataPlacements placements, - AccordTables accordTables, AccordFastPath accordFastPath, LockedRanges lockedRanges, InProgressSequences inProgressSequences, @@ -194,7 +188,6 @@ public class ClusterMetadata this.directory = directory; this.tokenMap = tokenMap; this.placements = placements; - this.accordTables = accordTables; this.accordFastPath = accordFastPath; this.lockedRanges = lockedRanges; this.inProgressSequences = inProgressSequences; @@ -205,12 +198,12 @@ public class ClusterMetadata public ClusterMetadata withDirectory(Directory directory) { - return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordTables, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions); + return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions); } public ClusterMetadata withPlacements(DataPlacements placements) { - return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordTables, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions); + return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions); } public Set fullCMSMembers() @@ -262,7 +255,6 @@ public class ClusterMetadata capLastModified(directory, epoch), capLastModified(tokenMap, epoch), capLastModified(placements, epoch), - capLastModified(accordTables, epoch), capLastModified(accordFastPath, epoch), capLastModified(lockedRanges, epoch), capLastModified(inProgressSequences, epoch), @@ -285,7 +277,6 @@ public class ClusterMetadata directory, tokenMap, placements, - accordTables, accordFastPath, lockedRanges, inProgressSequences, @@ -413,7 +404,6 @@ public class ClusterMetadata private Directory directory; private TokenMap tokenMap; private DataPlacements placements; - private AccordTables accordTables; private AccordFastPath accordFastPath; private LockedRanges lockedRanges; private InProgressSequences inProgressSequences; @@ -430,7 +420,6 @@ public class ClusterMetadata this.directory = metadata.directory; this.tokenMap = metadata.tokenMap; this.placements = metadata.placements; - this.accordTables = metadata.accordTables; this.accordFastPath = metadata.accordFastPath; this.lockedRanges = metadata.lockedRanges; this.inProgressSequences = metadata.inProgressSequences; @@ -439,6 +428,11 @@ public class ClusterMetadata modifiedKeys = new HashSet<>(); } + public Epoch epoch() + { + return epoch; + } + public Transformer with(DistributedSchema schema) { this.schema = schema; @@ -552,12 +546,6 @@ public class ClusterMetadata return this; } - public Transformer withAccordTable(TableId table) - { - accordTables = accordTables.with(table); - return this; - } - public Transformer withFastPathStatusSince(Node.Id node, AccordFastPath.Status status, long updateTimeMillis, long updateDelayMillis) { accordFastPath = accordFastPath.withNodeStatusSince(node, status, updateTimeMillis, updateDelayMillis); @@ -601,6 +589,12 @@ public class ClusterMetadata return this; } + public Transformer with(ConsensusMigrationState consensusMigrationState) + { + this.consensusMigrationState = consensusMigrationState; + return this; + } + public Transformer with(ExtensionKey key, ExtensionValue obj) { if (MetadataKeys.CORE_METADATA.contains(key)) @@ -678,12 +672,6 @@ public class ClusterMetadata placements = placements.withLastModified(epoch); } - if (accordTables != base.accordTables) - { - modifiedKeys.add(MetadataKeys.ACCORD_TABLES); - accordTables = accordTables.withLastModified(epoch); - } - if (accordFastPath != base.accordFastPath) { modifiedKeys.add(MetadataKeys.ACCORD_FAST_PATH); @@ -708,6 +696,11 @@ public class ClusterMetadata consensusMigrationState = consensusMigrationState.withLastModified(epoch); } + if (consensusMigrationState != base.consensusMigrationState || schema != base.schema) + { + consensusMigrationState.validateAgainstSchema(schema); + } + return new Transformed(new ClusterMetadata(base.metadataIdentifier, epoch, partitioner, @@ -715,7 +708,6 @@ public class ClusterMetadata directory, tokenMap, placements, - accordTables, accordFastPath, lockedRanges, inProgressSequences, @@ -733,7 +725,6 @@ public class ClusterMetadata directory, tokenMap, placements, - accordTables, accordFastPath, lockedRanges, inProgressSequences, @@ -866,7 +857,7 @@ public class ClusterMetadata directory.equals(that.directory) && tokenMap.equals(that.tokenMap) && placements.equals(that.placements) && - accordTables.equals(that.accordTables) && + accordFastPath.equals(that.accordFastPath) && lockedRanges.equals(that.lockedRanges) && inProgressSequences.equals(that.inProgressSequences) && consensusMigrationState.equals(that.consensusMigrationState) && @@ -918,7 +909,7 @@ public class ClusterMetadata @Override public int hashCode() { - return Objects.hash(epoch, schema, directory, tokenMap, placements, accordTables, lockedRanges, inProgressSequences, consensusMigrationState, extensions); + return Objects.hash(epoch, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions); } public static ClusterMetadata current() @@ -997,12 +988,11 @@ public class ClusterMetadata DataPlacements.serializer.serialize(metadata.placements, out, version); if (version.isAtLeast(V2)) { - AccordTables.serializer.serialize(metadata.accordTables, out, version); AccordFastPath.serializer.serialize(metadata.accordFastPath, out, version); + ConsensusMigrationState.serializer.serialize(metadata.consensusMigrationState, out, version); } LockedRanges.serializer.serialize(metadata.lockedRanges, out, version); InProgressSequences.serializer.serialize(metadata.inProgressSequences, out, version); - ConsensusMigrationState.serializer.serialize(metadata.consensusMigrationState, out, version); out.writeInt(metadata.extensions.size()); for (Map.Entry, ExtensionValue> entry : metadata.extensions.entrySet()) { @@ -1037,21 +1027,20 @@ public class ClusterMetadata Directory dir = Directory.serializer.deserialize(in, version); TokenMap tokenMap = TokenMap.serializer.deserialize(in, version); DataPlacements placements = DataPlacements.serializer.deserialize(in, version); - AccordTables accordTables; AccordFastPath accordFastPath; + ConsensusMigrationState consensusMigrationState; if (version.isAtLeast(V2)) { - accordTables = AccordTables.serializer.deserialize(in, version); accordFastPath = AccordFastPath.serializer.deserialize(in, version); + consensusMigrationState = ConsensusMigrationState.serializer.deserialize(in, version); } else { - accordTables = AccordTables.EMPTY; accordFastPath = AccordFastPath.EMPTY; + consensusMigrationState = ConsensusMigrationState.EMPTY; } LockedRanges lockedRanges = LockedRanges.serializer.deserialize(in, version); InProgressSequences ips = InProgressSequences.serializer.deserialize(in, version); - ConsensusMigrationState consensusMigrationState = ConsensusMigrationState.serializer.deserialize(in, version); int items = in.readInt(); Map, ExtensionValue> extensions = new HashMap<>(items); for (int i = 0; i < items; i++) @@ -1068,7 +1057,6 @@ public class ClusterMetadata dir, tokenMap, placements, - accordTables, accordFastPath, lockedRanges, ips, @@ -1092,14 +1080,12 @@ public class ClusterMetadata DistributedSchema.serializer.serializedSize(metadata.schema, version) + Directory.serializer.serializedSize(metadata.directory, version) + TokenMap.serializer.serializedSize(metadata.tokenMap, version) + - DataPlacements.serializer.serializedSize(metadata.placements, version) + - ConsensusMigrationState.serializer.serializedSize(metadata.consensusMigrationState, version); DataPlacements.serializer.serializedSize(metadata.placements, version); if (version.isAtLeast(V2)) { - size += AccordTables.serializer.serializedSize(metadata.accordTables, version) + - AccordFastPath.serializer.serializedSize(metadata.accordFastPath, version); + size += AccordFastPath.serializer.serializedSize(metadata.accordFastPath, version) + + ConsensusMigrationState.serializer.serializedSize(metadata.consensusMigrationState, version); } size += LockedRanges.serializer.serializedSize(metadata.lockedRanges, version) + diff --git a/src/java/org/apache/cassandra/tcm/MetadataKeys.java b/src/java/org/apache/cassandra/tcm/MetadataKeys.java index 1794a63889..68306ce313 100644 --- a/src/java/org/apache/cassandra/tcm/MetadataKeys.java +++ b/src/java/org/apache/cassandra/tcm/MetadataKeys.java @@ -39,7 +39,6 @@ public class MetadataKeys public static final MetadataKey NODE_DIRECTORY = make(CORE_NS, "membership", "node_directory"); public static final MetadataKey TOKEN_MAP = make(CORE_NS, "ownership", "token_map"); public static final MetadataKey DATA_PLACEMENTS = make(CORE_NS, "ownership", "data_placements"); - public static final MetadataKey ACCORD_TABLES = make(CORE_NS, "ownership", "accord_tables"); public static final MetadataKey ACCORD_FAST_PATH = make(CORE_NS, "ownership", "accord_fast_path"); public static final MetadataKey LOCKED_RANGES = make(CORE_NS, "sequences", "locked_ranges"); public static final MetadataKey IN_PROGRESS_SEQUENCES = make(CORE_NS, "sequences", "in_progress"); @@ -49,7 +48,6 @@ public class MetadataKeys NODE_DIRECTORY, TOKEN_MAP, DATA_PLACEMENTS, - ACCORD_TABLES, ACCORD_FAST_PATH, LOCKED_RANGES, IN_PROGRESS_SEQUENCES, diff --git a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java index 150b3934bb..d3318ecb74 100644 --- a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java @@ -29,12 +29,11 @@ import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.service.accord.AccordFastPath; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.tcm.Commit.Replicator; import org.apache.cassandra.tcm.log.Entry; import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.membership.Directory; -import org.apache.cassandra.tcm.ownership.AccordTables; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.PlacementProvider; import org.apache.cassandra.tcm.ownership.TokenMap; @@ -175,11 +174,10 @@ public class StubClusterMetadataService extends ClusterMetadataService Directory.EMPTY, new TokenMap(partitioner), DataPlacements.EMPTY, - AccordTables.EMPTY, AccordFastPath.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, - ConsensusTableMigrationState.ConsensusMigrationState.EMPTY, + ConsensusMigrationState.EMPTY, ImmutableMap.of()); } return new StubClusterMetadataService(new UniformRangePlacement(), diff --git a/src/java/org/apache/cassandra/tcm/Transformation.java b/src/java/org/apache/cassandra/tcm/Transformation.java index f6f0aa226d..1ccdb683e9 100644 --- a/src/java/org/apache/cassandra/tcm/Transformation.java +++ b/src/java/org/apache/cassandra/tcm/Transformation.java @@ -38,7 +38,6 @@ import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; -import org.apache.cassandra.tcm.transformations.AddAccordTable; import org.apache.cassandra.tcm.transformations.AlterSchema; import org.apache.cassandra.tcm.transformations.AlterTopology; import org.apache.cassandra.tcm.transformations.Assassinate; @@ -53,7 +52,6 @@ import org.apache.cassandra.tcm.transformations.PrepareMove; import org.apache.cassandra.tcm.transformations.PrepareReplace; import org.apache.cassandra.tcm.transformations.ReconfigureAccordFastPath; import org.apache.cassandra.tcm.transformations.Register; -import org.apache.cassandra.tcm.transformations.SetConsensusMigrationTargetProtocol; import org.apache.cassandra.tcm.transformations.Startup; import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import org.apache.cassandra.tcm.transformations.Unregister; @@ -240,11 +238,10 @@ public interface Transformation CANCEL_CMS_RECONFIGURATION(34, () -> CancelCMSReconfiguration.serializer), ALTER_TOPOLOGY(35, () -> AlterTopology.serializer), - ADD_ACCORD_TABLE(36, () -> AddAccordTable.serializer), - UPDATE_AVAILABILITY(37, () -> ReconfigureAccordFastPath.serializer), - BEGIN_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(38, () -> BeginConsensusMigrationForTableAndRange.serializer), - MAYBE_FINISH_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(39, () -> MaybeFinishConsensusMigrationForTableAndRange.serializer), - SET_CONSENSUS_MIGRATION_TARGET_PROTOCOL(40, () -> SetConsensusMigrationTargetProtocol.serializer) + UPDATE_AVAILABILITY(36, () -> ReconfigureAccordFastPath.serializer), + + BEGIN_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(37, () -> BeginConsensusMigrationForTableAndRange.serializer), + MAYBE_FINISH_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(38, () -> MaybeFinishConsensusMigrationForTableAndRange.serializer), ; private final Supplier> serializer; diff --git a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java index 390ead6f4d..e82572f53b 100644 --- a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java +++ b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java @@ -54,7 +54,7 @@ import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaKeyspace; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MultiStepOperation; @@ -67,7 +67,6 @@ import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.membership.NodeVersion; -import org.apache.cassandra.tcm.ownership.AccordTables; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.TokenMap; import org.apache.cassandra.tcm.ownership.UniformRangePlacement; @@ -297,7 +296,6 @@ public class GossipHelper Directory.EMPTY, new TokenMap(DatabaseDescriptor.getPartitioner()), DataPlacements.empty(), - AccordTables.EMPTY, AccordFastPath.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, @@ -387,7 +385,6 @@ public class GossipHelper directory, tokenMap, DataPlacements.empty(), - AccordTables.EMPTY, AccordFastPath.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, @@ -402,7 +399,6 @@ public class GossipHelper directory, tokenMap, placements, - AccordTables.EMPTY, AccordFastPath.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, diff --git a/src/java/org/apache/cassandra/tcm/ownership/AccordTables.java b/src/java/org/apache/cassandra/tcm/ownership/AccordTables.java deleted file mode 100644 index 22c6b5e8d0..0000000000 --- a/src/java/org/apache/cassandra/tcm/ownership/AccordTables.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.tcm.ownership; - -import java.io.IOException; -import java.util.Arrays; - -import com.google.common.collect.ImmutableSet; - -import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.tcm.MetadataValue; -import org.apache.cassandra.tcm.serialization.MetadataSerializer; -import org.apache.cassandra.tcm.serialization.Version; - -public class AccordTables implements MetadataValue -{ - public static final AccordTables EMPTY = new AccordTables(Epoch.EMPTY, ImmutableSet.of()); - private final Epoch lastModified; - private final ImmutableSet tables; - - public AccordTables(Epoch lastModified, ImmutableSet tables) - { - this.lastModified = lastModified; - this.tables = tables; - } - - public String toString() - { - return "AccordTables{" + lastModified + ", " + tables + '}'; - } - - public AccordTables withLastModified(Epoch epoch) - { - return new AccordTables(epoch, tables); - } - - public Epoch lastModified() - { - return lastModified; - } - - public boolean contains(TableId table) - { - return tables.contains(table); - } - - public AccordTables with(TableId table) - { - if (tables.contains(table)) - return this; - - return new AccordTables(lastModified, ImmutableSet.builder().addAll(tables).add(table).build()); - } - - public static final MetadataSerializer serializer = new MetadataSerializer() - { - public void serialize(AccordTables accordTables, DataOutputPlus out, Version version) throws IOException - { - int size = accordTables.tables.size(); - out.writeUnsignedVInt32(size); - TableId[] tables = new TableId[size]; - accordTables.tables.toArray(tables); - Arrays.sort(tables); - for (TableId table : tables) - table.serialize(out); - Epoch.serializer.serialize(accordTables.lastModified, out, version); - } - - public AccordTables deserialize(DataInputPlus in, Version version) throws IOException - { - int size = in.readUnsignedVInt32(); - ImmutableSet.Builder builder = ImmutableSet.builder(); - for (int i=0; i null, - (code, message) -> { - Invariants.checkState(code == ExceptionCode.ALREADY_EXISTS, - "Expected %s, got %s", ExceptionCode.ALREADY_EXISTS, code); - return null; - }); - } - - public static final AsymmetricMetadataSerializer serializer = new AsymmetricMetadataSerializer() - { - public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException - { - assert t instanceof AddAccordTable; - AddAccordTable addTable = (AddAccordTable) t; - addTable.table.serialize(out); - } - - public AddAccordTable deserialize(DataInputPlus in, Version version) throws IOException - { - return new AddAccordTable(TableId.deserialize(in)); - } - - public long serializedSize(Transformation t, Version version) - { - assert t instanceof AddAccordTable; - AddAccordTable addTable = (AddAccordTable) t; - return addTable.table.serializedSize(); - } - }; -} diff --git a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java index 5cdd1b3016..3fe49e63bb 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java @@ -19,20 +19,26 @@ package org.apache.cassandra.tcm.transformations; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Streams; +import org.apache.cassandra.config.AccordSpec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.cql3.statements.schema.AlterSchemaStatement; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.AlreadyExistsException; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -50,8 +56,7 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Tables; import org.apache.cassandra.schema.ViewMetadata; import org.apache.cassandra.schema.Views; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadata.Transformer; import org.apache.cassandra.tcm.ClusterMetadataService; @@ -72,7 +77,6 @@ import static org.apache.cassandra.exceptions.ExceptionCode.CONFIG_ERROR; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; import static org.apache.cassandra.exceptions.ExceptionCode.SERVER_ERROR; import static org.apache.cassandra.exceptions.ExceptionCode.SYNTAX_ERROR; -import static org.apache.cassandra.utils.Collectors3.toImmutableMap; public class AlterSchema implements Transformation { @@ -241,7 +245,7 @@ public class AlterSchema implements Transformation }); next = next.with(newPlacementsBuilder.build()); } - next = maybeUpdateConsensusTableMigrationStateForDroppedTables(prev.consensusMigrationState, next, diff.altered, diff.dropped); + next = maybeUpdateConsensusMigrationState(prev.consensusMigrationState, next, diff.altered, diff.dropped); return Transformation.success(next, LockedRanges.AffectedRanges.EMPTY); } @@ -257,22 +261,69 @@ public class AlterSchema implements Transformation return byReplication; } - private Transformer maybeUpdateConsensusTableMigrationStateForDroppedTables(ConsensusMigrationState prev, Transformer next, ImmutableList altered, Keyspaces dropped) + private Transformer maybeUpdateConsensusMigrationState(ConsensusMigrationState prev, Transformer next, ImmutableList altered, Keyspaces dropped) { - Set tableIds = Streams.concat( - altered.stream().flatMap(diff -> diff.tables.dropped.stream().map(TableMetadata::id)), - dropped.stream().flatMap(ks -> ks.tables.stream().map(TableMetadata::id))) + ConsensusMigrationState migrationState = prev; + + Set droppedIds = Streams.concat(altered.stream().flatMap(diff -> diff.tables.dropped.stream().map(TableMetadata::id)), + dropped.stream().flatMap(ks -> ks.tables.stream().map(TableMetadata::id))) + .collect(toImmutableSet()); + + if (!droppedIds.isEmpty()) + migrationState = migrationState.withMigrationsRemovedFor(droppedIds); + + Set completedIds = altered.stream() + .flatMap(diff -> diff.tables.altered.stream()) + .filter(alt -> alt.before.params.transactionalMigrationFrom.isMigrating() + && !alt.after.params.transactionalMigrationFrom.isMigrating()) + .map(alt -> alt.after.id) .collect(toImmutableSet()); - if (tableIds.stream().anyMatch(prev.tableStates.keySet()::contains)) + + if (!completedIds.isEmpty()) + migrationState = migrationState.withMigrationsCompletedFor(completedIds); + + Map reversals = altered.stream() + .flatMap(diff -> diff.tables.altered.stream()) + .filter(alt -> alt.before.params.transactionalMigrationFrom.from == alt.after.params.transactionalMode) + .map(alt -> alt.after) + .collect(Collectors.toMap(TableMetadata::id, Function.identity())); + + + // we treat explicitly switched migration types as a new migration here + Set started = altered.stream() + .flatMap(diff -> diff.tables.altered.stream()) + .filter(alt -> !reversals.containsKey(alt.after.id)) + .filter(alt -> alt.after.params.transactionalMigrationFrom.isMigrating() + && !alt.before.params.transactionalMigrationFrom.isMigrating()) + .map(alt -> alt.after) + .collect(Collectors.toUnmodifiableSet()); + + if (!started.isEmpty()) { - ImmutableMap newTableStates = - prev.tableStates.entrySet().stream().filter(e -> !tableIds.contains(e.getKey())).collect(toImmutableMap()); - return next.with(newTableStates); - } - else - { - return next; + List> ranges; + AccordSpec.TransactionalRangeMigration migration = DatabaseDescriptor.getTransactionalRangeMigration(); + switch (migration) + { + default: throw new IllegalStateException("Unhandled transactional range migration: " + migration); + case auto: + Token minToken = DatabaseDescriptor.getPartitioner().getMinimumToken(); + ranges = Range.normalize(Collections.singletonList(new Range<>(minToken, minToken))); + break; + case explicit: + ranges = Collections.emptyList(); + break; + } + + if (!ranges.isEmpty()) + migrationState = migrationState.withRangesMigrating(started, ranges, true); } + + migrationState = migrationState.withReversedMigrations(reversals, next.epoch()); + + if (migrationState != prev) + next = next.with(migrationState); + + return next; } private static Iterable normaliseTableEpochs(Epoch nextEpoch, Stream tables) diff --git a/src/java/org/apache/cassandra/tcm/transformations/BeginConsensusMigrationForTableAndRange.java b/src/java/org/apache/cassandra/tcm/transformations/BeginConsensusMigrationForTableAndRange.java index a00db104af..e74a18c81f 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/BeginConsensusMigrationForTableAndRange.java +++ b/src/java/org/apache/cassandra/tcm/transformations/BeginConsensusMigrationForTableAndRange.java @@ -19,15 +19,11 @@ package org.apache.cassandra.tcm.transformations; import java.io.IOException; +import java.util.Collection; import java.util.List; -import java.util.Map; -import java.util.function.Function; +import java.util.stream.Collectors; import javax.annotation.Nonnull; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; - -import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -35,9 +31,11 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigration; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; @@ -45,14 +43,10 @@ import org.apache.cassandra.tcm.serialization.Version; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; import static org.apache.cassandra.tcm.ClusterMetadata.Transformer; import static org.apache.cassandra.utils.CollectionSerializers.deserializeList; import static org.apache.cassandra.utils.CollectionSerializers.serializeCollection; import static org.apache.cassandra.utils.CollectionSerializers.serializedCollectionSize; -import static org.apache.cassandra.utils.Collectors3.toImmutableMap; public class BeginConsensusMigrationForTableAndRange implements Transformation { @@ -88,20 +82,10 @@ public class BeginConsensusMigrationForTableAndRange implements Transformation public Result execute(ClusterMetadata prev) { - Map tableStates = prev.consensusMigrationState.tableStates; - List columnFamilyStores = tables.stream().map(Schema.instance::getColumnFamilyStoreInstance).collect(toImmutableList()); - Transformer transformer = prev.transformer(); - - Map newStates = columnFamilyStores - .stream() - .map(cfs -> - tableStates.containsKey(cfs.getTableId()) ? - tableStates.get(cfs.getTableId()).withRangesMigrating(ranges, targetProtocol) : - new TableMigrationState(cfs.keyspace.getName(), cfs.name, cfs.getTableId(), targetProtocol, ImmutableSet.of(), ImmutableMap.of(Epoch.EMPTY, ranges))) - .collect(toImmutableMap(TableMigrationState::getTableId, Function.identity())); - - return Transformation.success(transformer.with(newStates), LockedRanges.AffectedRanges.EMPTY); + Collection metadata = tables.stream().map(Schema.instance::getTableMetadata).collect(Collectors.toList()); + ConsensusMigrationState consensusMigrationState = prev.consensusMigrationState.withRangesMigrating(metadata, ranges, false); + return Transformation.success(transformer.with(consensusMigrationState), LockedRanges.AffectedRanges.EMPTY); } static class Serializer implements AsymmetricMetadataSerializer @@ -111,14 +95,14 @@ public class BeginConsensusMigrationForTableAndRange implements Transformation { BeginConsensusMigrationForTableAndRange v = (BeginConsensusMigrationForTableAndRange)t; out.writeUTF(v.targetProtocol.toString()); - ConsensusTableMigrationState.rangesSerializer.serialize(v.ranges, out, version); + ConsensusTableMigration.rangesSerializer.serialize(v.ranges, out, version); serializeCollection(v.tables, out, version, TableId.metadataSerializer); } public BeginConsensusMigrationForTableAndRange deserialize(DataInputPlus in, Version version) throws IOException { ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(in.readUTF()); - List> ranges = ConsensusTableMigrationState.rangesSerializer.deserialize(in, version); + List> ranges = ConsensusTableMigration.rangesSerializer.deserialize(in, version); List tables = deserializeList(in, version, TableId.metadataSerializer); return new BeginConsensusMigrationForTableAndRange(targetProtocol, ranges, tables); } @@ -127,8 +111,8 @@ public class BeginConsensusMigrationForTableAndRange implements Transformation { BeginConsensusMigrationForTableAndRange v = (BeginConsensusMigrationForTableAndRange) t; return TypeSizes.sizeof(v.targetProtocol.toString()) - + ConsensusTableMigrationState.rangesSerializer.serializedSize(v.ranges, version) - + serializedCollectionSize(v.tables, version, TableId.metadataSerializer); + + ConsensusTableMigration.rangesSerializer.serializedSize(v.ranges, version) + + serializedCollectionSize(v.tables, version, TableId.metadataSerializer); } } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/transformations/MaybeFinishConsensusMigrationForTableAndRange.java b/src/java/org/apache/cassandra/tcm/transformations/MaybeFinishConsensusMigrationForTableAndRange.java index 64e6248c81..7dff0111ef 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/MaybeFinishConsensusMigrationForTableAndRange.java +++ b/src/java/org/apache/cassandra/tcm/transformations/MaybeFinishConsensusMigrationForTableAndRange.java @@ -22,19 +22,29 @@ import java.io.IOException; import java.util.List; import javax.annotation.Nonnull; -import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.DistributedSchema; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationRepairType; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; +import org.apache.cassandra.schema.TableParams; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationRepairType; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget; +import org.apache.cassandra.service.consensus.migration.ConsensusTableMigration; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.service.consensus.migration.TableMigrationState; +import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadata.Transformer; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.sequences.LockedRanges; @@ -47,12 +57,12 @@ import static java.lang.String.format; import static org.apache.cassandra.dht.Range.intersects; import static org.apache.cassandra.dht.Range.normalize; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; public class MaybeFinishConsensusMigrationForTableAndRange implements Transformation { + private static final Logger logger = LoggerFactory.getLogger(MaybeFinishConsensusMigrationForTableAndRange.class); + public static Serializer serializer = new Serializer(); @Nonnull @@ -96,9 +106,27 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma return Kind.MAYBE_FINISH_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE; } + private static Transformer resetMigrationOnSchema(ClusterMetadata prev, Transformer transformer, String ksName, String tblName, TableId id) + { + Keyspaces schema = prev.schema.getKeyspaces(); + KeyspaceMetadata keyspace = schema.getNullable(ksName); + + TableMetadata table = null == keyspace + ? null + : keyspace.getTableOrViewNullable(tblName); + + if (table == null || !table.id.equals(id)) + return transformer; + + TableParams params = table.params.unbuild().transactionalMigrationFrom(TransactionalMigrationFromMode.none).build(); + keyspace = keyspace.withSwapped(keyspace.tables.withSwapped(table.withSwapped(params))); + schema = schema.withAddedOrUpdated(keyspace); + return transformer.with(new DistributedSchema(schema)); + } + public Result execute(@Nonnull ClusterMetadata metadata) { - System.out.println("Completed repair " + repairType + " ranges " + repairedRanges); + logger.info("Completed repair {} ranges {}", repairType, repairedRanges); checkNotNull(metadata, "clusterMetadata should not be null"); String ksAndCF = keyspace + "." + cf; TableMetadata tbm = Schema.instance.getTableMetadata(keyspace, cf); @@ -106,7 +134,7 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma return new Rejected(INVALID, format("Table %s is not currently performing consensus migration", ksAndCF)); ConsensusMigrationState consensusMigrationState = metadata.consensusMigrationState; - ConsensusTableMigrationState.TableMigrationState tms = consensusMigrationState.tableStates.get(tbm.id); + TableMigrationState tms = consensusMigrationState.tableStates.get(tbm.id); if (tms == null) return new Rejected(INVALID, format("Table %s is not currently performing consensus migration", ksAndCF)); @@ -122,9 +150,16 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma if (!intersects(tms.migratingRanges, normalizedRepairedRanges)) return new Rejected(INVALID, format("Table %s is migrating ranges %s, which doesn't include repaired ranges %s", ksAndCF, tms.migratingRanges, normalizedRepairedRanges)); - TableMigrationState newTableMigrationState = tms.withRangesRepairedAtEpoch(normalizedRepairedRanges, minEpoch); + Transformer next = metadata.transformer(); + ConsensusMigrationState migrationState = metadata.consensusMigrationState.withRangesRepairedAtEpoch(tbm, normalizedRepairedRanges, minEpoch); + next = next.with(migrationState); - return Transformation.success(metadata.transformer().with(ImmutableMap.of(newTableMigrationState.tableId, newTableMigrationState)), LockedRanges.AffectedRanges.EMPTY); + // reset the migration value on the table if the migration has completed + TableMigrationState tableState = migrationState.tableStates.get(tbm.id); + if (tableState == null || tableState.hasMigratedFullTokenRange(metadata.partitioner)) + next = resetMigrationOnSchema(metadata, next, keyspace, cf, tbm.id); + + return Transformation.success(next, LockedRanges.AffectedRanges.EMPTY); } static class Serializer implements AsymmetricMetadataSerializer @@ -134,7 +169,7 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma MaybeFinishConsensusMigrationForTableAndRange v = (MaybeFinishConsensusMigrationForTableAndRange)t; out.writeUTF(v.keyspace); out.writeUTF(v.cf); - ConsensusTableMigrationState.rangesSerializer.serialize(v.repairedRanges, out, version); + ConsensusTableMigration.rangesSerializer.serialize(v.repairedRanges, out, version); Epoch.serializer.serialize(v.minEpoch, out, version); out.write(v.repairType.value); } @@ -143,7 +178,7 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma { String keyspace = in.readUTF(); String cf = in.readUTF(); - List> repairedRanges = ConsensusTableMigrationState.rangesSerializer.deserialize(in, version); + List> repairedRanges = ConsensusTableMigration.rangesSerializer.deserialize(in, version); Epoch minEpoch = Epoch.serializer.deserialize(in, version); ConsensusMigrationRepairType repairType = ConsensusMigrationRepairType.fromValue(in.readByte()); return new MaybeFinishConsensusMigrationForTableAndRange(keyspace, cf, repairedRanges, minEpoch, repairType); @@ -153,10 +188,10 @@ public class MaybeFinishConsensusMigrationForTableAndRange implements Transforma { MaybeFinishConsensusMigrationForTableAndRange v = (MaybeFinishConsensusMigrationForTableAndRange)t; return TypeSizes.sizeof(v.keyspace) - + TypeSizes.sizeof(v.cf) - + ConsensusTableMigrationState.rangesSerializer.serializedSize(v.repairedRanges, version) - + Epoch.serializer.serializedSize(v.minEpoch) - + TypeSizes.sizeof(v.repairType.value); + + TypeSizes.sizeof(v.cf) + + ConsensusTableMigration.rangesSerializer.serializedSize(v.repairedRanges, version) + + Epoch.serializer.serializedSize(v.minEpoch) + + TypeSizes.sizeof(v.repairType.value); } } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/transformations/SetConsensusMigrationTargetProtocol.java b/src/java/org/apache/cassandra/tcm/transformations/SetConsensusMigrationTargetProtocol.java deleted file mode 100644 index c0fd662d67..0000000000 --- a/src/java/org/apache/cassandra/tcm/transformations/SetConsensusMigrationTargetProtocol.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.tcm.transformations; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import javax.annotation.Nonnull; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; - -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.ClusterMetadata.Transformer; -import org.apache.cassandra.tcm.Transformation; -import org.apache.cassandra.tcm.sequences.LockedRanges; -import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; -import org.apache.cassandra.tcm.serialization.Version; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget.reset; -import static org.apache.cassandra.tcm.Transformation.Kind.SET_CONSENSUS_MIGRATION_TARGET_PROTOCOL; -import static org.apache.cassandra.utils.CollectionSerializers.deserializeList; -import static org.apache.cassandra.utils.CollectionSerializers.serializeCollection; -import static org.apache.cassandra.utils.CollectionSerializers.serializedCollectionSize; -import static org.apache.cassandra.utils.Collectors3.toImmutableMap; - -/* - * Narrowly focused on setting or changing the consensus migration protocol. The real use case - * is when a migration is already in progress or done and you want to change the target. - */ -public class SetConsensusMigrationTargetProtocol implements Transformation -{ - public static Serializer serializer = new Serializer(); - - @Nonnull - public final ConsensusMigrationTarget targetProtocol; - - @Nonnull - public final List tables; - - public SetConsensusMigrationTargetProtocol(@Nonnull ConsensusMigrationTarget targetProtocol, - @Nonnull List tables) - { - this.targetProtocol = targetProtocol; - this.tables = tables; - } - - @Override - public Kind kind() - { - return SET_CONSENSUS_MIGRATION_TARGET_PROTOCOL; - } - - @Override - public Result execute(ClusterMetadata metadata) - { - Map tableStates = metadata.consensusMigrationState.tableStates; - List columnFamilyStores = tables.stream().map(Schema.instance::getColumnFamilyStoreInstance).collect(toImmutableList()); - - Transformer transformer = metadata.transformer(); - - Map newStates; - - if (targetProtocol == reset) - { - newStates = tableStates.entrySet().stream().filter(entry -> !tables.contains(entry.getKey())).collect(toImmutableMap()); - } - else - { - newStates = columnFamilyStores - .stream() - .map(cfs -> - tableStates.containsKey(cfs.getTableId()) ? - tableStates.get(cfs.getTableId()).withMigrationTarget(targetProtocol) : - new TableMigrationState(cfs.keyspace.getName(), cfs.name, cfs.getTableId(), targetProtocol, ImmutableSet.of(), ImmutableMap.of())) - .collect(toImmutableMap(TableMigrationState::getTableId, Function.identity())); - } - - return Transformation.success(transformer.with(newStates, targetProtocol == reset ? false : true), LockedRanges.AffectedRanges.EMPTY); - } - - static class Serializer implements AsymmetricMetadataSerializer - { - public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException - { - SetConsensusMigrationTargetProtocol v = (SetConsensusMigrationTargetProtocol)t; - out.writeUTF(v.targetProtocol.toString()); - serializeCollection(v.tables, out, version, TableId.metadataSerializer); - } - - public SetConsensusMigrationTargetProtocol deserialize(DataInputPlus in, Version version) throws IOException - { - ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(in.readUTF()); - List tables = deserializeList(in, version, TableId.metadataSerializer); - return new SetConsensusMigrationTargetProtocol(targetProtocol, tables); - } - - public long serializedSize(Transformation t, Version version) - { - SetConsensusMigrationTargetProtocol v = (SetConsensusMigrationTargetProtocol) t; - return TypeSizes.sizeof(v.targetProtocol.toString()) - + serializedCollectionSize(v.tables, version, TableId.metadataSerializer); - } - } -} diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index 0c933fdf7d..00f2e66dfa 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -275,7 +275,6 @@ public class NodeTool .withDescription("List and mark ranges as migrating between consensus protocols") .withDefaultCommand(CassHelp.class) .withCommand(ConsensusMigrationAdmin.BeginMigration.class) - .withCommands(ConsensusMigrationAdmin.SetTargetProtocol.class) .withCommands(ConsensusMigrationAdmin.ListCmd.class) .withCommands(ConsensusMigrationAdmin.FinishMigration.class); diff --git a/src/java/org/apache/cassandra/tools/nodetool/ConsensusMigrationAdmin.java b/src/java/org/apache/cassandra/tools/nodetool/ConsensusMigrationAdmin.java index cd24bf91fa..9878034f88 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/ConsensusMigrationAdmin.java +++ b/src/java/org/apache/cassandra/tools/nodetool/ConsensusMigrationAdmin.java @@ -31,9 +31,7 @@ import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool; import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Collections.emptyList; import static java.util.Collections.singleton; -import static java.util.Collections.singletonList; /** * For managing migration from one consensus protocol to another. @@ -122,25 +120,4 @@ public abstract class ConsensusMigrationAdmin extends NodeTool.NodeToolCmd probe.output().out.printf("Finished consensus migration range (%s) of keyspaces %s and tables %s%n", maybeRangesStr, keyspaceNames, maybeTableNames); } } - - @Command(name = "set-target-protocol", description = "Set or change the target consensus protocol of the specified tables. If a migration is in progress then the migration will be reversed with migrating ranges still migrating, unmigrated ranges marked as migrated, and migrating ranges will need migration. Be aware that if no migration was in progress for a table it will immediately cause the table to run on the target protocol because the ranges requiring migration are derived from the migrated ranges that don't exist.") - public static class SetTargetProtocol extends ConsensusMigrationAdmin - { - @Arguments(usage = "[ ...]", description = "The keyspace followed by one or many tables") - private List schemaArgs = new ArrayList<>(); - - @Option(title = "target_protocol", name = {"-tp", "--target-protocol"}, description = "Use -tp to specify what consensus protocol should be migrated to", required=true) - private String targetProtocol = null; - - @Option(title = "force_completion", name = {"-f", "--force-completion"}, description = "Forces migration state for all ranges of the specified table regardless of whether migration completed successfully or not. Should only be used if table is empty or has had no writes since last repair.") - private boolean forceCompletion = false; - - protected void execute(NodeProbe probe) - { - checkArgument(schemaArgs.size() >= 2, "Must specify a keyspace and at least one table"); - List keyspaceNames = schemaArgs.size() > 0 ? singletonList(schemaArgs.get(0)) : emptyList(); - List maybeTableNames = schemaArgs.size() > 1 ? schemaArgs.subList(1, schemaArgs.size()) : null; - probe.getStorageService().setConsensusMigrationTargetProtocol(targetProtocol, keyspaceNames, maybeTableNames); - } - } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java index 5c6e276db3..b5e507a627 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java @@ -27,10 +27,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.util.concurrent.FutureCallback; +import org.apache.cassandra.distributed.api.*; import org.apache.cassandra.distributed.test.accord.AccordTestBase; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.*; import net.bytebuddy.ByteBuddy; import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; @@ -46,15 +45,11 @@ import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.Cluster; -import org.apache.cassandra.distributed.api.ConsistencyLevel; -import org.apache.cassandra.distributed.api.Feature; -import org.apache.cassandra.distributed.api.ICoordinator; -import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.api.IMessageFilters.Filter; -import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.reads.repair.BlockingReadRepair; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; import org.apache.cassandra.utils.concurrent.Condition; @@ -80,6 +75,34 @@ import static org.junit.Assert.fail; public class ReadRepairTest extends TestBaseImpl { + private static Cluster cluster; + private static int tableNum = 0; + private String tableName; + + @BeforeClass + public static void beforeClass() throws Throwable + { + cluster = init(Cluster.create(3, c -> c.with(Feature.GOSSIP, Feature.NETWORK))); + } + + @AfterClass + public static void afterClass() throws Throwable + { + if (cluster != null) + cluster.close(); + } + + private void incrementTableName() + { + tableName = "tbl" + tableNum++; + } + + @Before + public void setup() + { + incrementTableName(); + } + /** * Tests basic behaviour of read repair with {@code BLOCKING} read repair strategy. */ @@ -87,6 +110,7 @@ public class ReadRepairTest extends TestBaseImpl public void testBlockingReadRepair() throws Throwable { testReadRepair(ReadRepairStrategy.BLOCKING, false); + incrementTableName(); testReadRepair(ReadRepairStrategy.BLOCKING, true); } /** @@ -106,68 +130,63 @@ public class ReadRepairTest extends TestBaseImpl private void testReadRepair(ReadRepairStrategy strategy, boolean brrThroughAccord) throws Throwable { - try (Cluster cluster = init(Cluster.create(3, config -> config.set("non_serial_write_strategy", brrThroughAccord ? "migration" : "normal")))) - { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c int, v int, PRIMARY KEY (k, c)) " + - String.format("WITH read_repair='%s'", strategy))); - AccordTestBase.ensureTableIsAccordManaged(cluster, KEYSPACE, "t"); + TransactionalMode transactionalMode = brrThroughAccord ? TransactionalMode.unsafe_writes : TransactionalMode.off; + cluster.schemaChange(withKeyspace("CREATE TABLE %s." + tableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode.toString().toLowerCase() + '\'' + + String.format(" AND read_repair='%s'", strategy))); + AccordTestBase.ensureTableIsAccordManaged(cluster, KEYSPACE, "t"); - Object[] row = row(1, 1, 1); - String insertQuery = withKeyspace("INSERT INTO %s.t (k, c, v) VALUES (?, ?, ?)"); - String selectQuery = withKeyspace("SELECT * FROM %s.t WHERE k=1"); + Object[] row = row(1, 1, 1); + String insertQuery = withKeyspace("INSERT INTO %s." + tableName + " (k, c, v) VALUES (?, ?, ?)"); + String selectQuery = withKeyspace("SELECT * FROM %s." + tableName + " WHERE k=1"); - // insert data in two nodes, simulating a quorum write that has missed one node - cluster.get(1).executeInternal(insertQuery, row); - cluster.get(2).executeInternal(insertQuery, row); + // insert data in two nodes, simulating a quorum write that has missed one node + cluster.get(1).executeInternal(insertQuery, row); + cluster.get(2).executeInternal(insertQuery, row); - // verify that the third node doesn't have the row + // verify that the third node doesn't have the row + assertRows(cluster.get(3).executeInternal(selectQuery)); + + // read with CL=QUORUM to trigger read repair, force 3 to be involved in the read so that read repair + // will occur + Filter blockReadFromOne = cluster.filters().inbound().from(3).to(1).verbs(READ_REQ.id).drop(); + assertRows(cluster.coordinator(3).execute(selectQuery, QUORUM), row); + blockReadFromOne.off(); + + // verify whether the coordinator has the repaired row depending on the read repair strategy + if (strategy == ReadRepairStrategy.NONE) assertRows(cluster.get(3).executeInternal(selectQuery)); - - // read with CL=QUORUM to trigger read repair, force 3 to be involved in the read so that read repair - // will occur - Filter blockReadFromOne = cluster.filters().inbound().from(3).to(1).verbs(READ_REQ.id).drop(); - assertRows(cluster.coordinator(3).execute(selectQuery, QUORUM), row); - blockReadFromOne.off(); - - // verify whether the coordinator has the repaired row depending on the read repair strategy - if (strategy == ReadRepairStrategy.NONE) - assertRows(cluster.get(3).executeInternal(selectQuery)); - else - assertRows(cluster.get(3).executeInternal(selectQuery), row); - } + else + assertRows(cluster.get(3).executeInternal(selectQuery), row); } @Test public void readRepairTimeoutTest() throws Throwable { final long reducedReadTimeout = 3000L; - try (Cluster cluster = init(builder().withNodes(3).start())) + cluster.forEach(i -> i.runOnInstance(() -> DatabaseDescriptor.setReadRpcTimeout(reducedReadTimeout))); + cluster.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'"); + cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v) VALUES (1, 1, 1)"); + cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v) VALUES (1, 1, 1)"); + assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1")); + cluster.verbs(READ_REPAIR_RSP).to(1).drop(); + final long start = currentTimeMillis(); + try { - cluster.forEach(i -> i.runOnInstance(() -> DatabaseDescriptor.setReadRpcTimeout(reducedReadTimeout))); - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'"); - cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)"); - cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)"); - assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1")); - cluster.verbs(READ_REPAIR_RSP).to(1).drop(); - final long start = currentTimeMillis(); - try - { - cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.ALL); - fail("Read timeout expected but it did not occur"); - } - catch (Exception ex) - { - // the containing exception class was loaded by another class loader. Comparing the message as a workaround to assert the exception - assertTrue(ex.getClass().toString().contains("ReadTimeoutException")); - long actualTimeTaken = currentTimeMillis() - start; - long magicDelayAmount = 100L; // it might not be the best way to check if the time taken is around the timeout value. - // Due to the delays, the actual time taken from client perspective is slighly more than the timeout value - assertTrue(actualTimeTaken > reducedReadTimeout); - // But it should not exceed too much - assertTrue(actualTimeTaken < reducedReadTimeout + magicDelayAmount); - assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1"), - row(1, 1, 1)); // the partition happened when the repaired node sending back ack. The mutation should be in fact applied. - } + cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1", ConsistencyLevel.ALL); + fail("Read timeout expected but it did not occur"); + } + catch (Exception ex) + { + // the containing exception class was loaded by another class loader. Comparing the message as a workaround to assert the exception + assertTrue(ex.getClass().toString().contains("ReadTimeoutException")); + long actualTimeTaken = currentTimeMillis() - start; + long magicDelayAmount = 100L; // it might not be the best way to check if the time taken is around the timeout value. + // Due to the delays, the actual time taken from client perspective is slighly more than the timeout value + assertTrue(actualTimeTaken > reducedReadTimeout); + // But it should not exceed too much + assertTrue(actualTimeTaken < reducedReadTimeout + magicDelayAmount); + assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1"), + row(1, 1, 1)); // the partition happened when the repaired node sending back ack. The mutation should be in fact applied. } } @@ -176,20 +195,20 @@ public class ReadRepairTest extends TestBaseImpl { try (Cluster cluster = init(builder().withNodes(3).start())) { - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'"); + cluster.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'"); for (int i = 1 ; i <= 2 ; ++i) - cluster.get(i).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1)"); + cluster.get(i).executeInternal("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v) VALUES (1, 1, 1)"); - assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1")); + assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1")); cluster.filters().verbs(READ_REPAIR_REQ.id).to(3).drop(); - assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", + assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1", ConsistencyLevel.QUORUM), row(1, 1, 1)); // Data was not repaired - assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1")); + assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1")); } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java b/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java index 6d192db112..103b1fc4c0 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java @@ -28,7 +28,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.IntStream; -import org.apache.cassandra.distributed.test.accord.AccordTestBase; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -37,16 +36,13 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.apache.cassandra.config.Config.LWTStrategy; -import org.apache.cassandra.config.Config.NonSerialWriteStrategy; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.Pair; import static com.google.common.collect.Iterators.toArray; import static java.lang.String.format; @@ -88,17 +84,17 @@ public class ShortReadProtectionTest extends TestBaseImpl public boolean paging; @Parameterized.Parameter(3) - public Pair transactionStrategies; + public TransactionalMode transactionalMode; - @Parameterized.Parameters(name = "{index}: read_cl={0} flush={1} paging={2}, transactionStrategies={3}") + @Parameterized.Parameters(name = "{index}: read_cl={0} flush={1} paging={2}, transactionalMode={3}") public static Collection data() { List result = new ArrayList<>(); - for (Pair transactionStrategies : Arrays.asList(Pair.create(LWTStrategy.accord, NonSerialWriteStrategy.migration), Pair.create(LWTStrategy.migration, NonSerialWriteStrategy.normal))) + for (TransactionalMode mode : TransactionalMode.values()) for (ConsistencyLevel readConsistencyLevel : Arrays.asList(ALL, QUORUM, SERIAL)) for (boolean flush : BOOLEANS) for (boolean paging : BOOLEANS) - result.add(new Object[]{ readConsistencyLevel, flush, paging, transactionStrategies}); + result.add(new Object[]{ readConsistencyLevel, flush, paging, mode}); return result; } @@ -123,17 +119,14 @@ public class ShortReadProtectionTest extends TestBaseImpl @Before public void setupTester() { - String lwtStrategy = transactionStrategies.left.toString(); - String nonSerialWriteStrategy = transactionStrategies.right.toString(); - cluster.forEach(node -> { - node.runOnInstance(() -> { - DatabaseDescriptor.setLWTStrategy(LWTStrategy.valueOf(lwtStrategy)); - DatabaseDescriptor.setNonSerialWriteStrategy(NonSerialWriteStrategy.valueOf(nonSerialWriteStrategy)); - }); - }); tester = new Tester(readConsistencyLevel, flush, paging); } + private String transactionalModeCQL() + { + return " WITH transactional_mode='" + transactionalMode + '\''; + } + @After public void teardownTester() { @@ -150,7 +143,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testSkinnyTableWithoutLiveRows() { - tester.createTable("CREATE TABLE %s (id int PRIMARY KEY)") + tester.createTable("CREATE TABLE %s (id int PRIMARY KEY)" + transactionalModeCQL()) .allNodes("INSERT INTO %s (id) VALUES (0) USING TIMESTAMP 0") .toNode1("DELETE FROM %s WHERE id = 0") .assertRows("SELECT DISTINCT id FROM %s WHERE id = 0") @@ -167,7 +160,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testSkinnyTableWithLiveRows() { - tester.createTable("CREATE TABLE %s (id int PRIMARY KEY)") + tester.createTable("CREATE TABLE %s (id int PRIMARY KEY)" + transactionalModeCQL()) .allNodes(0, 10, i -> format("INSERT INTO %%s (id) VALUES (%d) USING TIMESTAMP 0", i)) // order is 5,1,8,0,2,4,7,6,9,3 .toNode1("DELETE FROM %s WHERE id IN (1, 0, 4, 6, 3)") // delete every other row .assertRows("SELECT DISTINCT token(id), id FROM %s", @@ -184,7 +177,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testSkinnyTableWithComplementaryDeletions() { - tester.createTable("CREATE TABLE %s (id int PRIMARY KEY)") + tester.createTable("CREATE TABLE %s (id int PRIMARY KEY)" + transactionalModeCQL()) .allNodes(0, 10, i -> format("INSERT INTO %%s (id) VALUES (%d) USING TIMESTAMP 0", i)) // order is 5,1,8,0,2,4,7,6,9,3 .toNode1("DELETE FROM %s WHERE id IN (5, 8, 2, 7, 9)") // delete every other row .toNode2("DELETE FROM %s WHERE id IN (1, 0, 4, 6)") // delete every other row but the last one @@ -202,7 +195,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testMultipleMissedRows() { - tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))") + tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))" + transactionalModeCQL()) .allNodes(0, 4, i -> format("INSERT INTO %%s (pk, ck) VALUES (0, %d) USING TIMESTAMP 0", i)) .toNode1("DELETE FROM %s WHERE pk = 0 AND ck IN (1, 2, 3)", "INSERT INTO %s (pk, ck) VALUES (0, 5)") @@ -221,7 +214,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testAscendingOrder() { - tester.createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))") + tester.createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))" + transactionalModeCQL()) .allNodes(1, 10, i -> format("INSERT INTO %%s (k, c, v) VALUES (0, %d, %d) USING TIMESTAMP 0", i, i * 10)) .toNode1("DELETE FROM %s WHERE k=0 AND c=1") .toNode2("DELETE FROM %s WHERE k=0 AND c=2") @@ -243,7 +236,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testDescendingOrder() { - tester.createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))") + tester.createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))" + transactionalModeCQL()) .allNodes(1, 10, i -> format("INSERT INTO %%s (k, c, v) VALUES (0, %d, %d) USING TIMESTAMP 0", i, i * 10)) .toNode1("DELETE FROM %s WHERE k=0 AND c=7") .toNode2("DELETE FROM %s WHERE k=0 AND c=8") @@ -266,7 +259,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testDeletePartition() { - tester.createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))") + tester.createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))" + transactionalModeCQL()) .allNodes("INSERT INTO %s (k, c, v) VALUES (0, 1, 10) USING TIMESTAMP 0", "INSERT INTO %s (k, c, v) VALUES (0, 2, 20) USING TIMESTAMP 0") .toNode2("DELETE FROM %s WHERE k=0") @@ -279,7 +272,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testDeletePartitionWithStatic() { - tester.createTable("CREATE TABLE %s (k int, c int, v int, s int STATIC, PRIMARY KEY(k, c))") + tester.createTable("CREATE TABLE %s (k int, c int, v int, s int STATIC, PRIMARY KEY(k, c))" + transactionalModeCQL()) .allNodes("INSERT INTO %s (k, c, v, s) VALUES (0, 1, 10, 100) USING TIMESTAMP 0", "INSERT INTO %s (k, c, v) VALUES (0, 2, 20) USING TIMESTAMP 0") .toNode2("DELETE FROM %s WHERE k=0") @@ -292,7 +285,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testDeleteClustering() { - tester.createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))") + tester.createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))" + transactionalModeCQL()) .allNodes("INSERT INTO %s (k, c, v) VALUES (0, 1, 10) USING TIMESTAMP 0", "INSERT INTO %s (k, c, v) VALUES (0, 2, 20) USING TIMESTAMP 0") .toNode2("DELETE FROM %s WHERE k=0 AND c=1") @@ -307,7 +300,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testDeleteClusteringWithStatic() { - tester.createTable("CREATE TABLE %s (k int, c int, v int, s int STATIC, PRIMARY KEY(k, c))") + tester.createTable("CREATE TABLE %s (k int, c int, v int, s int STATIC, PRIMARY KEY(k, c))" + transactionalModeCQL()) .allNodes("INSERT INTO %s (k, c, v, s) VALUES (0, 1, 10, 100) USING TIMESTAMP 0", "INSERT INTO %s (k, c, v) VALUES (0, 2, 20) USING TIMESTAMP 0") .toNode2("DELETE FROM %s WHERE k=0 AND c=1") @@ -324,7 +317,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testGroupByRegularRow() { - tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))") + tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))" + transactionalModeCQL()) .toNode1("INSERT INTO %s (pk, ck) VALUES (1, 1) USING TIMESTAMP 0", "DELETE FROM %s WHERE pk=0 AND ck=0", "INSERT INTO %s (pk, ck) VALUES (2, 2) USING TIMESTAMP 0") @@ -347,7 +340,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testGroupByStaticRow() { - tester.createTable("CREATE TABLE %s (pk int, ck int, s int static, PRIMARY KEY (pk, ck))") + tester.createTable("CREATE TABLE %s (pk int, ck int, s int static, PRIMARY KEY (pk, ck))" + transactionalModeCQL()) .toNode1("INSERT INTO %s (pk, s) VALUES (1, 1) USING TIMESTAMP 0", "INSERT INTO %s (pk, s) VALUES (0, null)", "INSERT INTO %s (pk, s) VALUES (2, 2) USING TIMESTAMP 0") @@ -370,7 +363,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testSkipEarlyTermination() { - tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))") + tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))" + transactionalModeCQL()) .toNode1("INSERT INTO %s (pk, ck) VALUES (0, 0)") .toNode2("DELETE FROM %s WHERE pk = 0 AND ck IN (1, 2)") .assertRows("SELECT DISTINCT pk FROM %s", row(0)); @@ -387,7 +380,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testSkipEarlyTerminationRows() { - tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))") + tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))" + transactionalModeCQL()) .toNode1("INSERT INTO %s (pk, ck) VALUES (0, 0) USING TIMESTAMP 0", "INSERT INTO %s (pk, ck) VALUES (0, 1) USING TIMESTAMP 0", "INSERT INTO %s (pk, ck) VALUES (2, 0) USING TIMESTAMP 0", @@ -411,7 +404,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Test public void testSkipEarlyTerminationPartitions() { - tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))") + tester.createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))" + transactionalModeCQL()) .toNode1("INSERT INTO %s (pk, ck) VALUES (0, 0) USING TIMESTAMP 0", "INSERT INTO %s (pk, ck) VALUES (0, 1) USING TIMESTAMP 0", "DELETE FROM %s USING TIMESTAMP 42 WHERE pk = 2 AND ck IN (0, 1)") @@ -455,8 +448,7 @@ public class ShortReadProtectionTest extends TestBaseImpl private Tester createTable(String query) { - cluster.schemaChange(format(query) + " WITH read_repair='NONE'"); - AccordTestBase.ensureTableIsAccordManaged(cluster, KEYSPACE, table); + cluster.schemaChange(format(query) + " AND read_repair='NONE'"); return this; } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java index 7d246a2b6c..f040e9d4db 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java @@ -206,7 +206,7 @@ public class AccordBootstrapTest extends TestBaseImpl } cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}"); - cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c))"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'"); long schemaChangeMax = maxEpoch(cluster); for (IInvokableInstance node : cluster) @@ -384,7 +384,7 @@ public class AccordBootstrapTest extends TestBaseImpl } cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}"); - cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c))"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'"); long schemaChangeMax = maxEpoch(cluster); for (IInvokableInstance node : cluster) diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java index 5eeee8108a..f7ef3afe7e 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java @@ -46,8 +46,6 @@ import org.slf4j.LoggerFactory; import accord.primitives.Unseekables; import accord.topology.Topologies; -import org.apache.cassandra.config.Config.NonSerialWriteStrategy; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.functions.types.utils.Bytes; import org.apache.cassandra.db.marshal.Int32Type; @@ -63,6 +61,7 @@ import org.apache.cassandra.distributed.api.SimpleQueryResult; import org.apache.cassandra.distributed.shared.AssertUtils; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTestUtils; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.utils.ByteBufferUtil; import org.assertj.core.api.Assertions; @@ -85,35 +84,28 @@ public class AccordCQLTest extends AccordTestBase } @Parameterized.Parameter - public String nonSerialWriteStrategyName; + public String transactionalModeName; - NonSerialWriteStrategy nonSerialWriteStrategy; + TransactionalMode transactionalMode; - @Parameterized.Parameters(name = "nonSerialWriteStrategy={0}") + @Parameterized.Parameters(name = "transactionalMode={0}") public static Collection data() { - return ImmutableList.of(new Object[] {NonSerialWriteStrategy.accord.toString()}, new Object[] {NonSerialWriteStrategy.migration.toString()}); + return ImmutableList.of(new Object[] {TransactionalMode.full.toString()}, + new Object[] {TransactionalMode.mixed_reads.toString()}); } @Before public void setNonSerialWriteStrategy() { - nonSerialWriteStrategy = NonSerialWriteStrategy.valueOf(nonSerialWriteStrategyName); - String nonSerialWriteStrategyName = this.nonSerialWriteStrategyName; - SHARED_CLUSTER.forEach(node -> { - node.runOnInstance(() -> { - DatabaseDescriptor.setNonSerialWriteStrategy(NonSerialWriteStrategy.valueOf(nonSerialWriteStrategyName)); - }); - }); + transactionalMode = TransactionalMode.valueOf(transactionalModeName); } @BeforeClass public static void setupClass() throws IOException { - AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("lwt_strategy", "accord") - .set("non_serial_write_strategy", "migration")), 2); + AccordTestBase.setupCluster(builder -> builder, 2); SHARED_CLUSTER.schemaChange("CREATE TYPE " + KEYSPACE + ".person (height int, age int)"); - SHARED_CLUSTER.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE)); } @Test @@ -177,7 +169,7 @@ public class AccordCQLTest extends AccordTestBase String currentTable = keyspace + ".tbl"; List ddls = Arrays.asList("DROP KEYSPACE IF EXISTS " + keyspace + ";", "CREATE KEYSPACE " + keyspace + " WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}", - "CREATE TABLE " + currentTable + " (k blob, c int, v int, primary key (k, c))"); + "CREATE TABLE " + currentTable + " (k blob, c int, v int, primary key (k, c)) WITH transactional_mode='" + transactionalMode + "'"); List tokens = tokens(); List keys = tokensToKeys(tokens); List keyStrings = keys.stream().map(bb -> "0x" + ByteBufferUtil.bytesToHex(bb)).collect(Collectors.toList()); @@ -262,13 +254,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testRegularScalarIsNull() throws Throwable { - testScalarIsNull("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c))"); + testScalarIsNull("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c)) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testStaticScalarIsNull() throws Throwable { - testScalarIsNull("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int static, primary key (k, c))"); + testScalarIsNull("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int static, primary key (k, c)) WITH transactional_mode='" + transactionalMode + "'"); } private void testScalarIsNull(String tableDDL) throws Exception { @@ -303,7 +295,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testQueryStaticColumn() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, s int static, v int, primary key (k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, s int static, v int, primary key (k, c)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { // select partition key, clustering key and static column, restrict on partition and clustering @@ -357,7 +349,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testUpdateStaticColumn() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, s int static, v int, primary key (k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, s int static, v int, primary key (k, c)) WITH transactional_mode='" + transactionalMode + '\'', cluster -> { checkUpdateStatic(cluster, "SET s=1 WHERE k=?", 101, "[[101, null, 1, null]]", "[]"); @@ -393,7 +385,7 @@ public class AccordCQLTest extends AccordTestBase private void assertResultsFromAccordMatches(Cluster cluster, String accordRead, String simpleRead, int key) { Object[][] simpleReadResult; - if (nonSerialWriteStrategy.ignoresSuppliedConsistencyLevel) + if (transactionalMode.ignoresSuppliedConsistencyLevel) // With accord non-SERIAL write strategy the commit CL is effectively ANY so we need to read at SERIAL simpleReadResult = cluster.coordinator(1).execute(simpleRead, ConsistencyLevel.SERIAL, key); else @@ -453,12 +445,12 @@ public class AccordCQLTest extends AccordTestBase @Test public void testStaticScalarEQ() throws Throwable { - testScalarCondition("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int static, primary key (k, c))", 3, "=", 3, "="); + testScalarCondition("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int static, primary key (k, c)) WITH transactional_mode='" + transactionalMode + "'", 3, "=", 3, "="); } private void testScalarCondition(int lhs, String operator, int rhs, String reversedOperator) throws Exception { - testScalarCondition("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c))", lhs, operator, rhs, reversedOperator); + testScalarCondition("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c)) WITH transactional_mode='" + transactionalMode + "'", lhs, operator, rhs, reversedOperator); } private void testScalarCondition(String tableDDL, int lhs, String operator, int rhs, String reversedOperator) throws Exception @@ -580,7 +572,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testReversedClusteringReference() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH CLUSTERING ORDER BY (c DESC)", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH CLUSTERING ORDER BY (c DESC) AND transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (1, 1, 1)", ConsistencyLevel.ALL); @@ -615,7 +607,7 @@ public class AccordCQLTest extends AccordTestBase private void testScalarShorthandOperation(int startingValue, String operation, int endingvalue) throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, v) VALUES (1, ?)", ConsistencyLevel.ALL, startingValue); @@ -637,7 +629,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testConstantNonStaticRowReadBeforeUpdate() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (1, 2, ?)", ConsistencyLevel.ALL, 3); @@ -659,7 +651,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testRangeDeletion() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (1, 2, ?)", ConsistencyLevel.ALL, 3); @@ -683,7 +675,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testPartitionKeyReferenceCondition() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k INT, c INT, v INT, PRIMARY KEY (k, c)) WITH CLUSTERING ORDER BY (c DESC)", + test("CREATE TABLE " + qualifiedTableName + " (k INT, c INT, v INT, PRIMARY KEY (k, c)) WITH CLUSTERING ORDER BY (c DESC) AND transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (1, 1, 1)", ConsistencyLevel.ALL); @@ -731,13 +723,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellListEqCondition() throws Exception { - testListEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list)"); + testListEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenListEqCondition() throws Exception { - testListEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>)"); + testListEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testListEqCondition(String ddl) throws Exception @@ -778,13 +770,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellSetEqCondition() throws Exception { - testSetEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set)"); + testSetEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenSetEqCondition() throws Exception { - testSetEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>)"); + testSetEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testSetEqCondition(String ddl) throws Exception @@ -825,13 +817,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellMapEqCondition() throws Exception { - testMapEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map)", true); + testMapEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map) WITH transactional_mode='" + transactionalMode + "'", true); } @Test public void testFrozenMapEqCondition() throws Exception { - testMapEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>)", false); + testMapEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>) WITH transactional_mode='" + transactionalMode + "'", false); } private void testMapEqCondition(String ddl, boolean isMultiCell) throws Exception @@ -872,13 +864,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellUDTEqCondition() throws Exception { - testUDTEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person)"); + testUDTEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenUDTEqCondition() throws Exception { - testUDTEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen)"); + testUDTEqCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen) WITH transactional_mode='" + transactionalMode + "'"); } private void testUDTEqCondition(String tableDDL) throws Exception @@ -918,7 +910,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testTupleEqCondition() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, pair tuple)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, pair tuple) WITH transactional_mode='" + transactionalMode + "'", cluster -> { Object initialTupleValue = CQLTester.tuple("age", 37); @@ -953,7 +945,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testIsNullWithComplexDeletion() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, int_list list, PRIMARY KEY (k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, int_list list, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { ListType listType = ListType.getInstance(Int32Type.instance, true); @@ -987,13 +979,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testNullMultiCellListConditions() throws Exception { - testNullListConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list)"); + testNullListConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testNullFrozenListConditions() throws Exception { - testNullListConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>)"); + testNullListConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testNullListConditions(String ddl) throws Exception @@ -1039,13 +1031,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testNullMultiCellSetConditions() throws Exception { - testNullSetConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set)"); + testNullSetConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testNullFrozenSetConditions() throws Exception { - testNullSetConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>)"); + testNullSetConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testNullSetConditions(String ddl) throws Exception @@ -1091,13 +1083,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testNullMultiCellMapConditions() throws Exception { - testNullMapConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map)", true); + testNullMapConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map) WITH transactional_mode='" + transactionalMode + "'", true); } @Test public void testNullFrozenMapConditions() throws Exception { - testNullMapConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>)", false); + testNullMapConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>) WITH transactional_mode='" + transactionalMode + "'", false); } private void testNullMapConditions(String ddl, boolean isMultiCell) throws Exception @@ -1148,13 +1140,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testNullMultiCellUDTCondition() throws Exception { - testNullUDTCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person)"); + testNullUDTCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testNullFrozenUDTCondition() throws Exception { - testNullUDTCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen)"); + testNullUDTCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen) WITH transactional_mode='" + transactionalMode + "'"); } private void testNullUDTCondition(String tableDDL) throws Exception @@ -1202,13 +1194,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testNullMultiCellSetElementConditions() throws Exception { - testNullSetElementConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set)"); + testNullSetElementConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testNullFrozenSetElementConditions() throws Exception { - testNullSetElementConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>)"); + testNullSetElementConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testNullSetElementConditions(String ddl) throws Exception @@ -1254,13 +1246,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testNullMultiCellMapElementConditions() throws Exception { - testNullMapElementConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map)", true); + testNullMapElementConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map) WITH transactional_mode='" + transactionalMode + "'", true); } @Test public void testNullFrozenMapElementConditions() throws Exception { - testNullMapElementConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>)", false); + testNullMapElementConditions("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>) WITH transactional_mode='" + transactionalMode + "'", false); } private void testNullMapElementConditions(String ddl, boolean isMultiCell) throws Exception @@ -1311,13 +1303,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testNullMultiCellUDTFieldCondition() throws Exception { - testNullUDTFieldCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person)"); + testNullUDTFieldCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testNullFrozenUDTFieldCondition() throws Exception { - testNullUDTFieldCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen)"); + testNullUDTFieldCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen) WITH transactional_mode='" + transactionalMode + "'"); } private void testNullUDTFieldCondition(String tableDDL) throws Exception @@ -1365,13 +1357,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellListSubstitution() throws Exception { - testListSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list)", true); + testListSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list) WITH transactional_mode='" + transactionalMode + "'", true); } @Test public void testFrozenListSubstitution() throws Exception { - testListSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>)", false); + testListSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>) WITH transactional_mode='" + transactionalMode + "'", false); } private void testListSubstitution(String ddl, boolean isMultiCell) throws Exception @@ -1405,13 +1397,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellSetSubstitution() throws Exception { - testSetSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set)", true); + testSetSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set) WITH transactional_mode='" + transactionalMode + "'", true); } @Test public void testFrozenSetSubstitution() throws Exception { - testSetSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>)", false); + testSetSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>) WITH transactional_mode='" + transactionalMode + "'", false); } private void testSetSubstitution(String ddl, boolean isMultiCell) throws Exception @@ -1445,13 +1437,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellMapSubstitution() throws Exception { - testMapSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map)", true); + testMapSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map) WITH transactional_mode='" + transactionalMode + "'", true); } @Test public void testFrozenMapSubstitution() throws Exception { - testMapSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>)", false); + testMapSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>) WITH transactional_mode='" + transactionalMode + "'", false); } private void testMapSubstitution(String ddl, boolean isMultiCell) throws Exception @@ -1485,13 +1477,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellUDTSubstitution() throws Exception { - testUDTSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person)"); + testUDTSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenUDTSubstitution() throws Exception { - testUDTSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen)"); + testUDTSubstitution("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen) WITH transactional_mode='" + transactionalMode + "'"); } private void testUDTSubstitution(String tableDDL) throws Exception @@ -1523,7 +1515,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testTupleSubstitution() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, pair tuple)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, pair tuple) WITH transactional_mode='" + transactionalMode + "'", cluster -> { Object initialTupleValue = CQLTester.tuple("age", 37); @@ -1550,13 +1542,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellListReplacement() throws Exception { - testListReplacement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list)"); + testListReplacement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenListReplacement() throws Exception { - testListReplacement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>)"); + testListReplacement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testListReplacement(String ddl) throws Exception @@ -1587,13 +1579,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellSetReplacement() throws Exception { - testSetReplacement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set)"); + testSetReplacement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenSetReplacement() throws Exception { - testSetReplacement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>)"); + testSetReplacement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testSetReplacement(String ddl) throws Exception @@ -1624,7 +1616,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testListAppendFromReference() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, int_list) VALUES (0, [1, 2]);", ConsistencyLevel.ALL); @@ -1650,13 +1642,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testSetByIndexFromMultiCellListElement() throws Exception { - testListSetByIndexFromListElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, src_int_list list, dest_int_list list)"); + testListSetByIndexFromListElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, src_int_list list, dest_int_list list) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testSetByIndexFromFrozenListElement() throws Exception { - testListSetByIndexFromListElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, src_int_list frozen>, dest_int_list list)"); + testListSetByIndexFromListElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, src_int_list frozen>, dest_int_list list) WITH transactional_mode='" + transactionalMode + "'"); } private void testListSetByIndexFromListElement(String ddl) throws Exception @@ -1685,7 +1677,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testListSetByIndexFromScalar() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, int_list) VALUES (0, [1, 2]);", ConsistencyLevel.ALL); @@ -1708,7 +1700,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testAutoReadSelectionConstruction() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, counter int, other_counter int, PRIMARY KEY (k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, counter int, other_counter int, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, counter, other_counter) VALUES (0, 0, 1, 1);", ConsistencyLevel.ALL); @@ -1732,7 +1724,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiMutationsSameKey() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, counter int, int_list list, PRIMARY KEY (k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, counter int, int_list list, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, counter, int_list) VALUES (0, 0, 0, [1, 2]);", ConsistencyLevel.ALL); @@ -1784,7 +1776,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testListSetByIndexMultiRow() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, int_list list, PRIMARY KEY (k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, int_list list, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, int_list) VALUES (0, 0, [1, 2]);", ConsistencyLevel.ALL); @@ -1812,7 +1804,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testSetAppend() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, int_set) VALUES (0, {1, 2});", ConsistencyLevel.ALL); @@ -1836,13 +1828,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testAssignmentFromMultiCellSetElement() throws Exception { - testAssignmentFromSetElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, int_set set)"); + testAssignmentFromSetElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, int_set set) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testAssignmentFromFrozenSetElement() throws Exception { - testAssignmentFromSetElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, int_set frozen>)"); + testAssignmentFromSetElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, int_set frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testAssignmentFromSetElement(String ddl) throws Exception @@ -1871,7 +1863,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMapAppend() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, int_map) VALUES (0, {'one': 2});", ConsistencyLevel.ALL); @@ -1895,13 +1887,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testAssignmentFromMultiCellMapElement() throws Exception { - testAssignmentFromMapElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, int_map map)"); + testAssignmentFromMapElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, int_map map) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testAssignmentFromFrozenMapElement() throws Exception { - testAssignmentFromMapElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, int_map frozen>)"); + testAssignmentFromMapElement("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, int_map frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testAssignmentFromMapElement(String ddl) throws Exception @@ -1930,13 +1922,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testAssignmentFromMultiCellUDTField() throws Exception { - testAssignmentFromUDTField("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, customer person)"); + testAssignmentFromUDTField("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, customer person) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testAssignmentFromFrozenUDTField() throws Exception { - testAssignmentFromUDTField("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, customer frozen)"); + testAssignmentFromUDTField("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, v int, customer frozen) WITH transactional_mode='" + transactionalMode + "'"); } private void testAssignmentFromUDTField(String tableDDL) throws Exception @@ -1967,7 +1959,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testSetMapElementFromMapElementReference() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, int_map) VALUES (0, {'one': 2});", ConsistencyLevel.ALL); @@ -1991,7 +1983,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testSetUDTFieldFromUDTFieldReference() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person) WITH transactional_mode='" + transactionalMode + "'", cluster -> { Object youngPerson = CQLTester.userType("height", 58, "age", 9); @@ -2020,13 +2012,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellListElementCondition() throws Exception { - testListElementCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list)"); + testListElementCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenListElementCondition() throws Exception { - testListElementCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>)"); + testListElementCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testListElementCondition(String ddl) throws Exception @@ -2057,13 +2049,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellMapElementCondition() throws Exception { - testMapElementCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map)"); + testMapElementCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenMapElementCondition() throws Exception { - testMapElementCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>)"); + testMapElementCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testMapElementCondition(String ddl) throws Exception @@ -2094,13 +2086,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellUDTFieldCondition() throws Exception { - testUDTFieldCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person)"); + testUDTFieldCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenUDTFieldCondition() throws Exception { - testUDTFieldCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen)"); + testUDTFieldCondition("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen) WITH transactional_mode='" + transactionalMode + "'"); } private void testUDTFieldCondition(String tableDDL) throws Exception @@ -2145,7 +2137,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testListSubtraction() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, int_list) VALUES (0, [1, 2, 3, 4]);", ConsistencyLevel.ALL); @@ -2171,7 +2163,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testSetSubtraction() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set)", + test("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set) WITH transactional_mode='" + transactionalMode + "'", cluster -> { cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, int_set) VALUES (0, {1, 2, 3, 4});", ConsistencyLevel.ALL); @@ -2197,13 +2189,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellMapSubtraction() throws Exception { - testMapSubtraction("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map, int_set set)"); + testMapSubtraction("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map, int_set set) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenMapSubtraction() throws Exception { - testMapSubtraction("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map, int_set frozen>)"); + testMapSubtraction("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map, int_set frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testMapSubtraction(String ddl) throws Exception @@ -2234,13 +2226,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellListSelection() throws Exception { - testListSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list)"); + testListSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list list) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenListSelection() throws Exception { - testListSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>)"); + testListSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_list frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testListSelection(String ddl) throws Exception @@ -2272,13 +2264,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiCellSetSelection() throws Exception { - testSetSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set)"); + testSetSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set set) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testFrozenSetSelection() throws Exception { - testSetSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>)"); + testSetSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_set frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testSetSelection(String ddl) throws Exception @@ -2316,7 +2308,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testFrozenMapSelection() throws Exception { - testMapSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>)"); + testMapSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map frozen>) WITH transactional_mode='" + transactionalMode + "'"); } private void testMapSelection(String ddl) throws Exception @@ -2349,8 +2341,8 @@ public class AccordCQLTest extends AccordTestBase { String KEYSPACE = "ks" + System.currentTimeMillis(); SHARED_CLUSTER.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 2}"); - SHARED_CLUSTER.schemaChange("CREATE TABLE " + qualifiedTableName + "1 (k int, c int, v int, primary key (k, c))"); - SHARED_CLUSTER.schemaChange("CREATE TABLE " + qualifiedTableName + "2 (k int, c int, v int, primary key (k, c))"); + SHARED_CLUSTER.schemaChange("CREATE TABLE " + qualifiedTableName + "1 (k int, c int, v int, primary key (k, c)) WITH transactional_mode='" + transactionalMode + "'"); + SHARED_CLUSTER.schemaChange("CREATE TABLE " + qualifiedTableName + "2 (k int, c int, v int, primary key (k, c)) WITH transactional_mode='" + transactionalMode + "'"); SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0))); SHARED_CLUSTER.coordinator(1).execute("INSERT INTO " + qualifiedTableName + "1 (k, c, v) VALUES (1, 2, 3);", ConsistencyLevel.ALL); SHARED_CLUSTER.coordinator(1).execute("INSERT INTO " + qualifiedTableName + "2 (k, c, v) VALUES (2, 2, 4);", ConsistencyLevel.ALL); @@ -2375,13 +2367,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testRegularScalarInsertSubstitution() throws Exception { - testScalarInsertSubstitution("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c))"); + testScalarInsertSubstitution("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testStaticScalarInsertSubstitution() throws Exception { - testScalarInsertSubstitution("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int static, PRIMARY KEY (k, c))"); + testScalarInsertSubstitution("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int static, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode + "'"); } private void testScalarInsertSubstitution(String tableDDL) throws Exception @@ -2411,13 +2403,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testSelectMultiCellUDTReference() throws Exception { - testSelectUDTReference("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person)"); + testSelectUDTReference("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testSelectFrozenUDTReference() throws Exception { - testSelectUDTReference("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen)"); + testSelectUDTReference("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen) WITH transactional_mode='" + transactionalMode + "'"); } private void testSelectUDTReference(String tableDDL) throws Exception @@ -2446,13 +2438,13 @@ public class AccordCQLTest extends AccordTestBase @Test public void testSelectMultiCellUDTFieldReference() throws Exception { - testSelectUDTFieldReference("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person)"); + testSelectUDTFieldReference("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer person) WITH transactional_mode='" + transactionalMode + "'"); } @Test public void testSelectFrozenUDTFieldReference() throws Exception { - testSelectUDTFieldReference("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen)"); + testSelectUDTFieldReference("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, customer frozen) WITH transactional_mode='" + transactionalMode + "'"); } private void testSelectUDTFieldReference(String tableDDL) throws Exception @@ -2483,7 +2475,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testMultiKeyQueryAndInsert() throws Throwable { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { String query1 = "BEGIN TRANSACTION\n" + @@ -2525,11 +2517,9 @@ public class AccordCQLTest extends AccordTestBase { SHARED_CLUSTER.schemaChange("DROP KEYSPACE IF EXISTS demo_ks;"); SHARED_CLUSTER.schemaChange("CREATE KEYSPACE demo_ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2};"); - SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.org_docs ( org_name text, doc_id int, contents_version int static, title text, permissions int, PRIMARY KEY (org_name, doc_id) );"); - SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.org_users ( org_name text, user text, members_version int static, permissions int, PRIMARY KEY (org_name, user) );"); - SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.user_docs ( user text, doc_id int, title text, org_name text, permissions int, PRIMARY KEY (user, doc_id) );"); - - SHARED_CLUSTER.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged("demo_ks")); + SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.org_docs ( org_name text, doc_id int, contents_version int static, title text, permissions int, PRIMARY KEY (org_name, doc_id) ) WITH transactional_mode='" + transactionalMode + "';"); + SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.org_users ( org_name text, user text, members_version int static, permissions int, PRIMARY KEY (org_name, user) ) WITH transactional_mode='" + transactionalMode + "';"); + SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.user_docs ( user text, doc_id int, title text, org_name text, permissions int, PRIMARY KEY (user, doc_id) ) WITH transactional_mode='" + transactionalMode + "';"); SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0))); @@ -2607,7 +2597,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testCASAndSerialRead() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (id int, c int, v int, s int static, PRIMARY KEY ((id), c));", + test("CREATE TABLE " + qualifiedTableName + " (id int, c int, v int, s int static, PRIMARY KEY ((id), c)) WITH transactional_mode='" + transactionalMode + "';", cluster -> { ICoordinator coordinator = cluster.coordinator(1); int startingAccordCoordinateCount = getAccordCoordinateCount(); @@ -2644,7 +2634,7 @@ public class AccordCQLTest extends AccordTestBase assertEquals(1, rangeDeletionCheck.length); // Make sure all the consensus using queries actually were run on Accord - if (nonSerialWriteStrategy.writesThroughAccord) + if (transactionalMode.writesThroughAccord) assertEquals( 20, getAccordCoordinateCount() - startingAccordCoordinateCount); else // Non-serial writes don't go through Accord in these modes @@ -2656,7 +2646,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testCASSimulatorLite() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (pk int, count int, seq1 text, seq2 list, PRIMARY KEY (pk))", + test("CREATE TABLE " + qualifiedTableName + " (pk int, count int, seq1 text, seq2 list, PRIMARY KEY (pk)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { ICoordinator coordinator = cluster.coordinator(1); coordinator.execute("INSERT INTO " + qualifiedTableName + " (pk, count, seq1, seq2) VALUES (1, 0, '', []) USING TIMESTAMP 0", ConsistencyLevel.ALL); @@ -2688,7 +2678,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testTransactionCasSimulatorLite() throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (pk int, count int, seq1 text, seq2 list, PRIMARY KEY (pk))", + test("CREATE TABLE " + qualifiedTableName + " (pk int, count int, seq1 text, seq2 list, PRIMARY KEY (pk)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { ICoordinator coordinator = cluster.coordinator(1); @@ -2729,7 +2719,7 @@ public class AccordCQLTest extends AccordTestBase @Test public void testSerialReadDescending() throws Throwable { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY(k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY(k, c)) WITH transactional_mode='" + transactionalMode + "'", cluster -> { ICoordinator coordinator = cluster.coordinator(1); for (int i = 1; i <= 10; i++) diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java index 827eebf6f9..133d2659a8 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java @@ -19,35 +19,16 @@ package org.apache.cassandra.distributed.test.accord; import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.junit.Test; -import org.apache.cassandra.db.virtual.AccordVirtualTables; -import org.apache.cassandra.db.virtual.SystemViewsKeyspace; -import org.apache.cassandra.db.virtual.VirtualTable; import org.apache.cassandra.distributed.Cluster; -import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.Feature; -import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.distributed.test.TestBaseImpl; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.service.accord.AccordService; -import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.AssertionUtils; import org.assertj.core.api.Assertions; -import static org.apache.cassandra.config.DatabaseDescriptor.NO_ACCORD_PAXOS_STRATEGY_WITH_ACCORD_DISABLED_MESSAGE; -import static org.apache.cassandra.cql3.statements.TransactionStatement.TRANSACTIONS_DISABLED_MESSAGE; -import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME; -import static org.junit.Assert.assertEquals; - public class AccordFeatureFlagTest extends TestBaseImpl { @Test @@ -58,46 +39,9 @@ public class AccordFeatureFlagTest extends TestBaseImpl .withConfig(c -> c.with(Feature.NETWORK).set("accord.enabled", "false")) .start())) { - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (k int, c int, v int, primary key (k, c))"); - - // Any transaction should fail to execute: - String query = "BEGIN TRANSACTION\n" + - " SELECT * FROM " + KEYSPACE + ".tbl WHERE k=0 AND c=0;\n" + - "COMMIT TRANSACTION"; - Assertions.assertThatThrownBy(() -> cluster.coordinator(1).executeWithResult(query, ConsistencyLevel.ANY)) - .has(AssertionUtils.isThrowableInstanceof(InvalidRequestException.class)) - .hasMessage(TRANSACTIONS_DISABLED_MESSAGE); - - // The Accord system keyspace should not be present: - assertEquals("The Accord system keyspace should not exist", - Optional.empty(), cluster.get(1).callOnInstance(() -> Schema.instance.localKeyspaces().get(ACCORD_KEYSPACE_NAME))); - - // Make sure virtual tables don't exist: - IIsolatedExecutor.SerializableCallable> hasAccordVirtualTables = - () -> SystemViewsKeyspace.instance.tables().stream().filter(t -> t.getClass().equals(AccordVirtualTables.Epoch.class)); - List tables = cluster.get(1).callOnInstance(hasAccordVirtualTables).collect(Collectors.toList()); - assertEquals("No Accord virtual tables should exist", Collections.emptyList(), tables); - - // Make sure we throw if someone tries to coordinate a transaction against the no-op service: - Assertions.assertThatThrownBy(() -> cluster.get(1).callOnInstance(() -> AccordService.instance().coordinate(null, null, Dispatcher.RequestTime.forImmediateExecution()))) - .isInstanceOf(UnsupportedOperationException.class); - } - } - - @SuppressWarnings("Convert2MethodRef") - @Test - public void shouldFailOnAccordMigrationWithAccordDisabled() throws IOException - { - try (Cluster cluster = Cluster.build(1) - .withoutVNodes() - .withConfig(c -> c.with(Feature.NETWORK) - .set("accord.enabled", "false") - .set("lwt_strategy", "accord")).createWithoutStarting()) - { - - Assertions.assertThatThrownBy(() -> cluster.startup()) - .has(AssertionUtils.isThrowableInstanceof(ConfigurationException.class)) - .hasMessage(NO_ACCORD_PAXOS_STRATEGY_WITH_ACCORD_DISABLED_MESSAGE); + Assertions.assertThatThrownBy(() -> cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'")) + .has(AssertionUtils.isThrowableInstanceof(InvalidRequestException.class)) + .hasMessageContaining("accord.enabled"); } } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteropReadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteropReadTest.java index 6022fdda55..72409dc871 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteropReadTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteropReadTest.java @@ -32,7 +32,6 @@ import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.distributed.util.QueryResultUtil; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; -import org.apache.cassandra.service.accord.AccordService; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; @@ -61,14 +60,10 @@ public class AccordInteropReadTest extends TestBaseImpl public void serialReadTest() throws Throwable { try (Cluster cluster = builder().withNodes(3) - .withConfig(config -> config.with(GOSSIP).with(NETWORK) - .set("non_serial_write_strategy", "mixed") - .set("lwt_strategy", "accord")) - .start()) + .withConfig(config -> config.with(GOSSIP).with(NETWORK)).start()) { cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':3}"); - cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, PRIMARY KEY (k, c))"); - cluster.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged("ks")); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode='unsafe_writes'"); cluster.get(1).runOnInstance(() -> localWrite("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 1)")); cluster.get(2).runOnInstance(() -> localWrite("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)")); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteroperabilityTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteroperabilityTest.java index 320a9f4e09..acef419889 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteroperabilityTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteroperabilityTest.java @@ -42,14 +42,13 @@ public class AccordInteroperabilityTest extends AccordTestBase @BeforeClass public static void setupClass() throws IOException { - AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("lwt_strategy", "accord") - .set("non_serial_write_strategy", "accord")), 3); + AccordTestBase.setupCluster(builder -> builder, 3); } @Test public void testSerialReadDescending() throws Throwable { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY(k, c))", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY(k, c)) WITH transactional_mode='full'", cluster -> { ICoordinator coordinator = cluster.coordinator(1); for (int i = 1; i <= 10; i++) diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java index 487fbf5f1d..235df7ebf7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.function.Function; import com.google.common.base.Throwables; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -89,14 +90,14 @@ public class AccordMetricsTest extends AccordTestBase public void beforeTest() { SHARED_CLUSTER.filters().reset(); - SHARED_CLUSTER.schemaChange("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c))"); - SHARED_CLUSTER.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, 0, 0)", ConsistencyLevel.ALL); + SHARED_CLUSTER.schemaChange("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH " + TransactionalMode.full.asCqlParam()); } @Test public void testRegularMetrics() throws Exception { countingMetrics0 = getMetrics(); + assertCoordinatorMetrics(0, "rw", 0, 0, 0, 0, 0); SHARED_CLUSTER.coordinator(1).executeWithResult(writeCql(), ConsistencyLevel.ALL, 0, 0, 0, 0); assertCoordinatorMetrics(0, "rw", 1, 0, 0, 0, 0); assertCoordinatorMetrics(1, "rw", 0, 0, 0, 0, 0); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java index f91cdecfaa..6cba8995ed 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java @@ -31,8 +31,10 @@ import java.util.function.Function; import javax.annotation.Nonnull; import com.google.common.collect.ImmutableList; + import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; @@ -58,21 +60,24 @@ import org.apache.cassandra.distributed.api.NodeToolResult; import org.apache.cassandra.gms.EndpointState; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState; import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationTarget; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget; +import org.apache.cassandra.service.consensus.migration.TableMigrationState; +import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.Ballot.Flag; import org.apache.cassandra.service.paxos.BallotGenerator; import org.apache.cassandra.service.paxos.Commit.Agreed; import org.apache.cassandra.service.paxos.Commit.Proposal; import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.ByteArrayUtil; @@ -86,6 +91,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.String.format; import static java.util.Collections.emptyList; import static org.apache.cassandra.Util.spinUntilSuccess; +import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; import static org.apache.cassandra.db.SystemKeyspace.CONSENSUS_MIGRATION_STATE; import static org.apache.cassandra.db.SystemKeyspace.PAXOS; import static org.apache.cassandra.dht.Range.normalize; @@ -96,8 +102,7 @@ import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME; import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.paxosV2; import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.PROMISE; import static org.assertj.core.api.Fail.fail; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /* * This test suite is intended to serve as an integration test with some pretty good visibility into actual execution @@ -146,11 +151,8 @@ public class AccordMigrationTest extends AccordTestBase ServerTestUtils.daemonInitialization(); // Otherwise repair complains if you don't specify a keyspace CassandraRelevantProperties.SYSTEM_TRACES_DEFAULT_RF.setInt(3); - AccordTestBase.setupCluster(builder -> - builder.appendConfig(config -> - config.set("paxos_variant", PaxosVariant.v2.name()) - .set("non_serial_write_strategy", "migration")), - 3); + AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("paxos_variant", PaxosVariant.v2.name()) + .set("accord.range_migration", "explicit")), 3); partitioner = FBUtilities.newPartitioner(SHARED_CLUSTER.get(1).callsOnInstance(() -> DatabaseDescriptor.getPartitioner().getClass().getSimpleName()).call()); StorageService.instance.setPartitionerUnsafe(partitioner); ServerTestUtils.prepareServerNoRegister(); @@ -177,9 +179,6 @@ public class AccordMigrationTest extends AccordTestBase ConsensusRequestRouter.resetInstance(); ConsensusKeyMigrationState.reset(); }); - SHARED_CLUSTER.get(1).runOnInstance(() -> { - ConsensusTableMigrationState.reset(); - }); SHARED_CLUSTER.coordinators().forEach(coordinator -> coordinator.execute(format("TRUNCATE TABLE %s.%s", SYSTEM_KEYSPACE_NAME, CONSENSUS_MIGRATION_STATE), ALL)); SHARED_CLUSTER.coordinators().forEach(coordinator -> coordinator.execute(format("TRUNCATE TABLE %s.%s", SYSTEM_KEYSPACE_NAME, PAXOS), ALL)); } @@ -243,10 +242,10 @@ public class AccordMigrationTest extends AccordTestBase boolean routed; @Override - protected ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull ColumnFamilyStore cfs, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + protected ConsensusRoutingDecision routeAndMaybeMigrate(ClusterMetadata cm, @Nonnull TableMetadata tmd, @Nonnull DecoratedKey key, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) { if (routed) - return super.routeAndMaybeMigrate(key, cfs, consistencyLevel, requestTime, timeoutNanos, isForWrite); + return super.routeAndMaybeMigrate(cm, tmd, key, consistencyLevel, requestTime, timeoutNanos, isForWrite); routed = true; return paxosV2; } @@ -279,10 +278,10 @@ public class AccordMigrationTest extends AccordTestBase boolean routed; @Override - protected ConsensusRoutingDecision routeAndMaybeMigrate(@Nonnull DecoratedKey key, @Nonnull ColumnFamilyStore cfs, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) + protected ConsensusRoutingDecision routeAndMaybeMigrate(ClusterMetadata cm, @Nonnull TableMetadata tmd, @Nonnull DecoratedKey key, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime, long timeoutNanos, boolean isForWrite) { if (routed) - return super.routeAndMaybeMigrate(key, cfs, consistencyLevel, requestTime, timeoutNanos, isForWrite); + return super.routeAndMaybeMigrate(cm, tmd, key, consistencyLevel, requestTime, timeoutNanos, isForWrite); routed = true; return ConsensusRoutingDecision.accord; } @@ -343,6 +342,21 @@ public class AccordMigrationTest extends AccordTestBase { test(format(TABLE_FMT, qualifiedTableName), cluster -> { + String table = tableName; + cluster.forEach(node -> node.runOnInstance(() -> { + TableMetadata tbl = Schema.instance.getTableMetadata(KEYSPACE, table); + Assert.assertEquals(TransactionalMode.off, tbl.params.transactionalMode); + Assert.assertEquals(TransactionalMigrationFromMode.none, tbl.params.transactionalMigrationFrom); + })); + + cluster.schemaChange(format("ALTER TABLE %s.%s WITH transactional_mode='%s'", KEYSPACE, tableName, TransactionalMode.full)); + + cluster.forEach(node -> node.runOnInstance(() -> { + TableMetadata tbl = Schema.instance.getTableMetadata(KEYSPACE, table); + Assert.assertEquals(TransactionalMode.full, tbl.params.transactionalMode); + Assert.assertEquals(TransactionalMigrationFromMode.off, tbl.params.transactionalMigrationFrom); + })); + String casCQL = format(CAS_FMT, qualifiedTableName, CLUSTERING_VALUE); Consumer runCasNoApply = key -> assertRowEquals(cluster, new Object[]{false}, casCQL, key); Consumer runCasApplies = key -> assertRowEquals(cluster, new Object[]{true}, casCQL, key); @@ -412,14 +426,18 @@ public class AccordMigrationTest extends AccordTestBase // key migration occurred assertTargetAccordWrite(runCasApplies, 1, migratingKey, 1, 0, 1, 0, 0); - // This will force the request to run on Paxos up to Accept - // and the accept will be rejected at both nodes and we are certain we need to retry the transaction + // This will force the write to use the normal write patch cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new PaxosToAccordMigrationNotHappeningUpToBegin())); // Update inserted row so the condition can apply, if the condition check doesn't apply // then it won't get to propose/accept migratingKey = testingKeys.next(); - Consumer makeCASApply = key -> cluster.coordinator(1).execute("UPDATE " + qualifiedTableName + " SET v = 42 WHERE id = ? AND c = ?", ALL, key, CLUSTERING_VALUE); + String query = "UPDATE " + qualifiedTableName + " SET v = 42 WHERE id = ? AND c = ?"; + Consumer makeCASApply = key -> cluster.forEach(instance -> instance.runOnInstance(() -> executeInternal(query, key, CLUSTERING_VALUE))); makeCASApply.accept(migratingKey); + + // This will force the request to run on Paxos up to Accept + // and the accept will be rejected at both nodes and we are certain we need to retry the transaction + cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new PaxosToAccordMigrationNotHappeningUpToBegin())); assertTargetAccordWrite(runCasApplies, 1, migratingKey, 1, 1, 1, 0, 1); // One node will now accept the other will reject and we are uncertain if we should retry the transaction @@ -429,7 +447,9 @@ public class AccordMigrationTest extends AccordTestBase cluster.get(1).runOnInstance(() -> ConsensusRequestRouter.setInstance(new PaxosToAccordMigrationNotHappeningUpToAccept())); try { + cluster.filters().allVerbs().to(3).from(3).drop(); runCasNoApply.accept(migratingKey); + cluster.filters().reset(); fail("Should have thrown timeout exception"); } catch (Throwable t) @@ -469,7 +489,7 @@ public class AccordMigrationTest extends AccordTestBase { test(format(TABLE_FMT, qualifiedTableName), cluster -> { - String tableName = qualifiedTableName.split("\\.")[1]; + cluster.schemaChange(format("ALTER TABLE %s.%s WITH transactional_mode='%s'", KEYSPACE, tableName, TransactionalMode.full)); String readCQL = format("SELECT * FROM %s WHERE id = ? and c = %s", qualifiedTableName, CLUSTERING_VALUE); Function runRead = key -> cluster.coordinator(1).execute(readCQL, SERIAL, key); Range migratingRange = new Range<>(new LongToken(Long.MIN_VALUE + 1), new LongToken(Long.MIN_VALUE)); @@ -490,6 +510,25 @@ public class AccordMigrationTest extends AccordTestBase }); } + private void alterTableTransactionalMode(TransactionalMode mode) + { + SHARED_CLUSTER.schemaChange(format("ALTER TABLE %s WITH %s", qualifiedTableName, mode.asCqlParam())); + } + + private void assertTransactionalModes(String keyspace, String table, TransactionalMode mode, TransactionalMigrationFromMode migration) + { + forEach(() -> { + TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table); + Assert.assertEquals(mode, metadata.params.transactionalMode); + Assert.assertEquals(migration, metadata.params.transactionalMigrationFrom); + }); + } + + private void assertTransactionalModes(TransactionalMode mode, TransactionalMigrationFromMode migration) + { + assertTransactionalModes(KEYSPACE, tableName, mode, migration); + } + @Test public void testAccordToPaxos() throws Exception { @@ -499,6 +538,9 @@ public class AccordMigrationTest extends AccordTestBase Consumer runCasNoApply = key -> assertRowEquals(cluster, new Object[]{false}, casCQL, key); String tableName = qualifiedTableName.split("\\.")[1]; + alterTableTransactionalMode(TransactionalMode.mixed_reads); + assertTransactionalModes(TransactionalMode.mixed_reads, TransactionalMigrationFromMode.off); + // Mark a subrange as migrating and finish migrating half of it nodetool(coordinator, "consensus_admin", "begin-migration", "-st", midToken.toString(), "-et", maxToken.toString(), "-tp", "accord", KEYSPACE, tableName); nodetool(coordinator, "consensus_admin", "finish-migration", "-st", midToken.toString(), "-et", "3074457345618258601"); @@ -508,7 +550,8 @@ public class AccordMigrationTest extends AccordTestBase assertMigrationState(tableName, ConsensusMigrationTarget.accord, ImmutableList.of(accordMigratedRange), ImmutableList.of(accordMigratingRange), 1); // Test that we can reverse the migration and go back to Paxos - nodetool(coordinator, "consensus_admin", "set-target-protocol", "-tp", "paxos", KEYSPACE, tableName); + alterTableTransactionalMode(TransactionalMode.off); + assertTransactionalModes(TransactionalMode.off, TransactionalMigrationFromMode.mixed_reads); assertMigrationState(tableName, ConsensusMigrationTarget.paxos, ImmutableList.of(new Range(minToken, midToken), new Range(maxToken, minToken)), ImmutableList.of(accordMigratingRange), 1); Iterator paxosNonMigratingKeys = getKeysBetweenTokens(minToken, midToken); Iterator paxosMigratingKeys = getKeysBetweenTokens(upperMidToken, maxToken); @@ -547,6 +590,38 @@ public class AccordMigrationTest extends AccordTestBase }); } + private static void assertCompletedMigrationState(String tableName) throws Throwable + { + // Validate nodetool consensus admin list output + String yamlResultString = nodetool(SHARED_CLUSTER.coordinator(1), "consensus_admin", "list"); + Map yamlStateMap = new Yaml().load(yamlResultString); + String minifiedYamlResultString = nodetool(SHARED_CLUSTER.coordinator(1), "consensus_admin", "list", "-f", "minified-yaml"); + Map minifiedYamlStateMap = new Yaml().load(minifiedYamlResultString); + String jsonResultString = nodetool(SHARED_CLUSTER.coordinator(1), "consensus_admin", "list", "-f", "json"); + Map jsonStateMap = JsonUtils.JSON_OBJECT_MAPPER.readValue(jsonResultString, new TypeReference>(){}); + String minifiedJsonResultString = nodetool(SHARED_CLUSTER.coordinator(1), "consensus_admin", "list", "-f", "minified-json"); + Map minifiedJsonStateMap = JsonUtils.JSON_OBJECT_MAPPER.readValue(minifiedJsonResultString, new TypeReference>(){}); + + for (Map migrationStateMap : ImmutableList.of(yamlStateMap, jsonStateMap, minifiedYamlStateMap, minifiedJsonStateMap)) { + assertEquals(PojoToString.CURRENT_VERSION, migrationStateMap.get("version")); + assertTrue(Epoch.EMPTY.getEpoch() < ((Number) migrationStateMap.get("lastModifiedEpoch")).longValue()); + List> tableStates = (List>) migrationStateMap.get("tableStates"); + assertEquals(0, tableStates.size()); + } + + spinUntilSuccess(() -> { + for (IInvokableInstance instance : SHARED_CLUSTER) + { + ConsensusMigrationState snapshot = getMigrationStateSnapshot(instance); + assertEquals(0, snapshot.tableStates.size()); + instance.runOnInstance(() -> { + TableMetadata tbl = Schema.instance.getTableMetadata(KEYSPACE, tableName); + Assert.assertEquals(TransactionalMigrationFromMode.none, tbl.params.transactionalMigrationFrom); + }); + } + }); + } + private static void assertMigrationState(String tableName, ConsensusMigrationTarget target, List> migratedRanges, List> migratingRanges, int numMigratingEpochs) throws Throwable { // Validate nodetool consensus admin list output @@ -564,9 +639,20 @@ public class AccordMigrationTest extends AccordTestBase { assertEquals(PojoToString.CURRENT_VERSION, migrationStateMap.get("version")); assertTrue(Epoch.EMPTY.getEpoch() < ((Number) migrationStateMap.get("lastModifiedEpoch")).longValue()); - List> tableStates = (List>) migrationStateMap.get("tableStates"); - assertEquals(tableStates.size(), 1); - Map tableStateMap = tableStates.get(0); + + Map tableStateMap = null; + for (Map stateMap : (List>) migrationStateMap.get("tableStates")) + { + Object table = stateMap.get("table"); + Object keyspace = stateMap.get("keyspace"); + if (KEYSPACE.equals(keyspace) && tableName.equals(table)) + { + tableStateMap = stateMap; + break; + } + } + assertNotNull(tableStateMap); + assertEquals(tableName, tableStateMap.get("table")); assertEquals(KEYSPACE, tableStateMap.get("keyspace")); tableIds.add((String) tableStateMap.get("tableId")); @@ -591,21 +677,22 @@ public class AccordMigrationTest extends AccordTestBase for (IInvokableInstance instance : SHARED_CLUSTER) { ConsensusMigrationState snapshot = getMigrationStateSnapshot(instance); - assertEquals(1, snapshot.tableStates.size()); - TableMigrationState state = snapshot.tableStates.values().iterator().next(); - assertEquals(KEYSPACE, state.keyspaceName); - assertEquals(tableName, state.tableName); for (String tableId : tableIds) - assertEquals(tableId, state.tableId.toString()); - assertEquals(target, state.targetProtocol); - assertEquals("Migrated ranges:", migratedRanges, state.migratedRanges); - assertEquals("Migrating ranges:", migratingRanges, state.migratingRanges); - assertEquals("Migrating and migrated ranges:", migratingAndMigratedRanges, state.migratingAndMigratedRanges); - assertEquals(numMigratingEpochs, state.migratingRangesByEpoch.size()); - if (migratingRanges.isEmpty()) - assertEquals(0, state.migratingRangesByEpoch.size()); - else - assertEquals(migratingRanges, state.migratingRangesByEpoch.values().iterator().next()); + { + TableMigrationState state = snapshot.tableStates.get(TableId.fromString(tableId)); + assertNotNull(state); + assertEquals(KEYSPACE, state.keyspaceName); + assertEquals(tableName, state.tableName); + assertEquals(target, state.targetProtocol); + assertEquals("Migrated ranges:", migratedRanges, state.migratedRanges); + assertEquals("Migrating ranges:", migratingRanges, state.migratingRanges); + assertEquals("Migrating and migrated ranges:", migratingAndMigratedRanges, state.migratingAndMigratedRanges); + assertEquals(numMigratingEpochs, state.migratingRangesByEpoch.size()); + if (migratingRanges.isEmpty()) + assertEquals(0, state.migratingRangesByEpoch.size()); + else + assertEquals(migratingRanges, state.migratingRangesByEpoch.values().iterator().next()); + } } }); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordSimpleFastPathTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordSimpleFastPathTest.java index 19d562a21c..abebaeb49d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordSimpleFastPathTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordSimpleFastPathTest.java @@ -27,6 +27,7 @@ import org.apache.cassandra.service.accord.AccordFastPath; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.junit.Assert; import org.junit.Test; @@ -90,7 +91,7 @@ public class AccordSimpleFastPathTest extends TestBaseImpl .start())) { cluster.schemaChange("CREATE KEYSPACE ks WITH replication={'class':'SimpleStrategy', 'replication_factor': 3}"); - cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key (k, c))"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key (k, c)) WITH " + TransactionalMode.full.asCqlParam()); String query = "BEGIN TRANSACTION\n" + " SELECT * FROM ks.tbl WHERE k=0 AND c=0;\n" + "COMMIT TRANSACTION"; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java index f5937f643b..4b0a90b173 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java @@ -37,6 +37,7 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,7 +71,7 @@ import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTestUtils; import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException; import org.apache.cassandra.service.accord.exceptions.WritePreemptedException; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.AssertionUtils; @@ -137,11 +138,10 @@ public abstract class AccordTestBase extends TestBaseImpl public static void ensureTableIsAccordManaged(Cluster cluster, String ksname, String tableName) { cluster.get(1).runOnInstance(() -> { - // TODO: remove when accord enabled is handled via schema TableMetadata metadata = Schema.instance.getTableMetadata(ksname, tableName); if (metadata == null) return; // bad plumbing from shared utils.... - AccordService.instance().ensureTableIsAccordManaged(metadata.id); + Assert.assertTrue(metadata.params.transactionalMode.accordIsEnabled); }); } @@ -150,7 +150,6 @@ public abstract class AccordTestBase extends TestBaseImpl for (String ddl : ddls) SHARED_CLUSTER.schemaChange(ddl); - ensureTableIsAccordManaged(SHARED_CLUSTER, KEYSPACE, tableName); // Evict commands from the cache immediately to expose problems loading from disk. SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0))); @@ -166,7 +165,7 @@ public abstract class AccordTestBase extends TestBaseImpl protected void test(FailingConsumer fn) throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c))", fn); + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", fn); } protected static ConsensusMigrationState getMigrationStateSnapshot(IInvokableInstance instance) throws IOException diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java index 084709e496..82c9e2f806 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java @@ -56,7 +56,7 @@ public class NewSchemaTest extends AccordTestBase String ks = "ks" + i; String table = ks + ".tbl" + i; SHARED_CLUSTER.schemaChange("CREATE KEYSPACE " + ks + " WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}"); - SHARED_CLUSTER.schemaChange(String.format("CREATE TABLE %s (pk blob primary key)", table)); + SHARED_CLUSTER.schemaChange(String.format("CREATE TABLE %s (pk blob primary key) WITH transactional_mode='full'", table)); SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0))); List keys = tokensToKeys(tokens()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java index 381ea8c6be..907a0e4375 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java @@ -69,7 +69,6 @@ import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.membership.NodeVersion; -import org.apache.cassandra.tcm.ownership.AccordTables; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.TokenMap; import org.apache.cassandra.tcm.ownership.UniformRangePlacement; @@ -150,7 +149,6 @@ public class ClusterMetadataTestHelper Directory.EMPTY, new TokenMap(partitioner), DataPlacements.empty(), - AccordTables.EMPTY, AccordFastPath.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, @@ -166,7 +164,6 @@ public class ClusterMetadataTestHelper null, null, DataPlacements.empty(), - AccordTables.EMPTY, AccordFastPath.EMPTY, null, null, @@ -182,7 +179,6 @@ public class ClusterMetadataTestHelper null, null, DataPlacements.empty(), - AccordTables.EMPTY, AccordFastPath.EMPTY, null, null, diff --git a/test/distributed/org/apache/cassandra/distributed/test/tcm/AccordAddTableTest.java b/test/distributed/org/apache/cassandra/distributed/test/tcm/AccordAddTableTest.java deleted file mode 100644 index c486da8f54..0000000000 --- a/test/distributed/org/apache/cassandra/distributed/test/tcm/AccordAddTableTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.distributed.test.tcm; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.Future; - -import org.junit.Test; - -import accord.primitives.Ranges; -import accord.primitives.Txn; -import org.apache.cassandra.distributed.Cluster; -import org.apache.cassandra.distributed.api.Feature; -import org.apache.cassandra.distributed.api.IInvokableInstance; -import org.apache.cassandra.distributed.test.TestBaseImpl; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.accord.AccordService; -import org.apache.cassandra.service.accord.TokenRange; -import org.apache.cassandra.service.accord.api.AccordRoutingKey; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.utils.FBUtilities; - -public class AccordAddTableTest extends TestBaseImpl -{ - @Test - public void test() throws IOException - { - try (Cluster cluster = builder().withNodes(6) - .withConfig(c -> c.with(Feature.GOSSIP, Feature.NETWORK)) - .start()) - { - List> results = new ArrayList<>(cluster.size()); - for (IInvokableInstance inst : cluster) - { - Future result = inst.asyncRunsOnInstance(() -> { - for (int i = 0; i < 100; i++) - { - AccordService.instance().maybeConvertTablesToAccord(fakeTxn(i)); - if (!ClusterMetadata.current().accordTables.contains(fromNum(i))) - throw new AssertionError("Table not found in TCM!"); - } - }).call(); - results.add(result); - } - FBUtilities.waitOnFutures(results); - } - } - - private static Txn fakeTxn(int i) - { - TableId id = fromNum(i); - - Ranges of = Ranges.of(new TokenRange(AccordRoutingKey.SentinelKey.min(id), AccordRoutingKey.SentinelKey.max(id))); - return new Txn.InMemory(of, null, null); - } - - private static TableId fromNum(int i) - { - return TableId.fromUUID(new UUID(i, 0)); // not valid... but do we care? - } -} diff --git a/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java b/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java index 94bc2d937b..ee0ad12bcf 100644 --- a/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java +++ b/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java @@ -442,7 +442,7 @@ public class AuditLoggerTest extends CQLTester @Test public void testTransactionAuditing() { - createTable("CREATE TABLE %s (key int PRIMARY KEY, val int)"); + createTable("CREATE TABLE %s (key int PRIMARY KEY, val int) WITH transactional_mode='full'"); Session session = sessionNet(); String fqTableName = KEYSPACE + "." + currentTable(); diff --git a/test/unit/org/apache/cassandra/auth/TxnAuthTest.java b/test/unit/org/apache/cassandra/auth/TxnAuthTest.java index d60c3cadc3..e26e025951 100644 --- a/test/unit/org/apache/cassandra/auth/TxnAuthTest.java +++ b/test/unit/org/apache/cassandra/auth/TxnAuthTest.java @@ -66,7 +66,7 @@ public class TxnAuthTest extends CQLTester @Before public void setUpTest() { - createTable("CREATE TABLE %s (k int, v int, PRIMARY KEY(k))"); + createTable("CREATE TABLE %s (k int, v int, PRIMARY KEY(k)) WITH transactional_mode='full'"); } @Test diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index eb10476ace..c580207634 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -78,6 +78,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.auth.INetworkAuthorizer", "org.apache.cassandra.auth.IRoleManager", "org.apache.cassandra.config.AccordSpec", + "org.apache.cassandra.config.AccordSpec$TransactionalRangeMigration", "org.apache.cassandra.config.CassandraRelevantProperties", "org.apache.cassandra.config.CassandraRelevantProperties$PropertyConverter", "org.apache.cassandra.config.Config", @@ -99,6 +100,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.Config$PaxosVariant", "org.apache.cassandra.config.Config$RepairCommandPoolFullStrategy", "org.apache.cassandra.config.Config$SSTableConfig", + "org.apache.cassandra.config.Config$TransactionalRangeMigration", "org.apache.cassandra.config.Config$TriggersPolicy", "org.apache.cassandra.config.Config$UserFunctionTimeoutPolicy", "org.apache.cassandra.config.ConfigBeanInfo", @@ -292,6 +294,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.service.CacheService$CacheType", "org.apache.cassandra.security.AbstractCryptoProvider", "org.apache.cassandra.tcm.RegistrationStateCallbacks", + "org.apache.cassandra.service.consensus.TransactionalMode", "org.apache.cassandra.transport.ProtocolException", "org.apache.cassandra.utils.Closeable", "org.apache.cassandra.utils.CloseableIterator", diff --git a/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java b/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java index 360b92ded5..be9fcb6136 100644 --- a/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java +++ b/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java @@ -93,7 +93,7 @@ public class NodeLocalConsistencyTest extends CQLTester @Test public void testTransaction() { - createTable("CREATE TABLE %s (key text, val int, PRIMARY KEY(key))"); + createTable("CREATE TABLE %s (key text, val int, PRIMARY KEY(key)) WITH transactional_mode='full'"); QueryProcessor.process(formatQuery("INSERT INTO %s (key, val) VALUES ('foo', 0)"), NODE_LOCAL); String query = "BEGIN TRANSACTION\n" + diff --git a/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java b/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java index 8819a1c7c6..70e49bc7fe 100644 --- a/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java +++ b/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java @@ -26,6 +26,9 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; @@ -76,6 +79,27 @@ public class PreparedStatementsTest extends CQLTester public void setup() { requireNetwork(); + for (int i=0; i<10; i++) + ClusterMetadataService.instance().log().waitForHighestConsecutive(); + } + + private static void runAndAwaitNextEpoch(Runnable runnable) + { + try + { + Epoch current = ClusterMetadata.current().epoch; + runnable.run(); + ClusterMetadataService.instance().awaitAtLeast(Epoch.create(current.getEpoch() + 1)); + } + catch (Throwable e) + { + throw new RuntimeException(e); + } + } + + private static void sessionSchemaUpdate(Session session, String update) + { + runAndAwaitNextEpoch(() -> session.execute(update)); } @Test @@ -173,30 +197,30 @@ public class PreparedStatementsTest extends CQLTester public void testInvalidatePreparedStatementsOnDrop() { Session session = sessionNet(ProtocolVersion.V5); - session.execute(dropKsStatement); - session.execute(createKsStatement); + sessionSchemaUpdate(session, dropKsStatement); + sessionSchemaUpdate(session, createKsStatement); - String createTableStatement = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_cleanup (id int PRIMARY KEY, cid int, val text);"; + String createTableStatement = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_cleanup (id int PRIMARY KEY, cid int, val text) WITH transactional_mode='unsafe';"; String dropTableStatement = "DROP TABLE IF EXISTS " + KEYSPACE + ".qp_cleanup;"; - session.execute(createTableStatement); + sessionSchemaUpdate(session, createTableStatement); String insert = "INSERT INTO " + KEYSPACE + ".qp_cleanup (id, cid, val) VALUES (?, ?, ?)"; PreparedStatement prepared = session.prepare(insert); PreparedStatement preparedBatch = session.prepare(batch(insert)); PreparedStatement preparedTxn = session.prepare(txn(insert)); - session.execute(dropTableStatement); - session.execute(createTableStatement); + sessionSchemaUpdate(session, dropTableStatement); + sessionSchemaUpdate(session, createTableStatement); updateTxnState(); session.execute(prepared.bind(1, 1, "value")); session.execute(preparedBatch.bind(2, 2, "value2")); session.execute(preparedTxn.bind(3, 3, "value3")); - session.execute(dropKsStatement); - session.execute(createKsStatement); - session.execute(createTableStatement); + sessionSchemaUpdate(session, dropKsStatement); + sessionSchemaUpdate(session, createKsStatement); + sessionSchemaUpdate(session, createTableStatement); updateTxnState(); // The driver will get a response about the prepared statement being invalid, causing it to transparently @@ -205,7 +229,7 @@ public class PreparedStatementsTest extends CQLTester session.execute(prepared.bind(1, 1, "value")); session.execute(preparedBatch.bind(2, 2, "value2")); session.execute(preparedTxn.bind(3, 3, "value3")); - session.execute(dropKsStatement); + sessionSchemaUpdate(session, dropKsStatement); } @Test @@ -223,12 +247,12 @@ public class PreparedStatementsTest extends CQLTester private void testInvalidatePreparedStatementOnAlter(ProtocolVersion version, boolean supportsMetadataChange) { Session session = sessionNet(version); - String createTableStatement = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_cleanup (a int PRIMARY KEY, b int, c int);"; + String createTableStatement = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_cleanup (a int PRIMARY KEY, b int, c int) WITH transactional_mode='unsafe';"; String alterTableStatement = "ALTER TABLE " + KEYSPACE + ".qp_cleanup ADD d int;"; - session.execute(dropKsStatement); - session.execute(createKsStatement); - session.execute(createTableStatement); + sessionSchemaUpdate(session, dropKsStatement); + sessionSchemaUpdate(session, createKsStatement); + sessionSchemaUpdate(session, createTableStatement); updateTxnState(); String select = "SELECT * FROM " + KEYSPACE + ".qp_cleanup"; @@ -247,7 +271,7 @@ public class PreparedStatementsTest extends CQLTester assertRowsNet(session.execute(preparedSelectTxn.bind(2)), row(2, 3, 4)); - session.execute(alterTableStatement); + sessionSchemaUpdate(session, alterTableStatement); updateTxnState(); session.execute("INSERT INTO " + KEYSPACE + ".qp_cleanup (a, b, c, d) VALUES (?, ?, ?, ?);", @@ -291,7 +315,7 @@ public class PreparedStatementsTest extends CQLTester } } - session.execute(dropKsStatement); + sessionSchemaUpdate(session, dropKsStatement); } @Test @@ -309,12 +333,12 @@ public class PreparedStatementsTest extends CQLTester private void testInvalidatePreparedStatementOnAlterUnchangedMetadata(ProtocolVersion version) { Session session = sessionNet(version); - String createTableStatement = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_cleanup (a int PRIMARY KEY, b int, c int);"; + String createTableStatement = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".qp_cleanup (a int PRIMARY KEY, b int, c int) WITH transactional_mode='unsafe';"; String alterTableStatement = "ALTER TABLE " + KEYSPACE + ".qp_cleanup ADD d int;"; - session.execute(dropKsStatement); - session.execute(createKsStatement); - session.execute(createTableStatement); + sessionSchemaUpdate(session, dropKsStatement); + sessionSchemaUpdate(session, createKsStatement); + sessionSchemaUpdate(session, createTableStatement); updateTxnState(); String select = "SELECT a, b, c FROM " + KEYSPACE + ".qp_cleanup"; @@ -338,7 +362,7 @@ public class PreparedStatementsTest extends CQLTester Assertions.assertThat(columnNames(rs)).containsExactlyInAnyOrder("a", "b", "c"); } - session.execute(alterTableStatement); + sessionSchemaUpdate(session, alterTableStatement); updateTxnState(); session.execute("INSERT INTO " + KEYSPACE + ".qp_cleanup (a, b, c, d) VALUES (?, ?, ?, ?);", @@ -358,18 +382,18 @@ public class PreparedStatementsTest extends CQLTester Assertions.assertThat(columnNames(rs)).containsExactlyInAnyOrder("a", "b", "c"); } - session.execute(dropKsStatement); + sessionSchemaUpdate(session, dropKsStatement); } @Test - public void testStatementRePreparationOnReconnect() + public void testStatementRePreparationOnReconnect() throws Throwable { Session session = sessionNet(ProtocolVersion.V5); session.execute("USE " + keyspace()); - session.execute(dropKsStatement); - session.execute(createKsStatement); - createTable("CREATE TABLE %s (id int PRIMARY KEY, cid int, val text);"); + sessionSchemaUpdate(session, dropKsStatement); + sessionSchemaUpdate(session, createKsStatement); + runAndAwaitNextEpoch(() -> createTable("CREATE TABLE %s (id int PRIMARY KEY, cid int, val text) WITH transactional_mode='unsafe';")); updateTxnState(); String insertCQL = "INSERT INTO " + currentTable() + " (id, cid, val) VALUES (?, ?, ?)"; @@ -414,14 +438,14 @@ public class PreparedStatementsTest extends CQLTester { Session session = sessionNet(ProtocolVersion.V5); - session.execute(dropKsStatement); - session.execute(createKsStatement); + sessionSchemaUpdate(session, dropKsStatement); + sessionSchemaUpdate(session, createKsStatement); String table = "custom_expr_test"; String index = "custom_index"; - session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id int PRIMARY KEY, cid int, val text);", + sessionSchemaUpdate(session, String.format("CREATE TABLE IF NOT EXISTS %s.%s (id int PRIMARY KEY, cid int, val text) WITH transactional_mode='unsafe';", KEYSPACE, table)); - session.execute(String.format("CREATE CUSTOM INDEX %s ON %s.%s(val) USING '%s'", + sessionSchemaUpdate(session, String.format("CREATE CUSTOM INDEX %s ON %s.%s(val) USING '%s'", index, KEYSPACE, table, StubIndex.class.getName())); updateTxnState(); @@ -460,7 +484,7 @@ public class PreparedStatementsTest extends CQLTester // Note: this test does not cover all aspects of 10786 (yet) - it was intended to test the // changes for CASSANDRA-13992. - createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))"); + runAndAwaitNextEpoch(() -> createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))")); execute("INSERT INTO %s (pk, v1, v2) VALUES (1,1,1)"); try (SimpleClient simpleClient = newSimpleClient(ProtocolVersion.BETA.orElse(ProtocolVersion.CURRENT))) @@ -646,7 +670,7 @@ public class PreparedStatementsTest extends CQLTester { Session session = sessionNet(version); session.execute("USE " + keyspace()); - createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))"); + runAndAwaitNextEpoch(() -> createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))")); PreparedStatement prepared1 = session.prepare(String.format("UPDATE %s SET v1 = ?, v2 = ? WHERE pk = 1 IF v1 = ?", currentTable())); PreparedStatement prepared2 = session.prepare(String.format("INSERT INTO %s (pk, v1, v2) VALUES (?, 200, 300) IF NOT EXISTS", currentTable())); @@ -710,7 +734,7 @@ public class PreparedStatementsTest extends CQLTester { Session session = sessionNet(version); session.execute("USE " + keyspace()); - createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))"); + runAndAwaitNextEpoch(() -> createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))")); PreparedStatement prepared1 = session.prepare("BEGIN BATCH " + "UPDATE " + currentTable() + " SET v1 = ? WHERE pk = 1 IF v1 = ?;" + @@ -745,7 +769,7 @@ public class PreparedStatementsTest extends CQLTester row(false, 1, 10, 20)); assertEquals(rs.getColumnDefinitions().size(), 4); - alterTable("ALTER TABLE %s ADD v3 int;"); + runAndAwaitNextEpoch(() -> alterTable("ALTER TABLE %s ADD v3 int;")); rs = session.execute(prepared2.bind()); assertRowsNet(rs, @@ -777,7 +801,7 @@ public class PreparedStatementsTest extends CQLTester int maxAttempts = 3; Session session = sessionNet(version); session.execute("USE " + keyspace()); - createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk))"); + runAndAwaitNextEpoch(() -> createTable("CREATE TABLE %s (pk int, v1 int, v2 int, PRIMARY KEY (pk)) WITH transactional_mode='full'")); updateTxnState(); PreparedStatement writeOnly = session.prepare(txn( diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index ccf3d9a288..8529d8ff80 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -1141,6 +1141,8 @@ public class DescribeStatementTest extends CQLTester " AND memtable_flush_period_in_ms = 0\n" + " AND min_index_interval = 128\n" + " AND read_repair = 'BLOCKING'\n" + + " AND transactional_mode = 'off'\n" + + " AND transactional_migration_from = 'none'\n" + " AND speculative_retry = '99p';"; } diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index d0d2295e42..3e642db7ff 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -347,6 +347,8 @@ public class SchemaCQLHelperTest extends CQLTester " AND memtable_flush_period_in_ms = 8\n" + " AND min_index_interval = 6\n" + " AND read_repair = 'BLOCKING'\n" + + " AND transactional_mode = 'off'\n" + + " AND transactional_migration_from = 'none'\n" + " AND speculative_retry = 'ALWAYS';" )); } diff --git a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java index 63f91b6729..2b22b0be3b 100644 --- a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java @@ -27,14 +27,13 @@ import java.util.Set; import com.google.common.collect.ImmutableMap; import org.apache.cassandra.service.accord.AccordFastPath; -import org.apache.cassandra.tcm.ownership.AccordTables; +import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.junit.Assert; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.schema.DistributedSchema; -import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.Directory; @@ -90,7 +89,6 @@ public class MetaStrategyTest directory, tokenMap, DataPlacements.EMPTY, - AccordTables.EMPTY, AccordFastPath.EMPTY, LockedRanges.EMPTY, InProgressSequences.EMPTY, diff --git a/test/unit/org/apache/cassandra/repair/FuzzTestBase.java b/test/unit/org/apache/cassandra/repair/FuzzTestBase.java index 7dcf4c6404..b9c7fe6b59 100644 --- a/test/unit/org/apache/cassandra/repair/FuzzTestBase.java +++ b/test/unit/org/apache/cassandra/repair/FuzzTestBase.java @@ -587,7 +587,9 @@ public abstract class FuzzTestBase extends CQLTester.InMemory { RepairType type = repairTypeGen.next(rs); PreviewType previewType = previewTypeGen.next(rs); - boolean accordRepair = type == RepairType.FULL && previewType == PreviewType.NONE ? rs.nextBoolean() : false; + // TODO (required - IR) add this back and expand as part of IR integration +// boolean accordRepair = type == RepairType.FULL && previewType == PreviewType.NONE ? rs.nextBoolean() : false; + boolean accordRepair = false; List args = new ArrayList<>(); args.add(ks); List tables = tablesGen.next(rs); @@ -1407,8 +1409,6 @@ public abstract class FuzzTestBase extends CQLTester.InMemory failures.add(new AssertionError(event.getMessage())); }); } - if (repair.state.options.accordRepair()) - AccordService.instance().ensureKeyspaceIsAccordManaged(repair.state.keyspace); return repair; } diff --git a/test/unit/org/apache/cassandra/schema/FastPathSchemaTest.java b/test/unit/org/apache/cassandra/schema/FastPathSchemaTest.java index 1a2dc1132d..e4c6030798 100644 --- a/test/unit/org/apache/cassandra/schema/FastPathSchemaTest.java +++ b/test/unit/org/apache/cassandra/schema/FastPathSchemaTest.java @@ -32,9 +32,6 @@ import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.service.accord.fastpath.FastPathStrategy; import org.apache.cassandra.service.accord.fastpath.ParameterizedFastPathStrategy; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.tcm.transformations.AddAccordTable; import static java.lang.String.format; @@ -70,14 +67,9 @@ public class FastPathSchemaTest KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE); Assert.assertSame(FastPathStrategy.simple(), ksm.params.fastPath); - process("CREATE TABLE %s.tbl (k int primary key, v int)", KEYSPACE); + process("CREATE TABLE %s.tbl (k int primary key, v int) WITH transactional_mode='full'", KEYSPACE); TableMetadata tbm = Schema.instance.getTableMetadata(KEYSPACE, "tbl"); Assert.assertSame(FastPathStrategy.inheritKeyspace(), tbm.params.fastPath); - - Epoch epoch = ClusterMetadata.current().epoch; - AddAccordTable.addTable(tbm.id); - - Assert.assertEquals(epoch.getEpoch() + 1, ClusterMetadata.current().epoch.getEpoch()); } @Test @@ -108,12 +100,10 @@ public class FastPathSchemaTest KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE); Assert.assertSame(FastPathStrategy.simple(), ksm.params.fastPath); - process("CREATE TABLE %s.tbl (k int primary key, v int)", KEYSPACE); + process("CREATE TABLE %s.tbl (k int primary key, v int) WITH transactional_mode='full'", KEYSPACE); TableMetadata tbm = Schema.instance.getTableMetadata(KEYSPACE, "tbl"); Assert.assertSame(FastPathStrategy.inheritKeyspace(), tbm.params.fastPath); - AddAccordTable.addTable(tbm.id); - process("ALTER TABLE %s.tbl WITH fast_path='simple'", KEYSPACE); tbm = Schema.instance.getTableMetadata(KEYSPACE, "tbl"); Assert.assertSame(FastPathStrategy.simple(), tbm.params.fastPath); diff --git a/test/unit/org/apache/cassandra/schema/TransactionalConfigSchemaTest.java b/test/unit/org/apache/cassandra/schema/TransactionalConfigSchemaTest.java new file mode 100644 index 0000000000..59c03fc3c7 --- /dev/null +++ b/test/unit/org/apache/cassandra/schema/TransactionalConfigSchemaTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.schema; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.service.consensus.TransactionalMode; +import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; + +import static java.lang.String.format; + +public class TransactionalConfigSchemaTest +{ + private static final String KEYSPACE = "ks"; + @BeforeClass + public static void setup() + { + DatabaseDescriptor.daemonInitialization(); + ServerTestUtils.prepareServer(); + SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create(KEYSPACE, KeyspaceParams.simple(1), Tables.of())); + } + + private static void process(String fmt, Object... objects) + { + QueryProcessor.process(format(fmt, objects), ConsistencyLevel.ANY); + } + + private static void assertTransactionalMode(String table, TransactionalMode mode, TransactionalMigrationFromMode migration) + { + TableMetadata metadata = Schema.instance.getTableMetadata(KEYSPACE, table); + Assert.assertEquals(mode, metadata.params.transactionalMode); + Assert.assertEquals(migration, metadata.params.transactionalMigrationFrom); + } + + // if a table is created with an accord transactional mode, it skips having to migrate + @Test + public void newTableSkipsMigration() + { + String table = "new_table"; + process("CREATE TABLE ks.%s (k int primary key, v int) WITH transactional_mode='%s'", table, TransactionalMode.full); + assertTransactionalMode(table, TransactionalMode.full, TransactionalMigrationFromMode.none); + } + + // if an existing table is set to an accord transactional mode, it should be set to migrating + @Test + public void existingTableMigration() + { + String table = "existing_table"; + process("CREATE TABLE ks.%s (k int primary key, v int)", table); + assertTransactionalMode(table, TransactionalMode.off, TransactionalMigrationFromMode.none); + + process("ALTER TABLE ks.%s WITH transactional_mode='%s'", table, TransactionalMode.full); + assertTransactionalMode(table, TransactionalMode.full, TransactionalMigrationFromMode.off); + } + + // changing transactional mode with an incomplete migration should fail, unless the migration mode is explicitly updated + @Test + public void incompleteMigrationFailure() + { + String table = "incomplete_table"; + process("CREATE TABLE ks.%s (k int primary key, v int)", table); + process("ALTER TABLE ks.%s WITH transactional_mode='%s'", table, TransactionalMode.full); + assertTransactionalMode(table, TransactionalMode.full, TransactionalMigrationFromMode.off); + + process("ALTER TABLE ks.%s WITH transactional_mode='%s'", table, TransactionalMode.off); + assertTransactionalMode(table, TransactionalMode.off, TransactionalMigrationFromMode.full); + + // explicitly setting the migration mode should work + process("ALTER TABLE ks.%s WITH transactional_mode='%s' AND transactional_migration_from='%s'", + table, TransactionalMode.off, TransactionalMigrationFromMode.none); + assertTransactionalMode(table, TransactionalMode.off, TransactionalMigrationFromMode.none); + } +} diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java index 626311fc03..af4e3f738b 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java @@ -57,9 +57,11 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.serializers.CommandSerializers; +import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.utils.Pair; import static accord.local.Status.Durability.Majority; @@ -82,7 +84,9 @@ public class AccordCommandStoreTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); + TableMetadata tbl = Schema.instance.getTableMetadata("ks", "tbl"); + Assert.assertEquals(TransactionalMode.full, tbl.params.transactionalMode); StorageService.instance.initServer(); } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index 93cdaeb5a4..5b1a1402a2 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -73,7 +73,7 @@ public class AccordCommandTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); StorageService.instance.initServer(); } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java b/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java index 3a0e0c7cb7..7b77791c6e 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java @@ -163,7 +163,7 @@ public class AccordConfigurationServiceTest ServerTestUtils.daemonInitialization(); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); } @Before diff --git a/test/unit/org/apache/cassandra/service/accord/AccordReadRepairTest.java b/test/unit/org/apache/cassandra/service/accord/AccordReadRepairTest.java index d62ab52946..1dd5bf4b16 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordReadRepairTest.java @@ -49,7 +49,7 @@ public class AccordReadRepairTest extends AccordTestBase @BeforeClass public static void setupClass() throws IOException { - AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("lwt_strategy", "accord").set("non_serial_write_strategy", "mixed")), 2); + AccordTestBase.setupCluster(builder -> builder, 2); SHARED_CLUSTER.schemaChange("CREATE TYPE " + KEYSPACE + ".person (height int, age int)"); } @@ -94,9 +94,9 @@ public class AccordReadRepairTest extends AccordTestBase void testReadRepair(Function accordTxn, Object[][] expected) throws Exception { - test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v1 int, v2 int, PRIMARY KEY ((k), c));", + test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v1 int, v2 int, PRIMARY KEY ((k), c)) WITH transactional_mode='unsafe_writes';", cluster -> { - Filter mutationFilter = cluster.filters().verbs(Verb.MUTATION_REQ.id).drop().on(); + Filter mutationFilter = cluster.filters().verbs(Verb.MUTATION_REQ.id).to(2).drop().on(); cluster.filters().verbs(Verb.HINT_REQ.id, Verb.HINT_RSP.id).drop().on(); cluster.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, v1, v2) VALUES (1, 1, 1, 1) USING TIMESTAMP 42;", ConsistencyLevel.ONE); mutationFilter.off(); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java b/test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java index 937b24350a..9dd9fc6a82 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java @@ -83,7 +83,7 @@ public class AccordTopologyTest { DatabaseDescriptor.daemonInitialization(); DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); - TableMetadata table = parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks").build(); + TableMetadata table = parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks").build(); tableId = table.id; keyspace = KeyspaceMetadata.create("ks", KeyspaceParams.simple(3), Tables.of(table)); } @@ -170,7 +170,7 @@ public class AccordTopologyTest Assert.assertEquals(partitioner.getMaximumTokenForSplitting(), ranges.get(2).right); ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace)); - Topology topology = AccordTopology.createAccordTopology(metadata, ks -> true); + Topology topology = AccordTopology.createAccordTopology(metadata); Topology expected = new Topology(1, new Shard(AccordTopology.minRange(tableId, ranges.get(0).right), NODE_LIST, NODE_SET), new Shard(AccordTopology.range(tableId, ranges.get(1)), NODE_LIST, NODE_SET), @@ -188,7 +188,7 @@ public class AccordTopologyTest range(100, -100)); ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace)); - Topology topology = AccordTopology.createAccordTopology(metadata, ks -> true); + Topology topology = AccordTopology.createAccordTopology(metadata); Topology expected = new Topology(1, new Shard(AccordTopology.minRange(tableId, ranges.get(0).left), NODE_LIST, NODE_SET), new Shard(AccordTopology.range(tableId, ranges.get(0)), NODE_LIST, NODE_SET), @@ -205,7 +205,7 @@ public class AccordTopologyTest range(-100, 100), range(token(100), partitioner.getMaximumTokenForSplitting())); ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace)); - Topology topology = AccordTopology.createAccordTopology(metadata, ks -> true); + Topology topology = AccordTopology.createAccordTopology(metadata); Topology expected = new Topology(1, new Shard(AccordTopology.minRange(tableId, ranges.get(0).right), NODE_LIST, NODE_SET), new Shard(AccordTopology.range(tableId, ranges.get(1)), NODE_LIST, NODE_SET), @@ -213,7 +213,7 @@ public class AccordTopologyTest new Shard(AccordTopology.maxRange(tableId, ranges.get(2).right), NODE_LIST, NODE_SET)); Assert.assertEquals(expected, topology); - topology = AccordTopology.createAccordTopology(metadata.transformer().withFastPathStatusSince(new Id(1), AccordFastPath.Status.UNAVAILABLE, 1, 1).build().metadata, ks -> true); + topology = AccordTopology.createAccordTopology(metadata.transformer().withFastPathStatusSince(new Id(1), AccordFastPath.Status.UNAVAILABLE, 1, 1).build().metadata); Set fastPath = new HashSet<>(NODE_SET); fastPath.remove(new Node.Id(1)); @@ -236,7 +236,7 @@ public class AccordTopologyTest range(-100, 100), range(token(100), partitioner.getMaximumTokenForSplitting())); ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace)); - Topology topology = AccordTopology.createAccordTopology(metadata, ks -> true); + Topology topology = AccordTopology.createAccordTopology(metadata); Topology expected = new Topology(1, new Shard(AccordTopology.minRange(tableId, ranges.get(0).right), NODE_LIST, NODE_SET), new Shard(AccordTopology.range(tableId, ranges.get(1)), NODE_LIST, NODE_SET), @@ -248,7 +248,7 @@ public class AccordTopologyTest .withFastPathStatusSince(new Id(1), AccordFastPath.Status.UNAVAILABLE, 1, 1) .withFastPathStatusSince(new Id(2), AccordFastPath.Status.UNAVAILABLE, 1, 1) .build().metadata; - topology = AccordTopology.createAccordTopology(metadata, ks -> true); + topology = AccordTopology.createAccordTopology(metadata); Set fastPath = new HashSet<>(NODE_SET); fastPath.remove(new Node.Id(1)); diff --git a/test/unit/org/apache/cassandra/service/accord/api/AccordKeyTest.java b/test/unit/org/apache/cassandra/service/accord/api/AccordKeyTest.java index ba80664c39..a3cc4504d9 100644 --- a/test/unit/org/apache/cassandra/service/accord/api/AccordKeyTest.java +++ b/test/unit/org/apache/cassandra/service/accord/api/AccordKeyTest.java @@ -45,8 +45,8 @@ public class AccordKeyTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl1 (k int, c int, v int, primary key (k, c))", "ks").id(TABLE1), - parse("CREATE TABLE tbl2 (k int, c int, v int, primary key (k, c))", "ks").id(TABLE2)); + parse("CREATE TABLE tbl1 (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks").id(TABLE1), + parse("CREATE TABLE tbl2 (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks").id(TABLE2)); } diff --git a/test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java b/test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java index 0f96516566..8c952f02c0 100644 --- a/test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java +++ b/test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java @@ -79,7 +79,7 @@ public class AsyncLoaderTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); StorageService.instance.initServer(); } diff --git a/test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java b/test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java index f421c993a0..b235d1ae59 100644 --- a/test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java +++ b/test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java @@ -107,7 +107,7 @@ public class AsyncOperationTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); StorageService.instance.initServer(); } diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandSerializersTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandSerializersTest.java index 9d04137587..d13742d257 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandSerializersTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandSerializersTest.java @@ -40,7 +40,7 @@ public class CommandSerializersTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); } diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 4be4c11115..e12b3fbf87 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -92,7 +92,7 @@ public class CommandsForKeySerializerTest // need to create the accord test table as generating random txn is not currently supported SchemaLoader.prepareServer(); SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); StorageService.instance.initServer(); } diff --git a/test/unit/org/apache/cassandra/service/accord/txn/AccordUpdateTest.java b/test/unit/org/apache/cassandra/service/accord/txn/AccordUpdateTest.java index fbfd1190cc..bad4074849 100644 --- a/test/unit/org/apache/cassandra/service/accord/txn/AccordUpdateTest.java +++ b/test/unit/org/apache/cassandra/service/accord/txn/AccordUpdateTest.java @@ -36,7 +36,7 @@ public class AccordUpdateTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); } diff --git a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java index 37b3815c1c..68f02de123 100644 --- a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java +++ b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java @@ -297,8 +297,6 @@ public class ClusterMetadataTransformationTest return metadata.lockedRanges; else if (key == IN_PROGRESS_SEQUENCES) return metadata.inProgressSequences; - else if (key == ACCORD_TABLES) - return metadata.accordTables; else if (key == ACCORD_FAST_PATH) return metadata.accordFastPath; else if (key == CONSENSUS_MIGRATION_STATE)