From f41c2e29d240a7f0cafe6455722d287edc80b1be Mon Sep 17 00:00:00 2001 From: Ke Han Date: Wed, 25 Oct 2023 01:25:56 +0000 Subject: [PATCH] Fix incorrect column identifier bytes problem when renaming a column patch by Ke Han; reviewed by Stefan Miklosovic and Brandon Williams for CASSANDRA-18956 --- CHANGES.txt | 1 + src/java/org/apache/cassandra/cql3/Cql.g | 12 +- .../cql3/statements/AlterTableStatement.java | 10 +- .../schema/LegacySchemaMigratorBaseTest.java | 816 ++++++++++++++++++ .../schema/LegacySchemaMigratorTest.java | 742 +--------------- .../LegacySchemaMigratorThriftTest.java | 90 ++ 6 files changed, 929 insertions(+), 742 deletions(-) create mode 100644 test/unit/org/apache/cassandra/schema/LegacySchemaMigratorBaseTest.java create mode 100644 test/unit/org/apache/cassandra/schema/LegacySchemaMigratorThriftTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 7ce9a42507..9804496fcb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.31 + * Fix incorrect column identifier bytes problem when renaming a column (CASSANDRA-18965) * Upgrade OWASP to 10.0.0 (CASSANDRA-19738) Merged from 2.2: * Add termin-8-jdk as a valid jdk8 candidate in the debian package (CASSANDRA-19752) diff --git a/src/java/org/apache/cassandra/cql3/Cql.g b/src/java/org/apache/cassandra/cql3/Cql.g index 02343274b5..2d10c95066 100644 --- a/src/java/org/apache/cassandra/cql3/Cql.g +++ b/src/java/org/apache/cassandra/cql3/Cql.g @@ -816,7 +816,7 @@ alterTableStatement returns [AlterTableStatement expr] @init { AlterTableStatement.Type type = null; TableAttributes attrs = new TableAttributes(); - Map renames = new HashMap(); + Map renames = new HashMap(); boolean isStatic = false; Long dropTimestamp = null; } @@ -829,8 +829,8 @@ alterTableStatement returns [AlterTableStatement expr] | K_DROP K_COMPACT K_STORAGE { type = AlterTableStatement.Type.DROP_COMPACT_STORAGE; } | K_WITH properties[attrs] { type = AlterTableStatement.Type.OPTS; } | K_RENAME { type = AlterTableStatement.Type.RENAME; } - id1=cident K_TO toId1=ident { renames.put(id1, toId1); } - ( K_AND idn=cident K_TO toIdn=ident { renames.put(idn, toIdn); } )* + id1=schema_cident K_TO toId1=schema_cident { renames.put(id1, toId1); } + ( K_AND idn=schema_cident K_TO toIdn=schema_cident { renames.put(idn, toIdn); } )* ) { $expr = new AlterTableStatement(cf, type, id, v, attrs, renames, isStatic, dropTimestamp); @@ -1180,6 +1180,12 @@ cident returns [ColumnIdentifier.Raw id] | EMPTY_QUOTED_NAME { $id = new ColumnIdentifier.Literal("", false); } ; +schema_cident returns [ColumnIdentifier.Raw id] + : t=IDENT { $id = new ColumnIdentifier.Literal($t.text, false); } + | t=QUOTED_NAME { $id = new ColumnIdentifier.Literal($t.text, true); } + | k=unreserved_keyword { $id = new ColumnIdentifier.Literal(k, false); } + ; + // Column identifiers where the comparator is known to be text ident returns [ColumnIdentifier id] : t=IDENT { $id = ColumnIdentifier.getInterned($t.text, false); } diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java index cd1041e6c1..2f516f7847 100644 --- a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java @@ -71,7 +71,7 @@ public class AlterTableStatement extends SchemaAlteringStatement public final CQL3Type.Raw validator; public final ColumnIdentifier.Raw rawColumnName; private final TableAttributes attrs; - private final Map renames; + private final Map renames; private final boolean isStatic; // Only for ALTER ADD private final Long deleteTimestamp; @@ -80,7 +80,7 @@ public class AlterTableStatement extends SchemaAlteringStatement ColumnIdentifier.Raw columnName, CQL3Type.Raw validator, TableAttributes attrs, - Map renames, + Map renames, boolean isStatic, Long deleteTimestamp) { @@ -328,10 +328,10 @@ public class AlterTableStatement extends SchemaAlteringStatement case RENAME: cfm = meta.copy(); - for (Map.Entry entry : renames.entrySet()) + for (Map.Entry entry : renames.entrySet()) { ColumnIdentifier from = entry.getKey().prepare(cfm); - ColumnIdentifier to = entry.getValue(); + ColumnIdentifier to = entry.getValue().prepare(cfm); cfm.renameColumn(from, to); @@ -342,7 +342,7 @@ public class AlterTableStatement extends SchemaAlteringStatement ViewDefinition viewCopy = view.copy(); ColumnIdentifier viewFrom = entry.getKey().prepare(viewCopy.metadata); - ColumnIdentifier viewTo = entry.getValue(); + ColumnIdentifier viewTo = entry.getValue().prepare(viewCopy.metadata); viewCopy.renameColumn(viewFrom, viewTo); if (viewUpdates == null) diff --git a/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorBaseTest.java b/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorBaseTest.java new file mode 100644 index 0000000000..49d99f8475 --- /dev/null +++ b/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorBaseTest.java @@ -0,0 +1,816 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.schema; + +import java.nio.ByteBuffer; +import java.util.*; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableList; + +import org.junit.Ignore; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.functions.*; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.index.internal.CassandraIndex; +import org.apache.cassandra.thrift.ThriftConversion; +import org.apache.cassandra.utils.*; + +import static java.lang.String.format; +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertTrue; +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; +import static org.apache.cassandra.utils.FBUtilities.json; + +@SuppressWarnings("deprecation") +@Ignore +public abstract class LegacySchemaMigratorBaseTest +{ + private static final long TIMESTAMP = 1435908994000000L; + + private static final String KEYSPACE_PREFIX = "LegacySchemaMigratorTest"; + + public static void loadLegacySchemaTables() + { + KeyspaceMetadata systemKeyspace = Schema.instance.getKSMetaData(SystemKeyspace.NAME); + + Tables systemTables = systemKeyspace.tables; + for (CFMetaData table : LegacySchemaMigrator.LegacySchemaTables) + systemTables = systemTables.with(table); + + LegacySchemaMigrator.LegacySchemaTables.forEach(Schema.instance::load); + + Schema.instance.setKeyspaceMetadata(systemKeyspace.withSwapped(systemTables)); + } + + public static Keyspaces keyspacesToMigrate() + { + Keyspaces.Builder keyspaces = Keyspaces.builder(); + + // A whole bucket of shorthand + String ks1 = KEYSPACE_PREFIX + "Keyspace1"; + String ks2 = KEYSPACE_PREFIX + "Keyspace2"; + String ks3 = KEYSPACE_PREFIX + "Keyspace3"; + String ks4 = KEYSPACE_PREFIX + "Keyspace4"; + String ks5 = KEYSPACE_PREFIX + "Keyspace5"; + String ks6 = KEYSPACE_PREFIX + "Keyspace6"; + String ks_rcs = KEYSPACE_PREFIX + "RowCacheSpace"; + String ks_nocommit = KEYSPACE_PREFIX + "NoCommitlogSpace"; + String ks_prsi = KEYSPACE_PREFIX + "PerRowSecondaryIndex"; + String ks_cql = KEYSPACE_PREFIX + "cql_keyspace"; + + // Make it easy to test compaction + Map compactionOptions = new HashMap<>(); + compactionOptions.put("tombstone_compaction_interval", "1"); + + Map leveledOptions = new HashMap<>(); + leveledOptions.put("sstable_size_in_mb", "1"); + + keyspaces.add(KeyspaceMetadata.create(ks1, + KeyspaceParams.simple(1), + Tables.of(SchemaLoader.standardCFMD(ks1, "Standard1") + .compaction(CompactionParams.scts(compactionOptions)), + SchemaLoader.standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0), + SchemaLoader.standardCFMD(ks1, "StandardLong1"), + SchemaLoader.keysIndexCFMD(ks1, "Indexed1", true), + SchemaLoader.keysIndexCFMD(ks1, "Indexed2", false), + SchemaLoader.jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance) + .addColumnDefinition(SchemaLoader.utf8Column(ks1, "JdbcUtf8")), + SchemaLoader.jdbcCFMD(ks1, "JdbcLong", LongType.instance), + SchemaLoader.jdbcCFMD(ks1, "JdbcBytes", BytesType.instance), + SchemaLoader.jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance), + SchemaLoader.standardCFMD(ks1, "StandardLeveled") + .compaction(CompactionParams.lcs(leveledOptions)), + SchemaLoader.standardCFMD(ks1, "legacyleveled") + .compaction(CompactionParams.lcs(leveledOptions)), + SchemaLoader.standardCFMD(ks1, "StandardLowIndexInterval") + .minIndexInterval(8) + .maxIndexInterval(256) + .caching(CachingParams.CACHE_NOTHING)))); + + // Keyspace 2 + keyspaces.add(KeyspaceMetadata.create(ks2, + KeyspaceParams.simple(1), + Tables.of(SchemaLoader.standardCFMD(ks2, "Standard1"), + SchemaLoader.keysIndexCFMD(ks2, "Indexed1", true), + SchemaLoader.compositeIndexCFMD(ks2, "Indexed2", true), + SchemaLoader.compositeIndexCFMD(ks2, "Indexed3", true) + .gcGraceSeconds(0)))); + + // Keyspace 3 + keyspaces.add(KeyspaceMetadata.create(ks3, + KeyspaceParams.simple(5), + Tables.of(SchemaLoader.standardCFMD(ks3, "Standard1"), + SchemaLoader.keysIndexCFMD(ks3, "Indexed1", true)))); + + // Keyspace 4 + keyspaces.add(KeyspaceMetadata.create(ks4, + KeyspaceParams.simple(3), + Tables.of(SchemaLoader.standardCFMD(ks4, "Standard1")))); + + // Keyspace 5 + keyspaces.add(KeyspaceMetadata.create(ks5, + KeyspaceParams.simple(2), + Tables.of(SchemaLoader.standardCFMD(ks5, "Standard1")))); + + // Keyspace 6 + keyspaces.add(KeyspaceMetadata.create(ks6, + KeyspaceParams.simple(1), + Tables.of(SchemaLoader.keysIndexCFMD(ks6, "Indexed1", true)))); + + // RowCacheSpace + keyspaces.add(KeyspaceMetadata.create(ks_rcs, + KeyspaceParams.simple(1), + Tables.of(SchemaLoader.standardCFMD(ks_rcs, "CFWithoutCache") + .caching(CachingParams.CACHE_NOTHING), + SchemaLoader.standardCFMD(ks_rcs, "CachedCF") + .caching(CachingParams.CACHE_EVERYTHING), + SchemaLoader.standardCFMD(ks_rcs, "CachedIntCF") + .caching(new CachingParams(true, 100))))); + + keyspaces.add(KeyspaceMetadata.create(ks_nocommit, + KeyspaceParams.simpleTransient(1), + Tables.of(SchemaLoader.standardCFMD(ks_nocommit, "Standard1")))); + + // PerRowSecondaryIndexTest + keyspaces.add(KeyspaceMetadata.create(ks_prsi, + KeyspaceParams.simple(1), + Tables.of(SchemaLoader.perRowIndexedCFMD(ks_prsi, "Indexed1")))); + + // CQLKeyspace + keyspaces.add(KeyspaceMetadata.create(ks_cql, + KeyspaceParams.simple(1), + Tables.of(CFMetaData.compile("CREATE TABLE table1 (" + + "k int PRIMARY KEY," + + "v1 text," + + "v2 int" + + ')', ks_cql), + + CFMetaData.compile("CREATE TABLE table2 (" + + "k text," + + "c text," + + "v text," + + "PRIMARY KEY (k, c))", ks_cql), + + CFMetaData.compile("CREATE TABLE foo (" + + "bar text, " + + "baz text, " + + "qux text, " + + "PRIMARY KEY(bar, baz) ) " + + "WITH COMPACT STORAGE", ks_cql), + + CFMetaData.compile("CREATE TABLE compact_pkonly (" + + "k int, " + + "c int, " + + "PRIMARY KEY (k, c)) " + + "WITH COMPACT STORAGE", + ks_cql), + + CFMetaData.compile("CREATE TABLE foofoo (" + + "bar text, " + + "baz text, " + + "qux text, " + + "quz text, " + + "foo text, " + + "PRIMARY KEY((bar, baz), qux, quz) ) " + + "WITH COMPACT STORAGE", ks_cql)))); + + // NTS keyspace + keyspaces.add(KeyspaceMetadata.create("nts", KeyspaceParams.nts("dc1", 1, "dc2", 2))); + + keyspaces.add(keyspaceWithDroppedCollections()); + keyspaces.add(keyspaceWithTriggers()); + keyspaces.add(keyspaceWithUDTs()); + keyspaces.add(keyspaceWithUDFs()); + keyspaces.add(keyspaceWithUDFsAndUDTs()); + keyspaces.add(keyspaceWithUDAs()); + keyspaces.add(keyspaceWithUDAsAndUDTs()); + + return keyspaces.build(); + } + + private static KeyspaceMetadata keyspaceWithDroppedCollections() + { + String keyspace = KEYSPACE_PREFIX + "DroppedCollections"; + + CFMetaData table = + CFMetaData.compile("CREATE TABLE dropped_columns (" + + "foo text," + + "bar text," + + "map1 map," + + "map2 map," + + "set1 set," + + "list1 list," + + "PRIMARY KEY ((foo), bar))", + keyspace); + + String[] collectionColumnNames = { "map1", "map2", "set1", "list1" }; + for (String name : collectionColumnNames) + { + ColumnDefinition column = table.getColumnDefinition(bytes(name)); + table.recordColumnDrop(column, FBUtilities.timestampMicros(), false); + table.removeColumnDefinition(column); + } + + return KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1), Tables.of(table)); + } + + private static KeyspaceMetadata keyspaceWithTriggers() + { + String keyspace = KEYSPACE_PREFIX + "Triggers"; + + Triggers.Builder triggers = Triggers.builder(); + CFMetaData table = SchemaLoader.standardCFMD(keyspace, "WithTriggers"); + for (int i = 0; i < 10; i++) + triggers.add(new TriggerMetadata("trigger" + i, "DummyTrigger" + i)); + table.triggers(triggers.build()); + + return KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1), Tables.of(table)); + } + + private static KeyspaceMetadata keyspaceWithUDTs() + { + String keyspace = KEYSPACE_PREFIX + "UDTs"; + + UserType udt1 = new UserType(keyspace, + bytes("udt1"), + new ArrayList() + {{ + add(bytes("col1")); + add(bytes("col2")); + }}, + new ArrayList>() + {{ + add(UTF8Type.instance); + add(Int32Type.instance); + }}); + + UserType udt2 = new UserType(keyspace, + bytes("udt2"), + new ArrayList() + {{ + add(bytes("col3")); + add(bytes("col4")); + }}, + new ArrayList>() + {{ + add(BytesType.instance); + add(BooleanType.instance); + }}); + + UserType udt3 = new UserType(keyspace, + bytes("udt3"), + new ArrayList() + {{ + add(bytes("col5")); + }}, + new ArrayList>() + {{ + add(AsciiType.instance); + }}); + + return KeyspaceMetadata.create(keyspace, + KeyspaceParams.simple(1), + Tables.none(), + Views.none(), + Types.of(udt1, udt2, udt3), + Functions.none()); + } + + private static KeyspaceMetadata keyspaceWithUDFs() + { + String keyspace = KEYSPACE_PREFIX + "UDFs"; + + UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf"), + ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), + ImmutableList.of(BytesType.instance, Int32Type.instance), + LongType.instance, + false, + "java", + "return 42L;"); + + // an overload with the same name, not a typo + UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf"), + ImmutableList.of(new ColumnIdentifier("col3", false), new ColumnIdentifier("col4", false)), + ImmutableList.of(AsciiType.instance, LongType.instance), + Int32Type.instance, + true, + "java", + "return 42;"); + + UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"), + ImmutableList.of(new ColumnIdentifier("col4", false)), + ImmutableList.of(UTF8Type.instance), + BooleanType.instance, + false, + "java", + "return true;"); + + return KeyspaceMetadata.create(keyspace, + KeyspaceParams.simple(1), + Tables.none(), + Views.none(), + Types.none(), + Functions.of(udf1, udf2, udf3)); + } + + private static KeyspaceMetadata keyspaceWithUDAs() + { + String keyspace = KEYSPACE_PREFIX + "UDAs"; + + UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf1"), + ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), + ImmutableList.of(Int32Type.instance, Int32Type.instance), + Int32Type.instance, + false, + "java", + "return 42;"); + + UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf2"), + ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), + ImmutableList.of(LongType.instance, Int32Type.instance), + LongType.instance, + false, + "java", + "return 42L;"); + + UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"), + ImmutableList.of(new ColumnIdentifier("col1", false)), + ImmutableList.of(LongType.instance), + DoubleType.instance, + false, + "java", + "return 42d;"); + + Functions udfs = Functions.builder().add(udf1).add(udf2).add(udf3).build(); + + UDAggregate uda1 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda1"), + ImmutableList.of(udf1.argTypes().get(1)), + udf1.returnType(), + udf1.name(), + null, + udf1.argTypes().get(0), + null + ); + + UDAggregate uda2 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda2"), + ImmutableList.of(udf2.argTypes().get(1)), + udf3.returnType(), + udf2.name(), + udf3.name(), + udf2.argTypes().get(0), + LongType.instance.decompose(0L) + ); + + return KeyspaceMetadata.create(keyspace, + KeyspaceParams.simple(1), + Tables.none(), + Views.none(), + Types.none(), + Functions.of(udf1, udf2, udf3, uda1, uda2)); + } + + private static KeyspaceMetadata keyspaceWithUDFsAndUDTs() + { + String keyspace = KEYSPACE_PREFIX + "UDFUDTs"; + + UserType udt1 = new UserType(keyspace, + bytes("udt1"), + new ArrayList() + {{ + add(bytes("col1")); + add(bytes("col2")); + }}, + new ArrayList>() + {{ + add(UTF8Type.instance); + add(Int32Type.instance); + }}); + + UserType udt2 = new UserType(keyspace, + bytes("udt2"), + new ArrayList() + {{ + add(bytes("col1")); + add(bytes("col2")); + }}, + new ArrayList>() + {{ + add(ListType.getInstance(udt1, false)); + add(Int32Type.instance); + }}); + + UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf"), + ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), + ImmutableList.of(udt1, udt2), + LongType.instance, + false, + "java", + "return 42L;"); + + // an overload with the same name, not a typo + UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf"), + ImmutableList.of(new ColumnIdentifier("col3", false), new ColumnIdentifier("col4", false)), + ImmutableList.of(AsciiType.instance, LongType.instance), + Int32Type.instance, + true, + "java", + "return 42;"); + + UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"), + ImmutableList.of(new ColumnIdentifier("col4", false)), + ImmutableList.of(new TupleType(Arrays.asList(udt1, udt2))), + BooleanType.instance, + false, + "java", + "return true;"); + + return KeyspaceMetadata.create(keyspace, + KeyspaceParams.simple(1), + Tables.none(), + Views.none(), + Types.of(udt1, udt2), + Functions.of(udf1, udf2, udf3)); + } + + private static KeyspaceMetadata keyspaceWithUDAsAndUDTs() + { + String keyspace = KEYSPACE_PREFIX + "UDAUDTs"; + + UserType udt1 = new UserType(keyspace, + bytes("udt1"), + new ArrayList() + {{ + add(bytes("col1")); + add(bytes("col2")); + }}, + new ArrayList>() + {{ + add(UTF8Type.instance); + add(Int32Type.instance); + }}); + + UserType udt2 = new UserType(keyspace, + bytes("udt2"), + new ArrayList() + {{ + add(bytes("col1")); + add(bytes("col2")); + }}, + new ArrayList>() + {{ + add(ListType.getInstance(udt1, false)); + add(Int32Type.instance); + }}); + + UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf1"), + ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), + ImmutableList.of(udt1, udt2), + udt1, + false, + "java", + "return null;"); + + UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf2"), + ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), + ImmutableList.of(udt2, udt1), + udt2, + false, + "java", + "return null;"); + + UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"), + ImmutableList.of(new ColumnIdentifier("col1", false)), + ImmutableList.of(udt2), + DoubleType.instance, + false, + "java", + "return 42d;"); + + Functions udfs = Functions.builder().add(udf1).add(udf2).add(udf3).build(); + + UDAggregate uda1 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda1"), + ImmutableList.of(udf1.argTypes().get(1)), + udf1.returnType(), + udf1.name(), + null, + udf1.argTypes().get(0), + null + ); + + ByteBuffer twoNullEntries = ByteBuffer.allocate(8); + twoNullEntries.putInt(-1); + twoNullEntries.putInt(-1); + twoNullEntries.flip(); + UDAggregate uda2 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda2"), + ImmutableList.of(udf2.argTypes().get(1)), + udf3.returnType(), + udf2.name(), + udf3.name(), + udf2.argTypes().get(0), + twoNullEntries + ); + + return KeyspaceMetadata.create(keyspace, + KeyspaceParams.simple(1), + Tables.none(), + Views.none(), + Types.of(udt1, udt2), + Functions.of(udf1, udf2, udf3, uda1, uda2)); + } + + /* + * Serializing keyspaces + */ + + public void legacySerializeKeyspace(KeyspaceMetadata keyspace) + { + makeLegacyCreateKeyspaceMutation(keyspace, TIMESTAMP).apply(); + setLegacyIndexStatus(keyspace); + } + + public Mutation makeLegacyCreateKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp) + { + // Note that because Keyspaces is a COMPACT TABLE, we're really only setting static columns internally and shouldn't set any clustering. + RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyKeyspaces, timestamp, keyspace.name); + + adder.add("durable_writes", keyspace.params.durableWrites) + .add("strategy_class", keyspace.params.replication.klass.getName()) + .add("strategy_options", json(keyspace.params.replication.options)); + + Mutation mutation = adder.build(); + + keyspace.tables.forEach(table -> addTableToSchemaMutation(table, timestamp, true, mutation)); + keyspace.types.forEach(type -> addTypeToSchemaMutation(type, timestamp, mutation)); + keyspace.functions.udfs().forEach(udf -> addFunctionToSchemaMutation(udf, timestamp, mutation)); + keyspace.functions.udas().forEach(uda -> addAggregateToSchemaMutation(uda, timestamp, mutation)); + + return mutation; + } + + /* + * Serializing tables + */ + + public void addTableToSchemaMutation(CFMetaData table, long timestamp, boolean withColumnsAndTriggers, Mutation mutation) + { + // For property that can be null (and can be changed), we insert tombstones, to make sure + // we don't keep a property the user has removed + RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyColumnfamilies, timestamp, mutation) + .clustering(table.cfName); + + adder.add("cf_id", table.cfId) + .add("type", table.isSuper() ? "Super" : "Standard"); + + if (table.isSuper()) + { + adder.add("comparator", table.comparator.subtype(0).toString()) + .add("subcomparator", ((MapType) table.compactValueColumn().type).getKeysType().toString()); + } + else + { + adder.add("comparator", LegacyLayout.makeLegacyComparator(table).toString()); + } + + adder.add("bloom_filter_fp_chance", table.params.bloomFilterFpChance) + .add("caching", cachingToString(table.params.caching)) + .add("comment", table.params.comment) + .add("compaction_strategy_class", table.params.compaction.klass().getName()) + .add("compaction_strategy_options", json(table.params.compaction.options())) + .add("compression_parameters", json(ThriftConversion.compressionParametersToThrift(table.params.compression))) + .add("default_time_to_live", table.params.defaultTimeToLive) + .add("gc_grace_seconds", table.params.gcGraceSeconds) + .add("key_validator", table.getKeyValidator().toString()) + .add("local_read_repair_chance", table.params.dcLocalReadRepairChance) + .add("max_compaction_threshold", table.params.compaction.maxCompactionThreshold()) + .add("max_index_interval", table.params.maxIndexInterval) + .add("memtable_flush_period_in_ms", table.params.memtableFlushPeriodInMs) + .add("min_compaction_threshold", table.params.compaction.minCompactionThreshold()) + .add("min_index_interval", table.params.minIndexInterval) + .add("read_repair_chance", table.params.readRepairChance) + .add("speculative_retry", table.params.speculativeRetry.toString()); + + for (Map.Entry entry : table.getDroppedColumns().entrySet()) + { + String name = UTF8Type.instance.getString(entry.getKey()); + CFMetaData.DroppedColumn column = entry.getValue(); + adder.addMapEntry("dropped_columns", name, column.droppedTime); + } + + adder.add("is_dense", table.isDense()); + + adder.add("default_validator", table.makeLegacyDefaultValidator().toString()); + + if (withColumnsAndTriggers) + { + for (ColumnDefinition column : table.allColumns()) + addColumnToSchemaMutation(table, column, timestamp, mutation); + + for (TriggerMetadata trigger : table.getTriggers()) + addTriggerToSchemaMutation(table, trigger, timestamp, mutation); + } + + adder.build(); + } + + private static String cachingToString(CachingParams caching) + { + return format("{\"keys\":\"%s\", \"rows_per_partition\":\"%s\"}", + caching.keysAsString(), + caching.rowsPerPartitionAsString()); + } + + public void addColumnToSchemaMutation(CFMetaData table, ColumnDefinition column, long timestamp, Mutation mutation) + { + // We need to special case pk-only dense tables. See CASSANDRA-9874. + String name = table.isDense() && column.kind == ColumnDefinition.Kind.REGULAR && column.type instanceof EmptyType + ? "" + : column.name.toString(); + + final RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyColumns, timestamp, mutation).clustering(table.cfName, name); + adder.add("validator", column.type.toString()) + .add("type", serializeKind(column.kind, table.isDense())) + .add("component_index", column.position()); + Optional index = findIndexForColumn(table.getIndexes(), table, column); + if (index.isPresent()) + { + IndexMetadata i = index.get(); + adder.add("index_name", i.name); + adder.add("index_type", i.kind.toString()); + adder.add("index_options", json(i.options)); + } + else + { + adder.add("index_name", null); + adder.add("index_type", null); + adder.add("index_options", null); + } + + adder.build(); + } + + private static Optional findIndexForColumn(Indexes indexes, + CFMetaData table, + ColumnDefinition column) + { + // makes the assumptions that the string option denoting the + // index targets can be parsed by CassandraIndex.parseTarget + // which should be true for any pre-3.0 index + for (IndexMetadata index : indexes) + if (CassandraIndex.parseTarget(table, index).left.equals(column)) + return Optional.of(index); + + return Optional.empty(); + } + + public abstract String serializeKind(ColumnDefinition.Kind kind, boolean isDense); + + + private static void addTriggerToSchemaMutation(CFMetaData table, TriggerMetadata trigger, long timestamp, Mutation mutation) + { + new RowUpdateBuilder(SystemKeyspace.LegacyTriggers, timestamp, mutation) + .clustering(table.cfName, trigger.name) + .addMapEntry("trigger_options", "class", trigger.classOption) + .build(); + } + + /* + * Serializing types + */ + + private static void addTypeToSchemaMutation(UserType type, long timestamp, Mutation mutation) + { + RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyUsertypes, timestamp, mutation) + .clustering(type.getNameAsString()); + + adder.resetCollection("field_names") + .resetCollection("field_types"); + + for (int i = 0; i < type.size(); i++) + { + adder.addListEntry("field_names", type.fieldName(i)) + .addListEntry("field_types", type.fieldType(i).toString()); + } + + adder.build(); + } + + /* + * Serializing functions + */ + + private static void addFunctionToSchemaMutation(UDFunction function, long timestamp, Mutation mutation) + { + RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyFunctions, timestamp, mutation) + .clustering(function.name().name, functionSignatureWithTypes(function)); + + adder.add("body", function.body()) + .add("language", function.language()) + .add("return_type", function.returnType().toString()) + .add("called_on_null_input", function.isCalledOnNullInput()); + + adder.resetCollection("argument_names") + .resetCollection("argument_types"); + + for (int i = 0; i < function.argNames().size(); i++) + { + adder.addListEntry("argument_names", function.argNames().get(i).bytes) + .addListEntry("argument_types", function.argTypes().get(i).toString()); + } + + adder.build(); + } + + /* + * Serializing aggregates + */ + + private static void addAggregateToSchemaMutation(UDAggregate aggregate, long timestamp, Mutation mutation) + { + RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyAggregates, timestamp, mutation) + .clustering(aggregate.name().name, functionSignatureWithTypes(aggregate)); + + adder.resetCollection("argument_types"); + + adder.add("return_type", aggregate.returnType().toString()) + .add("state_func", aggregate.stateFunction().name().name); + + if (aggregate.stateType() != null) + adder.add("state_type", aggregate.stateType().toString()); + if (aggregate.finalFunction() != null) + adder.add("final_func", aggregate.finalFunction().name().name); + if (aggregate.initialCondition() != null) + adder.add("initcond", aggregate.initialCondition()); + + for (AbstractType argType : aggregate.argTypes()) + adder.addListEntry("argument_types", argType.toString()); + + adder.build(); + } + + // We allow method overloads, so a function is not uniquely identified by its name only, but + // also by its argument types. To distinguish overloads of given function name in the schema + // we use a "signature" which is just a list of it's CQL argument types. + public static ByteBuffer functionSignatureWithTypes(AbstractFunction fun) + { + List arguments = + fun.argTypes() + .stream() + .map(argType -> argType.asCQL3Type().toString()) + .collect(Collectors.toList()); + + return ListType.getInstance(UTF8Type.instance, false).decompose(arguments); + } + + private static void setLegacyIndexStatus(KeyspaceMetadata keyspace) + { + keyspace.tables.forEach(LegacySchemaMigratorBaseTest::setLegacyIndexStatus); + } + + private static void setLegacyIndexStatus(CFMetaData table) + { + table.getIndexes().forEach((index) -> setLegacyIndexStatus(table.ksName, table.cfName, index)); + } + + private static void setLegacyIndexStatus(String keyspace, String table, IndexMetadata index) + { + SystemKeyspace.setIndexBuilt(keyspace, table + '.' + index.name); + } + + public static void verifyIndexBuildStatus(KeyspaceMetadata keyspace) + { + keyspace.tables.forEach(LegacySchemaMigratorBaseTest::verifyIndexBuildStatus); + } + + private static void verifyIndexBuildStatus(CFMetaData table) + { + table.getIndexes().forEach(index -> verifyIndexBuildStatus(table.ksName, table.cfName, index)); + } + + private static void verifyIndexBuildStatus(String keyspace, String table, IndexMetadata index) + { + assertFalse(SystemKeyspace.isIndexBuilt(keyspace, table + '.' + index.name)); + assertTrue(SystemKeyspace.isIndexBuilt(keyspace, index.name)); + } +} diff --git a/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java b/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java index 7643456ea0..c25f39cdeb 100644 --- a/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java +++ b/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java @@ -18,42 +18,23 @@ package org.apache.cassandra.schema; import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.*; -import java.util.stream.Collectors; -import com.google.common.collect.ImmutableList; +import org.apache.cassandra.config.ColumnDefinition; import org.junit.Assert; import org.junit.Test; -import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.config.Schema; import org.apache.cassandra.cql3.CQLTester; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.cql3.functions.*; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.index.internal.CassandraIndex; -import org.apache.cassandra.thrift.ThriftConversion; -import org.apache.cassandra.utils.*; import static java.lang.String.format; import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.assertTrue; import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal; -import static org.apache.cassandra.utils.ByteBufferUtil.bytes; -import static org.apache.cassandra.utils.FBUtilities.json; @SuppressWarnings("deprecation") -public class LegacySchemaMigratorTest +public class LegacySchemaMigratorTest extends LegacySchemaMigratorBaseTest { - private static final long TIMESTAMP = 1435908994000000L; - - private static final String KEYSPACE_PREFIX = "LegacySchemaMigratorTest"; - /* * 1. Write a variety of different keyspaces/tables/types/function in the legacy manner, using legacy schema tables * 2. Run the migrator @@ -66,10 +47,10 @@ public class LegacySchemaMigratorTest { CQLTester.cleanupAndLeaveDirs(); - Keyspaces expected = keyspacesToMigrate(); + Keyspaces expected = LegacySchemaMigratorBaseTest.keyspacesToMigrate(); // write the keyspaces into the legacy tables - expected.forEach(LegacySchemaMigratorTest::legacySerializeKeyspace); + expected.forEach(this::legacySerializeKeyspace); // run the migration LegacySchemaMigrator.migrate(); @@ -78,7 +59,7 @@ public class LegacySchemaMigratorTest Keyspaces actual = SchemaKeyspace.fetchNonSystemKeyspaces(); // need to load back CFMetaData of those tables (CFS instances will still be loaded) - loadLegacySchemaTables(); + LegacySchemaMigratorBaseTest.loadLegacySchemaTables(); // verify that nothing's left in the old schema tables for (CFMetaData table : LegacySchemaMigrator.LegacySchemaTables) @@ -94,7 +75,7 @@ public class LegacySchemaMigratorTest // check that the build status of all indexes has been updated to use the new // format of index name: the index_name column of system.IndexInfo used to // contain table_name.index_name. Now it should contain just the index_name. - expected.forEach(LegacySchemaMigratorTest::verifyIndexBuildStatus); + expected.forEach(LegacySchemaMigratorBaseTest::verifyIndexBuildStatus); } @Test @@ -122,588 +103,8 @@ public class LegacySchemaMigratorTest } } - private static void loadLegacySchemaTables() - { - KeyspaceMetadata systemKeyspace = Schema.instance.getKSMetaData(SystemKeyspace.NAME); - - Tables systemTables = systemKeyspace.tables; - for (CFMetaData table : LegacySchemaMigrator.LegacySchemaTables) - systemTables = systemTables.with(table); - - LegacySchemaMigrator.LegacySchemaTables.forEach(Schema.instance::load); - - Schema.instance.setKeyspaceMetadata(systemKeyspace.withSwapped(systemTables)); - } - - private static Keyspaces keyspacesToMigrate() - { - Keyspaces.Builder keyspaces = Keyspaces.builder(); - - // A whole bucket of shorthand - String ks1 = KEYSPACE_PREFIX + "Keyspace1"; - String ks2 = KEYSPACE_PREFIX + "Keyspace2"; - String ks3 = KEYSPACE_PREFIX + "Keyspace3"; - String ks4 = KEYSPACE_PREFIX + "Keyspace4"; - String ks5 = KEYSPACE_PREFIX + "Keyspace5"; - String ks6 = KEYSPACE_PREFIX + "Keyspace6"; - String ks_rcs = KEYSPACE_PREFIX + "RowCacheSpace"; - String ks_nocommit = KEYSPACE_PREFIX + "NoCommitlogSpace"; - String ks_prsi = KEYSPACE_PREFIX + "PerRowSecondaryIndex"; - String ks_cql = KEYSPACE_PREFIX + "cql_keyspace"; - - // Make it easy to test compaction - Map compactionOptions = new HashMap<>(); - compactionOptions.put("tombstone_compaction_interval", "1"); - - Map leveledOptions = new HashMap<>(); - leveledOptions.put("sstable_size_in_mb", "1"); - - keyspaces.add(KeyspaceMetadata.create(ks1, - KeyspaceParams.simple(1), - Tables.of(SchemaLoader.standardCFMD(ks1, "Standard1") - .compaction(CompactionParams.scts(compactionOptions)), - SchemaLoader.standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0), - SchemaLoader.standardCFMD(ks1, "StandardLong1"), - SchemaLoader.keysIndexCFMD(ks1, "Indexed1", true), - SchemaLoader.keysIndexCFMD(ks1, "Indexed2", false), - SchemaLoader.jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance) - .addColumnDefinition(SchemaLoader.utf8Column(ks1, "JdbcUtf8")), - SchemaLoader.jdbcCFMD(ks1, "JdbcLong", LongType.instance), - SchemaLoader.jdbcCFMD(ks1, "JdbcBytes", BytesType.instance), - SchemaLoader.jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance), - SchemaLoader.standardCFMD(ks1, "StandardLeveled") - .compaction(CompactionParams.lcs(leveledOptions)), - SchemaLoader.standardCFMD(ks1, "legacyleveled") - .compaction(CompactionParams.lcs(leveledOptions)), - SchemaLoader.standardCFMD(ks1, "StandardLowIndexInterval") - .minIndexInterval(8) - .maxIndexInterval(256) - .caching(CachingParams.CACHE_NOTHING)))); - - // Keyspace 2 - keyspaces.add(KeyspaceMetadata.create(ks2, - KeyspaceParams.simple(1), - Tables.of(SchemaLoader.standardCFMD(ks2, "Standard1"), - SchemaLoader.keysIndexCFMD(ks2, "Indexed1", true), - SchemaLoader.compositeIndexCFMD(ks2, "Indexed2", true), - SchemaLoader.compositeIndexCFMD(ks2, "Indexed3", true) - .gcGraceSeconds(0)))); - - // Keyspace 3 - keyspaces.add(KeyspaceMetadata.create(ks3, - KeyspaceParams.simple(5), - Tables.of(SchemaLoader.standardCFMD(ks3, "Standard1"), - SchemaLoader.keysIndexCFMD(ks3, "Indexed1", true)))); - - // Keyspace 4 - keyspaces.add(KeyspaceMetadata.create(ks4, - KeyspaceParams.simple(3), - Tables.of(SchemaLoader.standardCFMD(ks4, "Standard1")))); - - // Keyspace 5 - keyspaces.add(KeyspaceMetadata.create(ks5, - KeyspaceParams.simple(2), - Tables.of(SchemaLoader.standardCFMD(ks5, "Standard1")))); - - // Keyspace 6 - keyspaces.add(KeyspaceMetadata.create(ks6, - KeyspaceParams.simple(1), - Tables.of(SchemaLoader.keysIndexCFMD(ks6, "Indexed1", true)))); - - // RowCacheSpace - keyspaces.add(KeyspaceMetadata.create(ks_rcs, - KeyspaceParams.simple(1), - Tables.of(SchemaLoader.standardCFMD(ks_rcs, "CFWithoutCache") - .caching(CachingParams.CACHE_NOTHING), - SchemaLoader.standardCFMD(ks_rcs, "CachedCF") - .caching(CachingParams.CACHE_EVERYTHING), - SchemaLoader.standardCFMD(ks_rcs, "CachedIntCF") - .caching(new CachingParams(true, 100))))); - - keyspaces.add(KeyspaceMetadata.create(ks_nocommit, - KeyspaceParams.simpleTransient(1), - Tables.of(SchemaLoader.standardCFMD(ks_nocommit, "Standard1")))); - - // PerRowSecondaryIndexTest - keyspaces.add(KeyspaceMetadata.create(ks_prsi, - KeyspaceParams.simple(1), - Tables.of(SchemaLoader.perRowIndexedCFMD(ks_prsi, "Indexed1")))); - - // CQLKeyspace - keyspaces.add(KeyspaceMetadata.create(ks_cql, - KeyspaceParams.simple(1), - Tables.of(CFMetaData.compile("CREATE TABLE table1 (" - + "k int PRIMARY KEY," - + "v1 text," - + "v2 int" - + ')', ks_cql), - - CFMetaData.compile("CREATE TABLE table2 (" - + "k text," - + "c text," - + "v text," - + "PRIMARY KEY (k, c))", ks_cql), - - CFMetaData.compile("CREATE TABLE foo (" - + "bar text, " - + "baz text, " - + "qux text, " - + "PRIMARY KEY(bar, baz) ) " - + "WITH COMPACT STORAGE", ks_cql), - - CFMetaData.compile("CREATE TABLE compact_pkonly (" - + "k int, " - + "c int, " - + "PRIMARY KEY (k, c)) " - + "WITH COMPACT STORAGE", - ks_cql), - - CFMetaData.compile("CREATE TABLE foofoo (" - + "bar text, " - + "baz text, " - + "qux text, " - + "quz text, " - + "foo text, " - + "PRIMARY KEY((bar, baz), qux, quz) ) " - + "WITH COMPACT STORAGE", ks_cql)))); - - // NTS keyspace - keyspaces.add(KeyspaceMetadata.create("nts", KeyspaceParams.nts("dc1", 1, "dc2", 2))); - - keyspaces.add(keyspaceWithDroppedCollections()); - keyspaces.add(keyspaceWithTriggers()); - keyspaces.add(keyspaceWithUDTs()); - keyspaces.add(keyspaceWithUDFs()); - keyspaces.add(keyspaceWithUDFsAndUDTs()); - keyspaces.add(keyspaceWithUDAs()); - keyspaces.add(keyspaceWithUDAsAndUDTs()); - - return keyspaces.build(); - } - - private static KeyspaceMetadata keyspaceWithDroppedCollections() - { - String keyspace = KEYSPACE_PREFIX + "DroppedCollections"; - - CFMetaData table = - CFMetaData.compile("CREATE TABLE dropped_columns (" - + "foo text," - + "bar text," - + "map1 map," - + "map2 map," - + "set1 set," - + "list1 list," - + "PRIMARY KEY ((foo), bar))", - keyspace); - - String[] collectionColumnNames = { "map1", "map2", "set1", "list1" }; - for (String name : collectionColumnNames) - { - ColumnDefinition column = table.getColumnDefinition(bytes(name)); - table.recordColumnDrop(column, FBUtilities.timestampMicros(), false); - table.removeColumnDefinition(column); - } - - return KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1), Tables.of(table)); - } - - private static KeyspaceMetadata keyspaceWithTriggers() - { - String keyspace = KEYSPACE_PREFIX + "Triggers"; - - Triggers.Builder triggers = Triggers.builder(); - CFMetaData table = SchemaLoader.standardCFMD(keyspace, "WithTriggers"); - for (int i = 0; i < 10; i++) - triggers.add(new TriggerMetadata("trigger" + i, "DummyTrigger" + i)); - table.triggers(triggers.build()); - - return KeyspaceMetadata.create(keyspace, KeyspaceParams.simple(1), Tables.of(table)); - } - - private static KeyspaceMetadata keyspaceWithUDTs() - { - String keyspace = KEYSPACE_PREFIX + "UDTs"; - - UserType udt1 = new UserType(keyspace, - bytes("udt1"), - new ArrayList() {{ add(bytes("col1")); add(bytes("col2")); }}, - new ArrayList>() {{ add(UTF8Type.instance); add(Int32Type.instance); }}); - - UserType udt2 = new UserType(keyspace, - bytes("udt2"), - new ArrayList() {{ add(bytes("col3")); add(bytes("col4")); }}, - new ArrayList>() {{ add(BytesType.instance); add(BooleanType.instance); }}); - - UserType udt3 = new UserType(keyspace, - bytes("udt3"), - new ArrayList() {{ add(bytes("col5")); }}, - new ArrayList>() {{ add(AsciiType.instance); }}); - - return KeyspaceMetadata.create(keyspace, - KeyspaceParams.simple(1), - Tables.none(), - Views.none(), - Types.of(udt1, udt2, udt3), - Functions.none()); - } - - private static KeyspaceMetadata keyspaceWithUDFs() - { - String keyspace = KEYSPACE_PREFIX + "UDFs"; - - UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf"), - ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), - ImmutableList.of(BytesType.instance, Int32Type.instance), - LongType.instance, - false, - "java", - "return 42L;"); - - // an overload with the same name, not a typo - UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf"), - ImmutableList.of(new ColumnIdentifier("col3", false), new ColumnIdentifier("col4", false)), - ImmutableList.of(AsciiType.instance, LongType.instance), - Int32Type.instance, - true, - "java", - "return 42;"); - - UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"), - ImmutableList.of(new ColumnIdentifier("col4", false)), - ImmutableList.of(UTF8Type.instance), - BooleanType.instance, - false, - "java", - "return true;"); - - return KeyspaceMetadata.create(keyspace, - KeyspaceParams.simple(1), - Tables.none(), - Views.none(), - Types.none(), - Functions.of(udf1, udf2, udf3)); - } - - private static KeyspaceMetadata keyspaceWithUDAs() - { - String keyspace = KEYSPACE_PREFIX + "UDAs"; - - UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf1"), - ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), - ImmutableList.of(Int32Type.instance, Int32Type.instance), - Int32Type.instance, - false, - "java", - "return 42;"); - - UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf2"), - ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), - ImmutableList.of(LongType.instance, Int32Type.instance), - LongType.instance, - false, - "java", - "return 42L;"); - - UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"), - ImmutableList.of(new ColumnIdentifier("col1", false)), - ImmutableList.of(LongType.instance), - DoubleType.instance, - false, - "java", - "return 42d;"); - - Functions udfs = Functions.builder().add(udf1).add(udf2).add(udf3).build(); - - UDAggregate uda1 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda1"), - ImmutableList.of(udf1.argTypes().get(1)), - udf1.returnType(), - udf1.name(), - null, - udf1.argTypes().get(0), - null - ); - - UDAggregate uda2 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda2"), - ImmutableList.of(udf2.argTypes().get(1)), - udf3.returnType(), - udf2.name(), - udf3.name(), - udf2.argTypes().get(0), - LongType.instance.decompose(0L) - ); - - return KeyspaceMetadata.create(keyspace, - KeyspaceParams.simple(1), - Tables.none(), - Views.none(), - Types.none(), - Functions.of(udf1, udf2, udf3, uda1, uda2)); - } - - private static KeyspaceMetadata keyspaceWithUDFsAndUDTs() - { - String keyspace = KEYSPACE_PREFIX + "UDFUDTs"; - - UserType udt1 = new UserType(keyspace, - bytes("udt1"), - new ArrayList() {{ add(bytes("col1")); add(bytes("col2")); }}, - new ArrayList>() {{ add(UTF8Type.instance); add(Int32Type.instance); }}); - - UserType udt2 = new UserType(keyspace, - bytes("udt2"), - new ArrayList() {{ add(bytes("col1")); add(bytes("col2")); }}, - new ArrayList>() {{ add(ListType.getInstance(udt1, false)); add(Int32Type.instance); }}); - - UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf"), - ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), - ImmutableList.of(udt1, udt2), - LongType.instance, - false, - "java", - "return 42L;"); - - // an overload with the same name, not a typo - UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf"), - ImmutableList.of(new ColumnIdentifier("col3", false), new ColumnIdentifier("col4", false)), - ImmutableList.of(AsciiType.instance, LongType.instance), - Int32Type.instance, - true, - "java", - "return 42;"); - - UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"), - ImmutableList.of(new ColumnIdentifier("col4", false)), - ImmutableList.of(new TupleType(Arrays.asList(udt1, udt2))), - BooleanType.instance, - false, - "java", - "return true;"); - - return KeyspaceMetadata.create(keyspace, - KeyspaceParams.simple(1), - Tables.none(), - Views.none(), - Types.of(udt1, udt2), - Functions.of(udf1, udf2, udf3)); - } - - private static KeyspaceMetadata keyspaceWithUDAsAndUDTs() - { - String keyspace = KEYSPACE_PREFIX + "UDAUDTs"; - - UserType udt1 = new UserType(keyspace, - bytes("udt1"), - new ArrayList() {{ add(bytes("col1")); add(bytes("col2")); }}, - new ArrayList>() {{ add(UTF8Type.instance); add(Int32Type.instance); }}); - - UserType udt2 = new UserType(keyspace, - bytes("udt2"), - new ArrayList() {{ add(bytes("col1")); add(bytes("col2")); }}, - new ArrayList>() {{ add(ListType.getInstance(udt1, false)); add(Int32Type.instance); }}); - - UDFunction udf1 = UDFunction.create(new FunctionName(keyspace, "udf1"), - ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), - ImmutableList.of(udt1, udt2), - udt1, - false, - "java", - "return null;"); - - UDFunction udf2 = UDFunction.create(new FunctionName(keyspace, "udf2"), - ImmutableList.of(new ColumnIdentifier("col1", false), new ColumnIdentifier("col2", false)), - ImmutableList.of(udt2, udt1), - udt2, - false, - "java", - "return null;"); - - UDFunction udf3 = UDFunction.create(new FunctionName(keyspace, "udf3"), - ImmutableList.of(new ColumnIdentifier("col1", false)), - ImmutableList.of(udt2), - DoubleType.instance, - false, - "java", - "return 42d;"); - - Functions udfs = Functions.builder().add(udf1).add(udf2).add(udf3).build(); - - UDAggregate uda1 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda1"), - ImmutableList.of(udf1.argTypes().get(1)), - udf1.returnType(), - udf1.name(), - null, - udf1.argTypes().get(0), - null - ); - - ByteBuffer twoNullEntries = ByteBuffer.allocate(8); - twoNullEntries.putInt(-1); - twoNullEntries.putInt(-1); - twoNullEntries.flip(); - UDAggregate uda2 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda2"), - ImmutableList.of(udf2.argTypes().get(1)), - udf3.returnType(), - udf2.name(), - udf3.name(), - udf2.argTypes().get(0), - twoNullEntries - ); - - return KeyspaceMetadata.create(keyspace, - KeyspaceParams.simple(1), - Tables.none(), - Views.none(), - Types.of(udt1, udt2), - Functions.of(udf1, udf2, udf3, uda1, uda2)); - } - - /* - * Serializing keyspaces - */ - - private static void legacySerializeKeyspace(KeyspaceMetadata keyspace) - { - makeLegacyCreateKeyspaceMutation(keyspace, TIMESTAMP).apply(); - setLegacyIndexStatus(keyspace); - } - - private static Mutation makeLegacyCreateKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp) - { - // Note that because Keyspaces is a COMPACT TABLE, we're really only setting static columns internally and shouldn't set any clustering. - RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyKeyspaces, timestamp, keyspace.name); - - adder.add("durable_writes", keyspace.params.durableWrites) - .add("strategy_class", keyspace.params.replication.klass.getName()) - .add("strategy_options", json(keyspace.params.replication.options)); - - Mutation mutation = adder.build(); - - keyspace.tables.forEach(table -> addTableToSchemaMutation(table, timestamp, true, mutation)); - keyspace.types.forEach(type -> addTypeToSchemaMutation(type, timestamp, mutation)); - keyspace.functions.udfs().forEach(udf -> addFunctionToSchemaMutation(udf, timestamp, mutation)); - keyspace.functions.udas().forEach(uda -> addAggregateToSchemaMutation(uda, timestamp, mutation)); - - return mutation; - } - - /* - * Serializing tables - */ - - private static void addTableToSchemaMutation(CFMetaData table, long timestamp, boolean withColumnsAndTriggers, Mutation mutation) - { - // For property that can be null (and can be changed), we insert tombstones, to make sure - // we don't keep a property the user has removed - RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyColumnfamilies, timestamp, mutation) - .clustering(table.cfName); - - adder.add("cf_id", table.cfId) - .add("type", table.isSuper() ? "Super" : "Standard"); - - if (table.isSuper()) - { - adder.add("comparator", table.comparator.subtype(0).toString()) - .add("subcomparator", ((MapType)table.compactValueColumn().type).getKeysType().toString()); - } - else - { - adder.add("comparator", LegacyLayout.makeLegacyComparator(table).toString()); - } - - adder.add("bloom_filter_fp_chance", table.params.bloomFilterFpChance) - .add("caching", cachingToString(table.params.caching)) - .add("comment", table.params.comment) - .add("compaction_strategy_class", table.params.compaction.klass().getName()) - .add("compaction_strategy_options", json(table.params.compaction.options())) - .add("compression_parameters", json(ThriftConversion.compressionParametersToThrift(table.params.compression))) - .add("default_time_to_live", table.params.defaultTimeToLive) - .add("gc_grace_seconds", table.params.gcGraceSeconds) - .add("key_validator", table.getKeyValidator().toString()) - .add("local_read_repair_chance", table.params.dcLocalReadRepairChance) - .add("max_compaction_threshold", table.params.compaction.maxCompactionThreshold()) - .add("max_index_interval", table.params.maxIndexInterval) - .add("memtable_flush_period_in_ms", table.params.memtableFlushPeriodInMs) - .add("min_compaction_threshold", table.params.compaction.minCompactionThreshold()) - .add("min_index_interval", table.params.minIndexInterval) - .add("read_repair_chance", table.params.readRepairChance) - .add("speculative_retry", table.params.speculativeRetry.toString()); - - for (Map.Entry entry : table.getDroppedColumns().entrySet()) - { - String name = UTF8Type.instance.getString(entry.getKey()); - CFMetaData.DroppedColumn column = entry.getValue(); - adder.addMapEntry("dropped_columns", name, column.droppedTime); - } - - adder.add("is_dense", table.isDense()); - - adder.add("default_validator", table.makeLegacyDefaultValidator().toString()); - - if (withColumnsAndTriggers) - { - for (ColumnDefinition column : table.allColumns()) - addColumnToSchemaMutation(table, column, timestamp, mutation); - - for (TriggerMetadata trigger : table.getTriggers()) - addTriggerToSchemaMutation(table, trigger, timestamp, mutation); - } - - adder.build(); - } - - private static String cachingToString(CachingParams caching) - { - return format("{\"keys\":\"%s\", \"rows_per_partition\":\"%s\"}", - caching.keysAsString(), - caching.rowsPerPartitionAsString()); - } - - private static void addColumnToSchemaMutation(CFMetaData table, ColumnDefinition column, long timestamp, Mutation mutation) - { - // We need to special case pk-only dense tables. See CASSANDRA-9874. - String name = table.isDense() && column.kind == ColumnDefinition.Kind.REGULAR && column.type instanceof EmptyType - ? "" - : column.name.toString(); - - final RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyColumns, timestamp, mutation).clustering(table.cfName, name); - - adder.add("validator", column.type.toString()) - .add("type", serializeKind(column.kind, table.isDense())) - .add("component_index", column.position()); - - Optional index = findIndexForColumn(table.getIndexes(), table, column); - if (index.isPresent()) - { - IndexMetadata i = index.get(); - adder.add("index_name", i.name); - adder.add("index_type", i.kind.toString()); - adder.add("index_options", json(i.options)); - } - else - { - adder.add("index_name", null); - adder.add("index_type", null); - adder.add("index_options", null); - } - - adder.build(); - } - - private static Optional findIndexForColumn(Indexes indexes, - CFMetaData table, - ColumnDefinition column) - { - // makes the assumptions that the string option denoting the - // index targets can be parsed by CassandraIndex.parseTarget - // which should be true for any pre-3.0 index - for (IndexMetadata index : indexes) - if (CassandraIndex.parseTarget(table, index).left.equals(column)) - return Optional.of(index); - - return Optional.empty(); - } - - private static String serializeKind(ColumnDefinition.Kind kind, boolean isDense) + @Override + public String serializeKind(ColumnDefinition.Kind kind, boolean isDense) { // For backward compatibility, we special case CLUSTERING and the case where the table is dense. if (kind == ColumnDefinition.Kind.CLUSTERING) @@ -715,131 +116,4 @@ public class LegacySchemaMigratorTest return kind.toString().toLowerCase(); } - private static void addTriggerToSchemaMutation(CFMetaData table, TriggerMetadata trigger, long timestamp, Mutation mutation) - { - new RowUpdateBuilder(SystemKeyspace.LegacyTriggers, timestamp, mutation) - .clustering(table.cfName, trigger.name) - .addMapEntry("trigger_options", "class", trigger.classOption) - .build(); - } - - /* - * Serializing types - */ - - private static void addTypeToSchemaMutation(UserType type, long timestamp, Mutation mutation) - { - RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyUsertypes, timestamp, mutation) - .clustering(type.getNameAsString()); - - adder.resetCollection("field_names") - .resetCollection("field_types"); - - for (int i = 0; i < type.size(); i++) - { - adder.addListEntry("field_names", type.fieldName(i)) - .addListEntry("field_types", type.fieldType(i).toString()); - } - - adder.build(); - } - - /* - * Serializing functions - */ - - private static void addFunctionToSchemaMutation(UDFunction function, long timestamp, Mutation mutation) - { - RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyFunctions, timestamp, mutation) - .clustering(function.name().name, functionSignatureWithTypes(function)); - - adder.add("body", function.body()) - .add("language", function.language()) - .add("return_type", function.returnType().toString()) - .add("called_on_null_input", function.isCalledOnNullInput()); - - adder.resetCollection("argument_names") - .resetCollection("argument_types"); - - for (int i = 0; i < function.argNames().size(); i++) - { - adder.addListEntry("argument_names", function.argNames().get(i).bytes) - .addListEntry("argument_types", function.argTypes().get(i).toString()); - } - - adder.build(); - } - - /* - * Serializing aggregates - */ - - private static void addAggregateToSchemaMutation(UDAggregate aggregate, long timestamp, Mutation mutation) - { - RowUpdateBuilder adder = new RowUpdateBuilder(SystemKeyspace.LegacyAggregates, timestamp, mutation) - .clustering(aggregate.name().name, functionSignatureWithTypes(aggregate)); - - adder.resetCollection("argument_types"); - - adder.add("return_type", aggregate.returnType().toString()) - .add("state_func", aggregate.stateFunction().name().name); - - if (aggregate.stateType() != null) - adder.add("state_type", aggregate.stateType().toString()); - if (aggregate.finalFunction() != null) - adder.add("final_func", aggregate.finalFunction().name().name); - if (aggregate.initialCondition() != null) - adder.add("initcond", aggregate.initialCondition()); - - for (AbstractType argType : aggregate.argTypes()) - adder.addListEntry("argument_types", argType.toString()); - - adder.build(); - } - - // We allow method overloads, so a function is not uniquely identified by its name only, but - // also by its argument types. To distinguish overloads of given function name in the schema - // we use a "signature" which is just a list of it's CQL argument types. - public static ByteBuffer functionSignatureWithTypes(AbstractFunction fun) - { - List arguments = - fun.argTypes() - .stream() - .map(argType -> argType.asCQL3Type().toString()) - .collect(Collectors.toList()); - - return ListType.getInstance(UTF8Type.instance, false).decompose(arguments); - } - - private static void setLegacyIndexStatus(KeyspaceMetadata keyspace) - { - keyspace.tables.forEach(LegacySchemaMigratorTest::setLegacyIndexStatus); - } - - private static void setLegacyIndexStatus(CFMetaData table) - { - table.getIndexes().forEach((index) -> setLegacyIndexStatus(table.ksName, table.cfName, index)); - } - - private static void setLegacyIndexStatus(String keyspace, String table, IndexMetadata index) - { - SystemKeyspace.setIndexBuilt(keyspace, table + '.' + index.name); - } - - private static void verifyIndexBuildStatus(KeyspaceMetadata keyspace) - { - keyspace.tables.forEach(LegacySchemaMigratorTest::verifyIndexBuildStatus); - } - - private static void verifyIndexBuildStatus(CFMetaData table) - { - table.getIndexes().forEach(index -> verifyIndexBuildStatus(table.ksName, table.cfName, index)); - } - - private static void verifyIndexBuildStatus(String keyspace, String table, IndexMetadata index) - { - assertFalse(SystemKeyspace.isIndexBuilt(keyspace, table + '.' + index.name)); - assertTrue(SystemKeyspace.isIndexBuilt(keyspace, index.name)); - } - } diff --git a/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorThriftTest.java b/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorThriftTest.java new file mode 100644 index 0000000000..3686024f0f --- /dev/null +++ b/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorThriftTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.schema; + +import java.io.IOException; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.exceptions.ConfigurationException; + +import org.junit.Test; + +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.exceptions.InvalidRequestException; + +import static org.apache.cassandra.SchemaLoader.getCompressionParameters; +import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal; + +@SuppressWarnings("deprecation") +public class LegacySchemaMigratorThriftTest extends LegacySchemaMigratorBaseTest +{ + private static final String KEYSPACE_18956 = "ks18956"; + private static final String TABLE_18956 = "table18956"; + + @Test + public void testMigrate18956() throws IOException + { + CQLTester.cleanupAndLeaveDirs(); + Keyspaces expected = keyspacesToMigrate18956(); + expected.forEach(this::legacySerializeKeyspace); + LegacySchemaMigrator.migrate(); + Schema.instance.loadFromDisk(); + LegacySchemaMigratorBaseTest.loadLegacySchemaTables(); + try + { + // This should fail + executeOnceInternal(String.format("ALTER TABLE %s.%s RENAME key TO \"4f\"", KEYSPACE_18956, TABLE_18956)); + assert false; + } + catch (InvalidRequestException e) + { + assert e.toString().contains("another column of that name already exist"); + } + } + + public static Keyspaces keyspacesToMigrate18956() + { + Keyspaces.Builder keyspaces = Keyspaces.builder(); + keyspaces.add(KeyspaceMetadata.create(LegacySchemaMigratorThriftTest.KEYSPACE_18956, + KeyspaceParams.simple(1), + Tables.of( + bytesTypeComparatorCFMD18956(LegacySchemaMigratorThriftTest.KEYSPACE_18956, LegacySchemaMigratorThriftTest.TABLE_18956) + ))); + return keyspaces.build(); + } + + public static CFMetaData bytesTypeComparatorCFMD18956(String ksName, String cfName) throws ConfigurationException + { + return CFMetaData.Builder.createDense(ksName, cfName, false, false) + .addPartitionKey("key", BytesType.instance) + .addClusteringColumn("3d", BytesType.instance) + .addRegularColumn("4f", BytesType.instance) + .build() + .compression(getCompressionParameters()); + } + + @Override + public String serializeKind(ColumnDefinition.Kind kind, boolean isDense) + { + // Using cassandra-cli, it's possible to create legacy without compact_value + return kind == ColumnDefinition.Kind.CLUSTERING ? "clustering_key" : kind.toString().toLowerCase(); + } +}