diff --git a/CHANGES.txt b/CHANGES.txt index 812e02051a..d6ff2e121e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,7 @@ Merged from 3.0: * Forbid altering UDTs used in partition keys (CASSANDRA-15933) * Fix empty/null json string representation (CASSANDRA-15896) + * 3.x fails to start if commit log has range tombstones from a column which is also deleted (CASSANDRA-15970) Merged from 2.2: * Fix CQL parsing of collections when the column type is reversed (CASSANDRA-15814) diff --git a/src/java/org/apache/cassandra/db/LegacyLayout.java b/src/java/org/apache/cassandra/db/LegacyLayout.java index b28c72a932..673aa4f849 100644 --- a/src/java/org/apache/cassandra/db/LegacyLayout.java +++ b/src/java/org/apache/cassandra/db/LegacyLayout.java @@ -1600,6 +1600,8 @@ public abstract class LegacyLayout */ public Row getRow() { + if (clustering == null) + return null; if (!hasValidCells && invalidLivenessInfo != null) builder.addPrimaryKeyLivenessInfo(invalidLivenessInfo); return builder.build(); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 2c4e409741..8e712db9fe 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -285,7 +285,7 @@ public class NodeProbe implements AutoCloseable private void checkJobs(PrintStream out, int jobs) { // TODO this should get the configured number of concurrent_compactors via JMX and not using DatabaseDescriptor - DatabaseDescriptor.toolInitialization(); + DatabaseDescriptor.toolInitialization(false); // if running in dtest, this would fail if true (default) if (jobs > DatabaseDescriptor.getConcurrentCompactors()) out.println(String.format("jobs (%d) is bigger than configured concurrent_compactors (%d) on this host, using at most %d threads", jobs, DatabaseDescriptor.getConcurrentCompactors(), DatabaseDescriptor.getConcurrentCompactors())); } diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java index 41d524747f..1dc2774d31 100644 --- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java +++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java @@ -470,6 +470,8 @@ public class ByteBufferUtil return ByteBufferUtil.bytes((InetAddress) obj); else if (obj instanceof String) return ByteBufferUtil.bytes((String) obj); + else if (obj instanceof ByteBuffer) + return (ByteBuffer) obj; else throw new IllegalArgumentException(String.format("Cannot convert value %s of type %s", obj, diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns.java new file mode 100644 index 0000000000..c8c04d1317 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns.java @@ -0,0 +1,113 @@ +package org.apache.cassandra.distributed.upgrade; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Objects; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.distributed.api.QueryResults; +import org.apache.cassandra.distributed.api.SimpleQueryResult; +import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.distributed.shared.Versions; +import org.apache.cassandra.distributed.test.ThriftClientUtils; +import org.apache.cassandra.thrift.Deletion; +import org.apache.cassandra.thrift.Mutation; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.SliceRange; +import org.apache.cassandra.utils.ByteBufferUtil; + +public abstract class MigrateDropColumns extends UpgradeTestBase +{ + private static final MapType MAP_TYPE = MapType.getInstance(Int32Type.instance, Int32Type.instance, true); + + private final Versions.Major initial; + private final Versions.Major[] upgrade; + + protected MigrateDropColumns(Versions.Major initial, Versions.Major... upgrade) + { + this.initial = Objects.requireNonNull(initial, "initial"); + this.upgrade = Objects.requireNonNull(upgrade, "upgrade"); + } + + @Test + public void dropColumns() throws Throwable + { + new TestCase() + .upgrade(initial, upgrade) + .withConfig(c -> c.with(Feature.NATIVE_PROTOCOL)) + .setup(cluster -> { + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl(pk int, tables map, PRIMARY KEY (pk))")); + + ICoordinator coordinator = cluster.coordinator(1); + + // write a RT to pk=0 + ThriftClientUtils.thriftClient(cluster.get(1), thrift -> { + thrift.set_keyspace(KEYSPACE); + + Mutation mutation = new Mutation(); + Deletion deletion = new Deletion(); + SlicePredicate slice = new SlicePredicate(); + SliceRange range = new SliceRange(); + range.setStart(CompositeType.build(ByteBufferUtil.bytes("tables"))); + range.setFinish(CompositeType.build(ByteBufferUtil.bytes("tables"))); + slice.setSlice_range(range); + deletion.setPredicate(slice); + deletion.setTimestamp(System.currentTimeMillis()); + mutation.setDeletion(deletion); + + thrift.batch_mutate(Collections.singletonMap(ByteBufferUtil.bytes(0), + Collections.singletonMap("tbl", Arrays.asList(mutation))), + org.apache.cassandra.thrift.ConsistencyLevel.ALL); + }); + + // write table to pk=1 + // NOTE: because jvm-dtest doesn't support collections in the execute interface (see CASSANDRA-15969) + // need to encode to a ByteBuffer first + coordinator.execute(withKeyspace("INSERT INTO %s.tbl (pk, tables) VALUES (?, ?)"), ConsistencyLevel.ONE, 1, MAP_TYPE.decompose(ImmutableMap.of(1, 1))); + + cluster.forEach(inst -> inst.flush(KEYSPACE)); + + cluster.schemaChange(withKeyspace("ALTER TABLE %s.tbl DROP tables")); + }) + .runAfterClusterUpgrade(cluster -> { + ICoordinator coordinator = cluster.coordinator(1); + SimpleQueryResult qr = coordinator.executeWithResult("SELECT column_name " + + "FROM system_schema.dropped_columns " + + "WHERE keyspace_name=?" + + " AND table_name=?;", + ConsistencyLevel.ALL, KEYSPACE, "tbl"); + Assert.assertEquals(ImmutableSet.of("tables"), Sets.newHashSet(qr.map(r -> r.getString("column_name")))); + + assertRows(coordinator); + + // upgradesstables, make sure everything is still working + cluster.forEach(n -> n.nodetoolResult("upgradesstables", KEYSPACE).asserts().success()); + + assertRows(coordinator); + }) + .run(); + } + + private static void assertRows(ICoordinator coordinator) + { + // since only a RT was written to this row there is no liveness information, so the row will be skipped + AssertUtils.assertRows( + coordinator.executeWithResult(withKeyspace("SELECT * FROM %s.tbl WHERE pk=?"), ConsistencyLevel.ALL, 0), + QueryResults.empty()); + + AssertUtils.assertRows( + coordinator.executeWithResult(withKeyspace("SELECT * FROM %s.tbl WHERE pk=?"), ConsistencyLevel.ALL, 1), + QueryResults.builder().row(1).build()); + } +} \ No newline at end of file diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns22To30To311Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns22To30To311Test.java new file mode 100644 index 0000000000..e98b0232c7 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns22To30To311Test.java @@ -0,0 +1,11 @@ +package org.apache.cassandra.distributed.upgrade; + +import org.apache.cassandra.distributed.shared.Versions; + +public class MigrateDropColumns22To30To311Test extends MigrateDropColumns +{ + public MigrateDropColumns22To30To311Test() + { + super(Versions.Major.v22, Versions.Major.v30, Versions.Major.v3X); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns22To311Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns22To311Test.java new file mode 100644 index 0000000000..b1670bb0b3 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns22To311Test.java @@ -0,0 +1,11 @@ +package org.apache.cassandra.distributed.upgrade; + +import org.apache.cassandra.distributed.shared.Versions; + +public class MigrateDropColumns22To311Test extends MigrateDropColumns +{ + public MigrateDropColumns22To311Test() + { + super(Versions.Major.v22, Versions.Major.v3X); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns30To311Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns30To311Test.java new file mode 100644 index 0000000000..a0e0d1cb92 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MigrateDropColumns30To311Test.java @@ -0,0 +1,11 @@ +package org.apache.cassandra.distributed.upgrade; + +import org.apache.cassandra.distributed.shared.Versions; + +public class MigrateDropColumns30To311Test extends MigrateDropColumns +{ + public MigrateDropColumns30To311Test() + { + super(Versions.Major.v30, Versions.Major.v3X); + } +}