- * <=oa overflow policy triggers at year 2038. That could be <=4.1 or 5.0 with 4.x storage compatibility - * >oa overflow policy triggers at year 2106. That is >=5.0 using >=5.x storage compatibility - * - * @see StorageCompatibilityMode - */ -public class MixedModeTTLOverflowAfterUpgradeTest extends UpgradeTestBase +public class MixedModeTTLOverflowAfterUpgradeTest extends MixedModeTTLOverflowUpgradeTestBase { @Test public void testTTLOverflowAfterUpgrade() throws Throwable { - MixedModeTTLOverflowUpgradeTest.testTTLOverflow((cluster, node) -> { + testTTLOverflow((cluster, node) -> { + cluster.disableAutoCompaction(KEYSPACE); if (node == 1) // only node1 is upgraded, and the cluster is in mixed versions mode { - assertPolicyTriggersAt2038(cluster.coordinator(1)); - assertPolicyTriggersAt2038(cluster.coordinator(2)); + verify(Step.NODE1_40_NODE2_PREV, cluster, true); } else // both nodes have been upgraded, and the cluster isn't in mixed version mode anymore { - assertPolicyTriggersAt2038(cluster.coordinator(1)); - assertPolicyTriggersAt2038(cluster.coordinator(2)); + verify(Step.NODE1_40_NODE2_40, cluster, true); - // We restart one node on 5.0 >oa hence 2038 should still be the limit as the other node is 5.0 <=oa - // We're on compatibility mode where oa and oa nodes are a possibility + // We restart node1 with compatibility mode UPGRADING restartNodeWithCompatibilityMode(cluster, 1, UPGRADING); - assertPolicyTriggersAt2038(cluster.coordinator(1)); - assertPolicyTriggersAt2038(cluster.coordinator(2)); + // since node2 is still in 4.0 compatibility mode, the limit should remain 2038 + verify(Step.NODE1_UPGRADING_NODE2_40, cluster, true); - // We restart the other node so they're all on 5.0 >oa hence 2106 should be the limit + // We restart node2 in UPGRADING compatibility mode restartNodeWithCompatibilityMode(cluster, 2, UPGRADING); - assertPolicyTriggersAt2106(cluster.coordinator(1)); - assertPolicyTriggersAt2106(cluster.coordinator(2)); + // Both nodes are in UPGRADING compatibility mode, so the limit should be 2106 + verify(Step.NODE1_UPGRADING_NODE2_UPGRADING, cluster, false); - // We restart the cluster out of compatibility mode once everything is 5.0oa TTL 2106 + // We restart the cluster out of compatibility mode, so the limit should be 2106 restartNodeWithCompatibilityMode(cluster, 1, NONE); + verify(Step.NODE1_NONE_NODE2_UPGRADING, cluster, false); + restartNodeWithCompatibilityMode(cluster, 2, NONE); - assertPolicyTriggersAt2106(cluster.coordinator(1)); - assertPolicyTriggersAt2106(cluster.coordinator(2)); + verify(Step.NODE1_NONE_NODE2_NONE, cluster, false); } }); } diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeTTLOverflowDuringUpgradeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeTTLOverflowDuringUpgradeTest.java new file mode 100644 index 0000000000..c78a4ae252 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeTTLOverflowDuringUpgradeTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.upgrade; + +import org.junit.Test; + +import static org.apache.cassandra.utils.StorageCompatibilityMode.NONE; +import static org.apache.cassandra.utils.StorageCompatibilityMode.UPGRADING; + +public class MixedModeTTLOverflowDuringUpgradeTest extends MixedModeTTLOverflowUpgradeTestBase +{ + @Test + public void testTTLOverflowDuringUpgrade() throws Throwable + { + testTTLOverflow((cluster, node) -> { + cluster.disableAutoCompaction(KEYSPACE); + if (node == 1) // only node1 is upgraded, and the cluster is in mixed versions mode + { + verify(Step.NODE1_40_NODE2_PREV, cluster, true); + + // We restart the upgraded node 1 with compatibility mode = UPGRADING + restartNodeWithCompatibilityMode(cluster, 1, UPGRADING); + // 2038 should still be the limit, because node2 is not upgraded yet + verify(Step.NODE1_UPGRADING_NODE2_PREV, cluster, true); + } + else // both nodes have been upgraded, and the cluster isn't in mixed version mode anymore + { + // Once we have completed the upgrade, 2038 should still be the limit because + // node2 is still in 4.x compatibility mode + verify(Step.NODE1_UPGRADING_NODE2_40, cluster, true); + + // We restart the last upgraded node in UPGRADING compatibility mode + restartNodeWithCompatibilityMode(cluster, 2, UPGRADING); + // Both nodes are in UPGRADING compatibility mode, so the limit should be 2106 + verify(Step.NODE1_UPGRADING_NODE2_UPGRADING, cluster, false); + + // We restart get both nodes out of compatibility mode, so the limit should be 2106. + restartNodeWithCompatibilityMode(cluster, 1, NONE); + verify(Step.NODE1_NONE_NODE2_UPGRADING, cluster, false); + restartNodeWithCompatibilityMode(cluster, 2, NONE); + verify(Step.NODE1_NONE_NODE2_NONE, cluster, false); + } + }); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeTTLOverflowUpgradeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeTTLOverflowUpgradeTest.java deleted file mode 100644 index 390865ddb5..0000000000 --- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeTTLOverflowUpgradeTest.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.distributed.upgrade; - -import org.junit.Test; - -import org.apache.cassandra.cql3.Attributes; -import org.apache.cassandra.db.rows.Cell; -import org.apache.cassandra.distributed.UpgradeableCluster; -import org.apache.cassandra.distributed.api.Feature; -import org.apache.cassandra.distributed.api.ICoordinator; -import org.apache.cassandra.utils.Clock; -import org.apache.cassandra.utils.StorageCompatibilityMode; -import org.assertj.core.api.Assertions; - -import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; -import static org.apache.cassandra.utils.StorageCompatibilityMode.NONE; -import static org.apache.cassandra.utils.StorageCompatibilityMode.UPGRADING; - -/** - * Tests TTL the overflow policy triggers at the correct limit: year 2038 <=nb or 2186 >=oa - *
- * <=oa overflow policy triggers at year 2038. That could be <=4.1 or 5.0 with 4.x storage compatibility - * >oa overflow policy triggers at year 2106. That is >=5.0 using >=5.x storage compatibility - * - * @see StorageCompatibilityMode - */ -public class MixedModeTTLOverflowUpgradeTest extends UpgradeTestBase -{ - @Test - public void testTTLOverflowDuringUpgrade() throws Throwable - { - testTTLOverflow((cluster, node) -> { - if (node == 1) // only node1 is upgraded, and the cluster is in mixed versions mode - { - assertPolicyTriggersAt2038(cluster.coordinator(1)); - assertPolicyTriggersAt2038(cluster.coordinator(2)); - - // We restart the upgraded node out of 4.0 storage compatibility, - // and we set it to be compatible with 5.0. - // 2038 should still be the limit because there is still a not upgraded node. - restartNodeWithCompatibilityMode(cluster, 1, UPGRADING); - assertPolicyTriggersAt2038(cluster.coordinator(1)); - assertPolicyTriggersAt2038(cluster.coordinator(2)); - } - else // both nodes have been upgraded, and the cluster isn't in mixed version mode anymore - { - // Once we have completed the upgrade, 2038 should still be the limit because there is still one node - // in 5.0 storage compatibility mode. - assertPolicyTriggersAt2038(cluster.coordinator(1)); - assertPolicyTriggersAt2038(cluster.coordinator(2)); - - // We restart the last upgraded node in 5.0 compatibility mode, so both nodes are now in 5.0 - // compatibility mode, and the limit should be 2106. - restartNodeWithCompatibilityMode(cluster, 2, UPGRADING); - assertPolicyTriggersAt2106(cluster.coordinator(1)); - assertPolicyTriggersAt2106(cluster.coordinator(2)); - - // We restart get both nodes out of compatibility mode, so the limit should be 2106. - restartNodeWithCompatibilityMode(cluster, 1, NONE); - restartNodeWithCompatibilityMode(cluster, 2, NONE); - assertPolicyTriggersAt2106(cluster.coordinator(1)); - assertPolicyTriggersAt2106(cluster.coordinator(2)); - } - }); - } - - static void testTTLOverflow(RunOnClusterAndNode runAfterNodeUpgrade) throws Throwable - { - new TestCase() - .nodes(2) - .nodesToUpgradeOrdered(1, 2) - // all upgrades from v40 to current, excluding v50 -> v51 - .singleUpgradeToCurrentFrom(v40) - .singleUpgradeToCurrentFrom(v41) - .withConfig(c -> c.with(Feature.GOSSIP).set("storage_compatibility_mode", "CASSANDRA_4")) - .setup(cluster -> { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int PRIMARY KEY, v int)")); - - assertPolicyTriggersAt2038(cluster.coordinator(1)); - assertPolicyTriggersAt2038(cluster.coordinator(2)); - }) - .runAfterNodeUpgrade(runAfterNodeUpgrade) - .run(); - } - - static void restartNodeWithCompatibilityMode(UpgradeableCluster cluster, int node, StorageCompatibilityMode mode) throws Throwable - { - cluster.get(node).shutdown().get(); - cluster.get(node).config().set("storage_compatibility_mode", mode.toString()); - cluster.get(node).startup(); - } - - static void assertPolicyTriggersAt2038(ICoordinator coordinator) - { - Assertions.assertThatThrownBy(() -> coordinator.execute(withKeyspace("INSERT INTO %s.t (k, v) VALUES (0, 0) USING TTL " + Attributes.MAX_TTL), ALL)) - .hasMessageContaining("exceeds maximum supported expiration date") - .hasMessageContaining("2038"); - } - - static void assertPolicyTriggersAt2106(ICoordinator coordinator) - { - boolean overflowPoliciesApply = (Clock.Global.currentTimeMillis() / 1000) > (Cell.MAX_DELETION_TIME - Attributes.MAX_TTL); - - if (overflowPoliciesApply) - { - // This code won't run until 2086 - Assertions.assertThatThrownBy(() -> coordinator.execute(withKeyspace("INSERT INTO %s.t (k, v) VALUES (0, 0) USING TTL " + Attributes.MAX_TTL), ALL)) - .hasMessageContaining("exceeds maximum supported expiration date") - .hasMessageContaining("2106"); - } - else - coordinator.execute(withKeyspace("INSERT INTO %s.t (k, v) VALUES (0, 0) USING TTL " + Attributes.MAX_TTL), ALL); - } -} diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeTTLOverflowUpgradeTestBase.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeTTLOverflowUpgradeTestBase.java new file mode 100644 index 0000000000..c0a4b5b747 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeTTLOverflowUpgradeTestBase.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.upgrade; + +import java.util.List; +import java.util.function.BiConsumer; +import java.util.stream.Stream; + +import org.apache.cassandra.cql3.Attributes; +import org.apache.cassandra.distributed.UpgradeableCluster; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.StorageCompatibilityMode; +import org.assertj.core.api.Assertions; +import org.assertj.core.data.Offset; + +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.LOCAL_ONE; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests TTL the overflow policy triggers at the correct limit + *
+ * sstable version < BIG:oa, overflow policy triggers at year 2038. That could be <=4.1 or 5.0 with 4.x storage compatibility
+ * sstable version >= BIG:oa or BTI - overflow policy triggers at year 2106. That is >=5.0 using no storage compatibility
+ *
+ * @see StorageCompatibilityMode
+ *
+ * This test has been split by subclassing in order to avoid OOMs.
+ */
+public abstract class MixedModeTTLOverflowUpgradeTestBase extends UpgradeTestBase
+{
+ static final int SMALL_TTL = 3600;
+
+ static final String T_REGULAR = "table_regular";
+ static final String T_CLUST = "table_clust";
+ static final String T_STATIC = "table_static";
+ static final String T_COMPLEX = "table_complex";
+ static final String T_FROZEN = "table_frozen";
+ static final String T_INDEX = "table_indexed";
+ static final String INDEX = "idx";
+ static final String TYPE = "complex_type";
+
+ static final int NODE_1_MAX_TTL_KEY_OFFSET = 1000;
+ static final int NODE_2_MAX_TTL_KEY_OFFSET = 2000;
+ static final int NODE_1_MIXED_TTL_KEY_OFFSET = 3000;
+ static final int NODE_2_MIXED_TTL_KEY_OFFSET = 4000;
+
+ enum Step
+ {
+ NODE1_PREV_NODE2_PREV,
+ NODE1_40_NODE2_PREV,
+ NODE1_UPGRADING_NODE2_PREV,
+ NODE1_40_NODE2_40,
+ NODE1_UPGRADING_NODE2_40,
+ NODE1_UPGRADING_NODE2_UPGRADING,
+ NODE1_NONE_NODE2_UPGRADING,
+ NODE1_NONE_NODE2_NONE,
+ }
+
+ static volatile long clusterStatupTime = 0;
+
+ static void testTTLOverflow(RunOnClusterAndNode runAfterNodeUpgrade) throws Throwable
+ {
+ new TestCase()
+ .nodes(2)
+ .nodesToUpgradeOrdered(1, 2)
+ // all upgrades from v40 to current, excluding v50 -> v51
+ .singleUpgradeToCurrentFrom(v40)
+ .singleUpgradeToCurrentFrom(v41)
+ .withConfig(c -> c.with(Feature.GOSSIP).set("storage_compatibility_mode", "CASSANDRA_4"))
+ .setup(cluster -> {
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (k int PRIMARY KEY, v1 int, v2 int)", KEYSPACE, T_REGULAR));
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (k int, c int, v1 int, v2 int, PRIMARY KEY (k, c))", KEYSPACE, T_CLUST));
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (k int, c int, v1 int static, v2 int, PRIMARY KEY (k, c))", KEYSPACE, T_STATIC));
+ cluster.schemaChange(String.format("CREATE TYPE %s.%s (a int, b int)", KEYSPACE, TYPE));
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (k int PRIMARY KEY, v1 %s, v2 int)", KEYSPACE, T_COMPLEX, TYPE));
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (k int PRIMARY KEY, v1 frozen<%s>, v2 int)", KEYSPACE, T_FROZEN, TYPE));
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (k int PRIMARY KEY, v1 int, v2 int)", KEYSPACE, T_INDEX));
+ cluster.schemaChange(String.format("CREATE INDEX %s ON %s.%s (v1)", INDEX, KEYSPACE, T_INDEX));
+
+ cluster.disableAutoCompaction(KEYSPACE);
+ clusterStatupTime = Clock.Global.currentTimeMillis();
+ verify(Step.NODE1_PREV_NODE2_PREV, cluster, true);
+ })
+ .runAfterNodeUpgrade(runAfterNodeUpgrade)
+ .run();
+ }
+
+ /**
+ * Verifies that the TTL overflow policy triggers at the correct limit for a variety types
+ * @param step the step in the upgrade process (manily use a unique primary key for each verification)
+ * @param cluster the cluster
+ * @param expectPolicyTriggerAt2038 when true, we expect the overflow policy to trigger at 2038 and attempts to set
+ * a TTL which would result in expiration date after 2038 to fail. Otherwise, the
+ * allowed expiration date is 2106, and we cannot test that for now because of
+ * {@link Attributes#MAX_TTL} limit of 20 years.
+ */
+ static void verify(Step step, UpgradeableCluster cluster, boolean expectPolicyTriggerAt2038)
+ {
+ insert(cluster, step.ordinal(), expectPolicyTriggerAt2038);
+ query(cluster, step.ordinal(), expectPolicyTriggerAt2038);
+ cluster.coordinator(1).instance().flush(KEYSPACE);
+ query(cluster, step.ordinal(), expectPolicyTriggerAt2038);
+ cluster.coordinator(2).instance().flush(KEYSPACE);
+ query(cluster, step.ordinal(), expectPolicyTriggerAt2038);
+ }
+
+ private static void insert(UpgradeableCluster cluster, int step, boolean expectPolicyTriggerAt2038)
+ {
+ BiConsumer