From b6d8e2ce6bac816279ce0490fd6292aa7e4124b6 Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Wed, 7 Sep 2022 16:23:01 +0200 Subject: [PATCH] Make disabling auto snapshot on selected tables possible patch by Tommy Stendahl; reviewed by Stefan Miklosovic and Aleksey Yeschenko for CASSANDRA-10383 Co-authored-by: Tommy Stendahl Co-authored-by: Stefan Miklosovic --- CHANGES.txt | 1 + NEWS.txt | 5 + pylib/cqlshlib/cql3handling.py | 3 + pylib/cqlshlib/test/test_cqlsh_completion.py | 8 +- pylib/cqlshlib/test/test_cqlsh_output.py | 1 + .../statements/schema/TableAttributes.java | 3 + .../cassandra/db/ColumnFamilyStore.java | 11 +- .../cassandra/schema/SchemaKeyspace.java | 54 +++--- .../cassandra/schema/TableMetadata.java | 6 + .../apache/cassandra/schema/TableParams.java | 18 +- .../test/AllowAutoSnapshotTest.java | 156 ++++++++++++++++++ .../statements/DescribeStatementTest.java | 2 + .../cassandra/db/SchemaCQLHelperTest.java | 1 + .../cassandra/schema/SchemaKeyspaceTest.java | 40 +++++ 14 files changed, 281 insertions(+), 28 deletions(-) create mode 100644 test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java diff --git a/CHANGES.txt b/CHANGES.txt index c36017f157..3328974dad 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.2 + * Make disabling auto snapshot on selected tables possible (CASSANDRA-10383) * Introduce compaction priorities to prevent upgrade compaction inability to finish (CASSANDRA-17851) * Prevent a user from manually removing ephemeral snapshots (CASSANDRA-17757) * Remove dependency on Maven Ant Tasks (CASSANDRA-17750) diff --git a/NEWS.txt b/NEWS.txt index b488acbf20..9457c343bb 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -85,6 +85,9 @@ New features isEmptyWithoutStatus() our usage of hbState() + applicationState), however there are other failure cases which block host replacements and require intrusive workarounds and human intervention to recover from when you have something in hbState() you don't expect. See CASSANDRA-17842 for further details. + - Added new CQL table property 'allow_auto_snapshot' which is by default true. When set to false and 'auto_snapshot: true' + in cassandra.yaml, there will be no snapshot taken when a table is truncated or dropped. When auto_snapshot in + casandra.yaml is set to false, the newly added table property does not have any effect. Upgrading --------- @@ -92,6 +95,8 @@ Upgrading there is a dedicated flag in snapshot manifest instead. On upgrade of a node to version 4.2, on node's start, in case there are such ephemeral snapshots on disk, they will be deleted (same behaviour as before) and any new ephemeral snapshots will stop to create ephemeral marker files as flag in a snapshot manifest was introduced instead. + - There was new table property introduced called 'allow_auto_snapshot' (see section 'New features'). Hence, upgraded + node will be on a new schema version. Please do a rolling upgrade of nodes of a cluster to converge to one schema version. Deprecation ----------- diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py index 7e123bd67a..0e14676ad5 100644 --- a/pylib/cqlshlib/cql3handling.py +++ b/pylib/cqlshlib/cql3handling.py @@ -43,6 +43,7 @@ NONALTERBALE_KEYSPACES = ('system', 'system_schema', 'system_views', 'system_vir class Cql3ParsingRuleSet(CqlParsingRuleSet): columnfamily_layout_options = ( + ('allow_auto_snapshot', None), ('bloom_filter_fp_chance', None), ('comment', None), ('gc_grace_seconds', None), @@ -514,6 +515,8 @@ def cf_prop_val_completer(ctxt, cass): return [Hint('')] if this_opt in ('read_repair'): return [Hint('<\'none\'|\'blocking\'>')] + if this_opt == 'allow_auto_snapshot': + return [Hint('')] return [Hint('')] diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py index 06f7664712..8a2679dbde 100644 --- a/pylib/cqlshlib/test/test_cqlsh_completion.py +++ b/pylib/cqlshlib/test/test_cqlsh_completion.py @@ -616,7 +616,8 @@ class TestCqlshCompletion(CqlshCompletionCase): self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) W', immediate='ITH ') self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ', - choices=['bloom_filter_fp_chance', 'compaction', + choices=['allow_auto_snapshot', + 'bloom_filter_fp_chance', 'compaction', 'compression', 'default_time_to_live', 'gc_grace_seconds', 'max_index_interval', @@ -625,7 +626,8 @@ class TestCqlshCompletion(CqlshCompletionCase): 'COMPACT', 'caching', 'comment', 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ', - choices=['bloom_filter_fp_chance', 'compaction', + choices=['allow_auto_snapshot', + 'bloom_filter_fp_chance', 'compaction', 'compression', 'default_time_to_live', 'gc_grace_seconds', 'max_index_interval', @@ -673,7 +675,7 @@ class TestCqlshCompletion(CqlshCompletionCase): choices=[';', 'AND']) self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = " + "{'class': 'SizeTieredCompactionStrategy'} AND ", - choices=['bloom_filter_fp_chance', 'compaction', + choices=['allow_auto_snapshot', 'bloom_filter_fp_chance', 'compaction', 'compression', 'default_time_to_live', 'gc_grace_seconds', 'max_index_interval', diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index c430f46766..e54bf24234 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -652,6 +652,7 @@ class TestCqlshOutput(BaseTestCase): varcharcol text, varintcol varint ) WITH additional_write_policy = '99p' + AND allow_auto_snapshot = true AND bloom_filter_fp_chance = 0.01 AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} AND cdc = false diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index fd31e43ec0..21cf2c984c 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -98,6 +98,9 @@ public final class TableAttributes extends PropertyDefinitions private TableParams build(TableParams.Builder builder) { + if (hasOption(Option.ALLOW_AUTO_SNAPSHOT)) + builder.allowAutoSnapshot(getBoolean(Option.ALLOW_AUTO_SNAPSHOT.toString(), true)); + if (hasOption(Option.BLOOM_FILTER_FP_CHANCE)) builder.bloomFilterFpChance(getDouble(Option.BLOOM_FILTER_FP_CHANCE)); diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index e4b9d781e7..118069b38d 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -2581,7 +2581,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner if (!noSnapshot && ((keyspace.getMetadata().params.durableWrites && !memtableWritesAreDurable()) // need to clear dirty regions - || DatabaseDescriptor.isAutoSnapshot())) // need sstable for snapshot + || isAutoSnapshotEnabled())) { replayAfter = forceBlockingFlush(FlushReason.TRUNCATE); viewManager.forceBlockingFlush(FlushReason.TRUNCATE); @@ -2614,7 +2614,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner "Stopping parent sessions {} due to truncation of tableId="+metadata.id); data.notifyTruncated(truncatedAt); - if (!noSnapshot && DatabaseDescriptor.isAutoSnapshot()) + if (!noSnapshot && isAutoSnapshotEnabled()) snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, SNAPSHOT_TRUNCATE_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl()); discardSSTables(truncatedAt); @@ -3073,6 +3073,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return metadata().params.caching.cacheKeys() && CacheService.instance.keyCache.getCapacity() > 0; } + public boolean isAutoSnapshotEnabled() + { + return metadata().params.allowAutoSnapshot && DatabaseDescriptor.isAutoSnapshot(); + } + /** * Discard all SSTables that were created before given timestamp. * @@ -3207,7 +3212,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner CompactionManager.instance.interruptCompactionForCFs(concatWithIndexes(), (sstable) -> true, true); - if (DatabaseDescriptor.isAutoSnapshot()) + if (isAutoSnapshotEnabled()) snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, ColumnFamilyStore.SNAPSHOT_DROP_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl()); CommitLog.instance.forceRecycleAllSegments(Collections.singleton(metadata.id)); diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 33d2b7d6c8..cbcba70caa 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -100,6 +100,7 @@ public final class SchemaKeyspace "CREATE TABLE %s (" + "keyspace_name text," + "table_name text," + + "allow_auto_snapshot boolean," + "bloom_filter_fp_chance double," + "caching frozen>," + "comment text," @@ -168,6 +169,7 @@ public final class SchemaKeyspace + "base_table_id uuid," + "base_table_name text," + "where_clause text," + + "allow_auto_snapshot boolean," + "bloom_filter_fp_chance double," + "caching frozen>," + "comment text," @@ -563,6 +565,11 @@ public final class SchemaKeyspace // in mixed operation with pre-4.1 versioned node during upgrades. if (params.memtable != MemtableParams.DEFAULT) builder.add("memtable", params.memtable.configurationKey()); + + // As above, only add the allow_auto_snapshot column if the value is not default (true) and + // auto-snapshotting is enabled, to avoid RTE in pre-4.2 versioned node during upgrades + if (!params.allowAutoSnapshot) + builder.add("allow_auto_snapshot", false); } private static void addAlterTableToSchemaMutation(TableMetadata oldTable, TableMetadata newTable, Mutation.SimpleBuilder builder) @@ -954,27 +961,32 @@ public final class SchemaKeyspace @VisibleForTesting static TableParams createTableParamsFromRow(UntypedResultSet.Row row) { - return TableParams.builder() - .bloomFilterFpChance(row.getDouble("bloom_filter_fp_chance")) - .caching(CachingParams.fromMap(row.getFrozenTextMap("caching"))) - .comment(row.getString("comment")) - .compaction(CompactionParams.fromMap(row.getFrozenTextMap("compaction"))) - .compression(CompressionParams.fromMap(row.getFrozenTextMap("compression"))) - .memtable(MemtableParams.get(row.has("memtable") ? row.getString("memtable") : null)) // memtable column was introduced in 4.1 - .defaultTimeToLive(row.getInt("default_time_to_live")) - .extensions(row.getFrozenMap("extensions", UTF8Type.instance, BytesType.instance)) - .gcGraceSeconds(row.getInt("gc_grace_seconds")) - .maxIndexInterval(row.getInt("max_index_interval")) - .memtableFlushPeriodInMs(row.getInt("memtable_flush_period_in_ms")) - .minIndexInterval(row.getInt("min_index_interval")) - .crcCheckChance(row.getDouble("crc_check_chance")) - .speculativeRetry(SpeculativeRetryPolicy.fromString(row.getString("speculative_retry"))) - .additionalWritePolicy(row.has("additional_write_policy") ? - SpeculativeRetryPolicy.fromString(row.getString("additional_write_policy")) : - SpeculativeRetryPolicy.fromString("99PERCENTILE")) - .cdc(row.has("cdc") && row.getBoolean("cdc")) - .readRepair(getReadRepairStrategy(row)) - .build(); + TableParams.Builder builder = TableParams.builder() + .bloomFilterFpChance(row.getDouble("bloom_filter_fp_chance")) + .caching(CachingParams.fromMap(row.getFrozenTextMap("caching"))) + .comment(row.getString("comment")) + .compaction(CompactionParams.fromMap(row.getFrozenTextMap("compaction"))) + .compression(CompressionParams.fromMap(row.getFrozenTextMap("compression"))) + .memtable(MemtableParams.get(row.has("memtable") ? row.getString("memtable") : null)) // memtable column was introduced in 4.1 + .defaultTimeToLive(row.getInt("default_time_to_live")) + .extensions(row.getFrozenMap("extensions", UTF8Type.instance, BytesType.instance)) + .gcGraceSeconds(row.getInt("gc_grace_seconds")) + .maxIndexInterval(row.getInt("max_index_interval")) + .memtableFlushPeriodInMs(row.getInt("memtable_flush_period_in_ms")) + .minIndexInterval(row.getInt("min_index_interval")) + .crcCheckChance(row.getDouble("crc_check_chance")) + .speculativeRetry(SpeculativeRetryPolicy.fromString(row.getString("speculative_retry"))) + .additionalWritePolicy(row.has("additional_write_policy") ? + SpeculativeRetryPolicy.fromString(row.getString("additional_write_policy")) : + SpeculativeRetryPolicy.fromString("99PERCENTILE")) + .cdc(row.has("cdc") && row.getBoolean("cdc")) + .readRepair(getReadRepairStrategy(row)); + + // allow_auto_snapshot column was introduced in 4.2 + if (row.has("allow_auto_snapshot")) + builder.allowAutoSnapshot(row.getBoolean("allow_auto_snapshot")); + + return builder.build(); } private static List fetchColumns(String keyspace, String table, Types types) diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 2e9d507ba9..98fb0540c7 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -777,6 +777,12 @@ public class TableMetadata implements SchemaElement return this; } + public Builder allowAutoSnapshot(boolean val) + { + params.allowAutoSnapshot(val); + return this; + } + public Builder bloomFilterFpChance(double val) { params.bloomFilterFpChance(val); diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 440729c4ef..920bb829fc 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -41,6 +41,7 @@ public final class TableParams { public enum Option { + ALLOW_AUTO_SNAPSHOT, BLOOM_FILTER_FP_CHANCE, CACHING, COMMENT, @@ -67,6 +68,7 @@ public final class TableParams } public final String comment; + public final boolean allowAutoSnapshot; public final double bloomFilterFpChance; public final double crcCheckChance; public final int gcGraceSeconds; @@ -87,6 +89,7 @@ public final class TableParams private TableParams(Builder builder) { comment = builder.comment; + allowAutoSnapshot = builder.allowAutoSnapshot; bloomFilterFpChance = builder.bloomFilterFpChance == null ? builder.compaction.defaultBloomFilterFbChance() : builder.bloomFilterFpChance; @@ -114,7 +117,8 @@ public final class TableParams public static Builder builder(TableParams params) { - return new Builder().bloomFilterFpChance(params.bloomFilterFpChance) + return new Builder().allowAutoSnapshot(params.allowAutoSnapshot) + .bloomFilterFpChance(params.bloomFilterFpChance) .caching(params.caching) .comment(params.comment) .compaction(params.compaction) @@ -204,6 +208,7 @@ public final class TableParams TableParams p = (TableParams) o; return comment.equals(p.comment) + && allowAutoSnapshot == p.allowAutoSnapshot && bloomFilterFpChance == p.bloomFilterFpChance && crcCheckChance == p.crcCheckChance && gcGraceSeconds == p.gcGraceSeconds @@ -225,6 +230,7 @@ public final class TableParams public int hashCode() { return Objects.hashCode(comment, + allowAutoSnapshot, bloomFilterFpChance, crcCheckChance, gcGraceSeconds, @@ -247,6 +253,7 @@ public final class TableParams { return MoreObjects.toStringHelper(this) .add(Option.COMMENT.toString(), comment) + .add(Option.ALLOW_AUTO_SNAPSHOT.toString(), allowAutoSnapshot) .add(Option.BLOOM_FILTER_FP_CHANCE.toString(), bloomFilterFpChance) .add(Option.CRC_CHECK_CHANCE.toString(), crcCheckChance) .add(Option.GC_GRACE_SECONDS.toString(), gcGraceSeconds) @@ -269,6 +276,8 @@ public final class TableParams { // option names should be in alphabetical order builder.append("additional_write_policy = ").appendWithSingleQuotes(additionalWritePolicy.toString()) + .newLine() + .append("AND allow_auto_snapshot = ").append(allowAutoSnapshot) .newLine() .append("AND bloom_filter_fp_chance = ").append(bloomFilterFpChance) .newLine() @@ -315,6 +324,7 @@ public final class TableParams public static final class Builder { private String comment = ""; + private boolean allowAutoSnapshot = true; private Double bloomFilterFpChance; private double crcCheckChance = 1.0; private int gcGraceSeconds = 864000; // 10 days @@ -347,6 +357,12 @@ public final class TableParams return this; } + public Builder allowAutoSnapshot(boolean val) + { + allowAutoSnapshot = val; + return this; + } + public Builder bloomFilterFpChance(double val) { bloomFilterFpChance = val; diff --git a/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java new file mode 100644 index 0000000000..7a94905abb --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.io.IOException; +import java.util.Collections; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.service.StorageService; + +import static org.apache.cassandra.distributed.Cluster.build; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.awaitility.Awaitility.await; + +public class AllowAutoSnapshotTest extends TestBaseImpl +{ + @Test + public void testAllowAutoSnapshotOnAutoSnapshotEnabled() throws Exception + { + try (Cluster c = getCluster(true)) + { + c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH allow_auto_snapshot = true")); + c.schemaChange(withKeyspace("DROP TABLE %s.test_table;")); + + checkSnapshots(c, true, "test_table", "dropped"); + + c.schemaChange(withKeyspace("CREATE TABLE %s.test_table2 (a text primary key, b int) WITH allow_auto_snapshot = false")); + c.schemaChange(withKeyspace("DROP TABLE %s.test_table2;")); + + checkSnapshots(c, false, "test_table2", "dropped"); + } + } + + @Test + public void testAllowAutoSnapshotOnAutoSnapshotDisabled() throws Exception + { + try (Cluster c = getCluster(false)) + { + c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH allow_auto_snapshot = true")); + c.schemaChange(withKeyspace("DROP TABLE %s.test_table;")); + + checkSnapshots(c, false, "test_table", "dropped"); + + c.schemaChange(withKeyspace("CREATE TABLE %s.test_table2 (a text primary key, b int) WITH allow_auto_snapshot = false")); + c.schemaChange(withKeyspace("DROP TABLE %s.test_table2;")); + + checkSnapshots(c, false, "test_table2", "dropped"); + } + } + + @Test + public void testDisableAndEnableAllowAutoSnapshot() throws Exception + { + try (Cluster c = getCluster(true)) + { + c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH allow_auto_snapshot = true")); + + c.schemaChange(withKeyspace("ALTER TABLE %s.test_table WITH allow_auto_snapshot = false")); + c.schemaChange(withKeyspace("ALTER TABLE %s.test_table WITH allow_auto_snapshot = true")); + + c.schemaChange(withKeyspace("DROP TABLE %s.test_table;")); + + checkSnapshots(c, true, "test_table", "dropped"); + } + } + + @Test + public void testTruncateAllowAutoSnapshot() throws Exception + { + try (Cluster c = getCluster(true)) + { + // allow_auto_snapshot = true + c.schemaChange(withKeyspace("CREATE TABLE %s.test_table (a text primary key, b int) WITH allow_auto_snapshot = true")); + + c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table (a, b) VALUES ('a', 1);"), ALL); + + c.schemaChange(withKeyspace("TRUNCATE TABLE %s.test_table;")); + + checkSnapshots(c, true, "test_table", "truncated"); + + // allow_auto_snapshot = false + c.schemaChange(withKeyspace("CREATE TABLE %s.test_table2 (a text primary key, b int) WITH allow_auto_snapshot = false")); + + c.coordinator(1).execute(withKeyspace("INSERT INTO %s.test_table2 (a, b) VALUES ('a', 1);"), ALL); + + c.schemaChange(withKeyspace("TRUNCATE TABLE %s.test_table2;")); + + checkSnapshots(c, false, "test_table2", "truncated"); + } + } + + @Test + public void testMaterializedViewAllowAutoSnapshot() throws Exception + { + try (Cluster c = getCluster(true)) + { + // materialized view allow_auto_snapshot = false + c.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c1 int, c2 int, v1 int, v2 int, PRIMARY KEY (k, c1, c2))")); + c.schemaChange(withKeyspace("CREATE MATERIALIZED VIEW %s.mv1 AS SELECT * FROM t WHERE k IS NOT NULL AND c1 IS NOT NULL AND c2 IS NOT NULL PRIMARY KEY (c1, k, c2) WITH allow_auto_snapshot = false")); + + c.coordinator(1).execute(withKeyspace("INSERT INTO %s.t (k, c1, c2, v1, v2) VALUES (1, 2, 3, 4, 5);"), ALL); + + c.schemaChange(withKeyspace("DROP MATERIALIZED VIEW %s.mv1;")); + + checkSnapshots(c, false, "mv1", "dropped"); + + // materialized view allow_auto_snapshot = true + c.schemaChange(withKeyspace("CREATE MATERIALIZED VIEW %s.mv1 AS SELECT * FROM t WHERE k IS NOT NULL AND c1 IS NOT NULL AND c2 IS NOT NULL PRIMARY KEY (c1, k, c2) WITH allow_auto_snapshot = true")); + + c.schemaChange(withKeyspace("DROP MATERIALIZED VIEW %s.mv1;")); + checkSnapshots(c, true, "mv1", "dropped"); + } + } + + private Cluster getCluster(boolean autoSnapshotEnabled) throws IOException + { + return init(build(2).withConfig(c -> c.with(GOSSIP) + .set("auto_snapshot", autoSnapshotEnabled) + .set("materialized_views_enabled", true)).start()); + } + + private void checkSnapshots(Cluster cluster, boolean shouldContain, String table, String snapshotPrefix) + { + for (int i = 1; i < cluster.size() + 1; i++) + { + final int node = i; // has to be effectively final for the usage in "until" method + await().until(() -> cluster.get(node).appliesOnInstance((IIsolatedExecutor.SerializableTriFunction) (shouldContainSnapshot, tableName, prefix) -> { + Stream stream = StorageService.instance.getSnapshotDetails(Collections.emptyMap()).keySet().stream(); + Predicate predicate = tag -> tag.startsWith(prefix + '-') && tag.endsWith('-' + tableName); + return shouldContainSnapshot ? stream.anyMatch(predicate) : stream.noneMatch(predicate); + }).apply(shouldContain, table, snapshotPrefix)); + } + } +} diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index 9ea04b112e..83b320c5c3 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -839,6 +839,7 @@ public class DescribeStatementTest extends CQLTester private static String tableParametersCql() { return "additional_write_policy = '99p'\n" + + " AND allow_auto_snapshot = true\n" + " AND bloom_filter_fp_chance = 0.01\n" + " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" + " AND cdc = false\n" + @@ -860,6 +861,7 @@ public class DescribeStatementTest extends CQLTester private static String mvParametersCql() { return "additional_write_policy = '99p'\n" + + " AND allow_auto_snapshot = true\n" + " AND bloom_filter_fp_chance = 0.01\n" + " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" + " AND cdc = false\n" + diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index 63ef861101..ef62acc160 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -297,6 +297,7 @@ public class SchemaCQLHelperTest extends CQLTester assertThat(SchemaCQLHelper.getTableMetadataAsCQL(cfs.metadata(), cfs.keyspace.getMetadata()), containsString("CLUSTERING ORDER BY (cl1 ASC)\n" + " AND additional_write_policy = 'ALWAYS'\n" + + " AND allow_auto_snapshot = true\n" + " AND bloom_filter_fp_chance = 1.0\n" + " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" + " AND cdc = false\n" + diff --git a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java index e80773047a..77071d6d0e 100644 --- a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -33,11 +34,13 @@ import java.util.stream.Stream; import com.google.common.collect.ImmutableMap; import org.junit.Assert; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; @@ -61,6 +64,7 @@ import org.jboss.byteman.contrib.bmunit.BMUnitRunner; import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; @RunWith(BMUnitRunner.class) public class SchemaKeyspaceTest @@ -181,6 +185,42 @@ public class SchemaKeyspaceTest } + @Test + public void testAutoSnapshotEnabledOnTable() + { + Assume.assumeTrue(DatabaseDescriptor.isAutoSnapshot()); + String keyspaceName = "AutoSnapshot"; + String tableName = "table1"; + + createTable(keyspaceName, "CREATE TABLE " + tableName + " (a text primary key, b int) WITH allow_auto_snapshot = true"); + + ColumnFamilyStore cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName); + + assertTrue(cfs.isAutoSnapshotEnabled()); + + SchemaTestUtil.announceTableDrop(keyspaceName, tableName); + + assertFalse(cfs.listSnapshots().isEmpty()); + } + + @Test + public void testAutoSnapshotDisabledOnTable() + { + Assume.assumeTrue(DatabaseDescriptor.isAutoSnapshot()); + String keyspaceName = "AutoSnapshot"; + String tableName = "table2"; + + createTable(keyspaceName, "CREATE TABLE " + tableName + " (a text primary key, b int) WITH allow_auto_snapshot = false"); + + ColumnFamilyStore cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName); + + assertFalse(cfs.isAutoSnapshotEnabled()); + + SchemaTestUtil.announceTableDrop(keyspaceName, tableName); + + assertTrue(cfs.listSnapshots().isEmpty()); + } + private static void updateTable(String keyspace, TableMetadata oldTable, TableMetadata newTable) { KeyspaceMetadata ksm = Schema.instance.getKeyspaceInstance(keyspace).getMetadata();