Merge branch cassandra-3.11 into cassandra-4.0.0

CHANGES.txt

@@ -19,6 +19,7 @@ Merged from 3.11:
  * Nodetool garbagecollect should retain SSTableLevel for LCS (CASSANDRA-16634)
  * Ignore stale acks received in the shadow round (CASSANDRA-16588)
 Merged from 3.0:
+ * Add flag to disable ALTER...DROP COMPACT STORAGE statements (CASSANDRA-16733)
  * CQL shell should prefer newer TLS version by default (CASSANDRA-16695)
  * Ensure that existing empty rows are properly returned (CASSANDRA-16671)
  * Failure to execute queries should emit a KPI other than read timeout/unavailable so it can be alerted/tracked (CASSANDRA-16581)

NEWS.txt

@@ -298,6 +298,13 @@ Deprecation
       and will be removed in a subsequent major version.
     - cqlsh support of 2.7 is deprecated and will warn when running with Python 2.7.
 
+ALTER ... DROP COMPACT STORAGE
+------------------------------
+   - Following a discussion regarding concerns about the safety of the 'ALTER ... DROP COMPACT STORAGE' statement,
+     the C* development community does not recommend its use in production and considers it experimental
+     (see https://www.mail-archive.com/dev@cassandra.apache.org/msg16789.html).
+   - An 'enable_drop_compact_storage' flag has been added to cassandra.yaml to allow operators to prevent its use.
+
 Materialized Views
 -------------------
     - Following a discussion regarding concerns about the design and safety of Materialized Views, the C* development
@@ -315,7 +322,6 @@ Windows Support Removed
       who use Windows 10 still can run Apache Cassandra locally using WSL2 (Windows Subsystem for Linux version 2),
       Docker for Windows, or virtualization platform like Hyper-V and VirtualBox.
 
-
 3.11.10
 ======
 
@@ -421,7 +427,8 @@ Upgrading
 
 Compact Storage (only when upgrading from 3.X or any version lower than 3.0.15)
 ---------------
-    - Starting version 4.0, Thrift and COMPACT STORAGE is no longer supported.
+    - Starting version 4.0, Thrift is no longer supported.
+      Starting version 5.0, COMPACT STORAGE will no longer be supported.
       'ALTER ... DROP COMPACT STORAGE' statement makes Compact Tables CQL-compatible,
       exposing internal structure of Thrift/Compact Tables. You can find more details
       on exposed internal structure under: 

conf/cassandra.yaml

@@ -1434,3 +1434,7 @@ enable_sasi_indexes: false
 # Enables creation of transiently replicated keyspaces on this node.
 # Transient replication is experimental and is not recommended for production use.
 enable_transient_replication: false
+
+# Enables the used of 'ALTER ... DROP COMPACT STORAGE' statements on this node.
+# 'ALTER ... DROP COMPACT STORAGE' is considered experimental and is not recommended for production use.
+enable_drop_compact_storage: false

doc/source/configuration/cass_yaml_file.rst

@@ -2099,3 +2099,11 @@ Enables creation of transiently replicated keyspaces on this node.
 Transient replication is experimental and is not recommended for production use.
 
 *Default Value:* false
+
+``enable_drop_compact_storage``
+--------------------------------
+
+Enables the used of 'ALTER ... DROP COMPACT STORAGE' statements on this node.
+'ALTER ... DROP COMPACT STORAGE' is considered experimental and is not recommended for production use.
+
+*Default Value:* false

src/java/org/apache/cassandra/config/Config.java

@@ -398,6 +398,8 @@ public class Config
 
     public boolean enable_sasi_indexes = false;
 
+    public volatile boolean enable_drop_compact_storage = false;
+
     /**
      * Optionally disable asynchronous UDF execution.
      * Disabling asynchronous UDF execution also implicitly disables the security-manager!

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -2950,6 +2950,17 @@ public class DatabaseDescriptor
         conf.enable_transient_replication = enabled;
     }
 
+    public static boolean enableDropCompactStorage()
+    {
+        return conf.enable_drop_compact_storage;
+    }
+
+    @VisibleForTesting
+    public static void setEnableDropCompactStorage(boolean enableDropCompactStorage)
+    {
+        conf.enable_drop_compact_storage = enableDropCompactStorage;
+    }
+
     public static long getUserDefinedFunctionFailTimeout()
     {
         return conf.user_defined_function_fail_timeout;

src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java

@@ -36,7 +36,7 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
-
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.CQLStatement;
 import org.apache.cassandra.cql3.ColumnIdentifier;
@@ -431,6 +431,9 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
 
         public KeyspaceMetadata apply(KeyspaceMetadata keyspace, TableMetadata table)
         {
+            if (!DatabaseDescriptor.enableDropCompactStorage())
+                throw new InvalidRequestException("DROP COMPACT STORAGE is disabled. Enable in cassandra.yaml to use.");
+
             if (!table.isCompactTable())
                 throw AlterTableStatement.ire("Cannot DROP COMPACT STORAGE on table without COMPACT STORAGE");
 

test/conf/cassandra.yaml

@@ -50,4 +50,5 @@ stream_entire_sstables: true
 stream_throughput_outbound_megabits_per_sec: 200000000
 enable_sasi_indexes: true
 enable_materialized_views: true
+enable_drop_compact_storage: true
 file_cache_enabled: true

test/distributed/org/apache/cassandra/distributed/test/CountersTest.java

@@ -45,7 +45,7 @@ public class CountersTest extends TestBaseImpl
 
     private static void testUpdateCounter(boolean droppedCompactStorage) throws Throwable
     {
-        try (Cluster cluster = Cluster.build(2).withConfig(c -> c.with(GOSSIP, NATIVE_PROTOCOL)).start())
+        try (Cluster cluster = Cluster.build(2).withConfig(c -> c.with(GOSSIP, NATIVE_PROTOCOL).set("enable_drop_compact_storage", true)).start())
         {
             cluster.schemaChange("CREATE KEYSPACE k WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
 

test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorageUpgradeTest.java

@@ -146,7 +146,7 @@ public class CompactStorageUpgradeTest extends UpgradeTestBase
         .nodes(2)
         .nodesToUpgrade(1, 2)
         .upgrade(Versions.Major.v30, Versions.Major.v4)
-        .withConfig(config -> config.with(GOSSIP, NETWORK))
+        .withConfig(config -> config.with(GOSSIP, NETWORK).set("enable_drop_compact_storage", true))
         .setup((cluster) -> {
             cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, PRIMARY KEY (pk, ck)) WITH COMPACT STORAGE");
             cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck) VALUES (1,1)", ConsistencyLevel.ALL);

test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java

@@ -95,6 +95,7 @@ public class UpgradeTestBase extends DistributedTestBase
         private final List<TestVersions> upgrade = new ArrayList<>();
         private int nodeCount = 3;
         private RunOnCluster setup;
+        private RunOnClusterAndNode runBeforeNodeRestart;
         private RunOnClusterAndNode runAfterNodeUpgrade;
         private RunOnCluster runAfterClusterUpgrade;
         private final Set<Integer> nodesToUpgrade = new LinkedHashSet<>();
@@ -137,6 +138,12 @@ public class UpgradeTestBase extends DistributedTestBase
             return this;
         }
 
+        public TestCase runBeforeNodeRestart(RunOnClusterAndNode runBeforeNodeRestart)
+        {
+            this.runBeforeNodeRestart = runBeforeNodeRestart;
+            return this;
+        }
+
         public TestCase runAfterNodeUpgrade(RunOnClusterAndNode runAfterNodeUpgrade)
         {
             this.runAfterNodeUpgrade = runAfterNodeUpgrade;
@@ -163,6 +170,8 @@ public class UpgradeTestBase extends DistributedTestBase
                 throw new AssertionError();
             if (runAfterClusterUpgrade == null && runAfterNodeUpgrade == null)
                 throw new AssertionError();
+            if (runBeforeNodeRestart == null)
+                runBeforeNodeRestart = (c, n) -> {};
             if (runAfterClusterUpgrade == null)
                 runAfterClusterUpgrade = (c) -> {};
             if (runAfterNodeUpgrade == null)
@@ -183,6 +192,7 @@ public class UpgradeTestBase extends DistributedTestBase
                         {
                             cluster.get(n).shutdown().get();
                             cluster.get(n).setVersion(version);
+                            runBeforeNodeRestart.run(cluster, n);
                             cluster.get(n).startup();
                             runAfterNodeUpgrade.run(cluster, n);
                         }

test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java

@@ -25,6 +25,7 @@ import org.junit.runner.RunWith;
 
 import com.datastax.driver.core.PreparedStatement;
 import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
@@ -710,4 +711,14 @@ public class AlterTest extends CQLTester
         assertInvalidMessage(table2, format("ALTER TYPE %s.%s ADD v2 int;", keyspace(), type1));
         assertInvalidMessage(table3, format("ALTER TYPE %s.%s ADD v2 int;", keyspace(), type1));
     }
+
+    @Test
+    public void testAlterDropCompactStorageDisabled() throws Throwable
+    {
+        DatabaseDescriptor.setEnableDropCompactStorage(false);
+
+        createTable("CREATE TABLE %s (k text, i int, PRIMARY KEY (k, i)) WITH COMPACT STORAGE");
+
+        assertInvalidMessage("DROP COMPACT STORAGE is disabled. Enable in cassandra.yaml to use.", "ALTER TABLE %s DROP COMPACT STORAGE");
+    }
 }