Merge branch cassandra-3.0 into cassandra-3.11

CHANGES.txt

@@ -8,6 +8,7 @@
  * Make sure sstables with moved starts are removed correctly in LeveledGenerations (CASSANDRA-16552)
  * Upgrade jackson-databind to 2.9.10.8 (CASSANDRA-16462)
 Merged from 3.0:
+ * Add flag to disable ALTER...DROP COMPACT STORAGE statements (CASSANDRA-16733)
  * Clean transaction log leftovers at the beginning of sstablelevelreset and sstableofflinerelevel (CASSANDRA-12519)
  * CQL shell should prefer newer TLS version by default (CASSANDRA-16695)
  * Ensure that existing empty rows are properly returned (CASSANDRA-16671)

NEWS.txt

@@ -42,8 +42,19 @@ restore snapshots created with the previous major version using the
 'sstableloader' tool. You can upgrade the file format of your snapshots
 using the provided 'sstableupgrade' tool.
 
+3.11.11
+=======
+
+ALTER ... DROP COMPACT STORAGE
+------------------------------
+   - Following a discussion regarding concerns about the safety of the 'ALTER ... DROP COMPACT STORAGE' statement, 
+     the C* development community does not recommend its use in production and considers it experimental
+     (see https://www.mail-archive.com/dev@cassandra.apache.org/msg16789.html).
+   - An 'enable_drop_compact_storage' flag has been added to cassandra.yaml to allow operators to prevent its use.
+
 3.11.10
-=====
+======
+
 Upgrading
 ---------
     - This release fix a correctness issue with SERIAL reads, and LWT writes that do not apply.
@@ -200,7 +211,8 @@ Upgrading
 
 Compact Storage (only when upgrading from 3.X or any version lower than 3.0.15)
 ---------------
-    - Starting version 4.0, Thrift and COMPACT STORAGE is no longer supported.
+    - Starting version 4.0, Thrift is no longer supported.
+      Starting version 5.0, COMPACT STORAGE will no longer be supported.
       'ALTER ... DROP COMPACT STORAGE' statement makes Compact Tables CQL-compatible,
       exposing internal structure of Thrift/Compact Tables. You can find more details
       on exposed internal structure under: 

conf/cassandra.yaml

@@ -1278,3 +1278,7 @@ enable_materialized_views: true
 # Enables SASI index creation on this node.
 # SASI indexes are considered experimental and are not recommended for production use.
 enable_sasi_indexes: true
+
+# Enables the used of 'ALTER ... DROP COMPACT STORAGE' statements on this node.
+# 'ALTER ... DROP COMPACT STORAGE' is considered experimental and is not recommended for production use.
+enable_drop_compact_storage: false

src/java/org/apache/cassandra/config/Config.java

@@ -355,6 +355,8 @@ public class Config
 
     public boolean enable_sasi_indexes = true;
 
+    public volatile boolean enable_drop_compact_storage = true;
+
     /**
      * Optionally disable asynchronous UDF execution.
      * Disabling asynchronous UDF execution also implicitly disables the security-manager!

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -2522,6 +2522,17 @@ public class DatabaseDescriptor
         conf.enable_sasi_indexes = enableSASIIndexes;
     }
 
+    public static boolean enableDropCompactStorage()
+    {
+        return conf.enable_drop_compact_storage;
+    }
+
+    @VisibleForTesting
+    public static void setEnableDropCompactStorage(boolean enableDropCompactStorage)
+    {
+        conf.enable_drop_compact_storage = enableDropCompactStorage;
+    }
+
     public static long getUserDefinedFunctionFailTimeout()
     {
         return conf.user_defined_function_fail_timeout;

src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java

@@ -300,6 +300,10 @@ public class AlterTableStatement extends SchemaAlteringStatement
 
                 break;
             case DROP_COMPACT_STORAGE:
+
+                if (!DatabaseDescriptor.enableDropCompactStorage())
+                    throw new InvalidRequestException("DROP COMPACT STORAGE is disabled. Enable in cassandra.yaml to use.");
+
                 if (!meta.isCompactTable())
                     throw new InvalidRequestException("Cannot DROP COMPACT STORAGE on table without COMPACT STORAGE");
 

test/conf/cassandra.yaml

@@ -45,3 +45,4 @@ row_cache_size_in_mb: 16
 enable_user_defined_functions: true
 enable_scripted_user_defined_functions: true
 prepared_statements_cache_size_mb: 1
+enable_drop_compact_storage: true

test/distributed/org/apache/cassandra/distributed/upgrade/CompactStorage2to3UpgradeTest.java

@@ -163,6 +163,11 @@ public class CompactStorage2to3UpgradeTest extends UpgradeTestBase
                           KEYSPACE, table, partitions - 8, rowsPerPartition - 3),
 
             });
+        }).runBeforeNodeRestart((cluster, node) ->
+        {
+            cluster.get(node).config().set("enable_drop_compact_storage", true);
+
+
         }).runAfterClusterUpgrade(cluster ->
                                           {
                                               for (int i = 1; i <= cluster.size(); i++)
@@ -205,7 +210,7 @@ public class CompactStorage2to3UpgradeTest extends UpgradeTestBase
         new TestCase()
         .nodes(2)
         .upgrade(Versions.Major.v22, Versions.Major.v3X)
-        .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
+        .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL).set("enable_drop_compact_storage", true))
         .setup(cluster -> {
             cluster.schemaChange(String.format(
             "CREATE TABLE %s.%s (key int, c1 int, c2 int, c3 int, PRIMARY KEY (key, c1, c2)) WITH COMPACT STORAGE",

test/distributed/org/apache/cassandra/distributed/upgrade/DropCompactStorageTest.java

@@ -21,7 +21,6 @@ package org.apache.cassandra.distributed.upgrade;
 import org.junit.Test;
 
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
-import org.apache.cassandra.distributed.api.NodeToolResult;
 import org.apache.cassandra.distributed.shared.Versions;
 
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
@@ -29,7 +28,6 @@ import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.catchThrowable;
-import static org.junit.Assert.assertEquals;
 
 public class DropCompactStorageTest extends UpgradeTestBase
 {
@@ -50,7 +48,7 @@ public class DropCompactStorageTest extends UpgradeTestBase
         new TestCase()
         .nodes(1)
         .upgrade(Versions.Major.v22, upgradeTo)
-        .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
+        .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL).set("enable_drop_compact_storage", true))
         .setup((cluster) -> {
             cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (id int, ck int, v int, PRIMARY KEY (id, ck)) WITH COMPACT STORAGE");
             for (int i = 0; i < 5; i++)

test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java

@@ -89,6 +89,7 @@ public class UpgradeTestBase extends DistributedTestBase
         private final List<TestVersions> upgrade = new ArrayList<>();
         private int nodeCount = 3;
         private RunOnCluster setup;
+        private RunOnClusterAndNode runBeforeNodeRestart;
         private RunOnClusterAndNode runAfterNodeUpgrade;
         private RunOnCluster runAfterClusterUpgrade;
         private final Set<Integer> nodesToUpgrade = new HashSet<>();
@@ -131,6 +132,12 @@ public class UpgradeTestBase extends DistributedTestBase
             return this;
         }
 
+        public TestCase runBeforeNodeRestart(RunOnClusterAndNode runBeforeNodeRestart)
+        {
+            this.runBeforeNodeRestart = runBeforeNodeRestart;
+            return this;
+        }
+
         public TestCase runAfterNodeUpgrade(RunOnClusterAndNode runAfterNodeUpgrade)
         {
             this.runAfterNodeUpgrade = runAfterNodeUpgrade;
@@ -157,6 +164,8 @@ public class UpgradeTestBase extends DistributedTestBase
                 throw new AssertionError();
             if (runAfterClusterUpgrade == null && runAfterNodeUpgrade == null)
                 throw new AssertionError();
+            if (runBeforeNodeRestart == null)
+                runBeforeNodeRestart = (c, n) -> {};
             if (runAfterClusterUpgrade == null)
                 runAfterClusterUpgrade = (c) -> {};
             if (runAfterNodeUpgrade == null)
@@ -177,6 +186,7 @@ public class UpgradeTestBase extends DistributedTestBase
                         {
                             cluster.get(n).shutdown().get();
                             cluster.get(n).setVersion(version);
+                            runBeforeNodeRestart.run(cluster, n);
                             cluster.get(n).startup();
                             runAfterNodeUpgrade.run(cluster, n);
                         }

test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java

@@ -21,6 +21,7 @@ import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.cassandra.config.SchemaConstants;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
@@ -566,4 +567,14 @@ public class AlterTest extends CQLTester
         assertInvalidMessage(table2, format("ALTER TYPE %s.%s ADD v2 int;", keyspace(), type1));
         assertInvalidMessage(table3, format("ALTER TYPE %s.%s ADD v2 int;", keyspace(), type1));
     }
+
+    @Test
+    public void testAlterDropCompactStorageDisabled() throws Throwable
+    {
+        DatabaseDescriptor.setEnableDropCompactStorage(false);
+
+        createTable("CREATE TABLE %s (k text, i int, PRIMARY KEY (k, i)) WITH COMPACT STORAGE");
+
+        assertInvalidMessage("DROP COMPACT STORAGE is disabled. Enable in cassandra.yaml to use.", "ALTER TABLE %s DROP COMPACT STORAGE");
+    }
 }