CASSANDRA-19195 Added guardrail to enforce minimum CMS size

Adds the minimum_cms_size_fail_threshold guardrail that fails an operator-requested CMS reconfiguration (nodetool cms reconfigure / JMX) when the resulting CMS size (aggregate replication factor across DCs) would fall below the threshold.

- Config-driven, default -1 (disabled); valid values are -1 or >= 3
- Enforced only on operator-initiated reconfiguration, not the automatic reconfiguration on topology changes
- Skipped for clusters smaller than the threshold
- Fail-only (no warning threshold)

 patch by nvharikrishna; reviewed by <reviewer> for CASSANDRA-19195

CHANGES.txt

@@ -1,4 +1,5 @@
 7.0
+ * Add a guardrail to disallow reconfiguring CMS below a minimum size (CASSANDRA-19195)
  * Don't increment client metrics on messaging service connection unpause (CASSANDRA-21491)
  * Add nodetool getreplicas (CASSANDRA-17665)
  * Implementation of CEP-49: Hardware-accelerated compression (CASSANDRA-20975)

conf/cassandra.yaml

@@ -2585,6 +2585,12 @@ drop_compact_storage_enabled: false
 # maximum_replication_factor_warn_threshold: -1
 # maximum_replication_factor_fail_threshold: -1
 #
+
+# Guardrail to fail an operator-requested CMS reconfiguration (nodetool cms reconfigure / JMX) when the resulting
+# CMS size would be below this threshold. The CMS size is the aggregate replication factor across all datacenters.
+# The check is skipped when the cluster has fewer nodes than the threshold. Defaults to -1 to disable.
+# minimum_cms_size_fail_threshold: -1
+
 # Guardrail to warn or fail when a statement is prepared without bind markers (parameters).
 # This prevents the Prepared Statement Cache from being flooded with unique entries caused
 # by hardcoded literals. When warned is true, logs a warning on the usage of misprepared statements.

conf/cassandra_latest.yaml

@@ -2359,6 +2359,11 @@ drop_compact_storage_enabled: false
 # maximum_replication_factor_warn_threshold: -1
 # maximum_replication_factor_fail_threshold: -1
 #
+# Guardrail to fail an operator-requested CMS reconfiguration (nodetool cms reconfigure / JMX) when the resulting
+# CMS size would be below this threshold. The CMS size is the aggregate replication factor across all datacenters.
+# The check is skipped when the cluster has fewer nodes than the threshold. Defaults to -1 to disable.
+# minimum_cms_size_fail_threshold: -1
+#
 # Guardrail to warn or fail when a statement is prepared without bind markers (parameters).
 # This prevents the Prepared Statement Cache from being flooded with unique entries caused
 # by hardcoded literals. When warned is true, logs a warning on the usage of misprepared statements.

src/java/org/apache/cassandra/config/Config.java

@@ -1056,6 +1056,7 @@ public class Config
     public volatile int minimum_replication_factor_fail_threshold = -1;
     public volatile int maximum_replication_factor_warn_threshold = -1;
     public volatile int maximum_replication_factor_fail_threshold = -1;
+    public volatile int minimum_cms_size_fail_threshold = -1;
     public volatile boolean zero_ttl_on_twcs_warned = true;
     public volatile boolean zero_ttl_on_twcs_enabled = true;
     public volatile boolean non_partition_restricted_index_query_enabled = true;

src/java/org/apache/cassandra/config/GuardrailsOptions.java

@@ -110,6 +110,7 @@ public class GuardrailsOptions implements GuardrailsConfig
         validateDataDiskUsageMaxDiskSize(config.data_disk_usage_max_disk_size);
         validateMinRFThreshold(config.minimum_replication_factor_warn_threshold, config.minimum_replication_factor_fail_threshold);
         validateMaxRFThreshold(config.maximum_replication_factor_warn_threshold, config.maximum_replication_factor_fail_threshold);
+        validateMinCmsSizeThreshold(config.minimum_cms_size_fail_threshold, "minimum_cms_size_fail_threshold");
         validateTimestampThreshold(config.maximum_timestamp_warn_threshold, config.maximum_timestamp_fail_threshold, "maximum_timestamp");
         validateTimestampThreshold(config.minimum_timestamp_warn_threshold, config.minimum_timestamp_fail_threshold, "minimum_timestamp");
         validateMaxLongThreshold(config.sai_sstable_indexes_per_query_warn_threshold,
@@ -1051,6 +1052,21 @@ public class GuardrailsOptions implements GuardrailsConfig
                                   x -> config.minimum_replication_factor_fail_threshold = x);
     }
 
+    @Override
+    public int getMinimumCmsSizeFailThreshold()
+    {
+        return config.minimum_cms_size_fail_threshold;
+    }
+
+    public void setMinimumCmsSizeFailThreshold(int fail)
+    {
+        validateMinCmsSizeThreshold(fail, "minimum_cms_size_fail_threshold");
+        updatePropertyWithLogging("minimum_cms_size_fail_threshold",
+                                  fail,
+                                  () -> config.minimum_cms_size_fail_threshold,
+                                  x -> config.minimum_cms_size_fail_threshold = x);
+    }
+
     @Override
     public int getMaximumReplicationFactorWarnThreshold()
     {
@@ -1521,6 +1537,17 @@ public class GuardrailsOptions implements GuardrailsConfig
                                                       fail, DatabaseDescriptor.getDefaultKeyspaceRF()));
     }
 
+    private static void validateMinCmsSizeThreshold(int value, String name)
+    {
+        if (value == -1)
+            return;
+
+        if (value < 3)
+            throw new IllegalArgumentException(format("Invalid value %d for %s: minimum allowed value is 3, " +
+                                                      "as CMS requires at least 3 replicas to maintain quorum safely. " +
+                                                      "Use -1 to disable this guardrail", value, name));
+    }
+
     public static void validateTimestampThreshold(DurationSpec.LongMicrosecondsBound warn,
                                                   DurationSpec.LongMicrosecondsBound fail,
                                                   String name)

src/java/org/apache/cassandra/db/guardrails/Guardrails.java

@@ -615,6 +615,21 @@ public final class Guardrails implements GuardrailsMBean
                      format("The keyspace %s has a replication factor of %s, below the %s threshold of %s.",
                             what, value, isWarning ? "warning" : "failure", threshold));
 
+    /**
+     * Guardrail on the minimum CMS size (aggregate replication factor across DCs).
+     */
+    public static final MinThreshold minimumCmsSize =
+    (MinThreshold) new MinThreshold("minimum_cms_size",
+                     null,
+                     state -> -1,
+                     state -> CONFIG_PROVIDER.getOrCreate(state).getMinimumCmsSizeFailThreshold(),
+                     (isWarning, what, value, threshold) ->
+                     format("The CMS size of %s is below the failure threshold of %s. " +
+                            "Reconfigure CMS so its total size (sum of replication factors across all datacenters) is at least %s.",
+                            value, threshold, threshold))
+                     // CMS reconfiguration is a system operation with no ClientState; make it abort rather than just log.
+                     .throwOnNullClientState(true);
+
     /**
      * Guardrail on the maximum replication factor.
      */
@@ -1670,6 +1685,18 @@ public final class Guardrails implements GuardrailsMBean
         DEFAULT_CONFIG.setMinimumReplicationFactorThreshold(warn, fail);
     }
 
+    @Override
+    public int getMinimumCmsSizeFailThreshold()
+    {
+        return DEFAULT_CONFIG.getMinimumCmsSizeFailThreshold();
+    }
+
+    @Override
+    public void setMinimumCmsSizeFailThreshold(int fail)
+    {
+        DEFAULT_CONFIG.setMinimumCmsSizeFailThreshold(fail);
+    }
+
     @Override
     public boolean getZeroTTLOnTWCSEnabled()
     {

src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java

@@ -466,6 +466,11 @@ public interface GuardrailsConfig
      */
     int getMinimumReplicationFactorFailThreshold();
 
+    /**
+     * @return The threshold to fail when the CMS size (aggregate replication factor across DCs) is below the threshold.
+     */
+    int getMinimumCmsSizeFailThreshold();
+
     /**
      * @return The threshold to warn when replication factor is greater than threshold.
      */

src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java

@@ -910,6 +910,17 @@ public interface GuardrailsMBean
      */
     void setMinimumReplicationFactorThreshold (int warn, int fail);
 
+    /**
+     * @return The threshold to fail when the CMS size (aggregate replication factor across DCs) is below the threshold.
+     */
+    int getMinimumCmsSizeFailThreshold();
+
+    /**
+     * @param fail The threshold to fail when the CMS size (aggregate replication factor across DCs) is below it.
+     *             -1 means disabled.
+     */
+    void setMinimumCmsSizeFailThreshold(int fail);
+
     /**
      * @return The threshold to fail when replication factor is greater than threshold.
      */

src/java/org/apache/cassandra/tcm/CMSOperations.java

@@ -33,6 +33,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.virtual.ClusterMetadataDirectoryTable;
 import org.apache.cassandra.db.virtual.ClusterMetadataLogTable;
 import org.apache.cassandra.schema.ReplicationParams;
@@ -162,13 +163,38 @@ public class CMSOperations implements CMSOperationsMBean
     @Override
     public void reconfigureCMS(int rf)
     {
-        cms.reconfigureCMS(ReplicationParams.simpleMeta(rf, ClusterMetadata.current().directory.knownDatacenters()));
+        ReplicationParams params = ReplicationParams.simpleMeta(rf, ClusterMetadata.current().directory.knownDatacenters());
+        guardMinimumCmsSize(params);
+        cms.reconfigureCMS(params);
     }
 
     @Override
     public void reconfigureCMS(Map<String, Integer> rf)
     {
-        cms.reconfigureCMS(ReplicationParams.ntsMeta(rf));
+        ReplicationParams params = ReplicationParams.ntsMeta(rf);
+        guardMinimumCmsSize(params);
+        cms.reconfigureCMS(params);
+    }
+
+    /**
+     * Enforces the minimum CMS size guardrail against an operator-requested reconfiguration. This is only applied to
+     * explicit reconfigure requests (nodetool / JMX), not to the automatic reconfiguration that happens on topology
+     * changes, which reuses the already-committed replication params.
+     */
+    private void guardMinimumCmsSize(ReplicationParams params)
+    {
+        ClusterMetadata metadata = ClusterMetadata.current();
+        int minCmsSize = DatabaseDescriptor.getGuardrailsConfig().getMinimumCmsSizeFailThreshold();
+        int totalNodes = metadata.directory.allJoinedEndpoints().size();
+        // Skip clusters smaller than the threshold: they cannot host that many replicas yet (e.g. during bootstrap).
+        // When the guardrail is disabled this is never true, and guard() below is then a no-op.
+        if (totalNodes < minCmsSize)
+            return;
+
+        int cmsSize = params.options.values().stream()
+                                    .mapToInt(Integer::parseInt)
+                                    .sum();
+        Guardrails.minimumCmsSize.guard(cmsSize, "CMS", false, null);
     }
 
     @Override

test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailCmsSizeReconfigurationTest.java

@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.guardrails;
+
+import java.io.IOException;
+import java.util.function.Consumer;
+
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInstanceConfig;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+
+import static org.apache.cassandra.distributed.Cluster.build;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+
+/**
+ * Exercises the minimum CMS size guardrail via the real {@code nodetool cms reconfigure} path against a live in-JVM
+ * cluster. The guardrail only applies once the cluster has at least as many nodes as the threshold, so multiple nodes
+ * are required to trigger it.
+ */
+public class GuardrailCmsSizeReconfigurationTest extends TestBaseImpl
+{
+    private static final int THRESHOLD = 3;
+
+    @Test
+    public void reconfigureBelowThresholdIsRejected() throws IOException
+    {
+        try (Cluster cluster = build(3).withConfig(config(THRESHOLD)).start())
+        {
+            cluster.get(1).nodetoolResult("cms", "reconfigure", "2")
+                   .asserts()
+                   .failure()
+                   .stderrContains("failure threshold of " + THRESHOLD);
+        }
+    }
+
+    @Test
+    public void reconfigureAtThresholdIsAllowed() throws IOException
+    {
+        try (Cluster cluster = build(3).withConfig(config(THRESHOLD)).start())
+        {
+            cluster.get(1).nodetoolResult("cms", "reconfigure", "3")
+                   .asserts()
+                   .success();
+        }
+    }
+
+    @Test
+    public void reconfigureBelowThresholdIsAllowedWhenGuardrailDisabled() throws IOException
+    {
+        // Default threshold is -1 (disabled), so a below-safe reconfiguration is permitted.
+        try (Cluster cluster = build(3).withConfig(config(null)).start())
+        {
+            cluster.get(1).nodetoolResult("cms", "reconfigure", "2")
+                   .asserts()
+                   .success();
+        }
+    }
+
+    @Test
+    public void reconfigureBelowThresholdIsAllowedWhenClusterTooSmall() throws IOException
+    {
+        // The cluster has fewer nodes than the threshold, so it cannot host that many replicas and the guardrail is skipped.
+        try (Cluster cluster = build(2).withConfig(config(THRESHOLD)).start())
+        {
+            cluster.get(1).nodetoolResult("cms", "reconfigure", "2")
+                   .asserts()
+                   .success();
+        }
+    }
+
+    @Test
+    public void reconfigureMultiDcBelowThresholdIsRejected() throws IOException
+    {
+        // 2 DCs x 2 nodes = 4 nodes; requested aggregate size is 1 + 1 = 2, below the threshold of 3.
+        try (Cluster cluster = builder().withRacks(2, 1, 2).withConfig(config(THRESHOLD)).start())
+        {
+            cluster.get(1).nodetoolResult("cms", "reconfigure", "datacenter1:1", "datacenter2:1")
+                   .asserts()
+                   .failure()
+                   .stderrContains("failure threshold of " + THRESHOLD);
+        }
+    }
+
+    @Test
+    public void reconfigureMultiDcAtOrAboveThresholdIsAllowed() throws IOException
+    {
+        // 2 DCs x 2 nodes = 4 nodes; requested aggregate size is 2 + 2 = 4, at or above the threshold of 3.
+        try (Cluster cluster = builder().withRacks(2, 1, 2).withConfig(config(THRESHOLD)).start())
+        {
+            cluster.get(1).nodetoolResult("cms", "reconfigure", "datacenter1:2", "datacenter2:2")
+                   .asserts()
+                   .success();
+        }
+    }
+
+    private static Consumer<IInstanceConfig> config(Integer threshold)
+    {
+        return config -> {
+            config.with(GOSSIP).with(NETWORK);
+            if (threshold != null)
+                config.set("minimum_cms_size_fail_threshold", threshold);
+        };
+    }
+}

test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumCmsSizeTest.java

@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.lang.management.ManagementFactory;
+import java.util.Arrays;
+import java.util.Collection;
+
+import javax.management.JMX;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.GuardrailsOptions;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+
+@RunWith(Enclosed.class)
+public class GuardrailMinimumCmsSizeTest
+{
+    private static final int DISABLED = -1;
+
+    /**
+     * Returns a proxy that invokes the Guardrails MBean through the platform MBeanServer, so the tests exercise the
+     * real (local, no-network) JMX path rather than a direct method call. The MBean is registered defensively in case
+     * MBean registration is disabled in the unit test JVM.
+     */
+    private static GuardrailsMBean guardrailsJmxProxy()
+    {
+        try
+        {
+            MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+            ObjectName name = new ObjectName(Guardrails.MBEAN_NAME);
+            if (!mbs.isRegistered(name))
+                mbs.registerMBean(Guardrails.instance, name);
+            return JMX.newMBeanProxy(mbs, name, GuardrailsMBean.class);
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @RunWith(Parameterized.class)
+    public static class ValidValueAcceptedTest
+    {
+        @Parameterized.Parameter
+        public int validValue;
+
+        @Parameterized.Parameters(name = "value={0}")
+        public static Collection<Object[]> validValues()
+        {
+            return Arrays.asList(new Object[][]{ { DISABLED }, { 3 }, { 5 }, { 7 } });
+        }
+
+        @BeforeClass
+        public static void setupClass()
+        {
+            DatabaseDescriptor.daemonInitialization();
+        }
+
+        @After
+        public void teardown()
+        {
+            DatabaseDescriptor.getGuardrailsConfig().setMinimumCmsSizeFailThreshold(DISABLED);
+        }
+
+        @Test
+        public void testConfigValidationValueIsAccepted()
+        {
+            GuardrailsOptions guardrails = DatabaseDescriptor.getGuardrailsConfig();
+            guardrails.setMinimumCmsSizeFailThreshold(validValue);
+            assertEquals(validValue, guardrails.getMinimumCmsSizeFailThreshold());
+        }
+
+        @Test
+        public void testJmxValidationValueIsAccepted()
+        {
+            GuardrailsMBean jmx = guardrailsJmxProxy();
+            jmx.setMinimumCmsSizeFailThreshold(validValue);
+            assertEquals(validValue, jmx.getMinimumCmsSizeFailThreshold());
+        }
+    }
+
+    @RunWith(Parameterized.class)
+    public static class BelowMinimumRejectedTest
+    {
+        @Parameterized.Parameter
+        public int invalidValue;
+
+        @Parameterized.Parameters(name = "value={0}")
+        public static Collection<Object[]> invalidValues()
+        {
+            return Arrays.asList(new Object[][]{ { 2 }, { 1 }, { 0 } });
+        }
+
+        @BeforeClass
+        public static void setupClass()
+        {
+            DatabaseDescriptor.daemonInitialization();
+        }
+
+        @Test
+        public void testConfigValidationBelowMinimumIsRejected()
+        {
+            GuardrailsOptions guardrails = DatabaseDescriptor.getGuardrailsConfig();
+            assertThatThrownBy(() -> guardrails.setMinimumCmsSizeFailThreshold(invalidValue))
+                      .isInstanceOf(IllegalArgumentException.class)
+                      .hasMessageContaining("minimum allowed value is 3");
+        }
+
+        @Test
+        public void testJmxValidationBelowMinimumIsRejected()
+        {
+            GuardrailsMBean jmx = guardrailsJmxProxy();
+            // The setter throws IllegalArgumentException; the MBeanServer wraps it in RuntimeMBeanException, but the
+            // JMX proxy (MBeanServerInvocationHandler) unwraps it back to the original exception.
+            assertThatThrownBy(() -> jmx.setMinimumCmsSizeFailThreshold(invalidValue))
+                      .isInstanceOf(IllegalArgumentException.class)
+                      .hasMessageContaining("minimum allowed value is 3");
+        }
+    }
+}

test/unit/org/apache/cassandra/tools/nodetool/GuardrailsConfigCommandsTest.java

@@ -172,9 +172,12 @@ public class GuardrailsConfigCommandsTest extends CQLTester
             for (Method method : entry.getValue())
             {
                 String guardrailName = GuardrailsConfigCommand.toSnakeCase(method.getName().substring(3));
-                if (entry.getValue().size() == 1)
+                // Thresholds are grouped under a collapsed "<name>_threshold" key that intentionally differs from
+                // the field name (e.g. "<name>_fail_threshold"), whether or not the guardrail has both warn and
+                // fail halves. Everything else keys directly by its field name.
+                boolean isThreshold = method.getName().endsWith("Threshold");
+                if (!isThreshold)
                     assertEquals(entry.getKey(), guardrailName);
-                // else it is threshold, so it does not match the key
 
                 // assert converted snake-case guardrail name is actually in Config / cassandra.yaml
                 assertTrue(configFieldNames.contains(guardrailName));
@@ -240,6 +243,7 @@ public class GuardrailsConfigCommandsTest extends CQLTester
     "materialized_views_per_table_threshold       [-1, -1]      \n" +
     "maximum_replication_factor_threshold         [-1, -1]      \n" +
     "maximum_timestamp_threshold                  [null, null]  \n" +
+    "minimum_cms_size_threshold                   -1            \n" +
     "minimum_replication_factor_threshold         [-1, -1]      \n" +
     "minimum_timestamp_threshold                  [null, null]  \n" +
     "page_size_threshold                          [-1, -1]      \n" +
@@ -289,6 +293,7 @@ public class GuardrailsConfigCommandsTest extends CQLTester
     "maximum_replication_factor_warn_threshold         -1   \n" +
     "maximum_timestamp_fail_threshold                  null \n" +
     "maximum_timestamp_warn_threshold                  null \n" +
+    "minimum_cms_size_fail_threshold                   -1   \n" +
     "minimum_replication_factor_fail_threshold         -1   \n" +
     "minimum_replication_factor_warn_threshold         -1   \n" +
     "minimum_timestamp_fail_threshold                  null \n" +