Add nodetool getseeds and reloadseeds commands

patch by Samuel Fink; reviewed by jasobrown for CASSANDRA-14190

CHANGES.txt

@@ -1,4 +1,5 @@
 4.0
+ * Non-disruptive seed node list reload (CASSANDRA-14190)
  * Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185)
  * Migrate dtests to use pytest and python3 (CASSANDRA-14134)
  * Allow storage port to be configurable per node (CASSANDRA-7544)

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -1661,6 +1661,16 @@ public class DatabaseDescriptor
         return ImmutableSet.<InetAddressAndPort>builder().addAll(seedProvider.getSeeds()).build();
     }
 
+    public static SeedProvider getSeedProvider()
+    {
+        return seedProvider;
+    }
+
+    public static void setSeedProvider(SeedProvider newSeedProvider)
+    {
+        seedProvider = newSeedProvider;
+    }
+
     public static InetAddress getListenAddress()
     {
         return listenAddress;

src/java/org/apache/cassandra/gms/Gossiper.java

@@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.Uninterruptibles;
 
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.SeedProvider;
 import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.Pair;
 import org.slf4j.Logger;
@@ -89,7 +90,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     public final static int intervalInMillis = 1000;
     public final static int QUARANTINE_DELAY = StorageService.RING_DELAY * 2;
     private static final Logger logger = LoggerFactory.getLogger(Gossiper.class);
-    public static final Gossiper instance = new Gossiper();
+    public static final Gossiper instance = new Gossiper(true);
 
     // Timestamp to prevent processing any in-flight messages for we've not send any SYN yet, see CASSANDRA-12653.
     volatile long firstSynSendAt = 0L;
@@ -199,7 +200,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         }
     }
 
-    private Gossiper()
+    Gossiper(boolean registerJmx)
     {
         // half of QUARATINE_DELAY, to ensure justRemovedEndpoints has enough leeway to prevent re-gossip
         fatClientTimeout = (QUARANTINE_DELAY / 2);
@@ -207,14 +208,17 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         FailureDetector.instance.registerFailureDetectionEventListener(this);
 
         // Register this instance with JMX
-        try
+        if (registerJmx)
         {
-            MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+            try
-            mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
+            {
-        }
+                MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
-        catch (Exception e)
+                mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
-        {
+            }
-            throw new RuntimeException(e);
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
         }
     }
 
@@ -1468,6 +1472,67 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         }
     }
 
+    /**
+     * JMX interface for triggering an update of the seed node list.
+     */
+    public List<String> reloadSeeds()
+    {
+        logger.trace("Triggering reload of seed node list");
+
+        // Get the new set in the same that buildSeedsList does
+        Set<InetAddressAndPort> tmp = new HashSet<>();
+        try
+        {
+            for (InetAddressAndPort seed : DatabaseDescriptor.getSeeds())
+            {
+                if (seed.equals(FBUtilities.getBroadcastAddressAndPort()))
+                    continue;
+                tmp.add(seed);
+            }
+        }
+        // If using the SimpleSeedProvider invalid yaml added to the config since startup could
+        // cause this to throw. Additionally, third party seed providers may throw exceptions.
+        // Handle the error and return a null to indicate that there was a problem.
+        catch (Throwable e)
+        {
+            JVMStabilityInspector.inspectThrowable(e);
+            logger.warn("Error while getting seed node list: {}", e.getLocalizedMessage());
+            return null;
+        }
+
+        if (tmp.size() == 0)
+        {
+            logger.trace("New seed node list is empty. Not updating seed list.");
+            return getSeeds();
+        }
+
+        if (tmp.equals(seeds))
+        {
+            logger.trace("New seed node list matches the existing list.");
+            return getSeeds();
+        }
+
+        // Add the new entries
+        seeds.addAll(tmp);
+        // Remove the old entries
+        seeds.retainAll(tmp);
+        logger.trace("New seed node list after reload {}", seeds);
+        return getSeeds();
+    }
+
+    /**
+     * JMX endpoint for getting the list of seeds from the node
+     */
+    public List<String> getSeeds()
+    {
+        List<String> seedList = new ArrayList<String>();
+        for (InetAddressAndPort seed : seeds)
+        {
+            seedList.add(seed.toString());
+        }
+        return seedList;
+    }
+
     // initialize local HB state if needed, i.e., if gossiper has never been started before.
     public void maybeInitializeLocalState(int generationNbr)
     {

src/java/org/apache/cassandra/gms/GossiperMBean.java

@@ -18,6 +18,7 @@
 package org.apache.cassandra.gms;
 
 import java.net.UnknownHostException;
+import java.util.List;
 
 public interface GossiperMBean
 {
@@ -29,4 +30,8 @@ public interface GossiperMBean
 
     public void assassinateEndpoint(String address) throws UnknownHostException;
 
+    public List<String> reloadSeeds();
+
+    public List<String> getSeeds();
+
 }

src/java/org/apache/cassandra/tools/NodeProbe.java

@@ -711,6 +711,16 @@ public class NodeProbe implements AutoCloseable
         gossProxy.assassinateEndpoint(address);
     }
 
+    public List<String> reloadSeeds()
+    {
+        return gossProxy.reloadSeeds();
+    }
+
+    public List<String> getSeeds()
+    {
+        return gossProxy.getSeeds();
+    }
+
     /**
      * Set the compaction threshold
      *

src/java/org/apache/cassandra/tools/NodeTool.java

@@ -87,6 +87,7 @@ public class NodeTool
                 GetTraceProbability.class,
                 GetInterDCStreamThroughput.class,
                 GetEndpoints.class,
+                GetSeeds.class,
                 GetSSTables.class,
                 GetMaxHintWindow.class,
                 GossipInfo.class,
@@ -102,6 +103,7 @@ public class NodeTool
                 Refresh.class,
                 RemoveNode.class,
                 Assassinate.class,
+                ReloadSeeds.class,
                 ResetFullQueryLog.class,
                 Repair.class,
                 RepairAdmin.class,

src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.List;
+
+import io.airlift.airline.Command;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "getseeds", description = "Get the currently in use seed node IP list excluding the node IP")
+public class GetSeeds extends NodeToolCmd
+{
+    @Override
+    public void execute(NodeProbe probe)
+    {
+        List<String> seedList = probe.getSeeds();
+        if (seedList.isEmpty())
+        {
+            System.out.println("Seed node list does not contain any remote node IPs");
+        }
+        else
+        {
+            System.out.println("Current list of seed node IPs, excluding the current node's IP: " + String.join(" ", seedList));
+        }
+
+    }
+}

src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.List;
+
+import io.airlift.airline.Command;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "reloadseeds", description = "Reload the seed node list from the seed node provider")
+public class ReloadSeeds extends NodeToolCmd
+{
+    @Override
+    public void execute(NodeProbe probe)
+    {
+        List<String> seedList = probe.reloadSeeds();
+        if (seedList == null)
+        {
+            System.out.println("Failed to reload the seed node list.");
+        }
+        else if (seedList.isEmpty())
+        {
+            System.out.println("Seed node list does not contain any remote node IPs");
+        }
+        else
+        {
+            System.out.println("Updated seed node IP list, excluding the current node's IP: " + String.join(" ", seedList));
+        }
+    }
+}

test/unit/org/apache/cassandra/gms/GossiperTest.java

@@ -18,12 +18,16 @@
 
 package org.apache.cassandra.gms;
 
+import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.common.net.InetAddresses;
+import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -33,6 +37,7 @@ import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.RandomPartitioner;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.SeedProvider;
 import org.apache.cassandra.locator.TokenMetadata;
 import org.apache.cassandra.service.StorageService;
 
@@ -44,6 +49,7 @@ public class GossiperTest
     {
         DatabaseDescriptor.daemonInitialization();
     }
+
     static final IPartitioner partitioner = new RandomPartitioner();
     StorageService ss = StorageService.instance;
     TokenMetadata tmd = StorageService.instance.getTokenMetadata();
@@ -52,11 +58,20 @@ public class GossiperTest
     List<InetAddressAndPort> hosts = new ArrayList<>();
     List<UUID> hostIds = new ArrayList<>();
 
+    private SeedProvider originalSeedProvider;
+
     @Before
     public void setup()
     {
         tmd.clearUnsafe();
-    };
+        originalSeedProvider = DatabaseDescriptor.getSeedProvider();
+    }
+
+    @After
+    public void tearDown()
+    {
+        DatabaseDescriptor.setSeedProvider(originalSeedProvider);
+    }
 
     @Test
     public void testLargeGenerationJump() throws UnknownHostException, InterruptedException
@@ -90,4 +105,127 @@ public class GossiperTest
         //The generation should not have been updated because it is over Gossiper.MAX_GENERATION_DIFFERENCE in the future
         assertEquals(proposedRemoteHeartBeat.getGeneration(), actualRemoteHeartBeat.getGeneration());
     }
+
+    // Note: This test might fail if for some reason the node broadcast address is in 127.99.0.0/16
+    @Test
+    public void testReloadSeeds() throws UnknownHostException
+    {
+        Gossiper gossiper = new Gossiper(false);
+        List<String> loadedList;
+
+        // Initialize the seed list directly to a known set to start with
+        gossiper.seeds.clear();
+        InetAddressAndPort addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.1.1"));
+        int nextSize = 4;
+        List<InetAddressAndPort> nextSeeds = new ArrayList<>(nextSize);
+        for (int i = 0; i < nextSize; i ++)
+        {
+            gossiper.seeds.add(addr);
+            nextSeeds.add(addr);
+            addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+        }
+        Assert.assertEquals(nextSize, gossiper.seeds.size());
+
+        // Add another unique address to the list
+        addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+        nextSeeds.add(addr);
+        nextSize++;
+        DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
+        loadedList = gossiper.reloadSeeds();
+
+        // Check that the new entry was added
+        Assert.assertEquals(nextSize, loadedList.size());
+        for (InetAddressAndPort a : nextSeeds)
+            Assert.assertTrue(loadedList.contains(a.toString()));
+
+        // Check that the return value of the reloadSeeds matches the content of the getSeeds call
+        // and that they both match the internal contents of the Gossiper seeds list
+        Assert.assertEquals(loadedList.size(), gossiper.getSeeds().size());
+        for (InetAddressAndPort a : gossiper.seeds)
+        {
+            Assert.assertTrue(loadedList.contains(a.toString()));
+            Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
+        }
+
+        // Add a duplicate of the last address to the seed provider list
+        int uniqueSize = nextSize;
+        nextSeeds.add(addr);
+        nextSize++;
+        DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
+        loadedList = gossiper.reloadSeeds();
+
+        // Check that the number of seed nodes reported hasn't increased
+        Assert.assertEquals(uniqueSize, loadedList.size());
+        for (InetAddressAndPort a : nextSeeds)
+            Assert.assertTrue(loadedList.contains(a.toString()));
+
+        // Create a new list that has no overlaps with the previous list
+        addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.2.1"));
+        int disjointSize = 3;
+        List<InetAddressAndPort> disjointSeeds = new ArrayList<>(disjointSize);
+        for (int i = 0; i < disjointSize; i ++)
+        {
+            disjointSeeds.add(addr);
+            addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+        }
+        DatabaseDescriptor.setSeedProvider(new TestSeedProvider(disjointSeeds));
+        loadedList = gossiper.reloadSeeds();
+
+        // Check that the list now contains exactly the new other list.
+        Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
+        Assert.assertEquals(disjointSize, loadedList.size());
+        for (InetAddressAndPort a : disjointSeeds)
+        {
+            Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
+            Assert.assertTrue(loadedList.contains(a.toString()));
+        }
+
+        // Set the seed node provider to return an empty list
+        DatabaseDescriptor.setSeedProvider(new TestSeedProvider(new ArrayList<InetAddressAndPort>()));
+        loadedList = gossiper.reloadSeeds();
+
+        // Check that the in memory seed node list was not modified
+        Assert.assertEquals(disjointSize, loadedList.size());
+        for (InetAddressAndPort a : disjointSeeds)
+            Assert.assertTrue(loadedList.contains(a.toString()));
+
+        // Change the seed provider to one that throws an unchecked exception
+        DatabaseDescriptor.setSeedProvider(new ErrorSeedProvider());
+        loadedList = gossiper.reloadSeeds();
+
+        // Check for the expected null response from a reload error
+        Assert.assertNull(loadedList);
+
+        // Check that the in memory seed node list was not modified and the exception was caught
+        Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
+        for (InetAddressAndPort a : disjointSeeds)
+            Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
+    }
+
+    static class TestSeedProvider implements SeedProvider
+    {
+        private List<InetAddressAndPort> seeds;
+
+        TestSeedProvider(List<InetAddressAndPort> seeds)
+        {
+            this.seeds = seeds;
+        }
+
+        @Override
+        public List<InetAddressAndPort> getSeeds()
+        {
+            return seeds;
+        }
+    }
+
+    // A seed provider for testing which throws assertion errors when queried
+    static class ErrorSeedProvider implements SeedProvider
+    {
+        @Override
+        public List<InetAddressAndPort> getSeeds()
+        {
+            assert(false);
+            return new ArrayList<>();
+        }
+    }
 }