natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add nodetool getseeds and reloadseeds commands 

patch by Samuel Fink; reviewed by jasobrown for CASSANDRA-14190
This commit is contained in:
Samuel Fink 2018-01-24 13:28:54 -05:00 committed by Jason Brown
parent 69db2359ee
commit bfecdf5205
9 changed files with 332 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.0
* Non-disruptive seed node list reload (CASSANDRA-14190)
* Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185)
* Migrate dtests to use pytest and python3 (CASSANDRA-14134)
* Allow storage port to be configurable per node (CASSANDRA-7544)

10
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -1661,6 +1661,16 @@ public class DatabaseDescriptor
return ImmutableSet.<InetAddressAndPort>builder().addAll(seedProvider.getSeeds()).build();
}
public static SeedProvider getSeedProvider()
{
return seedProvider;
}
public static void setSeedProvider(SeedProvider newSeedProvider)
{
seedProvider = newSeedProvider;
}
public static InetAddress getListenAddress()
{
return listenAddress;

83
src/java/org/apache/cassandra/gms/Gossiper.java
View File

@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.SeedProvider;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.Pair;
import org.slf4j.Logger;
@ -89,7 +90,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
public final static int intervalInMillis = 1000;
public final static int QUARANTINE_DELAY = StorageService.RING_DELAY * 2;
private static final Logger logger = LoggerFactory.getLogger(Gossiper.class);
public static final Gossiper instance = new Gossiper();
public static final Gossiper instance = new Gossiper(true);
// Timestamp to prevent processing any in-flight messages for we've not send any SYN yet, see CASSANDRA-12653.
volatile long firstSynSendAt = 0L;
@ -199,7 +200,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
}
}
private Gossiper()
Gossiper(boolean registerJmx)
{
// half of QUARATINE_DELAY, to ensure justRemovedEndpoints has enough leeway to prevent re-gossip
fatClientTimeout = (QUARANTINE_DELAY / 2);
@ -207,14 +208,17 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
FailureDetector.instance.registerFailureDetectionEventListener(this);
// Register this instance with JMX
try
if (registerJmx)
{
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
}
catch (Exception e)
{
throw new RuntimeException(e);
try
{
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
}
@ -1468,6 +1472,67 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
}
}
/**
* JMX interface for triggering an update of the seed node list.
*/
public List<String> reloadSeeds()
{
logger.trace("Triggering reload of seed node list");
// Get the new set in the same that buildSeedsList does
Set<InetAddressAndPort> tmp = new HashSet<>();
try
{
for (InetAddressAndPort seed : DatabaseDescriptor.getSeeds())
{
if (seed.equals(FBUtilities.getBroadcastAddressAndPort()))
continue;
tmp.add(seed);
}
}
// If using the SimpleSeedProvider invalid yaml added to the config since startup could
// cause this to throw. Additionally, third party seed providers may throw exceptions.
// Handle the error and return a null to indicate that there was a problem.
catch (Throwable e)
{
JVMStabilityInspector.inspectThrowable(e);
logger.warn("Error while getting seed node list: {}", e.getLocalizedMessage());
return null;
}
if (tmp.size() == 0)
{
logger.trace("New seed node list is empty. Not updating seed list.");
return getSeeds();
}
if (tmp.equals(seeds))
{
logger.trace("New seed node list matches the existing list.");
return getSeeds();
}
// Add the new entries
seeds.addAll(tmp);
// Remove the old entries
seeds.retainAll(tmp);
logger.trace("New seed node list after reload {}", seeds);
return getSeeds();
}
/**
* JMX endpoint for getting the list of seeds from the node
*/
public List<String> getSeeds()
{
List<String> seedList = new ArrayList<String>();
for (InetAddressAndPort seed : seeds)
{
seedList.add(seed.toString());
}
return seedList;
}
// initialize local HB state if needed, i.e., if gossiper has never been started before.
public void maybeInitializeLocalState(int generationNbr)
{

5
src/java/org/apache/cassandra/gms/GossiperMBean.java
View File

@ -18,6 +18,7 @@
package org.apache.cassandra.gms;
import java.net.UnknownHostException;
import java.util.List;
public interface GossiperMBean
{
@ -29,4 +30,8 @@ public interface GossiperMBean
public void assassinateEndpoint(String address) throws UnknownHostException;
public List<String> reloadSeeds();
public List<String> getSeeds();
}

10
src/java/org/apache/cassandra/tools/NodeProbe.java
View File

@ -711,6 +711,16 @@ public class NodeProbe implements AutoCloseable
gossProxy.assassinateEndpoint(address);
}
public List<String> reloadSeeds()
{
return gossProxy.reloadSeeds();
}
public List<String> getSeeds()
{
return gossProxy.getSeeds();
}
/**
* Set the compaction threshold
*

2
src/java/org/apache/cassandra/tools/NodeTool.java
View File

@ -87,6 +87,7 @@ public class NodeTool
GetTraceProbability.class,
GetInterDCStreamThroughput.class,
GetEndpoints.class,
GetSeeds.class,
GetSSTables.class,
GetMaxHintWindow.class,
GossipInfo.class,
@ -102,6 +103,7 @@ public class NodeTool
Refresh.class,
RemoveNode.class,
Assassinate.class,
ReloadSeeds.class,
ResetFullQueryLog.class,
Repair.class,
RepairAdmin.class,

44
src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools.nodetool;
import java.util.List;
import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@Command(name = "getseeds", description = "Get the currently in use seed node IP list excluding the node IP")
public class GetSeeds extends NodeToolCmd
{
@Override
public void execute(NodeProbe probe)
{
List<String> seedList = probe.getSeeds();
if (seedList.isEmpty())
{
System.out.println("Seed node list does not contain any remote node IPs");
}
else
{
System.out.println("Current list of seed node IPs, excluding the current node's IP: " + String.join(" ", seedList));
}
}
}

47
src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools.nodetool;
import java.util.List;
import io.airlift.airline.Command;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@Command(name = "reloadseeds", description = "Reload the seed node list from the seed node provider")
public class ReloadSeeds extends NodeToolCmd
{
@Override
public void execute(NodeProbe probe)
{
List<String> seedList = probe.reloadSeeds();
if (seedList == null)
{
System.out.println("Failed to reload the seed node list.");
}
else if (seedList.isEmpty())
{
System.out.println("Seed node list does not contain any remote node IPs");
}
else
{
System.out.println("Updated seed node IP list, excluding the current node's IP: " + String.join(" ", seedList));
}
}
}

140
test/unit/org/apache/cassandra/gms/GossiperTest.java
View File

@ -18,12 +18,16 @@
package org.apache.cassandra.gms;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import com.google.common.collect.ImmutableMap;
import com.google.common.net.InetAddresses;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -33,6 +37,7 @@ import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.SeedProvider;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
@ -44,6 +49,7 @@ public class GossiperTest
{
DatabaseDescriptor.daemonInitialization();
}
static final IPartitioner partitioner = new RandomPartitioner();
StorageService ss = StorageService.instance;
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
@ -52,11 +58,20 @@ public class GossiperTest
List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<>();
private SeedProvider originalSeedProvider;
@Before
public void setup()
{
tmd.clearUnsafe();
};
originalSeedProvider = DatabaseDescriptor.getSeedProvider();
}
@After
public void tearDown()
{
DatabaseDescriptor.setSeedProvider(originalSeedProvider);
}
@Test
public void testLargeGenerationJump() throws UnknownHostException, InterruptedException
@ -90,4 +105,127 @@ public class GossiperTest
//The generation should not have been updated because it is over Gossiper.MAX_GENERATION_DIFFERENCE in the future
assertEquals(proposedRemoteHeartBeat.getGeneration(), actualRemoteHeartBeat.getGeneration());
}
// Note: This test might fail if for some reason the node broadcast address is in 127.99.0.0/16
@Test
public void testReloadSeeds() throws UnknownHostException
{
Gossiper gossiper = new Gossiper(false);
List<String> loadedList;
// Initialize the seed list directly to a known set to start with
gossiper.seeds.clear();
InetAddressAndPort addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.1.1"));
int nextSize = 4;
List<InetAddressAndPort> nextSeeds = new ArrayList<>(nextSize);
for (int i = 0; i < nextSize; i ++)
{
gossiper.seeds.add(addr);
nextSeeds.add(addr);
addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
}
Assert.assertEquals(nextSize, gossiper.seeds.size());
// Add another unique address to the list
addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
nextSeeds.add(addr);
nextSize++;
DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
loadedList = gossiper.reloadSeeds();
// Check that the new entry was added
Assert.assertEquals(nextSize, loadedList.size());
for (InetAddressAndPort a : nextSeeds)
Assert.assertTrue(loadedList.contains(a.toString()));
// Check that the return value of the reloadSeeds matches the content of the getSeeds call
// and that they both match the internal contents of the Gossiper seeds list
Assert.assertEquals(loadedList.size(), gossiper.getSeeds().size());
for (InetAddressAndPort a : gossiper.seeds)
{
Assert.assertTrue(loadedList.contains(a.toString()));
Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
}
// Add a duplicate of the last address to the seed provider list
int uniqueSize = nextSize;
nextSeeds.add(addr);
nextSize++;
DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
loadedList = gossiper.reloadSeeds();
// Check that the number of seed nodes reported hasn't increased
Assert.assertEquals(uniqueSize, loadedList.size());
for (InetAddressAndPort a : nextSeeds)
Assert.assertTrue(loadedList.contains(a.toString()));
// Create a new list that has no overlaps with the previous list
addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.2.1"));
int disjointSize = 3;
List<InetAddressAndPort> disjointSeeds = new ArrayList<>(disjointSize);
for (int i = 0; i < disjointSize; i ++)
{
disjointSeeds.add(addr);
addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
}
DatabaseDescriptor.setSeedProvider(new TestSeedProvider(disjointSeeds));
loadedList = gossiper.reloadSeeds();
// Check that the list now contains exactly the new other list.
Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
Assert.assertEquals(disjointSize, loadedList.size());
for (InetAddressAndPort a : disjointSeeds)
{
Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
Assert.assertTrue(loadedList.contains(a.toString()));
}
// Set the seed node provider to return an empty list
DatabaseDescriptor.setSeedProvider(new TestSeedProvider(new ArrayList<InetAddressAndPort>()));
loadedList = gossiper.reloadSeeds();
// Check that the in memory seed node list was not modified
Assert.assertEquals(disjointSize, loadedList.size());
for (InetAddressAndPort a : disjointSeeds)
Assert.assertTrue(loadedList.contains(a.toString()));
// Change the seed provider to one that throws an unchecked exception
DatabaseDescriptor.setSeedProvider(new ErrorSeedProvider());
loadedList = gossiper.reloadSeeds();
// Check for the expected null response from a reload error
Assert.assertNull(loadedList);
// Check that the in memory seed node list was not modified and the exception was caught
Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
for (InetAddressAndPort a : disjointSeeds)
Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
}
static class TestSeedProvider implements SeedProvider
{
private List<InetAddressAndPort> seeds;
TestSeedProvider(List<InetAddressAndPort> seeds)
{
this.seeds = seeds;
}
@Override
public List<InetAddressAndPort> getSeeds()
{
return seeds;
}
}
// A seed provider for testing which throws assertion errors when queried
static class ErrorSeedProvider implements SeedProvider
{
@Override
public List<InetAddressAndPort> getSeeds()
{
assert(false);
return new ArrayList<>();
}
}
}