From 94c361d1a7a63490f1b66f5b10e02579516c5eae Mon Sep 17 00:00:00 2001 From: Paulo Motta Date: Thu, 26 Nov 2015 13:00:20 -0800 Subject: [PATCH] Add property to allow listening on broadcast_address (CASSANDRA-9748) patch by pauloricardomg; reviewed by aweisberg for CASSANDRA-9748 --- CHANGES.txt | 1 + conf/cassandra.yaml | 8 ++++++++ .../org/apache/cassandra/config/Config.java | 1 + .../cassandra/config/DatabaseDescriptor.java | 5 +++++ .../locator/ReconnectableSnitchHelper.java | 2 +- .../cassandra/net/MessagingService.java | 20 +++++++++++++++---- .../cassandra/service/StorageService.java | 8 ++++---- .../apache/cassandra/utils/FBUtilities.java | 16 +++++++++++++++ .../apache/cassandra/service/RemoveTest.java | 2 +- 9 files changed, 53 insertions(+), 10 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5da0d428a4..035e47118c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.2.5 + * Add property to allow listening on broadcast interface (CASSANDRA-9748) * Fix regression in split size on CqlInputFormat (CASSANDRA-10835) * Better handling of SSL connection errors inter-node (CASSANDRA-10816) * Disable reloading of GossipingPropertyFileSnitch (CASSANDRA-9474) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index dc4ef32e62..ec30488937 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -421,6 +421,14 @@ listen_address: localhost # Leaving this blank will set it to the same value as listen_address # broadcast_address: 1.2.3.4 +# When using multiple physical network interfaces, set this +# to true to listen on broadcast_address in addition to +# the listen_address, allowing nodes to communicate in both +# interfaces. +# Ignore this property if the network configuration automatically +# routes between the public and private networks such as EC2. +# listen_on_broadcast_address: false + # Internode authentication backend, implementing IInternodeAuthenticator; # used to allow/disallow connections from peer nodes. # internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index a8be5f4030..c8ce0d7279 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -112,6 +112,7 @@ public class Config public String listen_interface; public Boolean listen_interface_prefer_ipv6 = false; public String broadcast_address; + public Boolean listen_on_broadcast_address = false; public String internode_authenticator; /* intentionally left set to true, despite being set to false in stock 2.2 cassandra.yaml diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 4fcc1fe18e..af0595daa8 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -1226,6 +1226,11 @@ public class DatabaseDescriptor return broadcastAddress; } + public static Boolean shouldListenOnBroadcastAddress() + { + return conf.listen_on_broadcast_address; + } + public static IInternodeAuthenticator getInternodeAuthenticator() { return internodeAuthenticator; diff --git a/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java b/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java index 8b34fc0c9f..3277af7e21 100644 --- a/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java +++ b/src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java @@ -63,7 +63,7 @@ public class ReconnectableSnitchHelper implements IEndpointStateChangeSubscriber && !MessagingService.instance().getConnectionPool(publicAddress).endPoint().equals(localAddress)) { MessagingService.instance().getConnectionPool(publicAddress).reset(localAddress); - logger.trace(String.format("Intiated reconnect to an Internal IP %s for the %s", localAddress, publicAddress)); + logger.debug(String.format("Intiated reconnect to an Internal IP %s for the %s", localAddress, publicAddress)); } } diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index 09ee73f1f1..61e58c20a2 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -447,21 +447,31 @@ public final class MessagingService implements MessagingServiceMBean getConnectionPool(ep).reset(); } + public void listen() + { + callbacks.reset(); // hack to allow tests to stop/restart MS + listen(FBUtilities.getLocalAddress()); + if (DatabaseDescriptor.shouldListenOnBroadcastAddress() + && !FBUtilities.getLocalAddress().equals(FBUtilities.getBroadcastAddress())) + { + listen(FBUtilities.getBroadcastAddress()); + } + listenGate.signalAll(); + } + /** * Listen on the specified port. * * @param localEp InetAddress whose port to listen on. */ - public void listen(InetAddress localEp) throws ConfigurationException + private void listen(InetAddress localEp) throws ConfigurationException { - callbacks.reset(); // hack to allow tests to stop/restart MS for (ServerSocket ss : getServerSockets(localEp)) { SocketThread th = new SocketThread(ss, "ACCEPT-" + localEp); th.start(); socketThreads.add(th); } - listenGate.signalAll(); } @SuppressWarnings("resource") @@ -524,7 +534,9 @@ public final class MessagingService implements MessagingServiceMBean FileUtils.closeQuietly(socket); throw new RuntimeException(e); } - logger.info("Starting Messaging Service on port {}", DatabaseDescriptor.getStoragePort()); + String nic = FBUtilities.getNetworkInterface(localEp); + logger.info("Starting Messaging Service on {}:{}{}", localEp, DatabaseDescriptor.getStoragePort(), + nic == null? "" : String.format(" (%s)", nic)); ss.add(socket); } return ss; diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 5a8d0c2fe9..e8e7daf358 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -509,7 +509,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { logger.info("Gathering node replacement information for {}", DatabaseDescriptor.getReplaceAddress()); if (!MessagingService.instance().isListening()) - MessagingService.instance().listen(FBUtilities.getLocalAddress()); + MessagingService.instance().listen(); // make magic happen Gossiper.instance.doShadowRound(); @@ -540,7 +540,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { logger.debug("Starting shadow gossip round to check for endpoint collision"); if (!MessagingService.instance().isListening()) - MessagingService.instance().listen(FBUtilities.getLocalAddress()); + MessagingService.instance().listen(); Gossiper.instance.doShadowRound(); if (!Gossiper.instance.isSafeForBootstrap(FBUtilities.getBroadcastAddress())) { @@ -573,7 +573,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE Gossiper.instance.start((int) (System.currentTimeMillis() / 1000)); // needed for node-ring gathering. Gossiper.instance.addLocalApplicationState(ApplicationState.NET_VERSION, valueFactory.networkVersion()); if (!MessagingService.instance().isListening()) - MessagingService.instance().listen(FBUtilities.getLocalAddress()); + MessagingService.instance().listen(); } public synchronized void initServer() throws ConfigurationException @@ -781,7 +781,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE Schema.instance.updateVersionAndAnnounce(); // Ensure we know our own actual Schema UUID in preparation for updates if (!MessagingService.instance().isListening()) - MessagingService.instance().listen(FBUtilities.getLocalAddress()); + MessagingService.instance().listen(); LoadBroadcaster.instance.startBroadcasting(); HintedHandOffManager.instance.start(); diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index a16fa132ae..9eda87826e 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -166,6 +166,22 @@ public class FBUtilities return localAddresses; } + public static String getNetworkInterface(InetAddress localAddress) + { + try { + for(NetworkInterface ifc : Collections.list(NetworkInterface.getNetworkInterfaces())) { + if(ifc.isUp()) { + for(InetAddress addr : Collections.list(ifc.getInetAddresses())) { + if (addr.equals(localAddress)) + return ifc.getDisplayName(); + } + } + } + } + catch (SocketException e) {} + return null; + } + /** * Given two bit arrays represented as BigIntegers, containing the given * number of significant bits, calculate a midpoint. diff --git a/test/unit/org/apache/cassandra/service/RemoveTest.java b/test/unit/org/apache/cassandra/service/RemoveTest.java index 6d7cac86b8..9f1d6a8d2a 100644 --- a/test/unit/org/apache/cassandra/service/RemoveTest.java +++ b/test/unit/org/apache/cassandra/service/RemoveTest.java @@ -78,7 +78,7 @@ public class RemoveTest // create a ring of 5 nodes Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 6); - MessagingService.instance().listen(FBUtilities.getBroadcastAddress()); + MessagingService.instance().listen(); Gossiper.instance.start(1); removalhost = hosts.get(5); hosts.remove(removalhost);