Resolve JMX output inconsistencies from CASSANDRA-7544 storage-port-configurable-per-node

patch by Jon Meredith; reviewed by David Capwell for CASSANDRA-15937

CHANGES.txt

@@ -18,6 +18,7 @@
  * Fix version parsing logic when upgrading from 3.0 (CASSANDRA-15973)
  * Optimize NoSpamLogger use in hot paths (CASSANDRA-15766)
  * Verify sstable components on startup (CASSANDRA-15945)
+ * Resolve JMX output inconsistencies from CASSANDRA-7544 storage-port-configurable-per-node (CASSANDRA-15937)
 Merged from 3.11:
  * Correctly interpret SASI's `max_compaction_flush_memory_in_mb` setting in megabytes not bytes (CASSANDRA-16071)
  * Fix short read protection for GROUP BY queries (CASSANDRA-15459)

src/java/org/apache/cassandra/gms/Gossiper.java

@@ -1949,7 +1949,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
                 hosts = new ArrayList<>();
                 results.put(stringVersion, hosts);
             }
-            hosts.add(host.getHostAddress(true));
+            hosts.add(host.getHostAddressAndPort());
         }
 
         return results;

src/java/org/apache/cassandra/gms/GossiperEvent.java

@@ -95,7 +95,7 @@ public final class GossiperEvent extends DiagnosticEvent
     {
         // be extra defensive against nulls and bugs
         HashMap<String, Serializable> ret = new HashMap<>();
-        if (endpoint != null) ret.put("endpoint", endpoint.getHostAddress(true));
+        if (endpoint != null) ret.put("endpoint", endpoint.getHostAddressAndPort());
         ret.put("quarantineExpiration", quarantineExpiration);
         ret.put("localState", String.valueOf(localState));
         ret.put("endpointStateMap", String.valueOf(endpointStateMap));

src/java/org/apache/cassandra/gms/VersionedValue.java

@@ -143,7 +143,7 @@ public class VersionedValue implements Comparable<VersionedValue>
 
         public VersionedValue bootReplacingWithPort(InetAddressAndPort oldNode)
         {
-            return new VersionedValue(versionString(VersionedValue.STATUS_BOOTSTRAPPING_REPLACE, oldNode.toString()));
+            return new VersionedValue(versionString(VersionedValue.STATUS_BOOTSTRAPPING_REPLACE, oldNode.getHostAddressAndPort()));
         }
 
         public VersionedValue bootstrapping(Collection<Token> tokens)
@@ -258,7 +258,7 @@ public class VersionedValue implements Comparable<VersionedValue>
 
         public VersionedValue nativeaddressAndPort(InetAddressAndPort address)
         {
-            return new VersionedValue(address.toString());
+            return new VersionedValue(address.getHostAddressAndPort());
         }
 
         public VersionedValue releaseVersion()
@@ -277,14 +277,14 @@ public class VersionedValue implements Comparable<VersionedValue>
             return new VersionedValue(String.valueOf(MessagingService.current_version));
         }
 
-        public VersionedValue internalIP(String private_ip)
+        public VersionedValue internalIP(InetAddress private_ip)
         {
-            return new VersionedValue(private_ip);
+            return new VersionedValue(private_ip.getHostAddress());
         }
 
-        public VersionedValue internalAddressAndPort(InetAddressAndPort address)
+        public VersionedValue internalAddressAndPort(InetAddressAndPort private_ip_and_port)
         {
-            return new VersionedValue(address.toString());
+            return new VersionedValue(private_ip_and_port.getHostAddressAndPort());
         }
 
         public VersionedValue severity(double value)

src/java/org/apache/cassandra/hints/HintEvent.java

@@ -89,7 +89,7 @@ final class HintEvent extends DiagnosticEvent
         // be extra defensive against nulls and bugs
         HashMap<String, Serializable> ret = new HashMap<>();
         ret.put("targetHostId", targetHostId);
-        ret.put("targetAddress", targetAddress.getHostAddress(true));
+        ret.put("targetAddress", targetAddress.getHostAddressAndPort());
         if (dispatchResult != null) ret.put("dispatchResult", dispatchResult.name());
         if (pageHintsSuccessful != null || pageHintsFailed != null || pageHintsTimeout != null)
         {

src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java

@@ -323,9 +323,9 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements Lat
         return scores.entrySet().stream().collect(Collectors.toMap(address -> address.getKey().address, Map.Entry::getValue));
     }
 
-    public Map<InetAddressAndPort, Double> getScoresWithPort()
+    public Map<String, Double> getScoresWithPort()
     {
-        return scores;
+        return scores.entrySet().stream().collect(Collectors.toMap(address -> address.getKey().toString(true), Map.Entry::getValue));
     }
 
     public int getUpdateInterval()

src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java

@@ -24,7 +24,7 @@ import java.util.List;
 
 public interface DynamicEndpointSnitchMBean 
 {
-    public Map<InetAddressAndPort, Double> getScoresWithPort();
+    public Map<String, Double> getScoresWithPort();
     @Deprecated
     public Map<InetAddress, Double> getScores();
     public int getUpdateInterval();

src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java

@@ -73,7 +73,7 @@ public class Ec2MultiRegionSnitch extends Ec2Snitch
             throw new RuntimeException(e);
         }
         Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT, StorageService.instance.valueFactory.internalAddressAndPort(address));
-        Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP, StorageService.instance.valueFactory.internalIP(localPrivateAddress));
+        Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP, StorageService.instance.valueFactory.internalIP(address.address));
         Gossiper.instance.register(new ReconnectableSnitchHelper(this, ec2region, true));
     }
 }

src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java

@@ -140,7 +140,7 @@ public class GossipingPropertyFileSnitch extends AbstractNetworkTopologySnitch//
         Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT,
                                                    StorageService.instance.valueFactory.internalAddressAndPort(FBUtilities.getLocalAddressAndPort()));
         Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP,
-                StorageService.instance.valueFactory.internalIP(FBUtilities.getJustLocalAddress().getHostAddress()));
+                StorageService.instance.valueFactory.internalIP(FBUtilities.getJustLocalAddress()));
 
         loadGossiperState();
     }

src/java/org/apache/cassandra/locator/InetAddressAndPort.java

@@ -24,6 +24,8 @@ import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
+import java.util.Objects;
+import java.util.regex.Pattern;
 
 import com.google.common.base.Preconditions;
 import com.google.common.net.HostAndPort;
@@ -119,11 +121,30 @@ public final class InetAddressAndPort implements Comparable<InetAddressAndPort>,
         return Integer.compare(port, o.port);
     }
 
+    public String getHostAddressAndPort()
+    {
+        return getHostAddress(true);
+    }
+
+    private static final Pattern JMX_INCOMPATIBLE_CHARS = Pattern.compile("[\\[\\]:]");
+
+
+    /**
+     * Return a version of getHostAddressAndPort suitable for use in JMX object names without
+     * requiring any escaping.  Replaces each character invalid for JMX names with an underscore.
+     *
+     * @return String with JMX-safe representation of the IP address and port
+     */
+    public String getHostAddressAndPortForJMX()
+    {
+        return JMX_INCOMPATIBLE_CHARS.matcher(getHostAddressAndPort()).replaceAll("_");
+    }
+
     public String getHostAddress(boolean withPort)
     {
         if (withPort)
         {
-            return toString();
+            return HostAndPort.fromParts(address.getHostAddress(), port).toString();
         }
         else
         {
@@ -149,9 +170,42 @@ public final class InetAddressAndPort implements Comparable<InetAddressAndPort>,
         }
     }
 
+    /** Format an InetAddressAndPort in the same style as InetAddress.toString.
+     *  The string returned is of the form: hostname / literal IP address : port
+     *  (without the whitespace). Literal IPv6 addresses will be wrapped with [ ]
+     *  to make the port number clear.
+     *
+     *  If the host name is unresolved, no reverse name service lookup
+     *  is performed. The hostname part will be represented by an empty string.
+     *
+     * @param address InetAddress to convert String
+     * @param port Port number to convert to String
+     * @return String representation of the IP address and port
+     */
     public static String toString(InetAddress address, int port)
     {
-        return HostAndPort.fromParts(address.getHostAddress(), port).toString();
+        String addressToString = address.toString(); // cannot use getHostName as it resolves
+        int nameLength = addressToString.lastIndexOf('/'); // use last index to prevent ambiguity if host name contains /
+        assert nameLength >= 0 : "InetAddress.toString format may have changed, expecting /";
+
+        // Check if need to wrap address with [ ] for IPv6 addresses
+        if (addressToString.indexOf(':', nameLength) >= 0)
+        {
+            StringBuilder sb = new StringBuilder(addressToString.length() + 16);
+            sb.append(addressToString, 0, nameLength + 1); // append optional host and / char
+            sb.append('[');
+            sb.append(addressToString, nameLength + 1, addressToString.length());
+            sb.append("]:");
+            sb.append(port);
+            return sb.toString();
+        }
+        else // can just append :port
+        {
+            StringBuilder sb = new StringBuilder(addressToString); // will have enough capacity for port
+            sb.append(":");
+            sb.append(port);
+            return sb.toString();
+        }
     }
 
     public static InetAddressAndPort getByName(String name) throws UnknownHostException

src/java/org/apache/cassandra/metrics/ClientMetrics.java

@@ -136,7 +136,7 @@ public final class ClientMetrics
 
         for (Server server : servers)
             for (ClientStat stat : server.recentClientStats())
-                stats.add(stat.asMap());
+                stats.add(new HashMap(stat.asMap())); // asMap returns guava, so need to convert to java for jmx
 
         stats.sort(Comparator.comparing(map -> map.get(ClientStat.PROTOCOL_VERSION)));
 

src/java/org/apache/cassandra/metrics/InternodeInboundMetrics.java

@@ -50,7 +50,7 @@ public class InternodeInboundMetrics
     public InternodeInboundMetrics(InetAddressAndPort peer, InboundMessageHandlers handlers)
     {
         // ipv6 addresses will contain colons, which are invalid in a JMX ObjectName
-        MetricNameFactory factory = new DefaultNameFactory("InboundConnection", peer.toString().replace(':', '_'));
+        MetricNameFactory factory = new DefaultNameFactory("InboundConnection", peer.getHostAddressAndPortForJMX());
 
         register(corruptFramesRecovered = factory.createMetricName("CorruptFramesRecovered"), handlers::corruptFramesRecovered);
         register(corruptFramesUnrecovered = factory.createMetricName("CorruptFramesUnrecovered"), handlers::corruptFramesUnrecovered);

src/java/org/apache/cassandra/metrics/InternodeOutboundMetrics.java

@@ -117,7 +117,7 @@ public class InternodeOutboundMetrics
     public InternodeOutboundMetrics(InetAddressAndPort ip, final OutboundConnections messagingPool)
     {
         // ipv6 addresses will contain colons, which are invalid in a JMX ObjectName
-        address = ip.toString().replace(':', '_');
+        address = ip.getHostAddressAndPortForJMX();
 
         factory = new DefaultNameFactory("Connection", address);
 

src/java/org/apache/cassandra/repair/SystemDistributedKeyspace.java

@@ -223,7 +223,7 @@ public final class SystemDistributedKeyspace
         for (InetAddressAndPort endpoint : commonRange.endpoints)
         {
             participants.add(endpoint.getHostAddress(false));
-            participants_v2.add(endpoint.toString());
+            participants_v2.add(endpoint.getHostAddressAndPort());
         }
 
         String query =

src/java/org/apache/cassandra/repair/consistent/LocalSessionInfo.java

@@ -63,7 +63,7 @@ public class LocalSessionInfo
         m.put(LAST_UPDATE, Integer.toString(session.getLastUpdate()));
         m.put(COORDINATOR, session.coordinator.toString());
         m.put(PARTICIPANTS, Joiner.on(',').join(Iterables.transform(session.participants.stream().map(peer -> peer.address).collect(Collectors.toList()), InetAddress::getHostAddress)));
-        m.put(PARTICIPANTS_WP, Joiner.on(',').join(Iterables.transform(session.participants, InetAddressAndPort::toString)));
+        m.put(PARTICIPANTS_WP, Joiner.on(',').join(Iterables.transform(session.participants, InetAddressAndPort::getHostAddressAndPort)));
         m.put(TABLES, Joiner.on(',').join(Iterables.transform(session.tableIds, LocalSessionInfo::tableString)));
 
         return m;

src/java/org/apache/cassandra/repair/consistent/LocalSessions.java

@@ -502,7 +502,7 @@ public class LocalSessions
                                        session.coordinator.address,
                                        session.coordinator.port,
                                        session.participants.stream().map(participant -> participant.address).collect(Collectors.toSet()),
-                                       session.participants.stream().map(participant -> participant.toString()).collect(Collectors.toSet()),
+                                       session.participants.stream().map(participant -> participant.getHostAddressAndPort()).collect(Collectors.toSet()),
                                        serializeRanges(session.ranges),
                                        tableIdToUuid(session.tableIds));
 

src/java/org/apache/cassandra/schema/SchemaMigrationEvent.java

@@ -100,7 +100,7 @@ final class SchemaMigrationEvent extends DiagnosticEvent
     public Map<String, Serializable> toMap()
     {
         HashMap<String, Serializable> ret = new HashMap<>();
-        if (endpoint != null) ret.put("endpoint", endpoint.getHostAddress(true));
+        if (endpoint != null) ret.put("endpoint", endpoint.getHostAddressAndPort());
         ret.put("endpointSchemaVersion", Schema.schemaVersionToString(endpointSchemaVersion));
         ret.put("localSchemaVersion", Schema.schemaVersionToString(localSchemaVersion));
         if (endpointMessagingVersion != null) ret.put("endpointMessagingVersion", endpointMessagingVersion);

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -2813,7 +2813,7 @@ public class StorageProxy implements StorageProxyMBean
 
     public String getIdealConsistencyLevel()
     {
-        return DatabaseDescriptor.getIdealConsistencyLevel().toString();
+        return Objects.toString(DatabaseDescriptor.getIdealConsistencyLevel(), "");
     }
 
     public String setIdealConsistencyLevel(String cl)

src/java/org/apache/cassandra/service/StorageService.java

@@ -1754,7 +1754,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     public String getNativeaddress(InetAddressAndPort endpoint, boolean withPort)
     {
         if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
-            return FBUtilities.getBroadcastNativeAddressAndPort().toString(withPort);
+            return FBUtilities.getBroadcastNativeAddressAndPort().getHostAddress(withPort);
         else if (Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT) != null)
         {
             try
@@ -3127,7 +3127,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             map.put(entry.getKey().getHostAddress(withPort), FileUtils.stringifyFileSize(entry.getValue()));
         }
         // gossiper doesn't see its own updates, so we need to special-case the local node
-        map.put(withPort ? FBUtilities.getJustBroadcastAddress().getHostAddress() : FBUtilities.getBroadcastAddressAndPort().toString(), getLoadString());
+        map.put(FBUtilities.getBroadcastAddressAndPort().getHostAddress(withPort), getLoadString());
         return map;
     }
 
@@ -3226,7 +3226,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
         for (Pair<Token, InetAddressAndPort> node : tokenMetadata.getMovingEndpoints())
         {
-            endpoints.add(node.right.toString());
+            endpoints.add(node.right.getHostAddressAndPort());
         }
 
         return endpoints;

src/java/org/apache/cassandra/streaming/StreamSession.java

@@ -652,7 +652,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
             {
                 logger.error("[Stream #{}] Socket closed before session completion, peer {} is probably down.",
                              planId(),
-                             peer.address.getHostAddress(),
+                             peer.getHostAddressAndPort(),
                              e);
 
                 return closeSession(State.FAILED);
@@ -673,16 +673,16 @@ public class StreamSession implements IEndpointStateChangeSubscriber
         {
             logger.error("[Stream #{}] Did not receive response from peer {}{} for {} secs. Is peer down? " +
                          "If not, maybe try increasing streaming_keep_alive_period_in_secs.", planId(),
-                         peer.getHostAddress(true),
-                         template.connectTo == null ? "" : " through " + template.connectTo.getHostAddress(true),
+                         peer.getHostAddressAndPort(),
+                         template.connectTo == null ? "" : " through " + template.connectTo.getHostAddressAndPort(),
                          2 * DatabaseDescriptor.getStreamingKeepAlivePeriod(),
                          e);
         }
         else
         {
             logger.error("[Stream #{}] Streaming error occurred on session with peer {}{}", planId(),
-                         peer.getHostAddress(true),
-                         template.connectTo == null ? "" : " through " + template.connectTo.getHostAddress(true),
+                         peer.getHostAddressAndPort(),
+                         template.connectTo == null ? "" : " through " + template.connectTo.getHostAddressAndPort(),
                          e);
         }
     }

test/distributed/org/apache/cassandra/distributed/impl/DistributedTestSnitch.java

@@ -123,6 +123,6 @@ public class DistributedTestSnitch extends AbstractNetworkTopologySnitch
         Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT,
                                                    StorageService.instance.valueFactory.internalAddressAndPort(FBUtilities.getLocalAddressAndPort()));
         Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP,
-                                                   StorageService.instance.valueFactory.internalIP(FBUtilities.getJustLocalAddress().getHostAddress()));
+                                                   StorageService.instance.valueFactory.internalIP(FBUtilities.getJustLocalAddress()));
     }
 }

test/distributed/org/apache/cassandra/distributed/test/GossipSettlesTest.java

@@ -18,9 +18,26 @@
 
 package org.apache.cassandra.distributed.test;
 
+import java.net.InetAddress;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.junit.Assert;
 import org.junit.Test;
 
-import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.SystemDistributedKeyspace;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
@@ -28,17 +45,116 @@ import static org.apache.cassandra.distributed.api.Feature.NETWORK;
 // TODO: this test should be removed after running in-jvm dtests is set up via the shared API repository
 public class GossipSettlesTest extends TestBaseImpl
 {
-
     @Test
     public void testGossipSettles() throws Throwable
     {
         /* Use withSubnet(1) to prove seed provider is set correctly - without the fix to pass a seed provider, this test fails */
-        try (ICluster cluster = builder().withNodes(3)
-                                         .withConfig(config -> config.with(GOSSIP).with(NETWORK))
-                                         .withSubnet(1)
-                                         .start())
+        try (Cluster cluster = builder().withNodes(3)
+                                        .withConfig(config -> config.with(GOSSIP).with(NETWORK))
+                                        .withSubnet(1)
+                                        .start())
         {
+            // Verify the 4.0 WithPort versions of status reporting methods match their InetAddress
+            // counterparts.  Disable Gossip first to prevent any bump in heartbeats that would
+            // invalidate the comparison.  Compare the newer WithPort versions by adding the
+            // storage port to IP addresses in keys/values/strings as appropriate.
+            cluster.forEach(i -> i.runOnInstance(() -> { Gossiper.instance.stop(); }));
+            cluster.get(1).runOnInstance(() -> {
+
+                // First prove that the storage port is added
+                Assert.assertEquals("stuff 127.0.0.1:7012 morestuff 127.0.0.2:7012", addStoragePortToIP("stuff 127.0.0.1 morestuff 127.0.0.2"));
+
+                FailureDetector fd = ((FailureDetector) FailureDetector.instance);
+                Assert.assertEquals(addStoragePortToInstanceName(fd.getAllEndpointStates(false)),
+                                    fd.getAllEndpointStates(true));
+                Assert.assertEquals(addPortToKeys(fd.getSimpleStates()), fd.getSimpleStatesWithPort());
+
+                StorageProxy sp = StorageProxy.instance;
+                Assert.assertEquals(addPortToSchemaVersions(sp.getSchemaVersions()), sp.getSchemaVersionsWithPort());
+
+                StorageService ss = StorageService.instance;
+                Assert.assertEquals(addPortToValues(ss.getTokenToEndpointMap()), ss.getTokenToEndpointWithPortMap());
+                Assert.assertEquals(addPortToKeys(ss.getEndpointToHostId()), ss.getEndpointWithPortToHostId());
+                Assert.assertEquals(addPortToValues(ss.getHostIdToEndpoint()), ss.getHostIdToEndpointWithPort());
+                Assert.assertEquals(addPortToKeys(ss.getLoadMap()), ss.getLoadMapWithPort());
+                Assert.assertEquals(addPortToList(ss.getLiveNodes()), ss.getLiveNodesWithPort());
+                List<String> naturalEndpointsAddedPort = ss.getNaturalEndpoints(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
+                                                                                SystemDistributedKeyspace.VIEW_BUILD_STATUS, "dummy").stream()
+                                                           .map(e -> addStoragePortToIP(e.getHostAddress())).collect(Collectors.toList());
+                Assert.assertEquals(naturalEndpointsAddedPort,
+                                    ss.getNaturalEndpointsWithPort(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
+                                                                   SystemDistributedKeyspace.VIEW_BUILD_STATUS, "dummy"));
+                naturalEndpointsAddedPort = ss.getNaturalEndpoints(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, ByteBufferUtil.EMPTY_BYTE_BUFFER).stream()
+                                              .map(e -> addStoragePortToIP(e.getHostAddress())).collect(Collectors.toList());
+                Assert.assertEquals(naturalEndpointsAddedPort,
+                                    ss.getNaturalEndpointsWithPort(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, ByteBufferUtil.EMPTY_BYTE_BUFFER));
+
+
+                // Difference in key type... convert to String and add the port to the older format
+                Map<String, Float> getOwnershipKeyAddedPort = ss.getOwnership().entrySet().stream()
+                                                                .collect(Collectors.<Map.Entry<InetAddress, Float>, String, Float>toMap(
+                                                                e -> addStoragePortToIP(e.getKey().toString()),
+                                                                Map.Entry::getValue));
+                Assert.assertEquals(getOwnershipKeyAddedPort, ss.getOwnershipWithPort());
+
+                MessagingService ms = MessagingService.instance();
+                Assert.assertEquals(addPortToKeys(ms.getTimeoutsPerHost()), ms.getTimeoutsPerHostWithPort());
+                Assert.assertEquals(addPortToKeys(ms.getLargeMessagePendingTasks()), ms.getLargeMessagePendingTasksWithPort());
+                Assert.assertEquals(addPortToKeys(ms.getLargeMessageCompletedTasks()), ms.getLargeMessageCompletedTasksWithPort());
+                Assert.assertEquals(addPortToKeys(ms.getLargeMessageDroppedTasks()), ms.getLargeMessageDroppedTasksWithPort());
+                Assert.assertEquals(addPortToKeys(ms.getSmallMessagePendingTasks()), ms.getSmallMessagePendingTasksWithPort());
+                Assert.assertEquals(addPortToKeys(ms.getSmallMessageCompletedTasks()), ms.getSmallMessageCompletedTasksWithPort());
+                Assert.assertEquals(addPortToKeys(ms.getSmallMessageDroppedTasks()), ms.getSmallMessageDroppedTasksWithPort());
+                Assert.assertEquals(addPortToKeys(ms.getGossipMessagePendingTasks()), ms.getGossipMessagePendingTasksWithPort());
+                Assert.assertEquals(addPortToKeys(ms.getGossipMessageCompletedTasks()), ms.getGossipMessageCompletedTasksWithPort());
+                Assert.assertEquals(addPortToKeys(ms.getGossipMessageDroppedTasks()), ms.getGossipMessageDroppedTasksWithPort());
+            });
         }
     }
 
+
+    final static Pattern IP4_ADDRESS = Pattern.compile("(127\\.0\\.\\d{1,3}\\.\\d{1,3})");
+
+    static String addStoragePortToIP(String s)
+    {
+        return IP4_ADDRESS.matcher(s).replaceAll("$1:" + DatabaseDescriptor.getStoragePort());
+    }
+
+    static String addStoragePortToInstanceName(String s)
+    {
+        return Arrays.stream(s.split("\n")).map(line -> {
+            if (line.startsWith(" "))
+            {
+                return line;
+            }
+            else // Host header line
+            {
+                return addStoragePortToIP(line);
+            }
+        }).collect(Collectors.joining("\n", "", "\n")); // to match final blank line
+    }
+
+    static <V> Map<String, V> addPortToKeys(Map<String, V> source)
+    {
+        return source.entrySet().stream().collect(Collectors.toMap(entry -> addStoragePortToIP(entry.getKey()),
+                                                                   Map.Entry::getValue));
+    }
+
+    static <K> Map<K, String> addPortToValues(Map<K, String> source)
+    {
+        return source.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey,
+                                                                   entry -> addStoragePortToIP(entry.getValue())));
+    }
+
+    static List<String> addPortToList(List<String> list)
+    {
+        return list.stream().map(GossipSettlesTest::addStoragePortToIP).collect(Collectors.toList());
+    }
+
+    static Map<String, List<String>> addPortToSchemaVersions(Map<String, List<String>> source)
+    {
+        return source.entrySet().stream()
+                     .collect(Collectors.toMap(Map.Entry::getKey,
+                                               hostAndPortEntry -> addPortToList(hostAndPortEntry.getValue())));
+    }
 }

test/distributed/org/apache/cassandra/distributed/test/RepairCoordinatorNeighbourDown.java

@@ -74,7 +74,7 @@ public abstract class RepairCoordinatorNeighbourDown extends RepairCoordinatorBa
         String table = tableName("neighbourdown");
         assertTimeoutPreemptively(Duration.ofMinutes(1), () -> {
             CLUSTER.schemaChange(format("CREATE TABLE %s.%s (key text, value text, PRIMARY KEY (key))", KEYSPACE, table));
-            String downNodeAddress = CLUSTER.get(2).callOnInstance(() -> FBUtilities.getBroadcastAddressAndPort().toString());
+            String downNodeAddress = CLUSTER.get(2).callOnInstance(() -> FBUtilities.getBroadcastAddressAndPort().getHostAddressAndPort());
             Future<Void> shutdownFuture = CLUSTER.get(2).shutdown();
             try
             {
@@ -170,8 +170,8 @@ public abstract class RepairCoordinatorNeighbourDown extends RepairCoordinatorBa
             if (withNotifications)
             {
                 result.asserts()
-                      .errorContains("Endpoint 127.0.0.2:7012 died")
-                      .notificationContains(NodeToolResult.ProgressEventType.ERROR, "Endpoint 127.0.0.2:7012 died")
+                      .errorContains("/127.0.0.2:7012 died")
+                      .notificationContains(NodeToolResult.ProgressEventType.ERROR, "/127.0.0.2:7012 died")
                       .notificationContains(NodeToolResult.ProgressEventType.COMPLETE, "finished with error");
             }
             else
@@ -188,7 +188,7 @@ public abstract class RepairCoordinatorNeighbourDown extends RepairCoordinatorBa
             Assert.assertEquals(repairExceptions + 1, getRepairExceptions(CLUSTER, 1));
             if (repairType != RepairType.PREVIEW)
             {
-                assertParentRepairFailedWithMessageContains(CLUSTER, KEYSPACE, table, "Endpoint 127.0.0.2:7012 died");
+                assertParentRepairFailedWithMessageContains(CLUSTER, KEYSPACE, table, "/127.0.0.2:7012 died");
             }
             else
             {

test/distributed/org/apache/cassandra/distributed/test/SimpleReadWriteTest.java

@@ -238,8 +238,8 @@ public class SimpleReadWriteTest extends TestBaseImpl
                 // I suppose it has to do with some classloader manipulation going on
                 Assert.assertTrue(e.getClass().toString().contains("WriteFailureException"));
                 // we may see 1 or 2 failures in here, because of the fail-fast behavior of AbstractWriteResponseHandler
-                Assert.assertTrue(e.getMessage().contains("INCOMPATIBLE_SCHEMA from 127.0.0.2")
-                                  || e.getMessage().contains("INCOMPATIBLE_SCHEMA from 127.0.0.3"));
+                Assert.assertTrue(e.getMessage().contains("INCOMPATIBLE_SCHEMA from ") &&
+                                  (e.getMessage().contains("/127.0.0.2") || e.getMessage().contains("/127.0.0.3")));
 
             }
         }
@@ -270,8 +270,8 @@ public class SimpleReadWriteTest extends TestBaseImpl
                 // I suppose it has to do with some classloader manipulation going on
                 Assert.assertTrue(e.getClass().toString().contains("ReadFailureException"));
                 // we may see 1 or 2 failures in here, because of the fail-fast behavior of ReadCallback
-                Assert.assertTrue(e.getMessage().contains("INCOMPATIBLE_SCHEMA from 127.0.0.2")
-                                  || e.getMessage().contains("INCOMPATIBLE_SCHEMA from 127.0.0.3"));
+                Assert.assertTrue(e.getMessage().contains("INCOMPATIBLE_SCHEMA from ") &&
+                                  (e.getMessage().contains("/127.0.0.2") || e.getMessage().contains("/127.0.0.3")));
             }
 
         }

test/distributed/org/apache/cassandra/distributed/test/TopologyChangeTest.java

@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.distributed.test;
 
+import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -63,19 +64,19 @@ public class TopologyChangeTest extends TestBaseImpl
 
         static class Event
         {
-            String host;
+            InetSocketAddress host;
             EventType type;
 
             Event(EventType type, Host host)
             {
                 this.type = type;
-                this.host = host.getBroadcastSocketAddress().toString();
+                this.host = host.getBroadcastSocketAddress();
             }
 
             public Event(EventType type, IInvokableInstance iInvokableInstance)
             {
                 this.type = type;
-                this.host = iInvokableInstance.broadcastAddress().toString();
+                this.host = iInvokableInstance.broadcastAddress();
             }
 
 

test/unit/org/apache/cassandra/gms/EndpointStateTest.java

@@ -18,6 +18,8 @@
 
 package org.apache.cassandra.gms;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.Collections;
 import java.util.EnumMap;
 import java.util.List;
@@ -31,6 +33,7 @@ import org.junit.Test;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -105,7 +108,7 @@ public class EndpointStateTest
     }
 
     @Test
-    public void testMultiThreadWriteConsistency() throws InterruptedException
+    public void testMultiThreadWriteConsistency() throws InterruptedException, UnknownHostException
     {
         for (int i = 0; i < 500; i++)
             innerTestMultiThreadWriteConsistency();
@@ -114,11 +117,11 @@ public class EndpointStateTest
     /**
      * Test that two threads can update the state map concurrently.
      */
-    private void innerTestMultiThreadWriteConsistency() throws InterruptedException
+    private void innerTestMultiThreadWriteConsistency() throws InterruptedException, UnknownHostException
     {
         final Token token = DatabaseDescriptor.getPartitioner().getRandomToken();
         final List<Token> tokens = Collections.singletonList(token);
-        final String ip = "127.0.0.1";
+        final InetAddress ip = InetAddress.getByAddress(null, new byte[] { 127, 0, 0, 1});
         final UUID hostId = UUID.randomUUID();
         final HeartBeatState hb = new HeartBeatState(0);
         final EndpointState state = new EndpointState(hb);

test/unit/org/apache/cassandra/locator/GossipingPropertyFileSnitchTest.java

@@ -61,6 +61,6 @@ public class GossipingPropertyFileSnitchTest
     public void testLoadConfig() throws Exception
     {
         final GossipingPropertyFileSnitch snitch = new GossipingPropertyFileSnitch();
-        checkEndpoint(snitch, FBUtilities.getBroadcastAddressAndPort().toString(), "DC1", "RAC1");
+        checkEndpoint(snitch, FBUtilities.getBroadcastAddressAndPort().getHostAddressAndPort(), "DC1", "RAC1");
     }
 }

test/unit/org/apache/cassandra/locator/InetAddressAndPortTest.java

@@ -119,12 +119,51 @@ public class InetAddressAndPortTest
     @Test
     public void toStringTest() throws Exception
     {
-        String ipv4 = "127.0.0.1:42";
-        String ipv6 = "[2001:db8:0:0:0:ff00:42:8329]:42";
-        assertEquals(ipv4, InetAddressAndPort.getByName(ipv4).toString());
-        assertEquals(ipv6, InetAddressAndPort.getByName(ipv6).toString());
+        InetAddress resolvedIPv4 = InetAddress.getByAddress("resolved4", new byte[] { 127, 0, 0, 1});
+        assertEquals("resolved4", resolvedIPv4.getHostName());
+        assertEquals("resolved4/127.0.0.1:42", InetAddressAndPort.getByAddressOverrideDefaults(resolvedIPv4, 42).toString());
+
+        InetAddress strangeIPv4 = InetAddress.getByAddress("strange/host/name4", new byte[] { 127, 0, 0, 1});
+        assertEquals("strange/host/name4", strangeIPv4.getHostName());
+        assertEquals("strange/host/name4/127.0.0.1:42", InetAddressAndPort.getByAddressOverrideDefaults(strangeIPv4, 42).toString());
+
+        InetAddress unresolvedIPv4 = InetAddress.getByAddress(null, new byte[] { 127, 0, 0, 1}); // don't call getHostName and resolve
+        assertEquals("/127.0.0.1:42", InetAddressAndPort.getByAddressOverrideDefaults(unresolvedIPv4, 42).toString());
+
+        InetAddress resolvedIPv6 = InetAddress.getByAddress("resolved6", new byte[] { 0x20, 0x01, 0xd, (byte) 0xb8, 0, 0, 0, 0, 0, 0, (byte) 0xff, 0, 0x00, 0x42, (byte) 0x83, 0x29});
+        assertEquals("resolved6", resolvedIPv6.getHostName());
+        assertEquals("resolved6/[2001:db8:0:0:0:ff00:42:8329]:42", InetAddressAndPort.getByAddressOverrideDefaults(resolvedIPv6, 42).toString());
+
+        InetAddress strangeIPv6 = InetAddress.getByAddress("strange/host/name6", new byte[] { 0x20, 0x01, 0xd, (byte) 0xb8, 0, 0, 0, 0, 0, 0, (byte) 0xff, 0, 0x00, 0x42, (byte) 0x83, 0x29});
+        assertEquals("strange/host/name6", strangeIPv6.getHostName());
+        assertEquals("strange/host/name6/[2001:db8:0:0:0:ff00:42:8329]:42", InetAddressAndPort.getByAddressOverrideDefaults(strangeIPv6, 42).toString());
+
+        InetAddress unresolvedIPv6 = InetAddress.getByAddress(null, new byte[] { 0x20, 0x01, 0xd, (byte) 0xb8, 0, 0, 0, 0, 0, 0, (byte) 0xff, 0, 0x00, 0x42, (byte) 0x83, 0x29});
+        assertEquals("/[2001:db8:0:0:0:ff00:42:8329]:42", InetAddressAndPort.getByAddressOverrideDefaults(unresolvedIPv6, 42).toString());
     }
 
+    @Test
+    public void getHostAddressAndPortTest() throws Exception
+    {
+        String ipv4withoutPort = "127.0.0.1";
+        String ipv6withoutPort = "2001:db8:0:0:0:ff00:42:8329";
+        String ipv4 = ipv4withoutPort + ":42";
+        String ipv6 = "[" + ipv6withoutPort + "]:42";
+        String ipv4forJMX = ipv4.replace("[", "_").replace("]", "_").replace(":","_");
+        String ipv6forJMX = ipv6.replace("[", "_").replace("]", "_").replace(":","_");
+
+        assertEquals(ipv4, InetAddressAndPort.getByName(ipv4).getHostAddressAndPort());
+        assertEquals(ipv6, InetAddressAndPort.getByName(ipv6).getHostAddressAndPort());
+
+        assertEquals(ipv4, InetAddressAndPort.getByName(ipv4).getHostAddress(true));
+        assertEquals(ipv6, InetAddressAndPort.getByName(ipv6).getHostAddress(true));
+
+        assertEquals(ipv4withoutPort, InetAddressAndPort.getByName(ipv4).getHostAddress(false));
+        assertEquals(ipv6withoutPort, InetAddressAndPort.getByName(ipv6).getHostAddress(false));
+
+        assertEquals(ipv4forJMX, InetAddressAndPort.getByName(ipv4).getHostAddressAndPortForJMX());
+        assertEquals(ipv6forJMX, InetAddressAndPort.getByName(ipv6).getHostAddressAndPortForJMX());
+    }
 
     private void shouldThrow(ThrowingRunnable t, Class expectedClass)
     {

test/unit/org/apache/cassandra/locator/PropertyFileSnitchTest.java

@@ -120,13 +120,13 @@ public class PropertyFileSnitchTest
             if (info.length == 2 && !line.startsWith("#") && !line.startsWith("default="))
             {
                 InetAddressAndPort address = InetAddressAndPort.getByName(info[0].replaceAll(Matcher.quoteReplacement("\\:"), ":"));
-                String replacement = replacements.get(address.toString());
+                String replacement = replacements.get(address.getHostAddressAndPort());
                 if (replacement != null)
                 {
                     if (!replacement.isEmpty()) // empty means remove this line
                         newLines.add(info[0] + '=' + replacement);
 
-                    replaced.add(address.toString());
+                    replaced.add(address.getHostAddressAndPort());
                 }
                 else
                 {
@@ -189,23 +189,23 @@ public class PropertyFileSnitchTest
     {
         final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.1");
         final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
-        checkEndpoint(snitch, host.toString(), "DC1", "RAC1");
+        checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "RAC1");
 
         try
         {
             setNodeLive(host);
 
             Files.copy(effectiveFile, backupFile);
-            replaceConfigFile(Collections.singletonMap(host.toString(), "DC1:RAC2"));
+            replaceConfigFile(Collections.singletonMap(host.getHostAddressAndPort(), "DC1:RAC2"));
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC1", "RAC1");
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "RAC1");
 
             setNodeShutdown(host);
-            replaceConfigFile(Collections.singletonMap(host.toString(), "DC1:RAC2"));
+            replaceConfigFile(Collections.singletonMap(host.getHostAddressAndPort(), "DC1:RAC2"));
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC1", "RAC2");
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "RAC2");
         }
         finally
         {
@@ -223,23 +223,23 @@ public class PropertyFileSnitchTest
     {
         final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.1");
         final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
-        checkEndpoint(snitch, host.toString(), "DC1", "RAC1");
+        checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "RAC1");
 
         try
         {
             setNodeLive(host);
 
             Files.copy(effectiveFile, backupFile);
-            replaceConfigFile(Collections.singletonMap(host.toString(), "DC2:RAC1"));
+            replaceConfigFile(Collections.singletonMap(host.getHostAddressAndPort(), "DC2:RAC1"));
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC1", "RAC1");
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "RAC1");
 
             setNodeShutdown(host);
-            replaceConfigFile(Collections.singletonMap(host.toString(), "DC2:RAC1"));
+            replaceConfigFile(Collections.singletonMap(host.getHostAddressAndPort(), "DC2:RAC1"));
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC2", "RAC1");
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC2", "RAC1");
         }
         finally
         {
@@ -258,23 +258,23 @@ public class PropertyFileSnitchTest
     {
         final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.9");
         final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
-        checkEndpoint(snitch, host.toString(), "DC1", "r1"); // default
+        checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "r1"); // default
 
         try
         {
             setNodeLive(host);
 
             Files.copy(effectiveFile, backupFile);
-            replaceConfigFile(Collections.singletonMap(host.toString(), "DC2:RAC2")); // add this line if not yet there
+            replaceConfigFile(Collections.singletonMap(host.getHostAddressAndPort(), "DC2:RAC2")); // add this line if not yet there
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC1", "r1"); // unchanged
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "r1"); // unchanged
 
             setNodeShutdown(host);
-            replaceConfigFile(Collections.singletonMap(host.toString(), "DC2:RAC2")); // add this line if not yet there
+            replaceConfigFile(Collections.singletonMap(host.getHostAddressAndPort(), "DC2:RAC2")); // add this line if not yet there
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC2", "RAC2"); // changed
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC2", "RAC2"); // changed
         }
         finally
         {
@@ -293,23 +293,23 @@ public class PropertyFileSnitchTest
     {
         final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.2");
         final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
-        checkEndpoint(snitch, host.toString(), "DC1", "RAC2");
+        checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "RAC2");
 
         try
         {
             setNodeLive(host);
 
             Files.copy(effectiveFile, backupFile);
-            replaceConfigFile(Collections.singletonMap(host.toString(), "")); // removes line if found
+            replaceConfigFile(Collections.singletonMap(host.getHostAddressAndPort(), "")); // removes line if found
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC1", "RAC2"); // unchanged
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "RAC2"); // unchanged
 
             setNodeShutdown(host);
-            replaceConfigFile(Collections.singletonMap(host.toString(), "")); // removes line if found
+            replaceConfigFile(Collections.singletonMap(host.getHostAddressAndPort(), "")); // removes line if found
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC1", "r1"); // default
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "r1"); // default
         }
         finally
         {
@@ -328,7 +328,7 @@ public class PropertyFileSnitchTest
     {
         final InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.9");
         final PropertyFileSnitch snitch = new PropertyFileSnitch(/*refreshPeriodInSeconds*/1);
-        checkEndpoint(snitch, host.toString(), "DC1", "r1"); // default
+        checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "r1"); // default
 
         try
         {
@@ -338,13 +338,13 @@ public class PropertyFileSnitchTest
             replaceConfigFile(Collections.singletonMap("default", "DC2:r2")); // change default
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC1", "r1"); // unchanged
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC1", "r1"); // unchanged
 
             setNodeShutdown(host);
             replaceConfigFile(Collections.singletonMap("default", "DC2:r2")); // change default again (refresh file update)
 
             Thread.sleep(1500);
-            checkEndpoint(snitch, host.toString(), "DC2", "r2"); // default updated
+            checkEndpoint(snitch, host.getHostAddressAndPort(), "DC2", "r2"); // default updated
         }
         finally
         {

test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java

@@ -216,7 +216,7 @@ public class ActiveRepairServiceTest
         }
 
         expected.remove(FBUtilities.getBroadcastAddressAndPort());
-        Collection<String> hosts = Arrays.asList(FBUtilities.getBroadcastAddressAndPort().toString(),expected.get(0).toString());
+        Collection<String> hosts = Arrays.asList(FBUtilities.getBroadcastAddressAndPort().getHostAddressAndPort(),expected.get(0).getHostAddressAndPort());
         Iterable<Range<Token>> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges();
 
         assertEquals(expected.get(0), ActiveRepairService.getNeighbors(KEYSPACE5, ranges,