Allow storage port to be configurable per node

Patch by Ariel Weisberg; Reviewed by Jason Brown for CASSANDRA-7544

CHANGES.txt

@@ -1,4 +1,5 @@
 4.0
+ * Allow storage port to be configurable per node (CASSANDRA-7544)
  * Make sub-range selection for non-frozen collections return null instead of empty (CASSANDRA-14182)
  * BloomFilter serialization format should not change byte ordering (CASSANDRA-9067)
  * Remove unused on-heap BloomFilter implementation (CASSANDRA-14152)

NEWS.txt

@@ -39,7 +39,7 @@ New features
      This property can be modified at runtime through both JMX and the new `setconcurrentviewbuilders`
      and `getconcurrentviewbuilders` nodetool commands. See CASSANDRA-12245 for more details.
    - There is now a binary full query log based on Chronicle Queue that can be controlled using
-     nodetool enablefullquerylog, disablefullquerylog, and resetfullquerylog. The log 
+     nodetool enablefullquerylog, disablefullquerylog, and resetfullquerylog. The log
      contains all queries invoked, approximate time they were invoked, any parameters necessary
      to bind wildcard values, and all query options. A human readable version of the log can be
      dumped or tailed using the new bin/fqltool utility. The full query log is designed to be safe
@@ -103,6 +103,22 @@ Upgrading
 	  but blocks for up to a configurable number of milliseconds between disk flushes.
 	- nodetool clearsnapshot now required the --all flag to remove all snapshots.
 	  Previous behavior would delete all snapshots by default.
+    - Nodes are now identified by a combination of IP, and storage port.
+      Existing JMX APIs, nodetool, and system tables continue to work
+      and accept/return just an IP, but there is a new
+      version of each that works with the full unambiguous identifier.
+      You should prefer these over the deprecated ambiguous versions that only
+      work with an IP. This was done to support multiple instances per IP.
+      Additionally we are moving to only using a single port for encrypted and
+      unencrypted traffic and if you want multiple instances per IP you must
+      first switch encrypted traffic to the storage port and not a separate
+      encrypted port. If you want to use multiple instances per IP
+      with SSL you will need to use StartTLS on storage_port and set
+      outgoing_encrypted_port_source to gossip outbound connections
+      know what port to connect to for each instance. Before changing
+      storage port or native port at nodes you must first upgrade the entire cluster
+      and clients to 4.0 so they can handle the port not being consistent across
+      the cluster.
 
 Materialized Views
 -------------------

bin/cqlsh.py

@@ -455,7 +455,8 @@ class Shell(cmd.Cmd):
                  single_statement=None,
                  request_timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS,
                  protocol_version=None,
-                 connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS):
+                 connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS,
+                 allow_server_port_discovery=False):
         cmd.Cmd.__init__(self, completekey=completekey)
         self.hostname = hostname
         self.port = port
@@ -470,6 +471,7 @@ class Shell(cmd.Cmd):
         self.tracing_enabled = tracing_enabled
         self.page_size = self.default_page_size
         self.expand_enabled = expand_enabled
+        self.allow_server_port_discovery = allow_server_port_discovery
         if use_conn:
             self.conn = use_conn
         else:
@@ -482,6 +484,7 @@ class Shell(cmd.Cmd):
                                 load_balancing_policy=WhiteListRoundRobinPolicy([self.hostname]),
                                 control_connection_timeout=connect_timeout,
                                 connect_timeout=connect_timeout,
+                                allow_server_port_discovery=allow_server_port_discovery,
                                 **kwargs)
         self.owns_connection = not use_conn
 
@@ -1768,7 +1771,8 @@ class Shell(cmd.Cmd):
                          display_timezone=self.display_timezone,
                          max_trace_wait=self.max_trace_wait, ssl=self.ssl,
                          request_timeout=self.session.default_timeout,
-                         connect_timeout=self.conn.connect_timeout)
+                         connect_timeout=self.conn.connect_timeout,
+                         allow_server_port_discovery=self.allow_server_port_discovery)
         subshell.cmdloop()
         f.close()
 
@@ -2252,6 +2256,7 @@ def read_options(cmdlineargs, environment):
     optvalues.connect_timeout = option_with_default(configs.getint, 'connection', 'timeout', DEFAULT_CONNECT_TIMEOUT_SECONDS)
     optvalues.request_timeout = option_with_default(configs.getint, 'connection', 'request_timeout', DEFAULT_REQUEST_TIMEOUT_SECONDS)
     optvalues.execute = None
+    optvalues.allow_server_port_discovery = option_with_default(configs.getboolean, 'connection', 'allow_server_port_discovery', 'False')
 
     (options, arguments) = parser.parse_args(cmdlineargs, values=optvalues)
 
@@ -2415,7 +2420,8 @@ def main(options, hostname, port):
                       single_statement=options.execute,
                       request_timeout=options.request_timeout,
                       connect_timeout=options.connect_timeout,
-                      encoding=options.encoding)
+                      encoding=options.encoding,
+                      allow_server_port_discovery=options.allow_server_port_discovery)
     except KeyboardInterrupt:
         sys.exit('Connection aborted.')
     except CQL_ERRORS, e:

build.xml

@@ -306,7 +306,7 @@
       <!-- define the remote repositories we use -->
       <artifact:remoteRepository id="central"   url="${artifact.remoteRepository.central}"/>
       <artifact:remoteRepository id="apache"    url="${artifact.remoteRepository.apache}"/>
-      
+
       <macrodef name="install">
         <attribute name="pomFile"/>
         <attribute name="file"/>
@@ -437,7 +437,7 @@
           <dependency groupId="com.google.code.findbugs" artifactId="jsr305" version="2.0.2" />
           <dependency groupId="com.clearspring.analytics" artifactId="stream" version="2.5.2" />
 	  <!-- UPDATE AND UNCOMMENT ON THE DRIVER RELEASE, BEFORE 4.0 RELEASE
-          <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" version="3.0.1" classifier="shaded">
+          <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" version="4.0.0-SNAPSHOT" classifier="shaded">
             <exclusion groupId="io.netty" artifactId="netty-buffer"/>
             <exclusion groupId="io.netty" artifactId="netty-codec"/>
             <exclusion groupId="io.netty" artifactId="netty-handler"/>
@@ -522,7 +522,7 @@
       	<dependency groupId="org.apache.hadoop" artifactId="hadoop-minicluster"/>
       	<dependency groupId="com.google.code.findbugs" artifactId="jsr305"/>
         <dependency groupId="org.antlr" artifactId="antlr"/>
-	<!-- UPDATE AND UNCOMMENT ON THE DRIVER RELEASE, BEFORE 4.0 RELEASE	
+	<!-- UPDATE AND UNCOMMENT ON THE DRIVER RELEASE, BEFORE 4.0 RELEASE
         <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded"/>
 	-->
         <dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj"/>
@@ -543,7 +543,7 @@
         <dependency groupId="junit" artifactId="junit"/>
 	<!-- UPDATE AND UNCOMMENT ON THE DRIVER RELEASE, BEFORE 4.0 RELEASE
         <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded"/>
-	-->     
+	-->
         <dependency groupId="io.netty" artifactId="netty-all"/>
         <dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj"/>
         <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core"/>
@@ -624,7 +624,7 @@
           <exclusion groupId="io.netty" artifactId="netty-transport"/>
         </dependency>
 	-->
-	
+
         <!-- don't need jna to run, but nice to have -->
         <dependency groupId="net.java.dev.jna" artifactId="jna"/>
 

conf/cassandra.yaml

@@ -416,7 +416,7 @@ seed_provider:
       parameters:
           # seeds is actually a comma-delimited list of addresses.
           # Ex: "<ip1>,<ip2>,<ip3>"
-          - seeds: "127.0.0.1"
+          - seeds: "127.0.0.1:7000"
 
 # For workloads with more data than can fit in memory, Cassandra's
 # bottleneck will be reads that need to fetch data from
@@ -945,6 +945,13 @@ dynamic_snitch_badness_threshold: 0.1
 # the keystore and truststore.  For instructions on generating these files, see:
 # http://download.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore
 #
+# If you are taking advantage of StartTLS outbound connections will have the issue that they can't know
+# what encrypted port to connect to in a foolproof way. outgoing_encrypted_port_source deals with this confusion
+# by allowing you to specify how you want a node to pick an outgoing port for intra-cluster connections.
+# Valid values are "gossip" and "yaml". Gossip will always connect to the storage port for a node that is
+# published via a gossip which is always going to be the plain storage port. "yaml" will always select
+# the port configured as ssl_storage_port on THIS node. If you want to use SSL and have different storage
+# ports across the cluster you must select "gossip" and use StartTLS on storage_port.
 server_encryption_options:
     # set to true for allowing secure incoming connections
     enabled: false

src/java/org/apache/cassandra/batchlog/BatchlogManager.java

@@ -19,7 +19,6 @@ package org.apache.cassandra.batchlog;
 
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.*;
@@ -34,6 +33,7 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.*;
@@ -250,7 +250,7 @@ public class BatchlogManager implements BatchlogManagerMBean
         int positionInPage = 0;
         ArrayList<ReplayingBatch> unfinishedBatches = new ArrayList<>(pageSize);
 
-        Set<InetAddress> hintedNodes = new HashSet<>();
+        Set<InetAddressAndPort> hintedNodes = new HashSet<>();
         Set<UUID> replayedBatches = new HashSet<>();
 
         // Sending out batches for replay without waiting for them, so that one stuck batch doesn't affect others
@@ -295,7 +295,7 @@ public class BatchlogManager implements BatchlogManagerMBean
         replayedBatches.forEach(BatchlogManager::remove);
     }
 
-    private void finishAndClearBatches(ArrayList<ReplayingBatch> batches, Set<InetAddress> hintedNodes, Set<UUID> replayedBatches)
+    private void finishAndClearBatches(ArrayList<ReplayingBatch> batches, Set<InetAddressAndPort> hintedNodes, Set<UUID> replayedBatches)
     {
         // schedule hints for timed out deliveries
         for (ReplayingBatch batch : batches)
@@ -330,7 +330,7 @@ public class BatchlogManager implements BatchlogManagerMBean
             this.replayedBytes = addMutations(version, serializedMutations);
         }
 
-        public int replay(RateLimiter rateLimiter, Set<InetAddress> hintedNodes) throws IOException
+        public int replay(RateLimiter rateLimiter, Set<InetAddressAndPort> hintedNodes) throws IOException
         {
             logger.trace("Replaying batch {}", id);
 
@@ -348,7 +348,7 @@ public class BatchlogManager implements BatchlogManagerMBean
             return replayHandlers.size();
         }
 
-        public void finish(Set<InetAddress> hintedNodes)
+        public void finish(Set<InetAddressAndPort> hintedNodes)
         {
             for (int i = 0; i < replayHandlers.size(); i++)
             {
@@ -396,7 +396,7 @@ public class BatchlogManager implements BatchlogManagerMBean
                 mutations.add(mutation);
         }
 
-        private void writeHintsForUndeliveredEndpoints(int startFrom, Set<InetAddress> hintedNodes)
+        private void writeHintsForUndeliveredEndpoints(int startFrom, Set<InetAddressAndPort> hintedNodes)
         {
             int gcgs = gcgs(mutations);
 
@@ -420,7 +420,7 @@ public class BatchlogManager implements BatchlogManagerMBean
 
         private static List<ReplayWriteResponseHandler<Mutation>> sendReplays(List<Mutation> mutations,
                                                                               long writtenAt,
-                                                                              Set<InetAddress> hintedNodes)
+                                                                              Set<InetAddressAndPort> hintedNodes)
         {
             List<ReplayWriteResponseHandler<Mutation>> handlers = new ArrayList<>(mutations.size());
             for (Mutation mutation : mutations)
@@ -440,15 +440,15 @@ public class BatchlogManager implements BatchlogManagerMBean
          */
         private static ReplayWriteResponseHandler<Mutation> sendSingleReplayMutation(final Mutation mutation,
                                                                                      long writtenAt,
-                                                                                     Set<InetAddress> hintedNodes)
+                                                                                     Set<InetAddressAndPort> hintedNodes)
         {
-            Set<InetAddress> liveEndpoints = new HashSet<>();
+            Set<InetAddressAndPort> liveEndpoints = new HashSet<>();
             String ks = mutation.getKeyspaceName();
             Token tk = mutation.key().getToken();
 
-            for (InetAddress endpoint : StorageService.instance.getNaturalAndPendingEndpoints(ks, tk))
+            for (InetAddressAndPort endpoint : StorageService.instance.getNaturalAndPendingEndpoints(ks, tk))
             {
-                if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+                if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
                 {
                     mutation.apply();
                 }
@@ -469,7 +469,7 @@ public class BatchlogManager implements BatchlogManagerMBean
 
             ReplayWriteResponseHandler<Mutation> handler = new ReplayWriteResponseHandler<>(liveEndpoints, System.nanoTime());
             MessageOut<Mutation> message = mutation.createMessage();
-            for (InetAddress endpoint : liveEndpoints)
+            for (InetAddressAndPort endpoint : liveEndpoints)
                 MessagingService.instance().sendRR(message, endpoint, handler, false);
             return handler;
         }
@@ -488,11 +488,11 @@ public class BatchlogManager implements BatchlogManagerMBean
          */
         private static class ReplayWriteResponseHandler<T> extends WriteResponseHandler<T>
         {
-            private final Set<InetAddress> undelivered = Collections.newSetFromMap(new ConcurrentHashMap<>());
+            private final Set<InetAddressAndPort> undelivered = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
-            ReplayWriteResponseHandler(Collection<InetAddress> writeEndpoints, long queryStartNanoTime)
+            ReplayWriteResponseHandler(Collection<InetAddressAndPort> writeEndpoints, long queryStartNanoTime)
             {
-                super(writeEndpoints, Collections.<InetAddress>emptySet(), null, null, null, WriteType.UNLOGGED_BATCH, queryStartNanoTime);
+                super(writeEndpoints, Collections.<InetAddressAndPort>emptySet(), null, null, null, WriteType.UNLOGGED_BATCH, queryStartNanoTime);
                 undelivered.addAll(writeEndpoints);
             }
 
@@ -505,7 +505,7 @@ public class BatchlogManager implements BatchlogManagerMBean
             @Override
             public void response(MessageIn<T> m)
             {
-                boolean removed = undelivered.remove(m == null ? FBUtilities.getBroadcastAddress() : m.from);
+                boolean removed = undelivered.remove(m == null ? FBUtilities.getBroadcastAddressAndPort() : m.from);
                 assert removed;
                 super.response(m);
             }
@@ -515,9 +515,9 @@ public class BatchlogManager implements BatchlogManagerMBean
     public static class EndpointFilter
     {
         private final String localRack;
-        private final Multimap<String, InetAddress> endpoints;
+        private final Multimap<String, InetAddressAndPort> endpoints;
 
-        public EndpointFilter(String localRack, Multimap<String, InetAddress> endpoints)
+        public EndpointFilter(String localRack, Multimap<String, InetAddressAndPort> endpoints)
         {
             this.localRack = localRack;
             this.endpoints = endpoints;
@@ -526,15 +526,15 @@ public class BatchlogManager implements BatchlogManagerMBean
         /**
          * @return list of candidates for batchlog hosting. If possible these will be two nodes from different racks.
          */
-        public Collection<InetAddress> filter()
+        public Collection<InetAddressAndPort> filter()
         {
             // special case for single-node data centers
             if (endpoints.values().size() == 1)
                 return endpoints.values();
 
             // strip out dead endpoints and localhost
-            ListMultimap<String, InetAddress> validated = ArrayListMultimap.create();
-            for (Map.Entry<String, InetAddress> entry : endpoints.entries())
+            ListMultimap<String, InetAddressAndPort> validated = ArrayListMultimap.create();
+            for (Map.Entry<String, InetAddressAndPort> entry : endpoints.entries())
                 if (isValid(entry.getValue()))
                     validated.put(entry.getKey(), entry.getValue());
 
@@ -554,7 +554,7 @@ public class BatchlogManager implements BatchlogManagerMBean
                  * pick two random nodes from there; we are guaranteed to have at least two nodes in the single remaining rack
                  * because of the preceding if block.
                  */
-                List<InetAddress> otherRack = Lists.newArrayList(validated.values());
+                List<InetAddressAndPort> otherRack = Lists.newArrayList(validated.values());
                 shuffle(otherRack);
                 return otherRack.subList(0, 2);
             }
@@ -572,10 +572,10 @@ public class BatchlogManager implements BatchlogManagerMBean
             }
 
             // grab a random member of up to two racks
-            List<InetAddress> result = new ArrayList<>(2);
+            List<InetAddressAndPort> result = new ArrayList<>(2);
             for (String rack : Iterables.limit(racks, 2))
             {
-                List<InetAddress> rackMembers = validated.get(rack);
+                List<InetAddressAndPort> rackMembers = validated.get(rack);
                 result.add(rackMembers.get(getRandomInt(rackMembers.size())));
             }
 
@@ -583,9 +583,9 @@ public class BatchlogManager implements BatchlogManagerMBean
         }
 
         @VisibleForTesting
-        protected boolean isValid(InetAddress input)
+        protected boolean isValid(InetAddressAndPort input)
         {
-            return !input.equals(FBUtilities.getBroadcastAddress()) && FailureDetector.instance.isAlive(input);
+            return !input.equals(FBUtilities.getBroadcastAddressAndPort()) && FailureDetector.instance.isAlive(input);
         }
 
         @VisibleForTesting

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -55,6 +55,7 @@ import org.apache.cassandra.io.util.SsdDiskOptimizationStrategy;
 import org.apache.cassandra.locator.DynamicEndpointSnitch;
 import org.apache.cassandra.locator.EndpointSnitchInfo;
 import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.SeedProvider;
 import org.apache.cassandra.net.BackPressureStrategy;
 import org.apache.cassandra.net.RateBasedBackPressure;
@@ -110,7 +111,7 @@ public class DatabaseDescriptor
     private static long indexSummaryCapacityInMB;
 
     private static String localDC;
-    private static Comparator<InetAddress> localComparator;
+    private static Comparator<InetAddressAndPort> localComparator;
     private static EncryptionContext encryptionContext;
     private static boolean hasLoggedConfig;
 
@@ -307,6 +308,7 @@ public class DatabaseDescriptor
 
     private static void applyAll() throws ConfigurationException
     {
+        //InetAddressAndPort cares that applySimpleConfig runs first
         applySimpleConfig();
 
         applyPartitioner();
@@ -324,6 +326,9 @@ public class DatabaseDescriptor
 
     private static void applySimpleConfig()
     {
+        //Doing this first before all other things in case other pieces of config want to construct
+        //InetAddressAndPort and get the right defaults
+        InetAddressAndPort.initializeDefaultPort(getStoragePort());
 
         if (conf.commitlog_sync == null)
         {
@@ -827,7 +832,7 @@ public class DatabaseDescriptor
         }
         else
         {
-            rpcAddress = FBUtilities.getLocalAddress();
+            rpcAddress = FBUtilities.getJustLocalAddress();
         }
 
         /* RPC address to broadcast */
@@ -956,10 +961,10 @@ public class DatabaseDescriptor
         snitch = createEndpointSnitch(conf.dynamic_snitch, conf.endpoint_snitch);
         EndpointSnitchInfo.create();
 
-        localDC = snitch.getDatacenter(FBUtilities.getBroadcastAddress());
-        localComparator = new Comparator<InetAddress>()
+        localDC = snitch.getDatacenter(FBUtilities.getBroadcastAddressAndPort());
+        localComparator = new Comparator<InetAddressAndPort>()
         {
-            public int compare(InetAddress endpoint1, InetAddress endpoint2)
+            public int compare(InetAddressAndPort endpoint1, InetAddressAndPort endpoint2)
             {
                 boolean local1 = localDC.equals(snitch.getDatacenter(endpoint1));
                 boolean local2 = localDC.equals(snitch.getDatacenter(endpoint2));
@@ -1319,14 +1324,14 @@ public class DatabaseDescriptor
         return conf.num_tokens;
     }
 
-    public static InetAddress getReplaceAddress()
+    public static InetAddressAndPort getReplaceAddress()
     {
         try
         {
             if (System.getProperty(Config.PROPERTY_PREFIX + "replace_address", null) != null)
-                return InetAddress.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address", null));
+                return InetAddressAndPort.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address", null));
             else if (System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null) != null)
-                return InetAddress.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null));
+                return InetAddressAndPort.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null));
             return null;
         }
         catch (UnknownHostException e)
@@ -1651,9 +1656,9 @@ public class DatabaseDescriptor
         return conf.saved_caches_directory;
     }
 
-    public static Set<InetAddress> getSeeds()
+    public static Set<InetAddressAndPort> getSeeds()
     {
-        return ImmutableSet.<InetAddress>builder().addAll(seedProvider.getSeeds()).build();
+        return ImmutableSet.<InetAddressAndPort>builder().addAll(seedProvider.getSeeds()).build();
     }
 
     public static InetAddress getListenAddress()
@@ -2206,7 +2211,7 @@ public class DatabaseDescriptor
         return localDC;
     }
 
-    public static Comparator<InetAddress> getLocalComparator()
+    public static Comparator<InetAddressAndPort> getLocalComparator()
     {
         return localComparator;
     }

src/java/org/apache/cassandra/db/ConsistencyLevel.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.db;
 
-import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -28,6 +27,7 @@ import com.google.common.collect.Iterables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.service.ReadRepairDecision;
@@ -145,21 +145,21 @@ public enum ConsistencyLevel
         return isDCLocal;
     }
 
-    public boolean isLocal(InetAddress endpoint)
+    public boolean isLocal(InetAddressAndPort endpoint)
     {
         return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint));
     }
 
-    public int countLocalEndpoints(Iterable<InetAddress> liveEndpoints)
+    public int countLocalEndpoints(Iterable<InetAddressAndPort> liveEndpoints)
     {
         int count = 0;
-        for (InetAddress endpoint : liveEndpoints)
+        for (InetAddressAndPort endpoint : liveEndpoints)
             if (isLocal(endpoint))
                 count++;
         return count;
     }
 
-    private Map<String, Integer> countPerDCEndpoints(Keyspace keyspace, Iterable<InetAddress> liveEndpoints)
+    private Map<String, Integer> countPerDCEndpoints(Keyspace keyspace, Iterable<InetAddressAndPort> liveEndpoints)
     {
         NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy();
 
@@ -167,7 +167,7 @@ public enum ConsistencyLevel
         for (String dc: strategy.getDatacenters())
             dcEndpoints.put(dc, 0);
 
-        for (InetAddress endpoint : liveEndpoints)
+        for (InetAddressAndPort endpoint : liveEndpoints)
         {
             String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint);
             dcEndpoints.put(dc, dcEndpoints.get(dc) + 1);
@@ -175,12 +175,12 @@ public enum ConsistencyLevel
         return dcEndpoints;
     }
 
-    public List<InetAddress> filterForQuery(Keyspace keyspace, List<InetAddress> liveEndpoints)
+    public List<InetAddressAndPort> filterForQuery(Keyspace keyspace, List<InetAddressAndPort> liveEndpoints)
     {
         return filterForQuery(keyspace, liveEndpoints, ReadRepairDecision.NONE);
     }
 
-    public List<InetAddress> filterForQuery(Keyspace keyspace, List<InetAddress> liveEndpoints, ReadRepairDecision readRepair)
+    public List<InetAddressAndPort> filterForQuery(Keyspace keyspace, List<InetAddressAndPort> liveEndpoints, ReadRepairDecision readRepair)
     {
         /*
          * If we are doing an each quorum query, we have to make sure that the endpoints we select
@@ -206,9 +206,9 @@ public enum ConsistencyLevel
             case GLOBAL:
                 return liveEndpoints;
             case DC_LOCAL:
-                List<InetAddress> local = new ArrayList<InetAddress>();
-                List<InetAddress> other = new ArrayList<InetAddress>();
-                for (InetAddress add : liveEndpoints)
+                List<InetAddressAndPort> local = new ArrayList<>();
+                List<InetAddressAndPort> other = new ArrayList<>();
+                for (InetAddressAndPort add : liveEndpoints)
                 {
                     if (isLocal(add))
                         local.add(add);
@@ -225,7 +225,7 @@ public enum ConsistencyLevel
         }
     }
 
-    private List<InetAddress> filterForEachQuorum(Keyspace keyspace, List<InetAddress> liveEndpoints, ReadRepairDecision readRepair)
+    private List<InetAddressAndPort> filterForEachQuorum(Keyspace keyspace, List<InetAddressAndPort> liveEndpoints, ReadRepairDecision readRepair)
     {
         NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy();
 
@@ -233,20 +233,20 @@ public enum ConsistencyLevel
         if (readRepair == ReadRepairDecision.GLOBAL)
             return liveEndpoints;
 
-        Map<String, List<InetAddress>> dcsEndpoints = new HashMap<>();
+        Map<String, List<InetAddressAndPort>> dcsEndpoints = new HashMap<>();
         for (String dc: strategy.getDatacenters())
             dcsEndpoints.put(dc, new ArrayList<>());
 
-        for (InetAddress add : liveEndpoints)
+        for (InetAddressAndPort add : liveEndpoints)
         {
             String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(add);
             dcsEndpoints.get(dc).add(add);
         }
 
-        List<InetAddress> waitSet = new ArrayList<>();
-        for (Map.Entry<String, List<InetAddress>> dcEndpoints : dcsEndpoints.entrySet())
+        List<InetAddressAndPort> waitSet = new ArrayList<>();
+        for (Map.Entry<String, List<InetAddressAndPort>> dcEndpoints : dcsEndpoints.entrySet())
         {
-            List<InetAddress> dcEndpoint = dcEndpoints.getValue();
+            List<InetAddressAndPort> dcEndpoint = dcEndpoints.getValue();
             if (readRepair == ReadRepairDecision.DC_LOCAL && dcEndpoints.getKey().equals(DatabaseDescriptor.getLocalDataCenter()))
                 waitSet.addAll(dcEndpoint);
             else
@@ -256,7 +256,7 @@ public enum ConsistencyLevel
         return waitSet;
     }
 
-    public boolean isSufficientLiveNodes(Keyspace keyspace, Iterable<InetAddress> liveEndpoints)
+    public boolean isSufficientLiveNodes(Keyspace keyspace, Iterable<InetAddressAndPort> liveEndpoints)
     {
         switch (this)
         {
@@ -283,7 +283,7 @@ public enum ConsistencyLevel
         }
     }
 
-    public void assureSufficientLiveNodes(Keyspace keyspace, Iterable<InetAddress> liveEndpoints) throws UnavailableException
+    public void assureSufficientLiveNodes(Keyspace keyspace, Iterable<InetAddressAndPort> liveEndpoints) throws UnavailableException
     {
         int blockFor = blockFor(keyspace);
         switch (this)
@@ -302,7 +302,7 @@ public enum ConsistencyLevel
                     if (logger.isTraceEnabled())
                     {
                         StringBuilder builder = new StringBuilder("Local replicas [");
-                        for (InetAddress endpoint : liveEndpoints)
+                        for (InetAddressAndPort endpoint : liveEndpoints)
                         {
                             if (isLocal(endpoint))
                                 builder.append(endpoint).append(",");

src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java

@@ -37,7 +37,7 @@ public class CounterMutationVerbHandler implements IVerbHandler<CounterMutation>
         final CounterMutation cm = message.payload;
         logger.trace("Applying forwarded {}", cm);
 
-        String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
+        String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort());
         // We should not wait for the result of the write in this thread,
         // otherwise we could have a distributed deadlock between replicas
         // running this VerbHandler (see #4578).

src/java/org/apache/cassandra/db/DiskBoundaryManager.java

@@ -80,14 +80,14 @@ public class DiskBoundaryManager
                 && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally
             {
                 PendingRangeCalculatorService.instance.blockUntilFinished();
-                localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddress());
+                localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddressAndPort());
             }
             else
             {
                 // Reason we use use the future settled TMD is that if we decommission a node, we want to stream
                 // from that node to the correct location on disk, if we didn't, we would put new files in the wrong places.
                 // We do this to minimize the amount of data we need to move in rebalancedisks once everything settled
-                localRanges = cfs.keyspace.getReplicationStrategy().getAddressRanges(tmd.cloneAfterAllSettled()).get(FBUtilities.getBroadcastAddress());
+                localRanges = cfs.keyspace.getReplicationStrategy().getAddressRanges(tmd.cloneAfterAllSettled()).get(FBUtilities.getBroadcastAddressAndPort());
             }
             logger.debug("Got local ranges {} (ringVersion = {})", localRanges, ringVersion);
         }

src/java/org/apache/cassandra/db/Keyspace.java

@@ -352,6 +352,7 @@ public class Keyspace
 
     private void createReplicationStrategy(KeyspaceMetadata ksm)
     {
+        logger.info("Creating replication strategy " + ksm .name + " params " + ksm.params);
         replicationStrategy = AbstractReplicationStrategy.createReplicationStrategy(ksm.name,
                                                                                     ksm.params.replication.klass,
                                                                                     StorageService.instance.getTokenMetadata(),

src/java/org/apache/cassandra/db/Mutation.java

@@ -43,9 +43,6 @@ public class Mutation implements IMutation
 {
     public static final MutationSerializer serializer = new MutationSerializer();
 
-    public static final String FORWARD_TO = "FWD_TO";
-    public static final String FORWARD_FROM = "FWD_FRM";
-
     // todo this is redundant
     // when we remove it, also restore SerializationsTest.testMutationRead to not regenerate new Mutations each test
     private final String keyspaceName;

src/java/org/apache/cassandra/db/MutationVerbHandler.java

@@ -17,18 +17,17 @@
  */
 package org.apache.cassandra.db;
 
-import java.io.DataInputStream;
 import java.io.IOException;
-import java.net.InetAddress;
+import java.util.Iterator;
 
 import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.io.util.FastByteArrayInputStream;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.*;
 import org.apache.cassandra.tracing.Tracing;
 
 public class MutationVerbHandler implements IVerbHandler<Mutation>
 {
-    private void reply(int id, InetAddress replyTo)
+    private void reply(int id, InetAddressAndPort replyTo)
     {
         Tracing.trace("Enqueuing response to {}", replyTo);
         MessagingService.instance().sendReply(WriteResponse.createMessage(), id, replyTo);
@@ -42,18 +41,19 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
     public void doVerb(MessageIn<Mutation> message, int id)  throws IOException
     {
         // Check if there were any forwarding headers in this message
-        byte[] from = message.parameters.get(Mutation.FORWARD_FROM);
-        InetAddress replyTo;
+        InetAddressAndPort from = (InetAddressAndPort)message.parameters.get(ParameterType.FORWARD_FROM);
+        InetAddressAndPort replyTo;
         if (from == null)
         {
             replyTo = message.from;
-            byte[] forwardBytes = message.parameters.get(Mutation.FORWARD_TO);
-            if (forwardBytes != null)
-                forwardToLocalNodes(message.payload, message.verb, forwardBytes, message.from);
+            ForwardToContainer forwardTo = (ForwardToContainer)message.parameters.get(ParameterType.FORWARD_TO);
+            if (forwardTo != null)
+                forwardToLocalNodes(message.payload, message.verb, forwardTo, message.from);
         }
         else
         {
-            replyTo = InetAddress.getByAddress(from);
+
+            replyTo = from;
         }
 
         try
@@ -69,22 +69,17 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
         }
     }
 
-    private static void forwardToLocalNodes(Mutation mutation, MessagingService.Verb verb, byte[] forwardBytes, InetAddress from) throws IOException
+    private static void forwardToLocalNodes(Mutation mutation, MessagingService.Verb verb, ForwardToContainer forwardTo, InetAddressAndPort from) throws IOException
     {
-        try (DataInputStream in = new DataInputStream(new FastByteArrayInputStream(forwardBytes)))
+        // tell the recipients who to send their ack to
+        MessageOut<Mutation> message = new MessageOut<>(verb, mutation, Mutation.serializer).withParameter(ParameterType.FORWARD_FROM, from);
+        Iterator<InetAddressAndPort> iterator = forwardTo.targets.iterator();
+        // Send a message to each of the addresses on our Forward List
+        for (int i = 0; i < forwardTo.targets.size(); i++)
         {
-            int size = in.readInt();
-
-            // tell the recipients who to send their ack to
-            MessageOut<Mutation> message = new MessageOut<>(verb, mutation, Mutation.serializer).withParameter(Mutation.FORWARD_FROM, from.getAddress());
-            // Send a message to each of the addresses on our Forward List
-            for (int i = 0; i < size; i++)
-            {
-                InetAddress address = CompactEndpointSerializationHelper.deserialize(in);
-                int id = in.readInt();
-                Tracing.trace("Enqueuing forwarded write to {}", address);
-                MessagingService.instance().sendOneWay(message, id, address);
-            }
+            InetAddressAndPort address = iterator.next();
+            Tracing.trace("Enqueuing forwarded write to {}", address);
+            MessagingService.instance().sendOneWay(message, forwardTo.messageIds[i], address);
         }
     }
 }

src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java

@@ -60,7 +60,7 @@ public class SizeEstimatesRecorder extends SchemaChangeListener implements Runna
     public void run()
     {
         TokenMetadata metadata = StorageService.instance.getTokenMetadata().cloneOnlyTokenMap();
-        if (!metadata.isMember(FBUtilities.getBroadcastAddress()))
+        if (!metadata.isMember(FBUtilities.getBroadcastAddressAndPort()))
         {
             logger.debug("Node is not part of the ring; not recording size estimates");
             return;
@@ -71,7 +71,7 @@ public class SizeEstimatesRecorder extends SchemaChangeListener implements Runna
         for (Keyspace keyspace : Keyspace.nonLocalStrategy())
         {
             Collection<Range<Token>> localRanges = StorageService.instance.getPrimaryRangesForEndpoint(keyspace.getName(),
-                    FBUtilities.getBroadcastAddress());
+                    FBUtilities.getBroadcastAddressAndPort());
             for (ColumnFamilyStore table : keyspace.getColumnFamilyStores())
             {
                 long start = System.nanoTime();

src/java/org/apache/cassandra/db/SystemKeyspace.java

@@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.SetMultimap;
 import com.google.common.collect.Sets;
 import com.google.common.io.ByteStreams;
+import com.google.common.util.concurrent.ListenableFuture;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -52,6 +53,7 @@ import org.apache.cassandra.dht.*;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.util.*;
 import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.RestorableMeter;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.*;
@@ -91,47 +93,52 @@ public final class SystemKeyspace
     public static final String PAXOS = "paxos";
     public static final String BUILT_INDEXES = "IndexInfo";
     public static final String LOCAL = "local";
-    public static final String PEERS = "peers";
-    public static final String PEER_EVENTS = "peer_events";
+    public static final String PEERS_V2 = "peers_v2";
+    public static final String PEER_EVENTS_V2 = "peer_events_v2";
     public static final String RANGE_XFERS = "range_xfers";
     public static final String COMPACTION_HISTORY = "compaction_history";
     public static final String SSTABLE_ACTIVITY = "sstable_activity";
     public static final String SIZE_ESTIMATES = "size_estimates";
     public static final String AVAILABLE_RANGES = "available_ranges";
     public static final String TRANSFERRED_RANGES = "transferred_ranges";
+    public static final String TRANSFERRED_RANGES_V2 = "transferred_ranges_v2";
     public static final String VIEW_BUILDS_IN_PROGRESS = "view_builds_in_progress";
     public static final String BUILT_VIEWS = "built_views";
     public static final String PREPARED_STATEMENTS = "prepared_statements";
     public static final String REPAIRS = "repairs";
 
+    @Deprecated public static final String LEGACY_PEERS = "peers";
+    @Deprecated public static final String LEGACY_PEER_EVENTS = "peer_events";
+    @Deprecated public static final String LEGACY_TRANSFERRED_RANGES = "transferred_ranges";
+
     public static final TableMetadata Batches =
         parse(BATCHES,
-              "batches awaiting replay",
-              "CREATE TABLE %s ("
-              + "id timeuuid,"
-              + "mutations list<blob>,"
-              + "version int,"
-              + "PRIMARY KEY ((id)))")
-              .partitioner(new LocalPartitioner(TimeUUIDType.instance))
-              .compaction(CompactionParams.scts(singletonMap("min_threshold", "2")))
-              .build();
+                "batches awaiting replay",
+                "CREATE TABLE %s ("
+                + "id timeuuid,"
+                + "mutations list<blob>,"
+                + "version int,"
+                + "PRIMARY KEY ((id)))")
+                .partitioner(new LocalPartitioner(TimeUUIDType.instance))
+                .compaction(CompactionParams.scts(singletonMap("min_threshold", "2")))
+                .build();
 
     private static final TableMetadata Paxos =
         parse(PAXOS,
-              "in-progress paxos proposals",
-              "CREATE TABLE %s ("
-              + "row_key blob,"
-              + "cf_id UUID,"
-              + "in_progress_ballot timeuuid,"
-              + "most_recent_commit blob,"
-              + "most_recent_commit_at timeuuid,"
-              + "most_recent_commit_version int,"
-              + "proposal blob,"
-              + "proposal_ballot timeuuid,"
-              + "proposal_version int,"
-              + "PRIMARY KEY ((row_key), cf_id))")
-              .compaction(CompactionParams.lcs(emptyMap()))
-              .build();
+                "in-progress paxos proposals",
+                "CREATE TABLE %s ("
+                + "row_key blob,"
+                + "cf_id UUID,"
+                + "in_progress_ballot timeuuid,"
+                + "most_recent_commit blob,"
+                + "most_recent_commit_at timeuuid,"
+                + "most_recent_commit_version int,"
+                + "proposal blob,"
+                + "proposal_ballot timeuuid,"
+                + "proposal_version int,"
+                + "PRIMARY KEY ((row_key), cf_id))")
+                .compaction(CompactionParams.lcs(emptyMap()))
+                .build();
 
     private static final TableMetadata BuiltIndexes =
         parse(BUILT_INDEXES,
@@ -145,122 +152,130 @@ public final class SystemKeyspace
 
     private static final TableMetadata Local =
         parse(LOCAL,
-              "information about the local node",
-              "CREATE TABLE %s ("
-              + "key text,"
-              + "bootstrapped text,"
-              + "broadcast_address inet,"
-              + "cluster_name text,"
-              + "cql_version text,"
-              + "data_center text,"
-              + "gossip_generation int,"
-              + "host_id uuid,"
-              + "listen_address inet,"
-              + "native_protocol_version text,"
-              + "partitioner text,"
-              + "rack text,"
-              + "release_version text,"
-              + "rpc_address inet,"
-              + "schema_version uuid,"
-              + "tokens set<varchar>,"
-              + "truncated_at map<uuid, blob>,"
-              + "PRIMARY KEY ((key)))")
-              .recordDeprecatedSystemColumn("thrift_version", UTF8Type.instance)
-              .build();
+                "information about the local node",
+                "CREATE TABLE %s ("
+                + "key text,"
+                + "bootstrapped text,"
+                + "broadcast_address inet,"
+                + "broadcast_port int,"
+                + "cluster_name text,"
+                + "cql_version text,"
+                + "data_center text,"
+                + "gossip_generation int,"
+                + "host_id uuid,"
+                + "listen_address inet,"
+                + "listen_port int,"
+                + "native_protocol_version text,"
+                + "partitioner text,"
+                + "rack text,"
+                + "release_version text,"
+                + "rpc_address inet,"
+                + "rpc_port int,"
+                + "schema_version uuid,"
+                + "tokens set<varchar>,"
+                + "truncated_at map<uuid, blob>,"
+                + "PRIMARY KEY ((key)))"
+                ).recordDeprecatedSystemColumn("thrift_version", UTF8Type.instance)
+                .build();
 
-    private static final TableMetadata Peers =
-        parse(PEERS,
-              "information about known peers in the cluster",
-              "CREATE TABLE %s ("
-              + "peer inet,"
-              + "data_center text,"
-              + "host_id uuid,"
-              + "preferred_ip inet,"
-              + "rack text,"
-              + "release_version text,"
-              + "rpc_address inet,"
-              + "schema_version uuid,"
-              + "tokens set<varchar>,"
-              + "PRIMARY KEY ((peer)))")
-              .build();
+    private static final TableMetadata PeersV2 =
+        parse(PEERS_V2,
+                "information about known peers in the cluster",
+                "CREATE TABLE %s ("
+                + "peer inet,"
+                + "peer_port int,"
+                + "data_center text,"
+                + "host_id uuid,"
+                + "preferred_ip inet,"
+                + "preferred_port int,"
+                + "rack text,"
+                + "release_version text,"
+                + "native_address inet,"
+                + "native_port int,"
+                + "schema_version uuid,"
+                + "tokens set<varchar>,"
+                + "PRIMARY KEY ((peer), peer_port))")
+                .build();
 
-    private static final TableMetadata PeerEvents =
-        parse(PEER_EVENTS,
-              "events related to peers",
-              "CREATE TABLE %s ("
-              + "peer inet,"
-              + "hints_dropped map<uuid, int>,"
-              + "PRIMARY KEY ((peer)))")
-              .build();
+    private static final TableMetadata PeerEventsV2 =
+        parse(PEER_EVENTS_V2,
+                "events related to peers",
+                "CREATE TABLE %s ("
+                + "peer inet,"
+                + "peer_port int,"
+                + "hints_dropped map<uuid, int>,"
+                + "PRIMARY KEY ((peer), peer_port))")
+                .build();
 
     private static final TableMetadata RangeXfers =
         parse(RANGE_XFERS,
-              "ranges requested for transfer",
-              "CREATE TABLE %s ("
-              + "token_bytes blob,"
-              + "requested_at timestamp,"
-              + "PRIMARY KEY ((token_bytes)))")
-              .build();
+                "ranges requested for transfer",
+                "CREATE TABLE %s ("
+                + "token_bytes blob,"
+                + "requested_at timestamp,"
+                + "PRIMARY KEY ((token_bytes)))")
+                .build();
 
     private static final TableMetadata CompactionHistory =
         parse(COMPACTION_HISTORY,
-              "week-long compaction history",
-              "CREATE TABLE %s ("
-              + "id uuid,"
-              + "bytes_in bigint,"
-              + "bytes_out bigint,"
-              + "columnfamily_name text,"
-              + "compacted_at timestamp,"
-              + "keyspace_name text,"
-              + "rows_merged map<int, bigint>,"
-              + "PRIMARY KEY ((id)))")
-              .defaultTimeToLive((int) TimeUnit.DAYS.toSeconds(7))
-              .build();
+                "week-long compaction history",
+                "CREATE TABLE %s ("
+                + "id uuid,"
+                + "bytes_in bigint,"
+                + "bytes_out bigint,"
+                + "columnfamily_name text,"
+                + "compacted_at timestamp,"
+                + "keyspace_name text,"
+                + "rows_merged map<int, bigint>,"
+                + "PRIMARY KEY ((id)))")
+                .defaultTimeToLive((int) TimeUnit.DAYS.toSeconds(7))
+                .build();
 
     private static final TableMetadata SSTableActivity =
         parse(SSTABLE_ACTIVITY,
-              "historic sstable read rates",
-              "CREATE TABLE %s ("
-              + "keyspace_name text,"
-              + "columnfamily_name text,"
-              + "generation int,"
-              + "rate_120m double,"
-              + "rate_15m double,"
-              + "PRIMARY KEY ((keyspace_name, columnfamily_name, generation)))")
-              .build();
+                "historic sstable read rates",
+                "CREATE TABLE %s ("
+                + "keyspace_name text,"
+                + "columnfamily_name text,"
+                + "generation int,"
+                + "rate_120m double,"
+                + "rate_15m double,"
+                + "PRIMARY KEY ((keyspace_name, columnfamily_name, generation)))")
+                .build();
 
     private static final TableMetadata SizeEstimates =
         parse(SIZE_ESTIMATES,
-              "per-table primary range size estimates",
-              "CREATE TABLE %s ("
-              + "keyspace_name text,"
-              + "table_name text,"
-              + "range_start text,"
-              + "range_end text,"
-              + "mean_partition_size bigint,"
-              + "partitions_count bigint,"
-              + "PRIMARY KEY ((keyspace_name), table_name, range_start, range_end))")
-              .build();
+                "per-table primary range size estimates",
+                "CREATE TABLE %s ("
+                + "keyspace_name text,"
+                + "table_name text,"
+                + "range_start text,"
+                + "range_end text,"
+                + "mean_partition_size bigint,"
+                + "partitions_count bigint,"
+                + "PRIMARY KEY ((keyspace_name), table_name, range_start, range_end))")
+                .build();
 
     private static final TableMetadata AvailableRanges =
         parse(AVAILABLE_RANGES,
-              "available keyspace/ranges during bootstrap/replace that are ready to be served",
-              "CREATE TABLE %s ("
-              + "keyspace_name text,"
-              + "ranges set<blob>,"
-              + "PRIMARY KEY ((keyspace_name)))")
-              .build();
+                "available keyspace/ranges during bootstrap/replace that are ready to be served",
+                "CREATE TABLE %s ("
+                + "keyspace_name text,"
+                + "ranges set<blob>,"
+                + "PRIMARY KEY ((keyspace_name)))")
+                .build();
 
-    private static final TableMetadata TransferredRanges =
-        parse(TRANSFERRED_RANGES,
-              "record of transferred ranges for streaming operation",
-              "CREATE TABLE %s ("
-              + "operation text,"
-              + "peer inet,"
-              + "keyspace_name text,"
-              + "ranges set<blob>,"
-              + "PRIMARY KEY ((operation, keyspace_name), peer))")
-              .build();
+    private static final TableMetadata TransferredRangesV2 =
+        parse(TRANSFERRED_RANGES_V2,
+                "record of transferred ranges for streaming operation",
+                "CREATE TABLE %s ("
+                + "operation text,"
+                + "peer inet,"
+                + "peer_port int,"
+                + "keyspace_name text,"
+                + "ranges set<blob>,"
+                + "PRIMARY KEY ((operation, keyspace_name), peer, peer_port))")
+                .build();
 
     private static final TableMetadata ViewBuildsInProgress =
         parse(VIEW_BUILDS_IN_PROGRESS,
@@ -277,38 +292,79 @@ public final class SystemKeyspace
 
     private static final TableMetadata BuiltViews =
         parse(BUILT_VIEWS,
-              "built views",
-              "CREATE TABLE %s ("
-              + "keyspace_name text,"
-              + "view_name text,"
-              + "status_replicated boolean,"
-              + "PRIMARY KEY ((keyspace_name), view_name))")
-              .build();
+                "built views",
+                "CREATE TABLE %s ("
+                + "keyspace_name text,"
+                + "view_name text,"
+                + "status_replicated boolean,"
+                + "PRIMARY KEY ((keyspace_name), view_name))")
+                .build();
 
     private static final TableMetadata PreparedStatements =
         parse(PREPARED_STATEMENTS,
-              "prepared statements",
-              "CREATE TABLE %s ("
-              + "prepared_id blob,"
-              + "logged_keyspace text,"
-              + "query_string text,"
-              + "PRIMARY KEY ((prepared_id)))")
-              .build();
+                "prepared statements",
+                "CREATE TABLE %s ("
+                + "prepared_id blob,"
+                + "logged_keyspace text,"
+                + "query_string text,"
+                + "PRIMARY KEY ((prepared_id)))")
+                .build();
 
     private static final TableMetadata Repairs =
         parse(REPAIRS,
-              "repairs",
-              "CREATE TABLE %s ("
-              + "parent_id timeuuid, "
-              + "started_at timestamp, "
-              + "last_update timestamp, "
-              + "repaired_at timestamp, "
-              + "state int, "
-              + "coordinator inet, "
-              + "participants set<inet>, "
-              + "ranges set<blob>, "
-              + "cfids set<uuid>, "
-              + "PRIMARY KEY (parent_id))").build();
+          "repairs",
+          "CREATE TABLE %s ("
+          + "parent_id timeuuid, "
+          + "started_at timestamp, "
+          + "last_update timestamp, "
+          + "repaired_at timestamp, "
+          + "state int, "
+          + "coordinator inet, "
+          + "coordinator_port int,"
+          + "participants set<inet>,"
+          + "participants_wp set<text>,"
+          + "ranges set<blob>, "
+          + "cfids set<uuid>, "
+          + "PRIMARY KEY (parent_id))").build();
+
+    @Deprecated
+    private static final TableMetadata LegacyPeers =
+        parse(LEGACY_PEERS,
+            "information about known peers in the cluster",
+            "CREATE TABLE %s ("
+            + "peer inet,"
+            + "data_center text,"
+            + "host_id uuid,"
+            + "preferred_ip inet,"
+            + "rack text,"
+            + "release_version text,"
+            + "rpc_address inet,"
+            + "schema_version uuid,"
+            + "tokens set<varchar>,"
+            + "PRIMARY KEY ((peer)))")
+            .build();
+
+    @Deprecated
+    private static final TableMetadata LegacyPeerEvents =
+        parse(LEGACY_PEER_EVENTS,
+            "events related to peers",
+            "CREATE TABLE %s ("
+            + "peer inet,"
+            + "hints_dropped map<uuid, int>,"
+            + "PRIMARY KEY ((peer)))")
+            .build();
+
+    @Deprecated
+    private static final TableMetadata LegacyTransferredRanges =
+        parse(LEGACY_TRANSFERRED_RANGES,
+            "record of transferred ranges for streaming operation",
+            "CREATE TABLE %s ("
+            + "operation text,"
+            + "peer inet,"
+            + "keyspace_name text,"
+            + "ranges set<blob>,"
+            + "PRIMARY KEY ((operation, keyspace_name), peer))")
+            .build();
 
     private static TableMetadata.Builder parse(String table, String description, String cql)
     {
@@ -331,14 +387,17 @@ public final class SystemKeyspace
                          Batches,
                          Paxos,
                          Local,
-                         Peers,
-                         PeerEvents,
+                         PeersV2,
+                         LegacyPeers,
+                         PeerEventsV2,
+                         LegacyPeerEvents,
                          RangeXfers,
                          CompactionHistory,
                          SSTableActivity,
                          SizeEstimates,
                          AvailableRanges,
-                         TransferredRanges,
+                         TransferredRangesV2,
+                         LegacyTransferredRanges,
                          ViewBuildsInProgress,
                          BuiltViews,
                          PreparedStatements,
@@ -384,9 +443,12 @@ public final class SystemKeyspace
                      "rack," +
                      "partitioner," +
                      "rpc_address," +
+                     "rpc_port," +
                      "broadcast_address," +
-                     "listen_address" +
-                     ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+                     "broadcast_port," +
+                     "listen_address," +
+                     "listen_port" +
+                     ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
         IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
         executeOnceInternal(format(req, LOCAL),
                             LOCAL,
@@ -394,12 +456,15 @@ public final class SystemKeyspace
                             FBUtilities.getReleaseVersionString(),
                             QueryProcessor.CQL_VERSION.toString(),
                             String.valueOf(ProtocolVersion.CURRENT.asInt()),
-                            snitch.getDatacenter(FBUtilities.getBroadcastAddress()),
-                            snitch.getRack(FBUtilities.getBroadcastAddress()),
+                            snitch.getDatacenter(FBUtilities.getBroadcastAddressAndPort()),
+                            snitch.getRack(FBUtilities.getBroadcastAddressAndPort()),
                             DatabaseDescriptor.getPartitioner().getClass().getName(),
                             DatabaseDescriptor.getRpcAddress(),
-                            FBUtilities.getBroadcastAddress(),
-                            FBUtilities.getLocalAddress());
+                            DatabaseDescriptor.getNativeTransportPort(),
+                            FBUtilities.getJustBroadcastAddress(),
+                            DatabaseDescriptor.getStoragePort(),
+                            FBUtilities.getJustLocalAddress(),
+                            DatabaseDescriptor.getStoragePort());
     }
 
     public static void updateCompactionHistory(String ksname,
@@ -461,11 +526,10 @@ public final class SystemKeyspace
     {
         String buildReq = "DELETE FROM %s.%s WHERE keyspace_name = ? AND view_name = ?";
         executeInternal(String.format(buildReq, SchemaConstants.SYSTEM_KEYSPACE_NAME, VIEW_BUILDS_IN_PROGRESS), keyspaceName, viewName);
-        forceBlockingFlush(VIEW_BUILDS_IN_PROGRESS);
 
         String builtReq = "DELETE FROM %s.\"%s\" WHERE keyspace_name = ? AND view_name = ? IF EXISTS";
         executeInternal(String.format(builtReq, SchemaConstants.SYSTEM_KEYSPACE_NAME, BUILT_VIEWS), keyspaceName, viewName);
-        forceBlockingFlush(BUILT_VIEWS);
+        forceBlockingFlush(VIEW_BUILDS_IN_PROGRESS, BUILT_VIEWS);
     }
 
     public static void finishViewBuildStatus(String ksname, String viewName)
@@ -609,39 +673,64 @@ public final class SystemKeyspace
     /**
      * Record tokens being used by another node
      */
-    public static synchronized void updateTokens(InetAddress ep, Collection<Token> tokens)
+    public static synchronized void updateTokens(InetAddressAndPort ep, Collection<Token> tokens)
     {
-        if (ep.equals(FBUtilities.getBroadcastAddress()))
+        if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
             return;
 
         String req = "INSERT INTO system.%s (peer, tokens) VALUES (?, ?)";
-        executeInternal(format(req, PEERS), ep, tokensAsSet(tokens));
+        executeInternal(String.format(req, LEGACY_PEERS), ep.address, tokensAsSet(tokens));
+        req = "INSERT INTO system.%s (peer, peer_port, tokens) VALUES (?, ?, ?)";
+        executeInternal(String.format(req, PEERS_V2), ep.address, ep.port, tokensAsSet(tokens));
     }
 
-    public static synchronized void updatePreferredIP(InetAddress ep, InetAddress preferred_ip)
+    public static synchronized void updatePreferredIP(InetAddressAndPort ep, InetAddressAndPort preferred_ip)
     {
         if (getPreferredIP(ep) == preferred_ip)
             return;
 
         String req = "INSERT INTO system.%s (peer, preferred_ip) VALUES (?, ?)";
-        executeInternal(format(req, PEERS), ep, preferred_ip);
-        forceBlockingFlush(PEERS);
+        executeInternal(String.format(req, LEGACY_PEERS), ep.address, preferred_ip.address);
+        req = "INSERT INTO system.%s (peer, peer_port, preferred_ip, preferred_port) VALUES (?, ?, ?, ?)";
+        executeInternal(String.format(req, PEERS_V2), ep.address, ep.port, preferred_ip.address, preferred_ip.port);
+        forceBlockingFlush(LEGACY_PEERS, PEERS_V2);
     }
 
-    public static synchronized void updatePeerInfo(InetAddress ep, String columnName, Object value)
+    public static synchronized void updatePeerInfo(InetAddressAndPort ep, String columnName, Object value)
     {
-        if (ep.equals(FBUtilities.getBroadcastAddress()))
+        if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
             return;
 
         String req = "INSERT INTO system.%s (peer, %s) VALUES (?, ?)";
-        executeInternal(format(req, PEERS, columnName), ep, value);
+        executeInternal(String.format(req, LEGACY_PEERS, columnName), ep.address, value);
+        //This column doesn't match across the two tables
+        if (columnName.equals("rpc_address"))
+        {
+            columnName = "native_address";
+        }
+        req = "INSERT INTO system.%s (peer, peer_port, %s) VALUES (?, ?, ?)";
+        executeInternal(String.format(req, PEERS_V2, columnName), ep.address, ep.port, value);
     }
 
-    public static synchronized void updateHintsDropped(InetAddress ep, UUID timePeriod, int value)
+    public static synchronized void updatePeerNativeAddress(InetAddressAndPort ep, InetAddressAndPort address)
+    {
+        if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
+            return;
+
+        String req = "INSERT INTO system.%s (peer, rpc_address) VALUES (?, ?)";
+        executeInternal(String.format(req, LEGACY_PEERS), ep.address, address.address);
+        req = "INSERT INTO system.%s (peer, peer_port, native_address, native_port) VALUES (?, ?, ?, ?)";
+        executeInternal(String.format(req, PEERS_V2), ep.address, ep.port, address.address, address.port);
+    }
+
+
+    public static synchronized void updateHintsDropped(InetAddressAndPort ep, UUID timePeriod, int value)
     {
         // with 30 day TTL
         String req = "UPDATE system.%s USING TTL 2592000 SET hints_dropped[ ? ] = ? WHERE peer = ?";
-        executeInternal(format(req, PEER_EVENTS), timePeriod, value, ep);
+        executeInternal(String.format(req, LEGACY_PEER_EVENTS), timePeriod, value, ep.address);
+        req = "UPDATE system.%s USING TTL 2592000 SET hints_dropped[ ? ] = ? WHERE peer = ? AND peer_port = ?";
+        executeInternal(String.format(req, PEER_EVENTS_V2), timePeriod, value, ep.address, ep.port);
     }
 
     public static synchronized void updateSchemaVersion(UUID version)
@@ -673,11 +762,13 @@ public final class SystemKeyspace
     /**
      * Remove stored tokens being used by another node
      */
-    public static synchronized void removeEndpoint(InetAddress ep)
+    public static synchronized void removeEndpoint(InetAddressAndPort ep)
     {
         String req = "DELETE FROM system.%s WHERE peer = ?";
-        executeInternal(format(req, PEERS), ep);
-        forceBlockingFlush(PEERS);
+        executeInternal(String.format(req, LEGACY_PEERS), ep.address);
+        req = String.format("DELETE FROM system.%s WHERE peer = ? AND peer_port = ?", PEERS_V2);
+        executeInternal(req, ep.address, ep.port);
+        forceBlockingFlush(LEGACY_PEERS, PEERS_V2);
     }
 
     /**
@@ -696,22 +787,32 @@ public final class SystemKeyspace
         forceBlockingFlush(LOCAL);
     }
 
-    public static void forceBlockingFlush(String cfname)
+    public static void forceBlockingFlush(String ...cfnames)
     {
         if (!DatabaseDescriptor.isUnsafeSystem())
-            FBUtilities.waitOnFuture(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(cfname).forceFlush());
+        {
+            List<ListenableFuture<CommitLogPosition>> futures = new ArrayList<>();
+
+            for (String cfname : cfnames)
+            {
+                futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(cfname).forceFlush());
+            }
+            FBUtilities.waitOnFutures(futures);
+        }
     }
 
     /**
      * Return a map of stored tokens to IP addresses
      *
      */
-    public static SetMultimap<InetAddress, Token> loadTokens()
+    public static SetMultimap<InetAddressAndPort, Token> loadTokens()
     {
-        SetMultimap<InetAddress, Token> tokenMap = HashMultimap.create();
-        for (UntypedResultSet.Row row : executeInternal("SELECT peer, tokens FROM system." + PEERS))
+        SetMultimap<InetAddressAndPort, Token> tokenMap = HashMultimap.create();
+        for (UntypedResultSet.Row row : executeInternal("SELECT peer, peer_port, tokens FROM system." + PEERS_V2))
         {
-            InetAddress peer = row.getInetAddress("peer");
+            InetAddress address = row.getInetAddress("peer");
+            Integer port = row.getInt("peer_port");
+            InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
             if (row.has("tokens"))
                 tokenMap.putAll(peer, deserializeTokens(row.getSet("tokens", UTF8Type.instance)));
         }
@@ -723,12 +824,14 @@ public final class SystemKeyspace
      * Return a map of store host_ids to IP addresses
      *
      */
-    public static Map<InetAddress, UUID> loadHostIds()
+    public static Map<InetAddressAndPort, UUID> loadHostIds()
     {
-        Map<InetAddress, UUID> hostIdMap = new HashMap<>();
-        for (UntypedResultSet.Row row : executeInternal("SELECT peer, host_id FROM system." + PEERS))
+        Map<InetAddressAndPort, UUID> hostIdMap = new HashMap<>();
+        for (UntypedResultSet.Row row : executeInternal("SELECT peer, peer_port, host_id FROM system." + PEERS_V2))
         {
-            InetAddress peer = row.getInetAddress("peer");
+            InetAddress address = row.getInetAddress("peer");
+            Integer port = row.getInt("peer_port");
+            InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
             if (row.has("host_id"))
             {
                 hostIdMap.put(peer, row.getUUID("host_id"));
@@ -743,24 +846,29 @@ public final class SystemKeyspace
      * @param ep endpoint address to check
      * @return Preferred IP for given endpoint if present, otherwise returns given ep
      */
-    public static InetAddress getPreferredIP(InetAddress ep)
+    public static InetAddressAndPort getPreferredIP(InetAddressAndPort ep)
     {
-        String req = "SELECT preferred_ip FROM system.%s WHERE peer=?";
-        UntypedResultSet result = executeInternal(format(req, PEERS), ep);
+        String req = "SELECT preferred_ip, preferred_port FROM system.%s WHERE peer=? AND peer_port = ?";
+        UntypedResultSet result = executeInternal(String.format(req, PEERS_V2), ep.address, ep.port);
         if (!result.isEmpty() && result.one().has("preferred_ip"))
-            return result.one().getInetAddress("preferred_ip");
+        {
+            UntypedResultSet.Row row = result.one();
+            return InetAddressAndPort.getByAddressOverrideDefaults(row.getInetAddress("preferred_ip"), row.getInt("preferred_port"));
+        }
         return ep;
     }
 
     /**
      * Return a map of IP addresses containing a map of dc and rack info
      */
-    public static Map<InetAddress, Map<String,String>> loadDcRackInfo()
+    public static Map<InetAddressAndPort, Map<String,String>> loadDcRackInfo()
     {
-        Map<InetAddress, Map<String, String>> result = new HashMap<>();
-        for (UntypedResultSet.Row row : executeInternal("SELECT peer, data_center, rack from system." + PEERS))
+        Map<InetAddressAndPort, Map<String, String>> result = new HashMap<>();
+        for (UntypedResultSet.Row row : executeInternal("SELECT peer, peer_port, data_center, rack from system." + PEERS_V2))
         {
-            InetAddress peer = row.getInetAddress("peer");
+            InetAddress address = row.getInetAddress("peer");
+            Integer port = row.getInt("peer_port");
+            InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(address, port);
             if (row.has("data_center") && row.has("rack"))
             {
                 Map<String, String> dcRack = new HashMap<>();
@@ -779,16 +887,16 @@ public final class SystemKeyspace
      * @param ep endpoint address to check
      * @return Release version or null if version is unknown.
      */
-    public static CassandraVersion getReleaseVersion(InetAddress ep)
+    public static CassandraVersion getReleaseVersion(InetAddressAndPort ep)
     {
         try
         {
-            if (FBUtilities.getBroadcastAddress().equals(ep))
+            if (FBUtilities.getBroadcastAddressAndPort().equals(ep))
             {
                 return new CassandraVersion(FBUtilities.getReleaseVersionString());
             }
-            String req = "SELECT release_version FROM system.%s WHERE peer=?";
-            UntypedResultSet result = executeInternal(format(req, PEERS), ep);
+            String req = "SELECT release_version FROM system.%s WHERE peer=? AND peer_port=?";
+            UntypedResultSet result = executeInternal(String.format(req, PEERS_V2), ep.address, ep.port);
             if (result != null && result.one().has("release_version"))
             {
                 return new CassandraVersion(result.one().getString("release_version"));
@@ -1197,7 +1305,7 @@ public final class SystemKeyspace
     }
 
     public static synchronized void updateTransferredRanges(StreamOperation streamOperation,
-                                                         InetAddress peer,
+                                                         InetAddressAndPort peer,
                                                          String keyspace,
                                                          Collection<Range<Token>> streamedRanges)
     {
@@ -1207,17 +1315,21 @@ public final class SystemKeyspace
         {
             rangesToUpdate.add(rangeToBytes(range));
         }
-        executeInternal(format(cql, TRANSFERRED_RANGES), rangesToUpdate, streamOperation.getDescription(), peer, keyspace);
+        executeInternal(format(cql, LEGACY_TRANSFERRED_RANGES), rangesToUpdate, streamOperation.getDescription(), peer.address, keyspace);
+        cql = "UPDATE system.%s SET ranges = ranges + ? WHERE operation = ? AND peer = ? AND peer_port = ? AND keyspace_name = ?";
+        executeInternal(String.format(cql, TRANSFERRED_RANGES_V2), rangesToUpdate, streamOperation.getDescription(), peer.address, peer.port, keyspace);
     }
 
-    public static synchronized Map<InetAddress, Set<Range<Token>>> getTransferredRanges(String description, String keyspace, IPartitioner partitioner)
+    public static synchronized Map<InetAddressAndPort, Set<Range<Token>>> getTransferredRanges(String description, String keyspace, IPartitioner partitioner)
     {
-        Map<InetAddress, Set<Range<Token>>> result = new HashMap<>();
+        Map<InetAddressAndPort, Set<Range<Token>>> result = new HashMap<>();
         String query = "SELECT * FROM system.%s WHERE operation = ? AND keyspace_name = ?";
-        UntypedResultSet rs = executeInternal(format(query, TRANSFERRED_RANGES), description, keyspace);
+        UntypedResultSet rs = executeInternal(String.format(query, TRANSFERRED_RANGES_V2), description, keyspace);
         for (UntypedResultSet.Row row : rs)
         {
-            InetAddress peer = row.getInetAddress("peer");
+            InetAddress peerAddress = row.getInetAddress("peer");
+            int port = row.getInt("peer_port");
+            InetAddressAndPort peer = InetAddressAndPort.getByAddressOverrideDefaults(peerAddress, port);
             Set<ByteBuffer> rawRanges = row.getSet("ranges", BytesType.instance);
             Set<Range<Token>> ranges = Sets.newHashSetWithExpectedSize(rawRanges.size());
             for (ByteBuffer rawRange : rawRanges)

src/java/org/apache/cassandra/db/SystemKeyspaceMigrator40.java

@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+
+/**
+ * Migrate 3.0 versions of some tables to 4.0. In this case it's just extra columns and some keys
+ * that are changed.
+ *
+ * Can't just add the additional columns because they are primary key columns and C* doesn't support changing
+ * key columns even if it's just clustering columns.
+ */
+public class SystemKeyspaceMigrator40
+{
+    static final String legacyPeersName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_PEERS);
+    static final String peersName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PEERS_V2);
+    static final String legacyPeerEventsName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_PEER_EVENTS);
+    static final String peerEventsName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PEER_EVENTS_V2);
+    static final String legacyTransferredRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_TRANSFERRED_RANGES);
+    static final String transferredRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.TRANSFERRED_RANGES_V2);
+
+    private static final Logger logger = LoggerFactory.getLogger(SystemKeyspaceMigrator40.class);
+
+    private SystemKeyspaceMigrator40() {}
+
+    public static void migrate()
+    {
+        migratePeers();
+        migratePeerEvents();
+        migrateTransferredRanges();
+    }
+
+    private static void migratePeers()
+    {
+        ColumnFamilyStore newPeers = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PEERS_V2);
+
+        if (!newPeers.isEmpty())
+             return;
+
+        logger.info("{} table was empty, migrating legacy {}, if this fails you should fix the issue and then truncate {} to have it try again.",
+                                  peersName, legacyPeersName, peersName);
+
+        String query = String.format("SELECT * FROM %s",
+                                     legacyPeersName);
+
+        String insert = String.format("INSERT INTO %s ( "
+                                      + "peer, "
+                                      + "peer_port, "
+                                      + "data_center, "
+                                      + "host_id, "
+                                      + "preferred_ip, "
+                                      + "preferred_port, "
+                                      + "rack, "
+                                      + "release_version, "
+                                      + "native_address, "
+                                      + "native_port, "
+                                      + "schema_version, "
+                                      + "tokens) "
+                                      + " values ( ?, ?, ? , ? , ?, ?, ?, ?, ?, ?, ?, ?)",
+                                      peersName);
+
+        UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
+        int transferred = 0;
+        logger.info("Migrating rows from legacy {} to {}", legacyPeersName, peersName);
+        for (UntypedResultSet.Row row : rows)
+        {
+            logger.debug("Transferring row {}", transferred);
+            QueryProcessor.executeInternal(insert,
+                                           row.has("peer") ? row.getInetAddress("peer") : null,
+                                           DatabaseDescriptor.getStoragePort(),
+                                           row.has("data_center") ? row.getString("data_center") : null,
+                                           row.has("host_id") ? row.getUUID("host_id") : null,
+                                           row.has("preferred_ip") ? row.getInetAddress("preferred_ip") : null,
+                                           DatabaseDescriptor.getStoragePort(),
+                                           row.has("rack") ? row.getString("rack") : null,
+                                           row.has("release_version") ? row.getString("release_version") : null,
+                                           row.has("rpc_address") ? row.getInetAddress("rpc_address") : null,
+                                           DatabaseDescriptor.getNativeTransportPort(),
+                                           row.has("schema_version") ? row.getUUID("schema_version") : null,
+                                           row.has("tokens") ? row.getSet("tokens", UTF8Type.instance) : null);
+            transferred++;
+        }
+        logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyPeersName, peersName);
+    }
+
+    private static void migratePeerEvents()
+    {
+        ColumnFamilyStore newPeerEvents = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PEER_EVENTS_V2);
+
+        if (!newPeerEvents.isEmpty())
+            return;
+
+        logger.info("{} table was empty, migrating legacy {} to {}", peerEventsName, legacyPeerEventsName, peerEventsName);
+
+        String query = String.format("SELECT * FROM %s",
+                                     legacyPeerEventsName);
+
+        String insert = String.format("INSERT INTO %s ( "
+                                      + "peer, "
+                                      + "peer_port, "
+                                      + "hints_dropped) "
+                                      + " values ( ?, ?, ? )",
+                                      peerEventsName);
+
+        UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
+        int transferred = 0;
+        for (UntypedResultSet.Row row : rows)
+        {
+            logger.debug("Transferring row {}", transferred);
+            QueryProcessor.executeInternal(insert,
+                                           row.has("peer") ? row.getInetAddress("peer") : null,
+                                           DatabaseDescriptor.getStoragePort(),
+                                           row.has("hints_dropped") ? row.getMap("hints_dropped", UUIDType.instance, Int32Type.instance) : null);
+            transferred++;
+        }
+        logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyPeerEventsName, peerEventsName);
+    }
+
+    static void migrateTransferredRanges()
+    {
+        ColumnFamilyStore newTransferredRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.TRANSFERRED_RANGES_V2);
+
+        if (!newTransferredRanges.isEmpty())
+            return;
+
+        logger.info("{} table was empty, migrating legacy {} to {}", transferredRangesName, legacyTransferredRangesName, transferredRangesName);
+
+        String query = String.format("SELECT * FROM %s",
+                                     legacyTransferredRangesName);
+
+        String insert = String.format("INSERT INTO %s ("
+                                      + "operation, "
+                                      + "peer, "
+                                      + "peer_port, "
+                                      + "keyspace_name, "
+                                      + "ranges) "
+                                      + " values ( ?, ?, ? , ?, ?)",
+                                      transferredRangesName);
+
+        UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
+        int transferred = 0;
+        for (UntypedResultSet.Row row : rows)
+        {
+            logger.debug("Transferring row {}", transferred);
+            QueryProcessor.executeInternal(insert,
+                                           row.has("operation") ? row.getString("operation") : null,
+                                           row.has("peer") ? row.getInetAddress("peer") : null,
+                                           DatabaseDescriptor.getStoragePort(),
+                                           row.has("keyspace_name") ? row.getString("keyspace_name") : null,
+                                           row.has("ranges") ? row.getSet("ranges", BytesType.instance) : null);
+            transferred++;
+        }
+
+        logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyTransferredRangesName, transferredRangesName);
+    }
+
+}

src/java/org/apache/cassandra/db/view/ViewUtils.java

@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.db.view;
 
-import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -27,6 +26,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.NetworkTopologyStrategy;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -58,14 +58,14 @@ public final class ViewUtils
      *
      * @return Optional.empty() if this method is called using a base token which does not belong to this replica
      */
-    public static Optional<InetAddress> getViewNaturalEndpoint(String keyspaceName, Token baseToken, Token viewToken)
+    public static Optional<InetAddressAndPort> getViewNaturalEndpoint(String keyspaceName, Token baseToken, Token viewToken)
     {
         AbstractReplicationStrategy replicationStrategy = Keyspace.open(keyspaceName).getReplicationStrategy();
 
-        String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
-        List<InetAddress> baseEndpoints = new ArrayList<>();
-        List<InetAddress> viewEndpoints = new ArrayList<>();
-        for (InetAddress baseEndpoint : replicationStrategy.getNaturalEndpoints(baseToken))
+        String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort());
+        List<InetAddressAndPort> baseEndpoints = new ArrayList<>();
+        List<InetAddressAndPort> viewEndpoints = new ArrayList<>();
+        for (InetAddressAndPort baseEndpoint : replicationStrategy.getNaturalEndpoints(baseToken))
         {
             // An endpoint is local if we're not using Net
             if (!(replicationStrategy instanceof NetworkTopologyStrategy) ||
@@ -73,10 +73,10 @@ public final class ViewUtils
                 baseEndpoints.add(baseEndpoint);
         }
 
-        for (InetAddress viewEndpoint : replicationStrategy.getNaturalEndpoints(viewToken))
+        for (InetAddressAndPort viewEndpoint : replicationStrategy.getNaturalEndpoints(viewToken))
         {
             // If we are a base endpoint which is also a view replica, we use ourselves as our view replica
-            if (viewEndpoint.equals(FBUtilities.getBroadcastAddress()))
+            if (viewEndpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
                 return Optional.of(viewEndpoint);
 
             // We have to remove any endpoint which is shared between the base and the view, as it will select itself
@@ -92,7 +92,7 @@ public final class ViewUtils
         // Since the same replication strategy is used, the same placement should be used and we should get the same
         // number of replicas for all of the tokens in the ring.
         assert baseEndpoints.size() == viewEndpoints.size() : "Replication strategy should have the same number of endpoints for the base and the view";
-        int baseIdx = baseEndpoints.indexOf(FBUtilities.getBroadcastAddress());
+        int baseIdx = baseEndpoints.indexOf(FBUtilities.getBroadcastAddressAndPort());
 
         if (baseIdx < 0)
             //This node is not a base replica of this key, so we return empty

src/java/org/apache/cassandra/dht/BootStrapper.java

@@ -18,7 +18,6 @@
 package org.apache.cassandra.dht;
 
 import java.io.IOException;
-import java.net.InetAddress;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -38,6 +37,7 @@ import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.TokenMetadata;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.streaming.*;
@@ -51,12 +51,12 @@ public class BootStrapper extends ProgressEventNotifierSupport
     private static final Logger logger = LoggerFactory.getLogger(BootStrapper.class);
 
     /* endpoint that needs to be bootstrapped */
-    protected final InetAddress address;
+    protected final InetAddressAndPort address;
     /* token of the node being bootstrapped. */
     protected final Collection<Token> tokens;
     protected final TokenMetadata tokenMetadata;
 
-    public BootStrapper(InetAddress address, Collection<Token> tokens, TokenMetadata tmd)
+    public BootStrapper(InetAddressAndPort address, Collection<Token> tokens, TokenMetadata tmd)
     {
         assert address != null;
         assert tokens != null && !tokens.isEmpty();
@@ -159,7 +159,7 @@ public class BootStrapper extends ProgressEventNotifierSupport
      * otherwise, if allocationKeyspace is specified use the token allocation algorithm to generate suitable tokens
      * else choose num_tokens tokens at random
      */
-    public static Collection<Token> getBootstrapTokens(final TokenMetadata metadata, InetAddress address, int schemaWaitDelay) throws ConfigurationException
+    public static Collection<Token> getBootstrapTokens(final TokenMetadata metadata, InetAddressAndPort address, int schemaWaitDelay) throws ConfigurationException
     {
         String allocationKeyspace = DatabaseDescriptor.getAllocateTokensForKeyspace();
         Collection<String> initialTokens = DatabaseDescriptor.getInitialTokens();
@@ -199,13 +199,13 @@ public class BootStrapper extends ProgressEventNotifierSupport
     }
 
     static Collection<Token> allocateTokens(final TokenMetadata metadata,
-                                            InetAddress address,
+                                            InetAddressAndPort address,
                                             String allocationKeyspace,
                                             int numTokens,
                                             int schemaWaitDelay)
     {
         StorageService.instance.waitForSchema(schemaWaitDelay);
-        if (!FBUtilities.getBroadcastAddress().equals(InetAddress.getLoopbackAddress()))
+        if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
             Gossiper.waitToSettle();
 
         Keyspace ks = Keyspace.open(allocationKeyspace);

src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java

@@ -32,6 +32,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.FBUtilities;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -72,7 +73,7 @@ public class RangeFetchMapCalculator
 {
     private static final Logger logger = LoggerFactory.getLogger(RangeFetchMapCalculator.class);
     private static final long TRIVIAL_RANGE_LIMIT = 1000;
-    private final Multimap<Range<Token>, InetAddress> rangesWithSources;
+    private final Multimap<Range<Token>, InetAddressAndPort> rangesWithSources;
     private final Collection<RangeStreamer.ISourceFilter> sourceFilters;
     private final String keyspace;
     //We need two Vertices to act as source and destination in the algorithm
@@ -80,7 +81,7 @@ public class RangeFetchMapCalculator
     private final Vertex destinationVertex = OuterVertex.getDestinationVertex();
     private final Set<Range<Token>> trivialRanges;
 
-    public RangeFetchMapCalculator(Multimap<Range<Token>, InetAddress> rangesWithSources,
+    public RangeFetchMapCalculator(Multimap<Range<Token>, InetAddressAndPort> rangesWithSources,
                                    Collection<RangeStreamer.ISourceFilter> sourceFilters,
                                    String keyspace)
     {
@@ -108,16 +109,16 @@ public class RangeFetchMapCalculator
         return false;
     }
 
-    public Multimap<InetAddress, Range<Token>> getRangeFetchMap()
+    public Multimap<InetAddressAndPort, Range<Token>> getRangeFetchMap()
     {
-        Multimap<InetAddress, Range<Token>> fetchMap = HashMultimap.create();
+        Multimap<InetAddressAndPort, Range<Token>> fetchMap = HashMultimap.create();
         fetchMap.putAll(getRangeFetchMapForNonTrivialRanges());
         fetchMap.putAll(getRangeFetchMapForTrivialRanges(fetchMap));
         return fetchMap;
     }
 
     @VisibleForTesting
-    Multimap<InetAddress, Range<Token>> getRangeFetchMapForNonTrivialRanges()
+    Multimap<InetAddressAndPort, Range<Token>> getRangeFetchMapForNonTrivialRanges()
     {
         //Get the graph with edges between ranges and their source endpoints
         MutableCapacityGraph<Vertex, Integer> graph = getGraph();
@@ -148,19 +149,19 @@ public class RangeFetchMapCalculator
     }
 
     @VisibleForTesting
-    Multimap<InetAddress, Range<Token>> getRangeFetchMapForTrivialRanges(Multimap<InetAddress, Range<Token>> optimisedMap)
+    Multimap<InetAddressAndPort, Range<Token>> getRangeFetchMapForTrivialRanges(Multimap<InetAddressAndPort, Range<Token>> optimisedMap)
     {
-        Multimap<InetAddress, Range<Token>> fetchMap = HashMultimap.create();
+        Multimap<InetAddressAndPort, Range<Token>> fetchMap = HashMultimap.create();
         for (Range<Token> trivialRange : trivialRanges)
         {
             boolean added = false;
             boolean localDCCheck = true;
             while (!added)
             {
-                List<InetAddress> srcs = new ArrayList<>(rangesWithSources.get(trivialRange));
+                List<InetAddressAndPort> srcs = new ArrayList<>(rangesWithSources.get(trivialRange));
                 // sort with the endpoint having the least number of streams first:
                 srcs.sort(Comparator.comparingInt(o -> optimisedMap.get(o).size()));
-                for (InetAddress src : srcs)
+                for (InetAddressAndPort src : srcs)
                 {
                     if (passFilters(src, localDCCheck))
                     {
@@ -202,9 +203,9 @@ public class RangeFetchMapCalculator
      * @param result Flow algorithm result
      * @return  Multi Map of Machine to Ranges
      */
-    private Multimap<InetAddress, Range<Token>> getRangeFetchMapFromGraphResult(MutableCapacityGraph<Vertex, Integer> graph, MaximumFlowAlgorithmResult<Integer, CapacityEdge<Vertex, Integer>> result)
+    private Multimap<InetAddressAndPort, Range<Token>> getRangeFetchMapFromGraphResult(MutableCapacityGraph<Vertex, Integer> graph, MaximumFlowAlgorithmResult<Integer, CapacityEdge<Vertex, Integer>> result)
     {
-        final Multimap<InetAddress, Range<Token>> rangeFetchMapMap = HashMultimap.create();
+        final Multimap<InetAddressAndPort, Range<Token>> rangeFetchMapMap = HashMultimap.create();
         if(result == null)
             return rangeFetchMapMap;
         final Function<CapacityEdge<Vertex, Integer>, Integer> flowFunction = result.calcFlowFunction();
@@ -346,13 +347,13 @@ public class RangeFetchMapCalculator
     private boolean addEndpoints(MutableCapacityGraph<Vertex, Integer> capacityGraph, RangeVertex rangeVertex, boolean localDCCheck)
     {
         boolean sourceFound = false;
-        for (InetAddress endpoint : rangesWithSources.get(rangeVertex.getRange()))
+        for (InetAddressAndPort endpoint : rangesWithSources.get(rangeVertex.getRange()))
         {
             if (passFilters(endpoint, localDCCheck))
             {
                 sourceFound = true;
                 // if we pass filters, it means that we don't filter away localhost and we can count it as a source:
-                if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+                if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
                     continue; // but don't add localhost to the graph to avoid streaming locally
                 final Vertex endpointVertex = new EndpointVertex(endpoint);
                 capacityGraph.insertVertex(rangeVertex);
@@ -363,7 +364,7 @@ public class RangeFetchMapCalculator
         return sourceFound;
     }
 
-    private boolean isInLocalDC(InetAddress endpoint)
+    private boolean isInLocalDC(InetAddressAndPort endpoint)
     {
         return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint));
     }
@@ -374,7 +375,7 @@ public class RangeFetchMapCalculator
      * @param localDCCheck Allow endpoints with local DC
      * @return   True if filters pass this endpoint
      */
-    private boolean passFilters(final InetAddress endpoint, boolean localDCCheck)
+    private boolean passFilters(final InetAddressAndPort endpoint, boolean localDCCheck)
     {
         for (RangeStreamer.ISourceFilter filter : sourceFilters)
         {
@@ -410,15 +411,15 @@ public class RangeFetchMapCalculator
      */
     private static class EndpointVertex extends Vertex
     {
-        private final InetAddress endpoint;
+        private final InetAddressAndPort endpoint;
 
-        public EndpointVertex(InetAddress endpoint)
+        public EndpointVertex(InetAddressAndPort endpoint)
         {
             assert endpoint != null;
             this.endpoint = endpoint;
         }
 
-        public InetAddress getEndpoint()
+        public InetAddressAndPort getEndpoint()
         {
             return endpoint;
         }

src/java/org/apache/cassandra/dht/RangeStreamer.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.dht;
 
-import java.net.InetAddress;
 import java.util.*;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -26,8 +25,9 @@ import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.LocalStrategy;
-import org.apache.cassandra.service.ActiveRepairService;
+
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -59,10 +59,10 @@ public class RangeStreamer
     /* current token ring */
     private final TokenMetadata metadata;
     /* address of this node */
-    private final InetAddress address;
+    private final InetAddressAndPort address;
     /* streaming description */
     private final String description;
-    private final Multimap<String, Map.Entry<InetAddress, Collection<Range<Token>>>> toFetch = HashMultimap.create();
+    private final Multimap<String, Map.Entry<InetAddressAndPort, Collection<Range<Token>>>> toFetch = HashMultimap.create();
     private final Set<ISourceFilter> sourceFilters = new HashSet<>();
     private final StreamPlan streamPlan;
     private final boolean useStrictConsistency;
@@ -74,7 +74,7 @@ public class RangeStreamer
      */
     public static interface ISourceFilter
     {
-        public boolean shouldInclude(InetAddress endpoint);
+        public boolean shouldInclude(InetAddressAndPort endpoint);
     }
 
     /**
@@ -90,7 +90,7 @@ public class RangeStreamer
             this.fd = fd;
         }
 
-        public boolean shouldInclude(InetAddress endpoint)
+        public boolean shouldInclude(InetAddressAndPort endpoint)
         {
             return fd.isAlive(endpoint);
         }
@@ -110,7 +110,7 @@ public class RangeStreamer
             this.snitch = snitch;
         }
 
-        public boolean shouldInclude(InetAddress endpoint)
+        public boolean shouldInclude(InetAddressAndPort endpoint)
         {
             return snitch.getDatacenter(endpoint).equals(sourceDc);
         }
@@ -121,9 +121,9 @@ public class RangeStreamer
      */
     public static class ExcludeLocalNodeFilter implements ISourceFilter
     {
-        public boolean shouldInclude(InetAddress endpoint)
+        public boolean shouldInclude(InetAddressAndPort endpoint)
         {
-            return !FBUtilities.getBroadcastAddress().equals(endpoint);
+            return !FBUtilities.getBroadcastAddressAndPort().equals(endpoint);
         }
     }
 
@@ -132,14 +132,14 @@ public class RangeStreamer
      */
     public static class WhitelistedSourcesFilter implements ISourceFilter
     {
-        private final Set<InetAddress> whitelistedSources;
+        private final Set<InetAddressAndPort> whitelistedSources;
 
-        public WhitelistedSourcesFilter(Set<InetAddress> whitelistedSources)
+        public WhitelistedSourcesFilter(Set<InetAddressAndPort> whitelistedSources)
         {
             this.whitelistedSources = whitelistedSources;
         }
 
-        public boolean shouldInclude(InetAddress endpoint)
+        public boolean shouldInclude(InetAddressAndPort endpoint)
         {
             return whitelistedSources.contains(endpoint);
         }
@@ -147,7 +147,7 @@ public class RangeStreamer
 
     public RangeStreamer(TokenMetadata metadata,
                          Collection<Token> tokens,
-                         InetAddress address,
+                         InetAddressAndPort address,
                          StreamOperation streamOperation,
                          boolean useStrictConsistency,
                          IEndpointSnitch snitch,
@@ -186,18 +186,18 @@ public class RangeStreamer
         }
 
         boolean useStrictSource = useStrictSourcesForRanges(keyspaceName);
-        Multimap<Range<Token>, InetAddress> rangesForKeyspace = useStrictSource
+        Multimap<Range<Token>, InetAddressAndPort> rangesForKeyspace = useStrictSource
                 ? getAllRangesWithStrictSourcesFor(keyspaceName, ranges) : getAllRangesWithSourcesFor(keyspaceName, ranges);
 
-        for (Map.Entry<Range<Token>, InetAddress> entry : rangesForKeyspace.entries())
+        for (Map.Entry<Range<Token>, InetAddressAndPort> entry : rangesForKeyspace.entries())
             logger.info("{}: range {} exists on {} for keyspace {}", description, entry.getKey(), entry.getValue(), keyspaceName);
 
         AbstractReplicationStrategy strat = Keyspace.open(keyspaceName).getReplicationStrategy();
-        Multimap<InetAddress, Range<Token>> rangeFetchMap = useStrictSource || strat == null || strat.getReplicationFactor() == 1
+        Multimap<InetAddressAndPort, Range<Token>> rangeFetchMap = useStrictSource || strat == null || strat.getReplicationFactor() == 1
                                                             ? getRangeFetchMap(rangesForKeyspace, sourceFilters, keyspaceName, useStrictConsistency)
                                                             : getOptimizedRangeFetchMap(rangesForKeyspace, sourceFilters, keyspaceName);
 
-        for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : rangeFetchMap.asMap().entrySet())
+        for (Map.Entry<InetAddressAndPort, Collection<Range<Token>>> entry : rangeFetchMap.asMap().entrySet())
         {
             if (logger.isTraceEnabled())
             {
@@ -226,19 +226,19 @@ public class RangeStreamer
      *
      * @throws java.lang.IllegalStateException when there is no source to get data streamed
      */
-    private Multimap<Range<Token>, InetAddress> getAllRangesWithSourcesFor(String keyspaceName, Collection<Range<Token>> desiredRanges)
+    private Multimap<Range<Token>, InetAddressAndPort> getAllRangesWithSourcesFor(String keyspaceName, Collection<Range<Token>> desiredRanges)
     {
         AbstractReplicationStrategy strat = Keyspace.open(keyspaceName).getReplicationStrategy();
-        Multimap<Range<Token>, InetAddress> rangeAddresses = strat.getRangeAddresses(metadata.cloneOnlyTokenMap());
+        Multimap<Range<Token>, InetAddressAndPort> rangeAddresses = strat.getRangeAddresses(metadata.cloneOnlyTokenMap());
 
-        Multimap<Range<Token>, InetAddress> rangeSources = ArrayListMultimap.create();
+        Multimap<Range<Token>, InetAddressAndPort> rangeSources = ArrayListMultimap.create();
         for (Range<Token> desiredRange : desiredRanges)
         {
             for (Range<Token> range : rangeAddresses.keySet())
             {
                 if (range.contains(desiredRange))
                 {
-                    List<InetAddress> preferred = snitch.getSortedListByProximity(address, rangeAddresses.get(range));
+                    List<InetAddressAndPort> preferred = snitch.getSortedListByProximity(address, rangeAddresses.get(range));
                     rangeSources.putAll(desiredRange, preferred);
                     break;
                 }
@@ -258,30 +258,30 @@ public class RangeStreamer
      *
      * @throws java.lang.IllegalStateException when there is no source to get data streamed, or more than 1 source found.
      */
-    private Multimap<Range<Token>, InetAddress> getAllRangesWithStrictSourcesFor(String keyspace, Collection<Range<Token>> desiredRanges)
+    private Multimap<Range<Token>, InetAddressAndPort> getAllRangesWithStrictSourcesFor(String keyspace, Collection<Range<Token>> desiredRanges)
     {
         assert tokens != null;
         AbstractReplicationStrategy strat = Keyspace.open(keyspace).getReplicationStrategy();
 
         // Active ranges
         TokenMetadata metadataClone = metadata.cloneOnlyTokenMap();
-        Multimap<Range<Token>, InetAddress> addressRanges = strat.getRangeAddresses(metadataClone);
+        Multimap<Range<Token>, InetAddressAndPort> addressRanges = strat.getRangeAddresses(metadataClone);
 
         // Pending ranges
         metadataClone.updateNormalTokens(tokens, address);
-        Multimap<Range<Token>, InetAddress> pendingRangeAddresses = strat.getRangeAddresses(metadataClone);
+        Multimap<Range<Token>, InetAddressAndPort> pendingRangeAddresses = strat.getRangeAddresses(metadataClone);
 
         // Collects the source that will have its range moved to the new node
-        Multimap<Range<Token>, InetAddress> rangeSources = ArrayListMultimap.create();
+        Multimap<Range<Token>, InetAddressAndPort> rangeSources = ArrayListMultimap.create();
 
         for (Range<Token> desiredRange : desiredRanges)
         {
-            for (Map.Entry<Range<Token>, Collection<InetAddress>> preEntry : addressRanges.asMap().entrySet())
+            for (Map.Entry<Range<Token>, Collection<InetAddressAndPort>> preEntry : addressRanges.asMap().entrySet())
             {
                 if (preEntry.getKey().contains(desiredRange))
                 {
-                    Set<InetAddress> oldEndpoints = Sets.newHashSet(preEntry.getValue());
-                    Set<InetAddress> newEndpoints = Sets.newHashSet(pendingRangeAddresses.get(desiredRange));
+                    Set<InetAddressAndPort> oldEndpoints = Sets.newHashSet(preEntry.getValue());
+                    Set<InetAddressAndPort> newEndpoints = Sets.newHashSet(pendingRangeAddresses.get(desiredRange));
 
                     // Due to CASSANDRA-5953 we can have a higher RF then we have endpoints.
                     // So we need to be careful to only be strict when endpoints == RF
@@ -296,14 +296,14 @@ public class RangeStreamer
             }
 
             // Validate
-            Collection<InetAddress> addressList = rangeSources.get(desiredRange);
+            Collection<InetAddressAndPort> addressList = rangeSources.get(desiredRange);
             if (addressList == null || addressList.isEmpty())
                 throw new IllegalStateException("No sources found for " + desiredRange);
 
             if (addressList.size() > 1)
                 throw new IllegalStateException("Multiple endpoints found for " + desiredRange);
 
-            InetAddress sourceIp = addressList.iterator().next();
+            InetAddressAndPort sourceIp = addressList.iterator().next();
             EndpointState sourceState = Gossiper.instance.getEndpointStateForEndpoint(sourceIp);
             if (Gossiper.instance.isEnabled() && (sourceState == null || !sourceState.isAlive()))
                 throw new RuntimeException("A node required to move the data consistently is down (" + sourceIp + "). " +
@@ -320,17 +320,17 @@ public class RangeStreamer
      * @param keyspace keyspace name
      * @return Map of source endpoint to collection of ranges
      */
-    private static Multimap<InetAddress, Range<Token>> getRangeFetchMap(Multimap<Range<Token>, InetAddress> rangesWithSources,
-                                                                        Collection<ISourceFilter> sourceFilters, String keyspace,
-                                                                        boolean useStrictConsistency)
+    private static Multimap<InetAddressAndPort, Range<Token>> getRangeFetchMap(Multimap<Range<Token>, InetAddressAndPort> rangesWithSources,
+                                                                               Collection<ISourceFilter> sourceFilters, String keyspace,
+                                                                               boolean useStrictConsistency)
     {
-        Multimap<InetAddress, Range<Token>> rangeFetchMapMap = HashMultimap.create();
+        Multimap<InetAddressAndPort, Range<Token>> rangeFetchMapMap = HashMultimap.create();
         for (Range<Token> range : rangesWithSources.keySet())
         {
             boolean foundSource = false;
 
             outer:
-            for (InetAddress address : rangesWithSources.get(range))
+            for (InetAddressAndPort address : rangesWithSources.get(range))
             {
                 for (ISourceFilter filter : sourceFilters)
                 {
@@ -338,7 +338,7 @@ public class RangeStreamer
                         continue outer;
                 }
 
-                if (address.equals(FBUtilities.getBroadcastAddress()))
+                if (address.equals(FBUtilities.getBroadcastAddressAndPort()))
                 {
                     // If localhost is a source, we have found one, but we don't add it to the map to avoid streaming locally
                     foundSource = true;
@@ -371,11 +371,11 @@ public class RangeStreamer
     }
 
 
-    private static Multimap<InetAddress, Range<Token>> getOptimizedRangeFetchMap(Multimap<Range<Token>, InetAddress> rangesWithSources,
-                                                                        Collection<ISourceFilter> sourceFilters, String keyspace)
+    private static Multimap<InetAddressAndPort, Range<Token>> getOptimizedRangeFetchMap(Multimap<Range<Token>, InetAddressAndPort> rangesWithSources,
+                                                                                        Collection<ISourceFilter> sourceFilters, String keyspace)
     {
         RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, sourceFilters, keyspace);
-        Multimap<InetAddress, Range<Token>> rangeFetchMapMap = calculator.getRangeFetchMap();
+        Multimap<InetAddressAndPort, Range<Token>> rangeFetchMapMap = calculator.getRangeFetchMap();
         logger.info("Output from RangeFetchMapCalculator for keyspace {}", keyspace);
         validateRangeFetchMap(rangesWithSources, rangeFetchMapMap, keyspace);
         return rangeFetchMapMap;
@@ -387,11 +387,11 @@ public class RangeStreamer
      * @param rangeFetchMapMap
      * @param keyspace
      */
-    private static void validateRangeFetchMap(Multimap<Range<Token>, InetAddress> rangesWithSources, Multimap<InetAddress, Range<Token>> rangeFetchMapMap, String keyspace)
+    private static void validateRangeFetchMap(Multimap<Range<Token>, InetAddressAndPort> rangesWithSources, Multimap<InetAddressAndPort, Range<Token>> rangeFetchMapMap, String keyspace)
     {
-        for (Map.Entry<InetAddress, Range<Token>> entry : rangeFetchMapMap.entries())
+        for (Map.Entry<InetAddressAndPort, Range<Token>> entry : rangeFetchMapMap.entries())
         {
-            if(entry.getKey().equals(FBUtilities.getBroadcastAddress()))
+            if(entry.getKey().equals(FBUtilities.getBroadcastAddressAndPort()))
             {
                 throw new IllegalStateException("Trying to stream locally. Range: " + entry.getValue()
                                         + " in keyspace " + keyspace);
@@ -407,26 +407,26 @@ public class RangeStreamer
         }
     }
 
-    public static Multimap<InetAddress, Range<Token>> getWorkMap(Multimap<Range<Token>, InetAddress> rangesWithSourceTarget, String keyspace,
-                                                                 IFailureDetector fd, boolean useStrictConsistency)
+    public static Multimap<InetAddressAndPort, Range<Token>> getWorkMap(Multimap<Range<Token>, InetAddressAndPort> rangesWithSourceTarget, String keyspace,
+                                                                        IFailureDetector fd, boolean useStrictConsistency)
     {
         return getRangeFetchMap(rangesWithSourceTarget, Collections.<ISourceFilter>singleton(new FailureDetectorSourceFilter(fd)), keyspace, useStrictConsistency);
     }
 
     // For testing purposes
     @VisibleForTesting
-    Multimap<String, Map.Entry<InetAddress, Collection<Range<Token>>>> toFetch()
+    Multimap<String, Map.Entry<InetAddressAndPort, Collection<Range<Token>>>> toFetch()
     {
         return toFetch;
     }
 
     public StreamResultFuture fetchAsync()
     {
-        for (Map.Entry<String, Map.Entry<InetAddress, Collection<Range<Token>>>> entry : toFetch.entries())
+        for (Map.Entry<String, Map.Entry<InetAddressAndPort, Collection<Range<Token>>>> entry : toFetch.entries())
         {
             String keyspace = entry.getKey();
-            InetAddress source = entry.getValue().getKey();
-            InetAddress preferred = SystemKeyspace.getPreferredIP(source);
+            InetAddressAndPort source = entry.getValue().getKey();
+            InetAddressAndPort preferred = SystemKeyspace.getPreferredIP(source);
             Collection<Range<Token>> ranges = entry.getValue().getValue();
 
             // filter out already streamed ranges

src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.dht.tokenallocator;
 
-import java.net.InetAddress;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -37,6 +36,7 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.NetworkTopologyStrategy;
 import org.apache.cassandra.locator.SimpleStrategy;
 import org.apache.cassandra.locator.TokenMetadata;
@@ -48,7 +48,7 @@ public class TokenAllocation
 
     public static Collection<Token> allocateTokens(final TokenMetadata tokenMetadata,
                                                    final AbstractReplicationStrategy rs,
-                                                   final InetAddress endpoint,
+                                                   final InetAddressAndPort endpoint,
                                                    int numTokens)
     {
         TokenMetadata tokenMetadataCopy = tokenMetadata.cloneOnlyTokenMap();
@@ -81,7 +81,7 @@ public class TokenAllocation
         {
             while (tokenMetadata.getEndpoint(t) != null)
             {
-                InetAddress other = tokenMetadata.getEndpoint(t);
+                InetAddressAndPort other = tokenMetadata.getEndpoint(t);
                 if (strategy.inAllocationRing(other))
                     throw new ConfigurationException(String.format("Allocated token %s already assigned to node %s. Is another node also allocating tokens?", t, other));
                 t = t.increaseSlightly();
@@ -92,9 +92,9 @@ public class TokenAllocation
     }
 
     // return the ratio of ownership for each endpoint
-    public static Map<InetAddress, Double> evaluateReplicatedOwnership(TokenMetadata tokenMetadata, AbstractReplicationStrategy rs)
+    public static Map<InetAddressAndPort, Double> evaluateReplicatedOwnership(TokenMetadata tokenMetadata, AbstractReplicationStrategy rs)
     {
-        Map<InetAddress, Double> ownership = Maps.newHashMap();
+        Map<InetAddressAndPort, Double> ownership = Maps.newHashMap();
         List<Token> sortedTokens = tokenMetadata.sortedTokens();
         Iterator<Token> it = sortedTokens.iterator();
         Token current = it.next();
@@ -109,11 +109,11 @@ public class TokenAllocation
         return ownership;
     }
 
-    static void addOwnership(final TokenMetadata tokenMetadata, final AbstractReplicationStrategy rs, Token current, Token next, Map<InetAddress, Double> ownership)
+    static void addOwnership(final TokenMetadata tokenMetadata, final AbstractReplicationStrategy rs, Token current, Token next, Map<InetAddressAndPort, Double> ownership)
     {
         double size = current.size(next);
         Token representative = current.getPartitioner().midpoint(current, next);
-        for (InetAddress n : rs.calculateNaturalEndpoints(representative, tokenMetadata))
+        for (InetAddressAndPort n : rs.calculateNaturalEndpoints(representative, tokenMetadata))
         {
             Double v = ownership.get(n);
             ownership.put(n, v != null ? v + size : size);
@@ -126,11 +126,11 @@ public class TokenAllocation
     }
 
     public static SummaryStatistics replicatedOwnershipStats(TokenMetadata tokenMetadata,
-                                                             AbstractReplicationStrategy rs, InetAddress endpoint)
+                                                             AbstractReplicationStrategy rs, InetAddressAndPort endpoint)
     {
         SummaryStatistics stat = new SummaryStatistics();
         StrategyAdapter strategy = getStrategy(tokenMetadata, rs, endpoint);
-        for (Map.Entry<InetAddress, Double> en : evaluateReplicatedOwnership(tokenMetadata, rs).entrySet())
+        for (Map.Entry<InetAddressAndPort, Double> en : evaluateReplicatedOwnership(tokenMetadata, rs).entrySet())
         {
             // Filter only in the same datacentre.
             if (strategy.inAllocationRing(en.getKey()))
@@ -139,10 +139,10 @@ public class TokenAllocation
         return stat;
     }
 
-    static TokenAllocator<InetAddress> create(TokenMetadata tokenMetadata, StrategyAdapter strategy)
+    static TokenAllocator<InetAddressAndPort> create(TokenMetadata tokenMetadata, StrategyAdapter strategy)
     {
-        NavigableMap<Token, InetAddress> sortedTokens = new TreeMap<>();
-        for (Map.Entry<Token, InetAddress> en : tokenMetadata.getNormalAndBootstrappingTokenToEndpointMap().entrySet())
+        NavigableMap<Token, InetAddressAndPort> sortedTokens = new TreeMap<>();
+        for (Map.Entry<Token, InetAddressAndPort> en : tokenMetadata.getNormalAndBootstrappingTokenToEndpointMap().entrySet())
         {
             if (strategy.inAllocationRing(en.getValue()))
                 sortedTokens.put(en.getKey(), en.getValue());
@@ -150,15 +150,15 @@ public class TokenAllocation
         return TokenAllocatorFactory.createTokenAllocator(sortedTokens, strategy, tokenMetadata.partitioner);
     }
 
-    interface StrategyAdapter extends ReplicationStrategy<InetAddress>
+    interface StrategyAdapter extends ReplicationStrategy<InetAddressAndPort>
     {
         // return true iff the provided endpoint occurs in the same virtual token-ring we are allocating for
         // i.e. the set of the nodes that share ownership with the node we are allocating
         // alternatively: return false if the endpoint's ownership is independent of the node we are allocating tokens for
-        boolean inAllocationRing(InetAddress other);
+        boolean inAllocationRing(InetAddressAndPort other);
     }
 
-    static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final AbstractReplicationStrategy rs, final InetAddress endpoint)
+    static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final AbstractReplicationStrategy rs, final InetAddressAndPort endpoint)
     {
         if (rs instanceof NetworkTopologyStrategy)
             return getStrategy(tokenMetadata, (NetworkTopologyStrategy) rs, rs.snitch, endpoint);
@@ -167,7 +167,7 @@ public class TokenAllocation
         throw new ConfigurationException("Token allocation does not support replication strategy " + rs.getClass().getSimpleName());
     }
 
-    static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final SimpleStrategy rs, final InetAddress endpoint)
+    static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final SimpleStrategy rs, final InetAddressAndPort endpoint)
     {
         final int replicas = rs.getReplicationFactor();
 
@@ -180,20 +180,20 @@ public class TokenAllocation
             }
 
             @Override
-            public Object getGroup(InetAddress unit)
+            public Object getGroup(InetAddressAndPort unit)
             {
                 return unit;
             }
 
             @Override
-            public boolean inAllocationRing(InetAddress other)
+            public boolean inAllocationRing(InetAddressAndPort other)
             {
                 return true;
             }
         };
     }
 
-    static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final NetworkTopologyStrategy rs, final IEndpointSnitch snitch, final InetAddress endpoint)
+    static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final NetworkTopologyStrategy rs, final IEndpointSnitch snitch, final InetAddressAndPort endpoint)
     {
         final String dc = snitch.getDatacenter(endpoint);
         final int replicas = rs.getReplicationFactor(dc);
@@ -210,13 +210,13 @@ public class TokenAllocation
                 }
 
                 @Override
-                public Object getGroup(InetAddress unit)
+                public Object getGroup(InetAddressAndPort unit)
                 {
                     return unit;
                 }
 
                 @Override
-                public boolean inAllocationRing(InetAddress other)
+                public boolean inAllocationRing(InetAddressAndPort other)
                 {
                     return dc.equals(snitch.getDatacenter(other));
                 }
@@ -237,13 +237,13 @@ public class TokenAllocation
                 }
 
                 @Override
-                public Object getGroup(InetAddress unit)
+                public Object getGroup(InetAddressAndPort unit)
                 {
                     return snitch.getRack(unit);
                 }
 
                 @Override
-                public boolean inAllocationRing(InetAddress other)
+                public boolean inAllocationRing(InetAddressAndPort other)
                 {
                     return dc.equals(snitch.getDatacenter(other));
                 }
@@ -261,13 +261,13 @@ public class TokenAllocation
                 }
 
                 @Override
-                public Object getGroup(InetAddress unit)
+                public Object getGroup(InetAddressAndPort unit)
                 {
                     return unit;
                 }
 
                 @Override
-                public boolean inAllocationRing(InetAddress other)
+                public boolean inAllocationRing(InetAddressAndPort other)
                 {
                     return dc.equals(snitch.getDatacenter(other));
                 }

src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocatorFactory.java

@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.dht.tokenallocator;
 
-import java.net.InetAddress;
 import java.util.NavigableMap;
 
 import org.slf4j.Logger;
@@ -26,13 +25,14 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 public class TokenAllocatorFactory
 {
     private static final Logger logger = LoggerFactory.getLogger(TokenAllocatorFactory.class);
-    public static TokenAllocator<InetAddress> createTokenAllocator(NavigableMap<Token, InetAddress> sortedTokens,
-                                                     ReplicationStrategy<InetAddress> strategy,
-                                                     IPartitioner partitioner)
+    public static TokenAllocator<InetAddressAndPort> createTokenAllocator(NavigableMap<Token, InetAddressAndPort> sortedTokens,
+                                                                          ReplicationStrategy<InetAddressAndPort> strategy,
+                                                                          IPartitioner partitioner)
     {
         if(strategy.replicas() == 1)
         {

src/java/org/apache/cassandra/exceptions/ReadFailureException.java

@@ -17,16 +17,16 @@
  */
 package org.apache.cassandra.exceptions;
 
-import java.net.InetAddress;
 import java.util.Map;
 
 import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 public class ReadFailureException extends RequestFailureException
 {
     public final boolean dataPresent;
 
-    public ReadFailureException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent, Map<InetAddress, RequestFailureReason> failureReasonByEndpoint)
+    public ReadFailureException(ConsistencyLevel consistency, int received, int blockFor, boolean dataPresent, Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint)
     {
         super(ExceptionCode.READ_FAILURE, consistency, received, blockFor, failureReasonByEndpoint);
         this.dataPresent = dataPresent;

src/java/org/apache/cassandra/exceptions/RequestFailureException.java

@@ -17,20 +17,20 @@
  */
 package org.apache.cassandra.exceptions;
 
-import java.net.InetAddress;
 import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 public class RequestFailureException extends RequestExecutionException
 {
     public final ConsistencyLevel consistency;
     public final int received;
     public final int blockFor;
-    public final Map<InetAddress, RequestFailureReason> failureReasonByEndpoint;
+    public final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint;
 
-    protected RequestFailureException(ExceptionCode code, ConsistencyLevel consistency, int received, int blockFor, Map<InetAddress, RequestFailureReason> failureReasonByEndpoint)
+    protected RequestFailureException(ExceptionCode code, ConsistencyLevel consistency, int received, int blockFor, Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint)
     {
         super(code, String.format("Operation failed - received %d responses and %d failures", received, failureReasonByEndpoint.size()));
         this.consistency = consistency;

src/java/org/apache/cassandra/exceptions/WriteFailureException.java

@@ -17,17 +17,17 @@
  */
 package org.apache.cassandra.exceptions;
 
-import java.net.InetAddress;
 import java.util.Map;
 
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 public class WriteFailureException extends RequestFailureException
 {
     public final WriteType writeType;
 
-    public WriteFailureException(ConsistencyLevel consistency, int received, int blockFor, WriteType writeType, Map<InetAddress, RequestFailureReason> failureReasonByEndpoint)
+    public WriteFailureException(ConsistencyLevel consistency, int received, int blockFor, WriteType writeType, Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint)
     {
         super(ExceptionCode.WRITE_FAILURE, consistency, received, blockFor, failureReasonByEndpoint);
         this.writeType = writeType;

src/java/org/apache/cassandra/gms/ApplicationState.java

@@ -19,15 +19,15 @@ package org.apache.cassandra.gms;
 
 public enum ApplicationState
 {
-    STATUS,
+    @Deprecated STATUS, //Deprecated and unsued in 4.0, stop publishing in 5.0, reclaim in 6.0
     LOAD,
     SCHEMA,
     DC,
     RACK,
     RELEASE_VERSION,
     REMOVAL_COORDINATOR,
-    INTERNAL_IP,
-    RPC_ADDRESS,
+    @Deprecated INTERNAL_IP, //Deprecated and unused in 4.0, stop publishing in 5.0, reclaim in 6.0
+    @Deprecated RPC_ADDRESS, // ^ Same
     X_11_PADDING, // padding specifically for 1.1
     SEVERITY,
     NET_VERSION,
@@ -35,8 +35,9 @@ public enum ApplicationState
     TOKENS,
     RPC_READY,
     // pad to allow adding new states to existing cluster
-    X1,
-    X2,
+    INTERNAL_ADDRESS_AND_PORT, //Replacement for INTERNAL_IP with up to two ports
+    NATIVE_ADDRESS_AND_PORT, //Replacement for RPC_ADDRESS
+    STATUS_WITH_PORT, //Replacement for STATUS
     X3,
     X4,
     X5,

src/java/org/apache/cassandra/gms/EndpointState.java

@@ -146,9 +146,15 @@ public class EndpointState
 
     public String getStatus()
     {
-        VersionedValue status = getApplicationState(ApplicationState.STATUS);
+        VersionedValue status = getApplicationState(ApplicationState.STATUS_WITH_PORT);
         if (status == null)
+        {
+            status = getApplicationState(ApplicationState.STATUS);
+        }
+        if (status == null)
+        {
             return "";
+        }
 
         String[] pieces = status.value.split(VersionedValue.DELIMITER_STR, -1);
         assert (pieces.length > 0);

src/java/org/apache/cassandra/gms/FailureDetector.java

@@ -22,7 +22,6 @@ import java.nio.file.StandardOpenOption;
 import java.nio.file.Path;
 import java.io.*;
 import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
@@ -38,7 +37,7 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -79,7 +78,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
     // change.
     private final double PHI_FACTOR = 1.0 / Math.log(10.0); // 0.434...
 
-    private final ConcurrentHashMap<InetAddress, ArrivalWindow> arrivalSamples = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<InetAddressAndPort, ArrivalWindow> arrivalSamples = new ConcurrentHashMap<>();
     private final List<IFailureDetectionEventListener> fdEvntListeners = new CopyOnWriteArrayList<>();
 
     public FailureDetector()
@@ -111,25 +110,45 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
     }
 
     public String getAllEndpointStates()
+    {
+        return getAllEndpointStates(false);
+    }
+
+    public String getAllEndpointStatesWithPort()
+    {
+        return getAllEndpointStates(true);
+    }
+
+    public String getAllEndpointStates(boolean withPort)
     {
         StringBuilder sb = new StringBuilder();
-        for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
+        for (Map.Entry<InetAddressAndPort, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
         {
-            sb.append(entry.getKey()).append("\n");
+            sb.append(entry.getKey().toString(withPort)).append("\n");
             appendEndpointState(sb, entry.getValue());
         }
         return sb.toString();
     }
 
     public Map<String, String> getSimpleStates()
+    {
+        return getSimpleStates(false);
+    }
+
+    public Map<String, String> getSimpleStatesWithPort()
+    {
+        return getSimpleStates(true);
+    }
+
+    private Map<String, String> getSimpleStates(boolean withPort)
     {
         Map<String, String> nodesStatus = new HashMap<String, String>(Gossiper.instance.endpointStateMap.size());
-        for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
+        for (Map.Entry<InetAddressAndPort, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
         {
             if (entry.getValue().isAlive())
-                nodesStatus.put(entry.getKey().toString(), "UP");
+                nodesStatus.put(entry.getKey().toString(withPort), "UP");
             else
-                nodesStatus.put(entry.getKey().toString(), "DOWN");
+                nodesStatus.put(entry.getKey().toString(withPort), "DOWN");
         }
         return nodesStatus;
     }
@@ -137,7 +156,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
     public int getDownEndpointCount()
     {
         int count = 0;
-        for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
+        for (Map.Entry<InetAddressAndPort, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
         {
             if (!entry.getValue().isAlive())
                 count++;
@@ -148,7 +167,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
     public int getUpEndpointCount()
     {
         int count = 0;
-        for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
+        for (Map.Entry<InetAddressAndPort, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet())
         {
             if (entry.getValue().isAlive())
                 count++;
@@ -158,6 +177,17 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
 
     @Override
     public TabularData getPhiValues() throws OpenDataException
+    {
+        return getPhiValues(false);
+    }
+
+    @Override
+    public TabularData getPhiValuesWithPort() throws OpenDataException
+    {
+        return getPhiValues(true);
+    }
+
+    private TabularData getPhiValues(boolean withPort) throws OpenDataException
     {
         final CompositeType ct = new CompositeType("Node", "Node",
                 new String[]{"Endpoint", "PHI"},
@@ -165,7 +195,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
                 new OpenType[]{SimpleType.STRING, SimpleType.DOUBLE});
         final TabularDataSupport results = new TabularDataSupport(new TabularType("PhiList", "PhiList", ct, new String[]{"Endpoint"}));
 
-        for (final Map.Entry<InetAddress, ArrivalWindow> entry : arrivalSamples.entrySet())
+        for (final Map.Entry<InetAddressAndPort, ArrivalWindow> entry : arrivalSamples.entrySet())
         {
             final ArrivalWindow window = entry.getValue();
             if (window.mean() > 0)
@@ -176,7 +206,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
                     // returned values are scaled by PHI_FACTOR so that the are on the same scale as PhiConvictThreshold
                     final CompositeData data = new CompositeDataSupport(ct,
                             new String[]{"Endpoint", "PHI"},
-                            new Object[]{entry.getKey().toString(), phi * PHI_FACTOR});
+                            new Object[]{entry.getKey().toString(withPort), phi * PHI_FACTOR});
                     results.put(data);
                 }
             }
@@ -187,7 +217,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
     public String getEndpointState(String address) throws UnknownHostException
     {
         StringBuilder sb = new StringBuilder();
-        EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(InetAddress.getByName(address));
+        EndpointState endpointState = Gossiper.instance.getEndpointStateForEndpoint(InetAddressAndPort.getByName(address));
         appendEndpointState(sb, endpointState);
         return sb.toString();
     }
@@ -243,9 +273,9 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
         return DatabaseDescriptor.getPhiConvictThreshold();
     }
 
-    public boolean isAlive(InetAddress ep)
+    public boolean isAlive(InetAddressAndPort ep)
     {
-        if (ep.equals(FBUtilities.getBroadcastAddress()))
+        if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
             return true;
 
         EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(ep);
@@ -257,7 +287,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
         return epState != null && epState.isAlive();
     }
 
-    public void report(InetAddress ep)
+    public void report(InetAddressAndPort ep)
     {
         long now = Clock.instance.nanoTime();
         ArrivalWindow heartbeatWindow = arrivalSamples.get(ep);
@@ -279,7 +309,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
             logger.trace("Average for {} is {}ns", ep, heartbeatWindow.mean());
     }
 
-    public void interpret(InetAddress ep)
+    public void interpret(InetAddressAndPort ep)
     {
         ArrivalWindow hbWnd = arrivalSamples.get(ep);
         if (hbWnd == null)
@@ -324,7 +354,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
         }
     }
 
-    public void forceConviction(InetAddress ep)
+    public void forceConviction(InetAddressAndPort ep)
     {
         logger.debug("Forcing conviction of {}", ep);
         for (IFailureDetectionEventListener listener : fdEvntListeners)
@@ -333,7 +363,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
         }
     }
 
-    public void remove(InetAddress ep)
+    public void remove(InetAddressAndPort ep)
     {
         arrivalSamples.remove(ep);
     }
@@ -351,10 +381,10 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
     public String toString()
     {
         StringBuilder sb = new StringBuilder();
-        Set<InetAddress> eps = arrivalSamples.keySet();
+        Set<InetAddressAndPort> eps = arrivalSamples.keySet();
 
         sb.append("-----------------------------------------------------------------------");
-        for (InetAddress ep : eps)
+        for (InetAddressAndPort ep : eps)
         {
             ArrivalWindow hWnd = arrivalSamples.get(ep);
             sb.append(ep).append(" : ");
@@ -447,7 +477,7 @@ class ArrivalWindow
         }
     }
 
-    synchronized void add(long value, InetAddress ep)
+    synchronized void add(long value, InetAddressAndPort ep)
     {
         assert tLast >= 0;
         if (tLast > 0L)

src/java/org/apache/cassandra/gms/FailureDetectorMBean.java

@@ -31,15 +31,18 @@ public interface FailureDetectorMBean
 
     public double getPhiConvictThreshold();
 
-    public String getAllEndpointStates();
+    @Deprecated public String getAllEndpointStates();
+    public String getAllEndpointStatesWithPort();
 
     public String getEndpointState(String address) throws UnknownHostException;
 
-    public Map<String, String> getSimpleStates();
+    @Deprecated public Map<String, String> getSimpleStates();
+    public Map<String, String> getSimpleStatesWithPort();
 
     public int getDownEndpointCount();
 
     public int getUpEndpointCount();
 
-    public TabularData getPhiValues() throws OpenDataException;
+    @Deprecated public TabularData getPhiValues() throws OpenDataException;
+    public TabularData getPhiValuesWithPort() throws OpenDataException;
 }

src/java/org/apache/cassandra/gms/GossipDigest.java

@@ -18,12 +18,12 @@
 package org.apache.cassandra.gms;
 
 import java.io.*;
-import java.net.InetAddress;
 
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.CompactEndpointSerializationHelper;
 
 /**
@@ -34,18 +34,18 @@ public class GossipDigest implements Comparable<GossipDigest>
 {
     public static final IVersionedSerializer<GossipDigest> serializer = new GossipDigestSerializer();
 
-    final InetAddress endpoint;
+    final InetAddressAndPort endpoint;
     final int generation;
     final int maxVersion;
 
-    GossipDigest(InetAddress ep, int gen, int version)
+    GossipDigest(InetAddressAndPort ep, int gen, int version)
     {
         endpoint = ep;
         generation = gen;
         maxVersion = version;
     }
 
-    InetAddress getEndpoint()
+    InetAddressAndPort getEndpoint()
     {
         return endpoint;
     }
@@ -83,14 +83,14 @@ class GossipDigestSerializer implements IVersionedSerializer<GossipDigest>
 {
     public void serialize(GossipDigest gDigest, DataOutputPlus out, int version) throws IOException
     {
-        CompactEndpointSerializationHelper.serialize(gDigest.endpoint, out);
+        CompactEndpointSerializationHelper.instance.serialize(gDigest.endpoint, out, version);
         out.writeInt(gDigest.generation);
         out.writeInt(gDigest.maxVersion);
     }
 
     public GossipDigest deserialize(DataInputPlus in, int version) throws IOException
     {
-        InetAddress endpoint = CompactEndpointSerializationHelper.deserialize(in);
+        InetAddressAndPort endpoint = CompactEndpointSerializationHelper.instance.deserialize(in, version);
         int generation = in.readInt();
         int maxVersion = in.readInt();
         return new GossipDigest(endpoint, generation, maxVersion);
@@ -98,7 +98,7 @@ class GossipDigestSerializer implements IVersionedSerializer<GossipDigest>
 
     public long serializedSize(GossipDigest gDigest, int version)
     {
-        long size = CompactEndpointSerializationHelper.serializedSize(gDigest.endpoint);
+        long size = CompactEndpointSerializationHelper.instance.serializedSize(gDigest.endpoint, version);
         size += TypeSizes.sizeof(gDigest.generation);
         size += TypeSizes.sizeof(gDigest.maxVersion);
         return size;

src/java/org/apache/cassandra/gms/GossipDigestAck.java

@@ -18,7 +18,6 @@
 package org.apache.cassandra.gms;
 
 import java.io.IOException;
-import java.net.InetAddress;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -27,6 +26,7 @@ import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.CompactEndpointSerializationHelper;
 
 /**
@@ -38,9 +38,9 @@ public class GossipDigestAck
     public static final IVersionedSerializer<GossipDigestAck> serializer = new GossipDigestAckSerializer();
 
     final List<GossipDigest> gDigestList;
-    final Map<InetAddress, EndpointState> epStateMap;
+    final Map<InetAddressAndPort, EndpointState> epStateMap;
 
-    GossipDigestAck(List<GossipDigest> gDigestList, Map<InetAddress, EndpointState> epStateMap)
+    GossipDigestAck(List<GossipDigest> gDigestList, Map<InetAddressAndPort, EndpointState> epStateMap)
     {
         this.gDigestList = gDigestList;
         this.epStateMap = epStateMap;
@@ -51,7 +51,7 @@ public class GossipDigestAck
         return gDigestList;
     }
 
-    Map<InetAddress, EndpointState> getEndpointStateMap()
+    Map<InetAddressAndPort, EndpointState> getEndpointStateMap()
     {
         return epStateMap;
     }
@@ -63,10 +63,10 @@ class GossipDigestAckSerializer implements IVersionedSerializer<GossipDigestAck>
     {
         GossipDigestSerializationHelper.serialize(gDigestAckMessage.gDigestList, out, version);
         out.writeInt(gDigestAckMessage.epStateMap.size());
-        for (Map.Entry<InetAddress, EndpointState> entry : gDigestAckMessage.epStateMap.entrySet())
+        for (Map.Entry<InetAddressAndPort, EndpointState> entry : gDigestAckMessage.epStateMap.entrySet())
         {
-            InetAddress ep = entry.getKey();
-            CompactEndpointSerializationHelper.serialize(ep, out);
+            InetAddressAndPort ep = entry.getKey();
+            CompactEndpointSerializationHelper.instance.serialize(ep, out, version);
             EndpointState.serializer.serialize(entry.getValue(), out, version);
         }
     }
@@ -75,11 +75,11 @@ class GossipDigestAckSerializer implements IVersionedSerializer<GossipDigestAck>
     {
         List<GossipDigest> gDigestList = GossipDigestSerializationHelper.deserialize(in, version);
         int size = in.readInt();
-        Map<InetAddress, EndpointState> epStateMap = new HashMap<InetAddress, EndpointState>(size);
+        Map<InetAddressAndPort, EndpointState> epStateMap = new HashMap<InetAddressAndPort, EndpointState>(size);
 
         for (int i = 0; i < size; ++i)
         {
-            InetAddress ep = CompactEndpointSerializationHelper.deserialize(in);
+            InetAddressAndPort ep = CompactEndpointSerializationHelper.instance.deserialize(in, version);
             EndpointState epState = EndpointState.serializer.deserialize(in, version);
             epStateMap.put(ep, epState);
         }
@@ -90,8 +90,8 @@ class GossipDigestAckSerializer implements IVersionedSerializer<GossipDigestAck>
     {
         int size = GossipDigestSerializationHelper.serializedSize(ack.gDigestList, version);
         size += TypeSizes.sizeof(ack.epStateMap.size());
-        for (Map.Entry<InetAddress, EndpointState> entry : ack.epStateMap.entrySet())
-            size += CompactEndpointSerializationHelper.serializedSize(entry.getKey())
+        for (Map.Entry<InetAddressAndPort, EndpointState> entry : ack.epStateMap.entrySet())
+            size += CompactEndpointSerializationHelper.instance.serializedSize(entry.getKey(), version)
                     + EndpointState.serializer.serializedSize(entry.getValue(), version);
         return size;
     }

src/java/org/apache/cassandra/gms/GossipDigestAck2.java

@@ -18,7 +18,6 @@
 package org.apache.cassandra.gms;
 
 import java.io.*;
-import java.net.InetAddress;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -26,6 +25,7 @@ import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.CompactEndpointSerializationHelper;
 
 /**
@@ -36,14 +36,14 @@ public class GossipDigestAck2
 {
     public static final IVersionedSerializer<GossipDigestAck2> serializer = new GossipDigestAck2Serializer();
 
-    final Map<InetAddress, EndpointState> epStateMap;
+    final Map<InetAddressAndPort, EndpointState> epStateMap;
 
-    GossipDigestAck2(Map<InetAddress, EndpointState> epStateMap)
+    GossipDigestAck2(Map<InetAddressAndPort, EndpointState> epStateMap)
     {
         this.epStateMap = epStateMap;
     }
 
-    Map<InetAddress, EndpointState> getEndpointStateMap()
+    Map<InetAddressAndPort, EndpointState> getEndpointStateMap()
     {
         return epStateMap;
     }
@@ -54,10 +54,10 @@ class GossipDigestAck2Serializer implements IVersionedSerializer<GossipDigestAck
     public void serialize(GossipDigestAck2 ack2, DataOutputPlus out, int version) throws IOException
     {
         out.writeInt(ack2.epStateMap.size());
-        for (Map.Entry<InetAddress, EndpointState> entry : ack2.epStateMap.entrySet())
+        for (Map.Entry<InetAddressAndPort, EndpointState> entry : ack2.epStateMap.entrySet())
         {
-            InetAddress ep = entry.getKey();
-            CompactEndpointSerializationHelper.serialize(ep, out);
+            InetAddressAndPort ep = entry.getKey();
+            CompactEndpointSerializationHelper.instance.serialize(ep, out, version);
             EndpointState.serializer.serialize(entry.getValue(), out, version);
         }
     }
@@ -65,11 +65,11 @@ class GossipDigestAck2Serializer implements IVersionedSerializer<GossipDigestAck
     public GossipDigestAck2 deserialize(DataInputPlus in, int version) throws IOException
     {
         int size = in.readInt();
-        Map<InetAddress, EndpointState> epStateMap = new HashMap<InetAddress, EndpointState>(size);
+        Map<InetAddressAndPort, EndpointState> epStateMap = new HashMap<>(size);
 
         for (int i = 0; i < size; ++i)
         {
-            InetAddress ep = CompactEndpointSerializationHelper.deserialize(in);
+            InetAddressAndPort ep = CompactEndpointSerializationHelper.instance.deserialize(in, version);
             EndpointState epState = EndpointState.serializer.deserialize(in, version);
             epStateMap.put(ep, epState);
         }
@@ -79,8 +79,8 @@ class GossipDigestAck2Serializer implements IVersionedSerializer<GossipDigestAck
     public long serializedSize(GossipDigestAck2 ack2, int version)
     {
         long size = TypeSizes.sizeof(ack2.epStateMap.size());
-        for (Map.Entry<InetAddress, EndpointState> entry : ack2.epStateMap.entrySet())
-            size += CompactEndpointSerializationHelper.serializedSize(entry.getKey())
+        for (Map.Entry<InetAddressAndPort, EndpointState> entry : ack2.epStateMap.entrySet())
+            size += CompactEndpointSerializationHelper.instance.serializedSize(entry.getKey(), version)
                     + EndpointState.serializer.serializedSize(entry.getValue(), version);
         return size;
     }

src/java/org/apache/cassandra/gms/GossipDigestAck2VerbHandler.java

@@ -17,12 +17,12 @@
  */
 package org.apache.cassandra.gms;
 
-import java.net.InetAddress;
 import java.util.Map;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.MessageIn;
 
@@ -34,7 +34,7 @@ public class GossipDigestAck2VerbHandler implements IVerbHandler<GossipDigestAck
     {
         if (logger.isTraceEnabled())
         {
-            InetAddress from = message.from;
+            InetAddressAndPort from = message.from;
             logger.trace("Received a GossipDigestAck2Message from {}", from);
         }
         if (!Gossiper.instance.isEnabled())
@@ -43,7 +43,7 @@ public class GossipDigestAck2VerbHandler implements IVerbHandler<GossipDigestAck
                 logger.trace("Ignoring GossipDigestAck2Message because gossip is disabled");
             return;
         }
-        Map<InetAddress, EndpointState> remoteEpStateMap = message.payload.getEndpointStateMap();
+        Map<InetAddressAndPort, EndpointState> remoteEpStateMap = message.payload.getEndpointStateMap();
         /* Notify the Failure Detector */
         Gossiper.instance.notifyFailureDetector(remoteEpStateMap);
         Gossiper.instance.applyStateLocally(remoteEpStateMap);

src/java/org/apache/cassandra/gms/GossipDigestAckVerbHandler.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.gms;
 
-import java.net.InetAddress;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -25,6 +24,7 @@ import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.MessageIn;
 import org.apache.cassandra.net.MessageOut;
@@ -36,7 +36,7 @@ public class GossipDigestAckVerbHandler implements IVerbHandler<GossipDigestAck>
 
     public void doVerb(MessageIn<GossipDigestAck> message, int id)
     {
-        InetAddress from = message.from;
+        InetAddressAndPort from = message.from;
         if (logger.isTraceEnabled())
             logger.trace("Received a GossipDigestAckMessage from {}", from);
         if (!Gossiper.instance.isEnabled() && !Gossiper.instance.isInShadowRound())
@@ -48,7 +48,7 @@ public class GossipDigestAckVerbHandler implements IVerbHandler<GossipDigestAck>
 
         GossipDigestAck gDigestAckMessage = message.payload;
         List<GossipDigest> gDigestList = gDigestAckMessage.getGossipDigestList();
-        Map<InetAddress, EndpointState> epStateMap = gDigestAckMessage.getEndpointStateMap();
+        Map<InetAddressAndPort, EndpointState> epStateMap = gDigestAckMessage.getEndpointStateMap();
         logger.trace("Received ack with {} digests and {} states", gDigestList.size(), epStateMap.size());
 
         if (Gossiper.instance.isInShadowRound())
@@ -79,10 +79,10 @@ public class GossipDigestAckVerbHandler implements IVerbHandler<GossipDigestAck>
         }
 
         /* Get the state required to send to this gossipee - construct GossipDigestAck2Message */
-        Map<InetAddress, EndpointState> deltaEpStateMap = new HashMap<InetAddress, EndpointState>();
+        Map<InetAddressAndPort, EndpointState> deltaEpStateMap = new HashMap<InetAddressAndPort, EndpointState>();
         for (GossipDigest gDigest : gDigestList)
         {
-            InetAddress addr = gDigest.getEndpoint();
+            InetAddressAndPort addr = gDigest.getEndpoint();
             EndpointState localEpStatePtr = Gossiper.instance.getStateForVersionBiggerThan(addr, gDigest.getMaxVersion());
             if (localEpStatePtr != null)
                 deltaEpStateMap.put(addr, localEpStatePtr);

src/java/org/apache/cassandra/gms/GossipDigestSynVerbHandler.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.gms;
 
-import java.net.InetAddress;
 import java.util.*;
 
 import com.google.common.collect.Maps;
@@ -26,6 +25,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.MessageIn;
 import org.apache.cassandra.net.MessageOut;
@@ -37,7 +37,7 @@ public class GossipDigestSynVerbHandler implements IVerbHandler<GossipDigestSyn>
 
     public void doVerb(MessageIn<GossipDigestSyn> message, int id)
     {
-        InetAddress from = message.from;
+        InetAddressAndPort from = message.from;
         if (logger.isTraceEnabled())
             logger.trace("Received a GossipDigestSynMessage from {}", from);
         if (!Gossiper.instance.isEnabled() && !Gossiper.instance.isInShadowRound())
@@ -102,7 +102,7 @@ public class GossipDigestSynVerbHandler implements IVerbHandler<GossipDigestSyn>
         doSort(gDigestList);
 
         List<GossipDigest> deltaGossipDigestList = new ArrayList<GossipDigest>();
-        Map<InetAddress, EndpointState> deltaEpStateMap = new HashMap<InetAddress, EndpointState>();
+        Map<InetAddressAndPort, EndpointState> deltaEpStateMap = new HashMap<InetAddressAndPort, EndpointState>();
         Gossiper.instance.examineGossiper(gDigestList, deltaGossipDigestList, deltaEpStateMap);
         logger.trace("sending {} digests and {} deltas", deltaGossipDigestList.size(), deltaEpStateMap.size());
         MessageOut<GossipDigestAck> gDigestAckMessage = new MessageOut<GossipDigestAck>(MessagingService.Verb.GOSSIP_DIGEST_ACK,
@@ -116,14 +116,14 @@ public class GossipDigestSynVerbHandler implements IVerbHandler<GossipDigestSyn>
     /*
      * First construct a map whose key is the endpoint in the GossipDigest and the value is the
      * GossipDigest itself. Then build a list of version differences i.e difference between the
-     * version in the GossipDigest and the version in the local state for a given InetAddress.
+     * version in the GossipDigest and the version in the local state for a given InetAddressAndPort.
      * Sort this list. Now loop through the sorted list and retrieve the GossipDigest corresponding
      * to the endpoint from the map that was initially constructed.
     */
     private void doSort(List<GossipDigest> gDigestList)
     {
         /* Construct a map of endpoint to GossipDigest. */
-        Map<InetAddress, GossipDigest> epToDigestMap = Maps.newHashMapWithExpectedSize(gDigestList.size());
+        Map<InetAddressAndPort, GossipDigest> epToDigestMap = Maps.newHashMapWithExpectedSize(gDigestList.size());
         for (GossipDigest gDigest : gDigestList)
         {
             epToDigestMap.put(gDigest.getEndpoint(), gDigest);
@@ -136,7 +136,7 @@ public class GossipDigestSynVerbHandler implements IVerbHandler<GossipDigestSyn>
         List<GossipDigest> diffDigests = new ArrayList<GossipDigest>(gDigestList.size());
         for (GossipDigest gDigest : gDigestList)
         {
-            InetAddress ep = gDigest.getEndpoint();
+            InetAddressAndPort ep = gDigest.getEndpoint();
             EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(ep);
             int version = (epState != null) ? Gossiper.instance.getMaxEndpointStateVersion(epState) : 0;
             int diffVersion = Math.abs(version - gDigest.getMaxVersion());

src/java/org/apache/cassandra/gms/Gossiper.java

@@ -18,12 +18,12 @@
 package org.apache.cassandra.gms;
 
 import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.*;
 import java.util.Map.Entry;
 import java.util.concurrent.*;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
 
 import javax.annotation.Nullable;
 import javax.management.MBeanServer;
@@ -34,6 +34,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.Uninterruptibles;
 
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.Pair;
 import org.slf4j.Logger;
@@ -99,43 +100,36 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     static final int MAX_GENERATION_DIFFERENCE = 86400 * 365;
     private long fatClientTimeout;
     private final Random random = new Random();
-    private final Comparator<InetAddress> inetcomparator = new Comparator<InetAddress>()
-    {
-        public int compare(InetAddress addr1, InetAddress addr2)
-        {
-            return addr1.getHostAddress().compareTo(addr2.getHostAddress());
-        }
-    };
 
     /* subscribers for interest in EndpointState change */
     private final List<IEndpointStateChangeSubscriber> subscribers = new CopyOnWriteArrayList<IEndpointStateChangeSubscriber>();
 
     /* live member set */
-    private final Set<InetAddress> liveEndpoints = new ConcurrentSkipListSet<InetAddress>(inetcomparator);
+    private final Set<InetAddressAndPort> liveEndpoints = new ConcurrentSkipListSet<>();
 
     /* unreachable member set */
-    private final Map<InetAddress, Long> unreachableEndpoints = new ConcurrentHashMap<InetAddress, Long>();
+    private final Map<InetAddressAndPort, Long> unreachableEndpoints = new ConcurrentHashMap<>();
 
     /* initial seeds for joining the cluster */
     @VisibleForTesting
-    final Set<InetAddress> seeds = new ConcurrentSkipListSet<InetAddress>(inetcomparator);
+    final Set<InetAddressAndPort> seeds = new ConcurrentSkipListSet<>();
 
     /* map where key is the endpoint and value is the state associated with the endpoint */
-    final ConcurrentMap<InetAddress, EndpointState> endpointStateMap = new ConcurrentHashMap<InetAddress, EndpointState>();
+    final ConcurrentMap<InetAddressAndPort, EndpointState> endpointStateMap = new ConcurrentHashMap<>();
 
     /* map where key is endpoint and value is timestamp when this endpoint was removed from
      * gossip. We will ignore any gossip regarding these endpoints for QUARANTINE_DELAY time
      * after removal to prevent nodes from falsely reincarnating during the time when removal
      * gossip gets propagated to all nodes */
-    private final Map<InetAddress, Long> justRemovedEndpoints = new ConcurrentHashMap<InetAddress, Long>();
+    private final Map<InetAddressAndPort, Long> justRemovedEndpoints = new ConcurrentHashMap<>();
 
-    private final Map<InetAddress, Long> expireTimeEndpointMap = new ConcurrentHashMap<InetAddress, Long>();
+    private final Map<InetAddressAndPort, Long> expireTimeEndpointMap = new ConcurrentHashMap<>();
 
     private volatile boolean inShadowRound = false;
     // seeds gathered during shadow round that indicated to be in the shadow round phase as well
-    private final Set<InetAddress> seedsInShadowRound = new ConcurrentSkipListSet<>(inetcomparator);
+    private final Set<InetAddressAndPort> seedsInShadowRound = new ConcurrentSkipListSet<>();
     // endpoint states as gathered during shadow round
-    private final Map<InetAddress, EndpointState> endpointShadowStateMap = new ConcurrentHashMap<>();
+    private final Map<InetAddressAndPort, EndpointState> endpointShadowStateMap = new ConcurrentHashMap<>();
 
     private volatile long lastProcessedMessageAt = System.currentTimeMillis();
 
@@ -151,9 +145,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
                 taskLock.lock();
 
                 /* Update the local heartbeat counter. */
-                endpointStateMap.get(FBUtilities.getBroadcastAddress()).getHeartBeatState().updateHeartBeat();
+                endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort()).getHeartBeatState().updateHeartBeat();
                 if (logger.isTraceEnabled())
-                    logger.trace("My heartbeat is now {}", endpointStateMap.get(FBUtilities.getBroadcastAddress()).getHeartBeatState().getHeartBeatVersion());
+                    logger.trace("My heartbeat is now {}", endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort()).getHeartBeatState().getHeartBeatVersion());
                 final List<GossipDigest> gDigests = new ArrayList<GossipDigest>();
                 Gossiper.instance.makeRandomGossipDigest(gDigests);
 
@@ -231,14 +225,24 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
 
     public boolean seenAnySeed()
     {
-        for (Map.Entry<InetAddress, EndpointState> entry : endpointStateMap.entrySet())
+        for (Map.Entry<InetAddressAndPort, EndpointState> entry : endpointStateMap.entrySet())
         {
             if (seeds.contains(entry.getKey()))
                 return true;
             try
             {
                 VersionedValue internalIp = entry.getValue().getApplicationState(ApplicationState.INTERNAL_IP);
-                if (internalIp != null && seeds.contains(InetAddress.getByName(internalIp.value)))
+                VersionedValue internalIpAndPort = entry.getValue().getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+                InetAddressAndPort endpoint = null;
+                if (internalIpAndPort != null)
+                {
+                    endpoint = InetAddressAndPort.getByName(internalIpAndPort.value);
+                }
+                else if (internalIp != null)
+                {
+                    endpoint = InetAddressAndPort.getByName(internalIp.value);
+                }
+                if (endpoint != null && seeds.contains(endpoint))
                     return true;
             }
             catch (UnknownHostException e)
@@ -272,18 +276,18 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     /**
      * @return a list of live gossip participants, including fat clients
      */
-    public Set<InetAddress> getLiveMembers()
+    public Set<InetAddressAndPort> getLiveMembers()
     {
-        Set<InetAddress> liveMembers = new HashSet<>(liveEndpoints);
-        if (!liveMembers.contains(FBUtilities.getBroadcastAddress()))
-            liveMembers.add(FBUtilities.getBroadcastAddress());
+        Set<InetAddressAndPort> liveMembers = new HashSet<>(liveEndpoints);
+        if (!liveMembers.contains(FBUtilities.getBroadcastAddressAndPort()))
+            liveMembers.add(FBUtilities.getBroadcastAddressAndPort());
         return liveMembers;
     }
 
     /**
      * @return a list of live ring members.
      */
-    public Set<InetAddress> getLiveTokenOwners()
+    public Set<InetAddressAndPort> getLiveTokenOwners()
     {
         return StorageService.instance.getLiveRingMembers(true);
     }
@@ -291,7 +295,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     /**
      * @return a list of unreachable gossip participants, including fat clients
      */
-    public Set<InetAddress> getUnreachableMembers()
+    public Set<InetAddressAndPort> getUnreachableMembers()
     {
         return unreachableEndpoints.keySet();
     }
@@ -299,10 +303,10 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     /**
      * @return a list of unreachable token owners
      */
-    public Set<InetAddress> getUnreachableTokenOwners()
+    public Set<InetAddressAndPort> getUnreachableTokenOwners()
     {
-        Set<InetAddress> tokenOwners = new HashSet<>();
-        for (InetAddress endpoint : unreachableEndpoints.keySet())
+        Set<InetAddressAndPort> tokenOwners = new HashSet<>();
+        for (InetAddressAndPort endpoint : unreachableEndpoints.keySet())
         {
             if (StorageService.instance.getTokenMetadata().isMember(endpoint))
                 tokenOwners.add(endpoint);
@@ -311,7 +315,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         return tokenOwners;
     }
 
-    public long getEndpointDowntime(InetAddress ep)
+    public long getEndpointDowntime(InetAddressAndPort ep)
     {
         Long downtime = unreachableEndpoints.get(ep);
         if (downtime != null)
@@ -320,14 +324,25 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
             return 0L;
     }
 
-    private boolean isShutdown(InetAddress endpoint)
+    private boolean isShutdown(InetAddressAndPort endpoint)
     {
         EndpointState epState = endpointStateMap.get(endpoint);
         if (epState == null)
+        {
             return false;
-        if (epState.getApplicationState(ApplicationState.STATUS) == null)
-            return false;
-        String value = epState.getApplicationState(ApplicationState.STATUS).value;
+        }
+
+        VersionedValue versionedValue = epState.getApplicationState(ApplicationState.STATUS_WITH_PORT);
+        if (versionedValue == null)
+        {
+            versionedValue = epState.getApplicationState(ApplicationState.STATUS);
+            if (versionedValue == null)
+            {
+                return false;
+            }
+        }
+
+        String value = versionedValue.value;
         String[] pieces = value.split(VersionedValue.DELIMITER_STR, -1);
         assert (pieces.length > 0);
         String state = pieces[0];
@@ -340,7 +355,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      *
      * @param endpoint end point that is convicted.
      */
-    public void convict(InetAddress endpoint, double phi)
+    public void convict(InetAddressAndPort endpoint, double phi)
     {
         EndpointState epState = endpointStateMap.get(endpoint);
         if (epState == null)
@@ -366,11 +381,12 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      * This method is used to mark a node as shutdown; that is it gracefully exited on its own and told us about it
      * @param endpoint endpoint that has shut itself down
      */
-    protected void markAsShutdown(InetAddress endpoint)
+    protected void markAsShutdown(InetAddressAndPort endpoint)
     {
         EndpointState epState = endpointStateMap.get(endpoint);
         if (epState == null)
             return;
+        epState.addApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.shutdown(true));
         epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.shutdown(true));
         epState.addApplicationState(ApplicationState.RPC_READY, StorageService.instance.valueFactory.rpcReady(false));
         epState.getHeartBeatState().forceHighestPossibleVersionUnsafe();
@@ -397,7 +413,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      *
      * @param endpoint endpoint to be removed from the current membership.
      */
-    private void evictFromMembership(InetAddress endpoint)
+    private void evictFromMembership(InetAddressAndPort endpoint)
     {
         unreachableEndpoints.remove(endpoint);
         endpointStateMap.remove(endpoint);
@@ -411,7 +427,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     /**
      * Removes the endpoint from Gossip but retains endpoint state
      */
-    public void removeEndpoint(InetAddress endpoint)
+    public void removeEndpoint(InetAddressAndPort endpoint)
     {
         // do subscribers first so anything in the subscriber that depends on gossiper state won't get confused
         for (IEndpointStateChangeSubscriber subscriber : subscribers)
@@ -438,7 +454,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      *
      * @param endpoint
      */
-    private void quarantineEndpoint(InetAddress endpoint)
+    private void quarantineEndpoint(InetAddressAndPort endpoint)
     {
         quarantineEndpoint(endpoint, System.currentTimeMillis());
     }
@@ -449,7 +465,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      * @param endpoint
      * @param quarantineExpiration
      */
-    private void quarantineEndpoint(InetAddress endpoint, long quarantineExpiration)
+    private void quarantineEndpoint(InetAddressAndPort endpoint, long quarantineExpiration)
     {
         justRemovedEndpoints.put(endpoint, quarantineExpiration);
     }
@@ -458,7 +474,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      * Quarantine endpoint specifically for replacement purposes.
      * @param endpoint
      */
-    public void replacementQuarantine(InetAddress endpoint)
+    public void replacementQuarantine(InetAddressAndPort endpoint)
     {
         // remember, quarantineEndpoint will effectively already add QUARANTINE_DELAY, so this is 2x
         logger.debug("");
@@ -471,7 +487,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      *
      * @param endpoint The endpoint that has been replaced
      */
-    public void replacedEndpoint(InetAddress endpoint)
+    public void replacedEndpoint(InetAddressAndPort endpoint)
     {
         removeEndpoint(endpoint);
         evictFromMembership(endpoint);
@@ -491,9 +507,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         int maxVersion = 0;
 
         // local epstate will be part of endpointStateMap
-        List<InetAddress> endpoints = new ArrayList<InetAddress>(endpointStateMap.keySet());
+        List<InetAddressAndPort> endpoints = new ArrayList<>(endpointStateMap.keySet());
         Collections.shuffle(endpoints, random);
-        for (InetAddress endpoint : endpoints)
+        for (InetAddressAndPort endpoint : endpoints)
         {
             epState = endpointStateMap.get(endpoint);
             if (epState != null)
@@ -524,7 +540,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      * @param hostId      - the ID of the host being removed
      * @param localHostId - my own host ID for replication coordination
      */
-    public void advertiseRemoving(InetAddress endpoint, UUID hostId, UUID localHostId)
+    public void advertiseRemoving(InetAddressAndPort endpoint, UUID hostId, UUID localHostId)
     {
         EndpointState epState = endpointStateMap.get(endpoint);
         // remember this node's generation
@@ -541,6 +557,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         epState.updateTimestamp(); // make sure we don't evict it too soon
         epState.getHeartBeatState().forceNewerGenerationUnsafe();
         Map<ApplicationState, VersionedValue> states = new EnumMap<>(ApplicationState.class);
+        states.put(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.removingNonlocal(hostId));
         states.put(ApplicationState.STATUS, StorageService.instance.valueFactory.removingNonlocal(hostId));
         states.put(ApplicationState.REMOVAL_COORDINATOR, StorageService.instance.valueFactory.removalCoordinator(localHostId));
         epState.addApplicationStates(states);
@@ -554,12 +571,13 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      * @param endpoint
      * @param hostId
      */
-    public void advertiseTokenRemoved(InetAddress endpoint, UUID hostId)
+    public void advertiseTokenRemoved(InetAddressAndPort endpoint, UUID hostId)
     {
         EndpointState epState = endpointStateMap.get(endpoint);
         epState.updateTimestamp(); // make sure we don't evict it too soon
         epState.getHeartBeatState().forceNewerGenerationUnsafe();
         long expireTime = computeExpireTime();
+        epState.addApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.removedNonlocal(hostId, expireTime));
         epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.removedNonlocal(hostId, expireTime));
         logger.info("Completing removal of {}", endpoint);
         addExpireTimeForEndpoint(endpoint, expireTime);
@@ -584,7 +602,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      */
     public void assassinateEndpoint(String address) throws UnknownHostException
     {
-        InetAddress endpoint = InetAddress.getByName(address);
+        InetAddressAndPort endpoint = InetAddressAndPort.getByName(address);
         EndpointState epState = endpointStateMap.get(endpoint);
         Collection<Token> tokens = null;
         logger.warn("Assassinating {} via gossip", endpoint);
@@ -624,18 +642,20 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         }
 
         // do not pass go, do not collect 200 dollars, just gtfo
-        epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.left(tokens, computeExpireTime()));
+        long expireTime = computeExpireTime();
+        epState.addApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.left(tokens, expireTime));
+        epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.left(tokens, expireTime));
         handleMajorStateChange(endpoint, epState);
         Uninterruptibles.sleepUninterruptibly(intervalInMillis * 4, TimeUnit.MILLISECONDS);
         logger.warn("Finished assassinating {}", endpoint);
     }
 
-    public boolean isKnownEndpoint(InetAddress endpoint)
+    public boolean isKnownEndpoint(InetAddressAndPort endpoint)
     {
         return endpointStateMap.containsKey(endpoint);
     }
 
-    public int getCurrentGenerationNumber(InetAddress endpoint)
+    public int getCurrentGenerationNumber(InetAddressAndPort endpoint)
     {
         return endpointStateMap.get(endpoint).getHeartBeatState().getGeneration();
     }
@@ -647,16 +667,16 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      * @param epSet   a set of endpoint from which a random endpoint is chosen.
      * @return true if the chosen endpoint is also a seed.
      */
-    private boolean sendGossip(MessageOut<GossipDigestSyn> message, Set<InetAddress> epSet)
+    private boolean sendGossip(MessageOut<GossipDigestSyn> message, Set<InetAddressAndPort> epSet)
     {
-        List<InetAddress> liveEndpoints = ImmutableList.copyOf(epSet);
+        List<InetAddressAndPort> liveEndpoints = ImmutableList.copyOf(epSet);
 
         int size = liveEndpoints.size();
         if (size < 1)
             return false;
         /* Generate a random number from 0 -> size */
         int index = (size == 1) ? 0 : random.nextInt(size);
-        InetAddress to = liveEndpoints.get(index);
+        InetAddressAndPort to = liveEndpoints.get(index);
         if (logger.isTraceEnabled())
             logger.trace("Sending a GossipDigestSyn to {} ...", to);
         if (firstSynSendAt == 0)
@@ -695,7 +715,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         int size = seeds.size();
         if (size > 0)
         {
-            if (size == 1 && seeds.contains(FBUtilities.getBroadcastAddress()))
+            if (size == 1 && seeds.contains(FBUtilities.getBroadcastAddressAndPort()))
             {
                 return;
             }
@@ -715,7 +735,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         }
     }
 
-    public boolean isGossipOnlyMember(InetAddress endpoint)
+    public boolean isGossipOnlyMember(InetAddressAndPort endpoint)
     {
         EndpointState epState = endpointStateMap.get(endpoint);
         if (epState == null)
@@ -740,8 +760,8 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      * @param epStates - endpoint states in the cluster
      * @return true if it is safe to start the node, false otherwise
      */
-    public boolean isSafeForStartup(InetAddress endpoint, UUID localHostUUID, boolean isBootstrapping,
-                                    Map<InetAddress, EndpointState> epStates)
+    public boolean isSafeForStartup(InetAddressAndPort endpoint, UUID localHostUUID, boolean isBootstrapping,
+                                    Map<InetAddressAndPort, EndpointState> epStates)
     {
         EndpointState epState = epStates.get(endpoint);
         // if there's no previous state, or the node was previously removed from the cluster, we're good
@@ -792,10 +812,10 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
             }
         }
 
-        Set<InetAddress> eps = endpointStateMap.keySet();
-        for (InetAddress endpoint : eps)
+        Set<InetAddressAndPort> eps = endpointStateMap.keySet();
+        for (InetAddressAndPort endpoint : eps)
         {
-            if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+            if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
                 continue;
 
             FailureDetector.instance.interpret(endpoint);
@@ -829,7 +849,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
 
         if (!justRemovedEndpoints.isEmpty())
         {
-            for (Entry<InetAddress, Long> entry : justRemovedEndpoints.entrySet())
+            for (Entry<InetAddressAndPort, Long> entry : justRemovedEndpoints.entrySet())
             {
                 if ((now - entry.getValue()) > QUARANTINE_DELAY)
                 {
@@ -841,34 +861,34 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         }
     }
 
-    protected long getExpireTimeForEndpoint(InetAddress endpoint)
+    protected long getExpireTimeForEndpoint(InetAddressAndPort endpoint)
     {
         /* default expireTime is aVeryLongTime */
         Long storedTime = expireTimeEndpointMap.get(endpoint);
         return storedTime == null ? computeExpireTime() : storedTime;
     }
 
-    public EndpointState getEndpointStateForEndpoint(InetAddress ep)
+    public EndpointState getEndpointStateForEndpoint(InetAddressAndPort ep)
     {
         return endpointStateMap.get(ep);
     }
 
-    public Set<Entry<InetAddress, EndpointState>> getEndpointStates()
+    public Set<Entry<InetAddressAndPort, EndpointState>> getEndpointStates()
     {
         return endpointStateMap.entrySet();
     }
 
-    public UUID getHostId(InetAddress endpoint)
+    public UUID getHostId(InetAddressAndPort endpoint)
     {
         return getHostId(endpoint, endpointStateMap);
     }
 
-    public UUID getHostId(InetAddress endpoint, Map<InetAddress, EndpointState> epStates)
+    public UUID getHostId(InetAddressAndPort endpoint, Map<InetAddressAndPort, EndpointState> epStates)
     {
         return UUID.fromString(epStates.get(endpoint).getApplicationState(ApplicationState.HOST_ID).value);
     }
 
-    EndpointState getStateForVersionBiggerThan(InetAddress forEndpoint, int version)
+    EndpointState getStateForVersionBiggerThan(InetAddressAndPort forEndpoint, int version)
     {
         EndpointState epState = endpointStateMap.get(forEndpoint);
         EndpointState reqdEndpointState = null;
@@ -919,7 +939,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     /**
      * determine which endpoint started up earlier
      */
-    public int compareEndpointStartup(InetAddress addr1, InetAddress addr2)
+    public int compareEndpointStartup(InetAddressAndPort addr1, InetAddressAndPort addr2)
     {
         EndpointState ep1 = getEndpointStateForEndpoint(addr1);
         EndpointState ep2 = getEndpointStateForEndpoint(addr2);
@@ -927,15 +947,15 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         return ep1.getHeartBeatState().getGeneration() - ep2.getHeartBeatState().getGeneration();
     }
 
-    void notifyFailureDetector(Map<InetAddress, EndpointState> remoteEpStateMap)
+    void notifyFailureDetector(Map<InetAddressAndPort, EndpointState> remoteEpStateMap)
     {
-        for (Entry<InetAddress, EndpointState> entry : remoteEpStateMap.entrySet())
+        for (Entry<InetAddressAndPort, EndpointState> entry : remoteEpStateMap.entrySet())
         {
             notifyFailureDetector(entry.getKey(), entry.getValue());
         }
     }
 
-    void notifyFailureDetector(InetAddress endpoint, EndpointState remoteEndpointState)
+    void notifyFailureDetector(InetAddressAndPort endpoint, EndpointState remoteEndpointState)
     {
         EndpointState localEndpointState = endpointStateMap.get(endpoint);
         /*
@@ -976,7 +996,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
 
     }
 
-    private void markAlive(final InetAddress addr, final EndpointState localState)
+    private void markAlive(final InetAddressAndPort addr, final EndpointState localState)
     {
         localState.markDead();
 
@@ -999,7 +1019,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     }
 
     @VisibleForTesting
-    public void realMarkAlive(final InetAddress addr, final EndpointState localState)
+    public void realMarkAlive(final InetAddressAndPort addr, final EndpointState localState)
     {
         if (logger.isTraceEnabled())
             logger.trace("marking as alive {}", addr);
@@ -1017,7 +1037,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     }
 
     @VisibleForTesting
-    public void markDead(InetAddress addr, EndpointState localState)
+    public void markDead(InetAddressAndPort addr, EndpointState localState)
     {
         if (logger.isTraceEnabled())
             logger.trace("marking as down {}", addr);
@@ -1037,7 +1057,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      * @param ep      endpoint
      * @param epState EndpointState for the endpoint
      */
-    private void handleMajorStateChange(InetAddress ep, EndpointState epState)
+    private void handleMajorStateChange(InetAddressAndPort ep, EndpointState epState)
     {
         EndpointState localEpState = endpointStateMap.get(ep);
         if (!isDeadState(epState))
@@ -1071,7 +1091,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
             markAsShutdown(ep);
     }
 
-    public boolean isAlive(InetAddress endpoint)
+    public boolean isAlive(InetAddressAndPort endpoint)
     {
         EndpointState epState = getEndpointStateForEndpoint(endpoint);
         if (epState == null)
@@ -1099,21 +1119,33 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
 
     private static String getGossipStatus(EndpointState epState)
     {
-        if (epState == null || epState.getApplicationState(ApplicationState.STATUS) == null)
+        if (epState == null)
+        {
             return "";
+        }
 
-        String value = epState.getApplicationState(ApplicationState.STATUS).value;
+        VersionedValue versionedValue = epState.getApplicationState(ApplicationState.STATUS_WITH_PORT);
+        if (versionedValue == null)
+        {
+            versionedValue = epState.getApplicationState(ApplicationState.STATUS);
+            if (versionedValue == null)
+            {
+                return "";
+            }
+        }
+
+        String value = versionedValue.value;
         String[] pieces = value.split(VersionedValue.DELIMITER_STR, -1);
         assert (pieces.length > 0);
         return pieces[0];
     }
 
-    void applyStateLocally(Map<InetAddress, EndpointState> epStateMap)
+    void applyStateLocally(Map<InetAddressAndPort, EndpointState> epStateMap)
     {
-        for (Entry<InetAddress, EndpointState> entry : epStateMap.entrySet())
+        for (Entry<InetAddressAndPort, EndpointState> entry : epStateMap.entrySet())
         {
-            InetAddress ep = entry.getKey();
-            if ( ep.equals(FBUtilities.getBroadcastAddress()) && !isInShadowRound())
+            InetAddressAndPort ep = entry.getKey();
+            if ( ep.equals(FBUtilities.getBroadcastAddressAndPort()) && !isInShadowRound())
                 continue;
             if (justRemovedEndpoints.containsKey(ep))
             {
@@ -1181,7 +1213,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         }
     }
 
-    private void applyNewStates(InetAddress addr, EndpointState localState, EndpointState remoteState)
+    private void applyNewStates(InetAddressAndPort addr, EndpointState localState, EndpointState remoteState)
     {
         // don't assert here, since if the node restarts the version will go back to zero
         int oldVersion = localState.getHeartBeatState().getHeartBeatVersion();
@@ -1194,12 +1226,27 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         assert remoteState.getHeartBeatState().getGeneration() == localState.getHeartBeatState().getGeneration();
         localState.addApplicationStates(remoteStates);
 
-        for (Entry<ApplicationState, VersionedValue> remoteEntry : remoteStates)
+        //Filter out pre-4.0 versions of data for more complete 4.0 versions
+        Set<Entry<ApplicationState, VersionedValue>> filtered = remoteStates.stream().filter(entry -> {
+           switch (entry.getKey())
+           {
+               case INTERNAL_IP:
+                    return remoteState.getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT) == null;
+               case STATUS:
+                   return remoteState.getApplicationState(ApplicationState.STATUS_WITH_PORT) == null;
+               case RPC_ADDRESS:
+                   return remoteState.getApplicationState(ApplicationState.NATIVE_ADDRESS_AND_PORT) == null;
+               default:
+                   return true;
+           }
+        }).collect(Collectors.toSet());
+
+        for (Entry<ApplicationState, VersionedValue> remoteEntry : filtered)
             doOnChangeNotifications(addr, remoteEntry.getKey(), remoteEntry.getValue());
     }
 
     // notify that a local application state is going to change (doesn't get triggered for remote changes)
-    private void doBeforeChangeNotifications(InetAddress addr, EndpointState epState, ApplicationState apState, VersionedValue newValue)
+    private void doBeforeChangeNotifications(InetAddressAndPort addr, EndpointState epState, ApplicationState apState, VersionedValue newValue)
     {
         for (IEndpointStateChangeSubscriber subscriber : subscribers)
         {
@@ -1208,7 +1255,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     }
 
     // notify that an application state has changed
-    private void doOnChangeNotifications(InetAddress addr, ApplicationState state, VersionedValue value)
+    private void doOnChangeNotifications(InetAddressAndPort addr, ApplicationState state, VersionedValue value)
     {
         for (IEndpointStateChangeSubscriber subscriber : subscribers)
         {
@@ -1226,7 +1273,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     }
 
     /* Send all the data with version greater than maxRemoteVersion */
-    private void sendAll(GossipDigest gDigest, Map<InetAddress, EndpointState> deltaEpStateMap, int maxRemoteVersion)
+    private void sendAll(GossipDigest gDigest, Map<InetAddressAndPort, EndpointState> deltaEpStateMap, int maxRemoteVersion)
     {
         EndpointState localEpStatePtr = getStateForVersionBiggerThan(gDigest.getEndpoint(), maxRemoteVersion);
         if (localEpStatePtr != null)
@@ -1237,7 +1284,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         This method is used to figure the state that the Gossiper has but Gossipee doesn't. The delta digests
         and the delta state are built up.
     */
-    void examineGossiper(List<GossipDigest> gDigestList, List<GossipDigest> deltaGossipDigestList, Map<InetAddress, EndpointState> deltaEpStateMap)
+    void examineGossiper(List<GossipDigest> gDigestList, List<GossipDigest> deltaGossipDigestList, Map<InetAddressAndPort, EndpointState> deltaEpStateMap)
     {
         if (gDigestList.size() == 0)
         {
@@ -1245,7 +1292,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
               If this is happening then the node is attempting shadow gossip, and we should reply with everything we know.
             */
             logger.debug("Shadow request received, adding all states");
-            for (Map.Entry<InetAddress, EndpointState> entry : endpointStateMap.entrySet())
+            for (Map.Entry<InetAddressAndPort, EndpointState> entry : endpointStateMap.entrySet())
             {
                 gDigestList.add(new GossipDigest(entry.getKey(), 0, 0));
             }
@@ -1320,7 +1367,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         buildSeedsList();
         /* initialize the heartbeat state for this localEndpoint */
         maybeInitializeLocalState(generationNbr);
-        EndpointState localState = endpointStateMap.get(FBUtilities.getBroadcastAddress());
+        EndpointState localState = endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort());
         localState.addApplicationStates(preloadLocalStates);
 
         //notify snitches that Gossiper is about to start
@@ -1345,14 +1392,14 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      * </ul>
      *
      * Method is synchronized, as we use an in-progress flag to indicate that shadow round must be cleared
-     * again by calling {@link Gossiper#maybeFinishShadowRound(InetAddress, boolean, Map)}. This will update
+     * again by calling {@link Gossiper#maybeFinishShadowRound(InetAddressAndPort, boolean, Map)}. This will update
      * {@link Gossiper#endpointShadowStateMap} with received values, in order to return an immutable copy to the
      * caller of {@link Gossiper#doShadowRound()}. Therefor only a single shadow round execution is permitted at
      * the same time.
      *
      * @return endpoint states gathered during shadow round or empty map
      */
-    public synchronized Map<InetAddress, EndpointState> doShadowRound()
+    public synchronized Map<InetAddressAndPort, EndpointState> doShadowRound()
     {
         buildSeedsList();
         // it may be that the local address is the only entry in the seed
@@ -1381,7 +1428,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
                 { // CASSANDRA-8072, retry at the beginning and every 5 seconds
                     logger.trace("Sending shadow round GOSSIP DIGEST SYN to seeds {}", seeds);
 
-                    for (InetAddress seed : seeds)
+                    for (InetAddressAndPort seed : seeds)
                         MessagingService.instance().sendOneWay(message, seed);
                 }
 
@@ -1393,8 +1440,8 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
                 if (slept > StorageService.RING_DELAY)
                 {
                     // if we don't consider ourself to be a seed, fail out
-                    if (!DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddress()))
-                        throw new RuntimeException("Unable to gossip with any seeds");
+                    if (!DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddressAndPort()))
+                        throw new RuntimeException("Unable to gossip with any seeds " + DatabaseDescriptor.getSeeds() + " and " + FBUtilities.getBroadcastAddressAndPort());
 
                     logger.warn("Unable to gossip with any seeds but continuing since node is in its own seed list");
                     inShadowRound = false;
@@ -1413,9 +1460,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     @VisibleForTesting
     void buildSeedsList()
     {
-        for (InetAddress seed : DatabaseDescriptor.getSeeds())
+        for (InetAddressAndPort seed : DatabaseDescriptor.getSeeds())
         {
-            if (seed.equals(FBUtilities.getBroadcastAddress()))
+            if (seed.equals(FBUtilities.getBroadcastAddressAndPort()))
                 continue;
             seeds.add(seed);
         }
@@ -1427,12 +1474,12 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         HeartBeatState hbState = new HeartBeatState(generationNbr);
         EndpointState localState = new EndpointState(hbState);
         localState.markAlive();
-        endpointStateMap.putIfAbsent(FBUtilities.getBroadcastAddress(), localState);
+        endpointStateMap.putIfAbsent(FBUtilities.getBroadcastAddressAndPort(), localState);
     }
 
     public void forceNewerGeneration()
     {
-        EndpointState epstate = endpointStateMap.get(FBUtilities.getBroadcastAddress());
+        EndpointState epstate = endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort());
         epstate.getHeartBeatState().forceNewerGenerationUnsafe();
     }
 
@@ -1440,9 +1487,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     /**
      * Add an endpoint we knew about previously, but whose state is unknown
      */
-    public void addSavedEndpoint(InetAddress ep)
+    public void addSavedEndpoint(InetAddressAndPort ep)
     {
-        if (ep.equals(FBUtilities.getBroadcastAddress()))
+        if (ep.equals(FBUtilities.getBroadcastAddressAndPort()))
         {
             logger.debug("Attempt to add self as saved endpoint");
             return;
@@ -1470,8 +1517,8 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     private void addLocalApplicationStateInternal(ApplicationState state, VersionedValue value)
     {
         assert taskLock.isHeldByCurrentThread();
-        EndpointState epState = endpointStateMap.get(FBUtilities.getBroadcastAddress());
-        InetAddress epAddr = FBUtilities.getBroadcastAddress();
+        EndpointState epState = endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort());
+        InetAddressAndPort epAddr = FBUtilities.getBroadcastAddressAndPort();
         assert epState != null;
         // Fire "before change" notifications:
         doBeforeChangeNotifications(epAddr, epState, state, value);
@@ -1508,13 +1555,14 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
 
     public void stop()
     {
-        EndpointState mystate = endpointStateMap.get(FBUtilities.getBroadcastAddress());
+        EndpointState mystate = endpointStateMap.get(FBUtilities.getBroadcastAddressAndPort());
         if (mystate != null && !isSilentShutdownState(mystate) && StorageService.instance.isJoined())
         {
             logger.info("Announcing shutdown");
+            addLocalApplicationState(ApplicationState.STATUS_WITH_PORT, StorageService.instance.valueFactory.shutdown(true));
             addLocalApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.shutdown(true));
             MessageOut message = new MessageOut(MessagingService.Verb.GOSSIP_SHUTDOWN);
-            for (InetAddress ep : liveEndpoints)
+            for (InetAddressAndPort ep : liveEndpoints)
                 MessagingService.instance().sendOneWay(message, ep);
             Uninterruptibles.sleepUninterruptibly(Integer.getInteger("cassandra.shutdown_announce_in_ms", 2000), TimeUnit.MILLISECONDS);
         }
@@ -1529,7 +1577,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         return (scheduledGossipTask != null) && (!scheduledGossipTask.isCancelled());
     }
 
-    protected void maybeFinishShadowRound(InetAddress respondent, boolean isInShadowRound, Map<InetAddress, EndpointState> epStateMap)
+    protected void maybeFinishShadowRound(InetAddressAndPort respondent, boolean isInShadowRound, Map<InetAddressAndPort, EndpointState> epStateMap)
     {
         if (inShadowRound)
         {
@@ -1565,7 +1613,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     }
 
     @VisibleForTesting
-    public void initializeNodeUnsafe(InetAddress addr, UUID uuid, int generationNbr)
+    public void initializeNodeUnsafe(InetAddressAndPort addr, UUID uuid, int generationNbr)
     {
         HeartBeatState hbState = new HeartBeatState(generationNbr);
         EndpointState newState = new EndpointState(hbState);
@@ -1581,7 +1629,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     }
 
     @VisibleForTesting
-    public void injectApplicationState(InetAddress endpoint, ApplicationState state, VersionedValue value)
+    public void injectApplicationState(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
     {
         EndpointState localState = endpointStateMap.get(endpoint);
         localState.addApplicationState(state, value);
@@ -1589,15 +1637,15 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
 
     public long getEndpointDowntime(String address) throws UnknownHostException
     {
-        return getEndpointDowntime(InetAddress.getByName(address));
+        return getEndpointDowntime(InetAddressAndPort.getByName(address));
     }
 
     public int getCurrentGenerationNumber(String address) throws UnknownHostException
     {
-        return getCurrentGenerationNumber(InetAddress.getByName(address));
+        return getCurrentGenerationNumber(InetAddressAndPort.getByName(address));
     }
 
-    public void addExpireTimeForEndpoint(InetAddress endpoint, long expireTime)
+    public void addExpireTimeForEndpoint(InetAddressAndPort endpoint, long expireTime)
     {
         if (logger.isDebugEnabled())
         {
@@ -1612,14 +1660,14 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     }
 
     @Nullable
-    public CassandraVersion getReleaseVersion(InetAddress ep)
+    public CassandraVersion getReleaseVersion(InetAddressAndPort ep)
     {
         EndpointState state = getEndpointStateForEndpoint(ep);
         return state != null ? state.getReleaseVersion() : null;
     }
 
     @Nullable
-    public UUID getSchemaVersion(InetAddress ep)
+    public UUID getSchemaVersion(InetAddressAndPort ep)
     {
         EndpointState state = getEndpointStateForEndpoint(ep);
         return state != null ? state.getSchemaVersion() : null;

src/java/org/apache/cassandra/gms/IEndpointStateChangeSubscriber.java

@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.gms;
 
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 /**
  * This is called by an instance of the IEndpointStateChangePublisher to notify
@@ -36,17 +36,17 @@ public interface IEndpointStateChangeSubscriber
      * @param endpoint endpoint for which the state change occurred.
      * @param epState  state that actually changed for the above endpoint.
      */
-    public void onJoin(InetAddress endpoint, EndpointState epState);
+    public void onJoin(InetAddressAndPort endpoint, EndpointState epState);
     
-    public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue);
+    public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue);
 
-    public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value);
+    public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value);
 
-    public void onAlive(InetAddress endpoint, EndpointState state);
+    public void onAlive(InetAddressAndPort endpoint, EndpointState state);
 
-    public void onDead(InetAddress endpoint, EndpointState state);
+    public void onDead(InetAddressAndPort endpoint, EndpointState state);
 
-    public void onRemove(InetAddress endpoint);
+    public void onRemove(InetAddressAndPort endpoint);
 
     /**
      * Called whenever a node is restarted.
@@ -54,5 +54,5 @@ public interface IEndpointStateChangeSubscriber
      * previously marked down. It will have only if {@code state.isAlive() == false}
      * as {@code state} is from before the restarted node is marked up.
      */
-    public void onRestart(InetAddress endpoint, EndpointState state);
+    public void onRestart(InetAddressAndPort endpoint, EndpointState state);
 }

src/java/org/apache/cassandra/gms/IFailureDetectionEventListener.java

@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.gms;
 
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 /**
  * Implemented by the Gossiper to convict an endpoint
@@ -33,5 +33,5 @@ public interface IFailureDetectionEventListener
      * @param ep  endpoint to be convicted
      * @param phi the value of phi with with ep was convicted
      */
-    public void convict(InetAddress ep, double phi);
+    public void convict(InetAddressAndPort ep, double phi);
 }

src/java/org/apache/cassandra/gms/IFailureDetector.java

@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.gms;
 
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 /**
  * An interface that provides an application with the ability
@@ -35,7 +35,7 @@ public interface IFailureDetector
      * @param ep endpoint in question.
      * @return true if UP and false if DOWN.
      */
-    public boolean isAlive(InetAddress ep);
+    public boolean isAlive(InetAddressAndPort ep);
 
     /**
      * This method is invoked by any entity wanting to interrogate the status of an endpoint.
@@ -44,7 +44,7 @@ public interface IFailureDetector
      *
      * param ep endpoint for which we interpret the inter arrival times.
      */
-    public void interpret(InetAddress ep);
+    public void interpret(InetAddressAndPort ep);
 
     /**
      * This method is invoked by the receiver of the heartbeat. In our case it would be
@@ -53,17 +53,17 @@ public interface IFailureDetector
      *
      * param ep endpoint being reported.
      */
-    public void report(InetAddress ep);
+    public void report(InetAddressAndPort ep);
 
     /**
      * remove endpoint from failure detector
      */
-    public void remove(InetAddress ep);
+    public void remove(InetAddressAndPort ep);
 
     /**
      * force conviction of endpoint in the failure detector
      */
-    public void forceConviction(InetAddress ep);
+    public void forceConviction(InetAddressAndPort ep);
 
     /**
      * Register interest for Failure Detector events.

src/java/org/apache/cassandra/gms/VersionedValue.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.commons.lang3.StringUtils;
@@ -133,11 +134,17 @@ public class VersionedValue implements Comparable<VersionedValue>
             return new VersionedValue(value.value);
         }
 
+        @Deprecated
         public VersionedValue bootReplacing(InetAddress oldNode)
         {
             return new VersionedValue(versionString(VersionedValue.STATUS_BOOTSTRAPPING_REPLACE, oldNode.getHostAddress()));
         }
 
+        public VersionedValue bootReplacingWithPort(InetAddressAndPort oldNode)
+        {
+            return new VersionedValue(versionString(VersionedValue.STATUS_BOOTSTRAPPING_REPLACE, oldNode.toString()));
+        }
+
         public VersionedValue bootstrapping(Collection<Token> tokens)
         {
             return new VersionedValue(versionString(VersionedValue.STATUS_BOOTSTRAPPING,
@@ -248,6 +255,11 @@ public class VersionedValue implements Comparable<VersionedValue>
             return new VersionedValue(endpoint.getHostAddress());
         }
 
+        public VersionedValue nativeaddressAndPort(InetAddressAndPort address)
+        {
+            return new VersionedValue(address.toString());
+        }
+
         public VersionedValue releaseVersion()
         {
             return new VersionedValue(FBUtilities.getReleaseVersionString());
@@ -263,6 +275,11 @@ public class VersionedValue implements Comparable<VersionedValue>
             return new VersionedValue(private_ip);
         }
 
+        public VersionedValue internalAddressAndPort(InetAddressAndPort address)
+        {
+            return new VersionedValue(address.toString());
+        }
+
         public VersionedValue severity(double value)
         {
             return new VersionedValue(String.valueOf(value));

src/java/org/apache/cassandra/hadoop/ConfigHelper.java

@@ -52,11 +52,13 @@ public class ConfigHelper
     private static final int DEFAULT_RANGE_BATCH_SIZE = 4096;
     private static final String INPUT_INITIAL_ADDRESS = "cassandra.input.address";
     private static final String OUTPUT_INITIAL_ADDRESS = "cassandra.output.address";
+    private static final String OUTPUT_INITIAL_PORT = "cassandra.output.port";
     private static final String READ_CONSISTENCY_LEVEL = "cassandra.consistencylevel.read";
     private static final String WRITE_CONSISTENCY_LEVEL = "cassandra.consistencylevel.write";
     private static final String OUTPUT_COMPRESSION_CLASS = "cassandra.output.compression.class";
     private static final String OUTPUT_COMPRESSION_CHUNK_LENGTH = "cassandra.output.compression.length";
     private static final String OUTPUT_LOCAL_DC_ONLY = "cassandra.output.local.dc.only";
+    private static final String DEFAULT_CASSANDRA_NATIVE_PORT = "7000";
 
     private static final Logger logger = LoggerFactory.getLogger(ConfigHelper.class);
 
@@ -349,6 +351,16 @@ public class ConfigHelper
         return conf.get(OUTPUT_INITIAL_ADDRESS);
     }
 
+    public static void setOutputInitialPort(Configuration conf, Integer port)
+    {
+        conf.set(OUTPUT_INITIAL_PORT, port.toString());
+    }
+
+    public static Integer getOutputInitialPort(Configuration conf)
+    {
+        return Integer.valueOf(conf.get(OUTPUT_INITIAL_PORT, DEFAULT_CASSANDRA_NATIVE_PORT));
+    }
+
     public static void setOutputInitialAddress(Configuration conf, String address)
     {
         conf.set(OUTPUT_INITIAL_ADDRESS, address);

src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java

@@ -21,11 +21,13 @@ import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
+import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.*;
 
+import com.google.common.net.HostAndPort;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,6 +41,7 @@ import org.apache.cassandra.hadoop.HadoopCompat;
 import org.apache.cassandra.io.sstable.CQLSSTableWriter;
 import org.apache.cassandra.io.sstable.SSTableLoader;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.streaming.StreamState;
 import org.apache.cassandra.utils.NativeSSTableLoaderClient;
@@ -80,7 +83,7 @@ public class CqlBulkRecordWriter extends RecordWriter<Object, List<ByteBuffer>>
     protected SSTableLoader loader;
     protected Progressable progress;
     protected TaskAttemptContext context;
-    protected final Set<InetAddress> ignores = new HashSet<>();
+    protected final Set<InetAddressAndPort> ignores = new HashSet<>();
 
     private String keyspace;
     private String table;
@@ -139,7 +142,7 @@ public class CqlBulkRecordWriter extends RecordWriter<Object, List<ByteBuffer>>
         try
         {
             for (String hostToIgnore : CqlBulkOutputFormat.getIgnoreHosts(conf))
-                ignores.add(InetAddress.getByName(hostToIgnore));
+                ignores.add(InetAddressAndPort.getByName(hostToIgnore));
         }
         catch (UnknownHostException e)
         {
@@ -285,20 +288,23 @@ public class CqlBulkRecordWriter extends RecordWriter<Object, List<ByteBuffer>>
         {
             super(resolveHostAddresses(conf),
                   CqlConfigHelper.getOutputNativePort(conf),
+                  ConfigHelper.getOutputInitialPort(conf),
                   ConfigHelper.getOutputKeyspaceUserName(conf),
                   ConfigHelper.getOutputKeyspacePassword(conf),
-                  CqlConfigHelper.getSSLOptions(conf).orNull());
+                  CqlConfigHelper.getSSLOptions(conf).orNull(),
+                  CqlConfigHelper.getAllowServerPortDiscovery(conf));
         }
 
-        private static Collection<InetAddress> resolveHostAddresses(Configuration conf)
+        private static Collection<InetSocketAddress> resolveHostAddresses(Configuration conf)
         {
-            Set<InetAddress> addresses = new HashSet<>();
-
+            Set<InetSocketAddress> addresses = new HashSet<>();
+            int port = CqlConfigHelper.getOutputNativePort(conf);
             for (String host : ConfigHelper.getOutputInitialAddress(conf).split(","))
             {
                 try
                 {
-                    addresses.add(InetAddress.getByName(host));
+                    HostAndPort hap = HostAndPort.fromString(host);
+                    addresses.add(new InetSocketAddress(InetAddress.getByName(hap.getHost()), hap.getPortOrDefault(port)));
                 }
                 catch (UnknownHostException e)
                 {

src/java/org/apache/cassandra/hadoop/cql3/CqlConfigHelper.java

@@ -88,6 +88,7 @@ public class CqlConfigHelper
 
     private static final String OUTPUT_CQL = "cassandra.output.cql";
     private static final String OUTPUT_NATIVE_PORT = "cassandra.output.native.port";
+    private static final String ALLOW_SERVER_PORT_DISCOVERY = "cassandra.allowserverportdiscovery";
 
     /**
      * Set the CQL columns for the input of this job.
@@ -651,4 +652,15 @@ public class CqlConfigHelper
                  new SecureRandom());
         return ctx;
     }
+
+    public static void setAllowServerPortDiscovery(Configuration conf, boolean allowServerPortDiscovery)
+    {
+        conf.set(ALLOW_SERVER_PORT_DISCOVERY, Boolean.toString(allowServerPortDiscovery));
+    }
+
+    public static boolean getAllowServerPortDiscovery(Configuration conf)
+    {
+        return Boolean.parseBoolean(conf.get(ALLOW_SERVER_PORT_DISCOVERY, "false"));
+    }
+
 }

src/java/org/apache/cassandra/hints/HintVerbHandler.java

@@ -18,13 +18,13 @@
  */
 package org.apache.cassandra.hints;
 
-import java.net.InetAddress;
 import java.util.UUID;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.MessageIn;
 import org.apache.cassandra.net.MessagingService;
@@ -47,7 +47,7 @@ public final class HintVerbHandler implements IVerbHandler<HintMessage>
     {
         UUID hostId = message.payload.hostId;
         Hint hint = message.payload.hint;
-        InetAddress address = StorageService.instance.getEndpointForHostId(hostId);
+        InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
 
         // If we see an unknown table id, it means the table, or one of the tables in the mutation, had been dropped.
         // In that case there is nothing we can really do, or should do, other than log it go on.
@@ -96,7 +96,7 @@ public final class HintVerbHandler implements IVerbHandler<HintMessage>
         }
     }
 
-    private static void reply(int id, InetAddress to)
+    private static void reply(int id, InetAddressAndPort to)
     {
         MessagingService.instance().sendReply(HintResponse.message, id, to);
     }

src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java

@@ -18,7 +18,6 @@
 package org.apache.cassandra.hints;
 
 import java.io.File;
-import java.net.InetAddress;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.*;
@@ -36,6 +35,7 @@ import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
 import org.apache.cassandra.concurrent.NamedThreadFactory;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.service.StorageService;
 
 /**
@@ -50,10 +50,10 @@ final class HintsDispatchExecutor
     private final File hintsDirectory;
     private final ExecutorService executor;
     private final AtomicBoolean isPaused;
-    private final Predicate<InetAddress> isAlive;
+    private final Predicate<InetAddressAndPort> isAlive;
     private final Map<UUID, Future> scheduledDispatches;
 
-    HintsDispatchExecutor(File hintsDirectory, int maxThreads, AtomicBoolean isPaused, Predicate<InetAddress> isAlive)
+    HintsDispatchExecutor(File hintsDirectory, int maxThreads, AtomicBoolean isPaused, Predicate<InetAddressAndPort> isAlive)
     {
         this.hintsDirectory = hintsDirectory;
         this.isPaused = isPaused;
@@ -154,7 +154,7 @@ final class HintsDispatchExecutor
         public void run()
         {
             UUID hostId = hostIdSupplier.get();
-            InetAddress address = StorageService.instance.getEndpointForHostId(hostId);
+            InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
             logger.info("Transferring all hints to {}: {}", address, hostId);
             if (transfer(hostId))
                 return;
@@ -257,7 +257,7 @@ final class HintsDispatchExecutor
         {
             logger.trace("Dispatching hints file {}", descriptor.fileName());
 
-            InetAddress address = StorageService.instance.getEndpointForHostId(hostId);
+            InetAddressAndPort address = StorageService.instance.getEndpointForHostId(hostId);
             if (address != null)
                 return deliver(descriptor, address);
 
@@ -266,7 +266,7 @@ final class HintsDispatchExecutor
             return true;
         }
 
-        private boolean deliver(HintsDescriptor descriptor, InetAddress address)
+        private boolean deliver(HintsDescriptor descriptor, InetAddressAndPort address)
         {
             File file = new File(hintsDirectory, descriptor.fileName());
             InputPosition offset = store.getDispatchOffset(descriptor);

src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java

@@ -23,7 +23,7 @@ import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.schema.Schema;
 
-import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddress;
+import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
 
 /**
  * A simple dispatch trigger that's being run every 10 seconds.

src/java/org/apache/cassandra/hints/HintsDispatcher.java

@@ -18,7 +18,6 @@
 package org.apache.cassandra.hints;
 
 import java.io.File;
-import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
@@ -31,6 +30,7 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.monitoring.ApproximateTime;
 import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.HintsServiceMetrics;
 import org.apache.cassandra.net.IAsyncCallbackWithFailure;
 import org.apache.cassandra.net.MessageIn;
@@ -51,13 +51,13 @@ final class HintsDispatcher implements AutoCloseable
 
     private final HintsReader reader;
     private final UUID hostId;
-    private final InetAddress address;
+    private final InetAddressAndPort address;
     private final int messagingVersion;
     private final BooleanSupplier abortRequested;
 
     private InputPosition currentPagePosition;
 
-    private HintsDispatcher(HintsReader reader, UUID hostId, InetAddress address, int messagingVersion, BooleanSupplier abortRequested)
+    private HintsDispatcher(HintsReader reader, UUID hostId, InetAddressAndPort address, int messagingVersion, BooleanSupplier abortRequested)
     {
         currentPagePosition = null;
 
@@ -68,7 +68,7 @@ final class HintsDispatcher implements AutoCloseable
         this.abortRequested = abortRequested;
     }
 
-    static HintsDispatcher create(File file, RateLimiter rateLimiter, InetAddress address, UUID hostId, BooleanSupplier abortRequested)
+    static HintsDispatcher create(File file, RateLimiter rateLimiter, InetAddressAndPort address, UUID hostId, BooleanSupplier abortRequested)
     {
         int messagingVersion = MessagingService.instance().getVersion(address);
         return new HintsDispatcher(HintsReader.open(file, rateLimiter), hostId, address, messagingVersion, abortRequested);
@@ -228,7 +228,7 @@ final class HintsDispatcher implements AutoCloseable
             return timedOut ? Outcome.TIMEOUT : outcome;
         }
 
-        public void onFailure(InetAddress from, RequestFailureReason failureReason)
+        public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
         {
             outcome = Outcome.FAILURE;
             condition.signalAll();

src/java/org/apache/cassandra/hints/HintsService.java

@@ -19,7 +19,6 @@ package org.apache.cassandra.hints;
 
 import java.io.File;
 import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Collections;
 import java.util.UUID;
@@ -40,6 +39,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.HintedHandoffMetrics;
 import org.apache.cassandra.metrics.StorageMetrics;
 import org.apache.cassandra.dht.Token;
@@ -267,10 +267,10 @@ public final class HintsService implements HintsServiceMBean
      */
     public void deleteAllHintsForEndpoint(String address)
     {
-        InetAddress target;
+        InetAddressAndPort target;
         try
         {
-            target = InetAddress.getByName(address);
+            target = InetAddressAndPort.getByName(address);
         }
         catch (UnknownHostException e)
         {
@@ -284,7 +284,7 @@ public final class HintsService implements HintsServiceMBean
      *
      * @param target inet address of the target node
      */
-    public void deleteAllHintsForEndpoint(InetAddress target)
+    public void deleteAllHintsForEndpoint(InetAddressAndPort target)
     {
         UUID hostId = StorageService.instance.getHostIdForEndpoint(target);
         if (hostId == null)

src/java/org/apache/cassandra/hints/HintsStore.java

@@ -19,7 +19,6 @@ package org.apache.cassandra.hints;
 
 import java.io.File;
 import java.io.IOException;
-import java.net.InetAddress;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
@@ -31,6 +30,7 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.SyncUtil;
 
@@ -77,14 +77,14 @@ final class HintsStore
         return new HintsStore(hostId, hintsDirectory, writerParams, descriptors);
     }
 
-    InetAddress address()
+    InetAddressAndPort address()
     {
         return StorageService.instance.getEndpointForHostId(hostId);
     }
 
     boolean isLive()
     {
-        InetAddress address = address();
+        InetAddressAndPort address = address();
         return address != null && FailureDetector.instance.isAlive(address);
     }
 

src/java/org/apache/cassandra/io/DummyByteVersionedSerializer.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io;
+
+import java.io.IOException;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
+
+/**
+ * Serializes a dummy byte that can't be set. Will always write 0 and return 0 in a correctly formed message.
+ */
+public class DummyByteVersionedSerializer implements IVersionedSerializer<byte[]>
+{
+    public static final DummyByteVersionedSerializer instance = new DummyByteVersionedSerializer();
+
+    private DummyByteVersionedSerializer() {}
+
+    public void serialize(byte[] bytes, DataOutputPlus out, int version) throws IOException
+    {
+        Preconditions.checkArgument(bytes == MessagingService.ONE_BYTE);
+        out.write(0);
+    }
+
+    public byte[] deserialize(DataInputPlus in, int version) throws IOException
+    {
+        assert(0 == in.readByte());
+        return MessagingService.ONE_BYTE;
+    }
+
+    public long serializedSize(byte[] bytes, int version)
+    {
+        //Payload
+        return 1;
+    }
+}

src/java/org/apache/cassandra/io/ShortVersionedSerializer.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+public class ShortVersionedSerializer implements IVersionedSerializer<Short>
+{
+
+    public static final ShortVersionedSerializer instance = new ShortVersionedSerializer();
+
+    private ShortVersionedSerializer() {}
+
+    public void serialize(Short aShort, DataOutputPlus out, int version) throws IOException
+    {
+        out.writeShort(aShort);
+    }
+
+    public Short deserialize(DataInputPlus in, int version) throws IOException
+    {
+        return in.readShort();
+    }
+
+    public long serializedSize(Short aShort, int version)
+    {
+        return 2;
+    }
+}

src/java/org/apache/cassandra/io/sstable/SSTableLoader.java

@@ -19,12 +19,12 @@ package org.apache.cassandra.io.sstable;
 
 import java.io.File;
 import java.io.IOException;
-import java.net.InetAddress;
 import java.util.*;
 
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.db.Directories;
@@ -32,7 +32,6 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.streaming.*;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Pair;
@@ -50,10 +49,10 @@ public class SSTableLoader implements StreamEventHandler
     private final Client client;
     private final int connectionsPerHost;
     private final OutputHandler outputHandler;
-    private final Set<InetAddress> failedHosts = new HashSet<>();
+    private final Set<InetAddressAndPort> failedHosts = new HashSet<>();
 
     private final List<SSTableReader> sstables = new ArrayList<>();
-    private final Multimap<InetAddress, StreamSession.SSTableStreamingSections> streamingDetails = HashMultimap.create();
+    private final Multimap<InetAddressAndPort, StreamSession.SSTableStreamingSections> streamingDetails = HashMultimap.create();
 
     public SSTableLoader(File directory, Client client, OutputHandler outputHandler)
     {
@@ -70,7 +69,7 @@ public class SSTableLoader implements StreamEventHandler
     }
 
     @SuppressWarnings("resource")
-    protected Collection<SSTableReader> openSSTables(final Map<InetAddress, Collection<Range<Token>>> ranges)
+    protected Collection<SSTableReader> openSSTables(final Map<InetAddressAndPort, Collection<Range<Token>>> ranges)
     {
         outputHandler.output("Opening sstables and calculating sections to stream");
 
@@ -124,9 +123,9 @@ public class SSTableLoader implements StreamEventHandler
 
                                               // calculate the sstable sections to stream as well as the estimated number of
                                               // keys per host
-                                              for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : ranges.entrySet())
+                                              for (Map.Entry<InetAddressAndPort, Collection<Range<Token>>> entry : ranges.entrySet())
                                               {
-                                                  InetAddress endpoint = entry.getKey();
+                                                  InetAddressAndPort endpoint = entry.getKey();
                                                   Collection<Range<Token>> tokenRanges = entry.getValue();
 
                                                   List<Pair<Long, Long>> sstableSections = sstable.getPositionsForRanges(tokenRanges);
@@ -153,17 +152,17 @@ public class SSTableLoader implements StreamEventHandler
 
     public StreamResultFuture stream()
     {
-        return stream(Collections.<InetAddress>emptySet());
+        return stream(Collections.<InetAddressAndPort>emptySet());
     }
 
-    public StreamResultFuture stream(Set<InetAddress> toIgnore, StreamEventHandler... listeners)
+    public StreamResultFuture stream(Set<InetAddressAndPort> toIgnore, StreamEventHandler... listeners)
     {
         client.init(keyspace);
         outputHandler.output("Established connection to initial hosts");
 
         StreamPlan plan = new StreamPlan(StreamOperation.BULK_LOAD, connectionsPerHost, false, false, null, PreviewKind.NONE).connectionFactory(client.getConnectionFactory());
 
-        Map<InetAddress, Collection<Range<Token>>> endpointToRanges = client.getEndpointToRangesMap();
+        Map<InetAddressAndPort, Collection<Range<Token>>> endpointToRanges = client.getEndpointToRangesMap();
         openSSTables(endpointToRanges);
         if (sstables.isEmpty())
         {
@@ -173,9 +172,9 @@ public class SSTableLoader implements StreamEventHandler
 
         outputHandler.output(String.format("Streaming relevant part of %s to %s", names(sstables), endpointToRanges.keySet()));
 
-        for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : endpointToRanges.entrySet())
+        for (Map.Entry<InetAddressAndPort, Collection<Range<Token>>> entry : endpointToRanges.entrySet())
         {
-            InetAddress remote = entry.getKey();
+            InetAddressAndPort remote = entry.getKey();
             if (toIgnore.contains(remote))
                 continue;
 
@@ -232,14 +231,14 @@ public class SSTableLoader implements StreamEventHandler
         return builder.toString();
     }
 
-    public Set<InetAddress> getFailedHosts()
+    public Set<InetAddressAndPort> getFailedHosts()
     {
         return failedHosts;
     }
 
     public static abstract class Client
     {
-        private final Map<InetAddress, Collection<Range<Token>>> endpointToRanges = new HashMap<>();
+        private final Map<InetAddressAndPort, Collection<Range<Token>>> endpointToRanges = new HashMap<>();
 
         /**
          * Initialize the client.
@@ -281,12 +280,12 @@ public class SSTableLoader implements StreamEventHandler
             throw new RuntimeException();
         }
 
-        public Map<InetAddress, Collection<Range<Token>>> getEndpointToRangesMap()
+        public Map<InetAddressAndPort, Collection<Range<Token>>> getEndpointToRangesMap()
         {
             return endpointToRanges;
         }
 
-        protected void addRangeForEndpoint(Range<Token> range, InetAddress endpoint)
+        protected void addRangeForEndpoint(Range<Token> range, InetAddressAndPort endpoint)
         {
             Collection<Range<Token>> ranges = endpointToRanges.get(endpoint);
             if (ranges == null)

src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java

@@ -17,14 +17,13 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.util.*;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 
 public abstract class AbstractEndpointSnitch implements IEndpointSnitch
 {
-    public abstract int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2);
+    public abstract int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2);
 
     /**
      * Sorts the <tt>Collection</tt> of node addresses by proximity to the given address
@@ -32,9 +31,9 @@ public abstract class AbstractEndpointSnitch implements IEndpointSnitch
      * @param unsortedAddress the nodes to sort
      * @return a new sorted <tt>List</tt>
      */
-    public List<InetAddress> getSortedListByProximity(InetAddress address, Collection<InetAddress> unsortedAddress)
+    public List<InetAddressAndPort> getSortedListByProximity(InetAddressAndPort address, Collection<InetAddressAndPort> unsortedAddress)
     {
-        List<InetAddress> preferred = new ArrayList<InetAddress>(unsortedAddress);
+        List<InetAddressAndPort> preferred = new ArrayList<>(unsortedAddress);
         sortByProximity(address, preferred);
         return preferred;
     }
@@ -44,11 +43,11 @@ public abstract class AbstractEndpointSnitch implements IEndpointSnitch
      * @param address the address to sort the proximity by
      * @param addresses the nodes to sort
      */
-    public void sortByProximity(final InetAddress address, List<InetAddress> addresses)
+    public void sortByProximity(final InetAddressAndPort address, List<InetAddressAndPort> addresses)
     {
-        Collections.sort(addresses, new Comparator<InetAddress>()
+        Collections.sort(addresses, new Comparator<InetAddressAndPort>()
         {
-            public int compare(InetAddress a1, InetAddress a2)
+            public int compare(InetAddressAndPort a1, InetAddressAndPort a2)
             {
                 return compareEndpoints(address, a1, a2);
             }
@@ -60,7 +59,7 @@ public abstract class AbstractEndpointSnitch implements IEndpointSnitch
         // noop by default
     }
 
-    public boolean isWorthMergingForRangeQuery(List<InetAddress> merged, List<InetAddress> l1, List<InetAddress> l2)
+    public boolean isWorthMergingForRangeQuery(List<InetAddressAndPort> merged, List<InetAddressAndPort> l1, List<InetAddressAndPort> l2)
     {
         // Querying remote DC is likely to be an order of magnitude slower than
         // querying locally, so 2 queries to local nodes is likely to still be
@@ -71,10 +70,10 @@ public abstract class AbstractEndpointSnitch implements IEndpointSnitch
              : true;
     }
 
-    private boolean hasRemoteNode(List<InetAddress> l)
+    private boolean hasRemoteNode(List<InetAddressAndPort> l)
     {
         String localDc = DatabaseDescriptor.getLocalDataCenter();
-        for (InetAddress ep : l)
+        for (InetAddressAndPort ep : l)
         {
             if (!localDc.equals(getDatacenter(ep)))
                 return true;

src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java

@@ -17,8 +17,6 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
-
 /**
  * An endpoint snitch tells Cassandra information about network topology that it can use to route
  * requests more efficiently.
@@ -30,16 +28,16 @@ public abstract class AbstractNetworkTopologySnitch extends AbstractEndpointSnit
      * @param endpoint a specified endpoint
      * @return string of rack
      */
-    abstract public String getRack(InetAddress endpoint);
+    abstract public String getRack(InetAddressAndPort endpoint);
 
     /**
      * Return the data center for which an endpoint resides in
      * @param endpoint a specified endpoint
      * @return string of data center
      */
-    abstract public String getDatacenter(InetAddress endpoint);
+    abstract public String getDatacenter(InetAddressAndPort endpoint);
 
-    public int compareEndpoints(InetAddress address, InetAddress a1, InetAddress a2)
+    public int compareEndpoints(InetAddressAndPort address, InetAddressAndPort a1, InetAddressAndPort a2)
     {
         if (address.equals(a1) && !address.equals(a2))
             return -1;

src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java

@@ -19,7 +19,6 @@ package org.apache.cassandra.locator;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
-import java.net.InetAddress;
 import java.util.*;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -74,9 +73,9 @@ public abstract class AbstractReplicationStrategy
         // lazy-initialize keyspace itself since we don't create them until after the replication strategies
     }
 
-    private final Map<Token, ArrayList<InetAddress>> cachedEndpoints = new NonBlockingHashMap<Token, ArrayList<InetAddress>>();
+    private final Map<Token, ArrayList<InetAddressAndPort>> cachedEndpoints = new NonBlockingHashMap<Token, ArrayList<InetAddressAndPort>>();
 
-    public ArrayList<InetAddress> getCachedEndpoints(Token t)
+    public ArrayList<InetAddressAndPort> getCachedEndpoints(Token t)
     {
         long lastVersion = tokenMetadata.getRingVersion();
 
@@ -103,21 +102,21 @@ public abstract class AbstractReplicationStrategy
      * @param searchPosition the position the natural endpoints are requested for
      * @return a copy of the natural endpoints for the given token
      */
-    public ArrayList<InetAddress> getNaturalEndpoints(RingPosition searchPosition)
+    public ArrayList<InetAddressAndPort> getNaturalEndpoints(RingPosition searchPosition)
     {
         Token searchToken = searchPosition.getToken();
         Token keyToken = TokenMetadata.firstToken(tokenMetadata.sortedTokens(), searchToken);
-        ArrayList<InetAddress> endpoints = getCachedEndpoints(keyToken);
+        ArrayList<InetAddressAndPort> endpoints = getCachedEndpoints(keyToken);
         if (endpoints == null)
         {
             TokenMetadata tm = tokenMetadata.cachedOnlyTokenMap();
             // if our cache got invalidated, it's possible there is a new token to account for too
             keyToken = TokenMetadata.firstToken(tm.sortedTokens(), searchToken);
-            endpoints = new ArrayList<InetAddress>(calculateNaturalEndpoints(searchToken, tm));
+            endpoints = new ArrayList<InetAddressAndPort>(calculateNaturalEndpoints(searchToken, tm));
             cachedEndpoints.put(keyToken, endpoints);
         }
 
-        return new ArrayList<InetAddress>(endpoints);
+        return new ArrayList<InetAddressAndPort>(endpoints);
     }
 
     /**
@@ -128,10 +127,10 @@ public abstract class AbstractReplicationStrategy
      * @param searchToken the token the natural endpoints are requested for
      * @return a copy of the natural endpoints for the given token
      */
-    public abstract List<InetAddress> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata);
+    public abstract List<InetAddressAndPort> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata);
 
-    public <T> AbstractWriteResponseHandler<T> getWriteResponseHandler(Collection<InetAddress> naturalEndpoints,
-                                                                       Collection<InetAddress> pendingEndpoints,
+    public <T> AbstractWriteResponseHandler<T> getWriteResponseHandler(Collection<InetAddressAndPort> naturalEndpoints,
+                                                                       Collection<InetAddressAndPort> pendingEndpoints,
                                                                        ConsistencyLevel consistency_level,
                                                                        Runnable callback,
                                                                        WriteType writeType,
@@ -140,8 +139,8 @@ public abstract class AbstractReplicationStrategy
         return getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, callback, writeType, queryStartNanoTime, DatabaseDescriptor.getIdealConsistencyLevel());
     }
 
-    public <T> AbstractWriteResponseHandler<T> getWriteResponseHandler(Collection<InetAddress> naturalEndpoints,
-                                                                       Collection<InetAddress> pendingEndpoints,
+    public <T> AbstractWriteResponseHandler<T> getWriteResponseHandler(Collection<InetAddressAndPort> naturalEndpoints,
+                                                                       Collection<InetAddressAndPort> pendingEndpoints,
                                                                        ConsistencyLevel consistency_level,
                                                                        Runnable callback,
                                                                        WriteType writeType,
@@ -211,14 +210,14 @@ public abstract class AbstractReplicationStrategy
      * (fixing this would probably require merging tokenmetadata into replicationstrategy,
      * so we could cache/invalidate cleanly.)
      */
-    public Multimap<InetAddress, Range<Token>> getAddressRanges(TokenMetadata metadata)
+    public Multimap<InetAddressAndPort, Range<Token>> getAddressRanges(TokenMetadata metadata)
     {
-        Multimap<InetAddress, Range<Token>> map = HashMultimap.create();
+        Multimap<InetAddressAndPort, Range<Token>> map = HashMultimap.create();
 
         for (Token token : metadata.sortedTokens())
         {
             Range<Token> range = metadata.getPrimaryRangeFor(token);
-            for (InetAddress ep : calculateNaturalEndpoints(token, metadata))
+            for (InetAddressAndPort ep : calculateNaturalEndpoints(token, metadata))
             {
                 map.put(ep, range);
             }
@@ -227,14 +226,14 @@ public abstract class AbstractReplicationStrategy
         return map;
     }
 
-    public Multimap<Range<Token>, InetAddress> getRangeAddresses(TokenMetadata metadata)
+    public Multimap<Range<Token>, InetAddressAndPort> getRangeAddresses(TokenMetadata metadata)
     {
-        Multimap<Range<Token>, InetAddress> map = HashMultimap.create();
+        Multimap<Range<Token>, InetAddressAndPort> map = HashMultimap.create();
 
         for (Token token : metadata.sortedTokens())
         {
             Range<Token> range = metadata.getPrimaryRangeFor(token);
-            for (InetAddress ep : calculateNaturalEndpoints(token, metadata))
+            for (InetAddressAndPort ep : calculateNaturalEndpoints(token, metadata))
             {
                 map.put(range, ep);
             }
@@ -243,17 +242,17 @@ public abstract class AbstractReplicationStrategy
         return map;
     }
 
-    public Multimap<InetAddress, Range<Token>> getAddressRanges()
+    public Multimap<InetAddressAndPort, Range<Token>> getAddressRanges()
     {
         return getAddressRanges(tokenMetadata.cloneOnlyTokenMap());
     }
 
-    public Collection<Range<Token>> getPendingAddressRanges(TokenMetadata metadata, Token pendingToken, InetAddress pendingAddress)
+    public Collection<Range<Token>> getPendingAddressRanges(TokenMetadata metadata, Token pendingToken, InetAddressAndPort pendingAddress)
     {
         return getPendingAddressRanges(metadata, Arrays.asList(pendingToken), pendingAddress);
     }
 
-    public Collection<Range<Token>> getPendingAddressRanges(TokenMetadata metadata, Collection<Token> pendingTokens, InetAddress pendingAddress)
+    public Collection<Range<Token>> getPendingAddressRanges(TokenMetadata metadata, Collection<Token> pendingTokens, InetAddressAndPort pendingAddress)
     {
         TokenMetadata temp = metadata.cloneOnlyTokenMap();
         temp.updateNormalTokens(pendingTokens, pendingAddress);

src/java/org/apache/cassandra/locator/CloudstackSnitch.java

@@ -24,7 +24,6 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.io.File;
 import java.net.HttpURLConnection;
-import java.net.InetAddress;
 import java.net.URL;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
@@ -56,7 +55,7 @@ public class CloudstackSnitch extends AbstractNetworkTopologySnitch
     protected static final Logger logger = LoggerFactory.getLogger(CloudstackSnitch.class);
     protected static final String ZONE_NAME_QUERY_URI = "/latest/meta-data/availability-zone";
 
-    private Map<InetAddress, Map<String, String>> savedEndpoints;
+    private Map<InetAddressAndPort, Map<String, String>> savedEndpoints;
 
     private static final String DEFAULT_DC = "UNKNOWN-DC";
     private static final String DEFAULT_RACK = "UNKNOWN-RACK";
@@ -83,9 +82,9 @@ public class CloudstackSnitch extends AbstractNetworkTopologySnitch
         csZoneRack = zone_parts[2];
     }
 
-    public String getRack(InetAddress endpoint)
+    public String getRack(InetAddressAndPort endpoint)
     {
-        if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+        if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return csZoneRack;
         EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
         if (state == null || state.getApplicationState(ApplicationState.RACK) == null)
@@ -99,9 +98,9 @@ public class CloudstackSnitch extends AbstractNetworkTopologySnitch
         return state.getApplicationState(ApplicationState.RACK).value;
     }
 
-    public String getDatacenter(InetAddress endpoint)
+    public String getDatacenter(InetAddressAndPort endpoint)
     {
-        if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+        if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return csZoneDc;
         EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
         if (state == null || state.getApplicationState(ApplicationState.DC) == null)

src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java

@@ -25,6 +25,7 @@ import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import com.codahale.metrics.ExponentiallyDecayingReservoir;
 import javax.management.MBeanServer;
@@ -63,8 +64,8 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
     private String mbeanName;
     private boolean registered = false;
 
-    private volatile HashMap<InetAddress, Double> scores = new HashMap<>();
-    private final ConcurrentHashMap<InetAddress, ExponentiallyDecayingReservoir> samples = new ConcurrentHashMap<>();
+    private volatile HashMap<InetAddressAndPort, Double> scores = new HashMap<>();
+    private final ConcurrentHashMap<InetAddressAndPort, ExponentiallyDecayingReservoir> samples = new ConcurrentHashMap<>();
 
     public final IEndpointSnitch subsnitch;
 
@@ -174,27 +175,27 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
         subsnitch.gossiperStarting();
     }
 
-    public String getRack(InetAddress endpoint)
+    public String getRack(InetAddressAndPort endpoint)
     {
         return subsnitch.getRack(endpoint);
     }
 
-    public String getDatacenter(InetAddress endpoint)
+    public String getDatacenter(InetAddressAndPort endpoint)
     {
         return subsnitch.getDatacenter(endpoint);
     }
 
-    public List<InetAddress> getSortedListByProximity(final InetAddress address, Collection<InetAddress> addresses)
+    public List<InetAddressAndPort> getSortedListByProximity(final InetAddressAndPort address, Collection<InetAddressAndPort> addresses)
     {
-        List<InetAddress> list = new ArrayList<InetAddress>(addresses);
+        List<InetAddressAndPort> list = new ArrayList<>(addresses);
         sortByProximity(address, list);
         return list;
     }
 
     @Override
-    public void sortByProximity(final InetAddress address, List<InetAddress> addresses)
+    public void sortByProximity(final InetAddressAndPort address, List<InetAddressAndPort> addresses)
     {
-        assert address.equals(FBUtilities.getBroadcastAddress()); // we only know about ourself
+        assert address.equals(FBUtilities.getBroadcastAddressAndPort()); // we only know about ourself
         if (dynamicBadnessThreshold == 0)
         {
             sortByProximityWithScore(address, addresses);
@@ -205,32 +206,32 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
         }
     }
 
-    private void sortByProximityWithScore(final InetAddress address, List<InetAddress> addresses)
+    private void sortByProximityWithScore(final InetAddressAndPort address, List<InetAddressAndPort> addresses)
     {
         // Scores can change concurrently from a call to this method. But Collections.sort() expects
         // its comparator to be "stable", that is 2 endpoint should compare the same way for the duration
         // of the sort() call. As we copy the scores map on write, it is thus enough to alias the current
         // version of it during this call.
-        final HashMap<InetAddress, Double> scores = this.scores;
-        Collections.sort(addresses, new Comparator<InetAddress>()
+        final HashMap<InetAddressAndPort, Double> scores = this.scores;
+        Collections.sort(addresses, new Comparator<InetAddressAndPort>()
         {
-            public int compare(InetAddress a1, InetAddress a2)
+            public int compare(InetAddressAndPort a1, InetAddressAndPort a2)
             {
                 return compareEndpoints(address, a1, a2, scores);
             }
         });
     }
 
-    private void sortByProximityWithBadness(final InetAddress address, List<InetAddress> addresses)
+    private void sortByProximityWithBadness(final InetAddressAndPort address, List<InetAddressAndPort> addresses)
     {
         if (addresses.size() < 2)
             return;
 
         subsnitch.sortByProximity(address, addresses);
-        HashMap<InetAddress, Double> scores = this.scores; // Make sure the score don't change in the middle of the loop below
+        HashMap<InetAddressAndPort, Double> scores = this.scores; // Make sure the score don't change in the middle of the loop below
                                                            // (which wouldn't really matter here but its cleaner that way).
         ArrayList<Double> subsnitchOrderedScores = new ArrayList<>(addresses.size());
-        for (InetAddress inet : addresses)
+        for (InetAddressAndPort inet : addresses)
         {
             Double score = scores.get(inet);
             if (score == null)
@@ -256,7 +257,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
     }
 
     // Compare endpoints given an immutable snapshot of the scores
-    private int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2, Map<InetAddress, Double> scores)
+    private int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2, Map<InetAddressAndPort, Double> scores)
     {
         Double scored1 = scores.get(a1);
         Double scored2 = scores.get(a2);
@@ -279,7 +280,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
             return 1;
     }
 
-    public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2)
+    public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2)
     {
         // That function is fundamentally unsafe because the scores can change at any time and so the result of that
         // method is not stable for identical arguments. This is why we don't rely on super.sortByProximity() in
@@ -287,7 +288,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
         throw new UnsupportedOperationException("You shouldn't wrap the DynamicEndpointSnitch (within itself or otherwise)");
     }
 
-    public void receiveTiming(InetAddress host, long latency) // this is cheap
+    public void receiveTiming(InetAddressAndPort host, long latency) // this is cheap
     {
         ExponentiallyDecayingReservoir sample = samples.get(host);
         if (sample == null)
@@ -315,23 +316,23 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
         }
         double maxLatency = 1;
 
-        Map<InetAddress, Snapshot> snapshots = new HashMap<>(samples.size());
-        for (Map.Entry<InetAddress, ExponentiallyDecayingReservoir> entry : samples.entrySet())
+        Map<InetAddressAndPort, Snapshot> snapshots = new HashMap<>(samples.size());
+        for (Map.Entry<InetAddressAndPort, ExponentiallyDecayingReservoir> entry : samples.entrySet())
         {
             snapshots.put(entry.getKey(), entry.getValue().getSnapshot());
         }
 
         // We're going to weight the latency for each host against the worst one we see, to
         // arrive at sort of a 'badness percentage' for them. First, find the worst for each:
-        HashMap<InetAddress, Double> newScores = new HashMap<>();
-        for (Map.Entry<InetAddress, Snapshot> entry : snapshots.entrySet())
+        HashMap<InetAddressAndPort, Double> newScores = new HashMap<>();
+        for (Map.Entry<InetAddressAndPort, Snapshot> entry : snapshots.entrySet())
         {
             double mean = entry.getValue().getMedian();
             if (mean > maxLatency)
                 maxLatency = mean;
         }
         // now make another pass to do the weighting based on the maximums we found before
-        for (Map.Entry<InetAddress, Snapshot> entry : snapshots.entrySet())
+        for (Map.Entry<InetAddressAndPort, Snapshot> entry : snapshots.entrySet())
         {
             double score = entry.getValue().getMedian() / maxLatency;
             // finally, add the severity without any weighting, since hosts scale this relative to their own load and the size of the task causing the severity.
@@ -350,6 +351,11 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
     }
 
     public Map<InetAddress, Double> getScores()
+    {
+        return scores.entrySet().stream().collect(Collectors.toMap(address -> address.getKey().address, Map.Entry::getValue));
+    }
+
+    public Map<InetAddressAndPort, Double> getScoresWithPort()
     {
         return scores;
     }
@@ -374,7 +380,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
 
     public List<Double> dumpTimings(String hostname) throws UnknownHostException
     {
-        InetAddress host = InetAddress.getByName(hostname);
+        InetAddressAndPort host = InetAddressAndPort.getByName(hostname);
         ArrayList<Double> timings = new ArrayList<Double>();
         ExponentiallyDecayingReservoir sample = samples.get(host);
         if (sample != null)
@@ -390,7 +396,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
         Gossiper.instance.addLocalApplicationState(ApplicationState.SEVERITY, StorageService.instance.valueFactory.severity(severity));
     }
 
-    private double getSeverity(InetAddress endpoint)
+    private double getSeverity(InetAddressAndPort endpoint)
     {
         EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
         if (state == null)
@@ -405,10 +411,10 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
 
     public double getSeverity()
     {
-        return getSeverity(FBUtilities.getBroadcastAddress());
+        return getSeverity(FBUtilities.getBroadcastAddressAndPort());
     }
 
-    public boolean isWorthMergingForRangeQuery(List<InetAddress> merged, List<InetAddress> l1, List<InetAddress> l2)
+    public boolean isWorthMergingForRangeQuery(List<InetAddressAndPort> merged, List<InetAddressAndPort> l1, List<InetAddressAndPort> l2)
     {
         if (!subsnitch.isWorthMergingForRangeQuery(merged, l1, l2))
             return false;
@@ -428,10 +434,10 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa
     }
 
     // Return the max score for the endpoint in the provided list, or -1.0 if no node have a score.
-    private double maxScore(List<InetAddress> endpoints)
+    private double maxScore(List<InetAddressAndPort> endpoints)
     {
         double maxScore = -1.0;
-        for (InetAddress endpoint : endpoints)
+        for (InetAddressAndPort endpoint : endpoints)
         {
             Double score = scores.get(endpoint);
             if (score == null)

src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java

@@ -24,6 +24,8 @@ import java.util.List;
 
 public interface DynamicEndpointSnitchMBean 
 {
+    public Map<InetAddressAndPort, Double> getScoresWithPort();
+    @Deprecated
     public Map<InetAddress, Double> getScores();
     public int getUpdateInterval();
     public int getResetInterval();

src/java/org/apache/cassandra/locator/Ec2MultiRegionSnitch.java

@@ -19,6 +19,7 @@ package org.apache.cassandra.locator;
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.net.UnknownHostException;
 
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.config.DatabaseDescriptor;
@@ -62,6 +63,16 @@ public class Ec2MultiRegionSnitch extends Ec2Snitch
     public void gossiperStarting()
     {
         super.gossiperStarting();
+        InetAddressAndPort address;
+        try
+        {
+            address = InetAddressAndPort.getByName(localPrivateAddress);
+        }
+        catch (UnknownHostException e)
+        {
+            throw new RuntimeException(e);
+        }
+        Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT, StorageService.instance.valueFactory.internalAddressAndPort(address));
         Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP, StorageService.instance.valueFactory.internalIP(localPrivateAddress));
         Gossiper.instance.register(new ReconnectableSnitchHelper(this, ec2region, true));
     }

src/java/org/apache/cassandra/locator/Ec2Snitch.java

@@ -21,7 +21,6 @@ import java.io.DataInputStream;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.net.HttpURLConnection;
-import java.net.InetAddress;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
@@ -46,7 +45,7 @@ public class Ec2Snitch extends AbstractNetworkTopologySnitch
     protected static final String ZONE_NAME_QUERY_URL = "http://169.254.169.254/latest/meta-data/placement/availability-zone";
     private static final String DEFAULT_DC = "UNKNOWN-DC";
     private static final String DEFAULT_RACK = "UNKNOWN-RACK";
-    private Map<InetAddress, Map<String, String>> savedEndpoints;
+    private Map<InetAddressAndPort, Map<String, String>> savedEndpoints;
     protected String ec2zone;
     protected String ec2region;
 
@@ -92,9 +91,9 @@ public class Ec2Snitch extends AbstractNetworkTopologySnitch
         }
     }
 
-    public String getRack(InetAddress endpoint)
+    public String getRack(InetAddressAndPort endpoint)
     {
-        if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+        if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return ec2zone;
         EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
         if (state == null || state.getApplicationState(ApplicationState.RACK) == null)
@@ -108,9 +107,9 @@ public class Ec2Snitch extends AbstractNetworkTopologySnitch
         return state.getApplicationState(ApplicationState.RACK).value;
     }
 
-    public String getDatacenter(InetAddress endpoint)
+    public String getDatacenter(InetAddressAndPort endpoint)
     {
-        if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+        if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return ec2region;
         EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
         if (state == null || state.getApplicationState(ApplicationState.DC) == null)

src/java/org/apache/cassandra/locator/EndpointSnitchInfo.java

@@ -19,7 +19,6 @@ package org.apache.cassandra.locator;
 
 
 import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
@@ -44,22 +43,22 @@ public class EndpointSnitchInfo implements EndpointSnitchInfoMBean
 
     public String getDatacenter(String host) throws UnknownHostException
     {
-        return DatabaseDescriptor.getEndpointSnitch().getDatacenter(InetAddress.getByName(host));
+        return DatabaseDescriptor.getEndpointSnitch().getDatacenter(InetAddressAndPort.getByName(host));
     }
 
     public String getRack(String host) throws UnknownHostException
     {
-        return DatabaseDescriptor.getEndpointSnitch().getRack(InetAddress.getByName(host));
+        return DatabaseDescriptor.getEndpointSnitch().getRack(InetAddressAndPort.getByName(host));
     }
 
     public String getDatacenter()
     {
-        return DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
+        return DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort());
     }
 
     public String getRack()
     {
-        return DatabaseDescriptor.getEndpointSnitch().getRack(FBUtilities.getBroadcastAddress());
+        return DatabaseDescriptor.getEndpointSnitch().getRack(FBUtilities.getBroadcastAddressAndPort());
     }
 
     public String getSnitchName()

src/java/org/apache/cassandra/locator/GoogleCloudSnitch.java

@@ -21,7 +21,6 @@ import java.io.DataInputStream;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.net.HttpURLConnection;
-import java.net.InetAddress;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
@@ -46,7 +45,7 @@ public class GoogleCloudSnitch extends AbstractNetworkTopologySnitch
     protected static final String ZONE_NAME_QUERY_URL = "http://metadata.google.internal/computeMetadata/v1/instance/zone";
     private static final String DEFAULT_DC = "UNKNOWN-DC";
     private static final String DEFAULT_RACK = "UNKNOWN-RACK";
-    private Map<InetAddress, Map<String, String>> savedEndpoints;
+    private Map<InetAddressAndPort, Map<String, String>> savedEndpoints;
     protected String gceZone;
     protected String gceRegion;
 
@@ -94,9 +93,9 @@ public class GoogleCloudSnitch extends AbstractNetworkTopologySnitch
         }
     }
 
-    public String getRack(InetAddress endpoint)
+    public String getRack(InetAddressAndPort endpoint)
     {
-        if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+        if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return gceZone;
         EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
         if (state == null || state.getApplicationState(ApplicationState.RACK) == null)
@@ -110,9 +109,9 @@ public class GoogleCloudSnitch extends AbstractNetworkTopologySnitch
         return state.getApplicationState(ApplicationState.RACK).value;
     }
 
-    public String getDatacenter(InetAddress endpoint)
+    public String getDatacenter(InetAddressAndPort endpoint)
     {
-        if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+        if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return gceRegion;
         EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
         if (state == null || state.getApplicationState(ApplicationState.DC) == null)

src/java/org/apache/cassandra/locator/GossipingPropertyFileSnitch.java

@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.Map;
 
@@ -45,7 +44,7 @@ public class GossipingPropertyFileSnitch extends AbstractNetworkTopologySnitch//
     private final boolean preferLocal;
     private final AtomicReference<ReconnectableSnitchHelper> snitchHelperReference;
 
-    private Map<InetAddress, Map<String, String>> savedEndpoints;
+    private Map<InetAddressAndPort, Map<String, String>> savedEndpoints;
     private static final String DEFAULT_DC = "UNKNOWN_DC";
     private static final String DEFAULT_RACK = "UNKNOWN_RACK";
 
@@ -84,9 +83,9 @@ public class GossipingPropertyFileSnitch extends AbstractNetworkTopologySnitch//
      * @param endpoint the endpoint to process
      * @return string of data center
      */
-    public String getDatacenter(InetAddress endpoint)
+    public String getDatacenter(InetAddressAndPort endpoint)
     {
-        if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+        if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return myDC;
 
         EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
@@ -112,9 +111,9 @@ public class GossipingPropertyFileSnitch extends AbstractNetworkTopologySnitch//
      * @param endpoint the endpoint to process
      * @return string of rack
      */
-    public String getRack(InetAddress endpoint)
+    public String getRack(InetAddressAndPort endpoint)
     {
-        if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+        if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
             return myRack;
 
         EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
@@ -138,8 +137,10 @@ public class GossipingPropertyFileSnitch extends AbstractNetworkTopologySnitch//
     {
         super.gossiperStarting();
 
+        Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT,
+                                                   StorageService.instance.valueFactory.internalAddressAndPort(FBUtilities.getLocalAddressAndPort()));
         Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP,
-                StorageService.instance.valueFactory.internalIP(FBUtilities.getLocalAddress().getHostAddress()));
+                StorageService.instance.valueFactory.internalIP(FBUtilities.getJustLocalAddress().getHostAddress()));
 
         loadGossiperState();
     }

src/java/org/apache/cassandra/locator/IEndpointSnitch.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.util.Collection;
 import java.util.List;
 
@@ -32,27 +31,27 @@ public interface IEndpointSnitch
     /**
      * returns a String representing the rack this endpoint belongs to
      */
-    public String getRack(InetAddress endpoint);
+    public String getRack(InetAddressAndPort endpoint);
 
     /**
      * returns a String representing the datacenter this endpoint belongs to
      */
-    public String getDatacenter(InetAddress endpoint);
+    public String getDatacenter(InetAddressAndPort endpoint);
 
     /**
      * returns a new <tt>List</tt> sorted by proximity to the given endpoint
      */
-    public List<InetAddress> getSortedListByProximity(InetAddress address, Collection<InetAddress> unsortedAddress);
+    public List<InetAddressAndPort> getSortedListByProximity(InetAddressAndPort address, Collection<InetAddressAndPort> unsortedAddress);
 
     /**
      * This method will sort the <tt>List</tt> by proximity to the given address.
      */
-    public void sortByProximity(InetAddress address, List<InetAddress> addresses);
+    public void sortByProximity(InetAddressAndPort address, List<InetAddressAndPort> addresses);
 
     /**
      * compares two endpoints in relation to the target endpoint, returning as Comparator.compare would
      */
-    public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2);
+    public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2);
 
     /**
      * called after Gossiper instance exists immediately before it starts gossiping
@@ -63,5 +62,5 @@ public interface IEndpointSnitch
      * Returns whether for a range query doing a query against merged is likely
      * to be faster than 2 sequential queries, one against l1 followed by one against l2.
      */
-    public boolean isWorthMergingForRangeQuery(List<InetAddress> merged, List<InetAddress> l1, List<InetAddress> l2);
+    public boolean isWorthMergingForRangeQuery(List<InetAddressAndPort> merged, List<InetAddressAndPort> l1, List<InetAddressAndPort> l2);
 }

src/java/org/apache/cassandra/locator/ILatencySubscriber.java

@@ -17,9 +17,7 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
-
 public interface ILatencySubscriber
 {
-    public void receiveTiming(InetAddress address, long latency);
+    public void receiveTiming(InetAddressAndPort address, long latency);
 }

src/java/org/apache/cassandra/locator/InetAddressAndPort.java

@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.locator;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.net.HostAndPort;
+
+import org.apache.cassandra.utils.FastByteOperations;
+
+/**
+ * A class to replace the usage of InetAddress to identify hosts in the cluster.
+ * Opting for a full replacement class so that in the future if we change the nature
+ * of the identifier the refactor will be easier in that we don't have to change the type
+ * just the methods.
+ *
+ * Because an IP might contain multiple C* instances the identification must be done
+ * using the IP + port. InetSocketAddress is undesirable for a couple of reasons. It's not comparable,
+ * it's toString() method doesn't correctly bracket IPv6, it doesn't handle optional default values,
+ * and a couple of other minor behaviors that are slightly less troublesome like handling the
+ * need to sometimes return a port and sometimes not.
+ *
+ */
+public final class InetAddressAndPort implements Comparable<InetAddressAndPort>, Serializable
+{
+    private static final long serialVersionUID = 0;
+
+    //Store these here to avoid requiring DatabaseDescriptor to be loaded. DatabaseDescriptor will set
+    //these when it loads the config. A lot of unit tests won't end up loading DatabaseDescriptor.
+    //Tools that might use this class also might not load database descriptor. Those tools are expected
+    //to always override the defaults.
+    static volatile int defaultPort = 7000;
+
+    public final InetAddress address;
+    public final byte[] addressBytes;
+    public final int port;
+
+    private InetAddressAndPort(InetAddress address, byte[] addressBytes, int port)
+    {
+        Preconditions.checkNotNull(address);
+        Preconditions.checkNotNull(addressBytes);
+        validatePortRange(port);
+        this.address = address;
+        this.port = port;
+        this.addressBytes = addressBytes;
+    }
+
+    private static void validatePortRange(int port)
+    {
+        if (port < 0 | port > 65535)
+        {
+            throw new IllegalArgumentException("Port " + port + " is not a valid port number in the range 0-65535");
+        }
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        InetAddressAndPort that = (InetAddressAndPort) o;
+
+        if (port != that.port) return false;
+        return address.equals(that.address);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        int result = address.hashCode();
+        result = 31 * result + port;
+        return result;
+    }
+
+    @Override
+    public int compareTo(InetAddressAndPort o)
+    {
+        int retval = FastByteOperations.compareUnsigned(addressBytes, 0, addressBytes.length, o.addressBytes, 0, o.addressBytes.length);
+        if (retval != 0)
+        {
+            return retval;
+        }
+
+        return Integer.compare(port, o.port);
+    }
+
+    public String getHostAddress(boolean withPort)
+    {
+        if (withPort)
+        {
+            return toString();
+        }
+        else
+        {
+            return address.getHostAddress();
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return toString(true);
+    }
+
+    public String toString(boolean withPort)
+    {
+        if (withPort)
+        {
+            return HostAndPort.fromParts(address.getHostAddress(), port).toString();
+        }
+        else
+        {
+            return address.toString();
+        }
+    }
+
+    public static InetAddressAndPort getByName(String name) throws UnknownHostException
+    {
+        return getByNameOverrideDefaults(name, null);
+    }
+
+    /**
+     *
+     * @param name Hostname + optional ports string
+     * @param port Port to connect on, overridden by values in hostname string, defaults to DatabaseDescriptor default if not specified anywhere.
+     * @return
+     * @throws UnknownHostException
+     */
+    public static InetAddressAndPort getByNameOverrideDefaults(String name, Integer port) throws UnknownHostException
+    {
+        HostAndPort hap = HostAndPort.fromString(name);
+        if (hap.hasPort())
+        {
+            port = hap.getPort();
+        }
+        return getByAddressOverrideDefaults(InetAddress.getByName(hap.getHost()), port);
+    }
+
+    public static InetAddressAndPort getByAddress(byte[] address) throws UnknownHostException
+    {
+        return getByAddressOverrideDefaults(InetAddress.getByAddress(address), address, null);
+    }
+
+    public static InetAddressAndPort getByAddress(InetAddress address)
+    {
+        return getByAddressOverrideDefaults(address, null);
+    }
+
+    public static InetAddressAndPort getByAddressOverrideDefaults(InetAddress address, Integer port)
+    {
+        if (port == null)
+        {
+            port = defaultPort;
+        }
+
+        return new InetAddressAndPort(address, address.getAddress(), port);
+    }
+
+    public static InetAddressAndPort getByAddressOverrideDefaults(InetAddress address, byte[] addressBytes, Integer port)
+    {
+        if (port == null)
+        {
+            port = defaultPort;
+        }
+
+        return new InetAddressAndPort(address, addressBytes, port);
+    }
+
+    public static InetAddressAndPort getLoopbackAddress()
+    {
+        return InetAddressAndPort.getByAddress(InetAddress.getLoopbackAddress());
+    }
+
+    public static InetAddressAndPort getLocalHost() throws UnknownHostException
+    {
+        return InetAddressAndPort.getByAddress(InetAddress.getLocalHost());
+    }
+
+    public static void initializeDefaultPort(int port)
+    {
+        defaultPort = port;
+    }
+}

src/java/org/apache/cassandra/locator/LocalStrategy.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Collection;
@@ -42,16 +41,16 @@ public class LocalStrategy extends AbstractReplicationStrategy
      * LocalStrategy may be used before tokens are set up.
      */
     @Override
-    public ArrayList<InetAddress> getNaturalEndpoints(RingPosition searchPosition)
+    public ArrayList<InetAddressAndPort> getNaturalEndpoints(RingPosition searchPosition)
     {
-        ArrayList<InetAddress> l = new ArrayList<InetAddress>(1);
-        l.add(FBUtilities.getBroadcastAddress());
+        ArrayList<InetAddressAndPort> l = new ArrayList<InetAddressAndPort>(1);
+        l.add(FBUtilities.getBroadcastAddressAndPort());
         return l;
     }
 
-    public List<InetAddress> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
+    public List<InetAddressAndPort> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
     {
-        return Collections.singletonList(FBUtilities.getBroadcastAddress());
+        return Collections.singletonList(FBUtilities.getBroadcastAddressAndPort());
     }
 
     public int getReplicationFactor()

src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.util.*;
 import java.util.Map.Entry;
 
@@ -72,7 +71,7 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
         }
 
         datacenters = Collections.unmodifiableMap(newDatacenters);
-        logger.trace("Configured datacenter replicas are {}", FBUtilities.toString(datacenters));
+        logger.info("Configured datacenter replicas are {}", FBUtilities.toString(datacenters));
     }
 
     /**
@@ -81,7 +80,7 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
     private static final class DatacenterEndpoints
     {
         /** List accepted endpoints get pushed into. */
-        Set<InetAddress> endpoints;
+        Set<InetAddressAndPort> endpoints;
         /**
          * Racks encountered so far. Replicas are put into separate racks while possible.
          * For efficiency the set is shared between the instances, using the location pair (dc, rack) to make sure
@@ -93,7 +92,7 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
         int rfLeft;
         int acceptableRackRepeats;
 
-        DatacenterEndpoints(int rf, int rackCount, int nodeCount, Set<InetAddress> endpoints, Set<Pair<String, String>> racks)
+        DatacenterEndpoints(int rf, int rackCount, int nodeCount, Set<InetAddressAndPort> endpoints, Set<Pair<String, String>> racks)
         {
             this.endpoints = endpoints;
             this.racks = racks;
@@ -108,7 +107,7 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
          * Attempts to add an endpoint to the replicas for this datacenter, adding to the endpoints set if successful.
          * Returns true if the endpoint was added, and this datacenter does not require further replicas.
          */
-        boolean addEndpointAndCheckIfDone(InetAddress ep, Pair<String,String> location)
+        boolean addEndpointAndCheckIfDone(InetAddressAndPort ep, Pair<String,String> location)
         {
             if (done())
                 return false;
@@ -143,17 +142,17 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
     /**
      * calculate endpoints in one pass through the tokens by tracking our progress in each DC.
      */
-    public List<InetAddress> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata)
+    public List<InetAddressAndPort> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata)
     {
         // we want to preserve insertion order so that the first added endpoint becomes primary
-        Set<InetAddress> replicas = new LinkedHashSet<>();
+        Set<InetAddressAndPort> replicas = new LinkedHashSet<>();
         Set<Pair<String, String>> seenRacks = new HashSet<>();
 
         Topology topology = tokenMetadata.getTopology();
         // all endpoints in each DC, so we can check when we have exhausted all the members of a DC
-        Multimap<String, InetAddress> allEndpoints = topology.getDatacenterEndpoints();
+        Multimap<String, InetAddressAndPort> allEndpoints = topology.getDatacenterEndpoints();
         // all racks in a DC so we can check when we have exhausted all racks in a DC
-        Map<String, Multimap<String, InetAddress>> racks = topology.getDatacenterRacks();
+        Map<String, Multimap<String, InetAddressAndPort>> racks = topology.getDatacenterRacks();
         assert !allEndpoints.isEmpty() && !racks.isEmpty() : "not aware of any cluster members";
 
         int dcsToFill = 0;
@@ -178,7 +177,7 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
         while (dcsToFill > 0 && tokenIter.hasNext())
         {
             Token next = tokenIter.next();
-            InetAddress ep = tokenMetadata.getEndpoint(next);
+            InetAddressAndPort ep = tokenMetadata.getEndpoint(next);
             Pair<String, String> location = topology.getLocation(ep);
             DatacenterEndpoints dcEndpoints = dcs.get(location.left);
             if (dcEndpoints != null && dcEndpoints.addEndpointAndCheckIfDone(ep, location))
@@ -227,9 +226,9 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
         final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
 
         // Add data center of localhost.
-        validDataCenters.add(snitch.getDatacenter(FBUtilities.getBroadcastAddress()));
+        validDataCenters.add(snitch.getDatacenter(FBUtilities.getBroadcastAddressAndPort()));
         // Fetch and add DCs of all peers.
-        for (final InetAddress peer : StorageService.instance.getTokenMetadata().getAllEndpoints())
+        for (final InetAddressAndPort peer : StorageService.instance.getTokenMetadata().getAllEndpoints())
         {
             validDataCenters.add(snitch.getDatacenter(peer));
         }

src/java/org/apache/cassandra/locator/OldNetworkTopologyStrategy.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Collection;
@@ -42,10 +41,10 @@ public class OldNetworkTopologyStrategy extends AbstractReplicationStrategy
         super(keyspaceName, tokenMetadata, snitch, configOptions);
     }
 
-    public List<InetAddress> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
+    public List<InetAddressAndPort> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
     {
         int replicas = getReplicationFactor();
-        List<InetAddress> endpoints = new ArrayList<InetAddress>(replicas);
+        List<InetAddressAndPort> endpoints = new ArrayList<>(replicas);
         ArrayList<Token> tokens = metadata.sortedTokens();
 
         if (tokens.isEmpty())

src/java/org/apache/cassandra/locator/PendingRangeMaps.java

@@ -26,10 +26,9 @@ import org.apache.cassandra.dht.Token;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.net.InetAddress;
 import java.util.*;
 
-public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<InetAddress>>>
+public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<InetAddressAndPort>>>
 {
     private static final Logger logger = LoggerFactory.getLogger(PendingRangeMaps.class);
 
@@ -39,7 +38,7 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
      * First two are for non-wrap-around ranges, and the last two are for wrap-around ranges.
      */
     // ascendingMap will sort the ranges by the ascending order of right token
-    final NavigableMap<Range<Token>, List<InetAddress>> ascendingMap;
+    final NavigableMap<Range<Token>, List<InetAddressAndPort>> ascendingMap;
     /**
      * sorting end ascending, if ends are same, sorting begin descending, so that token (end, end) will
      * come before (begin, end] with the same end, and (begin, end) will be selected in the tailMap.
@@ -58,7 +57,7 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
         };
 
     // ascendingMap will sort the ranges by the descending order of left token
-    final NavigableMap<Range<Token>, List<InetAddress>> descendingMap;
+    final NavigableMap<Range<Token>, List<InetAddressAndPort>> descendingMap;
     /**
      * sorting begin descending, if begins are same, sorting end descending, so that token (begin, begin) will
      * come after (begin, end] with the same begin, and (begin, end) won't be selected in the tailMap.
@@ -78,7 +77,7 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
         };
 
     // these two maps are for warp around ranges.
-    final NavigableMap<Range<Token>, List<InetAddress>> ascendingMapForWrapAround;
+    final NavigableMap<Range<Token>, List<InetAddressAndPort>> ascendingMapForWrapAround;
     /**
      * for wrap around range (begin, end], which begin > end.
      * Sorting end ascending, if ends are same, sorting begin ascending,
@@ -98,7 +97,7 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
         }
     };
 
-    final NavigableMap<Range<Token>, List<InetAddress>> descendingMapForWrapAround;
+    final NavigableMap<Range<Token>, List<InetAddressAndPort>> descendingMapForWrapAround;
     /**
      * for wrap around ranges, which begin > end.
      * Sorting end ascending, so that token (begin, begin) will come after (begin, end] with the same begin,
@@ -118,28 +117,28 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
 
     public PendingRangeMaps()
     {
-        this.ascendingMap = new TreeMap<Range<Token>, List<InetAddress>>(ascendingComparator);
-        this.descendingMap = new TreeMap<Range<Token>, List<InetAddress>>(descendingComparator);
-        this.ascendingMapForWrapAround = new TreeMap<Range<Token>, List<InetAddress>>(ascendingComparatorForWrapAround);
-        this.descendingMapForWrapAround = new TreeMap<Range<Token>, List<InetAddress>>(descendingComparatorForWrapAround);
+        this.ascendingMap = new TreeMap<Range<Token>, List<InetAddressAndPort>>(ascendingComparator);
+        this.descendingMap = new TreeMap<Range<Token>, List<InetAddressAndPort>>(descendingComparator);
+        this.ascendingMapForWrapAround = new TreeMap<Range<Token>, List<InetAddressAndPort>>(ascendingComparatorForWrapAround);
+        this.descendingMapForWrapAround = new TreeMap<Range<Token>, List<InetAddressAndPort>>(descendingComparatorForWrapAround);
     }
 
     static final void addToMap(Range<Token> range,
-                               InetAddress address,
-                               NavigableMap<Range<Token>, List<InetAddress>> ascendingMap,
-                               NavigableMap<Range<Token>, List<InetAddress>> descendingMap)
+                               InetAddressAndPort address,
+                               NavigableMap<Range<Token>, List<InetAddressAndPort>> ascendingMap,
+                               NavigableMap<Range<Token>, List<InetAddressAndPort>> descendingMap)
     {
-        List<InetAddress> addresses = ascendingMap.get(range);
+        List<InetAddressAndPort> addresses = ascendingMap.get(range);
         if (addresses == null)
         {
-            addresses = new ArrayList<InetAddress>(1);
+            addresses = new ArrayList<>(1);
             ascendingMap.put(range, addresses);
             descendingMap.put(range, addresses);
         }
         addresses.add(address);
     }
 
-    public void addPendingRange(Range<Token> range, InetAddress address)
+    public void addPendingRange(Range<Token> range, InetAddressAndPort address)
     {
         if (Range.isWrapAround(range.left, range.right))
         {
@@ -151,14 +150,14 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
         }
     }
 
-    static final void addIntersections(Set<InetAddress> endpointsToAdd,
-                                       NavigableMap<Range<Token>, List<InetAddress>> smallerMap,
-                                       NavigableMap<Range<Token>, List<InetAddress>> biggerMap)
+    static final void addIntersections(Set<InetAddressAndPort> endpointsToAdd,
+                                       NavigableMap<Range<Token>, List<InetAddressAndPort>> smallerMap,
+                                       NavigableMap<Range<Token>, List<InetAddressAndPort>> biggerMap)
     {
         // find the intersection of two sets
         for (Range<Token> range : smallerMap.keySet())
         {
-            List<InetAddress> addresses = biggerMap.get(range);
+            List<InetAddressAndPort> addresses = biggerMap.get(range);
             if (addresses != null)
             {
                 endpointsToAdd.addAll(addresses);
@@ -166,15 +165,15 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
         }
     }
 
-    public Collection<InetAddress> pendingEndpointsFor(Token token)
+    public Collection<InetAddressAndPort> pendingEndpointsFor(Token token)
     {
-        Set<InetAddress> endpoints = new HashSet<>();
+        Set<InetAddressAndPort> endpoints = new HashSet<>();
 
         Range searchRange = new Range(token, token);
 
         // search for non-wrap-around maps
-        NavigableMap<Range<Token>, List<InetAddress>> ascendingTailMap = ascendingMap.tailMap(searchRange, true);
-        NavigableMap<Range<Token>, List<InetAddress>> descendingTailMap = descendingMap.tailMap(searchRange, false);
+        NavigableMap<Range<Token>, List<InetAddressAndPort>> ascendingTailMap = ascendingMap.tailMap(searchRange, true);
+        NavigableMap<Range<Token>, List<InetAddressAndPort>> descendingTailMap = descendingMap.tailMap(searchRange, false);
 
         // add intersections of two maps
         if (ascendingTailMap.size() < descendingTailMap.size())
@@ -191,11 +190,11 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
         descendingTailMap = descendingMapForWrapAround.tailMap(searchRange, false);
 
         // add them since they are all necessary.
-        for (Map.Entry<Range<Token>, List<InetAddress>> entry : ascendingTailMap.entrySet())
+        for (Map.Entry<Range<Token>, List<InetAddressAndPort>> entry : ascendingTailMap.entrySet())
         {
             endpoints.addAll(entry.getValue());
         }
-        for (Map.Entry<Range<Token>, List<InetAddress>> entry : descendingTailMap.entrySet())
+        for (Map.Entry<Range<Token>, List<InetAddressAndPort>> entry : descendingTailMap.entrySet())
         {
             endpoints.addAll(entry.getValue());
         }
@@ -207,11 +206,11 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
     {
         StringBuilder sb = new StringBuilder();
 
-        for (Map.Entry<Range<Token>, List<InetAddress>> entry : this)
+        for (Map.Entry<Range<Token>, List<InetAddressAndPort>> entry : this)
         {
             Range<Token> range = entry.getKey();
 
-            for (InetAddress address : entry.getValue())
+            for (InetAddressAndPort address : entry.getValue())
             {
                 sb.append(address).append(':').append(range);
                 sb.append(System.getProperty("line.separator"));
@@ -222,7 +221,7 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, List<I
     }
 
     @Override
-    public Iterator<Map.Entry<Range<Token>, List<InetAddress>>> iterator()
+    public Iterator<Map.Entry<Range<Token>, List<InetAddressAndPort>>> iterator()
     {
         return Iterators.concat(ascendingMap.entrySet().iterator(), ascendingMapForWrapAround.entrySet().iterator());
     }

src/java/org/apache/cassandra/locator/PropertyFileSnitch.java

@@ -18,7 +18,6 @@
 package org.apache.cassandra.locator;
 
 import java.io.InputStream;
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -55,7 +54,7 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
     public static final String SNITCH_PROPERTIES_FILENAME = "cassandra-topology.properties";
     private static final int DEFAULT_REFRESH_PERIOD_IN_SECONDS = 5;
 
-    private static volatile Map<InetAddress, String[]> endpointMap;
+    private static volatile Map<InetAddressAndPort, String[]> endpointMap;
     private static volatile String[] defaultDCRack;
 
     private volatile boolean gossipStarted;
@@ -93,7 +92,7 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
      * @param endpoint endpoint to process
      * @return a array of string with the first index being the data center and the second being the rack
      */
-    public static String[] getEndpointInfo(InetAddress endpoint)
+    public static String[] getEndpointInfo(InetAddressAndPort endpoint)
     {
         String[] rawEndpointInfo = getRawEndpointInfo(endpoint);
         if (rawEndpointInfo == null)
@@ -101,7 +100,7 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
         return rawEndpointInfo;
     }
 
-    private static String[] getRawEndpointInfo(InetAddress endpoint)
+    private static String[] getRawEndpointInfo(InetAddressAndPort endpoint)
     {
         String[] value = endpointMap.get(endpoint);
         if (value == null)
@@ -118,7 +117,7 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
      * @param endpoint the endpoint to process
      * @return string of data center
      */
-    public String getDatacenter(InetAddress endpoint)
+    public String getDatacenter(InetAddressAndPort endpoint)
     {
         String[] info = getEndpointInfo(endpoint);
         assert info != null : "No location defined for endpoint " + endpoint;
@@ -131,7 +130,7 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
      * @param endpoint the endpoint to process
      * @return string of rack
      */
-    public String getRack(InetAddress endpoint)
+    public String getRack(InetAddressAndPort endpoint)
     {
         String[] info = getEndpointInfo(endpoint);
         assert info != null : "No location defined for endpoint " + endpoint;
@@ -140,7 +139,7 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
 
     public void reloadConfiguration(boolean isUpdate) throws ConfigurationException
     {
-        HashMap<InetAddress, String[]> reloadedMap = new HashMap<>();
+        HashMap<InetAddressAndPort, String[]> reloadedMap = new HashMap<>();
         String[] reloadedDefaultDCRack = null;
 
         Properties properties = new Properties();
@@ -168,11 +167,11 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
             }
             else
             {
-                InetAddress host;
+                InetAddressAndPort host;
                 String hostString = StringUtils.remove(key, '/');
                 try
                 {
-                    host = InetAddress.getByName(hostString);
+                    host = InetAddressAndPort.getByName(hostString);
                 }
                 catch (UnknownHostException e)
                 {
@@ -186,7 +185,7 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
                 reloadedMap.put(host, token);
             }
         }
-        InetAddress broadcastAddress = FBUtilities.getBroadcastAddress();
+        InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort();
         String[] localInfo = reloadedMap.get(broadcastAddress);
         if (reloadedDefaultDCRack == null && localInfo == null)
             throw new ConfigurationException(String.format("Snitch definitions at %s do not define a location for " +
@@ -194,7 +193,7 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
                                                            SNITCH_PROPERTIES_FILENAME, broadcastAddress));
         // internode messaging code converts our broadcast address to local,
         // make sure we can be found at that as well.
-        InetAddress localAddress = FBUtilities.getLocalAddress();
+        InetAddressAndPort localAddress = FBUtilities.getLocalAddressAndPort();
         if (!localAddress.equals(broadcastAddress) && !reloadedMap.containsKey(localAddress))
             reloadedMap.put(localAddress, localInfo);
 
@@ -204,7 +203,7 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
         if (logger.isTraceEnabled())
         {
             StringBuilder sb = new StringBuilder();
-            for (Map.Entry<InetAddress, String[]> entry : reloadedMap.entrySet())
+            for (Map.Entry<InetAddressAndPort, String[]> entry : reloadedMap.entrySet())
                 sb.append(entry.getKey()).append(':').append(Arrays.toString(entry.getValue())).append(", ");
             logger.trace("Loaded network topology from property file: {}", StringUtils.removeEnd(sb.toString(), ", "));
         }
@@ -231,17 +230,17 @@ public class PropertyFileSnitch extends AbstractNetworkTopologySnitch
      * @param reloadedDefaultDCRack - the default dc:rack or null if no default
      * @return true if we can continue updating (no live host had dc or rack updated)
      */
-    private static boolean livenessCheck(HashMap<InetAddress, String[]> reloadedMap, String[] reloadedDefaultDCRack)
+    private static boolean livenessCheck(HashMap<InetAddressAndPort, String[]> reloadedMap, String[] reloadedDefaultDCRack)
     {
         // If the default has changed we must check all live hosts but hopefully we will find a live
         // host quickly and interrupt the loop. Otherwise we only check the live hosts that were either
         // in the old set or in the new set
-        Set<InetAddress> hosts = Arrays.equals(defaultDCRack, reloadedDefaultDCRack)
+        Set<InetAddressAndPort> hosts = Arrays.equals(defaultDCRack, reloadedDefaultDCRack)
                                  ? Sets.intersection(StorageService.instance.getLiveRingMembers(), // same default
                                                      Sets.union(endpointMap.keySet(), reloadedMap.keySet()))
                                  : StorageService.instance.getLiveRingMembers(); // default updated
 
-        for (InetAddress host : hosts)
+        for (InetAddressAndPort host : hosts)
         {
             String[] origValue = endpointMap.containsKey(host) ? endpointMap.get(host) : defaultDCRack;
             String[] updateValue = reloadedMap.containsKey(host) ? reloadedMap.get(host) : reloadedDefaultDCRack;

src/java/org/apache/cassandra/locator/RackInferringSnitch.java

@@ -17,21 +17,19 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
-
 /**
  * A simple endpoint snitch implementation that assumes datacenter and rack information is encoded
  * in the 2nd and 3rd octets of the ip address, respectively.
  */
 public class RackInferringSnitch extends AbstractNetworkTopologySnitch
 {
-    public String getRack(InetAddress endpoint)
+    public String getRack(InetAddressAndPort endpoint)
     {
-        return Integer.toString(endpoint.getAddress()[2] & 0xFF, 10);
+        return Integer.toString(endpoint.address.getAddress()[2] & 0xFF, 10);
     }
 
-    public String getDatacenter(InetAddress endpoint)
+    public String getDatacenter(InetAddressAndPort endpoint)
     {
-        return Integer.toString(endpoint.getAddress()[1] & 0xFF, 10);
+        return Integer.toString(endpoint.address.getAddress()[1] & 0xFF, 10);
     }
 }

src/java/org/apache/cassandra/locator/ReconnectableSnitchHelper.java

@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -49,11 +48,11 @@ public class ReconnectableSnitchHelper implements IEndpointStateChangeSubscriber
         this.preferLocal = preferLocal;
     }
 
-    private void reconnect(InetAddress publicAddress, VersionedValue localAddressValue)
+    private void reconnect(InetAddressAndPort publicAddress, VersionedValue localAddressValue)
     {
         try
         {
-            reconnect(publicAddress, InetAddress.getByName(localAddressValue.value), snitch, localDc);
+            reconnect(publicAddress, InetAddressAndPort.getByName(localAddressValue.value), snitch, localDc);
         }
         catch (UnknownHostException e)
         {
@@ -62,9 +61,9 @@ public class ReconnectableSnitchHelper implements IEndpointStateChangeSubscriber
     }
 
     @VisibleForTesting
-    static void reconnect(InetAddress publicAddress, InetAddress localAddress, IEndpointSnitch snitch, String localDc)
+    static void reconnect(InetAddressAndPort publicAddress, InetAddressAndPort localAddress, IEndpointSnitch snitch, String localDc)
     {
-        if (!DatabaseDescriptor.getInternodeAuthenticator().authenticate(publicAddress, MessagingService.instance().portFor(publicAddress)))
+        if (!DatabaseDescriptor.getInternodeAuthenticator().authenticate(publicAddress.address, MessagingService.instance().portFor(publicAddress)))
         {
             logger.debug("InternodeAuthenticator said don't reconnect to {} on {}", publicAddress, localAddress);
             return;
@@ -78,40 +77,65 @@ public class ReconnectableSnitchHelper implements IEndpointStateChangeSubscriber
         }
     }
 
-    public void beforeChange(InetAddress endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue)
+    public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue)
     {
         // no-op
     }
 
-    public void onJoin(InetAddress endpoint, EndpointState epState)
+    public void onJoin(InetAddressAndPort endpoint, EndpointState epState)
     {
-        if (preferLocal && !Gossiper.instance.isDeadState(epState) && epState.getApplicationState(ApplicationState.INTERNAL_IP) != null)
-            reconnect(endpoint, epState.getApplicationState(ApplicationState.INTERNAL_IP));
+        if (preferLocal && !Gossiper.instance.isDeadState(epState))
+        {
+            VersionedValue address = epState.getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+            if (address == null)
+            {
+                address = epState.getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+            }
+            if (address != null)
+            {
+                reconnect(endpoint, address);
+            }
+        }
     }
 
-    public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value)
+    //Skeptical this will always do the right thing all the time port wise. It will converge on the right thing
+    //eventually once INTERNAL_ADDRESS_AND_PORT is populated
+    public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value)
     {
-        if (preferLocal && state == ApplicationState.INTERNAL_IP && !Gossiper.instance.isDeadState(Gossiper.instance.getEndpointStateForEndpoint(endpoint)))
-            reconnect(endpoint, value);
+        if (preferLocal && !Gossiper.instance.isDeadState(Gossiper.instance.getEndpointStateForEndpoint(endpoint)))
+        {
+            if (state == ApplicationState.INTERNAL_ADDRESS_AND_PORT)
+            {
+                reconnect(endpoint, value);
+            }
+            else if (state == ApplicationState.INTERNAL_IP &&
+                     null == Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT))
+            {
+                //Only use INTERNAL_IP if INTERNAL_ADDRESS_AND_PORT is unavailable
+                reconnect(endpoint, value);
+            }
+        }
     }
 
-    public void onAlive(InetAddress endpoint, EndpointState state)
+    public void onAlive(InetAddressAndPort endpoint, EndpointState state)
     {
-        if (preferLocal && state.getApplicationState(ApplicationState.INTERNAL_IP) != null)
-            reconnect(endpoint, state.getApplicationState(ApplicationState.INTERNAL_IP));
+        VersionedValue internalIP = state.getApplicationState(ApplicationState.INTERNAL_IP);
+        VersionedValue internalIPAndPorts = state.getApplicationState(ApplicationState.INTERNAL_ADDRESS_AND_PORT);
+        if (preferLocal && internalIP != null)
+            reconnect(endpoint, internalIPAndPorts != null ? internalIPAndPorts : internalIP);
     }
 
-    public void onDead(InetAddress endpoint, EndpointState state)
+    public void onDead(InetAddressAndPort endpoint, EndpointState state)
     {
         // do nothing.
     }
 
-    public void onRemove(InetAddress endpoint)
+    public void onRemove(InetAddressAndPort endpoint)
     {
         // do nothing.
     }
 
-    public void onRestart(InetAddress endpoint, EndpointState state)
+    public void onRestart(InetAddressAndPort endpoint, EndpointState state)
     {
         // do nothing.
     }

src/java/org/apache/cassandra/locator/SeedProvider.java

@@ -17,10 +17,9 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.util.List;
 
 public interface SeedProvider
 {
-    List<InetAddress> getSeeds();
+    List<InetAddressAndPort> getSeeds();
 }

src/java/org/apache/cassandra/locator/SimpleSeedProvider.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -26,6 +25,7 @@ import java.util.Map;
 
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,7 +35,7 @@ public class SimpleSeedProvider implements SeedProvider
 
     public SimpleSeedProvider(Map<String, String> args) {}
 
-    public List<InetAddress> getSeeds()
+    public List<InetAddressAndPort> getSeeds()
     {
         Config conf;
         try
@@ -47,12 +47,12 @@ public class SimpleSeedProvider implements SeedProvider
             throw new AssertionError(e);
         }
         String[] hosts = conf.seed_provider.parameters.get("seeds").split(",", -1);
-        List<InetAddress> seeds = new ArrayList<InetAddress>(hosts.length);
+        List<InetAddressAndPort> seeds = new ArrayList<>(hosts.length);
         for (String host : hosts)
         {
             try
             {
-                seeds.add(InetAddress.getByName(host.trim()));
+                seeds.add(InetAddressAndPort.getByName(host.trim()));
             }
             catch (UnknownHostException ex)
             {

src/java/org/apache/cassandra/locator/SimpleSnitch.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.util.List;
 
 /**
@@ -27,23 +26,23 @@ import java.util.List;
  */
 public class SimpleSnitch extends AbstractEndpointSnitch
 {
-    public String getRack(InetAddress endpoint)
+    public String getRack(InetAddressAndPort endpoint)
     {
         return "rack1";
     }
 
-    public String getDatacenter(InetAddress endpoint)
+    public String getDatacenter(InetAddressAndPort endpoint)
     {
         return "datacenter1";
     }
 
     @Override
-    public void sortByProximity(final InetAddress address, List<InetAddress> addresses)
+    public void sortByProximity(final InetAddressAndPort address, List<InetAddressAndPort> addresses)
     {
         // Optimization to avoid walking the list
     }
 
-    public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2)
+    public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2)
     {
         // Making all endpoints equal ensures we won't change the original ordering (since
         // Collections.sort is guaranteed to be stable)

src/java/org/apache/cassandra/locator/SimpleStrategy.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Collection;
@@ -42,11 +41,11 @@ public class SimpleStrategy extends AbstractReplicationStrategy
         super(keyspaceName, tokenMetadata, snitch, configOptions);
     }
 
-    public List<InetAddress> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
+    public List<InetAddressAndPort> calculateNaturalEndpoints(Token token, TokenMetadata metadata)
     {
         int replicas = getReplicationFactor();
         ArrayList<Token> tokens = metadata.sortedTokens();
-        List<InetAddress> endpoints = new ArrayList<InetAddress>(replicas);
+        List<InetAddressAndPort> endpoints = new ArrayList<InetAddressAndPort>(replicas);
 
         if (tokens.isEmpty())
             return endpoints;
@@ -55,7 +54,7 @@ public class SimpleStrategy extends AbstractReplicationStrategy
         Iterator<Token> iter = TokenMetadata.ringIterator(tokens, token, false);
         while (endpoints.size() < replicas && iter.hasNext())
         {
-            InetAddress ep = metadata.getEndpoint(iter.next());
+            InetAddressAndPort ep = metadata.getEndpoint(iter.next());
             if (!endpoints.contains(ep))
                 endpoints.add(ep);
         }

src/java/org/apache/cassandra/locator/TokenMetadata.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.locator;
 
-import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
@@ -52,12 +51,12 @@ public class TokenMetadata
      * Each Token is associated with exactly one Address, but each Address may have
      * multiple tokens.  Hence, the BiMultiValMap collection.
      */
-    private final BiMultiValMap<Token, InetAddress> tokenToEndpointMap;
+    private final BiMultiValMap<Token, InetAddressAndPort> tokenToEndpointMap;
 
     /** Maintains endpoint to host ID map of every node in the cluster */
-    private final BiMap<InetAddress, UUID> endpointToHostIdMap;
+    private final BiMap<InetAddressAndPort, UUID> endpointToHostIdMap;
 
-    // Prior to CASSANDRA-603, we just had <tt>Map<Range, InetAddress> pendingRanges<tt>,
+    // Prior to CASSANDRA-603, we just had <tt>Map<Range, InetAddressAndPort> pendingRanges<tt>,
     // which was added to when a node began bootstrap and removed from when it finished.
     //
     // This is inadequate when multiple changes are allowed simultaneously.  For example,
@@ -70,8 +69,8 @@ public class TokenMetadata
     //
     // So, we made two changes:
     //
-    // First, we changed pendingRanges to a <tt>Multimap<Range, InetAddress></tt> (now
-    // <tt>Map<String, Multimap<Range, InetAddress>></tt>, because replication strategy
+    // First, we changed pendingRanges to a <tt>Multimap<Range, InetAddressAndPort></tt> (now
+    // <tt>Map<String, Multimap<Range, InetAddressAndPort>></tt>, because replication strategy
     // and options are per-KeySpace).
     //
     // Second, we added the bootstrapTokens and leavingEndpoints collections, so we can
@@ -81,17 +80,17 @@ public class TokenMetadata
     // Finally, note that recording the tokens of joining nodes in bootstrapTokens also
     // means we can detect and reject the addition of multiple nodes at the same token
     // before one becomes part of the ring.
-    private final BiMultiValMap<Token, InetAddress> bootstrapTokens = new BiMultiValMap<>();
+    private final BiMultiValMap<Token, InetAddressAndPort> bootstrapTokens = new BiMultiValMap<>();
 
-    private final BiMap<InetAddress, InetAddress> replacementToOriginal = HashBiMap.create();
+    private final BiMap<InetAddressAndPort, InetAddressAndPort> replacementToOriginal = HashBiMap.create();
 
     // (don't need to record Token here since it's still part of tokenToEndpointMap until it's done leaving)
-    private final Set<InetAddress> leavingEndpoints = new HashSet<>();
+    private final Set<InetAddressAndPort> leavingEndpoints = new HashSet<>();
     // this is a cache of the calculation from {tokenToEndpointMap, bootstrapTokens, leavingEndpoints}
     private final ConcurrentMap<String, PendingRangeMaps> pendingRanges = new ConcurrentHashMap<String, PendingRangeMaps>();
 
     // nodes which are migrating to the new tokens in the ring
-    private final Set<Pair<Token, InetAddress>> movingEndpoints = new HashSet<>();
+    private final Set<Pair<Token, InetAddressAndPort>> movingEndpoints = new HashSet<>();
 
     /* Use this lock for manipulating the token map */
     private final ReadWriteLock lock = new ReentrantReadWriteLock(true);
@@ -100,26 +99,18 @@ public class TokenMetadata
     private final Topology topology;
     public final IPartitioner partitioner;
 
-    private static final Comparator<InetAddress> inetaddressCmp = new Comparator<InetAddress>()
-    {
-        public int compare(InetAddress o1, InetAddress o2)
-        {
-            return ByteBuffer.wrap(o1.getAddress()).compareTo(ByteBuffer.wrap(o2.getAddress()));
-        }
-    };
-
     // signals replication strategies that nodes have joined or left the ring and they need to recompute ownership
     private volatile long ringVersion = 0;
 
     public TokenMetadata()
     {
-        this(SortedBiMultiValMap.<Token, InetAddress>create(null, inetaddressCmp),
-             HashBiMap.<InetAddress, UUID>create(),
+        this(SortedBiMultiValMap.<Token, InetAddressAndPort>create(),
+             HashBiMap.create(),
              new Topology(),
              DatabaseDescriptor.getPartitioner());
     }
 
-    private TokenMetadata(BiMultiValMap<Token, InetAddress> tokenToEndpointMap, BiMap<InetAddress, UUID> endpointsMap, Topology topology, IPartitioner partitioner)
+    private TokenMetadata(BiMultiValMap<Token, InetAddressAndPort> tokenToEndpointMap, BiMap<InetAddressAndPort, UUID> endpointsMap, Topology topology, IPartitioner partitioner)
     {
         this.tokenToEndpointMap = tokenToEndpointMap;
         this.topology = topology;
@@ -143,7 +134,7 @@ public class TokenMetadata
     }
 
     /** @return the number of nodes bootstrapping into source's primary range */
-    public int pendingRangeChanges(InetAddress source)
+    public int pendingRangeChanges(InetAddressAndPort source)
     {
         int n = 0;
         Collection<Range<Token>> sourceRanges = getPrimaryRangesFor(getTokens(source));
@@ -165,14 +156,14 @@ public class TokenMetadata
     /**
      * Update token map with a single token/endpoint pair in normal state.
      */
-    public void updateNormalToken(Token token, InetAddress endpoint)
+    public void updateNormalToken(Token token, InetAddressAndPort endpoint)
     {
         updateNormalTokens(Collections.singleton(token), endpoint);
     }
 
-    public void updateNormalTokens(Collection<Token> tokens, InetAddress endpoint)
+    public void updateNormalTokens(Collection<Token> tokens, InetAddressAndPort endpoint)
     {
-        Multimap<InetAddress, Token> endpointTokens = HashMultimap.create();
+        Multimap<InetAddressAndPort, Token> endpointTokens = HashMultimap.create();
         for (Token token : tokens)
             endpointTokens.put(endpoint, token);
         updateNormalTokens(endpointTokens);
@@ -184,7 +175,7 @@ public class TokenMetadata
      * Prefer this whenever there are multiple pairs to update, as each update (whether a single or multiple)
      * is expensive (CASSANDRA-3831).
      */
-    public void updateNormalTokens(Multimap<InetAddress, Token> endpointTokens)
+    public void updateNormalTokens(Multimap<InetAddressAndPort, Token> endpointTokens)
     {
         if (endpointTokens.isEmpty())
             return;
@@ -193,7 +184,7 @@ public class TokenMetadata
         try
         {
             boolean shouldSortTokens = false;
-            for (InetAddress endpoint : endpointTokens.keySet())
+            for (InetAddressAndPort endpoint : endpointTokens.keySet())
             {
                 Collection<Token> tokens = endpointTokens.get(endpoint);
 
@@ -208,7 +199,7 @@ public class TokenMetadata
 
                 for (Token token : tokens)
                 {
-                    InetAddress prev = tokenToEndpointMap.put(token, endpoint);
+                    InetAddressAndPort prev = tokenToEndpointMap.put(token, endpoint);
                     if (!endpoint.equals(prev))
                     {
                         if (prev != null)
@@ -231,7 +222,7 @@ public class TokenMetadata
      * Store an end-point to host ID mapping.  Each ID must be unique, and
      * cannot be changed after the fact.
      */
-    public void updateHostId(UUID hostId, InetAddress endpoint)
+    public void updateHostId(UUID hostId, InetAddressAndPort endpoint)
     {
         assert hostId != null;
         assert endpoint != null;
@@ -239,7 +230,7 @@ public class TokenMetadata
         lock.writeLock().lock();
         try
         {
-            InetAddress storedEp = endpointToHostIdMap.inverse().get(hostId);
+            InetAddressAndPort storedEp = endpointToHostIdMap.inverse().get(hostId);
             if (storedEp != null)
             {
                 if (!storedEp.equals(endpoint) && (FailureDetector.instance.isAlive(storedEp)))
@@ -265,7 +256,7 @@ public class TokenMetadata
     }
 
     /** Return the unique host ID for an end-point. */
-    public UUID getHostId(InetAddress endpoint)
+    public UUID getHostId(InetAddressAndPort endpoint)
     {
         lock.readLock().lock();
         try
@@ -279,7 +270,7 @@ public class TokenMetadata
     }
 
     /** Return the end-point for a unique host ID */
-    public InetAddress getEndpointForHostId(UUID hostId)
+    public InetAddressAndPort getEndpointForHostId(UUID hostId)
     {
         lock.readLock().lock();
         try
@@ -293,12 +284,12 @@ public class TokenMetadata
     }
 
     /** @return a copy of the endpoint-to-id map for read-only operations */
-    public Map<InetAddress, UUID> getEndpointToHostIdMapForReading()
+    public Map<InetAddressAndPort, UUID> getEndpointToHostIdMapForReading()
     {
         lock.readLock().lock();
         try
         {
-            Map<InetAddress, UUID> readMap = new HashMap<>();
+            Map<InetAddressAndPort, UUID> readMap = new HashMap<>();
             readMap.putAll(endpointToHostIdMap);
             return readMap;
         }
@@ -309,17 +300,17 @@ public class TokenMetadata
     }
 
     @Deprecated
-    public void addBootstrapToken(Token token, InetAddress endpoint)
+    public void addBootstrapToken(Token token, InetAddressAndPort endpoint)
     {
         addBootstrapTokens(Collections.singleton(token), endpoint);
     }
 
-    public void addBootstrapTokens(Collection<Token> tokens, InetAddress endpoint)
+    public void addBootstrapTokens(Collection<Token> tokens, InetAddressAndPort endpoint)
     {
         addBootstrapTokens(tokens, endpoint, null);
     }
 
-    private void addBootstrapTokens(Collection<Token> tokens, InetAddress endpoint, InetAddress original)
+    private void addBootstrapTokens(Collection<Token> tokens, InetAddressAndPort endpoint, InetAddressAndPort original)
     {
         assert tokens != null && !tokens.isEmpty();
         assert endpoint != null;
@@ -328,7 +319,7 @@ public class TokenMetadata
         try
         {
 
-            InetAddress oldEndpoint;
+            InetAddressAndPort oldEndpoint;
 
             for (Token token : tokens)
             {
@@ -352,7 +343,7 @@ public class TokenMetadata
         }
     }
 
-    public void addReplaceTokens(Collection<Token> replacingTokens, InetAddress newNode, InetAddress oldNode)
+    public void addReplaceTokens(Collection<Token> replacingTokens, InetAddressAndPort newNode, InetAddressAndPort oldNode)
     {
         assert replacingTokens != null && !replacingTokens.isEmpty();
         assert newNode != null && oldNode != null;
@@ -379,12 +370,12 @@ public class TokenMetadata
         }
     }
 
-    public Optional<InetAddress> getReplacementNode(InetAddress endpoint)
+    public Optional<InetAddressAndPort> getReplacementNode(InetAddressAndPort endpoint)
     {
         return Optional.ofNullable(replacementToOriginal.inverse().get(endpoint));
     }
 
-    public Optional<InetAddress> getReplacingNode(InetAddress endpoint)
+    public Optional<InetAddressAndPort> getReplacingNode(InetAddressAndPort endpoint)
     {
         return Optional.ofNullable((replacementToOriginal.get(endpoint)));
     }
@@ -405,7 +396,7 @@ public class TokenMetadata
         }
     }
 
-    public void addLeavingEndpoint(InetAddress endpoint)
+    public void addLeavingEndpoint(InetAddressAndPort endpoint)
     {
         assert endpoint != null;
 
@@ -425,7 +416,7 @@ public class TokenMetadata
      * @param token token which is node moving to
      * @param endpoint address of the moving node
      */
-    public void addMovingEndpoint(Token token, InetAddress endpoint)
+    public void addMovingEndpoint(Token token, InetAddressAndPort endpoint)
     {
         assert endpoint != null;
 
@@ -441,7 +432,7 @@ public class TokenMetadata
         }
     }
 
-    public void removeEndpoint(InetAddress endpoint)
+    public void removeEndpoint(InetAddressAndPort endpoint)
     {
         assert endpoint != null;
 
@@ -469,7 +460,7 @@ public class TokenMetadata
     /**
      * This is called when the snitch properties for this endpoint are updated, see CASSANDRA-10238.
      */
-    public void updateTopology(InetAddress endpoint)
+    public void updateTopology(InetAddressAndPort endpoint)
     {
         assert endpoint != null;
 
@@ -509,14 +500,14 @@ public class TokenMetadata
      * Remove pair of token/address from moving endpoints
      * @param endpoint address of the moving node
      */
-    public void removeFromMoving(InetAddress endpoint)
+    public void removeFromMoving(InetAddressAndPort endpoint)
     {
         assert endpoint != null;
 
         lock.writeLock().lock();
         try
         {
-            for (Pair<Token, InetAddress> pair : movingEndpoints)
+            for (Pair<Token, InetAddressAndPort> pair : movingEndpoints)
             {
                 if (pair.right.equals(endpoint))
                 {
@@ -533,7 +524,7 @@ public class TokenMetadata
         }
     }
 
-    public Collection<Token> getTokens(InetAddress endpoint)
+    public Collection<Token> getTokens(InetAddressAndPort endpoint)
     {
         assert endpoint != null;
         assert isMember(endpoint); // don't want to return nulls
@@ -550,12 +541,12 @@ public class TokenMetadata
     }
 
     @Deprecated
-    public Token getToken(InetAddress endpoint)
+    public Token getToken(InetAddressAndPort endpoint)
     {
         return getTokens(endpoint).iterator().next();
     }
 
-    public boolean isMember(InetAddress endpoint)
+    public boolean isMember(InetAddressAndPort endpoint)
     {
         assert endpoint != null;
 
@@ -570,7 +561,7 @@ public class TokenMetadata
         }
     }
 
-    public boolean isLeaving(InetAddress endpoint)
+    public boolean isLeaving(InetAddressAndPort endpoint)
     {
         assert endpoint != null;
 
@@ -585,7 +576,7 @@ public class TokenMetadata
         }
     }
 
-    public boolean isMoving(InetAddress endpoint)
+    public boolean isMoving(InetAddressAndPort endpoint)
     {
         assert endpoint != null;
 
@@ -593,7 +584,7 @@ public class TokenMetadata
 
         try
         {
-            for (Pair<Token, InetAddress> pair : movingEndpoints)
+            for (Pair<Token, InetAddressAndPort> pair : movingEndpoints)
             {
                 if (pair.right.equals(endpoint))
                     return true;
@@ -618,7 +609,7 @@ public class TokenMetadata
         lock.readLock().lock();
         try
         {
-            return new TokenMetadata(SortedBiMultiValMap.create(tokenToEndpointMap, null, inetaddressCmp),
+            return new TokenMetadata(SortedBiMultiValMap.create(tokenToEndpointMap),
                                      HashBiMap.create(endpointToHostIdMap),
                                      new Topology(topology),
                                      partitioner);
@@ -673,9 +664,9 @@ public class TokenMetadata
         }
     }
 
-    private static TokenMetadata removeEndpoints(TokenMetadata allLeftMetadata, Set<InetAddress> leavingEndpoints)
+    private static TokenMetadata removeEndpoints(TokenMetadata allLeftMetadata, Set<InetAddressAndPort> leavingEndpoints)
     {
-        for (InetAddress endpoint : leavingEndpoints)
+        for (InetAddressAndPort endpoint : leavingEndpoints)
             allLeftMetadata.removeEndpoint(endpoint);
 
         return allLeftMetadata;
@@ -695,11 +686,11 @@ public class TokenMetadata
         {
             TokenMetadata metadata = cloneOnlyTokenMap();
 
-            for (InetAddress endpoint : leavingEndpoints)
+            for (InetAddressAndPort endpoint : leavingEndpoints)
                 metadata.removeEndpoint(endpoint);
 
 
-            for (Pair<Token, InetAddress> pair : movingEndpoints)
+            for (Pair<Token, InetAddressAndPort> pair : movingEndpoints)
                 metadata.updateNormalToken(pair.left, pair.right);
 
             return metadata;
@@ -710,7 +701,7 @@ public class TokenMetadata
         }
     }
 
-    public InetAddress getEndpoint(Token token)
+    public InetAddressAndPort getEndpoint(Token token)
     {
         lock.readLock().lock();
         try
@@ -742,17 +733,17 @@ public class TokenMetadata
         return sortedTokens;
     }
 
-    public Multimap<Range<Token>, InetAddress> getPendingRangesMM(String keyspaceName)
+    public Multimap<Range<Token>, InetAddressAndPort> getPendingRangesMM(String keyspaceName)
     {
-        Multimap<Range<Token>, InetAddress> map = HashMultimap.create();
+        Multimap<Range<Token>, InetAddressAndPort> map = HashMultimap.create();
         PendingRangeMaps pendingRangeMaps = this.pendingRanges.get(keyspaceName);
 
         if (pendingRangeMaps != null)
         {
-            for (Map.Entry<Range<Token>, List<InetAddress>> entry : pendingRangeMaps)
+            for (Map.Entry<Range<Token>, List<InetAddressAndPort>> entry : pendingRangeMaps)
             {
                 Range<Token> range = entry.getKey();
-                for (InetAddress address : entry.getValue())
+                for (InetAddressAndPort address : entry.getValue())
                 {
                     map.put(range, address);
                 }
@@ -768,10 +759,10 @@ public class TokenMetadata
         return this.pendingRanges.get(keyspaceName);
     }
 
-    public List<Range<Token>> getPendingRanges(String keyspaceName, InetAddress endpoint)
+    public List<Range<Token>> getPendingRanges(String keyspaceName, InetAddressAndPort endpoint)
     {
         List<Range<Token>> ranges = new ArrayList<>();
-        for (Map.Entry<Range<Token>, InetAddress> entry : getPendingRangesMM(keyspaceName).entries())
+        for (Map.Entry<Range<Token>, InetAddressAndPort> entry : getPendingRangesMM(keyspaceName).entries())
         {
             if (entry.getValue().equals(endpoint))
             {
@@ -824,9 +815,9 @@ public class TokenMetadata
                 long startedAt = System.currentTimeMillis();
 
                 // create clone of current state
-                BiMultiValMap<Token, InetAddress> bootstrapTokensClone = new BiMultiValMap<>();
-                Set<InetAddress> leavingEndpointsClone = new HashSet<>();
-                Set<Pair<Token, InetAddress>> movingEndpointsClone = new HashSet<>();
+                BiMultiValMap<Token, InetAddressAndPort> bootstrapTokensClone = new BiMultiValMap<>();
+                Set<InetAddressAndPort> leavingEndpointsClone = new HashSet<>();
+                Set<Pair<Token, InetAddressAndPort>> movingEndpointsClone = new HashSet<>();
                 TokenMetadata metadata;
 
                 lock.readLock().lock();
@@ -859,29 +850,29 @@ public class TokenMetadata
      */
     private static PendingRangeMaps calculatePendingRanges(AbstractReplicationStrategy strategy,
                                                            TokenMetadata metadata,
-                                                           BiMultiValMap<Token, InetAddress> bootstrapTokens,
-                                                           Set<InetAddress> leavingEndpoints,
-                                                           Set<Pair<Token, InetAddress>> movingEndpoints)
+                                                           BiMultiValMap<Token, InetAddressAndPort> bootstrapTokens,
+                                                           Set<InetAddressAndPort> leavingEndpoints,
+                                                           Set<Pair<Token, InetAddressAndPort>> movingEndpoints)
     {
         PendingRangeMaps newPendingRanges = new PendingRangeMaps();
 
-        Multimap<InetAddress, Range<Token>> addressRanges = strategy.getAddressRanges(metadata);
+        Multimap<InetAddressAndPort, Range<Token>> addressRanges = strategy.getAddressRanges(metadata);
 
         // Copy of metadata reflecting the situation after all leave operations are finished.
         TokenMetadata allLeftMetadata = removeEndpoints(metadata.cloneOnlyTokenMap(), leavingEndpoints);
 
         // get all ranges that will be affected by leaving nodes
         Set<Range<Token>> affectedRanges = new HashSet<Range<Token>>();
-        for (InetAddress endpoint : leavingEndpoints)
+        for (InetAddressAndPort endpoint : leavingEndpoints)
             affectedRanges.addAll(addressRanges.get(endpoint));
 
         // for each of those ranges, find what new nodes will be responsible for the range when
         // all leaving nodes are gone.
         for (Range<Token> range : affectedRanges)
         {
-            Set<InetAddress> currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, metadata));
-            Set<InetAddress> newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, allLeftMetadata));
-            for (InetAddress address : Sets.difference(newEndpoints, currentEndpoints))
+            Set<InetAddressAndPort> currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, metadata));
+            Set<InetAddressAndPort> newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, allLeftMetadata));
+            for (InetAddressAndPort address : Sets.difference(newEndpoints, currentEndpoints))
             {
                 newPendingRanges.addPendingRange(range, address);
             }
@@ -892,8 +883,8 @@ public class TokenMetadata
 
         // For each of the bootstrapping nodes, simply add and remove them one by one to
         // allLeftMetadata and check in between what their ranges would be.
-        Multimap<InetAddress, Token> bootstrapAddresses = bootstrapTokens.inverse();
-        for (InetAddress endpoint : bootstrapAddresses.keySet())
+        Multimap<InetAddressAndPort, Token> bootstrapAddresses = bootstrapTokens.inverse();
+        for (InetAddressAndPort endpoint : bootstrapAddresses.keySet())
         {
             Collection<Token> tokens = bootstrapAddresses.get(endpoint);
 
@@ -910,11 +901,11 @@ public class TokenMetadata
 
         // For each of the moving nodes, we do the same thing we did for bootstrapping:
         // simply add and remove them one by one to allLeftMetadata and check in between what their ranges would be.
-        for (Pair<Token, InetAddress> moving : movingEndpoints)
+        for (Pair<Token, InetAddressAndPort> moving : movingEndpoints)
         {
             //Calculate all the ranges which will could be affected. This will include the ranges before and after the move.
             Set<Range<Token>> moveAffectedRanges = new HashSet<>();
-            InetAddress endpoint = moving.right; // address of the moving node
+            InetAddressAndPort endpoint = moving.right; // address of the moving node
             //Add ranges before the move
             for (Range<Token> range : strategy.getAddressRanges(allLeftMetadata).get(endpoint))
             {
@@ -930,10 +921,10 @@ public class TokenMetadata
 
             for(Range<Token> range : moveAffectedRanges)
             {
-                Set<InetAddress> currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, metadata));
-                Set<InetAddress> newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, allLeftMetadata));
-                Set<InetAddress> difference = Sets.difference(newEndpoints, currentEndpoints);
-                for(final InetAddress address : difference)
+                Set<InetAddressAndPort> currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, metadata));
+                Set<InetAddressAndPort> newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, allLeftMetadata));
+                Set<InetAddressAndPort> difference = Sets.difference(newEndpoints, currentEndpoints);
+                for(final InetAddressAndPort address : difference)
                 {
                     Collection<Range<Token>> newRanges = strategy.getAddressRanges(allLeftMetadata).get(address);
                     Collection<Range<Token>> oldRanges = strategy.getAddressRanges(metadata).get(address);
@@ -973,12 +964,12 @@ public class TokenMetadata
     }
 
     /** @return a copy of the bootstrapping tokens map */
-    public BiMultiValMap<Token, InetAddress> getBootstrapTokens()
+    public BiMultiValMap<Token, InetAddressAndPort> getBootstrapTokens()
     {
         lock.readLock().lock();
         try
         {
-            return new BiMultiValMap<Token, InetAddress>(bootstrapTokens);
+            return new BiMultiValMap<>(bootstrapTokens);
         }
         finally
         {
@@ -986,7 +977,7 @@ public class TokenMetadata
         }
     }
 
-    public Set<InetAddress> getAllEndpoints()
+    public Set<InetAddressAndPort> getAllEndpoints()
     {
         lock.readLock().lock();
         try
@@ -1010,7 +1001,7 @@ public class TokenMetadata
     }
 
     /** caller should not modify leavingEndpoints */
-    public Set<InetAddress> getLeavingEndpoints()
+    public Set<InetAddressAndPort> getLeavingEndpoints()
     {
         lock.readLock().lock();
         try
@@ -1037,7 +1028,7 @@ public class TokenMetadata
      * Endpoints which are migrating to the new tokens
      * @return set of addresses of moving endpoints
      */
-    public Set<Pair<Token, InetAddress>> getMovingEndpoints()
+    public Set<Pair<Token, InetAddressAndPort>> getMovingEndpoints()
     {
         lock.readLock().lock();
         try
@@ -1148,14 +1139,14 @@ public class TokenMetadata
         lock.readLock().lock();
         try
         {
-            Multimap<InetAddress, Token> endpointToTokenMap = tokenToEndpointMap.inverse();
-            Set<InetAddress> eps = endpointToTokenMap.keySet();
+            Multimap<InetAddressAndPort, Token> endpointToTokenMap = tokenToEndpointMap.inverse();
+            Set<InetAddressAndPort> eps = endpointToTokenMap.keySet();
 
             if (!eps.isEmpty())
             {
                 sb.append("Normal Tokens:");
                 sb.append(System.getProperty("line.separator"));
-                for (InetAddress ep : eps)
+                for (InetAddressAndPort ep : eps)
                 {
                     sb.append(ep);
                     sb.append(':');
@@ -1168,7 +1159,7 @@ public class TokenMetadata
             {
                 sb.append("Bootstrapping Tokens:" );
                 sb.append(System.getProperty("line.separator"));
-                for (Map.Entry<Token, InetAddress> entry : bootstrapTokens.entrySet())
+                for (Map.Entry<Token, InetAddressAndPort> entry : bootstrapTokens.entrySet())
                 {
                     sb.append(entry.getValue()).append(':').append(entry.getKey());
                     sb.append(System.getProperty("line.separator"));
@@ -1179,7 +1170,7 @@ public class TokenMetadata
             {
                 sb.append("Leaving Endpoints:");
                 sb.append(System.getProperty("line.separator"));
-                for (InetAddress ep : leavingEndpoints)
+                for (InetAddressAndPort ep : leavingEndpoints)
                 {
                     sb.append(ep);
                     sb.append(System.getProperty("line.separator"));
@@ -1213,7 +1204,7 @@ public class TokenMetadata
         return sb.toString();
     }
 
-    public Collection<InetAddress> pendingEndpointsFor(Token token, String keyspaceName)
+    public Collection<InetAddressAndPort> pendingEndpointsFor(Token token, String keyspaceName)
     {
         PendingRangeMaps pendingRangeMaps = this.pendingRanges.get(keyspaceName);
         if (pendingRangeMaps == null)
@@ -1225,19 +1216,19 @@ public class TokenMetadata
     /**
      * @deprecated retained for benefit of old tests
      */
-    public Collection<InetAddress> getWriteEndpoints(Token token, String keyspaceName, Collection<InetAddress> naturalEndpoints)
+    public Collection<InetAddressAndPort> getWriteEndpoints(Token token, String keyspaceName, Collection<InetAddressAndPort> naturalEndpoints)
     {
         return ImmutableList.copyOf(Iterables.concat(naturalEndpoints, pendingEndpointsFor(token, keyspaceName)));
     }
 
     /** @return an endpoint to token multimap representation of tokenToEndpointMap (a copy) */
-    public Multimap<InetAddress, Token> getEndpointToTokenMapForReading()
+    public Multimap<InetAddressAndPort, Token> getEndpointToTokenMapForReading()
     {
         lock.readLock().lock();
         try
         {
-            Multimap<InetAddress, Token> cloned = HashMultimap.create();
-            for (Map.Entry<Token, InetAddress> entry : tokenToEndpointMap.entrySet())
+            Multimap<InetAddressAndPort, Token> cloned = HashMultimap.create();
+            for (Map.Entry<Token, InetAddressAndPort> entry : tokenToEndpointMap.entrySet())
                 cloned.put(entry.getValue(), entry.getKey());
             return cloned;
         }
@@ -1251,12 +1242,12 @@ public class TokenMetadata
      * @return a (stable copy, won't be modified) Token to Endpoint map for all the normal and bootstrapping nodes
      *         in the cluster.
      */
-    public Map<Token, InetAddress> getNormalAndBootstrappingTokenToEndpointMap()
+    public Map<Token, InetAddressAndPort> getNormalAndBootstrappingTokenToEndpointMap()
     {
         lock.readLock().lock();
         try
         {
-            Map<Token, InetAddress> map = new HashMap<>(tokenToEndpointMap.size() + bootstrapTokens.size());
+            Map<Token, InetAddressAndPort> map = new HashMap<>(tokenToEndpointMap.size() + bootstrapTokens.size());
             map.putAll(tokenToEndpointMap);
             map.putAll(bootstrapTokens);
             return map;
@@ -1302,11 +1293,11 @@ public class TokenMetadata
     public static class Topology
     {
         /** multi-map of DC to endpoints in that DC */
-        private final Multimap<String, InetAddress> dcEndpoints;
+        private final Multimap<String, InetAddressAndPort> dcEndpoints;
         /** map of DC to multi-map of rack to endpoints in that rack */
-        private final Map<String, Multimap<String, InetAddress>> dcRacks;
+        private final Map<String, Multimap<String, InetAddressAndPort>> dcRacks;
         /** reverse-lookup map for endpoint to current known dc/rack assignment */
-        private final Map<InetAddress, Pair<String, String>> currentLocations;
+        private final Map<InetAddressAndPort, Pair<String, String>> currentLocations;
 
         Topology()
         {
@@ -1337,7 +1328,7 @@ public class TokenMetadata
         /**
          * Stores current DC/rack assignment for ep
          */
-        void addEndpoint(InetAddress ep)
+        void addEndpoint(InetAddressAndPort ep)
         {
             IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
             String dc = snitch.getDatacenter(ep);
@@ -1353,12 +1344,12 @@ public class TokenMetadata
             doAddEndpoint(ep, dc, rack);
         }
 
-        private void doAddEndpoint(InetAddress ep, String dc, String rack)
+        private void doAddEndpoint(InetAddressAndPort ep, String dc, String rack)
         {
             dcEndpoints.put(dc, ep);
 
             if (!dcRacks.containsKey(dc))
-                dcRacks.put(dc, HashMultimap.<String, InetAddress>create());
+                dcRacks.put(dc, HashMultimap.create());
             dcRacks.get(dc).put(rack, ep);
 
             currentLocations.put(ep, Pair.create(dc, rack));
@@ -1367,7 +1358,7 @@ public class TokenMetadata
         /**
          * Removes current DC/rack assignment for ep
          */
-        void removeEndpoint(InetAddress ep)
+        void removeEndpoint(InetAddressAndPort ep)
         {
             if (!currentLocations.containsKey(ep))
                 return;
@@ -1375,13 +1366,13 @@ public class TokenMetadata
             doRemoveEndpoint(ep, currentLocations.remove(ep));
         }
 
-        private void doRemoveEndpoint(InetAddress ep, Pair<String, String> current)
+        private void doRemoveEndpoint(InetAddressAndPort ep, Pair<String, String> current)
         {
             dcRacks.get(current.left).remove(current.right, ep);
             dcEndpoints.remove(current.left, ep);
         }
 
-        void updateEndpoint(InetAddress ep)
+        void updateEndpoint(InetAddressAndPort ep)
         {
             IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
             if (snitch == null || !currentLocations.containsKey(ep))
@@ -1396,11 +1387,11 @@ public class TokenMetadata
             if (snitch == null)
                 return;
 
-            for (InetAddress ep : currentLocations.keySet())
+            for (InetAddressAndPort ep : currentLocations.keySet())
                 updateEndpoint(ep, snitch);
         }
 
-        private void updateEndpoint(InetAddress ep, IEndpointSnitch snitch)
+        private void updateEndpoint(InetAddressAndPort ep, IEndpointSnitch snitch)
         {
             Pair<String, String> current = currentLocations.get(ep);
             String dc = snitch.getDatacenter(ep);
@@ -1415,7 +1406,7 @@ public class TokenMetadata
         /**
          * @return multi-map of DC to endpoints in that DC
          */
-        public Multimap<String, InetAddress> getDatacenterEndpoints()
+        public Multimap<String, InetAddressAndPort> getDatacenterEndpoints()
         {
             return dcEndpoints;
         }
@@ -1423,7 +1414,7 @@ public class TokenMetadata
         /**
          * @return map of DC to multi-map of rack to endpoints in that rack
          */
-        public Map<String, Multimap<String, InetAddress>> getDatacenterRacks()
+        public Map<String, Multimap<String, InetAddressAndPort>> getDatacenterRacks()
         {
             return dcRacks;
         }
@@ -1431,7 +1422,7 @@ public class TokenMetadata
         /**
          * @return The DC and rack of the given endpoint.
          */
-        public Pair<String, String> getLocation(InetAddress addr)
+        public Pair<String, String> getLocation(InetAddressAndPort addr)
         {
             return currentLocations.get(addr);
         }

src/java/org/apache/cassandra/metrics/ConnectionMetrics.java

@@ -17,14 +17,14 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.net.InetAddress;
-
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Meter;
 import org.apache.cassandra.net.async.OutboundMessagingPool;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
+import org.apache.cassandra.locator.InetAddressAndPort;
+
 /**
  * Metrics for internode connections.
  */
@@ -65,10 +65,10 @@ public class ConnectionMetrics
      *
      * @param ip IP address to use for metrics label
      */
-    public ConnectionMetrics(InetAddress ip, final OutboundMessagingPool messagingPool)
+    public ConnectionMetrics(InetAddressAndPort ip, final OutboundMessagingPool messagingPool)
     {
         // ipv6 addresses will contain colons, which are invalid in a JMX ObjectName
-        address = ip.getHostAddress().replace(':', '.');
+        address = ip.toString().replace(':', '.');
 
         factory = new DefaultNameFactory("Connection", address);
 

src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.net.InetAddress;
 import java.util.Map.Entry;
 
 import com.google.common.util.concurrent.MoreExecutors;
@@ -27,6 +26,7 @@ import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
 
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.UUIDGen;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -43,28 +43,28 @@ public class HintedHandoffMetrics
     private static final MetricNameFactory factory = new DefaultNameFactory("HintedHandOffManager");
 
     /** Total number of hints which are not stored, This is not a cache. */
-    private final LoadingCache<InetAddress, DifferencingCounter> notStored = Caffeine.newBuilder()
-         .executor(MoreExecutors.directExecutor())
-         .build(DifferencingCounter::new);
+    private final LoadingCache<InetAddressAndPort, DifferencingCounter> notStored = Caffeine.newBuilder()
+                                                                                            .executor(MoreExecutors.directExecutor())
+                                                                                            .build(DifferencingCounter::new);
 
     /** Total number of hints that have been created, This is not a cache. */
-    private final LoadingCache<InetAddress, Counter> createdHintCounts = Caffeine.newBuilder()
-         .executor(MoreExecutors.directExecutor())
-         .build(address -> Metrics.counter(factory.createMetricName("Hints_created-" + address.getHostAddress().replace(':', '.'))));
+    private final LoadingCache<InetAddressAndPort, Counter> createdHintCounts = Caffeine.newBuilder()
+                                                                                        .executor(MoreExecutors.directExecutor())
+                                                                                        .build(address -> Metrics.counter(factory.createMetricName("Hints_created-" + address.toString().replace(':', '.'))));
 
-    public void incrCreatedHints(InetAddress address)
+    public void incrCreatedHints(InetAddressAndPort address)
     {
         createdHintCounts.get(address).inc();
     }
 
-    public void incrPastWindow(InetAddress address)
+    public void incrPastWindow(InetAddressAndPort address)
     {
         notStored.get(address).mark();
     }
 
     public void log()
     {
-        for (Entry<InetAddress, DifferencingCounter> entry : notStored.asMap().entrySet())
+        for (Entry<InetAddressAndPort, DifferencingCounter> entry : notStored.asMap().entrySet())
         {
             long difference = entry.getValue().difference();
             if (difference == 0)
@@ -79,9 +79,10 @@ public class HintedHandoffMetrics
         private final Counter meter;
         private long reported = 0;
 
-        public DifferencingCounter(InetAddress address)
+        public DifferencingCounter(InetAddressAndPort address)
         {
-            this.meter = Metrics.counter(factory.createMetricName("Hints_not_stored-" + address.getHostAddress().replace(':', '.')));
+            //This changes the name of the metric, people can update their monitoring when upgrading?
+            this.meter = Metrics.counter(factory.createMetricName("Hints_not_stored-" + address.toString().replace(':', '.')));
         }
 
         public long difference()

src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java

@@ -17,8 +17,6 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.net.InetAddress;
-
 import com.google.common.util.concurrent.MoreExecutors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -27,6 +25,7 @@ import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
@@ -47,11 +46,11 @@ public final class HintsServiceMetrics
     private static final Histogram globalDelayHistogram = Metrics.histogram(factory.createMetricName("Hint_delays"), false);
 
     /** Histograms per-endpoint of hint delivery delays, This is not a cache. */
-    private static final LoadingCache<InetAddress, Histogram> delayByEndpoint = Caffeine.newBuilder()
-        .executor(MoreExecutors.directExecutor())
-        .build(address -> Metrics.histogram(factory.createMetricName("Hint_delays-"+address.getHostAddress().replace(':', '.')), false));
+    private static final LoadingCache<InetAddressAndPort, Histogram> delayByEndpoint = Caffeine.newBuilder()
+                                                                                               .executor(MoreExecutors.directExecutor())
+                                                                                               .build(address -> Metrics.histogram(factory.createMetricName("Hint_delays-"+address.toString().replace(':', '.')), false));
 
-    public static void updateDelayMetrics(InetAddress endpoint, long delay)
+    public static void updateDelayMetrics(InetAddressAndPort endpoint, long delay)
     {
         if (delay <= 0)
         {

src/java/org/apache/cassandra/metrics/MessagingMetrics.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.net.InetAddress;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
@@ -26,6 +25,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.codahale.metrics.Timer;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
@@ -47,7 +47,7 @@ public class MessagingMetrics
         queueWaitLatency = new ConcurrentHashMap<>();
     }
 
-    public void addTimeTaken(InetAddress from, long timeTaken)
+    public void addTimeTaken(InetAddressAndPort from, long timeTaken)
     {
         String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(from);
         Timer timer = dcLatency.get(dc);

src/java/org/apache/cassandra/metrics/StreamingMetrics.java

@@ -17,11 +17,12 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.net.InetAddress;
 import java.util.concurrent.ConcurrentMap;
 
 
 import com.codahale.metrics.Counter;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
 import org.cliffc.high_scale_lib.NonBlockingHashMap;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -33,7 +34,7 @@ public class StreamingMetrics
 {
     public static final String TYPE_NAME = "Streaming";
 
-    private static final ConcurrentMap<InetAddress, StreamingMetrics> instances = new NonBlockingHashMap<InetAddress, StreamingMetrics>();
+    private static final ConcurrentMap<InetAddressAndPort, StreamingMetrics> instances = new NonBlockingHashMap<>();
 
     public static final Counter activeStreamsOutbound = Metrics.counter(DefaultNameFactory.createMetricName(TYPE_NAME, "ActiveOutboundStreams", null));
     public static final Counter totalIncomingBytes = Metrics.counter(DefaultNameFactory.createMetricName(TYPE_NAME, "TotalIncomingBytes", null));
@@ -41,7 +42,7 @@ public class StreamingMetrics
     public final Counter incomingBytes;
     public final Counter outgoingBytes;
 
-    public static StreamingMetrics get(InetAddress ip)
+    public static StreamingMetrics get(InetAddressAndPort ip)
     {
        StreamingMetrics metrics = instances.get(ip);
        if (metrics == null)
@@ -52,9 +53,9 @@ public class StreamingMetrics
        return metrics;
     }
 
-    public StreamingMetrics(final InetAddress peer)
+    public StreamingMetrics(final InetAddressAndPort peer)
     {
-        MetricNameFactory factory = new DefaultNameFactory("Streaming", peer.getHostAddress().replace(':', '.'));
+        MetricNameFactory factory = new DefaultNameFactory("Streaming", peer.toString().replace(':', '.'));
         incomingBytes = Metrics.counter(factory.createMetricName("IncomingBytes"));
         outgoingBytes= Metrics.counter(factory.createMetricName("OutgoingBytes"));
     }

src/java/org/apache/cassandra/net/BackPressureState.java

@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.net;
 
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 /**
  * Interface meant to track the back-pressure state per replica host.
@@ -47,5 +47,5 @@ public interface BackPressureState
     /**
      * Returns the host this state refers to.
      */
-    InetAddress getHost();
+    InetAddressAndPort getHost();
 }

src/java/org/apache/cassandra/net/BackPressureStrategy.java

@@ -17,10 +17,11 @@
  */
 package org.apache.cassandra.net;
 
-import java.net.InetAddress;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.cassandra.locator.InetAddressAndPort;
+
 /**
  * Back-pressure algorithm interface.
  * <p>
@@ -39,5 +40,5 @@ public interface BackPressureStrategy<S extends BackPressureState>
     /**
      * Creates a new {@link BackPressureState} initialized as needed by the specific implementation.
      */
-    S newState(InetAddress host);
+    S newState(InetAddressAndPort host);
 }

src/java/org/apache/cassandra/net/CallbackInfo.java

@@ -17,9 +17,8 @@
  */
 package org.apache.cassandra.net;
 
-import java.net.InetAddress;
-
 import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 /**
  * Encapsulates the callback information.
@@ -28,7 +27,7 @@ import org.apache.cassandra.io.IVersionedSerializer;
  */
 public class CallbackInfo
 {
-    protected final InetAddress target;
+    protected final InetAddressAndPort target;
     protected final IAsyncCallback callback;
     protected final IVersionedSerializer<?> serializer;
     private final boolean failureCallback;
@@ -41,7 +40,7 @@ public class CallbackInfo
      * @param serializer serializer to deserialize response message
      * @param failureCallback True when we have a callback to handle failures
      */
-    public CallbackInfo(InetAddress target, IAsyncCallback callback, IVersionedSerializer<?> serializer, boolean failureCallback)
+    public CallbackInfo(InetAddressAndPort target, IAsyncCallback callback, IVersionedSerializer<?> serializer, boolean failureCallback)
     {
         this.target = target;
         this.callback = callback;

src/java/org/apache/cassandra/net/CompactEndpointSerializationHelper.java

@@ -21,28 +21,108 @@ import java.io.*;
 import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
+import java.nio.ByteBuffer;
 
-public class CompactEndpointSerializationHelper
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+
+/*
+ * As of version 4.0 the endpoint description includes a port number as an unsigned short
+ */
+public class CompactEndpointSerializationHelper implements IVersionedSerializer<InetAddressAndPort>
 {
-    public static void serialize(InetAddress endpoint, DataOutput out) throws IOException
+    public static final IVersionedSerializer<InetAddressAndPort> instance = new CompactEndpointSerializationHelper();
+
+    /**
+     * Streaming uses its own version numbering so we need to ignore it and always use currrent version.
+     * There is no cross version streaming so it will always use the latest address serialization.
+     **/
+    public static final IVersionedSerializer<InetAddressAndPort> streamingInstance = new IVersionedSerializer<InetAddressAndPort>()
     {
-        byte[] buf = endpoint.getAddress();
-        out.writeByte(buf.length);
-        out.write(buf);
+        public void serialize(InetAddressAndPort inetAddressAndPort, DataOutputPlus out, int version) throws IOException
+        {
+            instance.serialize(inetAddressAndPort, out, MessagingService.current_version);
+        }
+
+        public InetAddressAndPort deserialize(DataInputPlus in, int version) throws IOException
+        {
+            return instance.deserialize(in, MessagingService.current_version);
+        }
+
+        public long serializedSize(InetAddressAndPort inetAddressAndPort, int version)
+        {
+            return instance.serializedSize(inetAddressAndPort, MessagingService.current_version);
+        }
+    };
+
+    private CompactEndpointSerializationHelper() {}
+
+    public void serialize(InetAddressAndPort endpoint, DataOutputPlus out, int version) throws IOException
+    {
+        if (version >= MessagingService.VERSION_40)
+        {
+            byte[] buf = endpoint.addressBytes;
+            out.writeByte(buf.length + 2);
+            out.write(buf);
+            out.writeShort(endpoint.port);
+        }
+        else
+        {
+            byte[] buf = endpoint.addressBytes;
+            out.writeByte(buf.length);
+            out.write(buf);
+        }
     }
 
-    public static InetAddress deserialize(DataInput in) throws IOException
+    public InetAddressAndPort deserialize(DataInputPlus in, int version) throws IOException
     {
-        byte[] bytes = new byte[in.readByte()];
-        in.readFully(bytes, 0, bytes.length);
-        return InetAddress.getByAddress(bytes);
+        int size = in.readByte() & 0xFF;
+        switch(size)
+        {
+            //The original pre-4.0 serialiation of just an address
+            case 4:
+            case 16:
+            {
+                byte[] bytes = new byte[size];
+                in.readFully(bytes, 0, bytes.length);
+                return InetAddressAndPort.getByAddress(bytes);
+            }
+            //Address and one port
+            case 6:
+            case 18:
+            {
+                byte[] bytes = new byte[size - 2];
+                in.readFully(bytes);
+
+                int port = in.readShort() & 0xFFFF;
+                return InetAddressAndPort.getByAddressOverrideDefaults(InetAddress.getByAddress(bytes), bytes, port);
+            }
+            default:
+                throw new AssertionError("Unexpected size " + size);
+
+        }
     }
 
-    public static int serializedSize(InetAddress from)
+    public long serializedSize(InetAddressAndPort from, int version)
     {
-        if (from instanceof Inet4Address)
-            return 1 + 4;
-        assert from instanceof Inet6Address;
-        return 1 + 16;
+        //4.0 includes a port number
+        if (version >= MessagingService.VERSION_40)
+        {
+            if (from.address instanceof Inet4Address)
+                return 1 + 4 + 2;
+            assert from.address instanceof Inet6Address;
+            return 1 + 16 + 2;
+        }
+        else
+        {
+            if (from.address instanceof Inet4Address)
+                return 1 + 4;
+            assert from.address instanceof Inet6Address;
+            return 1 + 16;
+        }
     }
 }

src/java/org/apache/cassandra/net/ForwardToContainer.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.Collection;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+/**
+ * Contains forward to information until it can be serialized as part of a message using a version
+ * specific serialization
+ */
+public class ForwardToContainer
+{
+    public final Collection<InetAddressAndPort> targets;
+    public final int[] messageIds;
+
+    public ForwardToContainer(Collection<InetAddressAndPort> targets,
+                              int[] messageIds)
+    {
+        Preconditions.checkArgument(targets.size() == messageIds.length);
+        this.targets = targets;
+        this.messageIds = messageIds;
+    }
+}

src/java/org/apache/cassandra/net/ForwardToSerializer.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+public class ForwardToSerializer implements IVersionedSerializer<ForwardToContainer>
+{
+    public static ForwardToSerializer instance = new ForwardToSerializer();
+
+    private ForwardToSerializer() {}
+
+    public void serialize(ForwardToContainer forwardToContainer, DataOutputPlus out, int version) throws IOException
+    {
+        out.writeInt(forwardToContainer.targets.size());
+        Iterator<InetAddressAndPort> iter = forwardToContainer.targets.iterator();
+        for (int ii = 0; ii < forwardToContainer.messageIds.length; ii++)
+        {
+            CompactEndpointSerializationHelper.instance.serialize(iter.next(), out, version);
+            out.writeInt(forwardToContainer.messageIds[ii]);
+        }
+    }
+
+    public ForwardToContainer deserialize(DataInputPlus in, int version) throws IOException
+    {
+        int[] ids = new int[in.readInt()];
+        List<InetAddressAndPort> hosts = new ArrayList<>(ids.length);
+        for (int ii = 0; ii < ids.length; ii++)
+        {
+           hosts.add(CompactEndpointSerializationHelper.instance.deserialize(in, version));
+           ids[ii] = in.readInt();
+        }
+        return new ForwardToContainer(hosts, ids);
+    }
+
+    public long serializedSize(ForwardToContainer forwardToContainer, int version)
+    {
+        //Number of forward addresses, 4 bytes per for each id
+        long size = 4 +
+                    (4 * forwardToContainer.targets.size());
+        //Depending on ipv6 or ipv4 the address size is different.
+        for (InetAddressAndPort forwardTo : forwardToContainer.targets)
+        {
+            size += CompactEndpointSerializationHelper.instance.serializedSize(forwardTo, version);
+        }
+
+        return size;
+    }
+
+    public static ForwardToContainer fromBytes(byte[] bytes, int version)
+    {
+        try (DataInputBuffer input = new DataInputBuffer(bytes))
+        {
+            return instance.deserialize(input, version);
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+}

src/java/org/apache/cassandra/net/IAsyncCallback.java

@@ -17,11 +17,10 @@
  */
 package org.apache.cassandra.net;
 
-import java.net.InetAddress;
-
 import com.google.common.base.Predicate;
 
 import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 /**
  * implementors of IAsyncCallback need to make sure that any public methods
@@ -31,9 +30,9 @@ import org.apache.cassandra.gms.FailureDetector;
  */
 public interface IAsyncCallback<T>
 {
-    Predicate<InetAddress> isAlive = new Predicate<InetAddress>()
+    Predicate<InetAddressAndPort> isAlive = new Predicate<InetAddressAndPort>()
     {
-        public boolean apply(InetAddress endpoint)
+        public boolean apply(InetAddressAndPort endpoint)
         {
             return FailureDetector.instance.isAlive(endpoint);
         }

src/java/org/apache/cassandra/net/IAsyncCallbackWithFailure.java

@@ -17,9 +17,8 @@
  */
 package org.apache.cassandra.net;
 
-import java.net.InetAddress;
-
 import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 public interface IAsyncCallbackWithFailure<T> extends IAsyncCallback<T>
 {
@@ -27,5 +26,5 @@ public interface IAsyncCallbackWithFailure<T> extends IAsyncCallback<T>
     /**
      * Called when there is an exception on the remote node or timeout happens
      */
-    void onFailure(InetAddress from, RequestFailureReason failureReason);
+    void onFailure(InetAddressAndPort from, RequestFailureReason failureReason);
 }

src/java/org/apache/cassandra/net/IMessageSink.java

@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.net;
 
-import java.net.InetAddress;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
 public interface IMessageSink
 {
@@ -26,7 +26,7 @@ public interface IMessageSink
      *
      * @return true if the message is allowed, false if it should be dropped
      */
-    boolean allowOutgoingMessage(MessageOut message, int id, InetAddress to);
+    boolean allowOutgoingMessage(MessageOut message, int id, InetAddressAndPort to);
 
     /**
      * Allow or drop an incoming message

src/java/org/apache/cassandra/net/MessageDeliveryTask.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.net;
 import java.io.IOException;
 import java.util.EnumSet;
 
+import com.google.common.primitives.Shorts;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,6 +29,7 @@ import org.apache.cassandra.db.monitoring.ApproximateTime;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.index.IndexNotAvailableException;
+import org.apache.cassandra.io.DummyByteVersionedSerializer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 
 public class MessageDeliveryTask implements Runnable
@@ -96,19 +98,11 @@ public class MessageDeliveryTask implements Runnable
         if (message.doCallbackOnFailure())
         {
             MessageOut response = new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE)
-                                                .withParameter(MessagingService.FAILURE_RESPONSE_PARAM, MessagingService.ONE_BYTE);
+                                                .withParameter(ParameterType.FAILURE_RESPONSE, MessagingService.ONE_BYTE);
 
             if (t instanceof TombstoneOverwhelmingException)
             {
-                try (DataOutputBuffer out = new DataOutputBuffer())
-                {
-                    out.writeShort(RequestFailureReason.READ_TOO_MANY_TOMBSTONES.code);
-                    response = response.withParameter(MessagingService.FAILURE_REASON_PARAM, out.getData());
-                }
-                catch (IOException ex)
-                {
-                    throw new RuntimeException(ex);
-                }
+                response = response.withParameter(ParameterType.FAILURE_REASON, Shorts.checkedCast(RequestFailureReason.READ_TOO_MANY_TOMBSTONES.code));
             }
 
             MessagingService.instance().sendReply(response, id, message.from);

src/java/org/apache/cassandra/net/MessageIn.java

@@ -18,7 +18,6 @@
 package org.apache.cassandra.net;
 
 import java.io.IOException;
-import java.net.InetAddress;
 import java.util.Collections;
 import java.util.Map;
 
@@ -31,8 +30,8 @@ import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.MessagingService.Verb;
-
 /**
  * The receiving node's view of a {@link MessageOut}. See documentation on {@link MessageOut} for details on the
  * serialization format.
@@ -41,16 +40,16 @@ import org.apache.cassandra.net.MessagingService.Verb;
  */
 public class MessageIn<T>
 {
-    public final InetAddress from;
+    public final InetAddressAndPort from;
     public final T payload;
-    public final Map<String, byte[]> parameters;
-    public final Verb verb;
+    public final Map<ParameterType, Object> parameters;
+    public final MessagingService.Verb verb;
     public final int version;
     public final long constructionTime;
 
-    private MessageIn(InetAddress from,
+    private MessageIn(InetAddressAndPort from,
                       T payload,
-                      Map<String, byte[]> parameters,
+                      Map<ParameterType, Object> parameters,
                       Verb verb,
                       int version,
                       long constructionTime)
@@ -63,9 +62,9 @@ public class MessageIn<T>
         this.constructionTime = constructionTime;
     }
 
-    public static <T> MessageIn<T> create(InetAddress from,
+    public static <T> MessageIn<T> create(InetAddressAndPort from,
                                           T payload,
-                                          Map<String, byte[]> parameters,
+                                          Map<ParameterType, Object> parameters,
                                           Verb verb,
                                           int version,
                                           long constructionTime)
@@ -73,9 +72,9 @@ public class MessageIn<T>
         return new MessageIn<>(from, payload, parameters, verb, version, constructionTime);
     }
 
-    public static <T> MessageIn<T> create(InetAddress from,
+    public static <T> MessageIn<T> create(InetAddressAndPort from,
                                           T payload,
-                                          Map<String, byte[]> parameters,
+                                          Map<ParameterType, Object> parameters,
                                           MessagingService.Verb verb,
                                           int version)
     {
@@ -89,37 +88,46 @@ public class MessageIn<T>
 
     public static <T2> MessageIn<T2> read(DataInputPlus in, int version, int id, long constructionTime) throws IOException
     {
-        InetAddress from = CompactEndpointSerializationHelper.deserialize(in);
+        InetAddressAndPort from = CompactEndpointSerializationHelper.instance.deserialize(in, version);
 
         MessagingService.Verb verb = MessagingService.Verb.fromId(in.readInt());
-        Map<String, byte[]> parameters = readParameters(in);
+        Map<ParameterType, Object> parameters = readParameters(in, version);
         int payloadSize = in.readInt();
         return read(in, version, id, constructionTime, from, payloadSize, verb, parameters);
     }
 
-    public static Map<String, byte[]> readParameters(DataInputPlus in) throws IOException
+    public static Map<ParameterType, Object> readParameters(DataInputPlus in, int version) throws IOException
     {
         int parameterCount = in.readInt();
+        Map<ParameterType, Object> parameters;
         if (parameterCount == 0)
         {
             return Collections.emptyMap();
         }
         else
         {
-            ImmutableMap.Builder<String, byte[]> builder = ImmutableMap.builder();
+            ImmutableMap.Builder<ParameterType, Object> builder = ImmutableMap.builder();
             for (int i = 0; i < parameterCount; i++)
             {
                 String key = in.readUTF();
-                byte[] value = new byte[in.readInt()];
-                in.readFully(value);
-                builder.put(key, value);
+                ParameterType type = ParameterType.byName.get(key);
+                if (type != null)
+                {
+                    byte[] value = new byte[in.readInt()];
+                    in.readFully(value);
+                    builder.put(type, type.serializer.deserialize(new DataInputBuffer(value), version));
+                }
+                else
+                {
+                    in.skipBytes(in.readInt());
+                }
             }
             return builder.build();
         }
     }
 
     public static <T2> MessageIn<T2> read(DataInputPlus in, int version, int id, long constructionTime,
-                                          InetAddress from, int payloadSize, Verb verb, Map<String, byte[]> parameters) throws IOException
+                                          InetAddressAndPort from, int payloadSize, Verb verb, Map<ParameterType, Object> parameters) throws IOException
     {
         IVersionedSerializer<T2> serializer = (IVersionedSerializer<T2>) MessagingService.verbSerializers.get(verb);
         if (serializer instanceof MessagingService.CallbackDeterminedSerializer)
@@ -140,7 +148,7 @@ public class MessageIn<T>
         return MessageIn.create(from, payload, parameters, verb, version, constructionTime);
     }
 
-    public static long deriveConstructionTime(InetAddress from, int messageTimestamp, long currentTime)
+    public static long deriveConstructionTime(InetAddressAndPort from, int messageTimestamp, long currentTime)
     {
         // Reconstruct the message construction time sent by the remote host (we sent only the lower 4 bytes, assuming the
         // higher 4 bytes wouldn't change between the sender and receiver)
@@ -182,36 +190,18 @@ public class MessageIn<T>
 
     public boolean doCallbackOnFailure()
     {
-        return parameters.containsKey(MessagingService.FAILURE_CALLBACK_PARAM);
+        return parameters.containsKey(ParameterType.FAILURE_CALLBACK);
     }
 
     public boolean isFailureResponse()
     {
-        return parameters.containsKey(MessagingService.FAILURE_RESPONSE_PARAM);
-    }
-
-    public boolean containsFailureReason()
-    {
-        return parameters.containsKey(MessagingService.FAILURE_REASON_PARAM);
+        return parameters.containsKey(ParameterType.FAILURE_RESPONSE);
     }
 
     public RequestFailureReason getFailureReason()
     {
-        if (containsFailureReason())
-        {
-            try (DataInputBuffer in = new DataInputBuffer(parameters.get(MessagingService.FAILURE_REASON_PARAM)))
-            {
-                return RequestFailureReason.fromCode(in.readUnsignedShort());
-            }
-            catch (IOException ex)
-            {
-                throw new RuntimeException(ex);
-            }
-        }
-        else
-        {
-            return RequestFailureReason.UNKNOWN;
-        }
+        Short code = (Short)parameters.get(ParameterType.FAILURE_REASON);
+        return code != null ? RequestFailureReason.fromCode(code) : RequestFailureReason.UNKNOWN;
     }
 
     public long getTimeout()

src/java/org/apache/cassandra/net/MessageOut.java

@@ -19,17 +19,18 @@
 package org.apache.cassandra.net;
 
 import java.io.IOException;
-import java.net.InetAddress;
-import java.util.Collections;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.List;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.primitives.Ints;
 
 import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
@@ -80,12 +81,20 @@ import static org.apache.cassandra.tracing.Tracing.isTracing;
 public class MessageOut<T>
 {
     private static final int SERIALIZED_SIZE_VERSION_UNDEFINED = -1;
+    //Parameters are stored in an object array as tuples of size two
+    public static final int PARAMETER_TUPLE_SIZE = 2;
+    //Offset in a parameter tuple containing the type of the parameter
+    public static final int PARAMETER_TUPLE_TYPE_OFFSET = 0;
+    //Offset in a parameter tuple containing the actual parameter represented as a POJO
+    public static final int PARAMETER_TUPLE_PARAMETER_OFFSET = 1;
 
-    public final InetAddress from;
+    public final InetAddressAndPort from;
     public final MessagingService.Verb verb;
     public final T payload;
     public final IVersionedSerializer<T> serializer;
-    public final Map<String, byte[]> parameters;
+    //A list of tuples, first object is the ParameterType enum value,
+    //the second object is the POJO to serialize
+    public final List<Object> parameters;
 
     /**
      * Memoization of the serialized size of the just the payload.
@@ -115,16 +124,16 @@ public class MessageOut<T>
              serializer,
              isTracing()
                  ? Tracing.instance.getTraceHeaders()
-                 : Collections.<String, byte[]>emptyMap());
+                 : ImmutableList.of());
     }
 
-    private MessageOut(MessagingService.Verb verb, T payload, IVersionedSerializer<T> serializer, Map<String, byte[]> parameters)
+    private MessageOut(MessagingService.Verb verb, T payload, IVersionedSerializer<T> serializer, List<Object> parameters)
     {
-        this(FBUtilities.getBroadcastAddress(), verb, payload, serializer, parameters);
+        this(FBUtilities.getBroadcastAddressAndPort(), verb, payload, serializer, parameters);
     }
 
     @VisibleForTesting
-    public MessageOut(InetAddress from, MessagingService.Verb verb, T payload, IVersionedSerializer<T> serializer, Map<String, byte[]> parameters)
+    public MessageOut(InetAddressAndPort from, MessagingService.Verb verb, T payload, IVersionedSerializer<T> serializer, List<Object> parameters)
     {
         this.from = from;
         this.verb = verb;
@@ -133,11 +142,13 @@ public class MessageOut<T>
         this.parameters = parameters;
     }
 
-    public MessageOut<T> withParameter(String key, byte[] value)
+    public <VT> MessageOut<T> withParameter(ParameterType type, VT value)
     {
-        ImmutableMap.Builder<String, byte[]> builder = ImmutableMap.builder();
-        builder.putAll(parameters).put(key, value);
-        return new MessageOut<T>(verb, payload, serializer, builder.build());
+        List<Object> newParameters = new ArrayList<>(parameters.size() + 2);
+        newParameters.addAll(parameters);
+        newParameters.add(type);
+        newParameters.add(value);
+        return new MessageOut<T>(verb, payload, serializer, newParameters);
     }
 
     public Stage getStage()
@@ -159,15 +170,19 @@ public class MessageOut<T>
 
     public void serialize(DataOutputPlus out, int version) throws IOException
     {
-        CompactEndpointSerializationHelper.serialize(from, out);
+        CompactEndpointSerializationHelper.instance.serialize(from, out, version);
 
         out.writeInt(verb.getId());
-        out.writeInt(parameters.size());
-        for (Map.Entry<String, byte[]> entry : parameters.entrySet())
+        assert parameters.size() % PARAMETER_TUPLE_SIZE == 0;
+        out.writeInt(parameters.size() / PARAMETER_TUPLE_SIZE);
+        for (int ii = 0; ii < parameters.size(); ii += PARAMETER_TUPLE_SIZE)
         {
-            out.writeUTF(entry.getKey());
-            out.writeInt(entry.getValue().length);
-            out.write(entry.getValue());
+            ParameterType type = (ParameterType)parameters.get(ii + PARAMETER_TUPLE_TYPE_OFFSET);
+            out.writeUTF(type.key);
+            IVersionedSerializer serializer = type.serializer;
+            Object parameter = parameters.get(ii + PARAMETER_TUPLE_PARAMETER_OFFSET);
+            out.writeInt(Ints.checkedCast(serializer.serializedSize(parameter, version)));
+            serializer.serialize(parameter, out, version);
         }
 
         if (payload != null)
@@ -187,15 +202,19 @@ public class MessageOut<T>
 
     private Pair<Long, Long> calculateSerializedSize(int version)
     {
-        long size = CompactEndpointSerializationHelper.serializedSize(from);
+        long size = 0;
+        size += CompactEndpointSerializationHelper.instance.serializedSize(from, version);
 
         size += TypeSizes.sizeof(verb.getId());
         size += TypeSizes.sizeof(parameters.size());
-        for (Map.Entry<String, byte[]> entry : parameters.entrySet())
+        for (int ii = 0; ii < parameters.size(); ii += PARAMETER_TUPLE_SIZE)
         {
-            size += TypeSizes.sizeof(entry.getKey());
-            size += TypeSizes.sizeof(entry.getValue().length);
-            size += entry.getValue().length;
+            ParameterType type = (ParameterType)parameters.get(ii + PARAMETER_TUPLE_TYPE_OFFSET);
+            size += TypeSizes.sizeof(type.key());
+            size += 4;//length prefix
+            IVersionedSerializer serializer = type.serializer;
+            Object parameter = parameters.get(ii + PARAMETER_TUPLE_PARAMETER_OFFSET);
+            size += serializer.serializedSize(parameter, version);
         }
 
         long payloadSize = payload == null ? 0 : serializer.serializedSize(payload, version);
@@ -237,4 +256,16 @@ public class MessageOut<T>
 
         return sizes.left.intValue();
     }
+
+    public Object getParameter(ParameterType type)
+    {
+        for (int ii = 0; ii < parameters.size(); ii += PARAMETER_TUPLE_SIZE)
+        {
+            if (((ParameterType)parameters.get(ii + PARAMETER_TUPLE_TYPE_OFFSET)).equals(type))
+            {
+                return parameters.get(ii + PARAMETER_TUPLE_PARAMETER_OFFSET);
+            }
+        }
+        return null;
+    }
 }

src/java/org/apache/cassandra/net/MessagingService.java

@@ -20,8 +20,6 @@ package org.apache.cassandra.net;
 import java.io.IOError;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -37,6 +35,7 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
 
@@ -91,6 +90,7 @@ import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.IEndpointSnitch;
 import org.apache.cassandra.locator.ILatencySubscriber;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.metrics.ConnectionMetrics;
 import org.apache.cassandra.metrics.DroppedMessageMetrics;
@@ -127,10 +127,7 @@ public final class MessagingService implements MessagingServiceMBean
     public static final int VERSION_40 = 12;
     public static final int current_version = VERSION_40;
 
-    public static final String FAILURE_CALLBACK_PARAM = "CAL_BAC";
     public static final byte[] ONE_BYTE = new byte[1];
-    public static final String FAILURE_RESPONSE_PARAM = "FAIL";
-    public static final String FAILURE_REASON_PARAM = "FAIL_REASON";
 
     /**
      * we preface every message with this number so the recipient can validate the sender is sane
@@ -447,7 +444,7 @@ public final class MessagingService implements MessagingServiceMBean
     private final Map<Verb, IVerbHandler> verbHandlers;
 
     @VisibleForTesting
-    public final ConcurrentMap<InetAddress, OutboundMessagingPool> channelManagers = new NonBlockingHashMap<>();
+    public final ConcurrentMap<InetAddressAndPort, OutboundMessagingPool> channelManagers = new NonBlockingHashMap<>();
     final List<ServerChannel> serverChannels = Lists.newArrayList();
 
     private static final Logger logger = LoggerFactory.getLogger(MessagingService.class);
@@ -506,7 +503,7 @@ public final class MessagingService implements MessagingServiceMBean
     private final List<ILatencySubscriber> subscribers = new ArrayList<ILatencySubscriber>();
 
     // protocol versions of the other nodes in the cluster
-    private final ConcurrentMap<InetAddress, Integer> versions = new NonBlockingHashMap<InetAddress, Integer>();
+    private final ConcurrentMap<InetAddressAndPort, Integer> versions = new NonBlockingHashMap<>();
 
     // message sinks are a testing hook
     private final Set<IMessageSink> messageSinks = new CopyOnWriteArraySet<>();
@@ -629,7 +626,7 @@ public final class MessagingService implements MessagingServiceMBean
      * @param callback The message callback.
      * @param message The actual message.
      */
-    public void updateBackPressureOnSend(InetAddress host, IAsyncCallback callback, MessageOut<?> message)
+    public void updateBackPressureOnSend(InetAddressAndPort host, IAsyncCallback callback, MessageOut<?> message)
     {
         if (DatabaseDescriptor.backPressureEnabled() && callback.supportsBackPressure())
         {
@@ -646,7 +643,7 @@ public final class MessagingService implements MessagingServiceMBean
      * @param callback The message callback.
      * @param timeout True if updated following a timeout, false otherwise.
      */
-    public void updateBackPressureOnReceive(InetAddress host, IAsyncCallback callback, boolean timeout)
+    public void updateBackPressureOnReceive(InetAddressAndPort host, IAsyncCallback callback, boolean timeout)
     {
         if (DatabaseDescriptor.backPressureEnabled() && callback.supportsBackPressure())
         {
@@ -669,14 +666,14 @@ public final class MessagingService implements MessagingServiceMBean
      * @param hosts The hosts to apply back-pressure to.
      * @param timeoutInNanos The max back-pressure timeout.
      */
-    public void applyBackPressure(Iterable<InetAddress> hosts, long timeoutInNanos)
+    public void applyBackPressure(Iterable<InetAddressAndPort> hosts, long timeoutInNanos)
     {
         if (DatabaseDescriptor.backPressureEnabled())
         {
             Set<BackPressureState> states = new HashSet<BackPressureState>();
-            for (InetAddress host : hosts)
+            for (InetAddressAndPort host : hosts)
             {
-                if (host.equals(FBUtilities.getBroadcastAddress()))
+                if (host.equals(FBUtilities.getBroadcastAddressAndPort()))
                     continue;
                 OutboundMessagingPool pool = getMessagingConnection(host);
                 if (pool != null)
@@ -686,13 +683,13 @@ public final class MessagingService implements MessagingServiceMBean
         }
     }
 
-    BackPressureState getBackPressureState(InetAddress host)
+    BackPressureState getBackPressureState(InetAddressAndPort host)
     {
         OutboundMessagingPool messagingConnection = getMessagingConnection(host);
         return messagingConnection != null ? messagingConnection.getBackPressureState() : null;
     }
 
-    void markTimeout(InetAddress addr)
+    void markTimeout(InetAddressAndPort addr)
     {
         OutboundMessagingPool conn = channelManagers.get(addr);
         if (conn != null)
@@ -706,13 +703,13 @@ public final class MessagingService implements MessagingServiceMBean
      * @param address the host that replied to the message
      * @param latency
      */
-    public void maybeAddLatency(IAsyncCallback cb, InetAddress address, long latency)
+    public void maybeAddLatency(IAsyncCallback cb, InetAddressAndPort address, long latency)
     {
         if (cb.isLatencyForSnitch())
             addLatency(address, latency);
     }
 
-    public void addLatency(InetAddress address, long latency)
+    public void addLatency(InetAddressAndPort address, long latency)
     {
         for (ILatencySubscriber subscriber : subscribers)
             subscriber.receiveTiming(address, latency);
@@ -721,7 +718,7 @@ public final class MessagingService implements MessagingServiceMBean
     /**
      * called from gossiper when it notices a node is not responding.
      */
-    public void convict(InetAddress ep)
+    public void convict(InetAddressAndPort ep)
     {
         logger.trace("Resetting pool for {}", ep);
         reset(ep);
@@ -735,24 +732,24 @@ public final class MessagingService implements MessagingServiceMBean
     public void listen(ServerEncryptionOptions serverEncryptionOptions)
     {
         callbacks.reset(); // hack to allow tests to stop/restart MS
-        listen(FBUtilities.getLocalAddress(), serverEncryptionOptions);
+        listen(FBUtilities.getLocalAddressAndPort(), serverEncryptionOptions);
         if (shouldListenOnBroadcastAddress())
-            listen(FBUtilities.getBroadcastAddress(), serverEncryptionOptions);
+            listen(FBUtilities.getBroadcastAddressAndPort(), serverEncryptionOptions);
         listenGate.signalAll();
     }
 
     public static boolean shouldListenOnBroadcastAddress()
     {
         return DatabaseDescriptor.shouldListenOnBroadcastAddress()
-               && !FBUtilities.getLocalAddress().equals(FBUtilities.getBroadcastAddress());
+               && !FBUtilities.getLocalAddressAndPort().equals(FBUtilities.getBroadcastAddressAndPort());
     }
 
     /**
      * Listen on the specified port.
      *
-     * @param localEp InetAddress whose port to listen on.
+     * @param localEp InetAddressAndPort whose port to listen on.
      */
-    private void listen(InetAddress localEp, ServerEncryptionOptions serverEncryptionOptions) throws ConfigurationException
+    private void listen(InetAddressAndPort localEp, ServerEncryptionOptions serverEncryptionOptions) throws ConfigurationException
     {
         IInternodeAuthenticator authenticator = DatabaseDescriptor.getInternodeAuthenticator();
         int receiveBufferSize = DatabaseDescriptor.getInternodeRecvBufferSize();
@@ -766,7 +763,7 @@ public final class MessagingService implements MessagingServiceMBean
             ServerEncryptionOptions legacyEncOptions = new ServerEncryptionOptions(serverEncryptionOptions);
             legacyEncOptions.optional = false;
 
-            InetSocketAddress localAddr = new InetSocketAddress(localEp, DatabaseDescriptor.getSSLStoragePort());
+            InetAddressAndPort localAddr = InetAddressAndPort.getByAddressOverrideDefaults(localEp.address, DatabaseDescriptor.getSSLStoragePort());
             ChannelGroup channelGroup = new DefaultChannelGroup("LegacyEncryptedInternodeMessagingGroup", NettyFactory.executorForChannelGroups());
             InboundInitializer initializer = new InboundInitializer(authenticator, legacyEncOptions, channelGroup);
             Channel encryptedChannel = NettyFactory.instance.createInboundChannel(localAddr, initializer, receiveBufferSize);
@@ -774,7 +771,8 @@ public final class MessagingService implements MessagingServiceMBean
         }
 
         // this is for the socket that can be plain, only ssl, or optional plain/ssl
-        InetSocketAddress localAddr = new InetSocketAddress(localEp, DatabaseDescriptor.getStoragePort());
+        assert localEp.port == DatabaseDescriptor.getStoragePort() : String.format("Local endpoint port %d doesn't match YAML configured port %d%n", localEp.port, DatabaseDescriptor.getStoragePort());
+        InetAddressAndPort localAddr = InetAddressAndPort.getByAddressOverrideDefaults(localEp.address, DatabaseDescriptor.getStoragePort());
         ChannelGroup channelGroup = new DefaultChannelGroup("InternodeMessagingGroup", NettyFactory.executorForChannelGroups());
         InboundInitializer initializer = new InboundInitializer(authenticator, serverEncryptionOptions, channelGroup);
         Channel channel = NettyFactory.instance.createInboundChannel(localAddr, initializer, receiveBufferSize);
@@ -809,10 +807,10 @@ public final class MessagingService implements MessagingServiceMBean
          * the inbound connections/channels can be closed when the listening socket itself is being closed.
          */
         private final ChannelGroup connectedChannels;
-        private final InetSocketAddress address;
+        private final InetAddressAndPort address;
         private final SecurityLevel securityLevel;
 
-        private ServerChannel(Channel channel, ChannelGroup channelGroup, InetSocketAddress address, SecurityLevel securityLevel)
+        private ServerChannel(Channel channel, ChannelGroup channelGroup, InetAddressAndPort address, SecurityLevel securityLevel)
         {
             this.channel = channel;
             this.connectedChannels = channelGroup;
@@ -840,7 +838,7 @@ public final class MessagingService implements MessagingServiceMBean
             return channel;
         }
 
-        InetSocketAddress getAddress()
+        InetAddressAndPort getAddress()
         {
             return address;
         }
@@ -869,7 +867,7 @@ public final class MessagingService implements MessagingServiceMBean
     }
 
 
-    public void destroyConnectionPool(InetAddress to)
+    public void destroyConnectionPool(InetAddressAndPort to)
     {
         OutboundMessagingPool pool = channelManagers.remove(to);
         if (pool != null)
@@ -884,26 +882,26 @@ public final class MessagingService implements MessagingServiceMBean
      * @param address IP Address to identify the peer
      * @param preferredAddress IP Address to use (and prefer) going forward for connecting to the peer
      */
-    public void reconnectWithNewIp(InetAddress address, InetAddress preferredAddress)
+    public void reconnectWithNewIp(InetAddressAndPort address, InetAddressAndPort preferredAddress)
     {
         SystemKeyspace.updatePreferredIP(address, preferredAddress);
 
         OutboundMessagingPool messagingPool = channelManagers.get(address);
         if (messagingPool != null)
-            messagingPool.reconnectWithNewIp(new InetSocketAddress(preferredAddress, portFor(address)));
+            messagingPool.reconnectWithNewIp(InetAddressAndPort.getByAddressOverrideDefaults(preferredAddress.address, portFor(address)));
     }
 
-    private void reset(InetAddress address)
+    private void reset(InetAddressAndPort address)
     {
         OutboundMessagingPool messagingPool = channelManagers.remove(address);
         if (messagingPool != null)
             messagingPool.close(false);
     }
 
-    public InetAddress getCurrentEndpoint(InetAddress publicAddress)
+    public InetAddressAndPort getCurrentEndpoint(InetAddressAndPort publicAddress)
     {
         OutboundMessagingPool messagingPool = getMessagingConnection(publicAddress);
-        return messagingPool != null ? messagingPool.getPreferredRemoteAddr().getAddress() : null;
+        return messagingPool != null ? messagingPool.getPreferredRemoteAddr() : null;
     }
 
     /**
@@ -931,7 +929,7 @@ public final class MessagingService implements MessagingServiceMBean
         return verbHandlers.get(type);
     }
 
-    public int addCallback(IAsyncCallback cb, MessageOut message, InetAddress to, long timeout, boolean failureCallback)
+    public int addCallback(IAsyncCallback cb, MessageOut message, InetAddressAndPort to, long timeout, boolean failureCallback)
     {
         assert message.verb != Verb.MUTATION; // mutations need to call the overload with a ConsistencyLevel
         int messageId = nextId();
@@ -942,7 +940,7 @@ public final class MessagingService implements MessagingServiceMBean
 
     public int addCallback(IAsyncCallback cb,
                            MessageOut<?> message,
-                           InetAddress to,
+                           InetAddressAndPort to,
                            long timeout,
                            ConsistencyLevel consistencyLevel,
                            boolean allowHints)
@@ -971,12 +969,12 @@ public final class MessagingService implements MessagingServiceMBean
         return idGen.incrementAndGet();
     }
 
-    public int sendRR(MessageOut message, InetAddress to, IAsyncCallback cb)
+    public int sendRR(MessageOut message, InetAddressAndPort to, IAsyncCallback cb)
     {
         return sendRR(message, to, cb, message.getTimeout(), false);
     }
 
-    public int sendRRWithFailure(MessageOut message, InetAddress to, IAsyncCallbackWithFailure cb)
+    public int sendRRWithFailure(MessageOut message, InetAddressAndPort to, IAsyncCallbackWithFailure cb)
     {
         return sendRR(message, to, cb, message.getTimeout(), true);
     }
@@ -992,11 +990,11 @@ public final class MessagingService implements MessagingServiceMBean
      * @param timeout the timeout used for expiration
      * @return an reference to message id used to match with the result
      */
-    public int sendRR(MessageOut message, InetAddress to, IAsyncCallback cb, long timeout, boolean failureCallback)
+    public int sendRR(MessageOut message, InetAddressAndPort to, IAsyncCallback cb, long timeout, boolean failureCallback)
     {
         int id = addCallback(cb, message, to, timeout, failureCallback);
         updateBackPressureOnSend(to, cb, message);
-        sendOneWay(failureCallback ? message.withParameter(FAILURE_CALLBACK_PARAM, ONE_BYTE) : message, id, to);
+        sendOneWay(failureCallback ? message.withParameter(ParameterType.FAILURE_CALLBACK, ONE_BYTE) : message, id, to);
         return id;
     }
 
@@ -1013,22 +1011,22 @@ public final class MessagingService implements MessagingServiceMBean
      * @return an reference to message id used to match with the result
      */
     public int sendRR(MessageOut<?> message,
-                      InetAddress to,
+                      InetAddressAndPort to,
                       AbstractWriteResponseHandler<?> handler,
                       boolean allowHints)
     {
         int id = addCallback(handler, message, to, message.getTimeout(), handler.consistencyLevel, allowHints);
         updateBackPressureOnSend(to, handler, message);
-        sendOneWay(message.withParameter(FAILURE_CALLBACK_PARAM, ONE_BYTE), id, to);
+        sendOneWay(message.withParameter(ParameterType.FAILURE_CALLBACK, ONE_BYTE), id, to);
         return id;
     }
 
-    public void sendOneWay(MessageOut message, InetAddress to)
+    public void sendOneWay(MessageOut message, InetAddressAndPort to)
     {
         sendOneWay(message, nextId(), to);
     }
 
-    public void sendReply(MessageOut message, int id, InetAddress to)
+    public void sendReply(MessageOut message, int id, InetAddressAndPort to)
     {
         sendOneWay(message, id, to);
     }
@@ -1040,12 +1038,12 @@ public final class MessagingService implements MessagingServiceMBean
      * @param message messages to be sent.
      * @param to      endpoint to which the message needs to be sent
      */
-    public void sendOneWay(MessageOut message, int id, InetAddress to)
+    public void sendOneWay(MessageOut message, int id, InetAddressAndPort to)
     {
         if (logger.isTraceEnabled())
-            logger.trace("{} sending {} to {}@{}", FBUtilities.getBroadcastAddress(), message.verb, id, to);
+            logger.trace("{} sending {} to {}@{}", FBUtilities.getBroadcastAddressAndPort(), message.verb, id, to);
 
-        if (to.equals(FBUtilities.getBroadcastAddress()))
+        if (to.equals(FBUtilities.getBroadcastAddressAndPort()))
             logger.trace("Message-to-self {} going over MessagingService", message);
 
         // message sinks are a testing hook
@@ -1058,7 +1056,7 @@ public final class MessagingService implements MessagingServiceMBean
             outboundMessagingPool.sendMessage(message, id);
     }
 
-    public <T> AsyncOneResponse<T> sendRR(MessageOut message, InetAddress to)
+    public <T> AsyncOneResponse<T> sendRR(MessageOut message, InetAddressAndPort to)
     {
         AsyncOneResponse<T> iar = new AsyncOneResponse<T>();
         sendRR(message, to, iar);
@@ -1176,7 +1174,7 @@ public final class MessagingService implements MessagingServiceMBean
     /**
      * @return the last version associated with address, or @param version if this is the first such version
      */
-    public int setVersion(InetAddress endpoint, int version)
+    public int setVersion(InetAddressAndPort endpoint, int version)
     {
         logger.trace("Setting version {} for {}", version, endpoint);
 
@@ -1184,7 +1182,7 @@ public final class MessagingService implements MessagingServiceMBean
         return v == null ? version : v;
     }
 
-    public void resetVersion(InetAddress endpoint)
+    public void resetVersion(InetAddressAndPort endpoint)
     {
         logger.trace("Resetting version for {}", endpoint);
         versions.remove(endpoint);
@@ -1194,7 +1192,7 @@ public final class MessagingService implements MessagingServiceMBean
      * Returns the messaging-version as announced by the given node but capped
      * to the min of the version as announced by the node and {@link #current_version}.
      */
-    public int getVersion(InetAddress endpoint)
+    public int getVersion(InetAddressAndPort endpoint)
     {
         Integer v = versions.get(endpoint);
         if (v == null)
@@ -1209,13 +1207,13 @@ public final class MessagingService implements MessagingServiceMBean
 
     public int getVersion(String endpoint) throws UnknownHostException
     {
-        return getVersion(InetAddress.getByName(endpoint));
+        return getVersion(InetAddressAndPort.getByName(endpoint));
     }
 
     /**
      * Returns the messaging-version exactly as announced by the given endpoint.
      */
-    public int getRawVersion(InetAddress endpoint)
+    public int getRawVersion(InetAddressAndPort endpoint)
     {
         Integer v = versions.get(endpoint);
         if (v == null)
@@ -1223,7 +1221,7 @@ public final class MessagingService implements MessagingServiceMBean
         return v;
     }
 
-    public boolean knowsVersion(InetAddress endpoint)
+    public boolean knowsVersion(InetAddressAndPort endpoint)
     {
         return versions.containsKey(endpoint);
     }
@@ -1358,72 +1356,144 @@ public final class MessagingService implements MessagingServiceMBean
     public Map<String, Integer> getLargeMessagePendingTasks()
     {
         Map<String, Integer> pendingTasks = new HashMap<String, Integer>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            pendingTasks.put(entry.getKey().getHostAddress(), entry.getValue().largeMessageChannel.getPendingMessages());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            pendingTasks.put(entry.getKey().toString(false), entry.getValue().largeMessageChannel.getPendingMessages());
         return pendingTasks;
     }
 
     public Map<String, Long> getLargeMessageCompletedTasks()
     {
         Map<String, Long> completedTasks = new HashMap<String, Long>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            completedTasks.put(entry.getKey().getHostAddress(), entry.getValue().largeMessageChannel.getCompletedMessages());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            completedTasks.put(entry.getKey().toString(false), entry.getValue().largeMessageChannel.getCompletedMessages());
         return completedTasks;
     }
 
     public Map<String, Long> getLargeMessageDroppedTasks()
     {
         Map<String, Long> droppedTasks = new HashMap<String, Long>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            droppedTasks.put(entry.getKey().getHostAddress(), entry.getValue().largeMessageChannel.getDroppedMessages());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            droppedTasks.put(entry.getKey().toString(false), entry.getValue().largeMessageChannel.getDroppedMessages());
         return droppedTasks;
     }
 
     public Map<String, Integer> getSmallMessagePendingTasks()
     {
         Map<String, Integer> pendingTasks = new HashMap<String, Integer>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            pendingTasks.put(entry.getKey().getHostAddress(), entry.getValue().smallMessageChannel.getPendingMessages());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            pendingTasks.put(entry.getKey().toString(false), entry.getValue().smallMessageChannel.getPendingMessages());
         return pendingTasks;
     }
 
     public Map<String, Long> getSmallMessageCompletedTasks()
     {
         Map<String, Long> completedTasks = new HashMap<String, Long>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            completedTasks.put(entry.getKey().getHostAddress(), entry.getValue().smallMessageChannel.getCompletedMessages());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            completedTasks.put(entry.getKey().toString(false), entry.getValue().smallMessageChannel.getCompletedMessages());
         return completedTasks;
     }
 
     public Map<String, Long> getSmallMessageDroppedTasks()
     {
         Map<String, Long> droppedTasks = new HashMap<String, Long>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            droppedTasks.put(entry.getKey().getHostAddress(), entry.getValue().smallMessageChannel.getDroppedMessages());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            droppedTasks.put(entry.getKey().toString(false), entry.getValue().smallMessageChannel.getDroppedMessages());
         return droppedTasks;
     }
 
     public Map<String, Integer> getGossipMessagePendingTasks()
     {
         Map<String, Integer> pendingTasks = new HashMap<String, Integer>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            pendingTasks.put(entry.getKey().getHostAddress(), entry.getValue().gossipChannel.getPendingMessages());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            pendingTasks.put(entry.getKey().toString(false), entry.getValue().gossipChannel.getPendingMessages());
         return pendingTasks;
     }
 
     public Map<String, Long> getGossipMessageCompletedTasks()
     {
         Map<String, Long> completedTasks = new HashMap<String, Long>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            completedTasks.put(entry.getKey().getHostAddress(), entry.getValue().gossipChannel.getCompletedMessages());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            completedTasks.put(entry.getKey().toString(false), entry.getValue().gossipChannel.getCompletedMessages());
         return completedTasks;
     }
 
     public Map<String, Long> getGossipMessageDroppedTasks()
     {
         Map<String, Long> droppedTasks = new HashMap<String, Long>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            droppedTasks.put(entry.getKey().getHostAddress(), entry.getValue().gossipChannel.getDroppedMessages());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            droppedTasks.put(entry.getKey().toString(false), entry.getValue().gossipChannel.getDroppedMessages());
+        return droppedTasks;
+    }
+
+    public Map<String, Integer> getLargeMessagePendingTasksWithPort()
+    {
+        Map<String, Integer> pendingTasks = new HashMap<String, Integer>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            pendingTasks.put(entry.getKey().toString(), entry.getValue().largeMessageChannel.getPendingMessages());
+        return pendingTasks;
+    }
+
+    public Map<String, Long> getLargeMessageCompletedTasksWithPort()
+    {
+        Map<String, Long> completedTasks = new HashMap<String, Long>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            completedTasks.put(entry.getKey().toString(), entry.getValue().largeMessageChannel.getCompletedMessages());
+        return completedTasks;
+    }
+
+    public Map<String, Long> getLargeMessageDroppedTasksWithPort()
+    {
+        Map<String, Long> droppedTasks = new HashMap<String, Long>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            droppedTasks.put(entry.getKey().toString(), entry.getValue().largeMessageChannel.getDroppedMessages());
+        return droppedTasks;
+    }
+
+    public Map<String, Integer> getSmallMessagePendingTasksWithPort()
+    {
+        Map<String, Integer> pendingTasks = new HashMap<String, Integer>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            pendingTasks.put(entry.getKey().toString(), entry.getValue().smallMessageChannel.getPendingMessages());
+        return pendingTasks;
+    }
+
+    public Map<String, Long> getSmallMessageCompletedTasksWithPort()
+    {
+        Map<String, Long> completedTasks = new HashMap<String, Long>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            completedTasks.put(entry.getKey().toString(), entry.getValue().smallMessageChannel.getCompletedMessages());
+        return completedTasks;
+    }
+
+    public Map<String, Long> getSmallMessageDroppedTasksWithPort()
+    {
+        Map<String, Long> droppedTasks = new HashMap<String, Long>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            droppedTasks.put(entry.getKey().toString(), entry.getValue().smallMessageChannel.getDroppedMessages());
+        return droppedTasks;
+    }
+
+    public Map<String, Integer> getGossipMessagePendingTasksWithPort()
+    {
+        Map<String, Integer> pendingTasks = new HashMap<String, Integer>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            pendingTasks.put(entry.getKey().toString(), entry.getValue().gossipChannel.getPendingMessages());
+        return pendingTasks;
+    }
+
+    public Map<String, Long> getGossipMessageCompletedTasksWithPort()
+    {
+        Map<String, Long> completedTasks = new HashMap<String, Long>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            completedTasks.put(entry.getKey().toString(), entry.getValue().gossipChannel.getCompletedMessages());
+        return completedTasks;
+    }
+
+    public Map<String, Long> getGossipMessageDroppedTasksWithPort()
+    {
+        Map<String, Long> droppedTasks = new HashMap<String, Long>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            droppedTasks.put(entry.getKey().toString(), entry.getValue().gossipChannel.getDroppedMessages());
         return droppedTasks;
     }
 
@@ -1435,7 +1505,6 @@ public final class MessagingService implements MessagingServiceMBean
         return map;
     }
 
-
     public long getTotalTimeouts()
     {
         return ConnectionMetrics.totalTimeouts.getCount();
@@ -1444,9 +1513,21 @@ public final class MessagingService implements MessagingServiceMBean
     public Map<String, Long> getTimeoutsPerHost()
     {
         Map<String, Long> result = new HashMap<String, Long>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
         {
-            String ip = entry.getKey().getHostAddress();
+            String ip = entry.getKey().toString(false);
+            long recent = entry.getValue().getTimeouts();
+            result.put(ip, recent);
+        }
+        return result;
+    }
+
+    public Map<String, Long> getTimeoutsPerHostWithPort()
+    {
+        Map<String, Long> result = new HashMap<String, Long>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+        {
+            String ip = entry.getKey().toString();
             long recent = entry.getValue().getTimeouts();
             result.put(ip, recent);
         }
@@ -1456,8 +1537,17 @@ public final class MessagingService implements MessagingServiceMBean
     public Map<String, Double> getBackPressurePerHost()
     {
         Map<String, Double> map = new HashMap<>(channelManagers.size());
-        for (Map.Entry<InetAddress, OutboundMessagingPool> entry : channelManagers.entrySet())
-            map.put(entry.getKey().getHostAddress(), entry.getValue().getBackPressureState().getBackPressureRateLimit());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            map.put(entry.getKey().toString(false), entry.getValue().getBackPressureState().getBackPressureRateLimit());
+
+        return map;
+    }
+
+    public Map<String, Double> getBackPressurePerHostWithPort()
+    {
+        Map<String, Double> map = new HashMap<>(channelManagers.size());
+        for (Map.Entry<InetAddressAndPort, OutboundMessagingPool> entry : channelManagers.entrySet())
+            map.put(entry.getKey().toString(false), entry.getValue().getBackPressureState().getBackPressureRateLimit());
 
         return map;
     }
@@ -1493,18 +1583,18 @@ public final class MessagingService implements MessagingServiceMBean
                                                    bounds.left.getPartitioner().getClass().getName()));
     }
 
-    private OutboundMessagingPool getMessagingConnection(InetAddress to)
+    private OutboundMessagingPool getMessagingConnection(InetAddressAndPort to)
     {
         OutboundMessagingPool pool = channelManagers.get(to);
         if (pool == null)
         {
             final boolean secure = isEncryptedConnection(to);
             final int port = portFor(to, secure);
-            if (!DatabaseDescriptor.getInternodeAuthenticator().authenticate(to, port))
+            if (!DatabaseDescriptor.getInternodeAuthenticator().authenticate(to.address, port))
                 return null;
 
-            InetSocketAddress preferredRemote = new InetSocketAddress(SystemKeyspace.getPreferredIP(to), port);
-            InetSocketAddress local = new InetSocketAddress(FBUtilities.getLocalAddress(), 0);
+            InetAddressAndPort preferredRemote = SystemKeyspace.getPreferredIP(to);
+            InetAddressAndPort local = FBUtilities.getLocalAddressAndPort();
             ServerEncryptionOptions encryptionOptions = secure ? DatabaseDescriptor.getServerEncryptionOptions() : null;
             IInternodeAuthenticator authenticator = DatabaseDescriptor.getInternodeAuthenticator();
 
@@ -1519,16 +1609,16 @@ public final class MessagingService implements MessagingServiceMBean
         return pool;
     }
 
-    public int portFor(InetAddress addr)
+    public int portFor(InetAddressAndPort addr)
     {
         final boolean secure = isEncryptedConnection(addr);
         return portFor(addr, secure);
     }
 
-    private int portFor(InetAddress address, boolean secure)
+    private int portFor(InetAddressAndPort address, boolean secure)
     {
         if (!secure)
-            return DatabaseDescriptor.getStoragePort();
+            return address.port;
 
         Integer v = versions.get(address);
         // if we don't know the version of the peer, assume it is 4.0 (or higher) as the only time is would be lower
@@ -1536,12 +1626,15 @@ public final class MessagingService implements MessagingServiceMBean
         // unfortunately fail - however the peer should connect to this node (at some point), and once we learn it's version, it'll be
         // in versions map. thus, when we attempt to reconnect to that node, we'll have the version and we can get the correct port.
         // we will be able to remove this logic at 5.0.
+        // Also as of 4.0 we will propagate the "regular" port (which will support both SSL and non-SSL) via gossip so
+        // for SSL and version 4.0 always connect to the gossiped port because if SSL is enabled it should ALWAYS
+        // listen for SSL on the "regular" port.
         int version = v != null ? v.intValue() : VERSION_40;
-        return version < VERSION_40 ? DatabaseDescriptor.getSSLStoragePort() : DatabaseDescriptor.getStoragePort();
+        return version < VERSION_40 ? DatabaseDescriptor.getSSLStoragePort() : address.port;
     }
 
     @VisibleForTesting
-    boolean isConnected(InetAddress address, MessageOut messageOut)
+    boolean isConnected(InetAddressAndPort address, MessageOut messageOut)
     {
         OutboundMessagingPool pool = channelManagers.get(address);
         if (pool == null)
@@ -1549,7 +1642,7 @@ public final class MessagingService implements MessagingServiceMBean
         return pool.getConnection(messageOut).isConnected();
     }
 
-    public static boolean isEncryptedConnection(InetAddress address)
+    public static boolean isEncryptedConnection(InetAddressAndPort address)
     {
         IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
         switch (DatabaseDescriptor.getServerEncryptionOptions().internode_encryption)
@@ -1559,13 +1652,13 @@ public final class MessagingService implements MessagingServiceMBean
             case all:
                 break;
             case dc:
-                if (snitch.getDatacenter(address).equals(snitch.getDatacenter(FBUtilities.getBroadcastAddress())))
+                if (snitch.getDatacenter(address).equals(snitch.getDatacenter(FBUtilities.getBroadcastAddressAndPort())))
                     return false;
                 break;
             case rack:
                 // for rack then check if the DC's are the same.
-                if (snitch.getRack(address).equals(snitch.getRack(FBUtilities.getBroadcastAddress()))
-                    && snitch.getDatacenter(address).equals(snitch.getDatacenter(FBUtilities.getBroadcastAddress())))
+                if (snitch.getRack(address).equals(snitch.getRack(FBUtilities.getBroadcastAddressAndPort()))
+                    && snitch.getDatacenter(address).equals(snitch.getDatacenter(FBUtilities.getBroadcastAddressAndPort())))
                     return false;
                 break;
         }

src/java/org/apache/cassandra/net/MessagingServiceMBean.java

@@ -30,47 +30,69 @@ public interface MessagingServiceMBean
     /**
      * Pending tasks for large message TCP Connections
      */
+    @Deprecated
     public Map<String, Integer> getLargeMessagePendingTasks();
+    public Map<String, Integer> getLargeMessagePendingTasksWithPort();
 
     /**
      * Completed tasks for large message) TCP Connections
      */
+    @Deprecated
     public Map<String, Long> getLargeMessageCompletedTasks();
+    public Map<String, Long> getLargeMessageCompletedTasksWithPort();
 
     /**
      * Dropped tasks for large message TCP Connections
      */
+    @Deprecated
     public Map<String, Long> getLargeMessageDroppedTasks();
+    public Map<String, Long> getLargeMessageDroppedTasksWithPort();
+
 
     /**
      * Pending tasks for small message TCP Connections
      */
+    @Deprecated
     public Map<String, Integer> getSmallMessagePendingTasks();
+    public Map<String, Integer> getSmallMessagePendingTasksWithPort();
+
 
     /**
      * Completed tasks for small message TCP Connections
      */
+    @Deprecated
     public Map<String, Long> getSmallMessageCompletedTasks();
+    public Map<String, Long> getSmallMessageCompletedTasksWithPort();
+
 
     /**
      * Dropped tasks for small message TCP Connections
      */
+    @Deprecated
     public Map<String, Long> getSmallMessageDroppedTasks();
+    public Map<String, Long> getSmallMessageDroppedTasksWithPort();
+
 
     /**
      * Pending tasks for gossip message TCP Connections
      */
+    @Deprecated
     public Map<String, Integer> getGossipMessagePendingTasks();
+    public Map<String, Integer> getGossipMessagePendingTasksWithPort();
 
     /**
      * Completed tasks for gossip message TCP Connections
      */
+    @Deprecated
     public Map<String, Long> getGossipMessageCompletedTasks();
+    public Map<String, Long> getGossipMessageCompletedTasksWithPort();
 
     /**
      * Dropped tasks for gossip message TCP Connections
      */
+    @Deprecated
     public Map<String, Long> getGossipMessageDroppedTasks();
+    public Map<String, Long> getGossipMessageDroppedTasksWithPort();
 
     /**
      * dropped message counts for server lifetime
@@ -85,12 +107,16 @@ public interface MessagingServiceMBean
     /**
      * Number of timeouts per host
      */
+    @Deprecated
     public Map<String, Long> getTimeoutsPerHost();
+    public Map<String, Long> getTimeoutsPerHostWithPort();
 
     /**
      * Back-pressure rate limiting per host
      */
+    @Deprecated
     public Map<String, Double> getBackPressurePerHost();
+    public Map<String, Double> getBackPressurePerHostWithPort();
 
     /**
      * Enable/Disable back-pressure

src/java/org/apache/cassandra/net/ParameterType.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.io.DummyByteVersionedSerializer;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.ShortVersionedSerializer;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+/**
+ * Type names and serializers for various parameters that
+ */
+public enum ParameterType
+{
+    FORWARD_TO("FORWARD_TO", ForwardToSerializer.instance),
+    FORWARD_FROM("FORWARD_FROM", CompactEndpointSerializationHelper.instance),
+    FAILURE_RESPONSE("FAIL", DummyByteVersionedSerializer.instance),
+    FAILURE_REASON("FAIL_REASON", ShortVersionedSerializer.instance),
+    FAILURE_CALLBACK("CAL_BAC", DummyByteVersionedSerializer.instance),
+    TRACE_SESSION("TraceSession", UUIDSerializer.serializer),
+    TRACE_TYPE("TraceType", Tracing.traceTypeSerializer);
+
+    public static final Map<String, ParameterType> byName;
+    public final String key;
+    public final IVersionedSerializer serializer;
+
+    static
+    {
+        ImmutableMap.Builder<String, ParameterType> builder = ImmutableMap.builder();
+        for (ParameterType type : values())
+        {
+            builder.put(type.key, type);
+        }
+        byName = builder.build();
+    }
+
+    ParameterType(String key, IVersionedSerializer serializer)
+    {
+        this.key = key;
+        this.serializer = serializer;
+    }
+
+    public String key()
+    {
+        return key;
+    }
+
+}