Remove nodes in total on restart.

Patch by brandonwilliams, reviewed by Vijay for CASSANDRA-4840

src/java/org/apache/cassandra/gms/Gossiper.java

@@ -316,6 +316,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         FailureDetector.instance.remove(endpoint);
         versions.remove(endpoint);
         quarantineEndpoint(endpoint);
+        MessagingService.instance().destroyConnectionPool(endpoint);
         if (logger.isDebugEnabled())
             logger.debug("removing endpoint " + endpoint);
     }

src/java/org/apache/cassandra/net/MessagingService.java

@@ -298,6 +298,18 @@ public final class MessagingService implements MessagingServiceMBean
         }
     }
 
+    public void destroyConnectionPool(InetAddress to)
+    {
+        OutboundTcpConnectionPool cp = connectionManagers_.get(to);
+        if (to == null)
+            return;
+        cp.ackCon.closeSocket();
+        cp.cmdCon.closeSocket();
+        connectionManagers_.remove(to);
+        recentTimeoutsPerHost.remove(to.getHostAddress());
+        timeoutsPerHost.remove(to.getHostAddress());
+    }
+
     public OutboundTcpConnectionPool getConnectionPool(InetAddress to)
     {
         OutboundTcpConnectionPool cp = connectionManagers_.get(to);

src/java/org/apache/cassandra/service/StorageService.java

@@ -1273,7 +1273,9 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe
                 // grab any data we are now responsible for and notify responsible node
                 restoreReplicaCount(endpoint, tokenMetadata_.getEndpoint(coordtoken));
             }
-        } // not a member, nothing to do
+        }
+        else // now that the gossiper has told us about this nonexistent member, notify the gossiper to remove it
+            Gossiper.instance.removeEndpoint(endpoint);
     }
 
     private void excise(Token token, InetAddress endpoint)