From d27f9b0b772ba60fece3f3b1101ff63aabdad1fb Mon Sep 17 00:00:00 2001
From: Paulo Motta <pauloricardomg@gmail.com>
Date: Wed, 18 May 2016 12:40:01 -0300
Subject: [PATCH] Set default streaming_socket_timeout_in_ms to 24 hours

Patch by Paulo Motta; reviewed by Yuki Morishita for CASSANDRA-11840
---
 CHANGES.txt                                         |  1 +
 conf/cassandra.yaml                                 | 10 ++++++----
 src/java/org/apache/cassandra/config/Config.java    |  2 +-
 .../apache/cassandra/streaming/StreamSession.java   | 13 ++++++++++++-
 4 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index d914420cdb..8dfa02aea2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.15
+ * Set default streaming_socket_timeout_in_ms to 24 hours (CASSANDRA-11840)
  * Do not consider local node a valid source during replace (CASSANDRA-11848)
  * Avoid holding SSTableReaders for duration of incremental repair (CASSANDRA-11739)
  * Add message dropped tasks to nodetool netstats (CASSANDRA-11855)
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 90c5be4e03..7f2a6fa9d2 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -635,10 +635,12 @@ request_timeout_in_ms: 10000
 cross_node_timeout: false
 
 # Set socket timeout for streaming operation.
-# The stream session is failed if no data is received by any of the
-# participants within that period.
-# Default value is 3600000, which means streams timeout after an hour.
-# streaming_socket_timeout_in_ms: 3600000
+# The stream session is failed if no data/ack is received by any of the participants
+# within that period, which means this should also be sufficient to stream a large
+# sstable or rebuild table indexes.
+# Default value is 86400000ms, which means stale streams timeout after 24 hours.
+# A value of zero means stream sockets should never time out.
+# streaming_socket_timeout_in_ms: 86400000
 
 # phi value that must be reached for a host to be marked down.
 # most users should never need to adjust this.
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 9ff7096449..2c5e36a2bf 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -85,7 +85,7 @@ public class Config
 
     public volatile Long truncate_request_timeout_in_ms = 60000L;
 
-    public Integer streaming_socket_timeout_in_ms = 3600000;
+    public Integer streaming_socket_timeout_in_ms = 86400000; //24 hours
 
     public boolean cross_node_timeout = false;
 
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 642e837afc..4eb855730e 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.streaming;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.Socket;
+import java.net.SocketTimeoutException;
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -502,7 +503,17 @@ public class StreamSession implements IEndpointStateChangeSubscriber
      */
     public void onError(Throwable e)
     {
-        logger.error("[Stream #{}] Streaming error occurred", planId(), e);
+        if (e instanceof SocketTimeoutException)
+        {
+            logger.error("[Stream #{}] Streaming socket timed out. This means the session peer stopped responding or " +
+                         "is still processing received data. If there is no sign of failure in the other end or a very " +
+                         "dense table is being transferred you may want to increase streaming_socket_timeout_in_ms " +
+                         "property. Current value is {}ms.", planId(), DatabaseDescriptor.getStreamingSocketTimeout(), e);
+        }
+        else
+        {
+            logger.error("[Stream #{}] Streaming error occurred", planId(), e);
+        }
         // send session failure message
         if (handler.isOutgoingConnected())
             handler.sendMessage(new SessionFailedMessage());