move streaming to use netty

patch by jasobrown, reviewed by aweisberg for CASSANDRA-12229

CHANGES.txt

@@ -1,4 +1,5 @@
 4.0
+ * use netty for streaming (CASSANDRA-12229)
  * Use netty for internode messaging (CASSANDRA-8457)
  * Add bytes repaired/unrepaired to nodetool tablestats (CASSANDRA-13774)
  * Don't delete incremental repair sessions if they still have sstables (CASSANDRA-13758)

src/java/org/apache/cassandra/config/Config.java

@@ -97,12 +97,6 @@ public class Config
 
     public volatile long truncate_request_timeout_in_ms = 60000L;
 
-    /**
-     * @deprecated use {@link #streaming_keep_alive_period_in_secs} instead
-     */
-    @Deprecated
-    public int streaming_socket_timeout_in_ms = 86400000; //24 hours
-
     public Integer streaming_connections_per_host = 1;
     public Integer streaming_keep_alive_period_in_secs = 300; //5 minutes
 

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -2060,21 +2060,6 @@ public class DatabaseDescriptor
         conf.counter_cache_keys_to_save = counterCacheKeysToSave;
     }
 
-    public static void setStreamingSocketTimeout(int value)
-    {
-        conf.streaming_socket_timeout_in_ms = value;
-    }
-
-    /**
-     * @deprecated use {@link #getStreamingKeepAlivePeriod()} instead
-     * @return streaming_socket_timeout_in_ms property
-     */
-    @Deprecated
-    public static int getStreamingSocketTimeout()
-    {
-        return conf.streaming_socket_timeout_in_ms;
-    }
-
     public static int getStreamingKeepAlivePeriod()
     {
         return conf.streaming_keep_alive_period_in_secs;

src/java/org/apache/cassandra/exceptions/ChecksumMismatchException.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.exceptions;
+
+import java.io.IOException;
+
+public class ChecksumMismatchException extends IOException
+{
+    public ChecksumMismatchException()
+    {
+        super();
+    }
+
+    public ChecksumMismatchException(String s)
+    {
+        super(s);
+    }
+}

src/java/org/apache/cassandra/io/compress/CompressionMetadata.java

@@ -145,7 +145,9 @@ public class CompressionMetadata
         this.chunkOffsetsSize = chunkOffsets.size();
     }
 
-    private CompressionMetadata(String filePath, CompressionParams parameters, SafeMemory offsets, long offsetsSize, long dataLength, long compressedLength)
+    // do not call this constructor directly, unless used in testing
+    @VisibleForTesting
+    public CompressionMetadata(String filePath, CompressionParams parameters, Memory offsets, long offsetsSize, long dataLength, long compressedLength)
     {
         this.indexFilePath = filePath;
         this.parameters = parameters;

src/java/org/apache/cassandra/io/sstable/SSTableLoader.java

@@ -208,7 +208,7 @@ public class SSTableLoader implements StreamEventHandler
         for (SSTableReader sstable : sstables)
         {
             sstable.selfRef().release();
-            assert sstable.selfRef().globalCount() == 0;
+            assert sstable.selfRef().globalCount() == 0 : String.format("for sstable = %s, ref count = %d", sstable, sstable.selfRef().globalCount());
         }
     }
 

src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java

@@ -20,9 +20,12 @@ package org.apache.cassandra.io.util;
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.zip.CheckedInputStream;
 import java.util.zip.Checksum;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.utils.ChecksumType;
@@ -57,6 +60,15 @@ public class DataIntegrityMetadata
             chunkSize = reader.readInt();
         }
 
+        @VisibleForTesting
+        protected ChecksumValidator(ChecksumType checksumType, RandomAccessReader reader, int chunkSize)
+        {
+            this.checksumType = checksumType;
+            this.reader = reader;
+            this.dataFilename = null;
+            this.chunkSize = chunkSize;
+        }
+
         public void seek(long offset)
         {
             long start = chunkStart(offset);
@@ -77,6 +89,20 @@ public class DataIntegrityMetadata
                 throw new IOException("Corrupted File : " + dataFilename);
         }
 
+        /**
+         * validates the checksum with the bytes from the specified buffer.
+         *
+         * Upon return, the buffer's position will
+         * be updated to its limit; its limit will not have been changed.
+         */
+        public void validate(ByteBuffer buffer) throws IOException
+        {
+            int current = (int) checksumType.of(buffer);
+            int actual = reader.readInt();
+            if (current != actual)
+                throw new IOException("Corrupted File : " + dataFilename);
+        }
+
         public void close()
         {
             reader.close();

src/java/org/apache/cassandra/net/IncomingStreamingConnection.java

@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.net;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.Socket;
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.streaming.StreamResultFuture;
-import org.apache.cassandra.streaming.messages.StreamInitMessage;
-import org.apache.cassandra.streaming.messages.StreamMessage;
-
-/**
- * Thread to consume stream init messages.
- */
-public class IncomingStreamingConnection extends Thread implements Closeable
-{
-    private static final Logger logger = LoggerFactory.getLogger(IncomingStreamingConnection.class);
-
-    private final int version;
-    public final Socket socket;
-    private final Set<Closeable> group;
-
-    public IncomingStreamingConnection(int version, Socket socket, Set<Closeable> group)
-    {
-        super("STREAM-INIT-" + socket.getRemoteSocketAddress());
-        this.version = version;
-        this.socket = socket;
-        this.group = group;
-    }
-
-    @Override
-    @SuppressWarnings("resource") // Not closing constructed DataInputPlus's as the stream needs to remain open.
-    public void run()
-    {
-        try
-        {
-            // streaming connections are per-session and have a fixed version.
-            // we can't do anything with a wrong-version stream connection, so drop it.
-            if (version != StreamMessage.CURRENT_VERSION)
-                throw new IOException(String.format("Received stream using protocol version %d (my version %d). Terminating connection", version, StreamMessage.CURRENT_VERSION));
-
-            DataInputPlus input = new DataInputStreamPlus(socket.getInputStream());
-            StreamInitMessage init = StreamInitMessage.serializer.deserialize(input, version);
-
-            //Set SO_TIMEOUT on follower side
-            if (!init.isForOutgoing)
-                socket.setSoTimeout(DatabaseDescriptor.getStreamingSocketTimeout());
-
-            // The initiator makes two connections, one for incoming and one for outgoing.
-            // The receiving side distinguish two connections by looking at StreamInitMessage#isForOutgoing.
-            // Note: we cannot use the same socket for incoming and outgoing streams because we want to
-            // parallelize said streams and the socket is blocking, so we might deadlock.
-            StreamResultFuture.initReceivingSide(init.sessionIndex, init.planId, init.streamOperation, init.from, this, init.isForOutgoing, version, init.keepSSTableLevel, init.pendingRepair, init.previewKind);
-        }
-        catch (Throwable t)
-        {
-            logger.error("Error while reading from socket from {}.", socket.getRemoteSocketAddress(), t);
-            close();
-        }
-    }
-
-    @Override
-    public void close()
-    {
-        try
-        {
-            if (!socket.isClosed())
-            {
-                socket.close();
-            }
-        }
-        catch (IOException e)
-        {
-            logger.debug("Error closing socket", e);
-        }
-        finally
-        {
-            group.remove(this);
-        }
-    }
-}

src/java/org/apache/cassandra/net/async/ByteBufDataInputPlus.java

@@ -24,8 +24,20 @@ import org.apache.cassandra.io.util.DataInputPlus;
 
 public class ByteBufDataInputPlus extends ByteBufInputStream implements DataInputPlus
 {
+    /**
+     * The parent class does not expose the buffer to derived classes, so we need
+     * to stash a reference here so it can be exposed via {@link #buffer()}.
+     */
+    private final ByteBuf buf;
+
     public ByteBufDataInputPlus(ByteBuf buffer)
     {
         super(buffer);
+        this.buf = buffer;
+    }
+
+    public ByteBuf buffer()
+    {
+        return buf;
     }
 }

src/java/org/apache/cassandra/net/async/ByteBufDataOutputStreamPlus.java

@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net.async;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.util.concurrent.Future;
+import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.streaming.StreamSession;
+
+/**
+ * A {@link DataOutputStreamPlus} that writes to a {@link ByteBuf}. The novelty here is that all writes
+ * actually get written in to a {@link ByteBuffer} that shares a backing buffer with a {@link ByteBuf}.
+ * The trick to do that is allocate the ByteBuf, get a ByteBuffer from it by calling {@link ByteBuf#nioBuffer()},
+ * and passing that to the super class as {@link #buffer}. When the {@link #buffer} is full or {@link #doFlush(int)}
+ * is invoked, the {@link #currentBuf} is published to the netty channel.
+ */
+public class ByteBufDataOutputStreamPlus extends BufferedDataOutputStreamPlus
+{
+    private final StreamSession session;
+    private final Channel channel;
+    private final int bufferSize;
+
+    /**
+     * Tracks how many bytes we've written to the netty channel. This more or less follows the channel's
+     * high/low water marks and ultimately the 'writablility' status of the channel. Unfortunately there's
+     * no notification mechanism that can poke a producer to let it know when the channel becomes writable
+     * (after it was unwritable); hence, the use of a {@link Semaphore}.
+     */
+    private final Semaphore channelRateLimiter;
+
+    /**
+     * This *must* be the owning {@link ByteBuf} for the {@link BufferedDataOutputStreamPlus#buffer}
+     */
+    private ByteBuf currentBuf;
+
+    private ByteBufDataOutputStreamPlus(StreamSession session, Channel channel, ByteBuf buffer, int bufferSize)
+    {
+        super(buffer.nioBuffer(0, bufferSize));
+        this.session = session;
+        this.channel = channel;
+        this.currentBuf = buffer;
+        this.bufferSize = bufferSize;
+
+        channelRateLimiter = new Semaphore(channel.config().getWriteBufferHighWaterMark(), true);
+    }
+
+    @Override
+    protected WritableByteChannel newDefaultChannel()
+    {
+        return new WritableByteChannel()
+        {
+            @Override
+            public int write(ByteBuffer src) throws IOException
+            {
+                assert src == buffer;
+                int size = src.position();
+                doFlush(size);
+                return size;
+            }
+
+            @Override
+            public boolean isOpen()
+            {
+                return channel.isOpen();
+            }
+
+            @Override
+            public void close()
+            {   }
+        };
+    }
+
+    public static ByteBufDataOutputStreamPlus create(StreamSession session, Channel channel, int bufferSize)
+    {
+        ByteBuf buf = channel.alloc().directBuffer(bufferSize, bufferSize);
+        return new ByteBufDataOutputStreamPlus(session, channel, buf, bufferSize);
+    }
+
+    /**
+     * Writes the incoming buffer directly to the backing {@link #channel}, without copying to the intermediate {@link #buffer}.
+     */
+    public ChannelFuture writeToChannel(ByteBuf buf) throws IOException
+    {
+        doFlush(buffer.position());
+
+        int byteCount = buf.readableBytes();
+        if (!Uninterruptibles.tryAcquireUninterruptibly(channelRateLimiter, byteCount, 5, TimeUnit.MINUTES))
+            throw new IOException("outbound channel was not writable");
+
+        // the (possibly naive) assumption that we should always flush after each incoming buf
+        ChannelFuture channelFuture = channel.writeAndFlush(buf);
+        channelFuture.addListener(future -> handleBuffer(future, byteCount));
+        return channelFuture;
+    }
+
+    /**
+     * Writes the incoming buffer directly to the backing {@link #channel}, without copying to the intermediate {@link #buffer}.
+     * The incoming buffer will be automatically released when the netty channel invokes the listeners of success/failure to
+     * send the buffer.
+     */
+    public ChannelFuture writeToChannel(ByteBuffer buffer) throws IOException
+    {
+        ChannelFuture channelFuture = writeToChannel(Unpooled.wrappedBuffer(buffer));
+        channelFuture.addListener(future -> FileUtils.clean(buffer));
+        return channelFuture;
+    }
+
+    @Override
+    protected void doFlush(int count) throws IOException
+    {
+        // flush the current backing write buffer only if there's any pending data
+        if (buffer.position() > 0 && channel.isOpen())
+        {
+            int byteCount = buffer.position();
+            currentBuf.writerIndex(byteCount);
+
+            if (!Uninterruptibles.tryAcquireUninterruptibly(channelRateLimiter, byteCount, 2, TimeUnit.MINUTES))
+                throw new IOException("outbound channel was not writable");
+
+            channel.writeAndFlush(currentBuf).addListener(future -> handleBuffer(future, byteCount));
+            currentBuf = channel.alloc().directBuffer(bufferSize, bufferSize);
+            buffer = currentBuf.nioBuffer(0, bufferSize);
+        }
+    }
+
+    /**
+     * Handles the result of publishing a buffer to the channel.
+     *
+     * Note: this will be executed on the event loop.
+     */
+    private void handleBuffer(Future<? super Void> future, int bytesWritten)
+    {
+        channelRateLimiter.release(bytesWritten);
+
+        if (!future.isSuccess() && channel.isOpen())
+            session.onError(future.cause());
+    }
+
+    public ByteBufAllocator getAllocator()
+    {
+        return channel.alloc();
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * Flush any last buffered (if the channel is open), and release any buffers. *Not* responsible for closing
+     * the netty channel as we might use it again for transferring more files.
+     *
+     * Note: should be called on the producer thread, not the netty event loop.
+     */
+    @Override
+    public void close() throws IOException
+    {
+        doFlush(0);
+        if (currentBuf.refCnt() > 0)
+            currentBuf.release();
+        currentBuf = null;
+        buffer = null;
+    }
+}

src/java/org/apache/cassandra/net/async/InboundHandshakeHandler.java

@@ -14,6 +14,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import io.netty.buffer.ByteBuf;
+import io.netty.channel.AdaptiveRecvByteBufAllocator;
 import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelPipeline;
@@ -25,6 +26,8 @@ import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.async.HandshakeProtocol.FirstHandshakeMessage;
 import org.apache.cassandra.net.async.HandshakeProtocol.SecondHandshakeMessage;
 import org.apache.cassandra.net.async.HandshakeProtocol.ThirdHandshakeMessage;
+import org.apache.cassandra.streaming.async.StreamingInboundHandler;
+import org.apache.cassandra.streaming.messages.StreamMessage;
 
 /**
  * 'Server'-side component that negotiates the internode handshake when establishing a new connection.
@@ -36,13 +39,13 @@ class InboundHandshakeHandler extends ByteToMessageDecoder
 {
     private static final Logger logger = LoggerFactory.getLogger(NettyFactory.class);
 
-    enum State { START, AWAITING_HANDSHAKE_BEGIN, AWAIT_STREAM_START_RESPONSE, AWAIT_MESSAGING_START_RESPONSE, MESSAGING_HANDSHAKE_COMPLETE, HANDSHAKE_FAIL }
+    enum State { START, AWAITING_HANDSHAKE_BEGIN, AWAIT_MESSAGING_START_RESPONSE, HANDSHAKE_COMPLETE, HANDSHAKE_FAIL }
 
     private State state;
 
     private final IInternodeAuthenticator authenticator;
-    private boolean hasAuthenticated;
 
+    private boolean hasAuthenticated;
     /**
      * The peer's declared messaging version.
      */
@@ -160,9 +163,16 @@ class InboundHandshakeHandler extends ByteToMessageDecoder
 
         if (msg.mode == NettyFactory.Mode.STREAMING)
         {
-            // TODO fill in once streaming is moved to netty
-            ctx.close();
-            return State.AWAIT_STREAM_START_RESPONSE;
+            // streaming connections are per-session and have a fixed version.  we can't do anything with a wrong-version stream connection, so drop it.
+            if (version != StreamMessage.CURRENT_VERSION)
+            {
+                logger.warn("Received stream using protocol version %d (my version %d). Terminating connection", version, MessagingService.current_version);
+                ctx.close();
+                return State.HANDSHAKE_FAIL;
+            }
+
+            setupStreamingPipeline(ctx, version);
+            return State.HANDSHAKE_COMPLETE;
         }
         else
         {
@@ -195,6 +205,18 @@ class InboundHandshakeHandler extends ByteToMessageDecoder
         }
     }
 
+    private void setupStreamingPipeline(ChannelHandlerContext ctx, int protocolVersion)
+    {
+        ChannelPipeline pipeline = ctx.pipeline();
+        InetSocketAddress address = (InetSocketAddress) ctx.channel().remoteAddress();
+        pipeline.addLast(NettyFactory.instance.streamingGroup, "streamInbound", new StreamingInboundHandler(address, protocolVersion, null));
+        pipeline.remove(this);
+
+        // pass a custom recv ByteBuf allocator to the channel. the default recv ByteBuf size is 1k, but in streaming we're
+        // dealing with large bulk blocks of data, let's default to larger sizes
+        ctx.channel().config().setRecvByteBufAllocator(new AdaptiveRecvByteBufAllocator(1 << 8, 1 << 13, 1 << 16));
+    }
+
     /**
      * Handles the third (and last) message in the internode messaging handshake protocol. Grabs the protocol version and
      * IP addr the peer wants to use.
@@ -227,7 +249,7 @@ class InboundHandshakeHandler extends ByteToMessageDecoder
         logger.trace("Set version for {} to {} (will use {})", from, maxVersion, MessagingService.instance().getVersion(from));
 
         setupMessagingPipeline(ctx.pipeline(), from, compressed, version);
-        return State.MESSAGING_HANDSHAKE_COMPLETE;
+        return State.HANDSHAKE_COMPLETE;
     }
 
     @VisibleForTesting
@@ -245,7 +267,7 @@ class InboundHandshakeHandler extends ByteToMessageDecoder
     {
         // we're not really racing on the handshakeTimeout as we're in the event loop,
         // but, hey, defensive programming is beautiful thing!
-        if (state == State.MESSAGING_HANDSHAKE_COMPLETE || (handshakeTimeout != null && handshakeTimeout.isCancelled()))
+        if (state == State.HANDSHAKE_COMPLETE || (handshakeTimeout != null && handshakeTimeout.isCancelled()))
             return;
 
         state = State.HANDSHAKE_FAIL;

src/java/org/apache/cassandra/net/async/NettyFactory.java

@@ -40,6 +40,7 @@ import io.netty.util.concurrent.DefaultThreadFactory;
 import io.netty.util.concurrent.EventExecutor;
 import io.netty.util.internal.logging.InternalLoggerFactory;
 import io.netty.util.internal.logging.Slf4JLoggerFactory;
+
 import net.jpountz.lz4.LZ4Factory;
 import net.jpountz.xxhash.XXHashFactory;
 import org.apache.cassandra.auth.IInternodeAuthenticator;
@@ -69,12 +70,18 @@ public final class NettyFactory
 
     private static final int LZ4_HASH_SEED = 0x9747b28c;
 
+    /**
+     * Default seed value for xxhash.
+     */
+    public static final int XXHASH_DEFAULT_SEED = 0x9747b28c;
+
     public enum Mode { MESSAGING, STREAMING }
 
     private static final String SSL_CHANNEL_HANDLER_NAME = "ssl";
-    static final String INBOUND_COMPRESSOR_HANDLER_NAME = "inboundCompressor";
-    static final String OUTBOUND_COMPRESSOR_HANDLER_NAME = "outboundCompressor";
-    private static final String HANDSHAKE_HANDLER_NAME = "handshakeHandler";
+    public static final String INBOUND_COMPRESSOR_HANDLER_NAME = "inboundCompressor";
+    public static final String OUTBOUND_COMPRESSOR_HANDLER_NAME = "outboundCompressor";
+    public static final String HANDSHAKE_HANDLER_NAME = "handshakeHandler";
+    public static final String INBOUND_STREAM_HANDLER_NAME = "inboundStreamHandler";
 
     /** a useful addition for debugging; simply set to true to get more data in your logs */
     private static final boolean WIRETRACE = false;
@@ -113,6 +120,7 @@ public final class NettyFactory
 
     private final EventLoopGroup inboundGroup;
     private final EventLoopGroup outboundGroup;
+    public final EventLoopGroup streamingGroup;
 
     /**
      * Constructor that allows modifying the {@link NettyFactory#useEpoll} for testing purposes. Otherwise, use the
@@ -126,6 +134,7 @@ public final class NettyFactory
                                         "MessagingService-NettyAcceptor-Threads", false);
         inboundGroup = getEventLoopGroup(useEpoll, FBUtilities.getAvailableProcessors(), "MessagingService-NettyInbound-Threads", false);
         outboundGroup = getEventLoopGroup(useEpoll, FBUtilities.getAvailableProcessors(), "MessagingService-NettyOutbound-Threads", true);
+        streamingGroup = getEventLoopGroup(useEpoll, FBUtilities.getAvailableProcessors(), "Streaming-Netty-Threads", false);
     }
 
     /**
@@ -257,7 +266,8 @@ public final class NettyFactory
                 SslContext sslContext = SSLFactory.getSslContext(encryptionOptions, true, true);
                 SslHandler sslHandler = sslContext.newHandler(channel.alloc());
                 logger.trace("creating inbound netty SslContext: context={}, engine={}", sslContext.getClass().getName(), sslHandler.engine().getClass().getName());
-                pipeline.addFirst(SSL_CHANNEL_HANDLER_NAME, sslHandler);            }
+                pipeline.addFirst(SSL_CHANNEL_HANDLER_NAME, sslHandler);
+            }
 
             if (WIRETRACE)
                 pipeline.addLast("logger", new LoggingHandler(LogLevel.INFO));
@@ -279,13 +289,14 @@ public final class NettyFactory
      * Create the {@link Bootstrap} for connecting to a remote peer. This method does <b>not</b> attempt to connect to the peer,
      * and thus does not block.
      */
+    @VisibleForTesting
     public Bootstrap createOutboundBootstrap(OutboundConnectionParams params)
     {
         logger.debug("creating outbound bootstrap to peer {}, compression: {}, encryption: {}, coalesce: {}", params.connectionId.connectionAddress(),
                      params.compress, encryptionLogStatement(params.encryptionOptions),
                      params.coalescingStrategy.isPresent() ? params.coalescingStrategy.get() : CoalescingStrategies.Strategy.DISABLED);
-        Class<? extends Channel>  transport = useEpoll ? EpollSocketChannel.class : NioSocketChannel.class;
-        Bootstrap bootstrap = new Bootstrap().group(outboundGroup)
+        Class<? extends Channel> transport = useEpoll ? EpollSocketChannel.class : NioSocketChannel.class;
+        Bootstrap bootstrap = new Bootstrap().group(params.mode == Mode.MESSAGING ? outboundGroup : streamingGroup)
                               .channel(transport)
                               .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 2000)
                               .option(ChannelOption.SO_KEEPALIVE, true)
@@ -349,6 +360,7 @@ public final class NettyFactory
         acceptGroup.shutdownGracefully();
         outboundGroup.shutdownGracefully();
         inboundGroup.shutdownGracefully();
+        streamingGroup.shutdownGracefully();
     }
 
     static Lz4FrameEncoder createLz4Encoder(int protocolVersion)

src/java/org/apache/cassandra/net/async/OutboundConnectionIdentifier.java

@@ -32,7 +32,7 @@ public class OutboundConnectionIdentifier
 {
     enum ConnectionType
     {
-        GOSSIP, LARGE_MESSAGE, SMALL_MESSAGE
+        GOSSIP, LARGE_MESSAGE, SMALL_MESSAGE, STREAM
     }
 
     /**
@@ -98,6 +98,15 @@ public class OutboundConnectionIdentifier
         return new OutboundConnectionIdentifier(localAddr, remoteAddr, ConnectionType.GOSSIP);
     }
 
+    /**
+     * Creates an identifier for a gossip connection and using the remote "identifying" address as its connection
+     * address.
+     */
+    public static OutboundConnectionIdentifier stream(InetSocketAddress localAddr, InetSocketAddress remoteAddr)
+    {
+        return new OutboundConnectionIdentifier(localAddr, remoteAddr, ConnectionType.STREAM);
+    }
+
     /**
      * Returns a newly created connection identifier to the same remote that this identifier, but using the provided
      * address as connection address.
@@ -106,7 +115,7 @@ public class OutboundConnectionIdentifier
      * @return a newly created connection identifier that differs from this one only by using {@code remoteConnectionAddr}
      * as connection address to the remote.
      */
-    OutboundConnectionIdentifier withNewConnectionAddress(InetSocketAddress remoteConnectionAddr)
+    public OutboundConnectionIdentifier withNewConnectionAddress(InetSocketAddress remoteConnectionAddr)
     {
         return new OutboundConnectionIdentifier(localAddr, remoteAddr, remoteConnectionAddr, connectionType);
     }
@@ -114,7 +123,7 @@ public class OutboundConnectionIdentifier
     /**
      * The local node address.
      */
-    InetAddress local()
+    public InetAddress local()
     {
         return localAddr.getAddress();
     }
@@ -122,7 +131,7 @@ public class OutboundConnectionIdentifier
     /**
      * The remote node identifying address (the one to use for anything else than connecting to the node).
      */
-    InetSocketAddress remoteAddress()
+    public InetSocketAddress remoteAddress()
     {
         return remoteAddr;
     }
@@ -130,7 +139,7 @@ public class OutboundConnectionIdentifier
     /**
      * The remote node identifying address (the one to use for anything else than connecting to the node).
      */
-    InetAddress remote()
+    public  InetAddress remote()
     {
         return remoteAddr.getAddress();
     }
@@ -138,7 +147,7 @@ public class OutboundConnectionIdentifier
     /**
      * The remote node connection address (the one to use to actually connect to the remote, and only that).
      */
-    InetSocketAddress connectionAddress()
+    public InetSocketAddress connectionAddress()
     {
         return remoteConnectionAddr;
     }

src/java/org/apache/cassandra/net/async/OutboundHandshakeHandler.java

@@ -36,6 +36,7 @@ import io.netty.channel.ChannelPipeline;
 import io.netty.handler.codec.ByteToMessageDecoder;
 import io.netty.handler.timeout.IdleStateHandler;
 import io.netty.util.concurrent.Future;
+
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.async.HandshakeProtocol.FirstHandshakeMessage;
@@ -95,7 +96,9 @@ public class OutboundHandshakeHandler extends ByteToMessageDecoder
     /**
      * {@inheritDoc}
      *
-     * Invoked when the channel is made active, and sends out the {@link FirstHandshakeMessage}
+     * Invoked when the channel is made active, and sends out the {@link FirstHandshakeMessage}.
+     * In the case of streaming, we do not require a full bi-directional handshake; the initial message,
+     * containing the streaming protocol version, is all that is required.
      */
     @Override
     public void channelActive(final ChannelHandlerContext ctx) throws Exception
@@ -103,6 +106,10 @@ public class OutboundHandshakeHandler extends ByteToMessageDecoder
         FirstHandshakeMessage msg = new FirstHandshakeMessage(messagingVersion, mode, params.compress);
         logger.trace("starting handshake with peer {}, msg = {}", connectionId.connectionAddress(), msg);
         ctx.writeAndFlush(msg.encode(ctx.alloc())).addListener(future -> firstHandshakeMessageListener(future, ctx));
+
+        if (mode == NettyFactory.Mode.STREAMING)
+            ctx.pipeline().remove(this);
+
         ctx.fireChannelActive();
     }
 

src/java/org/apache/cassandra/net/async/RebufferingByteBufDataInputPlus.java

@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net.async;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelConfig;
+import io.netty.util.ReferenceCountUtil;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+
+public class RebufferingByteBufDataInputPlus extends RebufferingInputStream implements ReadableByteChannel
+{
+    /**
+     * The parent, or owning, buffer of the current buffer being read from ({@link super#buffer}).
+     */
+    private ByteBuf currentBuf;
+
+    private final BlockingQueue<ByteBuf> queue;
+
+    /**
+     * The count of live bytes in all {@link ByteBuf}s held by this instance.
+     */
+    private final AtomicInteger queuedByteCount;
+
+    private final int lowWaterMark;
+    private final int highWaterMark;
+    private final ChannelConfig channelConfig;
+
+    private volatile boolean closed;
+
+    public RebufferingByteBufDataInputPlus(int lowWaterMark, int highWaterMark, ChannelConfig channelConfig)
+    {
+        super(Unpooled.EMPTY_BUFFER.nioBuffer());
+
+        if (lowWaterMark > highWaterMark)
+            throw new IllegalArgumentException(String.format("low water mark is greater than high water mark: %d vs %d", lowWaterMark, highWaterMark));
+
+        currentBuf = Unpooled.EMPTY_BUFFER;
+        this.lowWaterMark = lowWaterMark;
+        this.highWaterMark = highWaterMark;
+        this.channelConfig = channelConfig;
+        queue = new LinkedBlockingQueue<>();
+        queuedByteCount = new AtomicInteger();
+    }
+
+    /**
+     * Append a {@link ByteBuf} to the end of the einternal queue.
+     *
+     * Note: it's expected this method is invoked on the netty event loop.
+     */
+    public void append(ByteBuf buf) throws IllegalStateException
+    {
+        assert buf != null : "buffer cannot be null";
+
+        if (closed)
+        {
+            ReferenceCountUtil.release(buf);
+            throw new IllegalStateException("stream is already closed, so cannot add another buffer");
+        }
+
+        // this slightly undercounts the live count as it doesn't include the currentBuf's size.
+        // that's ok as the worst we'll do is allow another buffer in and add it to the queue,
+        // and that point we'll disable auto-read. this is a tradeoff versus making some other member field
+        // atomic or volatile.
+        int queuedCount = queuedByteCount.addAndGet(buf.readableBytes());
+        if (channelConfig.isAutoRead() && queuedCount > highWaterMark)
+            channelConfig.setAutoRead(false);
+
+        queue.add(buf);
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * Release open buffers and poll the {@link #queue} for more data.
+     * <p>
+     * This is best, and more or less expected, to be invoked on a consuming thread (not the event loop)
+     * becasue if we block on the queue we can't fill it on the event loop (as that's where the buffers are coming from).
+     */
+    @Override
+    protected void reBuffer() throws IOException
+    {
+        currentBuf.release();
+        buffer = null;
+        currentBuf = null;
+
+        // possibly re-enable auto-read, *before* blocking on the queue, because if we block on the queue
+        // without enabling auto-read we'll block forever :(
+        if (!channelConfig.isAutoRead() && queuedByteCount.get() < lowWaterMark)
+            channelConfig.setAutoRead(true);
+
+        try
+        {
+            currentBuf = queue.take();
+            int bytes;
+            // if we get an explicitly empty buffer, we treat that as an indicator that the input is closed
+            if (currentBuf == null || (bytes = currentBuf.readableBytes()) == 0)
+            {
+                releaseResources();
+                throw new EOFException();
+            }
+
+            buffer = currentBuf.nioBuffer(currentBuf.readerIndex(), bytes);
+            assert buffer.remaining() == bytes;
+            queuedByteCount.addAndGet(-bytes);
+            return;
+        }
+        catch (InterruptedException ie)
+        {
+            // nop - ignore
+        }
+    }
+
+    @Override
+    public int read(ByteBuffer dst) throws IOException
+    {
+        int readLength = dst.remaining();
+        int remaining = readLength;
+
+        while (remaining > 0)
+        {
+            if (closed)
+                throw new EOFException();
+
+            if (!buffer.hasRemaining())
+                reBuffer();
+            int copyLength = Math.min(remaining, buffer.remaining());
+
+            int originalLimit = buffer.limit();
+            buffer.limit(buffer.position() + copyLength);
+            dst.put(buffer);
+            buffer.limit(originalLimit);
+            remaining -= copyLength;
+        }
+
+        return readLength;
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * As long as this method is invoked on the consuming thread the returned value will be accurate.
+     */
+    @Override
+    public int available() throws EOFException
+    {
+        if (closed)
+            throw new EOFException();
+
+       final  int availableBytes = queuedByteCount.get() + (buffer != null ? buffer.remaining() : 0);
+
+        if (!channelConfig.isAutoRead() && availableBytes < lowWaterMark)
+            channelConfig.setAutoRead(true);
+
+        return availableBytes;
+    }
+
+    @Override
+    public boolean isOpen()
+    {
+        return !closed;
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * Note: This should invoked on the consuming thread.
+     */
+    @Override
+    public void close()
+    {
+        closed = true;
+        releaseResources();
+    }
+
+    private void releaseResources()
+    {
+        if (currentBuf != null)
+        {
+            if (currentBuf.refCnt() > 0)
+                currentBuf.release(currentBuf.refCnt());
+            currentBuf = null;
+            buffer = null;
+        }
+
+        ByteBuf buf;
+        while ((buf = queue.poll()) != null && buf.refCnt() > 0)
+            buf.release(buf.refCnt());
+    }
+
+    /**
+     * Mark this stream as closed, but do not release any of the resources.
+     *
+     * Note: this is best to be called from the producer thread.
+     */
+    public void markClose()
+    {
+        if (!closed)
+        {
+            closed = true;
+            queue.add(Unpooled.EMPTY_BUFFER);
+        }
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * Note: this is best to be called from the consumer thread.
+     */
+    @Override
+    public String toString()
+    {
+        return new StringBuilder(128).append("RebufferingByteBufDataInputPlus: currentBuf = ").append(currentBuf)
+                                  .append(" (super.buffer = ").append(buffer).append(')')
+                                  .append(", queuedByteCount = ").append(queuedByteCount)
+                                  .append(", queue buffers = ").append(queue)
+                                  .append(", closed = ").append(closed)
+                                  .toString();
+    }
+
+    public ByteBufAllocator getAllocator()
+    {
+        return channelConfig.getAllocator();
+    }
+}

src/java/org/apache/cassandra/security/SSLFactory.java

@@ -20,7 +20,6 @@ package org.apache.cassandra.security;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.InetAddress;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.security.KeyStore;
@@ -32,7 +31,6 @@ import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLParameters;
 import javax.net.ssl.SSLSocket;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
@@ -79,53 +77,6 @@ public final class SSLFactory
      */
     private static final AtomicReference<SslContext> serverSslContext = new AtomicReference<>();
 
-    /** Create a socket and connect */
-    public static SSLSocket getSocket(EncryptionOptions options, InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException
-    {
-        SSLContext ctx = createSSLContext(options, true);
-        SSLSocket socket = (SSLSocket) ctx.getSocketFactory().createSocket(address, port, localAddress, localPort);
-        try
-        {
-            prepareSocket(socket, options);
-            return socket;
-        }
-        catch (IllegalArgumentException e)
-        {
-            socket.close();
-            throw e;
-        }
-    }
-
-    /** Create a socket and connect, using any local address */
-    public static SSLSocket getSocket(EncryptionOptions options, InetAddress address, int port) throws IOException
-    {
-        SSLContext ctx = createSSLContext(options, true);
-        SSLSocket socket = (SSLSocket) ctx.getSocketFactory().createSocket(address, port);
-        try
-        {
-            prepareSocket(socket, options);
-            return socket;
-        }
-        catch (IllegalArgumentException e)
-        {
-            socket.close();
-            throw e;
-        }
-    }
-
-    /** Sets relevant socket options specified in encryption settings */
-    private static void prepareSocket(SSLSocket socket, EncryptionOptions options)
-    {
-        String[] suites = filterCipherSuites(socket.getSupportedCipherSuites(), options.cipher_suites);
-        if(options.require_endpoint_verification)
-        {
-            SSLParameters sslParameters = socket.getSSLParameters();
-            sslParameters.setEndpointIdentificationAlgorithm("HTTPS");
-            socket.setSSLParameters(sslParameters);
-        }
-        socket.setEnabledCipherSuites(suites);
-    }
-
     /**
      * Create a JSSE {@link SSLContext}.
      */

src/java/org/apache/cassandra/service/StorageService.java

@@ -1313,17 +1313,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         return DatabaseDescriptor.getTruncateRpcTimeout();
     }
 
-    public void setStreamingSocketTimeout(int value)
-    {
-        DatabaseDescriptor.setStreamingSocketTimeout(value);
-        logger.info("set streaming socket timeout to {} ms", value);
-    }
-
-    public int getStreamingSocketTimeout()
-    {
-        return DatabaseDescriptor.getStreamingSocketTimeout();
-    }
-
     public void setStreamThroughputMbPerSec(int value)
     {
         DatabaseDescriptor.setStreamThroughputOutboundMegabitsPerSec(value);

src/java/org/apache/cassandra/service/StorageServiceMBean.java

@@ -502,9 +502,6 @@ public interface StorageServiceMBean extends NotificationEmitter
     public void setTruncateRpcTimeout(long value);
     public long getTruncateRpcTimeout();
 
-    public void setStreamingSocketTimeout(int value);
-    public int getStreamingSocketTimeout();
-
     public void setStreamThroughputMbPerSec(int value);
     public int getStreamThroughputMbPerSec();
 

src/java/org/apache/cassandra/streaming/ConnectionHandler.java

@@ -1,428 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming;
-
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.net.Socket;
-import java.net.SocketException;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.concurrent.PriorityBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.util.concurrent.FastThreadLocalThread;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
-import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
-import org.apache.cassandra.net.IncomingStreamingConnection;
-import org.apache.cassandra.streaming.messages.StreamInitMessage;
-import org.apache.cassandra.streaming.messages.StreamMessage;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-
-/**
- * ConnectionHandler manages incoming/outgoing message exchange for the {@link StreamSession}.
- *
- * <p>
- * Internally, ConnectionHandler manages thread to receive incoming {@link StreamMessage} and thread to
- * send outgoing message. Messages are encoded/decoded on those thread and handed to
- * {@link StreamSession#messageReceived(org.apache.cassandra.streaming.messages.StreamMessage)}.
- */
-public class ConnectionHandler
-{
-    private static final Logger logger = LoggerFactory.getLogger(ConnectionHandler.class);
-
-    private final StreamSession session;
-
-    private IncomingMessageHandler incoming;
-    private OutgoingMessageHandler outgoing;
-    private final boolean isPreview;
-
-    ConnectionHandler(StreamSession session, int incomingSocketTimeout, boolean isPreview)
-    {
-        this.session = session;
-        this.isPreview = isPreview;
-        this.incoming = new IncomingMessageHandler(session, incomingSocketTimeout);
-        this.outgoing = new OutgoingMessageHandler(session);
-    }
-
-    /**
-     * Set up incoming message handler and initiate streaming.
-     *
-     * This method is called once on initiator.
-     *
-     * @throws IOException
-     */
-    @SuppressWarnings("resource")
-    public void initiate() throws IOException
-    {
-        logger.debug("[Stream #{}] Sending stream init for incoming stream", session.planId());
-        Socket incomingSocket = session.createConnection();
-        incoming.start(incomingSocket, StreamMessage.CURRENT_VERSION, true);
-
-        logger.debug("[Stream #{}] Sending stream init for outgoing stream", session.planId());
-        Socket outgoingSocket = session.createConnection();
-        outgoing.start(outgoingSocket, StreamMessage.CURRENT_VERSION, true);
-    }
-
-    /**
-     * Set up outgoing message handler on receiving side.
-     *
-     * @param connection Incoming connection to use for {@link OutgoingMessageHandler}.
-     * @param version Streaming message version
-     * @throws IOException
-     */
-    public void initiateOnReceivingSide(IncomingStreamingConnection connection, boolean isForOutgoing, int version) throws IOException
-    {
-        if (isForOutgoing)
-            outgoing.start(connection, version);
-        else
-            incoming.start(connection, version);
-    }
-
-    public ListenableFuture<?> close()
-    {
-        logger.debug("[Stream #{}] Closing stream connection handler on {}", session.planId(), session.peer);
-
-        ListenableFuture<?> inClosed = closeIncoming();
-        ListenableFuture<?> outClosed = closeOutgoing();
-
-        return Futures.allAsList(inClosed, outClosed);
-    }
-
-    public ListenableFuture<?> closeOutgoing()
-    {
-        return outgoing == null ? Futures.immediateFuture(null) : outgoing.close();
-    }
-
-    public ListenableFuture<?> closeIncoming()
-    {
-        return incoming == null ? Futures.immediateFuture(null) : incoming.close();
-    }
-
-    /**
-     * Enqueue messages to be sent.
-     *
-     * @param messages messages to send
-     */
-    public void sendMessages(Collection<? extends StreamMessage> messages)
-    {
-        for (StreamMessage message : messages)
-            sendMessage(message);
-    }
-
-    public void sendMessage(StreamMessage message)
-    {
-        if (outgoing.isClosed())
-            throw new RuntimeException("Outgoing stream handler has been closed");
-
-        if (message.type == StreamMessage.Type.FILE && isPreview)
-            throw new RuntimeException("Cannot send file messages for preview streaming sessions");
-
-        outgoing.enqueue(message);
-    }
-
-    /**
-     * @return true if outgoing connection is opened and ready to send messages
-     */
-    public boolean isOutgoingConnected()
-    {
-        return outgoing != null && !outgoing.isClosed();
-    }
-
-    abstract static class MessageHandler implements Runnable
-    {
-        protected final StreamSession session;
-
-        protected int protocolVersion;
-        private final boolean isOutgoingHandler;
-        protected Socket socket;
-
-        private final AtomicReference<SettableFuture<?>> closeFuture = new AtomicReference<>();
-        private IncomingStreamingConnection incomingConnection;
-
-        protected MessageHandler(StreamSession session, boolean isOutgoingHandler)
-        {
-            this.session = session;
-            this.isOutgoingHandler = isOutgoingHandler;
-        }
-
-        protected abstract String name();
-
-        @SuppressWarnings("resource")
-        protected static DataOutputStreamPlus getWriteChannel(Socket socket) throws IOException
-        {
-            WritableByteChannel out = socket.getChannel();
-            // socket channel is null when encrypted(SSL)
-            if (out == null)
-                return new WrappedDataOutputStreamPlus(new BufferedOutputStream(socket.getOutputStream()));
-            return new BufferedDataOutputStreamPlus(out);
-        }
-
-        protected static ReadableByteChannel getReadChannel(Socket socket) throws IOException
-        {
-            //we do this instead of socket.getChannel() so socketSoTimeout is respected
-            return Channels.newChannel(socket.getInputStream());
-        }
-
-        @SuppressWarnings("resource")
-        private void sendInitMessage() throws IOException
-        {
-            StreamInitMessage message = new StreamInitMessage(FBUtilities.getBroadcastAddress(),
-                                                              session.sessionIndex(),
-                                                              session.planId(),
-                                                              session.streamOperation(),
-                                                              !isOutgoingHandler,
-                                                              session.keepSSTableLevel(),
-                                                              session.getPendingRepair(),
-                                                              session.getPreviewKind());
-            ByteBuffer messageBuf = message.createMessage(false, protocolVersion);
-            DataOutputStreamPlus out = getWriteChannel(socket);
-            out.write(messageBuf);
-            out.flush();
-        }
-
-        public void start(IncomingStreamingConnection connection, int protocolVersion) throws IOException
-        {
-            this.incomingConnection = connection;
-            start(connection.socket, protocolVersion, false);
-        }
-
-        public void start(Socket socket, int protocolVersion, boolean initiator) throws IOException
-        {
-            this.socket = socket;
-            this.protocolVersion = protocolVersion;
-            if (initiator)
-                sendInitMessage();
-
-            new FastThreadLocalThread(this, name() + "-" + socket.getRemoteSocketAddress()).start();
-        }
-
-        public ListenableFuture<?> close()
-        {
-            // Assume it wasn't closed. Not a huge deal if we create a future on a race
-            SettableFuture<?> future = SettableFuture.create();
-            return closeFuture.compareAndSet(null, future)
-                 ? future
-                 : closeFuture.get();
-        }
-
-        public boolean isClosed()
-        {
-            return closeFuture.get() != null;
-        }
-
-        protected void signalCloseDone()
-        {
-            if (!isClosed())
-                close();
-
-            closeFuture.get().set(null);
-
-            // We can now close the socket
-            if (incomingConnection != null)
-            {
-                //this will close the underlying socket and remove it
-                //from active MessagingService connections (CASSANDRA-11854)
-                incomingConnection.close();
-            }
-            else
-            {
-                //this is an outgoing connection not registered in the MessagingService
-                //so we can close the socket directly
-                try
-                {
-                    socket.close();
-                }
-                catch (IOException e)
-                {
-                    // Erroring out while closing shouldn't happen but is not really a big deal, so just log
-                    // it at DEBUG and ignore otherwise.
-                    logger.debug("Unexpected error while closing streaming connection", e);
-                }
-            }
-        }
-    }
-
-    /**
-     * Incoming streaming message handler
-     */
-    static class IncomingMessageHandler extends MessageHandler
-    {
-        private final int socketTimeout;
-
-        IncomingMessageHandler(StreamSession session, int socketTimeout)
-        {
-            super(session, false);
-            this.socketTimeout = socketTimeout;
-        }
-
-        @Override
-        public void start(Socket socket, int version, boolean initiator) throws IOException
-        {
-            try
-            {
-                socket.setSoTimeout(socketTimeout);
-            }
-            catch (SocketException e)
-            {
-                logger.warn("Could not set incoming socket timeout to {}", socketTimeout, e);
-            }
-            super.start(socket, version, initiator);
-        }
-
-        protected String name()
-        {
-            return "STREAM-IN";
-        }
-
-        @SuppressWarnings("resource")
-        public void run()
-        {
-            try
-            {
-                ReadableByteChannel in = getReadChannel(socket);
-                while (!isClosed())
-                {
-                    // receive message
-                    StreamMessage message = StreamMessage.deserialize(in, protocolVersion, session);
-                    logger.debug("[Stream #{}] Received {}", session.planId(), message);
-                    // Might be null if there is an error during streaming (see FileMessage.deserialize). It's ok
-                    // to ignore here since we'll have asked for a retry.
-                    if (message != null)
-                    {
-                        session.messageReceived(message);
-                    }
-                }
-            }
-            catch (Throwable t)
-            {
-                JVMStabilityInspector.inspectThrowable(t);
-                session.onError(t);
-            }
-            finally
-            {
-                signalCloseDone();
-            }
-        }
-    }
-
-    /**
-     * Outgoing file transfer thread
-     */
-    static class OutgoingMessageHandler extends MessageHandler
-    {
-        /*
-         * All out going messages are queued up into messageQueue.
-         * The size will grow when received streaming request.
-         *
-         * Queue is also PriorityQueue so that prior messages can go out fast.
-         */
-        private final PriorityBlockingQueue<StreamMessage> messageQueue = new PriorityBlockingQueue<>(64, new Comparator<StreamMessage>()
-        {
-            public int compare(StreamMessage o1, StreamMessage o2)
-            {
-                return o2.getPriority() - o1.getPriority();
-            }
-        });
-
-        OutgoingMessageHandler(StreamSession session)
-        {
-            super(session, true);
-        }
-
-        protected String name()
-        {
-            return "STREAM-OUT";
-        }
-
-        public void enqueue(StreamMessage message)
-        {
-            messageQueue.put(message);
-        }
-
-        @SuppressWarnings("resource")
-        public void run()
-        {
-            try
-            {
-                DataOutputStreamPlus out = getWriteChannel(socket);
-
-                StreamMessage next;
-                while (!isClosed())
-                {
-                    if ((next = messageQueue.poll(1, TimeUnit.SECONDS)) != null)
-                    {
-                        logger.debug("[Stream #{}] Sending {}", session.planId(), next);
-                        sendMessage(out, next);
-                        if (next.type == StreamMessage.Type.SESSION_FAILED)
-                            close();
-                    }
-                }
-
-                // Sends the last messages on the queue
-                while ((next = messageQueue.poll()) != null)
-                    sendMessage(out, next);
-            }
-            catch (InterruptedException e)
-            {
-                throw new AssertionError(e);
-            }
-            catch (Throwable e)
-            {
-                session.onError(e);
-            }
-            finally
-            {
-                signalCloseDone();
-            }
-        }
-
-        private void sendMessage(DataOutputStreamPlus out, StreamMessage message)
-        {
-            try
-            {
-                StreamMessage.serialize(message, out, protocolVersion, session);
-                out.flush();
-                message.sent();
-            }
-            catch (SocketException e)
-            {
-                session.onError(e);
-                close();
-            }
-            catch (IOException e)
-            {
-                session.onError(e);
-            }
-        }
-    }
-}

src/java/org/apache/cassandra/streaming/DefaultConnectionFactory.java

@@ -15,83 +15,93 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.cassandra.streaming;
 
 import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
 
+import javax.annotation.Nullable;
+
+import com.google.common.util.concurrent.Uninterruptibles;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.config.Config;
+import io.netty.bootstrap.Bootstrap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.WriteBufferWaterMark;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.security.SSLFactory;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
+import org.apache.cassandra.net.async.NettyFactory;
+import org.apache.cassandra.net.async.OutboundConnectionIdentifier;
+import org.apache.cassandra.net.async.OutboundConnectionParams;
 
 public class DefaultConnectionFactory implements StreamConnectionFactory
 {
     private static final Logger logger = LoggerFactory.getLogger(DefaultConnectionFactory.class);
 
+    private static final int DEFAULT_CHANNEL_BUFFER_SIZE = 1 << 22;
+
+    private static final long MAX_WAIT_TIME_NANOS = TimeUnit.SECONDS.toNanos(30);
     private static final int MAX_CONNECT_ATTEMPTS = 3;
 
-    /**
-     * Connect to peer and start exchanging message.
-     * When connect attempt fails, this retries for maximum of MAX_CONNECT_ATTEMPTS times.
-     *
-     * @param peer the peer to connect to.
-     * @return the created socket.
-     *
-     * @throws IOException when connection failed.
-     */
-    public Socket createConnection(InetAddress peer) throws IOException
+    @Override
+    public Channel createConnection(OutboundConnectionIdentifier connectionId, int protocolVersion) throws IOException
     {
-        int attempts = 0;
+        ServerEncryptionOptions encryptionOptions = DatabaseDescriptor.getServerEncryptionOptions();
+
+        if (encryptionOptions.internode_encryption == ServerEncryptionOptions.InternodeEncryption.none)
+            encryptionOptions = null;
+
+        return createConnection(connectionId, protocolVersion, encryptionOptions);
+    }
+
+    protected Channel createConnection(OutboundConnectionIdentifier connectionId, int protocolVersion, @Nullable ServerEncryptionOptions encryptionOptions) throws IOException
+    {
+        // this is the amount of data to allow in memory before netty sets the channel writablility flag to false
+        int channelBufferSize = DEFAULT_CHANNEL_BUFFER_SIZE;
+        WriteBufferWaterMark waterMark = new WriteBufferWaterMark(channelBufferSize >> 2, channelBufferSize);
+
+        int sendBufferSize = DatabaseDescriptor.getInternodeSendBufferSize() > 0
+                             ? DatabaseDescriptor.getInternodeSendBufferSize()
+                             : OutboundConnectionParams.DEFAULT_SEND_BUFFER_SIZE;
+
+        OutboundConnectionParams params = OutboundConnectionParams.builder()
+                                                                  .connectionId(connectionId)
+                                                                  .encryptionOptions(encryptionOptions)
+                                                                  .mode(NettyFactory.Mode.STREAMING)
+                                                                  .protocolVersion(protocolVersion)
+                                                                  .sendBufferSize(sendBufferSize)
+                                                                  .waterMark(waterMark)
+                                                                  .build();
+
+        Bootstrap bootstrap = NettyFactory.instance.createOutboundBootstrap(params);
+
+        int connectionAttemptCount = 0;
+        long now = System.nanoTime();
+        final long end = now + MAX_WAIT_TIME_NANOS;
+        final Channel channel;
         while (true)
         {
-            try
+            ChannelFuture channelFuture = bootstrap.connect();
+            channelFuture.awaitUninterruptibly(end - now, TimeUnit.MILLISECONDS);
+            if (channelFuture.isSuccess())
             {
-                Socket socket = newSocket(peer);
-                socket.setSoTimeout(DatabaseDescriptor.getStreamingSocketTimeout());
-                socket.setKeepAlive(true);
-                return socket;
+                channel = channelFuture.channel();
+                break;
             }
-            catch (IOException e)
-            {
-                if (++attempts >= MAX_CONNECT_ATTEMPTS)
-                    throw e;
 
-                long waitms = DatabaseDescriptor.getRpcTimeout() * (long)Math.pow(2, attempts);
-                logger.warn("Failed attempt {} to connect to {}. Retrying in {} ms. ({})", attempts, peer, waitms, e.getMessage());
-                try
-                {
-                    Thread.sleep(waitms);
-                }
-                catch (InterruptedException wtf)
-                {
-                    throw new IOException("interrupted", wtf);
-                }
-            }
-        }
-    }
+            connectionAttemptCount++;
+            now = System.nanoTime();
+            if (connectionAttemptCount == MAX_CONNECT_ATTEMPTS || end - now <= 0)
+                throw new IOException("failed to connect to " + connectionId + " for streaming data", channelFuture.cause());
 
-    // TODO this is deliberately copied from (the now former) OutboundTcpConnectionPool, for CASSANDRA-8457.
-    // to be replaced in CASSANDRA-12229 (make streaming use 8457)
-    public static Socket newSocket(InetAddress endpoint) throws IOException
-    {
-        // zero means 'bind on any available port.'
-        if (MessagingService.isEncryptedConnection(endpoint))
-        {
-            return SSLFactory.getSocket(DatabaseDescriptor.getServerEncryptionOptions(), endpoint, DatabaseDescriptor.getSSLStoragePort());
-        }
-        else
-        {
-            SocketChannel channel = SocketChannel.open();
-            channel.connect(new InetSocketAddress(endpoint, DatabaseDescriptor.getStoragePort()));
-            return channel.socket();
+            long waitms = DatabaseDescriptor.getRpcTimeout() * (long)Math.pow(2, connectionAttemptCount);
+            logger.warn("Failed attempt {} to connect to {}. Retrying in {} ms.", connectionAttemptCount, connectionId, waitms);
+            Uninterruptibles.sleepUninterruptibly(waitms, TimeUnit.MILLISECONDS);
         }
+
+        return channel;
     }
 }

src/java/org/apache/cassandra/streaming/StreamConnectionFactory.java

@@ -15,16 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.cassandra.streaming;
 
 import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
 
-/**
- * Interface that creates connection used by streaming.
- */
+import io.netty.channel.Channel;
+import org.apache.cassandra.net.async.OutboundConnectionIdentifier;
+
 public interface StreamConnectionFactory
 {
-    Socket createConnection(InetAddress peer) throws IOException;
+    Channel createConnection(OutboundConnectionIdentifier connectionId, int protocolVersion) throws IOException;
 }

src/java/org/apache/cassandra/streaming/StreamCoordinator.java

@@ -37,8 +37,10 @@ public class StreamCoordinator
 {
     private static final Logger logger = LoggerFactory.getLogger(StreamCoordinator.class);
 
-    // Executor strictly for establishing the initial connections. Once we're connected to the other end the rest of the
-    // streaming is handled directly by the ConnectionHandler's incoming and outgoing threads.
+    /**
+     * Executor strictly for establishing the initial connections. Once we're connected to the other end the rest of the
+     * streaming is handled directly by the {@link StreamingMessageSender}'s incoming and outgoing threads.
+     */
     private static final DebuggableThreadPoolExecutor streamExecutor = DebuggableThreadPoolExecutor.createWithFixedPoolSize("StreamConnectionEstablisher",
                                                                                                                             FBUtilities.getAvailableProcessors());
     private final boolean connectSequentially;
@@ -55,8 +57,8 @@ public class StreamCoordinator
                              boolean connectSequentially, UUID pendingRepair, PreviewKind previewKind)
     {
         this.connectionsPerHost = connectionsPerHost;
-        this.factory = factory;
         this.keepSSTableLevel = keepSSTableLevel;
+        this.factory = factory;
         this.connectSequentially = connectSequentially;
         this.pendingRepair = pendingRepair;
         this.previewKind = previewKind;
@@ -163,6 +165,11 @@ public class StreamCoordinator
         return getOrCreateHostData(peer).getOrCreateSessionById(peer, id, connecting);
     }
 
+    public StreamSession getSessionById(InetAddress peer, int id)
+    {
+        return getHostData(peer).getSessionById(id);
+    }
+
     public synchronized void updateProgress(ProgressInfo info)
     {
         getHostData(info.peer).updateProgress(info);
@@ -274,8 +281,8 @@ public class StreamCoordinator
 
     private class HostStreamingData
     {
-        private Map<Integer, StreamSession> streamSessions = new HashMap<>();
-        private Map<Integer, SessionInfo> sessionInfos = new HashMap<>();
+        private final Map<Integer, StreamSession> streamSessions = new HashMap<>();
+        private final Map<Integer, SessionInfo> sessionInfos = new HashMap<>();
 
         private int lastReturned = -1;
 
@@ -333,6 +340,11 @@ public class StreamCoordinator
             return session;
         }
 
+        public StreamSession getSessionById(int id)
+        {
+            return streamSessions.get(id);
+        }
+
         public void updateProgress(ProgressInfo info)
         {
             sessionInfos.get(info.sessionIndex).updateProgress(info);

src/java/org/apache/cassandra/streaming/StreamManager.java

@@ -21,7 +21,6 @@ import java.net.InetAddress;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-
 import javax.management.ListenerNotFoundException;
 import javax.management.MBeanNotificationInfo;
 import javax.management.NotificationFilter;
@@ -136,7 +135,7 @@ public class StreamManager implements StreamManagerMBean
         initiatedStreams.put(result.planId, result);
     }
 
-    public void registerReceiving(final StreamResultFuture result)
+    public StreamResultFuture registerReceiving(final StreamResultFuture result)
     {
         result.addEventListener(notifier);
         // Make sure we remove the stream on completion (whether successful or not)
@@ -148,7 +147,8 @@ public class StreamManager implements StreamManagerMBean
             }
         }, MoreExecutors.directExecutor());
 
-        receivingStreams.put(result.planId, result);
+        StreamResultFuture previous = receivingStreams.putIfAbsent(result.planId, result);
+        return previous ==  null ? result : previous;
     }
 
     public StreamResultFuture getReceivingStream(UUID planId)
@@ -175,4 +175,22 @@ public class StreamManager implements StreamManagerMBean
     {
         return notifier.getNotificationInfo();
     }
+
+    public StreamSession findSession(InetAddress peer, UUID planId, int sessionIndex)
+    {
+        StreamSession session = findSession(initiatedStreams, peer, planId, sessionIndex);
+        if (session !=  null)
+            return session;
+
+        return findSession(receivingStreams, peer, planId, sessionIndex);
+    }
+
+    private StreamSession findSession(Map<UUID, StreamResultFuture> streams, InetAddress peer, UUID planId, int sessionIndex)
+    {
+        StreamResultFuture streamResultFuture = streams.get(planId);
+        if (streamResultFuture == null)
+            return null;
+
+        return streamResultFuture.getSession(peer, sessionIndex);
+    }
 }

src/java/org/apache/cassandra/streaming/StreamPlan.java

@@ -33,7 +33,7 @@ import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR
  */
 public class StreamPlan
 {
-    public static final String[] EMPTY_COLUMN_FAMILIES = new String[0];
+    private static final String[] EMPTY_COLUMN_FAMILIES = new String[0];
     private final UUID planId = UUIDGen.getTimeUUID();
     private final StreamOperation streamOperation;
     private final List<StreamEventHandler> handlers = new ArrayList<>();

src/java/org/apache/cassandra/streaming/StreamReader.java

@@ -18,8 +18,6 @@
 package org.apache.cassandra.streaming;
 
 import java.io.*;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
 import java.util.Collection;
 import java.util.UUID;
 
@@ -30,8 +28,7 @@ import com.google.common.collect.UnmodifiableIterator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.ning.compress.lzf.LZFInputStream;
-
+import org.apache.cassandra.io.util.TrackedDataInputPlus;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.db.*;
@@ -42,9 +39,10 @@ import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.streaming.compress.StreamCompressionInputStream;
 import org.apache.cassandra.streaming.messages.FileMessageHeader;
+import org.apache.cassandra.streaming.messages.StreamMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.io.util.TrackedInputStream;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 
@@ -88,12 +86,12 @@ public class StreamReader
     }
 
     /**
-     * @param channel where this reads data from
+     * @param inputPlus where this reads data from
      * @return SSTable transferred
      * @throws IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
      */
-    @SuppressWarnings("resource") // channel needs to remain open, streams on top of it can't be closed
-    public SSTableMultiWriter read(ReadableByteChannel channel) throws IOException
+    @SuppressWarnings("resource") // input needs to remain open, streams on top of it can't be closed
+    public SSTableMultiWriter read(DataInputPlus inputPlus) throws IOException
     {
         long totalSize = totalSize();
 
@@ -108,7 +106,8 @@ public class StreamReader
                      session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(),
                      cfs.getTableName(), pendingRepair);
 
-        TrackedInputStream in = new TrackedInputStream(new LZFInputStream(Channels.newInputStream(channel)));
+
+        TrackedDataInputPlus in = new TrackedDataInputPlus(new StreamCompressionInputStream(inputPlus, StreamMessage.CURRENT_VERSION));
         StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
         SSTableMultiWriter writer = null;
         try
@@ -179,10 +178,10 @@ public class StreamReader
         private Row staticRow;
         private IOException exception;
 
-        public StreamDeserializer(TableMetadata metadata, InputStream in, Version version, SerializationHeader header) throws IOException
+        public StreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header) throws IOException
         {
             this.metadata = metadata;
-            this.in = new DataInputPlus.DataInputStreamPlus(in);
+            this.in = in;
             this.helper = new SerializationHelper(metadata, version.correspondingMessagingVersion(), SerializationHelper.Flag.PRESERVE_SIZE);
             this.header = header;
         }
@@ -256,8 +255,8 @@ public class StreamReader
             // to what we do in hasNext)
             Unfiltered unfiltered = iterator.next();
             return metadata.isCounter() && unfiltered.kind() == Unfiltered.Kind.ROW
-                 ? maybeMarkLocalToBeCleared((Row) unfiltered)
-                 : unfiltered;
+                   ? maybeMarkLocalToBeCleared((Row) unfiltered)
+                   : unfiltered;
         }
 
         private Row maybeMarkLocalToBeCleared(Row row)

src/java/org/apache/cassandra/streaming/StreamReceiveException.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+public class StreamReceiveException extends RuntimeException
+{
+    public final StreamSession session;
+
+    public StreamReceiveException(StreamSession session, String msg)
+    {
+        super(msg);
+        this.session = session;
+    }
+
+    public StreamReceiveException(StreamSession session, Throwable t)
+    {
+        super(t);
+        this.session = session;
+    }
+}

src/java/org/apache/cassandra/streaming/StreamReceiveTask.java

@@ -104,6 +104,7 @@ public class StreamReceiveTask extends StreamTask
 
         remoteSSTablesReceived++;
         assert tableId.equals(sstable.getTableId());
+        logger.debug("recevied {} of {} total files", remoteSSTablesReceived, totalFiles);
 
         Collection<SSTableReader> finished = null;
         try

src/java/org/apache/cassandra/streaming/StreamResultFuture.java

@@ -17,8 +17,9 @@
  */
 package org.apache.cassandra.streaming;
 
-import java.io.IOException;
 import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
 import java.util.*;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
@@ -27,7 +28,7 @@ import com.google.common.util.concurrent.Futures;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.net.IncomingStreamingConnection;
+import io.netty.channel.Channel;
 import org.apache.cassandra.utils.FBUtilities;
 
 /**
@@ -103,12 +104,10 @@ public final class StreamResultFuture extends AbstractFuture<StreamState>
                                                                     UUID planId,
                                                                     StreamOperation streamOperation,
                                                                     InetAddress from,
-                                                                    IncomingStreamingConnection connection,
-                                                                    boolean isForOutgoing,
-                                                                    int version,
+                                                                    Channel channel,
                                                                     boolean keepSSTableLevel,
                                                                     UUID pendingRepair,
-                                                                    PreviewKind previewKind) throws IOException
+                                                                    PreviewKind previewKind)
     {
         StreamResultFuture future = StreamManager.instance.getReceivingStream(planId);
         if (future == null)
@@ -119,7 +118,7 @@ public final class StreamResultFuture extends AbstractFuture<StreamState>
             future = new StreamResultFuture(planId, streamOperation, keepSSTableLevel, pendingRepair, previewKind);
             StreamManager.instance.registerReceiving(future);
         }
-        future.attachConnection(from, sessionIndex, connection, isForOutgoing, version);
+        future.attachConnection(from, sessionIndex, channel);
         logger.info("[Stream #{}, ID#{}] Received streaming plan for {}", planId, sessionIndex, streamOperation.getDescription());
         return future;
     }
@@ -131,11 +130,18 @@ public final class StreamResultFuture extends AbstractFuture<StreamState>
         return future;
     }
 
-    private void attachConnection(InetAddress from, int sessionIndex, IncomingStreamingConnection connection, boolean isForOutgoing, int version) throws IOException
+    public StreamCoordinator getCoordinator()
     {
-        StreamSession session = coordinator.getOrCreateSessionById(from, sessionIndex, connection.socket.getInetAddress());
+        return coordinator;
+    }
+
+    private void attachConnection(InetAddress from, int sessionIndex, Channel channel)
+    {
+        SocketAddress addr = channel.remoteAddress();
+        InetAddress connecting = (addr instanceof InetSocketAddress ? ((InetSocketAddress) addr).getAddress() : from);
+        StreamSession session = coordinator.getOrCreateSessionById(from, sessionIndex, connecting);
         session.init(this);
-        session.handler.initiateOnReceivingSide(connection, isForOutgoing, version);
+        session.attach(channel);
     }
 
     public void addEventListener(StreamEventHandler listener)
@@ -206,6 +212,7 @@ public final class StreamResultFuture extends AbstractFuture<StreamState>
 
     private synchronized void maybeComplete()
     {
+        logger.warn("[Stream #{}] maybeComplete", planId);
         if (!coordinator.hasActiveSessions())
         {
             StreamState finalState = getCurrentState();
@@ -221,4 +228,9 @@ public final class StreamResultFuture extends AbstractFuture<StreamState>
             }
         }
     }
+
+    StreamSession getSession(InetAddress peer, int sessionIndex)
+    {
+        return coordinator.getSessionById(peer, sessionIndex);
+    }
 }

src/java/org/apache/cassandra/streaming/StreamSession.java

@@ -17,9 +17,8 @@
  */
 package org.apache.cassandra.streaming;
 
-import java.io.IOException;
 import java.net.InetAddress;
-import java.net.Socket;
+import java.net.InetSocketAddress;
 import java.net.SocketTimeoutException;
 import java.util.*;
 import java.util.concurrent.*;
@@ -29,17 +28,21 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
 import com.google.common.collect.*;
+import com.google.common.util.concurrent.Futures;
 
-import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.config.DatabaseDescriptor;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelId;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.PartitionPosition;
@@ -47,10 +50,12 @@ import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.gms.*;
 import org.apache.cassandra.metrics.StreamingMetrics;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.async.OutboundConnectionIdentifier;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.async.NettyStreamingMessageSender;
 import org.apache.cassandra.streaming.messages.*;
-import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;
@@ -59,79 +64,80 @@ import org.apache.cassandra.utils.concurrent.Refs;
 
 /**
  * Handles the streaming a one or more section of one of more sstables to and from a specific
- * remote node.
+ * remote node. The sending side performs a block-level transfer of the source sstable, while the receiver
+ * must deserilaize that data stream into an partitions and rows, and then write that out as an sstable.
  *
- * Both this node and the remote one will create a similar symmetrical StreamSession. A streaming
+ * Both this node and the remote one will create a similar symmetrical {@link StreamSession}. A streaming
  * session has the following life-cycle:
  *
- * 1. Connections Initialization
+ * 1. Session Initialization
  *
- *   (a) A node (the initiator in the following) create a new StreamSession, initialize it (init())
- *       and then start it (start()). Start will create a {@link ConnectionHandler} that will create
- *       two connections to the remote node (the follower in the following) with whom to stream and send
- *       a StreamInit message. The first connection will be the incoming connection for the
- *       initiator, and the second connection will be the outgoing.
- *   (b) Upon reception of that StreamInit message, the follower creates its own StreamSession,
- *       initialize it if it still does not exist, and attach connecting socket to its ConnectionHandler
- *       according to StreamInit message's isForOutgoing flag.
- *   (d) When the both incoming and outgoing connections are established, StreamSession calls
- *       StreamSession#onInitializationComplete method to start the streaming prepare phase
- *       (StreamResultFuture.startStreaming()).
+ *   (a) A node (the initiator in the following) create a new {@link StreamSession},
+ *       initialize it {@link #init(StreamResultFuture)}, and then start it ({@link #start()}).
+ *       Starting a session causes a {@link StreamInitMessage} to be sent.
+ *   (b) Upon reception of that {@link StreamInitMessage}, the follower creates its own {@link StreamSession},
+ *       and initializes it if it still does not exist.
+ *   (c) After the initiator sends the {@link StreamInitMessage}, it invokes
+ *       {@link StreamSession#onInitializationComplete()} to start the streaming prepare phase.
  *
  * 2. Streaming preparation phase
  *
- *   (a) This phase is started when the initiator onInitializationComplete() method is called. This method sends a
- *       PrepareMessage that includes what files/sections this node will stream to the follower
- *       (stored in a StreamTransferTask, each column family has it's own transfer task) and what
- *       the follower needs to stream back (StreamReceiveTask, same as above). If the initiator has
- *       nothing to receive from the follower, it goes directly to its Streaming phase. Otherwise,
- *       it waits for the follower PrepareMessage.
- *   (b) Upon reception of the PrepareMessage, the follower records which files/sections it will receive
- *       and send back its own PrepareMessage with a summary of the files/sections that will be sent to
- *       the initiator (prepare()). After having sent that message, the follower goes to its Streamning
- *       phase.
- *   (c) When the initiator receives the follower PrepareMessage, it records which files/sections it will
- *       receive and then goes to his own Streaming phase.
+ *   (a) A {@link PrepareSynMessage} is sent that includes a) what files/sections this node will stream to the follower
+ *       (stored locally in a {@link StreamTransferTask}, one for each table) and b) what the follower needs to
+ *       stream back (stored locally in a {@link StreamReceiveTask}, one for each table).
+ *   (b) Upon reception of the {@link PrepareSynMessage}, the follower records which files/sections it will receive
+ *       and send back a {@link PrepareSynAckMessage}, which contains a summary of the files/sections that will be sent to
+ *       the initiator.
+ *   (c) When the initiator receives the {@link PrepareSynAckMessage}, it records which files/sections it will
+ *       receive, and then goes to it's Streaming phase (see next section). If the intiator is to receive files,
+ *       it sends a {@link PrepareAckMessage} to the follower to indicate that it can start streaming to the initiator.
+ *   (d) (Optional) If the follower receives a {@link PrepareAckMessage}, it enters it's Streaming phase.
  *
  * 3. Streaming phase
  *
- *   (a) The streaming phase is started by each node (the sender in the follower, but note that each side
- *       of the StreamSession may be sender for some of the files) involved by calling startStreamingFiles().
- *       This will sequentially send a FileMessage for each file of each SteamTransferTask. Each FileMessage
- *       consists of a FileMessageHeader that indicates which file is coming and then start streaming the
- *       content for that file (StreamWriter in FileMessage.serialize()). When a file is fully sent, the
- *       fileSent() method is called for that file. If all the files for a StreamTransferTask are sent
- *       (StreamTransferTask.complete()), the task is marked complete (taskCompleted()).
- *   (b) On the receiving side, a SSTable will be written for the incoming file (StreamReader in
- *       FileMessage.deserialize()) and once the FileMessage is fully received, the file will be marked as
- *       complete (received()). When all files for the StreamReceiveTask have been received, the sstables
- *       are added to the CFS (and 2ndary index are built, StreamReceiveTask.complete()) and the task
- *       is marked complete (taskCompleted())
+ *   (a) The streaming phase is started at each node by calling {@link StreamSession#startStreamingFiles(boolean)}.
+ *       This will send, sequentially on each outbound streaming connection (see {@link NettyStreamingMessageSender}),
+ *       an {@link OutgoingFileMessage} for each file in each of the {@link StreamTransferTask}.
+ *       Each {@link OutgoingFileMessage} consists of a {@link FileMessageHeader} that contains metadata about the file
+ *       being streamed, followed by the file content itself. Once all the files for a {@link StreamTransferTask} are sent,
+ *       the task is marked complete {@link StreamTransferTask#complete(int)}.
+ *   (b) On the receiving side, a SSTable will be written for the incoming file, and once the file is fully received,
+ *       the file will be marked as complete ({@link StreamReceiveTask#received(SSTableMultiWriter)}). When all files
+ *       for the {@link StreamReceiveTask} have been received, the sstables are added to the CFS (and 2ndary indexes/MV are built),
+ *        and the task is marked complete ({@link #taskCompleted(StreamReceiveTask)}).
  *   (b) If during the streaming of a particular file an error occurs on the receiving end of a stream
- *       (FileMessage.deserialize), the node will send a SessionFailedMessage to the sender and close the stream session.
- *   (c) When all transfer and receive tasks for a session are complete, the move to the Completion phase
- *       (maybeCompleted()).
+ *       (it may be either the initiator or the follower), the node will send a {@link SessionFailedMessage}
+ *       to the sender and close the stream session.
+ *   (c) When all transfer and receive tasks for a session are complete, the session moves to the Completion phase
+ *       ({@link #maybeCompleted()}).
  *
  * 4. Completion phase
  *
- *   (a) When a node has finished all transfer and receive task, it enter the completion phase (maybeCompleted()).
- *       If it had already received a CompleteMessage from the other side (it is in the WAIT_COMPLETE state), that
- *       session is done is is closed (closeSession()). Otherwise, the node switch to the WAIT_COMPLETE state and
- *       send a CompleteMessage to the other side.
+ *   (a) When a node enters the completion phase, it sends a {@link CompleteMessage} to the peer, and then enter the
+ *       {@link StreamSession.State#WAIT_COMPLETE} state. If it has already received a {@link CompleteMessage}
+ *       from the peer, session is complete and is then closed ({@link #closeSession(State)}). Otherwise, the node
+ *       switch to the {@link StreamSession.State#WAIT_COMPLETE} state and send a {@link CompleteMessage} to the other side.
+ *
+ * In brief, the message passing looks like this (I for initiator, F for follwer):
+ * (session init)
+ * I: StreamInitMessage
+ * (session prepare)
+ * I: PrepareSynMessage
+ * F: PrepareSynAckMessage
+ * I: PrepareAckMessage
+ * (stream - this can happen in both directions)
+ * I: OutgoingFileMessage
+ * F: ReceivedMessage
+ * (completion)
+ * I/F: CompleteMessage
+ *
+ * All messages which derive from {@link StreamMessage} are sent by the standard internode messaging
+ * (via {@link org.apache.cassandra.net.MessagingService}, while the actual files themselves are sent by a special
+ * "streaming" connection type. See {@link NettyStreamingMessageSender} for details. Because of the asynchronous
  */
 public class StreamSession implements IEndpointStateChangeSubscriber
 {
-
-    /**
-     * Version where keep-alive support was added
-     */
-    private static final CassandraVersion STREAM_KEEP_ALIVE_VERSION = new CassandraVersion("3.10");
     private static final Logger logger = LoggerFactory.getLogger(StreamSession.class);
-    private static final DebuggableScheduledThreadPoolExecutor keepAliveExecutor = new DebuggableScheduledThreadPoolExecutor("StreamKeepAliveExecutor");
-    static {
-        // Immediately remove keep-alive task when cancelled.
-        keepAliveExecutor.setRemoveOnCancelPolicy(true);
-    }
 
     /**
      * Streaming endpoint.
@@ -139,7 +145,9 @@ public class StreamSession implements IEndpointStateChangeSubscriber
      * Each {@code StreamSession} is identified by this InetAddress which is broadcast address of the node streaming.
      */
     public final InetAddress peer;
+
     private final int index;
+
     /** Actual connecting address. Can be the same as {@linkplain #peer}. */
     public final InetAddress connecting;
 
@@ -154,20 +162,18 @@ public class StreamSession implements IEndpointStateChangeSubscriber
     // data receivers, filled after receiving prepare message
     private final Map<TableId, StreamReceiveTask> receivers = new ConcurrentHashMap<>();
     private final StreamingMetrics metrics;
-    /* can be null when session is created in remote */
-    private final StreamConnectionFactory factory;
 
-    public final Map<String, Set<Range<Token>>> transferredRangesPerKeyspace = new HashMap<>();
+    final Map<String, Set<Range<Token>>> transferredRangesPerKeyspace = new HashMap<>();
 
-    public final ConnectionHandler handler;
+    private final NettyStreamingMessageSender messageSender;
+    private final ConcurrentMap<ChannelId, Channel> incomingChannels = new ConcurrentHashMap<>();
 
-    private AtomicBoolean isAborted = new AtomicBoolean(false);
+    private final AtomicBoolean isAborted = new AtomicBoolean(false);
     private final boolean keepSSTableLevel;
-    private ScheduledFuture<?> keepAliveFuture = null;
     private final UUID pendingRepair;
     private final PreviewKind previewKind;
 
-    public static enum State
+    public enum State
     {
         INITIALIZED,
         PREPARING,
@@ -184,17 +190,16 @@ public class StreamSession implements IEndpointStateChangeSubscriber
      * Create new streaming session with the peer.
      *  @param peer Address of streaming peer
      * @param connecting Actual connecting address
-     * @param factory is used for establishing connection
      */
     public StreamSession(InetAddress peer, InetAddress connecting, StreamConnectionFactory factory, int index, boolean keepSSTableLevel, UUID pendingRepair, PreviewKind previewKind)
     {
         this.peer = peer;
         this.connecting = connecting;
         this.index = index;
-        this.factory = factory;
-        this.handler = new ConnectionHandler(this, isKeepAliveSupported()?
-                                                   (int)TimeUnit.SECONDS.toMillis(2 * DatabaseDescriptor.getStreamingKeepAlivePeriod()) :
-                                                   DatabaseDescriptor.getStreamingSocketTimeout(), previewKind.isPreview());
+
+        OutboundConnectionIdentifier id = OutboundConnectionIdentifier.stream(new InetSocketAddress(FBUtilities.getBroadcastAddress(), 0),
+                                                                              new InetSocketAddress(connecting, MessagingService.portFor(connecting)));
+        this.messageSender = new NettyStreamingMessageSender(this, id, factory, StreamMessage.CURRENT_VERSION, previewKind.isPreview());
         this.metrics = StreamingMetrics.get(connecting);
         this.keepSSTableLevel = keepSSTableLevel;
         this.pendingRepair = pendingRepair;
@@ -242,12 +247,6 @@ public class StreamSession implements IEndpointStateChangeSubscriber
         return receivers.get(tableId).getTransaction();
     }
 
-    private boolean isKeepAliveSupported()
-    {
-        CassandraVersion peerVersion = Gossiper.instance.getReleaseVersion(peer);
-        return peerVersion != null && peerVersion.compareTo(STREAM_KEEP_ALIVE_VERSION) >= 0;
-    }
-
     /**
      * Bind this session to report to specific {@link StreamResultFuture} and
      * perform pre-streaming initialization.
@@ -258,13 +257,18 @@ public class StreamSession implements IEndpointStateChangeSubscriber
     {
         this.streamResult = streamResult;
         StreamHook.instance.reportStreamFuture(this, streamResult);
-
-        if (isKeepAliveSupported())
-            scheduleKeepAliveTask();
-        else
-            logger.debug("Peer {} does not support keep-alive.", peer);
     }
 
+    public boolean attach(Channel channel)
+    {
+        if (!messageSender.hasControlChannel())
+            messageSender.injectControlMessageChannel(channel);
+        return incomingChannels.putIfAbsent(channel.id(), channel) == null;
+    }
+
+    /**
+     * invoked by the node that begins the stream session (it may be sending files, receiving files, or both)
+     */
     public void start()
     {
         if (requests.isEmpty() && transfers.isEmpty())
@@ -279,7 +283,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
             logger.info("[Stream #{}] Starting streaming to {}{}", planId(),
                                                                    peer,
                                                                    peer.equals(connecting) ? "" : " through " + connecting);
-            handler.initiate();
+            messageSender.initialize();
             onInitializationComplete();
         }
         catch (Exception e)
@@ -289,12 +293,6 @@ public class StreamSession implements IEndpointStateChangeSubscriber
         }
     }
 
-    public Socket createConnection() throws IOException
-    {
-        assert factory != null;
-        return factory.createConnection(connecting);
-    }
-
     /**
      * Request data fetch task to this session.
      *
@@ -317,7 +315,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
      * @param columnFamilies Transfer ColumnFamilies
      * @param flushTables flush tables?
      */
-    public synchronized void addTransferRanges(String keyspace, Collection<Range<Token>> ranges, Collection<String> columnFamilies, boolean flushTables)
+    synchronized void addTransferRanges(String keyspace, Collection<Range<Token>> ranges, Collection<String> columnFamilies, boolean flushTables)
     {
         failIfFinished();
         Collection<ColumnFamilyStore> stores = getColumnFamilyStores(keyspace, columnFamilies);
@@ -428,7 +426,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
         }
     }
 
-    public synchronized void addTransferFiles(Collection<SSTableStreamingSections> sstableDetails)
+    synchronized void addTransferFiles(Collection<SSTableStreamingSections> sstableDetails)
     {
         failIfFinished();
         Iterator<SSTableStreamingSections> iter = sstableDetails.iterator();
@@ -472,31 +470,37 @@ public class StreamSession implements IEndpointStateChangeSubscriber
         }
     }
 
-    private synchronized void closeSession(State finalState)
+    private synchronized Future closeSession(State finalState)
     {
+        Future abortedTasksFuture = null;
         if (isAborted.compareAndSet(false, true))
         {
             state(finalState);
 
+            // ensure aborting the tasks do not happen on the network IO thread (read: netty event loop)
+            // as we don't want any blocking disk IO to stop the network thread
             if (finalState == State.FAILED)
-            {
-                for (StreamTask task : Iterables.concat(receivers.values(), transfers.values()))
-                    task.abort();
-            }
+                abortedTasksFuture = ScheduledExecutors.nonPeriodicTasks.submit(this::abortTasks);
 
-            if (keepAliveFuture != null)
-            {
-                logger.debug("[Stream #{}] Finishing keep-alive task.", planId());
-                keepAliveFuture.cancel(false);
-                keepAliveFuture = null;
-            }
-
-            // Note that we shouldn't block on this close because this method is called on the handler
-            // incoming thread (so we would deadlock).
-            handler.close();
+            incomingChannels.values().stream().map(channel -> channel.close());
+            messageSender.close();
 
             streamResult.handleSessionComplete(this);
         }
+        return abortedTasksFuture != null ? abortedTasksFuture : Futures.immediateFuture(null);
+    }
+
+    private void abortTasks()
+    {
+        try
+        {
+            receivers.values().forEach(StreamReceiveTask::abort);
+            transfers.values().forEach(StreamTransferTask::abort);
+        }
+        catch (Exception e)
+        {
+            logger.warn("failed to abort some streaming tasks", e);
+        }
     }
 
     /**
@@ -517,6 +521,11 @@ public class StreamSession implements IEndpointStateChangeSubscriber
         return state;
     }
 
+    public NettyStreamingMessageSender getMessageSender()
+    {
+        return messageSender;
+    }
+
     /**
      * Return if this session completed successfully.
      *
@@ -531,27 +540,37 @@ public class StreamSession implements IEndpointStateChangeSubscriber
     {
         switch (message.type)
         {
-            case PREPARE:
-                PrepareMessage msg = (PrepareMessage) message;
+            case STREAM_INIT:
+                // nop
+                break;
+            case PREPARE_SYN:
+                PrepareSynMessage msg = (PrepareSynMessage) message;
                 prepare(msg.requests, msg.summaries);
                 break;
-
+            case PREPARE_SYNACK:
+                prepareSynAck((PrepareSynAckMessage) message);
+                break;
+            case PREPARE_ACK:
+                prepareAck((PrepareAckMessage) message);
+                break;
             case FILE:
                 receive((IncomingFileMessage) message);
                 break;
-
             case RECEIVED:
                 ReceivedMessage received = (ReceivedMessage) message;
                 received(received.tableId, received.sequenceNumber);
                 break;
-
             case COMPLETE:
                 complete();
                 break;
-
+            case KEEP_ALIVE:
+                // NOP - we only send/receive the KEEP_ALIVE to force the TCP connection to remain open
+                break;
             case SESSION_FAILED:
                 sessionFailed();
                 break;
+            default:
+                throw new AssertionError("unhandled StreamMessage type: " + message.getClass().getName());
         }
     }
 
@@ -562,55 +581,43 @@ public class StreamSession implements IEndpointStateChangeSubscriber
     {
         // send prepare message
         state(State.PREPARING);
-        PrepareMessage prepare = new PrepareMessage();
+        PrepareSynMessage prepare = new PrepareSynMessage();
         prepare.requests.addAll(requests);
         for (StreamTransferTask task : transfers.values())
             prepare.summaries.add(task.getSummary());
-        handler.sendMessage(prepare);
-
-        // if we don't need to prepare for receiving stream, start sending files immediately
-        if (requests.isEmpty())
-            startStreamingFiles();
+        messageSender.sendMessage(prepare);
     }
 
-    /**l
+    /**
      * Call back for handling exception during streaming.
-     *
-     * @param e thrown exception
      */
-    public void onError(Throwable e)
+    public Future onError(Throwable e)
     {
         logError(e);
         // send session failure message
-        if (handler.isOutgoingConnected())
-            handler.sendMessage(new SessionFailedMessage());
+        if (messageSender.connected())
+            messageSender.sendMessage(new SessionFailedMessage());
         // fail session
-        closeSession(State.FAILED);
+        return closeSession(State.FAILED);
     }
 
     private void logError(Throwable e)
     {
         if (e instanceof SocketTimeoutException)
         {
-            if (isKeepAliveSupported())
-                logger.error("[Stream #{}] Did not receive response from peer {}{} for {} secs. Is peer down? " +
-                             "If not, maybe try increasing streaming_keep_alive_period_in_secs.", planId(),
-                             peer.getHostAddress(),
-                             peer.equals(connecting) ? "" : " through " + connecting.getHostAddress(),
-                             2 * DatabaseDescriptor.getStreamingKeepAlivePeriod(),
-                             e);
-            else
-                logger.error("[Stream #{}] Streaming socket timed out. This means the session peer stopped responding or " +
-                             "is still processing received data. If there is no sign of failure in the other end or a very " +
-                             "dense table is being transferred you may want to increase streaming_socket_timeout_in_ms " +
-                             "property. Current value is {}ms.", planId(), DatabaseDescriptor.getStreamingSocketTimeout(), e);
+            logger.error("[Stream #{}] Did not receive response from peer {}{} for {} secs. Is peer down? " +
+                         "If not, maybe try increasing streaming_keep_alive_period_in_secs.", planId(),
+                         peer.getHostAddress(),
+                         peer.equals(connecting) ? "" : " through " + connecting.getHostAddress(),
+                         2 * DatabaseDescriptor.getStreamingKeepAlivePeriod(),
+                         e);
         }
         else
         {
             logger.error("[Stream #{}] Streaming error occurred on session with peer {}{}", planId(),
-                                                                                            peer.getHostAddress(),
-                                                                                            peer.equals(connecting) ? "" : " through " + connecting.getHostAddress(),
-                                                                                            e);
+                         peer.getHostAddress(),
+                         peer.equals(connecting) ? "" : " through " + connecting.getHostAddress(),
+                         e);
         }
     }
 
@@ -621,29 +628,55 @@ public class StreamSession implements IEndpointStateChangeSubscriber
     {
         // prepare tasks
         state(State.PREPARING);
+        ScheduledExecutors.nonPeriodicTasks.execute(() -> prepareAsync(requests, summaries));
+    }
+
+    /**
+     * Finish preparing the session. This method is blocking (memtables are flushed in {@link #addTransferRanges}),
+     * so the logic should not execute on the main IO thread (read: netty event loop).
+     */
+    private void prepareAsync(Collection<StreamRequest> requests, Collection<StreamSummary> summaries)
+    {
+
         for (StreamRequest request : requests)
             addTransferRanges(request.keyspace, request.ranges, request.columnFamilies, true); // always flush on stream request
         for (StreamSummary summary : summaries)
             prepareReceiving(summary);
 
-        // send back prepare message if prepare message contains stream request
-        if (!requests.isEmpty())
-        {
-            PrepareMessage prepare = new PrepareMessage();
+        PrepareSynAckMessage prepareSynAck = new PrepareSynAckMessage();
+        if (!peer.equals(FBUtilities.getBroadcastAddress()))
             for (StreamTransferTask task : transfers.values())
-                prepare.summaries.add(task.getSummary());
-            handler.sendMessage(prepare);
+                prepareSynAck.summaries.add(task.getSummary());
+        messageSender.sendMessage(prepareSynAck);
+
+
+        streamResult.handleSessionPrepared(this);
+        maybeCompleted();
+    }
+
+    private void prepareSynAck(PrepareSynAckMessage msg)
+    {
+        if (!msg.summaries.isEmpty())
+        {
+            for (StreamSummary summary : msg.summaries)
+                prepareReceiving(summary);
+
+            // only send the (final) ACK if we are expecting the peer to send this node (the initiator) some files
+            messageSender.sendMessage(new PrepareAckMessage());
         }
 
         if (isPreview())
-        {
             completePreview();
-            return;
-        }
+        else
+            startStreamingFiles(true);
+    }
 
-        // if there are files to stream
-        if (!maybeCompleted())
-            startStreamingFiles();
+    private void prepareAck(PrepareAckMessage msg)
+    {
+        if (isPreview())
+            completePreview();
+        else
+            startStreamingFiles(true);
     }
 
     /**
@@ -665,7 +698,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
     }
 
     /**
-     * Call back after receiving FileMessageHeader.
+     * Call back after receiving a streamed file.
      *
      * @param message received file
      */
@@ -680,7 +713,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
         StreamingMetrics.totalIncomingBytes.inc(headerSize);
         metrics.incomingBytes.inc(headerSize);
         // send back file received message
-        handler.sendMessage(new ReceivedMessage(message.header.tableId, message.header.sequenceNumber));
+        messageSender.sendMessage(new ReceivedMessage(message.header.tableId, message.header.sequenceNumber));
         receivers.get(message.header.tableId).received(message.sstable);
     }
 
@@ -700,11 +733,12 @@ public class StreamSession implements IEndpointStateChangeSubscriber
      */
     public synchronized void complete()
     {
+        logger.debug("handling Complete message, state = {}, completeSent = {}", state, completeSent);
         if (state == State.WAIT_COMPLETE)
         {
             if (!completeSent)
             {
-                handler.sendMessage(new CompleteMessage());
+                messageSender.sendMessage(new CompleteMessage());
                 completeSent = true;
             }
             closeSession(State.COMPLETE);
@@ -712,17 +746,6 @@ public class StreamSession implements IEndpointStateChangeSubscriber
         else
         {
             state(State.WAIT_COMPLETE);
-            handler.closeIncoming();
-        }
-    }
-
-    private synchronized void scheduleKeepAliveTask()
-    {
-        if (keepAliveFuture == null)
-        {
-            int keepAlivePeriod = DatabaseDescriptor.getStreamingKeepAlivePeriod();
-            logger.debug("[Stream #{}] Scheduling keep-alive task with {}s period.", planId(), keepAlivePeriod);
-            keepAliveFuture = keepAliveExecutor.scheduleAtFixedRate(new KeepAliveTask(), 0, keepAlivePeriod, TimeUnit.SECONDS);
         }
     }
 
@@ -804,7 +827,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
             {
                 if (!completeSent)
                 {
-                    handler.sendMessage(new CompleteMessage());
+                    messageSender.sendMessage(new CompleteMessage());
                     completeSent = true;
                 }
                 closeSession(State.COMPLETE);
@@ -812,10 +835,9 @@ public class StreamSession implements IEndpointStateChangeSubscriber
             else
             {
                 // notify peer that this session is completed
-                handler.sendMessage(new CompleteMessage());
+                messageSender.sendMessage(new CompleteMessage());
                 completeSent = true;
                 state(State.WAIT_COMPLETE);
-                handler.closeOutgoing();
             }
         }
         return completed;
@@ -840,46 +862,30 @@ public class StreamSession implements IEndpointStateChangeSubscriber
             receivers.put(summary.tableId, new StreamReceiveTask(this, summary.tableId, summary.files, summary.totalSize));
     }
 
-    private void startStreamingFiles()
+    private void startStreamingFiles(boolean notifyPrepared)
     {
-        streamResult.handleSessionPrepared(this);
+        if (notifyPrepared)
+            streamResult.handleSessionPrepared(this);
 
         state(State.STREAMING);
 
         for (StreamTransferTask task : transfers.values())
         {
             Collection<OutgoingFileMessage> messages = task.getFileMessages();
-            if (messages.size() > 0)
-                handler.sendMessages(messages);
-            else
-                taskCompleted(task); // there is no file to send
-        }
-    }
-
-    class KeepAliveTask implements Runnable
-    {
-        private KeepAliveMessage last = null;
-
-        public void run()
-        {
-            //to avoid jamming the message queue, we only send if the last one was sent
-            if (last == null || last.wasSent())
+            if (!messages.isEmpty())
             {
-                logger.trace("[Stream #{}] Sending keep-alive to {}.", planId(), peer);
-                last = new KeepAliveMessage();
-                try
+                for (OutgoingFileMessage ofm : messages)
                 {
-                    handler.sendMessage(last);
-                }
-                catch (RuntimeException e) //connection handler is closed
-                {
-                    logger.debug("[Stream #{}] Could not send keep-alive message (perhaps stream session is finished?).", planId(), e);
+                    // pass the session planId/index to the OFM (which is only set at init(), after the transfers have already been created)
+                    ofm.header.addSessionInfo(this);
+                    messageSender.sendMessage(ofm);
                 }
             }
             else
             {
-                logger.trace("[Stream #{}] Skip sending keep-alive to {} (previous was not yet sent).", planId(), peer);
+                taskCompleted(task); // there are no files to send
             }
         }
+        maybeCompleted();
     }
 }

src/java/org/apache/cassandra/streaming/StreamTransferTask.java

@@ -20,11 +20,15 @@ package org.apache.cassandra.streaming;
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Throwables;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.concurrent.NamedThreadFactory;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.TableId;
@@ -37,6 +41,7 @@ import org.apache.cassandra.utils.concurrent.Ref;
  */
 public class StreamTransferTask extends StreamTask
 {
+    private static final Logger logger = LoggerFactory.getLogger(StreamTransferTask.class);
     private static final ScheduledExecutorService timeoutExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("StreamingTransferTaskTimeouts"));
 
     private final AtomicInteger sequenceNumber = new AtomicInteger(0);
@@ -56,10 +61,10 @@ public class StreamTransferTask extends StreamTask
     public synchronized void addTransferFile(Ref<SSTableReader> ref, long estimatedKeys, List<Pair<Long, Long>> sections)
     {
         assert ref.get() != null && tableId.equals(ref.get().metadata().id);
-        OutgoingFileMessage message = new OutgoingFileMessage(ref, sequenceNumber.getAndIncrement(), estimatedKeys, sections, session.keepSSTableLevel());
+        OutgoingFileMessage message = new OutgoingFileMessage(ref, session, sequenceNumber.getAndIncrement(), estimatedKeys, sections, session.keepSSTableLevel());
         message = StreamHook.instance.reportOutgoingFile(session, ref.get(), message);
         files.put(message.header.sequenceNumber, message);
-        totalSize += message.header.size();
+                totalSize += message.header.size();
     }
 
     /**
@@ -80,6 +85,7 @@ public class StreamTransferTask extends StreamTask
             if (file != null)
                 file.complete();
 
+            logger.debug("recevied sequenceNumber {}, remaining files {}", sequenceNumber, files.keySet());
             signalComplete = files.isEmpty();
         }
 

src/java/org/apache/cassandra/streaming/StreamWriter.java

@@ -19,21 +19,21 @@ package org.apache.cassandra.streaming;
 
 import java.io.File;
 import java.io.IOException;
-import java.io.OutputStream;
+import java.nio.ByteBuffer;
 import java.util.Collection;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.ning.compress.lzf.LZFOutputStream;
-
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.ChannelProxy;
 import org.apache.cassandra.io.util.DataIntegrityMetadata;
 import org.apache.cassandra.io.util.DataIntegrityMetadata.ChecksumValidator;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.streaming.StreamManager.StreamRateLimiter;
+import org.apache.cassandra.streaming.compress.ByteBufCompressionDataOutputStreamPlus;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 
@@ -51,11 +51,6 @@ public class StreamWriter
     protected final StreamRateLimiter limiter;
     protected final StreamSession session;
 
-    private OutputStream compressedOutput;
-
-    // allocate buffer to use for transfers only once
-    private byte[] transferBuffer;
-
     public StreamWriter(SSTableReader sstable, Collection<Pair<Long, Long>> sections, StreamSession session)
     {
         this.session = session;
@@ -78,45 +73,48 @@ public class StreamWriter
         logger.debug("[Stream #{}] Start streaming file {} to {}, repairedAt = {}, totalSize = {}", session.planId(),
                      sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize);
 
-        try(RandomAccessReader file = sstable.openDataReader();
+        try(ChannelProxy proxy = sstable.getDataChannel().sharedCopy();
             ChecksumValidator validator = new File(sstable.descriptor.filenameFor(Component.CRC)).exists()
                                           ? DataIntegrityMetadata.checksumValidator(sstable.descriptor)
-                                          : null;)
+                                          : null)
         {
-            transferBuffer = validator == null ? new byte[DEFAULT_CHUNK_SIZE] : new byte[validator.chunkSize];
+            int bufferSize = validator == null ? DEFAULT_CHUNK_SIZE: validator.chunkSize;
 
             // setting up data compression stream
-            compressedOutput = new LZFOutputStream(output);
             long progress = 0L;
 
-            // stream each of the required sections of the file
-            for (Pair<Long, Long> section : sections)
+            try (DataOutputStreamPlus compressedOutput = new ByteBufCompressionDataOutputStreamPlus(output, limiter))
             {
-                long start = validator == null ? section.left : validator.chunkStart(section.left);
-                int readOffset = (int) (section.left - start);
-                // seek to the beginning of the section
-                file.seek(start);
-                if (validator != null)
-                    validator.seek(start);
-
-                // length of the section to read
-                long length = section.right - start;
-                // tracks write progress
-                long bytesRead = 0;
-                while (bytesRead < length)
+                // stream each of the required sections of the file
+                for (Pair<Long, Long> section : sections)
                 {
-                    long lastBytesRead = write(file, validator, readOffset, length, bytesRead);
-                    bytesRead += lastBytesRead;
-                    progress += (lastBytesRead - readOffset);
-                    session.progress(sstable.descriptor.filenameFor(Component.DATA), ProgressInfo.Direction.OUT, progress, totalSize);
-                    readOffset = 0;
-                }
+                    long start = validator == null ? section.left : validator.chunkStart(section.left);
+                    // if the transfer does not start on the valididator's chunk boundary, this is the number of bytes to offset by
+                    int transferOffset = (int) (section.left - start);
+                    if (validator != null)
+                        validator.seek(start);
 
-                // make sure that current section is sent
-                compressedOutput.flush();
+                    // length of the section to read
+                    long length = section.right - start;
+                    // tracks write progress
+                    long bytesRead = 0;
+                    while (bytesRead < length)
+                    {
+                        int toTransfer = (int) Math.min(bufferSize, length - bytesRead);
+                        long lastBytesRead = write(proxy, validator, compressedOutput, start, transferOffset, toTransfer, bufferSize);
+                        start += lastBytesRead;
+                        bytesRead += lastBytesRead;
+                        progress += (lastBytesRead - transferOffset);
+                        session.progress(sstable.descriptor.filenameFor(Component.DATA), ProgressInfo.Direction.OUT, progress, totalSize);
+                        transferOffset = 0;
+                    }
+
+                    // make sure that current section is sent
+                    output.flush();
+                }
+                logger.debug("[Stream #{}] Finished streaming file {} to {}, bytesTransferred = {}, totalSize = {}",
+                             session.planId(), sstable.getFilename(), session.peer, FBUtilities.prettyPrintMemory(progress), FBUtilities.prettyPrintMemory(totalSize));
             }
-            logger.debug("[Stream #{}] Finished streaming file {} to {}, bytesTransferred = {}, totalSize = {}",
-                         session.planId(), sstable.getFilename(), session.peer, FBUtilities.prettyPrintMemory(progress), FBUtilities.prettyPrintMemory(totalSize));
         }
     }
 
@@ -131,27 +129,44 @@ public class StreamWriter
     /**
      * Sequentially read bytes from the file and write them to the output stream
      *
-     * @param reader The file reader to read from
+     * @param proxy The file reader to read from
      * @param validator validator to verify data integrity
-     * @param start number of bytes to skip transfer, but include for validation.
-     * @param length The full length that should be read from {@code reader}
-     * @param bytesTransferred Number of bytes already read out of {@code length}
+     * @param start The readd offset from the beginning of the {@code proxy} file.
+     * @param transferOffset number of bytes to skip transfer, but include for validation.
+     * @param toTransfer The number of bytes to be transferred.
      *
-     * @return Number of bytes read
+     * @return Number of bytes transferred.
      *
      * @throws java.io.IOException on any I/O error
      */
-    protected long write(RandomAccessReader reader, ChecksumValidator validator, int start, long length, long bytesTransferred) throws IOException
+    protected long write(ChannelProxy proxy, ChecksumValidator validator, DataOutputStreamPlus output, long start, int transferOffset, int toTransfer, int bufferSize) throws IOException
     {
-        int toTransfer = (int) Math.min(transferBuffer.length, length - bytesTransferred);
-        int minReadable = (int) Math.min(transferBuffer.length, reader.length() - reader.getFilePointer());
+        // the count of bytes to read off disk
+        int minReadable = (int) Math.min(bufferSize, proxy.size() - start);
 
-        reader.readFully(transferBuffer, 0, minReadable);
-        if (validator != null)
-            validator.validate(transferBuffer, 0, minReadable);
+        // this buffer will hold the data from disk. as it will be compressed on the fly by
+        // ByteBufCompressionDataOutputStreamPlus.write(ByteBuffer), we can release this buffer as soon as we can.
+        ByteBuffer buffer = ByteBuffer.allocateDirect(minReadable);
+        try
+        {
+            int readCount = proxy.read(buffer, start);
+            assert readCount == minReadable : String.format("could not read required number of bytes from file to be streamed: read %d bytes, wanted %d bytes", readCount, minReadable);
+            buffer.flip();
 
-        limiter.acquire(toTransfer - start);
-        compressedOutput.write(transferBuffer, start, (toTransfer - start));
+            if (validator != null)
+            {
+                validator.validate(buffer);
+                buffer.flip();
+            }
+
+            buffer.position(transferOffset);
+            buffer.limit(transferOffset + (toTransfer - transferOffset));
+            output.write(buffer);
+        }
+        finally
+        {
+            FileUtils.clean(buffer);
+        }
 
         return toTransfer;
     }

src/java/org/apache/cassandra/streaming/StreamingMessageSender.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming;
+
+import java.io.IOException;
+
+import org.apache.cassandra.streaming.messages.StreamMessage;
+
+public interface StreamingMessageSender
+{
+    void initialize() throws IOException;
+
+    void sendMessage(StreamMessage message) throws IOException;
+
+    boolean connected();
+
+    void close();
+}

src/java/org/apache/cassandra/streaming/async/NettyStreamingMessageSender.java

@@ -0,0 +1,508 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelPipeline;
+import io.netty.util.AttributeKey;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.DataOutputBufferFixed;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.net.async.ByteBufDataOutputStreamPlus;
+import org.apache.cassandra.net.async.NettyFactory;
+import org.apache.cassandra.net.async.OutboundConnectionIdentifier;
+import org.apache.cassandra.streaming.StreamConnectionFactory;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamingMessageSender;
+import org.apache.cassandra.streaming.messages.IncomingFileMessage;
+import org.apache.cassandra.streaming.messages.KeepAliveMessage;
+import org.apache.cassandra.streaming.messages.OutgoingFileMessage;
+import org.apache.cassandra.streaming.messages.StreamInitMessage;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Responsible for sending {@link StreamMessage}s to a given peer. We manage an array of netty {@link Channel}s
+ * for sending {@link OutgoingFileMessage} instances; all other {@link StreamMessage} types are sent via
+ * a special control channel. The reason for this is to treat those messages carefully and not let them get stuck
+ * behind a file transfer.
+ *
+ * One of the challenges when sending files is we might need to delay shipping the file if:
+ *
+ * - we've exceeded our network I/O use due to rate limiting (at the cassandra level)
+ * - the receiver isn't keeping up, which causes the local TCP socket buffer to not empty, which causes epoll writes to not
+ * move any bytes to the socket, which causes buffers to stick around in user-land (a/k/a cassandra) memory.
+ *
+ * When those conditions occur, it's easy enough to reschedule processing the file once the resources pick up
+ * (we acquire the permits from the rate limiter, or the socket drains). However, we need to ensure that
+ * no other messages are submitted to the same channel while the current file is still being processed.
+ */
+public class NettyStreamingMessageSender implements StreamingMessageSender
+{
+    private static final Logger logger = LoggerFactory.getLogger(NettyStreamingMessageSender.class);
+
+    private static final int DEFAULT_MAX_PARALLEL_TRANSFERS = FBUtilities.getAvailableProcessors();
+    private static final int MAX_PARALLEL_TRANSFERS = Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "streaming.session.parallelTransfers", Integer.toString(DEFAULT_MAX_PARALLEL_TRANSFERS)));
+
+    // a simple mechansim for allowing a degree of fairnes across multiple sessions
+    private static final Semaphore fileTransferSemaphore = new Semaphore(DEFAULT_MAX_PARALLEL_TRANSFERS, true);
+
+    private final StreamSession session;
+    private final boolean isPreview;
+    private final int protocolVersion;
+    private final OutboundConnectionIdentifier connectionId;
+    private final StreamConnectionFactory factory;
+
+    private volatile boolean closed;
+
+    /**
+     * A special {@link Channel} for sending non-file streaming messages, basically anything that isn't an
+     * {@link OutgoingFileMessage} (or an {@link IncomingFileMessage}, but a node doesn't send that, it's only received).
+     */
+    private Channel controlMessageChannel;
+
+    // note: this really doesn't need to be a LBQ, just something that's thread safe
+    private final Collection<ScheduledFuture<?>> channelKeepAlives = new LinkedBlockingQueue<>();
+
+    private final ThreadPoolExecutor fileTransferExecutor;
+
+    /**
+     * A {@link ThreadLocal} used by the threads in {@link #fileTransferExecutor} to stash references to constructed
+     * and connected {@link Channel}s.
+     */
+    private final ConcurrentMap<Thread, Channel> threadLocalChannel = new ConcurrentHashMap<>();
+
+    /**
+     * A netty channel attribute used to indicate if a channel is currently transferring a file. This is primarily used
+     * to indicate to the {@link KeepAliveTask} if it is safe to send a {@link KeepAliveMessage}, as sending the
+     * (application level) keep-alive in the middle of streaming a file would be bad news.
+     */
+    @VisibleForTesting
+    static final AttributeKey<Boolean> TRANSFERRING_FILE_ATTR = AttributeKey.valueOf("transferringFile");
+
+    public NettyStreamingMessageSender(StreamSession session, OutboundConnectionIdentifier connectionId, StreamConnectionFactory factory, int protocolVersion, boolean isPreview)
+    {
+        this.session = session;
+        this.protocolVersion = protocolVersion;
+        this.connectionId = connectionId;
+        this.factory = factory;
+        this.isPreview = isPreview;
+
+        String name = session.peer.toString().replace(':', '.');
+        fileTransferExecutor = new DebuggableThreadPoolExecutor(1, MAX_PARALLEL_TRANSFERS, 1L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(),
+                                                                new NamedThreadFactory("NettyStreaming-Outbound-" + name));
+        fileTransferExecutor.allowCoreThreadTimeOut(true);
+    }
+
+    @Override
+    public void initialize() throws IOException
+    {
+        StreamInitMessage message = new StreamInitMessage(FBUtilities.getBroadcastAddress(),
+                                                          session.sessionIndex(),
+                                                          session.planId(),
+                                                          session.streamOperation(),
+                                                          session.keepSSTableLevel(),
+                                                          session.getPendingRepair(),
+                                                          session.getPreviewKind());
+        sendMessage(message);
+    }
+
+    public boolean hasControlChannel()
+    {
+        return controlMessageChannel != null;
+    }
+
+    public void injectControlMessageChannel(Channel channel)
+    {
+        this.controlMessageChannel = channel;
+        channel.attr(TRANSFERRING_FILE_ATTR).set(Boolean.FALSE);
+        scheduleKeepAliveTask(channel);
+    }
+
+    private void setupControlMessageChannel() throws IOException
+    {
+        if (controlMessageChannel == null)
+        {
+            controlMessageChannel = createChannel();
+            scheduleKeepAliveTask(controlMessageChannel);
+        }
+    }
+
+    private void scheduleKeepAliveTask(Channel channel)
+    {
+        int keepAlivePeriod = DatabaseDescriptor.getStreamingKeepAlivePeriod();
+        logger.debug("{} Scheduling keep-alive task with {}s period.", createLogTag(session, channel), keepAlivePeriod);
+
+        KeepAliveTask task = new KeepAliveTask(channel, session);
+        ScheduledFuture<?> scheduledFuture = channel.eventLoop().scheduleAtFixedRate(task, 0, keepAlivePeriod, TimeUnit.SECONDS);
+        channelKeepAlives.add(scheduledFuture);
+        task.future = scheduledFuture;
+    }
+    
+    private Channel createChannel() throws IOException
+    {
+        Channel channel = factory.createConnection(connectionId, protocolVersion);
+        ChannelPipeline pipeline = channel.pipeline();
+        pipeline.addLast(NettyFactory.instance.streamingGroup, NettyFactory.INBOUND_STREAM_HANDLER_NAME, new StreamingInboundHandler(connectionId.remoteAddress(), protocolVersion, session));
+        channel.attr(TRANSFERRING_FILE_ATTR).set(Boolean.FALSE);
+        return channel;
+    }
+
+     static String createLogTag(StreamSession session, Channel channel)
+    {
+        StringBuilder sb = new StringBuilder(64);
+        sb.append("[Stream");
+
+        if (session != null)
+                sb.append(" #").append(session.planId());
+
+        if (channel != null)
+                sb.append(" channel: ").append(channel.id());
+
+        sb.append(']');
+        return sb.toString();
+    }
+
+    @Override
+    public void sendMessage(StreamMessage message)
+    {
+        if (closed)
+            throw new RuntimeException("stream has been closed, cannot send " + message);
+
+        if (message instanceof OutgoingFileMessage)
+        {
+            if (isPreview)
+                throw new RuntimeException("Cannot send file messages for preview streaming sessions");
+            logger.debug("{} Sending {}", createLogTag(session, null), message);
+            fileTransferExecutor.submit(new FileStreamTask((OutgoingFileMessage)message));
+            return;
+        }
+
+        try
+        {
+            setupControlMessageChannel();
+            sendControlMessage(controlMessageChannel, message, future -> onControlMessageComplete(future, message));
+        }
+        catch (Exception e)
+        {
+            close();
+            session.onError(e);
+        }
+    }
+
+    private void sendControlMessage(Channel channel, StreamMessage message, GenericFutureListener listener) throws IOException
+    {
+        logger.debug("{} Sending {}", createLogTag(session, channel), message);
+
+        // we anticipate that the control messages are rather small, so allocating a ByteBuf shouldn't  blow out of memory.
+        long messageSize = StreamMessage.serializedSize(message, protocolVersion);
+        if (messageSize > 1 << 30)
+        {
+            throw new IllegalStateException(String.format("%s something is seriously wrong with the calculated stream control message's size: %d bytes, type is %s",
+                                                          createLogTag(session, channel), messageSize, message.type));
+        }
+
+        // as control messages are (expected to be) small, we can simply allocate a ByteBuf here, wrap it, and send via the channel
+        ByteBuf buf = channel.alloc().directBuffer((int) messageSize, (int) messageSize);
+        ByteBuffer nioBuf = buf.nioBuffer(0, (int) messageSize);
+        @SuppressWarnings("resource")
+        DataOutputBufferFixed out = new DataOutputBufferFixed(nioBuf);
+        StreamMessage.serialize(message, out, protocolVersion, session);
+        assert nioBuf.position() == nioBuf.limit();
+        buf.writerIndex(nioBuf.position());
+
+        ChannelFuture channelFuture = channel.writeAndFlush(buf);
+        channelFuture.addListener(future -> listener.operationComplete(future));
+    }
+
+    /**
+     * Decides what to do after a {@link StreamMessage} is processed.
+     *
+     * Note: this is called from the netty event loop.
+     *
+     * @return null if the message was processed sucessfully; else, a {@link java.util.concurrent.Future} to indicate
+     * the status of aborting any remaining tasks in the session.
+     */
+    java.util.concurrent.Future onControlMessageComplete(Future<?> future, StreamMessage msg)
+    {
+        ChannelFuture channelFuture = (ChannelFuture)future;
+        Throwable cause = future.cause();
+        if (cause == null)
+            return null;
+
+        Channel channel = channelFuture.channel();
+        logger.error("{} failed to send a stream message/file to peer {}: msg = {}",
+                     createLogTag(session, channel), connectionId, msg, future.cause());
+
+        // StreamSession will invoke close(), but we have to mark this sender as closed so the session doesn't try
+        // to send any failure messages
+        return session.onError(cause);
+    }
+
+    class FileStreamTask implements Runnable
+    {
+        /**
+         * Time interval, in minutes, to wait between logging a message indicating that we're waiting on a semaphore
+         * permit to become available.
+         */
+        private static final int SEMAPHORE_UNAVAILABLE_LOG_INTERVAL = 3;
+
+        /**
+         * Even though we expect only an {@link OutgoingFileMessage} at runtime, the type here is {@link StreamMessage}
+         * to facilitate simpler testing.
+         */
+        private final StreamMessage msg;
+
+        FileStreamTask(OutgoingFileMessage ofm)
+        {
+            this.msg = ofm;
+        }
+
+        /**
+         * For testing purposes
+         */
+        FileStreamTask(StreamMessage msg)
+        {
+            this.msg = msg;
+        }
+
+        @Override
+        public void run()
+        {
+            if (!acquirePermit(SEMAPHORE_UNAVAILABLE_LOG_INTERVAL))
+                return;
+
+            try
+            {
+                Channel channel = getOrCreateChannel();
+                if (!channel.attr(TRANSFERRING_FILE_ATTR).compareAndSet(false, true))
+                    throw new IllegalStateException("channel's transferring state is currently set to true. refusing to start new stream");
+
+                // close the DataOutputStreamPlus as we're done with it - but don't close the channel
+                try (DataOutputStreamPlus outPlus = ByteBufDataOutputStreamPlus.create(session, channel, 1 << 16))
+                {
+                    StreamMessage.serialize(msg, outPlus, protocolVersion, session);
+                    channel.flush();
+                }
+                finally
+                {
+                    channel.attr(TRANSFERRING_FILE_ATTR).set(Boolean.FALSE);
+                }
+            }
+            catch (Exception e)
+            {
+                session.onError(e);
+            }
+            finally
+            {
+                fileTransferSemaphore.release();
+            }
+        }
+
+        boolean acquirePermit(int logInterval)
+        {
+            long logIntervalNanos = TimeUnit.MINUTES.toNanos(logInterval);
+            long timeOfLastLogging = System.nanoTime();
+            while (true)
+            {
+                if (closed)
+                    return false;
+                try
+                {
+                    if (fileTransferSemaphore.tryAcquire(1, TimeUnit.SECONDS))
+                        return true;
+
+                    // log a helpful message to operators in case they are wondering why a given session might not be making progress.
+                    long now = System.nanoTime();
+                    if (now - timeOfLastLogging > logIntervalNanos)
+                    {
+                        timeOfLastLogging = now;
+                        OutgoingFileMessage ofm = (OutgoingFileMessage)msg;
+                        logger.info("{} waiting to acquire a permit to begin streaming file {}. This message logs every {} minutes",
+                                    createLogTag(session, null), ofm.getFilename(), logInterval);
+                    }
+                }
+                catch (InterruptedException ie)
+                {
+                    //ignore
+                }
+            }
+        }
+
+        private Channel getOrCreateChannel()
+        {
+            Thread currentThread = Thread.currentThread();
+            try
+            {
+                Channel channel = threadLocalChannel.get(currentThread);
+                if (channel != null)
+                    return channel;
+
+                channel = createChannel();
+                threadLocalChannel.put(currentThread, channel);
+                return channel;
+            }
+            catch (Exception e)
+            {
+                throw new IOError(e);
+            }
+        }
+
+        /**
+         * For testing purposes
+         */
+        void injectChannel(Channel channel)
+        {
+            Thread currentThread = Thread.currentThread();
+            if (threadLocalChannel.get(currentThread) != null)
+                throw new IllegalStateException("previous channel already set");
+
+            threadLocalChannel.put(currentThread, channel);
+        }
+
+        /**
+         * For testing purposes
+         */
+        void unsetChannel()
+        {
+            threadLocalChannel.remove(Thread.currentThread());
+        }
+    }
+
+    /**
+     * Periodically sends the {@link KeepAliveMessage}.
+     *
+     * NOTE: this task, and the callback function {@link #keepAliveListener(Future)} is executed in the netty event loop.
+     */
+    class KeepAliveTask implements Runnable
+    {
+        private final Channel channel;
+        private final StreamSession session;
+
+        /**
+         * A reference to the scheduled task for this instance so that it may be cancelled.
+         */
+        ScheduledFuture<?> future;
+
+        KeepAliveTask(Channel channel, StreamSession session)
+        {
+            this.channel = channel;
+            this.session = session;
+        }
+
+        public void run()
+        {
+            // if the channel has been closed, cancel the scheduled task and return
+            if (!channel.isOpen() || closed)
+            {
+                future.cancel(false);
+                return;
+            }
+
+            // if the channel is currently processing streaming, skip this execution. As this task executes
+            // on the event loop, even if there is a race with a FileStreamTask which changes the channel attribute
+            // after we check it, the FileStreamTask cannot send out any bytes as this KeepAliveTask is executing
+            // on the event loop (and FileStreamTask publishes it's buffer to the channel, consumed after we're done here).
+            if (channel.attr(TRANSFERRING_FILE_ATTR).get())
+                return;
+
+            try
+            {
+                logger.trace("{} Sending keep-alive to {}.", createLogTag(session, channel), session.peer);
+                sendControlMessage(channel, new KeepAliveMessage(), this::keepAliveListener);
+            }
+            catch (IOException ioe)
+            {
+                future.cancel(false);
+            }
+        }
+
+        private void keepAliveListener(Future<? super Void> future)
+        {
+            if (future.isSuccess() || future.isCancelled())
+                return;
+
+            logger.debug("{} Could not send keep-alive message (perhaps stream session is finished?).",
+                         createLogTag(session, channel), future.cause());
+        }
+    }
+
+    /**
+     * For testing purposes only.
+     */
+    void setClosed()
+    {
+        closed = true;
+    }
+
+    void setControlMessageChannel(Channel channel)
+    {
+        controlMessageChannel = channel;
+    }
+
+    int semaphoreAvailablePermits()
+    {
+        return fileTransferSemaphore.availablePermits();
+    }
+
+    @Override
+    public boolean connected()
+    {
+        return !closed;
+    }
+
+    @Override
+    public void close()
+    {
+        closed = true;
+        logger.debug("{} Closing stream connection channels on {}", createLogTag(session, null), connectionId);
+        channelKeepAlives.stream().map(scheduledFuture -> scheduledFuture.cancel(false));
+        channelKeepAlives.clear();
+
+        threadLocalChannel.values().stream().map(channel -> channel.close());
+        threadLocalChannel.clear();
+        fileTransferExecutor.shutdownNow();
+
+        if (controlMessageChannel != null)
+            controlMessageChannel.close();
+    }
+}

src/java/org/apache/cassandra/streaming/async/StreamCompressionSerializer.java

@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4FastDecompressor;
+import org.apache.cassandra.io.util.DataInputPlus;
+
+/**
+ * A serialiazer for stream compressed files (see package-level documentation). Much like a typical compressed
+ * output stream, this class operates on buffers or chunks of the data at a a time. The format for each compressed
+ * chunk is as follows:
+ *
+ * - int - compressed payload length
+ * - int - uncompressed payload length
+ * - bytes - compressed payload
+ */
+public class StreamCompressionSerializer
+{
+    private final ByteBufAllocator allocator;
+
+    public StreamCompressionSerializer(ByteBufAllocator allocator)
+    {
+        this.allocator = allocator;
+    }
+
+    /**
+     * Length of heaer data, which includes compressed length, uncompressed length.
+     */
+    private static final int HEADER_LENGTH = 8;
+
+    /**
+     * @return A buffer with decompressed data.
+     */
+    public ByteBuf serialize(LZ4Compressor compressor, ByteBuffer in, int version)
+    {
+        final int uncompressedLength = in.remaining();
+        int maxLength = compressor.maxCompressedLength(uncompressedLength);
+        ByteBuf out = allocator.directBuffer(maxLength);
+        try
+        {
+            ByteBuffer compressedNioBuffer = out.nioBuffer(HEADER_LENGTH, maxLength - HEADER_LENGTH);
+            compressor.compress(in, compressedNioBuffer);
+            final int compressedLength = compressedNioBuffer.position();
+            out.setInt(0, compressedLength);
+            out.setInt(4, uncompressedLength);
+            out.writerIndex(HEADER_LENGTH + compressedLength);
+        }
+        catch (Exception e)
+        {
+            if (out != null)
+                out.release();
+        }
+        return out;
+    }
+
+    /**
+     * @return A buffer with decompressed data.
+     */
+    public ByteBuf deserialize(LZ4FastDecompressor decompressor, DataInputPlus in, int version) throws IOException
+    {
+        final int compressedLength = in.readInt();
+        final int uncompressedLength = in.readInt();
+
+        // there's no guarantee the next compressed block is contained within one buffer in the input,
+        // so hence we need a 'staging' buffer to get all the bytes into one contiguous buffer for the decompressor
+        ByteBuf compressed = null;
+        ByteBuf uncompressed = null;
+        try
+        {
+            final ByteBuffer compressedNioBuffer;
+
+            // ReadableByteChannel allows us to keep the bytes off-heap because we pass a ByteBuffer to RBC.read(BB),
+            // DataInputPlus.read() takes a byte array (thus, an on-heap array).
+            if (in instanceof ReadableByteChannel)
+            {
+                compressed = allocator.directBuffer(compressedLength);
+                compressedNioBuffer = compressed.nioBuffer(0, compressedLength);
+                int readLength = ((ReadableByteChannel) in).read(compressedNioBuffer);
+                assert readLength == compressedNioBuffer.position();
+                compressedNioBuffer.flip();
+            }
+            else
+            {
+                byte[] compressedBytes = new byte[compressedLength];
+                in.readFully(compressedBytes);
+                compressedNioBuffer = ByteBuffer.wrap(compressedBytes);
+            }
+
+            uncompressed = allocator.directBuffer(uncompressedLength);
+            ByteBuffer uncompressedNioBuffer = uncompressed.nioBuffer(0, uncompressedLength);
+            decompressor.decompress(compressedNioBuffer, uncompressedNioBuffer);
+            uncompressed.writerIndex(uncompressedLength);
+            return uncompressed;
+        }
+        catch (Exception e)
+        {
+            if (uncompressed != null)
+                uncompressed.release();
+
+            if (e instanceof IOException)
+                throw e;
+            throw new IOException(e);
+        }
+        finally
+        {
+            if (compressed != null)
+                compressed.release();
+        }
+    }
+}

src/java/org/apache/cassandra/streaming/async/StreamingInboundHandler.java

@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.util.ReferenceCountUtil;
+import io.netty.util.concurrent.FastThreadLocalThread;
+import org.apache.cassandra.net.async.RebufferingByteBufDataInputPlus;
+import org.apache.cassandra.streaming.StreamManager;
+import org.apache.cassandra.streaming.StreamReceiveException;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.messages.FileMessageHeader;
+import org.apache.cassandra.streaming.messages.IncomingFileMessage;
+import org.apache.cassandra.streaming.messages.KeepAliveMessage;
+import org.apache.cassandra.streaming.messages.StreamInitMessage;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
+import static org.apache.cassandra.streaming.async.NettyStreamingMessageSender.createLogTag;
+
+/**
+ * Handles the inbound side of streaming messages and sstable data. From the incoming data, we derserialize the message
+ * and potentially reify partitions and rows and write those out to new sstable files. Because deserialization is a blocking affair,
+ * we can't block the netty event loop. Thus we have a background thread perform all the blocking deserialization.
+ */
+public class StreamingInboundHandler extends ChannelInboundHandlerAdapter
+{
+    private static final Logger logger = LoggerFactory.getLogger(StreamingInboundHandler.class);
+    static final Function<SessionIdentifier, StreamSession> DEFAULT_SESSION_PROVIDER = sid -> StreamManager.instance.findSession(sid.from, sid.planId, sid.sessionIndex);
+
+    private static final int AUTO_READ_LOW_WATER_MARK = 1 << 15;
+    private static final int AUTO_READ_HIGH_WATER_MARK = 1 << 16;
+
+    private final InetSocketAddress remoteAddress;
+    private final int protocolVersion;
+
+    private final StreamSession session;
+
+    /**
+     * A collection of {@link ByteBuf}s that are yet to be processed. Incoming buffers are first dropped into this
+     * structure, and then consumed.
+     * <p>
+     * For thread safety, this structure's resources are released on the consuming thread
+     * (via {@link RebufferingByteBufDataInputPlus#close()},
+     * but the producing side calls {@link RebufferingByteBufDataInputPlus#markClose()} to notify the input that is should close.
+     */
+    private RebufferingByteBufDataInputPlus buffers;
+
+    private volatile boolean closed;
+
+    public StreamingInboundHandler(InetSocketAddress remoteAddress, int protocolVersion, @Nullable StreamSession session)
+    {
+        this.remoteAddress = remoteAddress;
+        this.protocolVersion = protocolVersion;
+        this.session = session;
+    }
+
+    @Override
+    @SuppressWarnings("resource")
+    public void handlerAdded(ChannelHandlerContext ctx)
+    {
+        buffers = new RebufferingByteBufDataInputPlus(AUTO_READ_LOW_WATER_MARK, AUTO_READ_HIGH_WATER_MARK, ctx.channel().config());
+        Thread blockingIOThread = new FastThreadLocalThread(new StreamDeserializingTask(DEFAULT_SESSION_PROVIDER, session, ctx.channel()),
+                                                            String.format("Stream-Deserializer-%s-%s", remoteAddress.toString(), ctx.channel().id()));
+        blockingIOThread.setDaemon(true);
+        blockingIOThread.start();
+    }
+
+    @Override
+    public void channelRead(ChannelHandlerContext ctx, Object message)
+    {
+        if (!closed && message instanceof ByteBuf)
+            buffers.append((ByteBuf) message);
+        else
+            ReferenceCountUtil.release(message);
+    }
+
+    @Override
+    public void channelInactive(ChannelHandlerContext ctx)
+    {
+        close();
+        ctx.fireChannelInactive();
+    }
+
+    void close()
+    {
+        closed = true;
+        buffers.markClose();
+    }
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
+    {
+        if (cause instanceof IOException)
+            logger.trace("connection problem while streaming", cause);
+        else
+            logger.warn("exception occurred while in processing streaming file", cause);
+        close();
+    }
+
+    /**
+     * For testing only!!
+     */
+    void setPendingBuffers(RebufferingByteBufDataInputPlus bufChannel)
+    {
+        this.buffers = bufChannel;
+    }
+
+    /**
+     * The task that performs the actual deserialization.
+     */
+    class StreamDeserializingTask implements Runnable
+    {
+        private final Function<SessionIdentifier, StreamSession> sessionProvider;
+        private final Channel channel;
+
+        @VisibleForTesting
+        StreamSession session;
+
+        StreamDeserializingTask(Function<SessionIdentifier, StreamSession> sessionProvider, StreamSession session, Channel channel)
+        {
+            this.sessionProvider = sessionProvider;
+            this.session = session;
+            this.channel = channel;
+        }
+
+        @Override
+        public void run()
+        {
+            try
+            {
+                while (true)
+                {
+                    // do a check of available bytes and possibly sleep some amount of time (then continue).
+                    // this way we can break out of run() sanely or we end up blocking indefintely in StreamMessage.deserialize()
+                    while (buffers.available() == 0)
+                    {
+                        if (closed)
+                            return;
+
+                        Uninterruptibles.sleepUninterruptibly(400, TimeUnit.MILLISECONDS);
+                    }
+
+                    StreamMessage message = StreamMessage.deserialize(buffers, protocolVersion, null);
+
+                    // keep-alives don't necessarily need to be tied to a session (they could be arrive before or after
+                    // wrt session lifecycle, due to races), just log that we received the message and carry on
+                    if (message instanceof KeepAliveMessage)
+                    {
+                        logger.debug("{} Received {}", createLogTag(session, channel), message);
+                        continue;
+                    }
+
+                    if (session == null)
+                        session = deriveSession(message);
+                    logger.debug("{} Received {}", createLogTag(session, channel), message);
+                    session.messageReceived(message);
+                }
+            }
+            catch (EOFException eof)
+            {
+                // ignore
+            }
+            catch (Throwable t)
+            {
+                JVMStabilityInspector.inspectThrowable(t);
+                if (session != null)
+                {
+                    session.onError(t);
+                }
+                else if (t instanceof StreamReceiveException)
+                {
+                    ((StreamReceiveException)t).session.onError(t);
+                }
+                else
+                {
+                    logger.error("{} stream operation from {} failed", createLogTag(session, channel), remoteAddress, t);
+                }
+            }
+            finally
+            {
+                channel.close();
+                closed = true;
+
+                if (buffers != null)
+                    buffers.close();
+            }
+        }
+
+        StreamSession deriveSession(StreamMessage message) throws IOException
+        {
+            StreamSession streamSession = null;
+            // StreamInitMessage starts a new channel, and IncomingFileMessage potentially, as well.
+            // IncomingFileMessage needs a session to be established a priori, though
+            if (message instanceof StreamInitMessage)
+            {
+                assert session == null : "initiator of stream session received a StreamInitMessage";
+                StreamInitMessage init = (StreamInitMessage) message;
+                StreamResultFuture.initReceivingSide(init.sessionIndex, init.planId, init.streamOperation, init.from, channel, init.keepSSTableLevel, init.pendingRepair, init.previewKind);
+                streamSession = sessionProvider.apply(new SessionIdentifier(init.from, init.planId, init.sessionIndex));
+            }
+            else if (message instanceof IncomingFileMessage)
+            {
+                // TODO: it'd be great to check if the session actually exists before slurping in the entire sstable,
+                // but that's a refactoring for another day
+                FileMessageHeader header = ((IncomingFileMessage) message).header;
+                streamSession = sessionProvider.apply(new SessionIdentifier(header.sender, header.planId, header.sessionIndex));
+            }
+
+            if (streamSession == null)
+                throw new IllegalStateException(createLogTag(null, channel) + " no session found for message " + message);
+
+            streamSession.attach(channel);
+            return streamSession;
+        }
+    }
+
+    /**
+     * A simple struct to wrap the data points required to lookup a {@link StreamSession}
+     */
+    static class SessionIdentifier
+    {
+        final InetAddress from;
+        final UUID planId;
+        final int sessionIndex;
+
+        SessionIdentifier(InetAddress from, UUID planId, int sessionIndex)
+        {
+            this.from = from;
+            this.planId = planId;
+            this.sessionIndex = sessionIndex;
+        }
+    }
+}

src/java/org/apache/cassandra/streaming/async/package-info.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <h1>Non-blocking streaming with netty</h1>
+ * This document describes the implementation details of streaming protocol. A listener for a streaming
+ * session listens on the same socket as internode messaging, and participates in the same handshake protocol
+ * That protocol is described in the package-level documentation for {@link org.apache.cassandra.net.async}, and
+ * thus not here.
+ *
+ * Streaming 2.0 was implemented as CASSANDRA-5286. Streaming 2.0 used (the equivalent of) a single thread and
+ * a single socket to transfer sstables sequentially to a peer (either as part of a repair, bootstrap, and so on).
+ * Part of the motivation for switching to netty and a non-blocking model as to enable file transfers to occur
+ * in parallel for a given session.
+ *
+ * Thus, a more detailed approach is required for stream session management.
+ *
+ * <h2>Session setup and management</h2>
+ *
+ * The full details of the session lifecycle are documented in {@link org.apache.cassandra.streaming.StreamSession}.
+ *
+ *
+ * <h2>File transfer</h2>
+ *
+ * When tranferring whole or subsections of an sstable, only the DATA component is shipped. To that end,
+ * there are three "modes" of an sstable transfer that need to be handled somewhat differently:
+ *
+ * 1) uncompressed sstable - data needs to be read into user space so it can be manipulated: checksum validation,
+ * apply stream compression (see next section), and/or TLS encryption.
+ *
+ * 2) compressed sstable, transferred with SSL/TLS - data needs to be read into user space as that is where the TLS encryption
+ * needs to happen. Netty does not allow the pretense of doing zero-copy transfers when TLS is in the pipeline;
+ * data must explicitly be pulled into user-space memory for TLS encryption to work.
+ *
+ * 3) compressed sstable, transferred without SSL/TLS - data can be streamed via zero-copy transfer as the data does not
+ * need to be manipulated (it can be sent "as-is").
+ *
+ * <h3>Compressing the data</h3>
+ * We always want to transfer as few bytes as possible of the wire when streaming a file. If the
+ * sstable is not already compressed via table compression options, we apply an on-the-fly stream compression
+ * to the data. The stream compression format is documented in
+ * {@link org.apache.cassandra.streaming.async.StreamCompressionSerializer}
+ *
+ * You may be wondering: why implement your own compression scheme? why not use netty's built-in compression codecs,
+ * like {@link io.netty.handler.codec.compression.Lz4FrameEncoder}? That makes complete sense if all the sstables
+ * to be streamed are non using sstable compression (and obviously you wouldn't use stream compression when the sstables
+ * are using sstable compression). The problem is when you have a mix of files, some using sstable compression
+ * and some not. You can either:
+ *
+ * - send the files of one type over one kind of socket, and the others over another socket
+ * - send them both over the same socket, but then auto-adjust per each file type.
+ *
+ * I've opted for the latter to keep socket/channel management simpler and cleaner.
+ */
+package org.apache.cassandra.streaming.async;
+

src/java/org/apache/cassandra/streaming/compress/ByteBufCompressionDataOutputStreamPlus.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.compress;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import io.netty.buffer.ByteBuf;
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Factory;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
+import org.apache.cassandra.net.async.ByteBufDataOutputStreamPlus;
+import org.apache.cassandra.streaming.StreamManager.StreamRateLimiter;
+import org.apache.cassandra.streaming.async.StreamCompressionSerializer;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+
+/**
+ * The intent of this class is to only be used in a very narrow use-case: on the stream compression path of streaming.
+ * This class should really only get calls to {@link #write(ByteBuffer)}, where the incoming buffer is compressed and sent
+ * downstream.
+ */
+public class ByteBufCompressionDataOutputStreamPlus extends WrappedDataOutputStreamPlus
+{
+    private final StreamRateLimiter limiter;
+    private final LZ4Compressor compressor;
+    private final StreamCompressionSerializer serializer;
+
+    public ByteBufCompressionDataOutputStreamPlus(DataOutputStreamPlus out, StreamRateLimiter limiter)
+    {
+        super(out);
+        assert out instanceof ByteBufDataOutputStreamPlus;
+        compressor = LZ4Factory.fastestInstance().fastCompressor();
+        serializer = new StreamCompressionSerializer(((ByteBufDataOutputStreamPlus)out).getAllocator());
+        this.limiter = limiter;
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * Compress the incoming buffer and send the result downstream. The buffer parameter will not be used nor passed
+     * to downstream components, and thus callers can safely free the buffer upon return.
+     */
+    @Override
+    public void write(ByteBuffer buffer) throws IOException
+    {
+        ByteBuf compressed = serializer.serialize(compressor, buffer, StreamMessage.CURRENT_VERSION);
+
+        // this is a blocking call - you have been warned
+        limiter.acquire(compressed.readableBytes());
+
+        ((ByteBufDataOutputStreamPlus)out).writeToChannel(compressed);
+    }
+
+    @Override
+    public void close()
+    {
+        // explicitly overriding close() to avoid closing the wrapped stream; it will be closed via other means
+    }
+}

src/java/org/apache/cassandra/streaming/compress/CompressedInputStream.java

@@ -19,7 +19,8 @@ package org.apache.cassandra.streaming.compress;
 
 import java.io.EOFException;
 import java.io.IOException;
-import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
 import java.util.Iterator;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
@@ -27,42 +28,45 @@ import java.util.concurrent.ThreadLocalRandom;
 import java.util.function.Supplier;
 
 import com.google.common.collect.Iterators;
-import com.google.common.primitives.Ints;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import io.netty.util.concurrent.FastThreadLocalThread;
 import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+import org.apache.cassandra.streaming.StreamReader.StreamDeserializer;
 import org.apache.cassandra.utils.ChecksumType;
 import org.apache.cassandra.utils.WrappedRunnable;
 
 /**
- * InputStream which reads data from underlining source with given {@link CompressionInfo}.
+ * InputStream which reads data from underlining source with given {@link CompressionInfo}. Uses {@link #buffer} as a buffer
+ * for uncompressed data (which is read by stream consumers - {@link StreamDeserializer} in this case).
  */
-public class CompressedInputStream extends InputStream
+public class CompressedInputStream extends RebufferingInputStream
 {
 
     private static final Logger logger = LoggerFactory.getLogger(CompressedInputStream.class);
 
     private final CompressionInfo info;
     // chunk buffer
-    private final BlockingQueue<byte[]> dataBuffer;
+    private final BlockingQueue<ByteBuffer> dataBuffer;
     private final Supplier<Double> crcCheckChanceSupplier;
 
-    // uncompressed bytes
-    private final byte[] buffer;
+    /**
+     * The base offset of the current {@link #buffer} from the beginning of the stream.
+     */
+    private long bufferOffset = 0;
 
-    // offset from the beginning of the buffer
-    protected long bufferOffset = 0;
-    // current position in stream
+    /**
+     * The current {@link CompressedStreamReader#sections} offset in the stream.
+     */
     private long current = 0;
-    // number of bytes in the buffer that are actually valid
-    protected int validBufferBytes = -1;
 
     private final ChecksumType checksumType;
 
-    // raw checksum bytes
-    private final byte[] checksumBytes = new byte[4];
+    private static final int CHECKSUM_LENGTH = 4;
 
     /**
      * Indicates there was a problem when reading from source stream.
@@ -71,9 +75,9 @@ public class CompressedInputStream extends InputStream
      * with the cause of the error when reading from source stream, so it is
      * thrown to the consumer on subsequent read operation.
      */
-    private static final byte[] POISON_PILL = new byte[0];
+    private static final ByteBuffer POISON_PILL = ByteBuffer.wrap(new byte[0]);
 
-    protected volatile IOException readException = null;
+    private volatile IOException readException = null;
 
     private long totalCompressedBytesRead;
 
@@ -81,11 +85,11 @@ public class CompressedInputStream extends InputStream
      * @param source Input source to read compressed data from
      * @param info Compression info
      */
-    public CompressedInputStream(InputStream source, CompressionInfo info, ChecksumType checksumType, Supplier<Double> crcCheckChanceSupplier)
+    public CompressedInputStream(DataInputPlus source, CompressionInfo info, ChecksumType checksumType, Supplier<Double> crcCheckChanceSupplier)
     {
+        super(ByteBuffer.allocateDirect(info.parameters.chunkLength()));
+        buffer.limit(buffer.position()); // force the buffer to appear "consumed" so that it triggers reBuffer on the first read
         this.info = info;
-        this.buffer = new byte[info.parameters.chunkLength()];
-        // buffer is limited to store up to 1024 chunks
         this.dataBuffer = new ArrayBlockingQueue<>(Math.min(info.chunks.length, 1024));
         this.crcCheckChanceSupplier = crcCheckChanceSupplier;
         this.checksumType = checksumType;
@@ -93,19 +97,50 @@ public class CompressedInputStream extends InputStream
         new FastThreadLocalThread(new Reader(source, info, dataBuffer)).start();
     }
 
-    private void decompressNextChunk() throws IOException
+    /**
+     * Invoked when crossing into the next stream boundary in {@link CompressedStreamReader#sections}.
+     */
+    public void position(long position) throws IOException
     {
         if (readException != null)
             throw readException;
 
+        assert position >= current : "stream can only read forward.";
+        current = position;
+
+        if (current > bufferOffset + buffer.limit())
+            reBuffer(false);
+
+        buffer.position((int)(current - bufferOffset));
+    }
+
+    protected void reBuffer() throws IOException
+    {
+        reBuffer(true);
+    }
+
+    private void reBuffer(boolean updateCurrent) throws IOException
+    {
+        if (readException != null)
+        {
+            FileUtils.clean(buffer);
+            buffer = null;
+            throw readException;
+        }
+
+        // increment the offset into the stream based on the current buffer's read count
+        if (updateCurrent)
+            current += buffer.position();
+
         try
         {
-            byte[] compressedWithCRC = dataBuffer.take();
+            ByteBuffer compressedWithCRC = dataBuffer.take();
             if (compressedWithCRC == POISON_PILL)
             {
                 assert readException != null;
                 throw readException;
             }
+
             decompress(compressedWithCRC);
         }
         catch (InterruptedException e)
@@ -114,74 +149,49 @@ public class CompressedInputStream extends InputStream
         }
     }
 
-    @Override
-    public int read() throws IOException
+    private void decompress(ByteBuffer compressed) throws IOException
     {
-        if (current >= bufferOffset + buffer.length || validBufferBytes == -1)
-            decompressNextChunk();
+        final int compressedChunkLength = info.parameters.chunkLength();
+        int length = compressed.remaining();
 
-        assert current >= bufferOffset && current < bufferOffset + validBufferBytes;
-
-        return ((int) buffer[(int) (current++ - bufferOffset)]) & 0xff;
-    }
-
-    @Override
-    public int read(byte[] b, int off, int len) throws IOException
-    {
-        long nextCurrent = current + len;
-
-        if (current >= bufferOffset + buffer.length || validBufferBytes == -1)
-            decompressNextChunk();
-
-        assert nextCurrent >= bufferOffset;
-
-        int read = 0;
-        while (read < len)
+        // uncompress if the buffer size is less than chunk size. else, if the buffer size is equal to the compressedChunkLength,
+        // we assume the buffer is not compressed. see CASSANDRA-10520
+        final boolean releaseCompressedBuffer;
+        if (length - CHECKSUM_LENGTH < compressedChunkLength)
         {
-            int nextLen = Math.min((len - read), (int)((bufferOffset + validBufferBytes) - current));
-
-            System.arraycopy(buffer, (int)(current - bufferOffset), b, off + read, nextLen);
-            read += nextLen;
-
-            current += nextLen;
-            if (read != len)
-                decompressNextChunk();
+            buffer.clear();
+            compressed.limit(length - CHECKSUM_LENGTH);
+            info.parameters.getSstableCompressor().uncompress(compressed, buffer);
+            buffer.flip();
+            releaseCompressedBuffer = true;
         }
-
-        return len;
-    }
-
-    public void position(long position)
-    {
-        assert position >= current : "stream can only read forward.";
-        current = position;
-    }
-
-    private void decompress(byte[] compressed) throws IOException
-    {
-        // uncompress
-        if (compressed.length - checksumBytes.length < info.parameters.maxCompressedLength())
-            validBufferBytes = info.parameters.getSstableCompressor().uncompress(compressed, 0, compressed.length - checksumBytes.length, buffer, 0);
         else
         {
-            validBufferBytes = compressed.length - checksumBytes.length;
-            System.arraycopy(compressed, 0, buffer, 0, validBufferBytes);
+            FileUtils.clean(buffer);
+            buffer = compressed;
+            buffer.limit(length - CHECKSUM_LENGTH);
+            releaseCompressedBuffer = false;
         }
-        totalCompressedBytesRead += compressed.length;
+        totalCompressedBytesRead += length;
 
         // validate crc randomly
         double crcCheckChance = this.crcCheckChanceSupplier.get();
         if (crcCheckChance > 0d && crcCheckChance > ThreadLocalRandom.current().nextDouble())
         {
-            int checksum = (int) checksumType.of(compressed, 0, compressed.length - checksumBytes.length);
+            ByteBuffer crcBuf = compressed.duplicate();
+            crcBuf.limit(length - CHECKSUM_LENGTH).position(0);
+            int checksum = (int) checksumType.of(crcBuf);
 
-            System.arraycopy(compressed, compressed.length - checksumBytes.length, checksumBytes, 0, checksumBytes.length);
-            if (Ints.fromByteArray(checksumBytes) != checksum)
+            crcBuf.limit(length);
+            if (crcBuf.getInt() != checksum)
                 throw new IOException("CRC unmatched");
         }
 
+        if (releaseCompressedBuffer)
+            FileUtils.clean(compressed);
+
         // buffer offset is always aligned
-        bufferOffset = current & ~(buffer.length - 1);
+        bufferOffset = current & ~(compressedChunkLength - 1);
     }
 
     public long getTotalCompressedBytesRead()
@@ -189,13 +199,26 @@ public class CompressedInputStream extends InputStream
         return totalCompressedBytesRead;
     }
 
+    /**
+     * Releases the resources specific to this instance, but not the {@link DataInputPlus} that is used by the {@link Reader}.
+     */
+    @Override
+    public void close()
+    {
+        if (buffer != null)
+        {
+            FileUtils.clean(buffer);
+            buffer = null;
+        }
+    }
+
     class Reader extends WrappedRunnable
     {
-        private final InputStream source;
+        private final DataInputPlus source;
         private final Iterator<CompressionMetadata.Chunk> chunks;
-        private final BlockingQueue<byte[]> dataBuffer;
+        private final BlockingQueue<ByteBuffer> dataBuffer;
 
-        Reader(InputStream source, CompressionInfo info, BlockingQueue<byte[]> dataBuffer)
+        Reader(DataInputPlus source, CompressionInfo info, BlockingQueue<ByteBuffer> dataBuffer)
         {
             this.source = source;
             this.chunks = Iterators.forArray(info.chunks);
@@ -204,36 +227,54 @@ public class CompressedInputStream extends InputStream
 
         protected void runMayThrow() throws Exception
         {
-            byte[] compressedWithCRC;
+            byte[] tmp = null;
             while (chunks.hasNext())
             {
                 CompressionMetadata.Chunk chunk = chunks.next();
 
                 int readLength = chunk.length + 4; // read with CRC
-                compressedWithCRC = new byte[readLength];
-
-                int bufferRead = 0;
-                while (bufferRead < readLength)
+                ByteBuffer compressedWithCRC = null;
+                try
                 {
-                    try
+                    final int r;
+                    if (source instanceof ReadableByteChannel)
                     {
-                        int r = source.read(compressedWithCRC, bufferRead, readLength - bufferRead);
-                        if (r < 0)
-                        {
-                            readException = new EOFException("No chunk available");
-                            dataBuffer.put(POISON_PILL);
-                            return; // throw exception where we consume dataBuffer
-                        }
-                        bufferRead += r;
+                        compressedWithCRC = ByteBuffer.allocateDirect(readLength);
+                        r = ((ReadableByteChannel)source).read(compressedWithCRC);
+                        compressedWithCRC.flip();
                     }
-                    catch (IOException e)
+                    else
                     {
-                        logger.warn("Error while reading compressed input stream.", e);
-                        readException = e;
+                        // read into an on-heap araay, then copy over to an off-heap buffer. at a minumum snappy requires
+                        // off-heap buffers for decompression, else we could have just wrapped the plain byte array in a ByteBuffer
+                        if (tmp == null || tmp.length < info.parameters.chunkLength() + CHECKSUM_LENGTH)
+                            tmp = new byte[info.parameters.chunkLength() + CHECKSUM_LENGTH];
+                        source.readFully(tmp, 0, readLength);
+                        compressedWithCRC = ByteBuffer.allocateDirect(readLength);
+                        compressedWithCRC.put(tmp, 0, readLength);
+                        compressedWithCRC.position(0);
+                        r = readLength;
+                    }
+
+                    if (r < 0)
+                    {
+                        FileUtils.clean(compressedWithCRC);
+                        readException = new EOFException("No chunk available");
                         dataBuffer.put(POISON_PILL);
                         return; // throw exception where we consume dataBuffer
                     }
                 }
+                catch (IOException e)
+                {
+                    if (!(e instanceof EOFException))
+                        logger.warn("Error while reading compressed input stream.", e);
+                    if (compressedWithCRC != null)
+                        FileUtils.clean(compressedWithCRC);
+
+                    readException = e;
+                    dataBuffer.put(POISON_PILL);
+                    return; // throw exception where we consume dataBuffer
+                }
                 dataBuffer.put(compressedWithCRC);
             }
         }

src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java

@@ -18,8 +18,6 @@
 package org.apache.cassandra.streaming.compress;
 
 import java.io.IOException;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
 
 import com.google.common.base.Throwables;
 import org.slf4j.Logger;
@@ -28,7 +26,8 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.io.compress.CompressionMetadata;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.util.TrackedInputStream;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.TrackedDataInputPlus;
 import org.apache.cassandra.streaming.ProgressInfo;
 import org.apache.cassandra.streaming.StreamReader;
 import org.apache.cassandra.streaming.StreamSession;
@@ -59,8 +58,8 @@ public class CompressedStreamReader extends StreamReader
      * @throws java.io.IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
      */
     @Override
-    @SuppressWarnings("resource") // channel needs to remain open, streams on top of it can't be closed
-    public SSTableMultiWriter read(ReadableByteChannel channel) throws IOException
+    @SuppressWarnings("resource") // input needs to remain open, streams on top of it can't be closed
+    public SSTableMultiWriter read(DataInputPlus inputPlus) throws IOException
     {
         long totalSize = totalSize();
 
@@ -76,9 +75,9 @@ public class CompressedStreamReader extends StreamReader
                      session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(), pendingRepair,
                      cfs.getTableName());
 
-        CompressedInputStream cis = new CompressedInputStream(Channels.newInputStream(channel), compressionInfo,
+        CompressedInputStream cis = new CompressedInputStream(inputPlus, compressionInfo,
                                                               ChecksumType.CRC32, cfs::getCrcCheckChance);
-        TrackedInputStream in = new TrackedInputStream(cis);
+        TrackedDataInputPlus in = new TrackedDataInputPlus(cis);
 
         StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
         SSTableMultiWriter writer = null;
@@ -120,6 +119,10 @@ public class CompressedStreamReader extends StreamReader
                 throw e;
             throw Throwables.propagate(e);
         }
+        finally
+        {
+            cis.close();
+        }
     }
 
     @Override

src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java

@@ -18,6 +18,7 @@
 package org.apache.cassandra.streaming.compress;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -30,6 +31,8 @@ import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.ChannelProxy;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.net.async.ByteBufDataOutputStreamPlus;
 import org.apache.cassandra.streaming.ProgressInfo;
 import org.apache.cassandra.streaming.StreamSession;
 import org.apache.cassandra.streaming.StreamWriter;
@@ -41,7 +44,7 @@ import org.apache.cassandra.utils.Pair;
  */
 public class CompressedStreamWriter extends StreamWriter
 {
-    public static final int CHUNK_SIZE = 10 * 1024 * 1024;
+    private static final int CHUNK_SIZE = 1 << 16;
 
     private static final Logger logger = LoggerFactory.getLogger(CompressedStreamWriter.class);
 
@@ -56,6 +59,8 @@ public class CompressedStreamWriter extends StreamWriter
     @Override
     public void write(DataOutputStreamPlus out) throws IOException
     {
+        assert out instanceof ByteBufDataOutputStreamPlus;
+        ByteBufDataOutputStreamPlus output = (ByteBufDataOutputStreamPlus)out;
         long totalSize = totalSize();
         logger.debug("[Stream #{}] Start streaming file {} to {}, repairedAt = {}, totalSize = {}", session.planId(),
                      sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize);
@@ -79,10 +84,24 @@ public class CompressedStreamWriter extends StreamWriter
                 long bytesTransferred = 0;
                 while (bytesTransferred < length)
                 {
-                    final long bytesTransferredFinal = bytesTransferred;
                     final int toTransfer = (int) Math.min(CHUNK_SIZE, length - bytesTransferred);
                     limiter.acquire(toTransfer);
-                    long lastWrite = out.applyToChannel((wbc) -> fc.transferTo(section.left + bytesTransferredFinal, toTransfer, wbc));
+
+                    ByteBuffer outBuffer = ByteBuffer.allocateDirect(toTransfer);
+                    long lastWrite;
+                    try
+                    {
+                        lastWrite = fc.read(outBuffer, section.left + bytesTransferred);
+                        assert lastWrite == toTransfer : String.format("could not read required number of bytes from file to be streamed: read %d bytes, wanted %d bytes", lastWrite, toTransfer);
+                        outBuffer.flip();
+                        output.writeToChannel(outBuffer);
+                    }
+                    catch (IOException e)
+                    {
+                        FileUtils.clean(outBuffer);
+                        throw e;
+                    }
+
                     bytesTransferred += lastWrite;
                     progress += lastWrite;
                     session.progress(sstable.descriptor.filenameFor(Component.DATA), ProgressInfo.Direction.OUT, progress, totalSize);

src/java/org/apache/cassandra/streaming/compress/StreamCompressionInputStream.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.compress;
+
+import java.io.IOException;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocator;
+import io.netty.buffer.Unpooled;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+import org.apache.cassandra.net.async.RebufferingByteBufDataInputPlus;
+import org.apache.cassandra.streaming.async.StreamCompressionSerializer;
+
+public class StreamCompressionInputStream extends RebufferingInputStream
+{
+    /**
+     * The stream which contains buffers of compressed data that came from the peer.
+     */
+    private final DataInputPlus dataInputPlus;
+
+    private final LZ4FastDecompressor decompressor;
+    private final int protocolVersion;
+    private final StreamCompressionSerializer deserializer;
+
+    /**
+     * The parent, or owning, buffer of the current buffer being read from ({@link super#buffer}).
+     */
+    private ByteBuf currentBuf;
+
+    public StreamCompressionInputStream(DataInputPlus dataInputPlus, int protocolVersion)
+    {
+        super(Unpooled.EMPTY_BUFFER.nioBuffer());
+        currentBuf = Unpooled.EMPTY_BUFFER;
+
+        this.dataInputPlus = dataInputPlus;
+        this.protocolVersion = protocolVersion;
+        this.decompressor = LZ4Factory.fastestInstance().fastDecompressor();
+
+        ByteBufAllocator allocator = dataInputPlus instanceof RebufferingByteBufDataInputPlus
+                                     ? ((RebufferingByteBufDataInputPlus)dataInputPlus).getAllocator()
+                                     : PooledByteBufAllocator.DEFAULT;
+        deserializer = new StreamCompressionSerializer(allocator);
+    }
+
+    @Override
+    public void reBuffer() throws IOException
+    {
+        currentBuf.release();
+        currentBuf = deserializer.deserialize(decompressor, dataInputPlus, protocolVersion);
+        buffer = currentBuf.nioBuffer(0, currentBuf.readableBytes());
+    }
+
+    @Override
+    public void close()
+    {
+        currentBuf.release();
+    }
+}

src/java/org/apache/cassandra/streaming/messages/CompleteMessage.java

@@ -17,8 +17,7 @@
  */
 package org.apache.cassandra.streaming.messages;
 
-import java.nio.channels.ReadableByteChannel;
-
+import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.streaming.StreamSession;
 
@@ -26,12 +25,17 @@ public class CompleteMessage extends StreamMessage
 {
     public static Serializer<CompleteMessage> serializer = new Serializer<CompleteMessage>()
     {
-        public CompleteMessage deserialize(ReadableByteChannel in, int version, StreamSession session)
+        public CompleteMessage deserialize(DataInputPlus in, int version, StreamSession session)
         {
             return new CompleteMessage();
         }
 
         public void serialize(CompleteMessage message, DataOutputStreamPlus out, int version, StreamSession session) {}
+
+        public long serializedSize(CompleteMessage message, int version)
+        {
+            return 0;
+        }
     };
 
     public CompleteMessage()

src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java

@@ -18,6 +18,7 @@
 package org.apache.cassandra.streaming.messages;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
@@ -29,7 +30,10 @@ import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.net.CompactEndpointSerializationHelper;
+import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.StreamSession;
 import org.apache.cassandra.streaming.compress.CompressionInfo;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.UUIDSerializer;
@@ -42,6 +46,8 @@ public class FileMessageHeader
     public static FileMessageHeaderSerializer serializer = new FileMessageHeaderSerializer();
 
     public final TableId tableId;
+    public UUID planId;
+    public int sessionIndex;
     public final int sequenceNumber;
     /** SSTable version */
     public final Version version;
@@ -61,11 +67,15 @@ public class FileMessageHeader
     public final UUID pendingRepair;
     public final int sstableLevel;
     public final SerializationHeader.Component header;
+    public final InetAddress sender;
 
     /* cached size value */
     private transient final long size;
 
-    public FileMessageHeader(TableId tableId,
+    private FileMessageHeader(TableId tableId,
+                             InetAddress sender,
+                             UUID planId,
+                             int sessionIndex,
                              int sequenceNumber,
                              Version version,
                              SSTableFormat.Type format,
@@ -78,6 +88,9 @@ public class FileMessageHeader
                              SerializationHeader.Component header)
     {
         this.tableId = tableId;
+        this.sender = sender;
+        this.planId = planId;
+        this.sessionIndex = sessionIndex;
         this.sequenceNumber = sequenceNumber;
         this.version = version;
         this.format = format;
@@ -93,6 +106,9 @@ public class FileMessageHeader
     }
 
     public FileMessageHeader(TableId tableId,
+                             InetAddress sender,
+                             UUID planId,
+                             int sessionIndex,
                              int sequenceNumber,
                              Version version,
                              SSTableFormat.Type format,
@@ -105,6 +121,9 @@ public class FileMessageHeader
                              SerializationHeader.Component header)
     {
         this.tableId = tableId;
+        this.sender = sender;
+        this.planId = planId;
+        this.sessionIndex = sessionIndex;
         this.sequenceNumber = sequenceNumber;
         this.version = version;
         this.format = format;
@@ -188,11 +207,20 @@ public class FileMessageHeader
         return result;
     }
 
+    public void addSessionInfo(StreamSession session)
+    {
+        planId = session.planId();
+        sessionIndex = session.sessionIndex();
+    }
+
     static class FileMessageHeaderSerializer
     {
         public CompressionInfo serialize(FileMessageHeader header, DataOutputPlus out, int version) throws IOException
         {
             header.tableId.serialize(out);
+            CompactEndpointSerializationHelper.serialize(header.sender, out);
+            UUIDSerializer.serializer.serialize(header.planId, out, version);
+            out.writeInt(header.sessionIndex);
             out.writeInt(header.sequenceNumber);
             out.writeUTF(header.version.toString());
             out.writeUTF(header.format.name);
@@ -224,6 +252,9 @@ public class FileMessageHeader
         public FileMessageHeader deserialize(DataInputPlus in, int version) throws IOException
         {
             TableId tableId = TableId.deserialize(in);
+            InetAddress sender = CompactEndpointSerializationHelper.deserialize(in);
+            UUID planId = UUIDSerializer.serializer.deserialize(in, MessagingService.current_version);
+            int sessionIndex = in.readInt();
             int sequenceNumber = in.readInt();
             Version sstableVersion = SSTableFormat.Type.current().info.getVersion(in.readUTF());
             SSTableFormat.Type format = SSTableFormat.Type.validate(in.readUTF());
@@ -239,12 +270,15 @@ public class FileMessageHeader
             int sstableLevel = in.readInt();
             SerializationHeader.Component header =  SerializationHeader.serializer.deserialize(sstableVersion, in);
 
-            return new FileMessageHeader(tableId, sequenceNumber, sstableVersion, format, estimatedKeys, sections, compressionInfo, repairedAt, pendingRepair, sstableLevel, header);
+            return new FileMessageHeader(tableId, sender, planId, sessionIndex, sequenceNumber, sstableVersion, format, estimatedKeys, sections, compressionInfo, repairedAt, pendingRepair, sstableLevel, header);
         }
 
         public long serializedSize(FileMessageHeader header, int version)
         {
             long size = header.tableId.serializedSize();
+            size += CompactEndpointSerializationHelper.serializedSize(header.sender);
+            size += UUIDSerializer.serializer.serializedSize(header.planId, version);
+            size += TypeSizes.sizeof(header.sessionIndex);
             size += TypeSizes.sizeof(header.sequenceNumber);
             size += TypeSizes.sizeof(header.version.toString());
             size += TypeSizes.sizeof(header.format.name);

src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java

@@ -18,15 +18,15 @@
 package org.apache.cassandra.streaming.messages;
 
 import java.io.IOException;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
 
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
 
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.streaming.StreamManager;
 import org.apache.cassandra.streaming.StreamReader;
+import org.apache.cassandra.streaming.StreamReceiveException;
 import org.apache.cassandra.streaming.StreamSession;
 import org.apache.cassandra.streaming.compress.CompressedStreamReader;
 import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -40,21 +40,27 @@ public class IncomingFileMessage extends StreamMessage
     public static Serializer<IncomingFileMessage> serializer = new Serializer<IncomingFileMessage>()
     {
         @SuppressWarnings("resource")
-        public IncomingFileMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
+        public IncomingFileMessage deserialize(DataInputPlus input, int version, StreamSession session) throws IOException
         {
-            DataInputPlus input = new DataInputStreamPlus(Channels.newInputStream(in));
             FileMessageHeader header = FileMessageHeader.serializer.deserialize(input, version);
+            session = StreamManager.instance.findSession(header.sender, header.planId, header.sessionIndex);
+            if (session == null)
+                throw new IllegalStateException(String.format("unknown stream session: %s - %d", header.planId, header.sessionIndex));
+            ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(header.tableId);
+            if (cfs == null)
+                throw new StreamReceiveException(session, "CF " + header.tableId + " was dropped during streaming");
+
             StreamReader reader = !header.isCompressed() ? new StreamReader(header, session)
-                    : new CompressedStreamReader(header, session);
+                                                         : new CompressedStreamReader(header, session);
 
             try
             {
-                return new IncomingFileMessage(reader.read(in), header);
+                return new IncomingFileMessage(reader.read(input), header);
             }
             catch (Throwable t)
             {
                 JVMStabilityInspector.inspectThrowable(t);
-                throw t;
+                throw new StreamReceiveException(session, t);
             }
         }
 
@@ -62,6 +68,11 @@ public class IncomingFileMessage extends StreamMessage
         {
             throw new UnsupportedOperationException("Not allowed to call serialize on an incoming file");
         }
+
+        public long serializedSize(IncomingFileMessage message, int version)
+        {
+            throw new UnsupportedOperationException("Not allowed to call serializedSize on an incoming file");
+        }
     };
 
     public FileMessageHeader header;
@@ -77,7 +88,8 @@ public class IncomingFileMessage extends StreamMessage
     @Override
     public String toString()
     {
-        return "File (" + header + ", file: " + sstable.getFilename() + ")";
+        String filename = sstable != null ? sstable.getFilename() : null;
+        return "File (" + header + ", file: " + filename + ")";
     }
 }
 

src/java/org/apache/cassandra/streaming/messages/KeepAliveMessage.java

@@ -19,8 +19,8 @@
 package org.apache.cassandra.streaming.messages;
 
 import java.io.IOException;
-import java.nio.channels.ReadableByteChannel;
 
+import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.streaming.StreamSession;
 
@@ -28,13 +28,18 @@ public class KeepAliveMessage extends StreamMessage
 {
     public static Serializer<KeepAliveMessage> serializer = new Serializer<KeepAliveMessage>()
     {
-        public KeepAliveMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
+        public KeepAliveMessage deserialize(DataInputPlus in, int version, StreamSession session) throws IOException
         {
             return new KeepAliveMessage();
         }
 
         public void serialize(KeepAliveMessage message, DataOutputStreamPlus out, int version, StreamSession session)
         {}
+
+        public long serializedSize(KeepAliveMessage message, int version)
+        {
+            return 0;
+        }
     };
 
     public KeepAliveMessage()

src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java

@@ -18,17 +18,18 @@
 package org.apache.cassandra.streaming.messages;
 
 import java.io.IOException;
-import java.nio.channels.ReadableByteChannel;
 import java.util.List;
 
 import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.streaming.StreamSession;
 import org.apache.cassandra.streaming.StreamWriter;
 import org.apache.cassandra.streaming.compress.CompressedStreamWriter;
 import org.apache.cassandra.streaming.compress.CompressionInfo;
+import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Ref;
 
@@ -39,7 +40,7 @@ public class OutgoingFileMessage extends StreamMessage
 {
     public static Serializer<OutgoingFileMessage> serializer = new Serializer<OutgoingFileMessage>()
     {
-        public OutgoingFileMessage deserialize(ReadableByteChannel in, int version, StreamSession session)
+        public OutgoingFileMessage deserialize(DataInputPlus in, int version, StreamSession session)
         {
             throw new UnsupportedOperationException("Not allowed to call deserialize on an outgoing file");
         }
@@ -57,6 +58,11 @@ public class OutgoingFileMessage extends StreamMessage
                 message.finishTransfer();
             }
         }
+
+        public long serializedSize(OutgoingFileMessage message, int version)
+        {
+            return 0;
+        }
     };
 
     public final FileMessageHeader header;
@@ -65,7 +71,7 @@ public class OutgoingFileMessage extends StreamMessage
     private boolean completed = false;
     private boolean transferring = false;
 
-    public OutgoingFileMessage(Ref<SSTableReader> ref, int sequenceNumber, long estimatedKeys, List<Pair<Long, Long>> sections, boolean keepSSTableLevel)
+    public OutgoingFileMessage(Ref<SSTableReader> ref, StreamSession session, int sequenceNumber, long estimatedKeys, List<Pair<Long, Long>> sections, boolean keepSSTableLevel)
     {
         super(Type.FILE);
         this.ref = ref;
@@ -73,6 +79,9 @@ public class OutgoingFileMessage extends StreamMessage
         SSTableReader sstable = ref.get();
         filename = sstable.getFilename();
         this.header = new FileMessageHeader(sstable.metadata().id,
+                                            FBUtilities.getBroadcastAddress(),
+                                            session.planId(),
+                                            session.sessionIndex(),
                                             sequenceNumber,
                                             sstable.descriptor.version,
                                             sstable.descriptor.formatType,
@@ -93,12 +102,12 @@ public class OutgoingFileMessage extends StreamMessage
         }
 
         CompressionInfo compressionInfo = FileMessageHeader.serializer.serialize(header, out, version);
-
+        out.flush();
         final SSTableReader reader = ref.get();
         StreamWriter writer = compressionInfo == null ?
-                                      new StreamWriter(reader, header.sections, session) :
-                                      new CompressedStreamWriter(reader, header.sections,
-                                                                 compressionInfo, session);
+                              new StreamWriter(reader, header.sections, session) :
+                              new CompressedStreamWriter(reader, header.sections,
+                                                         compressionInfo, session);
         writer.write(out);
     }
 
@@ -140,5 +149,10 @@ public class OutgoingFileMessage extends StreamMessage
     {
         return "File (" + header + ", file: " + filename + ")";
     }
+
+    public String getFilename()
+    {
+        return filename;
+    }
 }
 

src/java/org/apache/cassandra/streaming/messages/PrepareAckMessage.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.messages;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.streaming.StreamSession;
+
+public class PrepareAckMessage extends StreamMessage
+{
+    public static Serializer<PrepareAckMessage> serializer = new Serializer<PrepareAckMessage>()
+    {
+        public void serialize(PrepareAckMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
+        {
+            //nop
+        }
+
+        public PrepareAckMessage deserialize(DataInputPlus in, int version, StreamSession session) throws IOException
+        {
+            return new PrepareAckMessage();
+        }
+
+        public long serializedSize(PrepareAckMessage message, int version)
+        {
+            return 0;
+        }
+    };
+
+    public PrepareAckMessage()
+    {
+        super(Type.PREPARE_ACK);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Prepare ACK";
+    }
+}

src/java/org/apache/cassandra/streaming/messages/PrepareSynAckMessage.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.messages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.StreamSummary;
+
+public class PrepareSynAckMessage extends StreamMessage
+{
+    public static Serializer<PrepareSynAckMessage> serializer = new Serializer<PrepareSynAckMessage>()
+    {
+        public void serialize(PrepareSynAckMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
+        {
+            out.writeInt(message.summaries.size());
+            for (StreamSummary summary : message.summaries)
+                StreamSummary.serializer.serialize(summary, out, version);
+        }
+
+        public PrepareSynAckMessage deserialize(DataInputPlus input, int version, StreamSession session) throws IOException
+        {
+            PrepareSynAckMessage message = new PrepareSynAckMessage();
+            int numSummaries = input.readInt();
+            for (int i = 0; i < numSummaries; i++)
+                message.summaries.add(StreamSummary.serializer.deserialize(input, version));
+            return message;
+        }
+
+        public long serializedSize(PrepareSynAckMessage message, int version)
+        {
+            long size = 4; // count of requests and count of summaries
+            for (StreamSummary summary : message.summaries)
+                size += StreamSummary.serializer.serializedSize(summary, version);
+            return size;
+        }
+    };
+
+    /**
+     * Summaries of streaming out
+     */
+    public final Collection<StreamSummary> summaries = new ArrayList<>();
+
+    public PrepareSynAckMessage()
+    {
+        super(Type.PREPARE_SYNACK);
+    }
+
+    @Override
+    public String toString()
+    {
+        final StringBuilder sb = new StringBuilder("Prepare SYNACK (");
+        int totalFile = 0;
+        for (StreamSummary summary : summaries)
+            totalFile += summary.files;
+        sb.append(" ").append(totalFile).append(" files");
+        sb.append('}');
+        return sb.toString();
+    }
+}

src/java/org/apache/cassandra/streaming/messages/PrepareSynMessage.java

@@ -18,27 +18,22 @@
 package org.apache.cassandra.streaming.messages;
 
 import java.io.*;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
 import java.util.ArrayList;
 import java.util.Collection;
 
 import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.streaming.StreamRequest;
 import org.apache.cassandra.streaming.StreamSession;
 import org.apache.cassandra.streaming.StreamSummary;
 
-public class PrepareMessage extends StreamMessage
+public class PrepareSynMessage extends StreamMessage
 {
-    public static Serializer<PrepareMessage> serializer = new Serializer<PrepareMessage>()
+    public static Serializer<PrepareSynMessage> serializer = new Serializer<PrepareSynMessage>()
     {
-        @SuppressWarnings("resource") // Not closing constructed DataInputPlus's as the channel needs to remain open.
-        public PrepareMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
+        public PrepareSynMessage deserialize(DataInputPlus input, int version, StreamSession session) throws IOException
         {
-            DataInputPlus input = new DataInputStreamPlus(Channels.newInputStream(in));
-            PrepareMessage message = new PrepareMessage();
+            PrepareSynMessage message = new PrepareSynMessage();
             // requests
             int numRequests = input.readInt();
             for (int i = 0; i < numRequests; i++)
@@ -50,7 +45,17 @@ public class PrepareMessage extends StreamMessage
             return message;
         }
 
-        public void serialize(PrepareMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
+        public long serializedSize(PrepareSynMessage message, int version)
+        {
+            long size = 4 + 4; // count of requests and count of summaries
+            for (StreamRequest request : message.requests)
+                size += StreamRequest.serializer.serializedSize(request, version);
+            for (StreamSummary summary : message.summaries)
+                size += StreamSummary.serializer.serializedSize(summary, version);
+            return size;
+        }
+
+        public void serialize(PrepareSynMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
         {
             // requests
             out.writeInt(message.requests.size());
@@ -73,15 +78,15 @@ public class PrepareMessage extends StreamMessage
      */
     public final Collection<StreamSummary> summaries = new ArrayList<>();
 
-    public PrepareMessage()
+    public PrepareSynMessage()
     {
-        super(Type.PREPARE);
+        super(Type.PREPARE_SYN);
     }
 
     @Override
     public String toString()
     {
-        final StringBuilder sb = new StringBuilder("Prepare (");
+        final StringBuilder sb = new StringBuilder("Prepare SYN (");
         sb.append(requests.size()).append(" requests, ");
         int totalFile = 0;
         for (StreamSummary summary : summaries)

src/java/org/apache/cassandra/streaming/messages/ReceivedMessage.java

@@ -18,11 +18,8 @@
 package org.apache.cassandra.streaming.messages;
 
 import java.io.*;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
 
 import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.streaming.StreamSession;
@@ -32,9 +29,8 @@ public class ReceivedMessage extends StreamMessage
     public static Serializer<ReceivedMessage> serializer = new Serializer<ReceivedMessage>()
     {
         @SuppressWarnings("resource") // Not closing constructed DataInputPlus's as the channel needs to remain open.
-        public ReceivedMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
+        public ReceivedMessage deserialize(DataInputPlus input, int version, StreamSession session) throws IOException
         {
-            DataInputPlus input = new DataInputStreamPlus(Channels.newInputStream(in));
             return new ReceivedMessage(TableId.deserialize(input), input.readInt());
         }
 
@@ -43,6 +39,11 @@ public class ReceivedMessage extends StreamMessage
             message.tableId.serialize(out);
             out.writeInt(message.sequenceNumber);
         }
+
+        public long serializedSize(ReceivedMessage message, int version)
+        {
+            return message.tableId.serializedSize() + 4;
+        }
     };
 
     public final TableId tableId;

src/java/org/apache/cassandra/streaming/messages/RetryMessage.java

@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.streaming.messages;
-
-import java.io.*;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.UUID;
-
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.streaming.StreamSession;
-import org.apache.cassandra.utils.UUIDSerializer;
-
-/**
- * @deprecated retry support removed on CASSANDRA-10992
- */
-@Deprecated
-public class RetryMessage extends StreamMessage
-{
-    public static Serializer<RetryMessage> serializer = new Serializer<RetryMessage>()
-    {
-        @SuppressWarnings("resource") // Not closing constructed DataInputPlus's as the channel needs to remain open.
-        public RetryMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
-        {
-            DataInputPlus input = new DataInputStreamPlus(Channels.newInputStream(in));
-            return new RetryMessage(UUIDSerializer.serializer.deserialize(input, MessagingService.current_version), input.readInt());
-        }
-
-        public void serialize(RetryMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
-        {
-            UUIDSerializer.serializer.serialize(message.cfId, out, MessagingService.current_version);
-            out.writeInt(message.sequenceNumber);
-        }
-    };
-
-    public final UUID cfId;
-    public final int sequenceNumber;
-
-    public RetryMessage(UUID cfId, int sequenceNumber)
-    {
-        super(Type.RETRY);
-        this.cfId = cfId;
-        this.sequenceNumber = sequenceNumber;
-    }
-
-    @Override
-    public String toString()
-    {
-        final StringBuilder sb = new StringBuilder("Retry (");
-        sb.append(cfId).append(", #").append(sequenceNumber).append(')');
-        return sb.toString();
-    }
-}

src/java/org/apache/cassandra/streaming/messages/SessionFailedMessage.java

@@ -17,8 +17,7 @@
  */
 package org.apache.cassandra.streaming.messages;
 
-import java.nio.channels.ReadableByteChannel;
-
+import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.streaming.StreamSession;
 
@@ -26,12 +25,17 @@ public class SessionFailedMessage extends StreamMessage
 {
     public static Serializer<SessionFailedMessage> serializer = new Serializer<SessionFailedMessage>()
     {
-        public SessionFailedMessage deserialize(ReadableByteChannel in, int version, StreamSession session)
+        public SessionFailedMessage deserialize(DataInputPlus in, int version, StreamSession session)
         {
             return new SessionFailedMessage();
         }
 
         public void serialize(SessionFailedMessage message, DataOutputStreamPlus out, int version, StreamSession session) {}
+
+        public long serializedSize(SessionFailedMessage message, int version)
+        {
+            return 0;
+        }
     };
 
     public SessionFailedMessage()

src/java/org/apache/cassandra/streaming/messages/StreamInitMessage.java

@@ -19,103 +19,64 @@ package org.apache.cassandra.streaming.messages;
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.nio.ByteBuffer;
 import java.util.UUID;
 
 import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.DataOutputBufferFixed;
-import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.net.CompactEndpointSerializationHelper;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.streaming.StreamOperation;
 import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamSession;
 import org.apache.cassandra.utils.UUIDSerializer;
 
 /**
  * StreamInitMessage is first sent from the node where {@link org.apache.cassandra.streaming.StreamSession} is started,
  * to initiate corresponding {@link org.apache.cassandra.streaming.StreamSession} on the other side.
  */
-public class StreamInitMessage
+public class StreamInitMessage extends StreamMessage
 {
-    public static IVersionedSerializer<StreamInitMessage> serializer = new StreamInitMessageSerializer();
+    public static Serializer<StreamInitMessage> serializer = new StreamInitMessageSerializer();
 
     public final InetAddress from;
     public final int sessionIndex;
     public final UUID planId;
     public final StreamOperation streamOperation;
 
-    // true if this init message is to connect for outgoing message on receiving side
-    public final boolean isForOutgoing;
     public final boolean keepSSTableLevel;
     public final UUID pendingRepair;
     public final PreviewKind previewKind;
 
-    public StreamInitMessage(InetAddress from, int sessionIndex, UUID planId, StreamOperation streamOperation, boolean isForOutgoing, boolean keepSSTableLevel, UUID pendingRepair, PreviewKind previewKind)
+    public StreamInitMessage(InetAddress from, int sessionIndex, UUID planId, StreamOperation streamOperation, boolean keepSSTableLevel, UUID pendingRepair, PreviewKind previewKind)
     {
+        super(Type.STREAM_INIT);
         this.from = from;
         this.sessionIndex = sessionIndex;
         this.planId = planId;
         this.streamOperation = streamOperation;
-        this.isForOutgoing = isForOutgoing;
         this.keepSSTableLevel = keepSSTableLevel;
         this.pendingRepair = pendingRepair;
         this.previewKind = previewKind;
     }
 
-    /**
-     * Create serialized message.
-     *
-     * @param compress true if message is compressed
-     * @param version Streaming protocol version
-     * @return serialized message in ByteBuffer format
-     */
-    public ByteBuffer createMessage(boolean compress, int version)
+    @Override
+    public String toString()
     {
-        int header = 0;
-        // set compression bit.
-        if (compress)
-            header |= 4;
-        // set streaming bit
-        header |= 8;
-        // Setting up the version bit
-        header |= (version << 8);
-
-        byte[] bytes;
-        try
-        {
-            int size = (int)StreamInitMessage.serializer.serializedSize(this, version);
-            try (DataOutputBuffer buffer = new DataOutputBufferFixed(size))
-            {
-                StreamInitMessage.serializer.serialize(this, buffer, version);
-                bytes = buffer.getData();
-            }
-        }
-        catch (IOException e)
-        {
-            throw new RuntimeException(e);
-        }
-        assert bytes.length > 0;
-
-        ByteBuffer buffer = ByteBuffer.allocate(4 + 4 + bytes.length);
-        buffer.putInt(MessagingService.PROTOCOL_MAGIC);
-        buffer.putInt(header);
-        buffer.put(bytes);
-        buffer.flip();
-        return buffer;
+        StringBuilder sb = new StringBuilder(128);
+        sb.append("StreamInitMessage: from = ").append(from);
+        sb.append(", planId = ").append(planId).append(", session index = ").append(sessionIndex);
+        return sb.toString();
     }
 
-    private static class StreamInitMessageSerializer implements IVersionedSerializer<StreamInitMessage>
+    private static class StreamInitMessageSerializer implements Serializer<StreamInitMessage>
     {
-        public void serialize(StreamInitMessage message, DataOutputPlus out, int version) throws IOException
+        public void serialize(StreamInitMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
         {
             CompactEndpointSerializationHelper.serialize(message.from, out);
             out.writeInt(message.sessionIndex);
             UUIDSerializer.serializer.serialize(message.planId, out, MessagingService.current_version);
             out.writeUTF(message.streamOperation.getDescription());
-            out.writeBoolean(message.isForOutgoing);
             out.writeBoolean(message.keepSSTableLevel);
 
             out.writeBoolean(message.pendingRepair != null);
@@ -126,18 +87,17 @@ public class StreamInitMessage
             out.writeInt(message.previewKind.getSerializationVal());
         }
 
-        public StreamInitMessage deserialize(DataInputPlus in, int version) throws IOException
+        public StreamInitMessage deserialize(DataInputPlus in, int version, StreamSession session) throws IOException
         {
             InetAddress from = CompactEndpointSerializationHelper.deserialize(in);
             int sessionIndex = in.readInt();
             UUID planId = UUIDSerializer.serializer.deserialize(in, MessagingService.current_version);
             String description = in.readUTF();
-            boolean sentByInitiator = in.readBoolean();
             boolean keepSSTableLevel = in.readBoolean();
 
             UUID pendingRepair = in.readBoolean() ? UUIDSerializer.serializer.deserialize(in, version) : null;
             PreviewKind previewKind = PreviewKind.deserialize(in.readInt());
-            return new StreamInitMessage(from, sessionIndex, planId, StreamOperation.fromString(description), sentByInitiator, keepSSTableLevel, pendingRepair, previewKind);
+            return new StreamInitMessage(from, sessionIndex, planId, StreamOperation.fromString(description), keepSSTableLevel, pendingRepair, previewKind);
         }
 
         public long serializedSize(StreamInitMessage message, int version)
@@ -146,7 +106,6 @@ public class StreamInitMessage
             size += TypeSizes.sizeof(message.sessionIndex);
             size += UUIDSerializer.serializer.serializedSize(message.planId, MessagingService.current_version);
             size += TypeSizes.sizeof(message.streamOperation.getDescription());
-            size += TypeSizes.sizeof(message.isForOutgoing);
             size += TypeSizes.sizeof(message.keepSSTableLevel);
             size += TypeSizes.sizeof(message.pendingRepair != null);
             if (message.pendingRepair != null)

src/java/org/apache/cassandra/streaming/messages/StreamMessage.java

@@ -18,10 +18,8 @@
 package org.apache.cassandra.streaming.messages;
 
 import java.io.IOException;
-import java.net.SocketException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ReadableByteChannel;
 
+import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.streaming.StreamSession;
 
@@ -36,67 +34,47 @@ public abstract class StreamMessage
     public static final int VERSION_40 = 5;
     public static final int CURRENT_VERSION = VERSION_40;
 
-    private transient volatile boolean sent = false;
-
     public static void serialize(StreamMessage message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException
     {
-        ByteBuffer buff = ByteBuffer.allocate(1);
         // message type
-        buff.put(message.type.type);
-        buff.flip();
-        out.write(buff);
+        out.writeByte(message.type.type);
         message.type.outSerializer.serialize(message, out, version, session);
     }
 
-    public static StreamMessage deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException
+    public static long serializedSize(StreamMessage message, int version) throws IOException
     {
-        ByteBuffer buff = ByteBuffer.allocate(1);
-        int readBytes = in.read(buff);
-        if (readBytes > 0)
-        {
-            buff.flip();
-            Type type = Type.get(buff.get());
-            return type.inSerializer.deserialize(in, version, session);
-        }
-        else if (readBytes == 0)
-        {
-            // input socket buffer was not filled yet
-            return null;
-        }
-        else
-        {
-            // possibly socket gets closed
-            throw new SocketException("End-of-stream reached");
-        }
+        return 1 + message.type.outSerializer.serializedSize(message, version);
     }
 
-    public void sent()
+    public static StreamMessage deserialize(DataInputPlus in, int version, StreamSession session) throws IOException
     {
-        sent = true;
-    }
-
-    public boolean wasSent()
-    {
-        return sent;
+        byte b = in.readByte();
+        if (b == 0)
+            b = -1;
+        Type type = Type.get(b);
+        return type.inSerializer.deserialize(in, version, session);
     }
 
     /** StreamMessage serializer */
     public static interface Serializer<V extends StreamMessage>
     {
-        V deserialize(ReadableByteChannel in, int version, StreamSession session) throws IOException;
+        V deserialize(DataInputPlus in, int version, StreamSession session) throws IOException;
         void serialize(V message, DataOutputStreamPlus out, int version, StreamSession session) throws IOException;
+        long serializedSize(V message, int version) throws IOException;
     }
 
     /** StreamMessage types */
-    public static enum Type
+    public enum Type
     {
-        PREPARE(1, 5, PrepareMessage.serializer),
+        PREPARE_SYN(1, 5, PrepareSynMessage.serializer),
         FILE(2, 0, IncomingFileMessage.serializer, OutgoingFileMessage.serializer),
         RECEIVED(3, 4, ReceivedMessage.serializer),
-        RETRY(4, 4, RetryMessage.serializer),
         COMPLETE(5, 1, CompleteMessage.serializer),
         SESSION_FAILED(6, 5, SessionFailedMessage.serializer),
-        KEEP_ALIVE(7, 5, KeepAliveMessage.serializer);
+        KEEP_ALIVE(7, 5, KeepAliveMessage.serializer),
+        PREPARE_SYNACK(8, 5, PrepareSynAckMessage.serializer),
+        PREPARE_ACK(9, 5, PrepareAckMessage.serializer),
+        STREAM_INIT(10, 5, StreamInitMessage.serializer);
 
         public static Type get(byte type)
         {

src/java/org/apache/cassandra/tools/BulkLoadConnectionFactory.java

@@ -15,20 +15,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.cassandra.tools;
 
 import java.io.IOException;
-import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.nio.channels.SocketChannel;
 
+import io.netty.channel.Channel;
 import org.apache.cassandra.config.EncryptionOptions;
-import org.apache.cassandra.security.SSLFactory;
+import org.apache.cassandra.net.async.OutboundConnectionIdentifier;
+import org.apache.cassandra.streaming.DefaultConnectionFactory;
 import org.apache.cassandra.streaming.StreamConnectionFactory;
-import org.apache.cassandra.utils.FBUtilities;
 
-public class BulkLoadConnectionFactory implements StreamConnectionFactory
+public class BulkLoadConnectionFactory extends DefaultConnectionFactory implements StreamConnectionFactory
 {
     private final boolean outboundBindAny;
     private final int storagePort;
@@ -43,24 +42,15 @@ public class BulkLoadConnectionFactory implements StreamConnectionFactory
         this.outboundBindAny = outboundBindAny;
     }
 
-    public Socket createConnection(InetAddress peer) throws IOException
+    public Channel createConnection(OutboundConnectionIdentifier connectionId, int protocolVersion) throws IOException
     {
         // Connect to secure port for all peers if ServerEncryptionOptions is configured other than 'none'
         // When 'all', 'dc' and 'rack', server nodes always have SSL port open, and since thin client like sstableloader
         // does not know which node is in which dc/rack, connecting to SSL port is always the option.
-        if (encryptionOptions != null && encryptionOptions.internode_encryption != EncryptionOptions.ServerEncryptionOptions.InternodeEncryption.none)
-        {
-            if (outboundBindAny)
-                return SSLFactory.getSocket(encryptionOptions, peer, secureStoragePort);
-            else
-                return SSLFactory.getSocket(encryptionOptions, peer, secureStoragePort, FBUtilities.getLocalAddress(), 0);
-        }
-        else
-        {
-            Socket socket = SocketChannel.open(new InetSocketAddress(peer, storagePort)).socket();
-            if (outboundBindAny && !socket.isBound())
-                socket.bind(new InetSocketAddress(FBUtilities.getLocalAddress(), 0));
-            return socket;
-        }
+        int port = encryptionOptions != null && encryptionOptions.internode_encryption != EncryptionOptions.ServerEncryptionOptions.InternodeEncryption.none ?
+                   secureStoragePort : storagePort;
+
+        connectionId = connectionId.withNewConnectionAddress(new InetSocketAddress(connectionId.remote(), port));
+        return createConnection(connectionId, protocolVersion, encryptionOptions);
     }
 }

src/java/org/apache/cassandra/tools/NodeProbe.java

@@ -1076,8 +1076,6 @@ public class NodeProbe implements AutoCloseable
                 return ssProxy.getCasContentionTimeout();
             case "truncate":
                 return ssProxy.getTruncateRpcTimeout();
-            case "streamingsocket":
-                return ssProxy.getStreamingSocketTimeout();
             default:
                 throw new RuntimeException("Timeout type requires one of (" + GetTimeout.TIMEOUT_TYPES + ")");
         }
@@ -1156,11 +1154,6 @@ public class NodeProbe implements AutoCloseable
             case "truncate":
                 ssProxy.setTruncateRpcTimeout(value);
                 break;
-            case "streamingsocket":
-                if (value > Integer.MAX_VALUE)
-                    throw new RuntimeException("streamingsocket timeout must be less than " + Integer.MAX_VALUE);
-                ssProxy.setStreamingSocketTimeout((int) value);
-                break;
             default:
                 throw new RuntimeException("Timeout type requires one of (" + GetTimeout.TIMEOUT_TYPES + ")");
         }

src/java/org/apache/cassandra/tools/nodetool/GetTimeout.java

@@ -31,7 +31,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 @Command(name = "gettimeout", description = "Print the timeout of the given type in ms")
 public class GetTimeout extends NodeToolCmd
 {
-    public static final String TIMEOUT_TYPES = "read, range, write, counterwrite, cascontention, truncate, streamingsocket, misc (general rpc_timeout_in_ms)";
+    public static final String TIMEOUT_TYPES = "read, range, write, counterwrite, cascontention, truncate, misc (general rpc_timeout_in_ms)";
 
     @Arguments(usage = "<timeout_type>", description = "The timeout type, one of (" + TIMEOUT_TYPES + ")")
     private List<String> args = new ArrayList<>();

src/java/org/apache/cassandra/utils/UUIDGen.java

@@ -40,6 +40,8 @@ public class UUIDGen
     private static final long START_EPOCH = -12219292800000L;
     private static final long clockSeqAndNode = makeClockSeqAndNode();
 
+    public static final int UUID_LEN = 16;
+
     /*
      * The min and max possible lsb for a UUID.
      * Note that his is not 0 and all 1's because Cassandra TimeUUIDType
@@ -106,10 +108,10 @@ public class UUIDGen
     }
 
     /**
-     * Similar to {@link getTimeUUIDFromMicros}, but randomize (using SecureRandom) the clock and sequence.
+     * Similar to {@link #getTimeUUIDFromMicros}, but randomize (using SecureRandom) the clock and sequence.
      * <p>
      * If you can guarantee that the {@code whenInMicros} argument is unique (for this JVM instance) for
-     * every call, then you should prefer {@link getTimeUUIDFromMicros} which is faster. If you can't
+     * every call, then you should prefer {@link #getTimeUUIDFromMicros} which is faster. If you can't
      * guarantee this however, this method will ensure the returned UUID are still unique (accross calls)
      * through randomization.
      *
@@ -143,7 +145,7 @@ public class UUIDGen
 
     public static ByteBuffer toByteBuffer(UUID uuid)
     {
-        ByteBuffer buffer = ByteBuffer.allocate(16);
+        ByteBuffer buffer = ByteBuffer.allocate(UUID_LEN);
         buffer.putLong(uuid.getMostSignificantBits());
         buffer.putLong(uuid.getLeastSignificantBits());
         buffer.flip();

test/long/org/apache/cassandra/streaming/LongStreamingTest.java

@@ -24,10 +24,12 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.io.Files;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.schema.CompressionParams;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.schema.Schema;
@@ -62,26 +64,42 @@ public class LongStreamingTest
     }
 
     @Test
-    public void testCompressedStream() throws InvalidRequestException, IOException, ExecutionException, InterruptedException
+    public void testSstableCompressionStreaming() throws InterruptedException, ExecutionException, IOException
     {
-        String KS = "cql_keyspace";
+        testStream(true);
+    }
+
+    @Test
+    public void testStreamCompressionStreaming() throws InterruptedException, ExecutionException, IOException
+    {
+        testStream(false);
+    }
+
+    private void testStream(boolean useSstableCompression) throws InvalidRequestException, IOException, ExecutionException, InterruptedException
+    {
+        String KS = useSstableCompression ? "sstable_compression_ks" : "stream_compression_ks";
         String TABLE = "table1";
 
         File tempdir = Files.createTempDir();
         File dataDir = new File(tempdir.getAbsolutePath() + File.separator + KS + File.separator + TABLE);
         assert dataDir.mkdirs();
 
-        String schema = "CREATE TABLE cql_keyspace.table1 ("
+        String schema = "CREATE TABLE " + KS + '.'  + TABLE + "  ("
                         + "  k int PRIMARY KEY,"
                         + "  v1 text,"
                         + "  v2 int"
-                        + ");";// with compression = {};";
-        String insert = "INSERT INTO cql_keyspace.table1 (k, v1, v2) VALUES (?, ?, ?)";
+                        + ") with compression = " + (useSstableCompression ? "{'class': 'LZ4Compressor'};" : "{};");
+        String insert = "INSERT INTO " + KS + '.'  + TABLE + " (k, v1, v2) VALUES (?, ?, ?)";
         CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                   .sorted()
                                                   .inDirectory(dataDir)
                                                   .forTable(schema)
                                                   .using(insert).build();
+
+        CompressionParams compressionParams = Keyspace.open(KS).getColumnFamilyStore(TABLE).metadata().params.compression;
+        Assert.assertEquals(useSstableCompression, compressionParams.isEnabled());
+
+
         long start = System.nanoTime();
 
         for (int i = 0; i < 10_000_000; i++)
@@ -103,7 +121,7 @@ public class LongStreamingTest
             private String ks;
             public void init(String keyspace)
             {
-                for (Range<Token> range : StorageService.instance.getLocalRanges("cql_keyspace"))
+                for (Range<Token> range : StorageService.instance.getLocalRanges(KS))
                     addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
 
                 this.ks = keyspace;
@@ -130,7 +148,7 @@ public class LongStreamingTest
             private String ks;
             public void init(String keyspace)
             {
-                for (Range<Token> range : StorageService.instance.getLocalRanges("cql_keyspace"))
+                for (Range<Token> range : StorageService.instance.getLocalRanges(KS))
                     addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
 
                 this.ks = keyspace;
@@ -160,7 +178,7 @@ public class LongStreamingTest
                                          millis / 1000d,
                                          (dataSize * 2 / (1 << 20) / (millis / 1000d)) * 8));
 
-        UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM cql_keyspace.table1 limit 100;");
+        UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM " + KS + '.'  + TABLE + " limit 100;");
         assertEquals(100, rs.size());
     }
 }

test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java

@@ -54,7 +54,7 @@ public class PreparedStatementsTest extends SchemaLoader
 
         // Currently the native server start method return before the server is fully binded to the socket, so we need
         // to wait slightly before trying to connect to it. We should fix this but in the meantime using a sleep.
-        Thread.sleep(500);
+        Thread.sleep(1500);
 
         cluster = Cluster.builder().addContactPoint("127.0.0.1")
                                    .withPort(DatabaseDescriptor.getNativeTransportPort())

test/unit/org/apache/cassandra/io/util/RewindableDataInputStreamPlusTest.java

@@ -378,7 +378,7 @@ public class RewindableDataInputStreamPlusTest
             //finish reading again previous sequence
 
             reader.mark();
-            //read 3 bytes - OK
+            //read 3 bytes - START
             assertEquals('a', reader.readChar());
             //read 1 more bytes - CAPACITY will exhaust when trying to reset :(
             assertEquals(1, reader.readShort());

test/unit/org/apache/cassandra/net/async/HandshakeHandlersTest.java

@@ -42,7 +42,7 @@ import org.apache.cassandra.net.MessageOut;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.KeyspaceParams;
 
-import static org.apache.cassandra.net.async.InboundHandshakeHandler.State.MESSAGING_HANDSHAKE_COMPLETE;
+import static org.apache.cassandra.net.async.InboundHandshakeHandler.State.HANDSHAKE_COMPLETE;
 import static org.apache.cassandra.net.async.OutboundMessagingConnection.State.READY;
 
 public class HandshakeHandlersTest
@@ -100,7 +100,7 @@ public class HandshakeHandlersTest
             inboundChannel.writeInbound(o);
 
         Assert.assertEquals(READY, imc.getState());
-        Assert.assertEquals(MESSAGING_HANDSHAKE_COMPLETE, inboundHandshakeHandler.getState());
+        Assert.assertEquals(HANDSHAKE_COMPLETE, inboundHandshakeHandler.getState());
     }
 
     @Test

test/unit/org/apache/cassandra/net/async/InboundHandshakeHandlerTest.java

@@ -181,7 +181,7 @@ public class InboundHandshakeHandlerTest
         buf = new ThirdHandshakeMessage(MESSAGING_VERSION, addr.getAddress()).encode(PooledByteBufAllocator.DEFAULT);
         state = handler.handleMessagingStartResponse(channel.pipeline().firstContext(), buf);
 
-        Assert.assertEquals(State.MESSAGING_HANDSHAKE_COMPLETE, state);
+        Assert.assertEquals(State.HANDSHAKE_COMPLETE, state);
         Assert.assertTrue(channel.isOpen());
         Assert.assertTrue(channel.isActive());
         Assert.assertFalse(channel.outboundMessages().isEmpty());
@@ -217,7 +217,7 @@ public class InboundHandshakeHandlerTest
         buf.writeInt(MESSAGING_VERSION);
         CompactEndpointSerializationHelper.serialize(addr.getAddress(), new ByteBufOutputStream(buf));
         State state = handler.handleMessagingStartResponse(channel.pipeline().firstContext(), buf);
-        Assert.assertEquals(State.MESSAGING_HANDSHAKE_COMPLETE, state);
+        Assert.assertEquals(State.HANDSHAKE_COMPLETE, state);
         Assert.assertTrue(channel.isOpen());
         Assert.assertTrue(channel.isActive());
     }
@@ -268,9 +268,9 @@ public class InboundHandshakeHandlerTest
         handler.setHandshakeTimeout(future);
         Assert.assertFalse(future.isCancelled());
         Assert.assertTrue(channel.isOpen());
-        handler.setState(State.MESSAGING_HANDSHAKE_COMPLETE);
+        handler.setState(State.HANDSHAKE_COMPLETE);
         handler.failHandshake(channel.pipeline().firstContext());
-        Assert.assertSame(State.MESSAGING_HANDSHAKE_COMPLETE, handler.getState());
+        Assert.assertSame(State.HANDSHAKE_COMPLETE, handler.getState());
         Assert.assertTrue(channel.isOpen());
     }
 

test/unit/org/apache/cassandra/net/async/OutboundMessagingConnectionTest.java

@@ -24,11 +24,7 @@ import java.net.InetSocketAddress;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.net.ssl.SSLHandshakeException;
 
@@ -475,45 +471,4 @@ public class OutboundMessagingConnectionTest
         Assert.assertNotSame(omc.getConnectionId(), originalId);
         Assert.assertSame(NOT_READY, omc.getState());
     }
-
-    private static class TestScheduledFuture implements ScheduledFuture<Object>
-    {
-        private boolean cancelled = false;
-
-        public long getDelay(TimeUnit unit)
-        {
-            return 0;
-        }
-
-        public int compareTo(Delayed o)
-        {
-            return 0;
-        }
-
-        public boolean cancel(boolean mayInterruptIfRunning)
-        {
-            cancelled = true;
-            return false;
-        }
-
-        public boolean isCancelled()
-        {
-            return cancelled;
-        }
-
-        public boolean isDone()
-        {
-            return false;
-        }
-
-        public Object get() throws InterruptedException, ExecutionException
-        {
-            return null;
-        }
-
-        public Object get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
-        {
-            return null;
-        }
-    }
 }

test/unit/org/apache/cassandra/net/async/RebufferingByteBufDataInputPlusTest.java

@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net.async;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.embedded.EmbeddedChannel;
+
+public class RebufferingByteBufDataInputPlusTest
+{
+    private EmbeddedChannel channel;
+    private RebufferingByteBufDataInputPlus inputPlus;
+    private ByteBuf buf;
+
+    @Before
+    public void setUp()
+    {
+        channel = new EmbeddedChannel();
+        inputPlus = new RebufferingByteBufDataInputPlus(1 << 10, 1 << 11, channel.config());
+    }
+
+    @After
+    public void tearDown()
+    {
+        inputPlus.close();
+        channel.close();
+
+        if (buf != null && buf.refCnt() > 0)
+            buf.release(buf.refCnt());
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void ctor_badWaterMarks()
+    {
+        inputPlus = new RebufferingByteBufDataInputPlus(2, 1, null);
+    }
+
+    @Test
+    public void isOpen()
+    {
+        Assert.assertTrue(inputPlus.isOpen());
+        inputPlus.markClose();
+        Assert.assertFalse(inputPlus.isOpen());
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void append_closed()
+    {
+        inputPlus.markClose();
+        buf = channel.alloc().buffer(4);
+        inputPlus.append(buf);
+    }
+
+    @Test
+    public void append_normal() throws EOFException
+    {
+        int size = 4;
+        buf = channel.alloc().buffer(size);
+        buf.writerIndex(size);
+        inputPlus.append(buf);
+        Assert.assertEquals(buf.readableBytes(), inputPlus.available());
+    }
+
+    @Test
+    public void read() throws IOException
+    {
+        // put two buffers of 8 bytes each into the queue.
+        // then read an int, then a long. the latter tests offset into the inputPlus, as well as spanning across queued buffers.
+        // the values of those int/long will both be '42', but spread across both queue buffers.
+        ByteBuf buf = channel.alloc().buffer(8);
+        buf.writeInt(42);
+        buf.writerIndex(8);
+        inputPlus.append(buf);
+        buf = channel.alloc().buffer(8);
+        buf.writeInt(42);
+        buf.writerIndex(8);
+        inputPlus.append(buf);
+        Assert.assertEquals(16, inputPlus.available());
+
+        ByteBuffer out = ByteBuffer.allocate(4);
+        int readCount = inputPlus.read(out);
+        Assert.assertEquals(4, readCount);
+        out.flip();
+        Assert.assertEquals(42, out.getInt());
+        Assert.assertEquals(12, inputPlus.available());
+
+        out = ByteBuffer.allocate(8);
+        readCount = inputPlus.read(out);
+        Assert.assertEquals(8, readCount);
+        out.flip();
+        Assert.assertEquals(42, out.getLong());
+        Assert.assertEquals(4, inputPlus.available());
+    }
+
+    @Test (expected = EOFException.class)
+    public void read_closed() throws IOException
+    {
+        inputPlus.markClose();
+        ByteBuffer buf = ByteBuffer.allocate(1);
+        inputPlus.read(buf);
+    }
+}

test/unit/org/apache/cassandra/net/async/TestScheduledFuture.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net.async;
+
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class TestScheduledFuture implements ScheduledFuture<Object>
+{
+    private boolean cancelled = false;
+
+    public long getDelay(TimeUnit unit)
+    {
+        return 0;
+    }
+
+    public int compareTo(Delayed o)
+    {
+        return 0;
+    }
+
+    public boolean cancel(boolean mayInterruptIfRunning)
+    {
+        cancelled = true;
+        return false;
+    }
+
+    public boolean isCancelled()
+    {
+        return cancelled;
+    }
+
+    public boolean isDone()
+    {
+        return false;
+    }
+
+    public Object get() throws InterruptedException, ExecutionException
+    {
+        return null;
+    }
+
+    public Object get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
+    {
+        return null;
+    }
+}

test/unit/org/apache/cassandra/service/RemoveTest.java

@@ -70,6 +70,7 @@ public class RemoveTest
     public static void setupClass() throws ConfigurationException
     {
         oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner);
+        MessagingService.instance().listen();
     }
 
     @AfterClass
@@ -86,7 +87,6 @@ public class RemoveTest
         // create a ring of 5 nodes
         Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 6);
 
-        MessagingService.instance().listen();
         removalhost = hosts.get(5);
         hosts.remove(removalhost);
         removalId = hostIds.get(5);
@@ -98,7 +98,6 @@ public class RemoveTest
     {
         MessagingService.instance().clearMessageSinks();
         MessagingService.instance().clearCallbacksUnsafe();
-        MessagingService.instance().shutdown();
     }
 
     @Test(expected = UnsupportedOperationException.class)

test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java

@@ -32,6 +32,7 @@ import org.junit.BeforeClass;
 import org.junit.After;
 import org.junit.Test;
 
+import io.netty.channel.embedded.EmbeddedChannel;
 import junit.framework.Assert;
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.db.ColumnFamilyStore;
@@ -74,7 +75,7 @@ public class StreamTransferTaskTest
     public void testScheduleTimeout() throws Exception
     {
         InetAddress peer = FBUtilities.getBroadcastAddress();
-        StreamSession session = new StreamSession(peer, peer, null, 0, true, null, PreviewKind.NONE);
+        StreamSession session = new StreamSession(peer, peer, (connectionId, protocolVersion) -> new EmbeddedChannel(), 0, true, UUID.randomUUID(), PreviewKind.ALL);
         ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
 
         // create two sstables
@@ -120,7 +121,7 @@ public class StreamTransferTaskTest
     public void testFailSessionDuringTransferShouldNotReleaseReferences() throws Exception
     {
         InetAddress peer = FBUtilities.getBroadcastAddress();
-        StreamCoordinator streamCoordinator = new StreamCoordinator(1, true, null, false, null, PreviewKind.NONE);
+        StreamCoordinator streamCoordinator = new StreamCoordinator(1, true, new DefaultConnectionFactory(), false, null, PreviewKind.NONE);
         StreamResultFuture future = StreamResultFuture.init(UUID.randomUUID(), StreamOperation.OTHER, Collections.<StreamEventHandler>emptyList(), streamCoordinator);
         StreamSession session = new StreamSession(peer, peer, null, 0, true, null, PreviewKind.NONE);
         session.init(future);
@@ -159,7 +160,7 @@ public class StreamTransferTaskTest
         }
 
         //fail stream session mid-transfer
-        session.onError(new Exception("Fake exception"));
+        session.onError(new Exception("Fake exception")).get(5, TimeUnit.SECONDS);
 
         //make sure reference was not released
         for (Ref<SSTableReader> ref : refs)

test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java

@@ -26,7 +26,6 @@ import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.slf4j.Logger;
@@ -47,7 +46,6 @@ import org.apache.cassandra.db.partitions.*;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.service.StorageService;
@@ -62,9 +60,6 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 @RunWith(OrderedJUnit4ClassRunner.class)
-
-// TODO:JEB intentionally breaking this with CASSANDRA-8457 until CASSANDRA-12229
-@Ignore
 public class StreamingTransferTest
 {
     private static final Logger logger = LoggerFactory.getLogger(StreamingTransferTest.class);
@@ -244,7 +239,6 @@ public class StreamingTransferTest
         ranges.add(new Range<Token>(p.getToken(ByteBufferUtil.bytes("key1")), p.getToken(ByteBufferUtil.bytes("key0"))));
         StreamPlan streamPlan = new StreamPlan(StreamOperation.OTHER).transferRanges(LOCAL, cfs.keyspace.getName(), ranges, cfs.getTableName());
         streamPlan.execute().get();
-        verifyConnectionsAreClosed();
 
         //cannot add ranges after stream session is finished
         try
@@ -262,7 +256,6 @@ public class StreamingTransferTest
     {
         StreamPlan streamPlan = new StreamPlan(StreamOperation.OTHER).transferFiles(LOCAL, makeStreamingDetails(ranges, Refs.tryRef(Arrays.asList(sstable))));
         streamPlan.execute().get();
-        verifyConnectionsAreClosed();
 
         //cannot add files after stream session is finished
         try
@@ -276,27 +269,6 @@ public class StreamingTransferTest
         }
     }
 
-    /**
-     * Test that finished incoming connections are removed from MessagingService (CASSANDRA-11854)
-     */
-    private void verifyConnectionsAreClosed() throws InterruptedException
-    {
-        // TODO:JEB intentionally breaking this with CASSANDRA-8457 until CASSANDRA-12229
-        //after stream session is finished, message handlers may take several milliseconds to be closed
-//        outer:
-//        for (int i = 0; i <= 100; i++)
-//        {
-//            for (MessagingService.SocketThread socketThread : MessagingService.instance().getSocketThreads())
-//                if (!socketThread.connections.isEmpty())
-//                {
-//                    Thread.sleep(100);
-//                    continue outer;
-//                }
-//            return;
-//        }
-//        fail("Streaming connections remain registered in MessagingService");
-    }
-
     private Collection<StreamSession.SSTableStreamingSections> makeStreamingDetails(List<Range<Token>> ranges, Refs<SSTableReader> sstables)
     {
         ArrayList<StreamSession.SSTableStreamingSections> details = new ArrayList<>();

test/unit/org/apache/cassandra/streaming/async/NettyStreamingMessageSenderTest.java

@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.net.InetSocketAddress;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.net.async.TestScheduledFuture;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.messages.CompleteMessage;
+
+public class NettyStreamingMessageSenderTest
+{
+    private static final InetSocketAddress REMOTE_ADDR = new InetSocketAddress("127.0.0.2", 0);
+
+    private EmbeddedChannel channel;
+    private StreamSession session;
+    private NettyStreamingMessageSender sender;
+    private NettyStreamingMessageSender.FileStreamTask fileStreamTask;
+
+    @BeforeClass
+    public static void before()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Before
+    public void setUp()
+    {
+        channel = new EmbeddedChannel();
+        channel.attr(NettyStreamingMessageSender.TRANSFERRING_FILE_ATTR).set(Boolean.FALSE);
+        UUID pendingRepair = UUID.randomUUID();
+        session = new StreamSession(REMOTE_ADDR.getAddress(), REMOTE_ADDR.getAddress(), (connectionId, protocolVersion) -> null, 0, true, pendingRepair, PreviewKind.ALL);
+        StreamResultFuture future = StreamResultFuture.initReceivingSide(0, UUID.randomUUID(), StreamOperation.REPAIR, REMOTE_ADDR.getAddress(), channel, true, pendingRepair, session.getPreviewKind());
+        session.init(future);
+        sender = session.getMessageSender();
+        sender.setControlMessageChannel(channel);
+    }
+
+    @After
+    public void tearDown()
+    {
+        if (fileStreamTask != null)
+            fileStreamTask.unsetChannel();
+    }
+
+    @Test
+    public void KeepAliveTask_normalSend()
+    {
+        Assert.assertTrue(channel.isOpen());
+        NettyStreamingMessageSender.KeepAliveTask task = sender.new KeepAliveTask(channel, session);
+        task.run();
+        Assert.assertTrue(channel.releaseOutbound());
+    }
+
+    @Test
+    public void KeepAliveTask_channelClosed()
+    {
+        channel.close();
+        Assert.assertFalse(channel.isOpen());
+        channel.releaseOutbound();
+        NettyStreamingMessageSender.KeepAliveTask task = sender.new KeepAliveTask(channel, session);
+        task.future = new TestScheduledFuture();
+        Assert.assertFalse(task.future.isCancelled());
+        task.run();
+        Assert.assertTrue(task.future.isCancelled());
+        Assert.assertFalse(channel.releaseOutbound());
+    }
+
+    @Test
+    public void KeepAliveTask_closed()
+    {
+        Assert.assertTrue(channel.isOpen());
+        NettyStreamingMessageSender.KeepAliveTask task = sender.new KeepAliveTask(channel, session);
+        task.future = new TestScheduledFuture();
+        Assert.assertFalse(task.future.isCancelled());
+
+        sender.setClosed();
+        Assert.assertFalse(sender.connected());
+        task.run();
+        Assert.assertTrue(task.future.isCancelled());
+        Assert.assertFalse(channel.releaseOutbound());
+    }
+
+    @Test
+    public void KeepAliveTask_CurrentlyStreaming()
+    {
+        Assert.assertTrue(channel.isOpen());
+        channel.attr(NettyStreamingMessageSender.TRANSFERRING_FILE_ATTR).set(Boolean.TRUE);
+        NettyStreamingMessageSender.KeepAliveTask task = sender.new KeepAliveTask(channel, session);
+        task.future = new TestScheduledFuture();
+        Assert.assertFalse(task.future.isCancelled());
+
+        Assert.assertTrue(sender.connected());
+        task.run();
+        Assert.assertFalse(task.future.isCancelled());
+        Assert.assertFalse(channel.releaseOutbound());
+    }
+
+    @Test
+    public void FileStreamTask_acquirePermit_closed()
+    {
+        fileStreamTask = sender.new FileStreamTask(null);
+        sender.setClosed();
+        Assert.assertFalse(fileStreamTask.acquirePermit(1));
+    }
+
+    @Test
+    public void FileStreamTask_acquirePermit_HapppyPath()
+    {
+        int permits = sender.semaphoreAvailablePermits();
+        fileStreamTask = sender.new FileStreamTask(null);
+        Assert.assertTrue(fileStreamTask.acquirePermit(1));
+        Assert.assertEquals(permits - 1, sender.semaphoreAvailablePermits());
+    }
+
+    @Test
+    public void FileStreamTask_BadChannelAttr()
+    {
+        int permits = sender.semaphoreAvailablePermits();
+        channel.attr(NettyStreamingMessageSender.TRANSFERRING_FILE_ATTR).set(Boolean.TRUE);
+        fileStreamTask = sender.new FileStreamTask(null);
+        fileStreamTask.injectChannel(channel);
+        fileStreamTask.run();
+        Assert.assertEquals(StreamSession.State.FAILED, session.state());
+        Assert.assertTrue(channel.releaseOutbound()); // when the session fails, it will send a SessionFailed msg
+        Assert.assertEquals(permits, sender.semaphoreAvailablePermits());
+    }
+
+    @Test
+    public void FileStreamTask_HappyPath()
+    {
+        int permits = sender.semaphoreAvailablePermits();
+        fileStreamTask = sender.new FileStreamTask(new CompleteMessage());
+        fileStreamTask.injectChannel(channel);
+        fileStreamTask.run();
+        Assert.assertNotEquals(StreamSession.State.FAILED, session.state());
+        Assert.assertTrue(channel.releaseOutbound());
+        Assert.assertEquals(permits, sender.semaphoreAvailablePermits());
+    }
+
+    @Test
+    public void onControlMessageComplete_HappyPath()
+    {
+        Assert.assertTrue(channel.isOpen());
+        Assert.assertTrue(sender.connected());
+        ChannelPromise promise = channel.newPromise();
+        promise.setSuccess();
+        Assert.assertNull(sender.onControlMessageComplete(promise, new CompleteMessage()));
+        Assert.assertTrue(channel.isOpen());
+        Assert.assertTrue(sender.connected());
+        Assert.assertNotEquals(StreamSession.State.FAILED, session.state());
+    }
+
+    @Test
+    public void onControlMessageComplete_Exception() throws InterruptedException, ExecutionException, TimeoutException
+    {
+        Assert.assertTrue(channel.isOpen());
+        Assert.assertTrue(sender.connected());
+        ChannelPromise promise = channel.newPromise();
+        promise.setFailure(new RuntimeException("this is just a testing exception"));
+        Future f = sender.onControlMessageComplete(promise, new CompleteMessage());
+
+        f.get(5, TimeUnit.SECONDS);
+
+        Assert.assertFalse(channel.isOpen());
+        Assert.assertFalse(sender.connected());
+        Assert.assertEquals(StreamSession.State.FAILED, session.state());
+    }
+}

test/unit/org/apache/cassandra/streaming/async/StreamCompressionSerializerTest.java

@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocator;
+import net.jpountz.lz4.LZ4Compressor;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+
+public class StreamCompressionSerializerTest
+{
+    private static final int VERSION = StreamMessage.CURRENT_VERSION;
+    private static final Random random = new Random(2347623847623L);
+
+    private final ByteBufAllocator allocator = PooledByteBufAllocator.DEFAULT;
+    private final StreamCompressionSerializer serializer = new StreamCompressionSerializer(allocator);
+    private final LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor();
+    private final LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor();
+
+    private ByteBuffer input;
+    private ByteBuf compressed;
+    private ByteBuf output;
+
+    @BeforeClass
+    public static void before()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @After
+    public void tearDown()
+    {
+        if (input != null)
+            FileUtils.clean(input);
+        if (compressed != null && compressed.refCnt() > 0)
+            compressed.release(compressed.refCnt());
+        if (output != null && output.refCnt() > 0)
+            output.release(output.refCnt());
+    }
+
+    @Test
+    public void roundTrip_HappyPath_NotReadabaleByteBuffer() throws IOException
+    {
+        populateInput();
+        compressed = serializer.serialize(compressor, input, VERSION);
+        input.flip();
+        ByteBuffer compressedNioBuffer = compressed.nioBuffer(0, compressed.writerIndex());
+        output = serializer.deserialize(decompressor, new DataInputBuffer(compressedNioBuffer, false), VERSION);
+        validateResults();
+    }
+
+    private void populateInput()
+    {
+        int bufSize = 1 << 14;
+        input = ByteBuffer.allocateDirect(bufSize);
+        for (int i = 0; i < bufSize; i += 4)
+            input.putInt(random.nextInt());
+        input.flip();
+    }
+
+    private void validateResults()
+    {
+        Assert.assertEquals(input.remaining(), output.readableBytes());
+        for (int i = 0; i < input.remaining(); i++)
+            Assert.assertEquals(input.get(i), output.readByte());
+    }
+
+    @Test
+    public void roundTrip_HappyPath_ReadabaleByteBuffer() throws IOException
+    {
+        populateInput();
+        compressed = serializer.serialize(compressor, input, VERSION);
+        input.flip();
+        output = serializer.deserialize(decompressor, new ByteBufRCH(compressed), VERSION);
+        validateResults();
+    }
+
+    private static class ByteBufRCH extends DataInputBuffer implements ReadableByteChannel
+    {
+        public ByteBufRCH(ByteBuf compressed)
+        {
+            super (compressed.nioBuffer(0, compressed.readableBytes()), false);
+        }
+
+        @Override
+        public int read(ByteBuffer dst) throws IOException
+        {
+            int len = dst.remaining();
+            dst.put(buffer);
+            return len;
+        }
+
+        @Override
+        public boolean isOpen()
+        {
+            return true;
+        }
+
+        @Override
+        public void close()
+        {   }
+    }
+}

test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java

@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.streaming.async;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.UUID;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.net.async.RebufferingByteBufDataInputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamManager;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.streaming.async.StreamingInboundHandler.SessionIdentifier;
+import org.apache.cassandra.streaming.messages.CompleteMessage;
+import org.apache.cassandra.streaming.messages.FileMessageHeader;
+import org.apache.cassandra.streaming.messages.IncomingFileMessage;
+import org.apache.cassandra.streaming.messages.StreamInitMessage;
+import org.apache.cassandra.streaming.messages.StreamMessage;
+
+public class StreamingInboundHandlerTest
+{
+    private static final int VERSION = StreamMessage.CURRENT_VERSION;
+    private static final InetSocketAddress REMOTE_ADDR = new InetSocketAddress("127.0.0.2", 0);
+
+    private StreamingInboundHandler handler;
+    private EmbeddedChannel channel;
+    private RebufferingByteBufDataInputPlus buffers;
+    private ByteBuf buf;
+
+    @BeforeClass
+    public static void before()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Before
+    public void setup()
+    {
+        handler = new StreamingInboundHandler(REMOTE_ADDR, VERSION, null);
+        channel = new EmbeddedChannel(handler);
+        buffers = new RebufferingByteBufDataInputPlus(1 << 9, 1 << 10, channel.config());
+        handler.setPendingBuffers(buffers);
+    }
+
+    @After
+    public void tearDown()
+    {
+        if (buf != null)
+        {
+            while (buf.refCnt() > 0)
+                buf.release();
+        }
+
+        channel.close();
+    }
+
+    @Test
+    public void channelRead_Normal() throws EOFException
+    {
+        Assert.assertEquals(0, buffers.available());
+        int size = 8;
+        buf = channel.alloc().buffer(size);
+        buf.writerIndex(size);
+        channel.writeInbound(buf);
+        Assert.assertEquals(size, buffers.available());
+        Assert.assertFalse(channel.releaseInbound());
+    }
+
+    @Test (expected = EOFException.class)
+    public void channelRead_Closed() throws EOFException
+    {
+        int size = 8;
+        buf = channel.alloc().buffer(size);
+        Assert.assertEquals(1, buf.refCnt());
+        buf.writerIndex(size);
+        handler.close();
+        channel.writeInbound(buf);
+        Assert.assertEquals(0, buffers.available());
+        Assert.assertEquals(0, buf.refCnt());
+        Assert.assertFalse(channel.releaseInbound());
+    }
+
+    @Test
+    public void channelRead_WrongObject() throws EOFException
+    {
+        channel.writeInbound("homer");
+        Assert.assertEquals(0, buffers.available());
+        Assert.assertFalse(channel.releaseInbound());
+    }
+
+    @Test
+    public void StreamDeserializingTask_deriveSession_StreamInitMessage() throws InterruptedException, IOException
+    {
+        StreamInitMessage msg = new StreamInitMessage(REMOTE_ADDR.getAddress(), 0, UUID.randomUUID(), StreamOperation.REPAIR, true, UUID.randomUUID(), PreviewKind.ALL);
+        StreamingInboundHandler.StreamDeserializingTask task = handler.new StreamDeserializingTask(sid -> createSession(sid), null, channel);
+        StreamSession session = task.deriveSession(msg);
+        Assert.assertNotNull(session);
+    }
+
+    private StreamSession createSession(SessionIdentifier sid)
+    {
+        return new StreamSession(sid.from, sid.from, (connectionId, protocolVersion) -> null, sid.sessionIndex, true, UUID.randomUUID(), PreviewKind.ALL);
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void StreamDeserializingTask_deriveSession_NoSession() throws InterruptedException, IOException
+    {
+        CompleteMessage msg = new CompleteMessage();
+        StreamingInboundHandler.StreamDeserializingTask task = handler.new StreamDeserializingTask(sid -> createSession(sid), null, channel);
+        task.deriveSession(msg);
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void StreamDeserializingTask_deriveSession_IFM_NoSession() throws InterruptedException, IOException
+    {
+        FileMessageHeader header = new FileMessageHeader(TableId.generate(), REMOTE_ADDR.getAddress(), UUID.randomUUID(), 0, 0,
+                                                         BigFormat.latestVersion, SSTableFormat.Type.BIG, 0, new ArrayList<>(), null, 0, UUID.randomUUID(), 0 , null);
+        IncomingFileMessage msg = new IncomingFileMessage(null, header);
+        StreamingInboundHandler.StreamDeserializingTask task = handler.new StreamDeserializingTask(sid -> StreamManager.instance.findSession(sid.from, sid.planId, sid.sessionIndex), null, channel);
+        task.deriveSession(msg);
+    }
+
+    @Test
+    public void StreamDeserializingTask_deriveSession_IFM_HasSession() throws InterruptedException, IOException
+    {
+        UUID planId = UUID.randomUUID();
+        StreamResultFuture future = StreamResultFuture.initReceivingSide(0, planId, StreamOperation.REPAIR, REMOTE_ADDR.getAddress(), channel, true, UUID.randomUUID(), PreviewKind.ALL);
+        StreamManager.instance.register(future);
+        FileMessageHeader header = new FileMessageHeader(TableId.generate(), REMOTE_ADDR.getAddress(), planId, 0, 0,
+                                                         BigFormat.latestVersion, SSTableFormat.Type.BIG, 0, new ArrayList<>(), null, 0, UUID.randomUUID(), 0 , null);
+        IncomingFileMessage msg = new IncomingFileMessage(null, header);
+        StreamingInboundHandler.StreamDeserializingTask task = handler.new StreamDeserializingTask(sid -> StreamManager.instance.findSession(sid.from, sid.planId, sid.sessionIndex), null, channel);
+        StreamSession session = task.deriveSession(msg);
+        Assert.assertNotNull(session);
+    }
+}

test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java

@@ -28,12 +28,11 @@ import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.io.compress.CompressedSequentialWriter;
 import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
 import org.apache.cassandra.io.util.SequentialWriterOption;
 import org.apache.cassandra.schema.CompressionParams;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.streaming.compress.CompressedInputStream;
 import org.apache.cassandra.streaming.compress.CompressionInfo;
@@ -174,7 +173,7 @@ public class CompressedInputStreamTest
             testException(sections, info);
             return;
         }
-        CompressedInputStream input = new CompressedInputStream(new ByteArrayInputStream(toRead), info, ChecksumType.CRC32, () -> 1.0);
+        CompressedInputStream input = new CompressedInputStream(new DataInputStreamPlus(new ByteArrayInputStream(toRead)), info, ChecksumType.CRC32, () -> 1.0);
 
         try (DataInputStream in = new DataInputStream(input))
         {
@@ -189,14 +188,14 @@ public class CompressedInputStreamTest
 
     private static void testException(List<Pair<Long, Long>> sections, CompressionInfo info) throws IOException
     {
-        CompressedInputStream input = new CompressedInputStream(new ByteArrayInputStream(new byte[0]), info, ChecksumType.CRC32, () -> 1.0);
+        CompressedInputStream input = new CompressedInputStream(new DataInputStreamPlus(new ByteArrayInputStream(new byte[0])), info, ChecksumType.CRC32, () -> 1.0);
 
         try (DataInputStream in = new DataInputStream(input))
         {
             for (int i = 0; i < sections.size(); i++)
             {
-                input.position(sections.get(i).left);
                 try {
+                    input.position(sections.get(i).left);
                     in.readLong();
                     fail("Should have thrown IOException");
                 }
@@ -208,3 +207,4 @@ public class CompressedInputStreamTest
         }
     }
 }
+