Faster sequential IO (CASSANDRA-8630)

Merge RandomAccessReader and NIODataInputStream class hierarchies
to share performance optimisation work across all readers.

patch by stefania; reviewed by ariel and benedict for CASSANDRA-8630

build.xml

@@ -415,8 +415,8 @@
           <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" version="2.1.5" classifier="shaded" />
           -->
           <dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj" version="4.4.2" />
-          <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="0.4" />
-          <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8" version="0.4" />
+          <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="0.4.2" />
+          <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8" version="0.4.2" />
           <dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations" version="1.2.0" />
           <dependency groupId="org.fusesource" artifactId="sigar" version="1.6.4">
           	<exclusion groupId="log4j" artifactId="log4j"/>
@@ -470,8 +470,8 @@
         <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" classifier="shaded"/>
         -->
         <dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj"/>
-        <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="0.4" />
-        <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8" version="0.4" />
+        <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="0.4.2" />
+        <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core-j8" version="0.4.2" />
         <dependency groupId="org.openjdk.jmh" artifactId="jmh-core"/>
         <dependency groupId="org.openjdk.jmh" artifactId="jmh-generator-annprocess"/>
         <dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations"/>

src/java/org/apache/cassandra/cache/AutoSavingCache.java

@@ -68,7 +68,7 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
     {
         public InputStream getInputStream(File dataPath, File crcPath) throws IOException
         {
-            return ChecksummedRandomAccessReader.open(dataPath, crcPath);
+            return new ChecksummedRandomAccessReader.Builder(dataPath, crcPath).build();
         }
 
         public OutputStream getOutputStream(File dataPath, File crcPath)

src/java/org/apache/cassandra/cache/OHCProvider.java

@@ -28,6 +28,7 @@ import org.apache.cassandra.db.partitions.CachedPartition;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBufferFixed;
 import org.apache.cassandra.io.util.NIODataInputStream;
+import org.apache.cassandra.io.util.RebufferingInputStream;
 import org.caffinitas.ohc.OHCache;
 import org.caffinitas.ohc.OHCacheBuilder;
 
@@ -172,7 +173,7 @@ public class OHCProvider implements CacheProvider<RowCacheKey, IRowCacheEntry>
         {
             try
             {
-                NIODataInputStream in = new DataInputBuffer(buf, false);
+                RebufferingInputStream in = new DataInputBuffer(buf, false);
                 boolean isSentinel = in.readBoolean();
                 if (isSentinel)
                     return new RowCacheSentinel(in.readLong());

src/java/org/apache/cassandra/db/SystemKeyspace.java

@@ -676,7 +676,7 @@ public final class SystemKeyspace
     {
         try
         {
-            NIODataInputStream in = new DataInputBuffer(bytes, true);
+            RebufferingInputStream in = new DataInputBuffer(bytes, true);
             return Pair.create(ReplayPosition.serializer.deserialize(in), in.available() > 0 ? in.readLong() : Long.MIN_VALUE);
         }
         catch (IOException e)

src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java

@@ -47,12 +47,13 @@ import org.apache.cassandra.db.rows.SerializationHelper;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.FileSegmentInputStream;
+import org.apache.cassandra.io.util.RebufferingInputStream;
 import org.apache.cassandra.schema.CompressionParams;
 import org.apache.cassandra.io.compress.ICompressor;
-import org.apache.cassandra.io.util.ByteBufferDataInput;
+import org.apache.cassandra.io.util.ChannelProxy;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.NIODataInputStream;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.utils.FBUtilities;
@@ -290,8 +291,8 @@ public class CommitLogReplayer
     public void recover(File file, boolean tolerateTruncation) throws IOException
     {
         CommitLogDescriptor desc = CommitLogDescriptor.fromFileName(file.getName());
-        RandomAccessReader reader = RandomAccessReader.open(new File(file.getAbsolutePath()));
-        try
+        try(ChannelProxy channel = new ChannelProxy(file);
+            RandomAccessReader reader = RandomAccessReader.open(channel))
         {
             if (desc.version < CommitLogDescriptor.VERSION_21)
             {
@@ -299,7 +300,7 @@ public class CommitLogReplayer
                     return;
                 if (globalPosition.segment == desc.id)
                     reader.seek(globalPosition.position);
-                replaySyncSection(reader, (int) reader.getPositionLimit(), desc, desc.fileName(), tolerateTruncation);
+                replaySyncSection(reader, (int) reader.length(), desc, desc.fileName(), tolerateTruncation);
                 return;
             }
 
@@ -388,7 +389,7 @@ public class CommitLogReplayer
                         if (uncompressedLength > uncompressedBuffer.length)
                             uncompressedBuffer = new byte[(int) (1.2 * uncompressedLength)];
                         compressedLength = compressor.uncompress(buffer, 0, compressedLength, uncompressedBuffer, 0);
-                        sectionReader = new ByteBufferDataInput(ByteBuffer.wrap(uncompressedBuffer), reader.getPath(), replayPos, 0);
+                        sectionReader = new FileSegmentInputStream(ByteBuffer.wrap(uncompressedBuffer), reader.getPath(), replayPos);
                         errorContext = "compressed section at " + start + " in " + errorContext;
                     }
                     catch (IOException | ArrayIndexOutOfBoundsException e)
@@ -403,10 +404,6 @@ public class CommitLogReplayer
                 if (!replaySyncSection(sectionReader, replayEnd, desc, errorContext, tolerateErrorsInSection))
                     break;
             }
-        }
-        finally
-        {
-            FileUtils.closeQuietly(reader);
             logger.info("Finished reading {}", file);
         }
     }
@@ -522,7 +519,7 @@ public class CommitLogReplayer
     {
 
         final Mutation mutation;
-        try (NIODataInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size))
+        try (RebufferingInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size))
         {
             mutation = Mutation.serializer.deserialize(bufIn,
                                                        desc.getMessagingVersion(),

src/java/org/apache/cassandra/hints/ChecksummedDataInput.java

@@ -17,98 +17,148 @@
  */
 package org.apache.cassandra.hints;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.zip.CRC32;
 
-import org.apache.cassandra.io.util.AbstractDataInput;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.FileMark;
+import org.apache.cassandra.io.util.RandomAccessReader;
 
 /**
- * An {@link AbstractDataInput} wrapper that calctulates the CRC in place.
+ * A {@link RandomAccessReader} wrapper that calctulates the CRC in place.
  *
  * Useful for {@link org.apache.cassandra.hints.HintsReader}, for example, where we must verify the CRC, yet don't want
- * to allocate an extra byte array just that purpose.
+ * to allocate an extra byte array just that purpose. The CRC can be embedded in the input stream and checked via checkCrc().
  *
- * In addition to calculating the CRC, allows to enforce a maximim known size. This is needed
+ * In addition to calculating the CRC, it allows to enforce a maximim known size. This is needed
  * so that {@link org.apache.cassandra.db.Mutation.MutationSerializer} doesn't blow up the heap when deserializing a
  * corrupted sequence by reading a huge corrupted length of bytes via
  * via {@link org.apache.cassandra.utils.ByteBufferUtil#readWithLength(java.io.DataInput)}.
  */
-public final class ChecksummedDataInput extends AbstractDataInput
+public final class ChecksummedDataInput extends RandomAccessReader.RandomAccessReaderWithOwnChannel
 {
     private final CRC32 crc;
-    private final AbstractDataInput source;
-    private int limit;
+    private int crcPosition;
+    private boolean crcUpdateDisabled;
 
-    private ChecksummedDataInput(AbstractDataInput source)
+    private long limit;
+    private FileMark limitMark;
+
+    private ChecksummedDataInput(Builder builder)
     {
-        this.source = source;
+        super(builder);
 
         crc = new CRC32();
-        limit = Integer.MAX_VALUE;
+        crcPosition = 0;
+        crcUpdateDisabled = false;
+
+        resetLimit();
     }
 
-    public static ChecksummedDataInput wrap(AbstractDataInput source)
+    public static ChecksummedDataInput open(File file)
     {
-        return new ChecksummedDataInput(source);
+        return new Builder(new ChannelProxy(file)).build();
     }
 
     public void resetCrc()
     {
         crc.reset();
+        crcPosition = buffer.position();
+    }
+
+    public void limit(long newLimit)
+    {
+        limit = newLimit;
+        limitMark = mark();
     }
 
     public void resetLimit()
     {
-        limit = Integer.MAX_VALUE;
+        limit = Long.MAX_VALUE;
+        limitMark = null;
     }
 
-    public void limit(int newLimit)
+    public void checkLimit(int length) throws IOException
     {
-        limit = newLimit;
+        if (limitMark == null)
+            return;
+
+        if ((bytesPastLimit() + length) > limit)
+            throw new IOException("Digest mismatch exception");
     }
 
-    public int bytesRemaining()
+    public long bytesPastLimit()
     {
-        return limit;
+        assert limitMark != null;
+        return bytesPastMark(limitMark);
     }
 
-    public int getCrc()
+    public boolean checkCrc() throws IOException
     {
-        return (int) crc.getValue();
-    }
+        try
+        {
+            updateCrc();
 
-    public void seek(long position) throws IOException
-    {
-        source.seek(position);
-    }
-
-    public long getPosition()
-    {
-        return source.getPosition();
-    }
-
-    public long getPositionLimit()
-    {
-        return source.getPositionLimit();
-    }
-
-    public int read() throws IOException
-    {
-        int b = source.read();
-        crc.update(b);
-        limit--;
-        return b;
+            // we must diable crc updates in case we rebuffer
+            // when called source.readInt()
+            crcUpdateDisabled = true;
+            return ((int) crc.getValue()) == readInt();
+        }
+        finally
+        {
+            crcPosition = buffer.position();
+            crcUpdateDisabled = false;
+        }
     }
 
     @Override
-    public int read(byte[] buff, int offset, int length) throws IOException
+    public void readFully(byte[] b) throws IOException
     {
-        if (length > limit)
-            throw new IOException("Digest mismatch exception");
+        checkLimit(b.length);
+        super.readFully(b);
+    }
 
-        int copied = source.read(buff, offset, length);
-        crc.update(buff, offset, copied);
-        limit -= copied;
-        return copied;
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException
+    {
+        checkLimit(len);
+        return super.read(b, off, len);
+    }
+
+    @Override
+    public void reBuffer()
+    {
+        updateCrc();
+        super.reBuffer();
+        crcPosition = buffer.position();
+    }
+
+    private void updateCrc()
+    {
+        if (crcPosition == buffer.position() | crcUpdateDisabled)
+            return;
+
+        assert crcPosition >= 0 && crcPosition < buffer.position();
+
+        ByteBuffer unprocessed = buffer.duplicate();
+        unprocessed.position(crcPosition)
+                   .limit(buffer.position());
+
+        crc.update(unprocessed);
+    }
+
+    public final static class Builder extends RandomAccessReader.Builder
+    {
+        public Builder(ChannelProxy channel)
+        {
+            super(channel);
+        }
+
+        public ChecksummedDataInput build()
+        {
+            return new ChecksummedDataInput(this);
+        }
     }
 }

src/java/org/apache/cassandra/hints/HintsReader.java

@@ -24,6 +24,7 @@ import java.util.Iterator;
 
 import javax.annotation.Nullable;
 
+import com.google.common.primitives.Ints;
 import com.google.common.util.concurrent.RateLimiter;
 
 import org.slf4j.Logger;
@@ -32,7 +33,6 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.db.UnknownColumnFamilyException;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.CLibrary;
@@ -57,25 +57,23 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
 
     private final HintsDescriptor descriptor;
     private final File file;
-    private final RandomAccessReader reader;
-    private final ChecksummedDataInput crcInput;
+    private final ChecksummedDataInput input;
 
     // we pass the RateLimiter into HintsReader itself because it's cheaper to calculate the size before the hint is deserialized
     @Nullable
     private final RateLimiter rateLimiter;
 
-    private HintsReader(HintsDescriptor descriptor, File file, RandomAccessReader reader, RateLimiter rateLimiter)
+    private HintsReader(HintsDescriptor descriptor, File file, ChecksummedDataInput reader, RateLimiter rateLimiter)
     {
         this.descriptor = descriptor;
         this.file = file;
-        this.reader = reader;
-        this.crcInput = ChecksummedDataInput.wrap(reader);
+        this.input = reader;
         this.rateLimiter = rateLimiter;
     }
 
     static HintsReader open(File file, RateLimiter rateLimiter)
     {
-        RandomAccessReader reader = RandomAccessReader.open(file);
+        ChecksummedDataInput reader = ChecksummedDataInput.open(file);
         try
         {
             HintsDescriptor descriptor = HintsDescriptor.deserialize(reader);
@@ -83,7 +81,7 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
         }
         catch (IOException e)
         {
-            reader.close();
+            FileUtils.closeQuietly(reader);
             throw new FSReadError(e, file);
         }
     }
@@ -95,7 +93,7 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
 
     public void close()
     {
-        FileUtils.closeQuietly(reader);
+        FileUtils.closeQuietly(input);
     }
 
     public HintsDescriptor descriptor()
@@ -105,7 +103,7 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
 
     void seek(long newPosition)
     {
-        reader.seek(newPosition);
+        input.seek(newPosition);
     }
 
     public Iterator<Page> iterator()
@@ -138,12 +136,12 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
         @SuppressWarnings("resource")
         protected Page computeNext()
         {
-            CLibrary.trySkipCache(reader.getChannel().getFileDescriptor(), 0, reader.getFilePointer(), reader.getPath());
+            CLibrary.trySkipCache(input.getChannel().getFileDescriptor(), 0, input.getFilePointer(), input.getPath());
 
-            if (reader.length() == reader.getFilePointer())
+            if (input.length() == input.getFilePointer())
                 return endOfData();
 
-            return new Page(reader.getFilePointer());
+            return new Page(input.getFilePointer());
         }
     }
 
@@ -166,9 +164,9 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
 
             do
             {
-                long position = reader.getFilePointer();
+                long position = input.getFilePointer();
 
-                if (reader.length() == position)
+                if (input.length() == position)
                     return endOfData(); // reached EOF
 
                 if (position - offset >= PAGE_SIZE)
@@ -190,13 +188,13 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
 
         private Hint computeNextInternal() throws IOException
         {
-            crcInput.resetCrc();
-            crcInput.resetLimit();
+            input.resetCrc();
+            input.resetLimit();
 
-            int size = crcInput.readInt();
+            int size = input.readInt();
 
             // if we cannot corroborate the size via crc, then we cannot safely skip this hint
-            if (reader.readInt() != crcInput.getCrc())
+            if (!input.checkCrc())
                 throw new IOException("Digest mismatch exception");
 
             return readHint(size);
@@ -206,12 +204,13 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
         {
             if (rateLimiter != null)
                 rateLimiter.acquire(size);
-            crcInput.limit(size);
+            input.limit(size);
 
             Hint hint;
             try
             {
-                hint = Hint.serializer.deserialize(crcInput, descriptor.messagingVersion());
+                hint = Hint.serializer.deserialize(input, descriptor.messagingVersion());
+                input.checkLimit(0);
             }
             catch (UnknownColumnFamilyException e)
             {
@@ -219,18 +218,18 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
                             descriptor.hostId,
                             e.cfId,
                             descriptor.fileName());
-                reader.skipBytes(crcInput.bytesRemaining());
+                input.skipBytes(Ints.checkedCast(size - input.bytesPastLimit()));
 
                 return null;
             }
 
-            if (reader.readInt() == crcInput.getCrc())
+            if (input.checkCrc())
                 return hint;
 
             // log a warning and skip the corrupted entry
             logger.warn("Failed to read a hint for {} - digest mismatch for hint at position {} in file {}",
                         descriptor.hostId,
-                        crcInput.getPosition() - size - 4,
+                        input.getPosition() - size - 4,
                         descriptor.fileName());
             return null;
         }
@@ -255,9 +254,9 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
 
             do
             {
-                long position = reader.getFilePointer();
+                long position = input.getFilePointer();
 
-                if (reader.length() == position)
+                if (input.length() == position)
                     return endOfData(); // reached EOF
 
                 if (position - offset >= PAGE_SIZE)
@@ -279,13 +278,13 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
 
         private ByteBuffer computeNextInternal() throws IOException
         {
-            crcInput.resetCrc();
-            crcInput.resetLimit();
+            input.resetCrc();
+            input.resetLimit();
 
-            int size = crcInput.readInt();
+            int size = input.readInt();
 
             // if we cannot corroborate the size via crc, then we cannot safely skip this hint
-            if (reader.readInt() != crcInput.getCrc())
+            if (!input.checkCrc())
                 throw new IOException("Digest mismatch exception");
 
             return readBuffer(size);
@@ -295,16 +294,16 @@ final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
         {
             if (rateLimiter != null)
                 rateLimiter.acquire(size);
-            crcInput.limit(size);
+            input.limit(size);
 
-            ByteBuffer buffer = ByteBufferUtil.read(crcInput, size);
-            if (reader.readInt() == crcInput.getCrc())
+            ByteBuffer buffer = ByteBufferUtil.read(input, size);
+            if (input.checkCrc())
                 return buffer;
 
             // log a warning and skip the corrupted entry
             logger.warn("Failed to read a hint for {} - digest mismatch for hint at position {} in file {}",
                         descriptor.hostId,
-                        crcInput.getPosition() - size - 4,
+                        input.getPosition() - size - 4,
                         descriptor.fileName());
             return null;
         }

src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java

@@ -19,11 +19,7 @@ package org.apache.cassandra.io.compress;
 
 import java.io.*;
 import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-import java.util.Map;
-import java.util.TreeMap;
 import java.util.concurrent.ThreadLocalRandom;
-import java.util.zip.Adler32;
 import java.util.zip.Checksum;
 
 import com.google.common.primitives.Ints;
@@ -31,7 +27,6 @@ import com.google.common.primitives.Ints;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.util.*;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.memory.BufferPool;
 
 /**
@@ -40,18 +35,6 @@ import org.apache.cassandra.utils.memory.BufferPool;
  */
 public class CompressedRandomAccessReader extends RandomAccessReader
 {
-    public static CompressedRandomAccessReader open(ChannelProxy channel, CompressionMetadata metadata)
-    {
-        return new CompressedRandomAccessReader(channel, metadata, null);
-    }
-
-    public static CompressedRandomAccessReader open(ICompressedFile file)
-    {
-        return new CompressedRandomAccessReader(file.channel(), file.getMetadata(), file);
-    }
-
-    private final TreeMap<Long, MappedByteBuffer> chunkSegments;
-
     private final CompressionMetadata metadata;
 
     // we read the raw compressed bytes into this buffer, then move the uncompressed ones into super.buffer.
@@ -63,39 +46,59 @@ public class CompressedRandomAccessReader extends RandomAccessReader
     // raw checksum bytes
     private ByteBuffer checksumBytes;
 
-    protected CompressedRandomAccessReader(ChannelProxy channel, CompressionMetadata metadata, ICompressedFile file)
+    protected CompressedRandomAccessReader(Builder builder)
     {
-        super(channel, metadata.chunkLength(), metadata.compressedFileLength, metadata.compressor().preferredBufferType());
-        this.metadata = metadata;
-        checksum = metadata.checksumType.newInstance();
+        super(builder.initializeBuffers(false));
+        this.metadata = builder.metadata;
+        this.checksum = metadata.checksumType.newInstance();
 
-        chunkSegments = file == null ? null : file.chunkSegments();
-        if (chunkSegments == null)
+        initializeBuffer();
+    }
+
+    @Override
+    protected int getBufferSize(RandomAccessReader.Builder builder)
+    {
+        // this is the chunk data length, throttling is OK with this
+        return builder.bufferSize;
+    }
+
+    @Override
+    protected void initializeBuffer()
+    {
+        buffer = allocateBuffer(bufferSize);
+        buffer.limit(0);
+
+        if (regions == null)
         {
-            compressed = allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength()), metadata.compressor().preferredBufferType());
+            compressed = allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength()));
             checksumBytes = ByteBuffer.wrap(new byte[4]);
         }
     }
 
-    protected int getBufferSize(int size)
-    {
-        assert Integer.bitCount(size) == 1; //must be a power of two
-        return size;
-    }
-
     @Override
-    public void close()
+    protected void releaseBuffer()
     {
-        super.close();
-
-        if (compressed != null)
+        try
         {
-            BufferPool.put(compressed);
-            compressed = null;
+            if (buffer != null)
+            {
+                BufferPool.put(buffer);
+                buffer = null;
+            }
+        }
+        finally
+        {
+            // this will always be null if using mmap access mode (unlike in parent, where buffer is set to a region)
+            if (compressed != null)
+            {
+                BufferPool.put(compressed);
+                compressed = null;
+            }
         }
     }
 
-    private void reBufferStandard()
+    @Override
+    protected void reBufferStandard()
     {
         try
         {
@@ -105,13 +108,19 @@ public class CompressedRandomAccessReader extends RandomAccessReader
             CompressionMetadata.Chunk chunk = metadata.chunkFor(position);
 
             if (compressed.capacity() < chunk.length)
-                compressed = allocateBuffer(chunk.length, metadata.compressor().preferredBufferType());
+            {
+                BufferPool.put(compressed);
+                compressed = allocateBuffer(chunk.length);
+            }
             else
+            {
                 compressed.clear();
-            compressed.limit(chunk.length);
+            }
 
+            compressed.limit(chunk.length);
             if (channel.read(compressed, chunk.offset) != chunk.length)
                 throw new CorruptBlockException(getPath(), chunk);
+
             compressed.flip();
             buffer.clear();
 
@@ -158,7 +167,8 @@ public class CompressedRandomAccessReader extends RandomAccessReader
         }
     }
 
-    private void reBufferMmap()
+    @Override
+    protected void reBufferMmap()
     {
         try
         {
@@ -167,10 +177,10 @@ public class CompressedRandomAccessReader extends RandomAccessReader
 
             CompressionMetadata.Chunk chunk = metadata.chunkFor(position);
 
-            Map.Entry<Long, MappedByteBuffer> entry = chunkSegments.floorEntry(chunk.offset);
-            long segmentOffset = entry.getKey();
+            MmappedRegions.Region region = regions.floor(chunk.offset);
+            long segmentOffset = region.bottom();
             int chunkOffset = Ints.checkedCast(chunk.offset - segmentOffset);
-            ByteBuffer compressedChunk = entry.getValue().duplicate(); // TODO: change to slice(chunkOffset) when we upgrade LZ4-java
+            ByteBuffer compressedChunk = region.buffer.duplicate(); // TODO: change to slice(chunkOffset) when we upgrade LZ4-java
 
             compressedChunk.position(chunkOffset).limit(chunkOffset + chunk.length);
 
@@ -218,19 +228,6 @@ public class CompressedRandomAccessReader extends RandomAccessReader
 
     }
 
-    @Override
-    protected void reBuffer()
-    {
-        if (chunkSegments != null)
-        {
-            reBufferMmap();
-        }
-        else
-        {
-            reBufferStandard();
-        }
-    }
-
     private int checksum(CompressionMetadata.Chunk chunk) throws IOException
     {
         long position = chunk.offset + chunk.length;
@@ -240,11 +237,6 @@ public class CompressedRandomAccessReader extends RandomAccessReader
         return checksumBytes.getInt(0);
     }
 
-    public int getTotalBufferSize()
-    {
-        return super.getTotalBufferSize() + (chunkSegments != null ? 0 : compressed.capacity());
-    }
-
     @Override
     public long length()
     {
@@ -256,4 +248,39 @@ public class CompressedRandomAccessReader extends RandomAccessReader
     {
         return String.format("%s - chunk length %d, data length %d.", getPath(), metadata.chunkLength(), metadata.dataLength);
     }
+
+    public final static class Builder extends RandomAccessReader.Builder
+    {
+        private final CompressionMetadata metadata;
+
+        public Builder(ICompressedFile file)
+        {
+            super(file.channel());
+            this.metadata = applyMetadata(file.getMetadata());
+            this.regions = file.regions();
+        }
+
+        public Builder(ChannelProxy channel, CompressionMetadata metadata)
+        {
+            super(channel);
+            this.metadata = applyMetadata(metadata);
+        }
+
+        private CompressionMetadata applyMetadata(CompressionMetadata metadata)
+        {
+            this.overrideLength = metadata.compressedFileLength;
+            this.bufferSize = metadata.chunkLength();
+            this.bufferType = metadata.compressor().preferredBufferType();
+
+            assert Integer.bitCount(this.bufferSize) == 1; //must be a power of two
+
+            return metadata;
+        }
+
+        @Override
+        public RandomAccessReader build()
+        {
+            return new CompressedRandomAccessReader(this);
+        }
+    }
 }

src/java/org/apache/cassandra/io/compress/CompressedThrottledReader.java

@@ -1,48 +0,0 @@
-package org.apache.cassandra.io.compress;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-import com.google.common.util.concurrent.RateLimiter;
-
-import org.apache.cassandra.io.util.ChannelProxy;
-import org.apache.cassandra.io.util.ICompressedFile;
-
-public class CompressedThrottledReader extends CompressedRandomAccessReader
-{
-    private final RateLimiter limiter;
-
-    public CompressedThrottledReader(ChannelProxy channel, CompressionMetadata metadata, ICompressedFile file, RateLimiter limiter)
-    {
-        super(channel, metadata, file);
-        this.limiter = limiter;
-    }
-
-    protected void reBuffer()
-    {
-        limiter.acquire(buffer.capacity());
-        super.reBuffer();
-    }
-
-    public static CompressedThrottledReader open(ICompressedFile file, RateLimiter limiter)
-    {
-        return new CompressedThrottledReader(file.channel(), file.getMetadata(), file, limiter);
-    }
-}

src/java/org/apache/cassandra/io/compress/CompressionMetadata.java

@@ -92,7 +92,7 @@ public class CompressionMetadata
     }
 
     @VisibleForTesting
-    CompressionMetadata(String indexFilePath, long compressedLength, ChecksumType checksumType)
+    public CompressionMetadata(String indexFilePath, long compressedLength, ChecksumType checksumType)
     {
         this.indexFilePath = indexFilePath;
         this.checksumType = checksumType;

src/java/org/apache/cassandra/io/sstable/KeyIterator.java

@@ -28,7 +28,6 @@ import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.RowIndexEntry;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.CloseableIterator;
 

src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java

@@ -828,8 +828,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
                     if (!summaryLoaded)
                     {
                         summaryBuilder.maybeAddEntry(decoratedKey, indexPosition);
-                        ibuilder.addPotentialBoundary(indexPosition);
-                        dbuilder.addPotentialBoundary(indexEntry.position);
                     }
                 }
 
@@ -868,8 +866,8 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
                     metadata.params.minIndexInterval, metadata.params.maxIndexInterval);
             first = decorateKey(ByteBufferUtil.readWithLength(iStream));
             last = decorateKey(ByteBufferUtil.readWithLength(iStream));
-            ibuilder.deserializeBounds(iStream);
-            dbuilder.deserializeBounds(iStream);
+            ibuilder.deserializeBounds(iStream, descriptor.version);
+            dbuilder.deserializeBounds(iStream, descriptor.version);
         }
         catch (IOException e)
         {
@@ -904,38 +902,34 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
         if (ifile == null)
             return false;
 
-        Iterator<FileDataInput> segments = ifile.iterator(0);
         int i = 0;
         int summaryEntriesChecked = 0;
         int expectedIndexInterval = getMinIndexInterval();
-        while (segments.hasNext())
+        String path = null;
+        try (FileDataInput in = ifile.createReader(0))
         {
-            String path = null;
-            try (FileDataInput in = segments.next())
+            path = in.getPath();
+            while (!in.isEOF())
             {
-                path = in.getPath();
-                while (!in.isEOF())
+                ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
+                if (i % expectedIndexInterval == 0)
                 {
-                    ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
-                    if (i % expectedIndexInterval == 0)
-                    {
-                        ByteBuffer summaryKey = ByteBuffer.wrap(indexSummary.getKey(i / expectedIndexInterval));
-                        if (!summaryKey.equals(indexKey))
-                            return false;
-                        summaryEntriesChecked++;
+                    ByteBuffer summaryKey = ByteBuffer.wrap(indexSummary.getKey(i / expectedIndexInterval));
+                    if (!summaryKey.equals(indexKey))
+                        return false;
+                    summaryEntriesChecked++;
 
-                        if (summaryEntriesChecked == Downsampling.BASE_SAMPLING_LEVEL)
-                            return true;
-                    }
-                    RowIndexEntry.Serializer.skip(in);
-                    i++;
+                    if (summaryEntriesChecked == Downsampling.BASE_SAMPLING_LEVEL)
+                        return true;
                 }
+                RowIndexEntry.Serializer.skip(in);
+                i++;
             }
-            catch (IOException e)
-            {
-                markSuspect();
-                throw new CorruptSSTableException(e, path);
-            }
+        }
+        catch (IOException e)
+        {
+            markSuspect();
+            throw new CorruptSSTableException(e, path);
         }
 
         return true;
@@ -972,8 +966,8 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
             IndexSummary.serializer.serialize(summary, oStream, descriptor.version.hasSamplingLevel());
             ByteBufferUtil.writeWithLength(first.getKey(), oStream);
             ByteBufferUtil.writeWithLength(last.getKey(), oStream);
-            ibuilder.serializeBounds(oStream);
-            dbuilder.serializeBounds(oStream);
+            ibuilder.serializeBounds(oStream, descriptor.version);
+            dbuilder.serializeBounds(oStream, descriptor.version);
         }
         catch (IOException e)
         {
@@ -1600,29 +1594,25 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
         if (ifile == null)
             return null;
 
-        Iterator<FileDataInput> segments = ifile.iterator(sampledPosition);
-        while (segments.hasNext())
+        String path = null;
+        try (FileDataInput in = ifile.createReader(sampledPosition))
         {
-            String path = null;
-            try (FileDataInput in = segments.next();)
+            path = in.getPath();
+            while (!in.isEOF())
             {
-                path = in.getPath();
-                while (!in.isEOF())
-                {
-                    ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
-                    DecoratedKey indexDecoratedKey = decorateKey(indexKey);
-                    if (indexDecoratedKey.compareTo(token) > 0)
-                        return indexDecoratedKey;
+                ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
+                DecoratedKey indexDecoratedKey = decorateKey(indexKey);
+                if (indexDecoratedKey.compareTo(token) > 0)
+                    return indexDecoratedKey;
 
-                    RowIndexEntry.Serializer.skip(in);
-                }
-            }
-            catch (IOException e)
-            {
-                markSuspect();
-                throw new CorruptSSTableException(e, path);
+                RowIndexEntry.Serializer.skip(in);
             }
         }
+        catch (IOException e)
+        {
+            markSuspect();
+            throw new CorruptSSTableException(e, path);
+        }
 
         return null;
     }
@@ -1744,11 +1734,9 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
      */
     public abstract ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, RateLimiter limiter, boolean isForThrift);
 
-
-
     public FileDataInput getFileDataInput(long position)
     {
-        return dfile.getSegment(position);
+        return dfile.createReader(position);
     }
 
     /**
@@ -1939,7 +1927,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
     public RandomAccessReader openDataReader(RateLimiter limiter)
     {
         assert limiter != null;
-        return dfile.createThrottledReader(limiter);
+        return dfile.createReader(limiter);
     }
 
     public RandomAccessReader openDataReader()
@@ -1954,6 +1942,16 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
         return null;
     }
 
+    public ChannelProxy getDataChannel()
+    {
+        return dfile.channel;
+    }
+
+    public ChannelProxy getIndexChannel()
+    {
+        return ifile.channel;
+    }
+
     /**
      * @param component component to get timestamp.
      * @return last modified time for given component. 0 if given component does not exist or IO error occurs.

src/java/org/apache/cassandra/io/sstable/format/Version.java

@@ -68,6 +68,8 @@ public abstract class Version
 
     public abstract boolean hasCompactionAncestors();
 
+    public abstract boolean hasBoundaries();
+
     public String getVersion()
     {
         return version;

src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java

@@ -134,6 +134,7 @@ public class BigFormat implements SSTableFormat
         private final boolean newFileName;
         public final boolean storeRows;
         public final int correspondingMessagingVersion; // Only use by storage that 'storeRows' so far
+        public final boolean hasBoundaries;
         /**
          * CASSANDRA-8413: 3.0 bloom filter representation changed (two longs just swapped)
          * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution.
@@ -176,6 +177,8 @@ public class BigFormat implements SSTableFormat
             correspondingMessagingVersion = storeRows
                                           ? MessagingService.VERSION_30
                                           : MessagingService.VERSION_21;
+
+            hasBoundaries = version.compareTo("ma") < 0;
         }
 
         @Override
@@ -250,6 +253,12 @@ public class BigFormat implements SSTableFormat
             return correspondingMessagingVersion;
         }
 
+        @Override
+        public boolean hasBoundaries()
+        {
+            return hasBoundaries;
+        }
+
         @Override
         public boolean isCompatible()
         {

src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java

@@ -178,78 +178,74 @@ public class BigTableReader extends SSTableReader
         // is lesser than the first key of next interval (and in that case we must return the position of the first key
         // of the next interval).
         int i = 0;
-        Iterator<FileDataInput> segments = ifile.iterator(sampledPosition);
-        while (segments.hasNext())
+        String path = null;
+        try (FileDataInput in = ifile.createReader(sampledPosition))
         {
-            String path = null;
-            try (FileDataInput in = segments.next())
+            path = in.getPath();
+            while (!in.isEOF())
             {
-                path = in.getPath();
-                while (!in.isEOF())
+                i++;
+
+                ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
+
+                boolean opSatisfied; // did we find an appropriate position for the op requested
+                boolean exactMatch; // is the current position an exact match for the key, suitable for caching
+
+                // Compare raw keys if possible for performance, otherwise compare decorated keys.
+                if (op == Operator.EQ && i <= effectiveInterval)
                 {
-                    i++;
-
-                    ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
-
-                    boolean opSatisfied; // did we find an appropriate position for the op requested
-                    boolean exactMatch; // is the current position an exact match for the key, suitable for caching
-
-                    // Compare raw keys if possible for performance, otherwise compare decorated keys.
-                    if (op == Operator.EQ && i <= effectiveInterval)
-                    {
-                        opSatisfied = exactMatch = indexKey.equals(((DecoratedKey) key).getKey());
-                    }
-                    else
-                    {
-                        DecoratedKey indexDecoratedKey = decorateKey(indexKey);
-                        int comparison = indexDecoratedKey.compareTo(key);
-                        int v = op.apply(comparison);
-                        opSatisfied = (v == 0);
-                        exactMatch = (comparison == 0);
-                        if (v < 0)
-                        {
-                            Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.generation);
-                            return null;
-                        }
-                    }
-
-                    if (opSatisfied)
-                    {
-                        // read data position from index entry
-                        RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in);
-                        if (exactMatch && updateCacheAndStats)
-                        {
-                            assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key
-                            DecoratedKey decoratedKey = (DecoratedKey)key;
-
-                            if (logger.isTraceEnabled())
-                            {
-                                // expensive sanity check!  see CASSANDRA-4687
-                                try (FileDataInput fdi = dfile.getSegment(indexEntry.position))
-                                {
-                                    DecoratedKey keyInDisk = decorateKey(ByteBufferUtil.readWithShortLength(fdi));
-                                    if (!keyInDisk.equals(key))
-                                        throw new AssertionError(String.format("%s != %s in %s", keyInDisk, key, fdi.getPath()));
-                                }
-                            }
-
-                            // store exact match for the key
-                            cacheKey(decoratedKey, indexEntry);
-                        }
-                        if (op == Operator.EQ && updateCacheAndStats)
-                            bloomFilterTracker.addTruePositive();
-                        Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndex().size(), descriptor.generation);
-                        return indexEntry;
-                    }
-
-                    RowIndexEntry.Serializer.skip(in);
+                    opSatisfied = exactMatch = indexKey.equals(((DecoratedKey) key).getKey());
                 }
+                else
+                {
+                    DecoratedKey indexDecoratedKey = decorateKey(indexKey);
+                    int comparison = indexDecoratedKey.compareTo(key);
+                    int v = op.apply(comparison);
+                    opSatisfied = (v == 0);
+                    exactMatch = (comparison == 0);
+                    if (v < 0)
+                    {
+                        Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.generation);
+                        return null;
+                    }
+                }
+
+                if (opSatisfied)
+                {
+                    // read data position from index entry
+                    RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in);
+                    if (exactMatch && updateCacheAndStats)
+                    {
+                        assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key
+                        DecoratedKey decoratedKey = (DecoratedKey)key;
+
+                        if (logger.isTraceEnabled())
+                        {
+                            // expensive sanity check!  see CASSANDRA-4687
+                            try (FileDataInput fdi = dfile.createReader(indexEntry.position))
+                            {
+                                DecoratedKey keyInDisk = decorateKey(ByteBufferUtil.readWithShortLength(fdi));
+                                if (!keyInDisk.equals(key))
+                                    throw new AssertionError(String.format("%s != %s in %s", keyInDisk, key, fdi.getPath()));
+                            }
+                        }
+
+                        // store exact match for the key
+                        cacheKey(decoratedKey, indexEntry);
+                    }
+                    if (op == Operator.EQ && updateCacheAndStats)
+                        bloomFilterTracker.addTruePositive();
+                    Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndex().size(), descriptor.generation);
+                    return indexEntry;
+                }
+
+                RowIndexEntry.Serializer.skip(in);
             }
-            catch (IOException e)
-            {
-                markSuspect();
-                throw new CorruptSSTableException(e, path);
-            }
+        }
+        catch (IOException e)
+        {
+            markSuspect();
+            throw new CorruptSSTableException(e, path);
         }
 
         if (op == SSTableReader.Operator.EQ && updateCacheAndStats)

src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java

@@ -121,7 +121,6 @@ public class BigTableWriter extends SSTableWriter
         if (logger.isTraceEnabled())
             logger.trace("wrote {} at {}", decoratedKey, dataEnd);
         iwriter.append(decoratedKey, index, dataEnd);
-        dbuilder.addPotentialBoundary(dataEnd);
     }
 
     /**
@@ -420,7 +419,6 @@ public class BigTableWriter extends SSTableWriter
                 logger.trace("wrote index entry: {} at {}", indexEntry, indexStart);
 
             summary.maybeAddEntry(key, indexStart, indexEnd, dataEnd);
-            builder.addPotentialBoundary(indexStart);
         }
 
         /**

src/java/org/apache/cassandra/io/util/AbstractDataInput.java

@@ -1,343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.util;
-
-import java.io.*;
-
-public abstract class AbstractDataInput extends InputStream implements DataInputPlus
-{
-    public abstract void seek(long position) throws IOException;
-    public abstract long getPosition();
-    public abstract long getPositionLimit();
-
-    public int skipBytes(int n) throws IOException
-    {
-        if (n <= 0)
-            return 0;
-        long oldPosition = getPosition();
-        seek(Math.min(getPositionLimit(), oldPosition + n));
-        long skipped = getPosition() - oldPosition;
-        assert skipped >= 0 && skipped <= n;
-        return (int) skipped;
-    }
-
-    /**
-     * Reads a boolean from the current position in this file. Blocks until one
-     * byte has been read, the end of the file is reached or an exception is
-     * thrown.
-     *
-     * @return the next boolean value from this file.
-     * @throws java.io.EOFException
-     *             if the end of this file is detected.
-     * @throws java.io.IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public final boolean readBoolean() throws IOException {
-        int temp = this.read();
-        if (temp < 0) {
-            throw new EOFException();
-        }
-        return temp != 0;
-    }
-
-    /**
-     * Reads an 8-bit byte from the current position in this file. Blocks until
-     * one byte has been read, the end of the file is reached or an exception is
-     * thrown.
-     *
-     * @return the next signed 8-bit byte value from this file.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public final byte readByte() throws IOException {
-        int temp = this.read();
-        if (temp < 0) {
-            throw new EOFException();
-        }
-        return (byte) temp;
-    }
-
-    /**
-     * Reads a 16-bit character from the current position in this file. Blocks until
-     * two bytes have been read, the end of the file is reached or an exception is
-     * thrown.
-     *
-     * @return the next char value from this file.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public final char readChar() throws IOException {
-        int ch1 = this.read();
-        int ch2 = this.read();
-        if ((ch1 | ch2) < 0)
-            throw new EOFException();
-        return (char)((ch1 << 8) + (ch2 << 0));
-    }
-
-    /**
-     * Reads a 64-bit double from the current position in this file. Blocks
-     * until eight bytes have been read, the end of the file is reached or an
-     * exception is thrown.
-     *
-     * @return the next double value from this file.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public final double readDouble() throws IOException {
-        return Double.longBitsToDouble(readLong());
-    }
-
-    /**
-     * Reads a 32-bit float from the current position in this file. Blocks
-     * until four bytes have been read, the end of the file is reached or an
-     * exception is thrown.
-     *
-     * @return the next float value from this file.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public final float readFloat() throws IOException {
-        return Float.intBitsToFloat(readInt());
-    }
-
-    /**
-     * Reads bytes from this file into {@code buffer}. Blocks until {@code
-     * buffer.length} number of bytes have been read, the end of the file is
-     * reached or an exception is thrown.
-     *
-     * @param buffer
-     *            the buffer to read bytes into.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     * @throws NullPointerException
-     *             if {@code buffer} is {@code null}.
-     */
-    public void readFully(byte[] buffer) throws IOException
-    {
-        readFully(buffer, 0, buffer.length);
-    }
-
-    /**
-     * Read bytes from this file into {@code buffer} starting at offset {@code
-     * offset}. This method blocks until {@code count} number of bytes have been
-     * read.
-     *
-     * @param buffer
-     *            the buffer to read bytes into.
-     * @param offset
-     *            the initial position in {@code buffer} to store the bytes read
-     *            from this file.
-     * @param count
-     *            the maximum number of bytes to store in {@code buffer}.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IndexOutOfBoundsException
-     *             if {@code offset < 0} or {@code count < 0}, or if {@code
-     *             offset + count} is greater than the length of {@code buffer}.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     * @throws NullPointerException
-     *             if {@code buffer} is {@code null}.
-     */
-    public void readFully(byte[] buffer, int offset, int count) throws IOException
-    {
-        if (buffer == null) {
-            throw new NullPointerException();
-        }
-        // avoid int overflow
-        if (offset < 0 || offset > buffer.length || count < 0
-                || count > buffer.length - offset) {
-            throw new IndexOutOfBoundsException();
-        }
-        while (count > 0) {
-            int result = read(buffer, offset, count);
-            if (result < 0) {
-                throw new EOFException();
-            }
-            offset += result;
-            count -= result;
-        }
-    }
-
-    /**
-     * Reads a 32-bit integer from the current position in this file. Blocks
-     * until four bytes have been read, the end of the file is reached or an
-     * exception is thrown.
-     *
-     * @return the next int value from this file.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public int readInt() throws IOException {
-        int ch1 = this.read();
-        int ch2 = this.read();
-        int ch3 = this.read();
-        int ch4 = this.read();
-        if ((ch1 | ch2 | ch3 | ch4) < 0)
-            throw new EOFException();
-        return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0));
-    }
-
-    /**
-     * Reads a line of text form the current position in this file. A line is
-     * represented by zero or more characters followed by {@code '\n'}, {@code
-     * '\r'}, {@code "\r\n"} or the end of file marker. The string does not
-     * include the line terminating sequence.
-     * <p>
-     * Blocks until a line terminating sequence has been read, the end of the
-     * file is reached or an exception is thrown.
-     *
-     * @return the contents of the line or {@code null} if no characters have
-     *         been read before the end of the file has been reached.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public final String readLine() throws IOException {
-        StringBuilder line = new StringBuilder(80); // Typical line length
-        boolean foundTerminator = false;
-        long unreadPosition = -1;
-        while (true) {
-            int nextByte = read();
-            switch (nextByte) {
-                case -1:
-                    return line.length() != 0 ? line.toString() : null;
-                case (byte) '\r':
-                    if (foundTerminator) {
-                        seek(unreadPosition);
-                        return line.toString();
-                    }
-                    foundTerminator = true;
-                    /* Have to be able to peek ahead one byte */
-                    unreadPosition = getPosition();
-                    break;
-                case (byte) '\n':
-                    return line.toString();
-                default:
-                    if (foundTerminator) {
-                        seek(unreadPosition);
-                        return line.toString();
-                    }
-                    line.append((char) nextByte);
-            }
-        }
-    }
-
-    /**
-     * Reads a 64-bit long from the current position in this file. Blocks until
-     * eight bytes have been read, the end of the file is reached or an
-     * exception is thrown.
-     *
-     * @return the next long value from this file.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public long readLong() throws IOException {
-        return ((long)(readInt()) << 32) + (readInt() & 0xFFFFFFFFL);
-    }
-
-    /**
-     * Reads a 16-bit short from the current position in this file. Blocks until
-     * two bytes have been read, the end of the file is reached or an exception
-     * is thrown.
-     *
-     * @return the next short value from this file.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public short readShort() throws IOException {
-        int ch1 = this.read();
-        int ch2 = this.read();
-        if ((ch1 | ch2) < 0)
-            throw new EOFException();
-        return (short)((ch1 << 8) + (ch2 << 0));
-    }
-
-    /**
-     * Reads an unsigned 8-bit byte from the current position in this file and
-     * returns it as an integer. Blocks until one byte has been read, the end of
-     * the file is reached or an exception is thrown.
-     *
-     * @return the next unsigned byte value from this file as an int.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public final int readUnsignedByte() throws IOException {
-        int temp = this.read();
-        if (temp < 0) {
-            throw new EOFException();
-        }
-        return temp;
-    }
-
-    /**
-     * Reads an unsigned 16-bit short from the current position in this file and
-     * returns it as an integer. Blocks until two bytes have been read, the end of
-     * the file is reached or an exception is thrown.
-     *
-     * @return the next unsigned short value from this file as an int.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     */
-    public int readUnsignedShort() throws IOException {
-        int ch1 = this.read();
-        int ch2 = this.read();
-        if ((ch1 | ch2) < 0)
-            throw new EOFException();
-        return (ch1 << 8) + (ch2 << 0);
-    }
-
-    /**
-     * Reads a string that is encoded in {@link java.io.DataInput modified UTF-8} from
-     * this file. The number of bytes that must be read for the complete string
-     * is determined by the first two bytes read from the file. Blocks until all
-     * required bytes have been read, the end of the file is reached or an
-     * exception is thrown.
-     *
-     * @return the next string encoded in {@link java.io.DataInput modified UTF-8} from
-     *         this file.
-     * @throws EOFException
-     *             if the end of this file is detected.
-     * @throws IOException
-     *             if this file is closed or another I/O error occurs.
-     * @throws java.io.UTFDataFormatException
-     *             if the bytes read cannot be decoded into a character string.
-     */
-    public final String readUTF() throws IOException {
-        return DataInputStream.readUTF(this);
-    }
-}

src/java/org/apache/cassandra/io/util/BufferedSegmentedFile.java

@@ -29,25 +29,8 @@ public class BufferedSegmentedFile extends SegmentedFile
         super(copy);
     }
 
-    private static class Cleanup extends SegmentedFile.Cleanup
-    {
-        protected Cleanup(ChannelProxy channel)
-        {
-            super(channel);
-        }
-        public void tidy()
-        {
-            super.tidy();
-        }
-    }
-
     public static class Builder extends SegmentedFile.Builder
     {
-        public void addPotentialBoundary(long boundary)
-        {
-            // only one segment in a standard-io file
-        }
-
         public SegmentedFile complete(ChannelProxy channel, int bufferSize, long overrideLength)
         {
             long length = overrideLength > 0 ? overrideLength : channel.size();
@@ -55,13 +38,6 @@ public class BufferedSegmentedFile extends SegmentedFile
         }
     }
 
-    public FileDataInput getSegment(long position)
-    {
-        RandomAccessReader reader = RandomAccessReader.open(channel, bufferSize, -1L);
-        reader.seek(position);
-        return reader;
-    }
-
     public BufferedSegmentedFile sharedCopy()
     {
         return new BufferedSegmentedFile(this);

src/java/org/apache/cassandra/io/util/ByteBufferDataInput.java

@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.util;
-
-import java.io.*;
-import java.nio.ByteBuffer;
-
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-public class ByteBufferDataInput extends AbstractDataInput implements FileDataInput, DataInput
-{
-    private final ByteBuffer buffer;
-    private final String filename;
-    private final long segmentOffset;
-    private int position;
-
-    public ByteBufferDataInput(ByteBuffer buffer, String filename, long segmentOffset, int position)
-    {
-        assert buffer != null;
-        this.buffer = buffer;
-        this.filename = filename;
-        this.segmentOffset = segmentOffset;
-        this.position = position;
-    }
-
-    // Only use when we know the seek in within the mapped segment. Throws an
-    // IOException otherwise.
-    public void seek(long pos) throws IOException
-    {
-        long inSegmentPos = pos - segmentOffset;
-        if (inSegmentPos < 0 || inSegmentPos > buffer.capacity())
-            throw new IOException(String.format("Seek position %d is not within mmap segment (seg offs: %d, length: %d)", pos, segmentOffset, buffer.capacity()));
-
-        position = (int) inSegmentPos;
-    }
-
-    public long getFilePointer()
-    {
-        return segmentOffset + position;
-    }
-
-    public long getPosition()
-    {
-        return segmentOffset + position;
-    }
-
-    public long getPositionLimit()
-    {
-        return segmentOffset + buffer.capacity();
-    }
-
-    @Override
-    public boolean markSupported()
-    {
-        return false;
-    }
-
-    public void reset(FileMark mark) throws IOException
-    {
-        assert mark instanceof MappedFileDataInputMark;
-        position = ((MappedFileDataInputMark) mark).position;
-    }
-
-    public FileMark mark()
-    {
-        return new MappedFileDataInputMark(position);
-    }
-
-    public long bytesPastMark(FileMark mark)
-    {
-        assert mark instanceof MappedFileDataInputMark;
-        assert position >= ((MappedFileDataInputMark) mark).position;
-        return position - ((MappedFileDataInputMark) mark).position;
-    }
-
-    public boolean isEOF() throws IOException
-    {
-        return position == buffer.capacity();
-    }
-
-    public long bytesRemaining() throws IOException
-    {
-        return buffer.capacity() - position;
-    }
-
-    public String getPath()
-    {
-        return filename;
-    }
-
-    public int read() throws IOException
-    {
-        if (isEOF())
-            return -1;
-        return buffer.get(position++) & 0xFF;
-    }
-
-    /**
-     * Does the same thing as <code>readFully</code> do but without copying data (thread safe)
-     * @param length length of the bytes to read
-     * @return buffer with portion of file content
-     * @throws IOException on any fail of I/O operation
-     */
-    public ByteBuffer readBytes(int length) throws IOException
-    {
-        int remaining = buffer.remaining() - position;
-        if (length > remaining)
-            throw new IOException(String.format("mmap segment underflow; remaining is %d but %d requested",
-                                                remaining, length));
-
-        if (length == 0)
-            return ByteBufferUtil.EMPTY_BYTE_BUFFER;
-
-        ByteBuffer bytes = buffer.duplicate();
-        bytes.position(buffer.position() + position).limit(buffer.position() + position + length);
-        position += length;
-
-        // we have to copy the data in case we unreference the underlying sstable.  See CASSANDRA-3179
-        ByteBuffer clone = ByteBuffer.allocate(bytes.remaining());
-        clone.put(bytes);
-        clone.flip();
-        return clone;
-    }
-
-    @Override
-    public final void readFully(byte[] bytes) throws IOException
-    {
-        ByteBufferUtil.arrayCopy(buffer, buffer.position() + position, bytes, 0, bytes.length);
-        position += bytes.length;
-    }
-
-    @Override
-    public final void readFully(byte[] bytes, int offset, int count) throws IOException
-    {
-        ByteBufferUtil.arrayCopy(buffer, buffer.position() + position, bytes, offset, count);
-        position += count;
-    }
-
-    private static class MappedFileDataInputMark implements FileMark
-    {
-        int position;
-
-        MappedFileDataInputMark(int position)
-        {
-            this.position = position;
-        }
-    }
-
-    @Override
-    public String toString() {
-        return getClass().getSimpleName() + "(" +
-               "filename='" + filename + "'" +
-               ", position=" + position +
-               ")";
-    }
-}

src/java/org/apache/cassandra/io/util/ChannelProxy.java

@@ -63,7 +63,7 @@ public final class ChannelProxy extends SharedCloseableImpl
 
     public ChannelProxy(File file)
     {
-        this(file.getAbsolutePath(), openChannel(file));
+        this(file.getPath(), openChannel(file));
     }
 
     public ChannelProxy(String filePath, FileChannel channel)
@@ -87,7 +87,7 @@ public final class ChannelProxy extends SharedCloseableImpl
         final String filePath;
         final FileChannel channel;
 
-        protected Cleanup(String filePath, FileChannel channel)
+        Cleanup(String filePath, FileChannel channel)
         {
             this.filePath = filePath;
             this.channel = channel;

src/java/org/apache/cassandra/io/util/ChecksummedRandomAccessReader.java

@@ -23,43 +23,33 @@ import java.util.zip.CRC32;
 
 import org.apache.cassandra.io.compress.BufferType;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Throwables;
 
 public class ChecksummedRandomAccessReader extends RandomAccessReader
 {
     @SuppressWarnings("serial")
     public static class CorruptFileException extends RuntimeException
     {
-        public final File file;
+        public final String filePath;
 
-        public CorruptFileException(Exception cause, File file)
+        public CorruptFileException(Exception cause, String filePath)
         {
             super(cause);
-            this.file = file;
+            this.filePath = filePath;
         }
     }
 
     private final DataIntegrityMetadata.ChecksumValidator validator;
-    private final File file;
 
-    protected ChecksummedRandomAccessReader(File file, ChannelProxy channel, DataIntegrityMetadata.ChecksumValidator validator)
+    private ChecksummedRandomAccessReader(Builder builder)
     {
-        super(channel, validator.chunkSize, -1, BufferType.ON_HEAP);
-        this.validator = validator;
-        this.file = file;
+        super(builder);
+        this.validator = builder.validator;
     }
 
     @SuppressWarnings("resource")
-    public static ChecksummedRandomAccessReader open(File file, File crcFile) throws IOException
-    {
-        ChannelProxy channel = new ChannelProxy(file);
-        RandomAccessReader crcReader = RandomAccessReader.open(crcFile);
-        DataIntegrityMetadata.ChecksumValidator validator =
-            new DataIntegrityMetadata.ChecksumValidator(new CRC32(), crcReader, file.getPath());
-        return new ChecksummedRandomAccessReader(file, channel, validator);
-    }
-
     @Override
-    protected void reBuffer()
+    protected void reBufferStandard()
     {
         long desiredPosition = current();
         // align with buffer size, as checksums were computed in chunks of buffer size each.
@@ -84,12 +74,18 @@ public class ChecksummedRandomAccessReader extends RandomAccessReader
         }
         catch (IOException e)
         {
-            throw new CorruptFileException(e, file);
+            throw new CorruptFileException(e, channel.filePath());
         }
 
         buffer.position((int) (desiredPosition - bufferOffset));
     }
 
+    @Override
+    protected void reBufferMmap()
+    {
+        throw new AssertionError("Unsupported operation");
+    }
+
     @Override
     public void seek(long newPosition)
     {
@@ -100,14 +96,32 @@ public class ChecksummedRandomAccessReader extends RandomAccessReader
     @Override
     public void close()
     {
-        try
+        Throwables.perform(channel.filePath(), Throwables.FileOpType.READ,
+                           super::close,
+                           validator::close,
+                           channel::close);
+    }
+
+    public static final class Builder extends RandomAccessReader.Builder
+    {
+        private final DataIntegrityMetadata.ChecksumValidator validator;
+
+        @SuppressWarnings("resource")
+        public Builder(File file, File crcFile) throws IOException
         {
-            super.close();
+            super(new ChannelProxy(file));
+            this.validator = new DataIntegrityMetadata.ChecksumValidator(new CRC32(),
+                                                                         RandomAccessReader.open(crcFile),
+                                                                         file.getPath());
+
+            super.bufferSize(validator.chunkSize)
+                 .bufferType(BufferType.ON_HEAP);
         }
-        finally
+
+        @Override
+        public RandomAccessReader build()
         {
-            channel.close();
-            validator.close();
+            return new ChecksummedRandomAccessReader(this);
         }
     }
 }

src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java

@@ -17,44 +17,49 @@
  */
 package org.apache.cassandra.io.util;
 
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.TreeMap;
-
 import com.google.common.util.concurrent.RateLimiter;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.compress.CompressedRandomAccessReader;
 import org.apache.cassandra.io.compress.CompressedSequentialWriter;
-import org.apache.cassandra.io.compress.CompressedThrottledReader;
 import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.concurrent.Ref;
 
 public class CompressedSegmentedFile extends SegmentedFile implements ICompressedFile
 {
-    public final CompressionMetadata metadata;
+    private static final Logger logger = LoggerFactory.getLogger(CompressedSegmentedFile.class);
     private static final boolean useMmap = DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap;
-    private static int MAX_SEGMENT_SIZE = Integer.MAX_VALUE;
-    private final TreeMap<Long, MappedByteBuffer> chunkSegments;
+
+    public final CompressionMetadata metadata;
+    private final MmappedRegions regions;
 
     public CompressedSegmentedFile(ChannelProxy channel, int bufferSize, CompressionMetadata metadata)
     {
-        this(channel, bufferSize, metadata, createMappedSegments(channel, metadata));
+        this(channel,
+             bufferSize,
+             metadata,
+             useMmap
+             ? MmappedRegions.map(channel, metadata)
+             : null);
     }
 
-    public CompressedSegmentedFile(ChannelProxy channel, int bufferSize, CompressionMetadata metadata, TreeMap<Long, MappedByteBuffer> chunkSegments)
+    public CompressedSegmentedFile(ChannelProxy channel, int bufferSize, CompressionMetadata metadata, MmappedRegions regions)
     {
-        super(new Cleanup(channel, metadata, chunkSegments), channel, bufferSize, metadata.dataLength, metadata.compressedFileLength);
+        super(new Cleanup(channel, metadata, regions), channel, bufferSize, metadata.dataLength, metadata.compressedFileLength);
         this.metadata = metadata;
-        this.chunkSegments = chunkSegments;
+        this.regions = regions;
     }
 
     private CompressedSegmentedFile(CompressedSegmentedFile copy)
     {
         super(copy);
         this.metadata = copy.metadata;
-        this.chunkSegments = copy.chunkSegments;
+        this.regions = copy.regions;
     }
 
     public ChannelProxy channel()
@@ -62,60 +67,36 @@ public class CompressedSegmentedFile extends SegmentedFile implements ICompresse
         return channel;
     }
 
-    public TreeMap<Long, MappedByteBuffer> chunkSegments()
+    public MmappedRegions regions()
     {
-        return chunkSegments;
-    }
-
-    static TreeMap<Long, MappedByteBuffer> createMappedSegments(ChannelProxy channel, CompressionMetadata metadata)
-    {
-        if (!useMmap)
-            return null;
-        TreeMap<Long, MappedByteBuffer> chunkSegments = new TreeMap<>();
-        long offset = 0;
-        long lastSegmentOffset = 0;
-        long segmentSize = 0;
-
-        while (offset < metadata.dataLength)
-        {
-            CompressionMetadata.Chunk chunk = metadata.chunkFor(offset);
-
-            //Reached a new mmap boundary
-            if (segmentSize + chunk.length + 4 > MAX_SEGMENT_SIZE)
-            {
-                chunkSegments.put(lastSegmentOffset, channel.map(FileChannel.MapMode.READ_ONLY, lastSegmentOffset, segmentSize));
-                lastSegmentOffset += segmentSize;
-                segmentSize = 0;
-            }
-
-            segmentSize += chunk.length + 4; //checksum
-            offset += metadata.chunkLength();
-        }
-
-        if (segmentSize > 0)
-            chunkSegments.put(lastSegmentOffset, channel.map(FileChannel.MapMode.READ_ONLY, lastSegmentOffset, segmentSize));
-        return chunkSegments;
+        return regions;
     }
 
     private static final class Cleanup extends SegmentedFile.Cleanup
     {
         final CompressionMetadata metadata;
-        final TreeMap<Long, MappedByteBuffer> chunkSegments;
-        protected Cleanup(ChannelProxy channel, CompressionMetadata metadata, TreeMap<Long, MappedByteBuffer> chunkSegments)
+        private final MmappedRegions regions;
+
+        protected Cleanup(ChannelProxy channel, CompressionMetadata metadata, MmappedRegions regions)
         {
             super(channel);
             this.metadata = metadata;
-            this.chunkSegments = chunkSegments;
+            this.regions = regions;
         }
         public void tidy()
         {
-            super.tidy();
-            metadata.close();
-            if (chunkSegments != null)
+            Throwable err = regions == null ? null : regions.close(null);
+            if (err != null)
             {
-                for (MappedByteBuffer segment : chunkSegments.values())
-                    FileUtils.clean(segment);
+                JVMStabilityInspector.inspectThrowable(err);
+
+                // This is not supposed to happen
+                logger.error("Error while closing mmapped regions", err);
             }
+
+            metadata.close();
+
+            super.tidy();
         }
     }
 
@@ -132,17 +113,12 @@ public class CompressedSegmentedFile extends SegmentedFile implements ICompresse
 
     public static class Builder extends SegmentedFile.Builder
     {
-        protected final CompressedSequentialWriter writer;
+        final CompressedSequentialWriter writer;
         public Builder(CompressedSequentialWriter writer)
         {
             this.writer = writer;
         }
 
-        public void addPotentialBoundary(long boundary)
-        {
-            // only one segment in a standard-io file
-        }
-
         protected CompressionMetadata metadata(String path, long overrideLength)
         {
             if (writer == null)
@@ -166,12 +142,12 @@ public class CompressedSegmentedFile extends SegmentedFile implements ICompresse
 
     public RandomAccessReader createReader()
     {
-        return CompressedRandomAccessReader.open(this);
+        return new CompressedRandomAccessReader.Builder(this).build();
     }
 
-    public RandomAccessReader createThrottledReader(RateLimiter limiter)
+    public RandomAccessReader createReader(RateLimiter limiter)
     {
-        return CompressedThrottledReader.open(this, limiter);
+        return new CompressedRandomAccessReader.Builder(this).limiter(limiter).build();
     }
 
     public CompressionMetadata getMetadata()

src/java/org/apache/cassandra/io/util/DataInputBuffer.java

@@ -21,13 +21,10 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
- * Input stream around a fixed ByteBuffer. Necessary to have this derived class to avoid NIODataInputStream's
- * shuffling of bytes behavior in readNext()
- *
+ * Input stream around a single ByteBuffer.
  */
-public class DataInputBuffer extends NIODataInputStream
+public class DataInputBuffer extends RebufferingInputStream
 {
-
     private static ByteBuffer slice(byte[] buffer, int offset, int length)
     {
         ByteBuffer buf = ByteBuffer.wrap(buffer);
@@ -41,13 +38,12 @@ public class DataInputBuffer extends NIODataInputStream
     }
 
     /**
-     *
-     * @param buf
+     * @param buffer
      * @param duplicate Whether or not to duplicate the buffer to ensure thread safety
      */
-    public DataInputBuffer(ByteBuffer buf, boolean duplicate)
+    public DataInputBuffer(ByteBuffer buffer, boolean duplicate)
     {
-        super(buf, duplicate);
+        super(duplicate ? buffer.duplicate() : buffer);
     }
 
     public DataInputBuffer(byte[] buffer, int offset, int length)
@@ -61,8 +57,14 @@ public class DataInputBuffer extends NIODataInputStream
     }
 
     @Override
-    protected int readNext() throws IOException
+    protected void reBuffer() throws IOException
     {
-        return -1;
+        //nope, we don't rebuffer, we are done!
+    }
+
+    @Override
+    public int available() throws IOException
+    {
+        return buffer.remaining();
     }
 }

src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java

@@ -34,6 +34,7 @@ import com.google.common.base.Charsets;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.utils.Throwables;
 
 public class DataIntegrityMetadata
 {
@@ -138,14 +139,8 @@ public class DataIntegrityMetadata
 
         public void close()
         {
-            try
-            {
-                this.digestReader.close();
-            }
-            finally
-            {
-                this.dataReader.close();
-            }
+            Throwables.perform(digestReader::close,
+                               dataReader::close);
         }
     }
 

src/java/org/apache/cassandra/io/util/FileDataInput.java

@@ -19,31 +19,22 @@ package org.apache.cassandra.io.util;
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.nio.ByteBuffer;
 
 public interface FileDataInput extends DataInputPlus, Closeable
 {
-    public String getPath();
+    String getPath();
 
-    public boolean isEOF() throws IOException;
+    boolean isEOF() throws IOException;
 
-    public long bytesRemaining() throws IOException;
+    long bytesRemaining() throws IOException;
 
-    public void seek(long pos) throws IOException;
+    void seek(long pos) throws IOException;
 
-    public FileMark mark();
+    FileMark mark();
 
-    public void reset(FileMark mark) throws IOException;
+    void reset(FileMark mark) throws IOException;
 
-    public long bytesPastMark(FileMark mark);
+    long bytesPastMark(FileMark mark);
 
-    public long getFilePointer();
-
-    /**
-     * Read length bytes from current file position
-     * @param length length of the bytes to read
-     * @return buffer with bytes read
-     * @throws IOException if any I/O operation failed
-     */
-    public ByteBuffer readBytes(int length) throws IOException;
+    long getFilePointer();
 }

src/java/org/apache/cassandra/io/util/FileSegmentInputStream.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This is the same as DataInputBuffer, i.e. a stream for a fixed byte buffer,
+ * except that we also implement FileDataInput by using an offset and a file path.
+ */
+public class FileSegmentInputStream extends DataInputBuffer implements FileDataInput
+{
+    private final String filePath;
+    private final long offset;
+
+    public FileSegmentInputStream(ByteBuffer buffer, String filePath, long offset)
+    {
+        super(buffer, false);
+        this.filePath = filePath;
+        this.offset = offset;
+    }
+
+    public String getPath()
+    {
+        return filePath;
+    }
+
+    private long size()
+    {
+        return offset + buffer.capacity();
+    }
+
+    public boolean isEOF()
+    {
+        return !buffer.hasRemaining();
+    }
+
+    public long bytesRemaining()
+    {
+        return buffer.remaining();
+    }
+
+    public void seek(long pos)
+    {
+        if (pos < 0 || pos > size())
+            throw new IllegalArgumentException(String.format("Unable to seek to position %d in %s (%d bytes) in partial mode",
+                                                             pos,
+                                                             getPath(),
+                                                             size()));
+
+
+        buffer.position((int) (pos - offset));
+    }
+
+    @Override
+    public boolean markSupported()
+    {
+        return false;
+    }
+
+    public FileMark mark()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    public void reset(FileMark mark)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    public long bytesPastMark(FileMark mark)
+    {
+        return 0;
+    }
+
+    public long getFilePointer()
+    {
+        return offset + buffer.position();
+    }
+}

src/java/org/apache/cassandra/io/util/ICompressedFile.java

@@ -17,14 +17,11 @@
  */
 package org.apache.cassandra.io.util;
 
-import java.nio.MappedByteBuffer;
-import java.util.TreeMap;
-
 import org.apache.cassandra.io.compress.CompressionMetadata;
 
 public interface ICompressedFile
 {
-    public ChannelProxy channel();
-    public CompressionMetadata getMetadata();
-    public TreeMap<Long, MappedByteBuffer> chunkSegments();
+    ChannelProxy channel();
+    CompressionMetadata getMetadata();
+    MmappedRegions regions();
 }

src/java/org/apache/cassandra/io/util/MemoryInputStream.java

@@ -19,50 +19,58 @@ package org.apache.cassandra.io.util;
 
 import java.io.DataInput;
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 
-public class MemoryInputStream extends AbstractDataInput implements DataInput
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+public class MemoryInputStream extends RebufferingInputStream implements DataInput
 {
     private final Memory mem;
-    private int position = 0;
+    private final int bufferSize;
+    private long offset;
+
 
     public MemoryInputStream(Memory mem)
     {
+        this(mem, Ints.saturatedCast(mem.size));
+    }
+
+    @VisibleForTesting
+    public MemoryInputStream(Memory mem, int bufferSize)
+    {
+        super(getByteBuffer(mem.peer, bufferSize));
         this.mem = mem;
+        this.bufferSize = bufferSize;
+        this.offset = mem.peer + bufferSize;
     }
 
-    public int read() throws IOException
+    @Override
+    protected void reBuffer() throws IOException
     {
-        return mem.getByte(position++) & 0xFF;
+        if (offset - mem.peer >= mem.size())
+            return;
+
+        buffer = getByteBuffer(offset, Math.min(bufferSize, Ints.saturatedCast(memRemaining())));
+        offset += buffer.capacity();
     }
 
-    public void readFully(byte[] buffer, int offset, int count) throws IOException
+    @Override
+    public int available()
     {
-        mem.getBytes(position, buffer, offset, count);
-        position += count;
+        return Ints.saturatedCast(buffer.remaining() + memRemaining());
     }
 
-    public void seek(long pos)
+    private long memRemaining()
     {
-        position = (int) pos;
+        return mem.size + mem.peer - offset;
     }
 
-    public long getPosition()
+    private static ByteBuffer getByteBuffer(long offset, int length)
     {
-        return position;
-    }
-
-    public long getPositionLimit()
-    {
-        return mem.size();
-    }
-
-    protected long length()
-    {
-        return mem.size();
-    }
-
-    public void close()
-    {
-        // do nothing.
+        return MemoryUtil.getByteBuffer(offset, length).order(ByteOrder.BIG_ENDIAN);
     }
 }

src/java/org/apache/cassandra/io/util/MmappedRegions.java

@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Arrays;
+
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.concurrent.RefCounted;
+import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
+
+import static java.util.stream.Stream.of;
+import static org.apache.cassandra.utils.Throwables.perform;
+
+public class MmappedRegions extends SharedCloseableImpl
+{
+    /** In a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size */
+    public static int MAX_SEGMENT_SIZE = Integer.MAX_VALUE;
+
+    /** When we need to grow the arrays, we add this number of region slots */
+    static final int REGION_ALLOC_SIZE = 15;
+
+    /** The original state, which is shared with the tidier and
+     * contains all the regions mapped so far. It also
+     * does the actual mapping. */
+    private final State state;
+
+    /** A copy of the latest state. We update this each time the original state is
+     * updated and we share this with copies. If we are a copy, then this
+     * is null. Copies can only access existing regions, they cannot create
+     * new ones. This is for thread safety and because MmappedRegions is
+     * reference counted, only the original state will be cleaned-up,
+     * therefore only the original state should create new mapped regions.
+     */
+    private volatile State copy;
+
+    private MmappedRegions(ChannelProxy channel, CompressionMetadata metadata, long length)
+    {
+        this(new State(channel), metadata, length);
+    }
+
+    private MmappedRegions(State state, CompressionMetadata metadata, long length)
+    {
+        super(new Tidier(state));
+
+        this.state = state;
+
+        if (metadata != null)
+        {
+            assert length == 0 : "expected no length with metadata";
+            updateState(metadata);
+        }
+        else if (length > 0)
+        {
+            updateState(length);
+        }
+
+        this.copy = new State(state);
+    }
+
+    private MmappedRegions(MmappedRegions original)
+    {
+        super(original);
+        this.state = original.copy;
+    }
+
+    public static MmappedRegions empty(ChannelProxy channel)
+    {
+        return new MmappedRegions(channel, null, 0);
+    }
+
+    public static MmappedRegions map(ChannelProxy channel, CompressionMetadata metadata)
+    {
+        if (metadata == null)
+            throw new IllegalArgumentException("metadata cannot be null");
+
+        return new MmappedRegions(channel, metadata, 0);
+    }
+
+    public static MmappedRegions map(ChannelProxy channel, long length)
+    {
+        if (length <= 0)
+            throw new IllegalArgumentException("Length must be positive");
+
+        return new MmappedRegions(channel, null, length);
+    }
+
+    /**
+     * @return a snapshot of the memory mapped regions. The snapshot can
+     * only use existing regions, it cannot create new ones.
+     */
+    public MmappedRegions sharedCopy()
+    {
+        return new MmappedRegions(this);
+    }
+
+    private boolean isCopy()
+    {
+        return copy == null;
+    }
+
+    public void extend(long length)
+    {
+        if (length < 0)
+            throw new IllegalArgumentException("Length must not be negative");
+
+        assert !isCopy() : "Copies cannot be extended";
+
+        if (length <= state.length)
+            return;
+
+        updateState(length);
+        copy = new State(state);
+    }
+
+    private void updateState(long length)
+    {
+        state.length = length;
+        long pos = state.getPosition();
+        while (pos < length)
+        {
+            long size = Math.min(MAX_SEGMENT_SIZE, length - pos);
+            state.add(pos, size);
+            pos += size;
+        }
+    }
+
+    private void updateState(CompressionMetadata metadata)
+    {
+        long offset = 0;
+        long lastSegmentOffset = 0;
+        long segmentSize = 0;
+
+        while (offset < metadata.dataLength)
+        {
+            CompressionMetadata.Chunk chunk = metadata.chunkFor(offset);
+
+            //Reached a new mmap boundary
+            if (segmentSize + chunk.length + 4 > MAX_SEGMENT_SIZE)
+            {
+                if (segmentSize > 0)
+                {
+                    state.add(lastSegmentOffset, segmentSize);
+                    lastSegmentOffset += segmentSize;
+                    segmentSize = 0;
+                }
+            }
+
+            segmentSize += chunk.length + 4; //checksum
+            offset += metadata.chunkLength();
+        }
+
+        if (segmentSize > 0)
+            state.add(lastSegmentOffset, segmentSize);
+
+        state.length = lastSegmentOffset + segmentSize;
+    }
+
+    public boolean isValid(ChannelProxy channel)
+    {
+        return state.isValid(channel);
+    }
+
+    public boolean isEmpty()
+    {
+        return state.isEmpty();
+    }
+
+    public Region floor(long position)
+    {
+        assert !isCleanedUp() : "Attempted to use closed region";
+        return state.floor(position);
+    }
+
+    public static final class Region
+    {
+        public final long offset;
+        public final ByteBuffer buffer;
+
+        public Region(long offset, ByteBuffer buffer)
+        {
+            this.offset = offset;
+            this.buffer = buffer;
+        }
+
+        public long bottom()
+        {
+            return offset;
+        }
+
+        public long top()
+        {
+            return offset + buffer.capacity();
+        }
+    }
+
+    private static final class State
+    {
+        /** The file channel */
+        private final ChannelProxy channel;
+
+        /** An array of region buffers, synchronized with offsets */
+        private ByteBuffer[] buffers;
+
+        /** An array of region offsets, synchronized with buffers */
+        private long[] offsets;
+
+        /** The maximum file length we have mapped */
+        private long length;
+
+        /** The index to the last region added */
+        private int last;
+
+        private State(ChannelProxy channel)
+        {
+            this.channel = channel.sharedCopy();
+            this.buffers = new ByteBuffer[REGION_ALLOC_SIZE];
+            this.offsets = new long[REGION_ALLOC_SIZE];
+            this.length = 0;
+            this.last = -1;
+        }
+
+        private State(State original)
+        {
+            this.channel = original.channel;
+            this.buffers = original.buffers;
+            this.offsets = original.offsets;
+            this.length = original.length;
+            this.last = original.last;
+        }
+
+        private boolean isEmpty()
+        {
+            return last < 0;
+        }
+
+        private boolean isValid(ChannelProxy channel)
+        {
+            return this.channel.filePath().equals(channel.filePath());
+        }
+
+        private Region floor(long position)
+        {
+            assert 0 <= position && position < length : String.format("%d >= %d", position, length);
+
+            int idx = Arrays.binarySearch(offsets, 0, last +1, position);
+            assert idx != -1 : String.format("Bad position %d for regions %s, last %d in %s", position, Arrays.toString(offsets), last, channel);
+            if (idx < 0)
+                idx = -(idx + 2); // round down to entry at insertion point
+
+            return new Region(offsets[idx], buffers[idx]);
+        }
+
+        private long getPosition()
+        {
+            return last < 0 ? 0 : offsets[last] + buffers[last].capacity();
+        }
+
+        private void add(long pos, long size)
+        {
+            ByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, pos, size);
+
+            ++last;
+
+            if (last == offsets.length)
+            {
+                offsets = Arrays.copyOf(offsets, offsets.length + REGION_ALLOC_SIZE);
+                buffers = Arrays.copyOf(buffers, buffers.length + REGION_ALLOC_SIZE);
+            }
+
+            offsets[last] = pos;
+            buffers[last] = buffer;
+        }
+
+        private Throwable close(Throwable accumulate)
+        {
+            accumulate = channel.close(accumulate);
+
+            /*
+             * Try forcing the unmapping of segments using undocumented unsafe sun APIs.
+             * If this fails (non Sun JVM), we'll have to wait for the GC to finalize the mapping.
+             * If this works and a thread tries to access any segment, hell will unleash on earth.
+             */
+            if (!FileUtils.isCleanerAvailable())
+                return accumulate;
+
+            return perform(accumulate, channel.filePath(), Throwables.FileOpType.READ,
+                           of(buffers)
+                           .map((buffer) ->
+                                () ->
+                                {
+                                    if (buffer != null)
+                                        FileUtils.clean(buffer);
+                                }));
+        }
+    }
+
+    public static final class Tidier implements RefCounted.Tidy
+    {
+        final State state;
+
+        Tidier(State state)
+        {
+            this.state = state;
+        }
+
+        public String name()
+        {
+            return state.channel.filePath();
+        }
+
+        public void tidy()
+        {
+            try
+            {
+                Throwables.maybeFail(state.close(null));
+            }
+            catch (Exception e)
+            {
+                throw new FSReadError(e, state.channel.filePath());
+            }
+        }
+    }
+
+}

src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java

@@ -18,41 +18,31 @@
 package org.apache.cassandra.io.util;
 
 import java.io.*;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
 
+import com.google.common.util.concurrent.RateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 
 public class MmappedSegmentedFile extends SegmentedFile
 {
     private static final Logger logger = LoggerFactory.getLogger(MmappedSegmentedFile.class);
 
-    // in a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size to stay sane.
-    public static long MAX_SEGMENT_SIZE = Integer.MAX_VALUE;
+    private final MmappedRegions regions;
 
-    /**
-     * Sorted array of segment offsets and MappedByteBuffers for segments. If mmap is completely disabled, or if the
-     * segment would be too long to mmap, the value for an offset will be null, indicating that we need to fall back
-     * to a RandomAccessFile.
-     */
-    private final Segment[] segments;
-
-    public MmappedSegmentedFile(ChannelProxy channel, int bufferSize, long length, Segment[] segments)
+    public MmappedSegmentedFile(ChannelProxy channel, int bufferSize, long length, MmappedRegions regions)
     {
-        super(new Cleanup(channel, segments), channel, bufferSize, length);
-        this.segments = segments;
+        super(new Cleanup(channel, regions), channel, bufferSize, length);
+        this.regions = regions;
     }
 
     private MmappedSegmentedFile(MmappedSegmentedFile copy)
     {
         super(copy);
-        this.segments = copy.segments;
+        this.regions = copy.regions;
     }
 
     public MmappedSegmentedFile sharedCopy()
@@ -60,78 +50,46 @@ public class MmappedSegmentedFile extends SegmentedFile
         return new MmappedSegmentedFile(this);
     }
 
-    /**
-     * @return The segment entry for the given position.
-     */
-    private Segment floor(long position)
+    public RandomAccessReader createReader()
     {
-        assert 0 <= position && position < length: String.format("%d >= %d in %s", position, length, path());
-        Segment seg = new Segment(position, null);
-        int idx = Arrays.binarySearch(segments, seg);
-        assert idx != -1 : String.format("Bad position %d for segments %s in %s", position, Arrays.toString(segments), path());
-        if (idx < 0)
-            // round down to entry at insertion point
-            idx = -(idx + 2);
-        return segments[idx];
+        return new RandomAccessReader.Builder(channel)
+               .overrideLength(length)
+               .regions(regions)
+               .build();
     }
 
-    /**
-     * @return The segment containing the given position: must be closed after use.
-     */
-    public FileDataInput getSegment(long position)
+    public RandomAccessReader createReader(RateLimiter limiter)
     {
-        Segment segment = floor(position);
-        if (segment.right != null)
-        {
-            // segment is mmap'd
-            return new ByteBufferDataInput(segment.right, path(), segment.left, (int) (position - segment.left));
-        }
-
-        // we can have single cells or partitions larger than 2Gb, which is our maximum addressable range in a single segment;
-        // in this case we open as a normal random access reader
-        // FIXME: brafs are unbounded, so this segment will cover the rest of the file, rather than just the row
-        RandomAccessReader file = RandomAccessReader.open(channel, bufferSize, -1L);
-        file.seek(position);
-        return file;
+        return new RandomAccessReader.Builder(channel)
+               .overrideLength(length)
+               .bufferSize(bufferSize)
+               .regions(regions)
+               .limiter(limiter)
+               .build();
     }
 
     private static final class Cleanup extends SegmentedFile.Cleanup
     {
-        final Segment[] segments;
-        protected Cleanup(ChannelProxy channel, Segment[] segments)
+        private final MmappedRegions regions;
+
+        Cleanup(ChannelProxy channel, MmappedRegions regions)
         {
             super(channel);
-            this.segments = segments;
+            this.regions = regions;
         }
 
         public void tidy()
         {
-            super.tidy();
-
-            if (!FileUtils.isCleanerAvailable())
-                return;
-
-        /*
-         * Try forcing the unmapping of segments using undocumented unsafe sun APIs.
-         * If this fails (non Sun JVM), we'll have to wait for the GC to finalize the mapping.
-         * If this works and a thread tries to access any segment, hell will unleash on earth.
-         */
-            try
+            Throwable err = regions.close(null);
+            if (err != null)
             {
-                for (Segment segment : segments)
-                {
-                    if (segment.right == null)
-                        continue;
-                    FileUtils.clean(segment.right);
-                }
-                logger.debug("All segments have been unmapped successfully");
-            }
-            catch (Exception e)
-            {
-                JVMStabilityInspector.inspectThrowable(e);
+                JVMStabilityInspector.inspectThrowable(err);
+
                 // This is not supposed to happen
-                logger.error("Error while unmapping segments", e);
+                logger.error("Error while closing mmapped regions", err);
             }
+
+            super.tidy();
         }
     }
 
@@ -140,104 +98,64 @@ public class MmappedSegmentedFile extends SegmentedFile
      */
     static class Builder extends SegmentedFile.Builder
     {
-        // planned segment boundaries
-        private List<Long> boundaries;
+        private MmappedRegions regions;
 
-        // offset of the open segment (first segment begins at 0).
-        private long currentStart = 0;
-
-        // current length of the open segment.
-        // used to allow merging multiple too-large-to-mmap segments, into a single buffered segment.
-        private long currentSize = 0;
-
-        public Builder()
+        Builder()
         {
             super();
-            boundaries = new ArrayList<>();
-            boundaries.add(0L);
-        }
-
-        public void addPotentialBoundary(long boundary)
-        {
-            if (boundary - currentStart <= MAX_SEGMENT_SIZE)
-            {
-                // boundary fits into current segment: expand it
-                currentSize = boundary - currentStart;
-                return;
-            }
-
-            // close the current segment to try and make room for the boundary
-            if (currentSize > 0)
-            {
-                currentStart += currentSize;
-                boundaries.add(currentStart);
-            }
-            currentSize = boundary - currentStart;
-
-            // if we couldn't make room, the boundary needs its own segment
-            if (currentSize > MAX_SEGMENT_SIZE)
-            {
-                currentStart = boundary;
-                boundaries.add(currentStart);
-                currentSize = 0;
-            }
         }
 
         public SegmentedFile complete(ChannelProxy channel, int bufferSize, long overrideLength)
         {
             long length = overrideLength > 0 ? overrideLength : channel.size();
-            // create the segments
-            return new MmappedSegmentedFile(channel, bufferSize, length, createSegments(channel, length));
+            updateRegions(channel, length);
+
+            return new MmappedSegmentedFile(channel, bufferSize, length, regions.sharedCopy());
         }
 
-        private Segment[] createSegments(ChannelProxy channel, long length)
+        private void updateRegions(ChannelProxy channel, long length)
         {
-            // if we're early finishing a range that doesn't span multiple segments, but the finished file now does,
-            // we remove these from the end (we loop incase somehow this spans multiple segments, but that would
-            // be a loco dataset
-            while (length < boundaries.get(boundaries.size() - 1))
-                boundaries.remove(boundaries.size() -1);
-
-            // add a sentinel value == length
-            List<Long> boundaries = new ArrayList<>(this.boundaries);
-            if (length != boundaries.get(boundaries.size() - 1))
-                boundaries.add(length);
-
-            int segcount = boundaries.size() - 1;
-            Segment[] segments = new Segment[segcount];
-            for (int i = 0; i < segcount; i++)
+            if (regions != null && !regions.isValid(channel))
             {
-                long start = boundaries.get(i);
-                long size = boundaries.get(i + 1) - start;
-                MappedByteBuffer segment = size <= MAX_SEGMENT_SIZE
-                                           ? channel.map(FileChannel.MapMode.READ_ONLY, start, size)
-                                           : null;
-                segments[i] = new Segment(start, segment);
+                Throwable err = regions.close(null);
+                if (err != null)
+                    logger.error("Failed to close mapped regions", err);
+
+                regions = null;
             }
-            return segments;
+
+            if (regions == null)
+                regions = MmappedRegions.map(channel, length);
+            else
+                regions.extend(length);
         }
 
         @Override
-        public void serializeBounds(DataOutput out) throws IOException
+        public void serializeBounds(DataOutput out, Version version) throws IOException
         {
-            super.serializeBounds(out);
-            out.writeInt(boundaries.size());
-            for (long position: boundaries)
-                out.writeLong(position);
+            if (!version.hasBoundaries())
+                return;
+
+            super.serializeBounds(out, version);
+            out.writeInt(0);
         }
 
         @Override
-        public void deserializeBounds(DataInput in) throws IOException
+        public void deserializeBounds(DataInput in, Version version) throws IOException
         {
-            super.deserializeBounds(in);
+            if (!version.hasBoundaries())
+                return;
 
-            int size = in.readInt();
-            List<Long> temp = new ArrayList<>(size);
-            
-            for (int i = 0; i < size; i++)
-                temp.add(in.readLong());
+            super.deserializeBounds(in, version);
+            in.skipBytes(in.readInt() * TypeSizes.sizeof(0L));
+        }
 
-            boundaries = temp;
+        @Override
+        public Throwable close(Throwable accumulate)
+        {
+            return super.close(regions == null
+                               ? accumulate
+                               : regions.close(accumulate));
         }
     }
 }

src/java/org/apache/cassandra/io/util/NIODataInputStream.java

@@ -18,17 +18,11 @@
 package org.apache.cassandra.io.util;
 
 import java.io.Closeable;
-import java.io.DataInputStream;
-import java.io.EOFException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.SeekableByteChannel;
 
-import org.apache.cassandra.utils.vint.VIntCoding;
-
 import com.google.common.base.Preconditions;
 
 /**
@@ -43,357 +37,59 @@ import com.google.common.base.Preconditions;
  *
  * NIODataInputStream is not thread safe.
  */
-public class NIODataInputStream extends InputStream implements DataInputPlus, Closeable
+public class NIODataInputStream extends RebufferingInputStream
 {
-    private final ReadableByteChannel rbc;
-    private final ByteBuffer buf;
+    protected final ReadableByteChannel channel;
 
-    /*
-     *  Used when wrapping a fixed buffer of data instead of a channel. Should never attempt
-     *  to read from it.
-     */
-    private static final ReadableByteChannel emptyReadableByteChannel = new ReadableByteChannel()
+    private static ByteBuffer makeBuffer(int bufferSize)
     {
+        ByteBuffer buffer = ByteBuffer.allocateDirect(bufferSize);
+        buffer.position(0);
+        buffer.limit(0);
 
-        @Override
-        public boolean isOpen()
-        {
-            return true;
-        }
-
-        @Override
-        public void close() throws IOException
-        {
-        }
-
-        @Override
-        public int read(ByteBuffer dst) throws IOException
-        {
-            throw new AssertionError();
-        }
-
-    };
-
-    public NIODataInputStream(ReadableByteChannel rbc, int bufferSize)
-    {
-        Preconditions.checkNotNull(rbc);
-        Preconditions.checkArgument(bufferSize >= 9, "Buffer size must be large enough to accomadate a varint");
-        this.rbc = rbc;
-        buf = ByteBuffer.allocateDirect(bufferSize);
-        buf.position(0);
-        buf.limit(0);
+        return buffer;
     }
 
-    protected NIODataInputStream(ByteBuffer buf, boolean duplicate)
+    public NIODataInputStream(ReadableByteChannel channel, ByteBuffer buffer)
     {
-        Preconditions.checkNotNull(buf);
-        if (duplicate)
-            this.buf = buf.duplicate();
-        else
-            this.buf = buf;
+        super(buffer);
 
-        this.rbc = emptyReadableByteChannel;
+        Preconditions.checkNotNull(channel);
+        this.channel = channel;
     }
 
-    /*
-     * The decision to duplicate or not really needs to conscious since it a real impact
-     * in terms of thread safety so don't expose this constructor with an implicit default.
-     */
-    protected NIODataInputStream(ByteBuffer buf)
+    public NIODataInputStream(ReadableByteChannel channel, int bufferSize)
     {
-        this(buf, false);
+        this(channel, makeBuffer(bufferSize));
     }
 
     @Override
-    public void readFully(byte[] b) throws IOException
+    protected void reBuffer() throws IOException
     {
-        readFully(b, 0, b.length);
-    }
+        Preconditions.checkState(buffer.remaining() == 0);
+        buffer.clear();
 
+        while ((channel.read(buffer)) == 0) {}
 
-    @Override
-    public void readFully(byte[] b, int off, int len) throws IOException
-    {
-        int copied = 0;
-        while (copied < len)
-        {
-            int read = read(b, off + copied, len - copied);
-            if (read < 0)
-                throw new EOFException();
-            copied += read;
-        }
-    }
-
-    @Override
-    public int read(byte b[], int off, int len) throws IOException {
-        if (b == null)
-            throw new NullPointerException();
-
-        // avoid int overflow
-        if (off < 0 || off > b.length || len < 0
-                || len > b.length - off)
-            throw new IndexOutOfBoundsException();
-
-        if (len == 0)
-            return 0;
-
-        int copied = 0;
-        while (copied < len)
-        {
-            if (buf.hasRemaining())
-            {
-                int toCopy = Math.min(len - copied, buf.remaining());
-                buf.get(b, off + copied, toCopy);
-                copied += toCopy;
-            }
-            else
-            {
-                int read = readNext();
-                if (read < 0 && copied == 0) return -1;
-                if (read <= 0) return copied;
-            }
-        }
-
-        return copied;
-    }
-
-    /*
-     * Refill the buffer, preserving any unread bytes remaining in the buffer
-     */
-    protected int readNext() throws IOException
-    {
-        Preconditions.checkState(buf.remaining() != buf.capacity());
-        assert(buf.remaining() < 9);
-
-        /*
-         * If there is data already at the start of the buffer, move the position to the end
-         * If there is data but not at the start, move it to the start
-         * Otherwise move the position to 0 so writes start at the beginning of the buffer
-         *
-         * We go to the trouble of shuffling the bytes remaining for cases where the buffer isn't fully drained
-         * while retrieving a multi-byte value while the position is in the middle.
-         */
-        if (buf.position() == 0 && buf.hasRemaining())
-        {
-            buf.position(buf.limit());
-        }
-        else if (buf.hasRemaining())
-        {
-            //FastByteOperations.copy failed to do the copy so inline a simple one here
-            int position = buf.position();
-            int remaining  = buf.remaining();
-            buf.clear();
-            for (int ii = 0; ii < remaining; ii++)
-                buf.put(buf.get(position + ii));
-        }
-        else
-        {
-            buf.position(0);
-        }
-
-        buf.limit(buf.capacity());
-
-        int read = 0;
-        while ((read = rbc.read(buf)) == 0) {}
-
-        buf.flip();
-
-        return read;
-    }
-
-    /*
-     * Read the minimum number of bytes and throw EOF if the minimum could not be read
-     */
-    private void readMinimum(int minimum) throws IOException
-    {
-        assert(buf.remaining() < 8);
-        while (buf.remaining() < minimum)
-        {
-            int read = readNext();
-            if (read == -1)
-            {
-                //DataInputStream consumes the bytes even if it doesn't get the entire value, match the behavior here
-                buf.position(0);
-                buf.limit(0);
-                throw new EOFException();
-            }
-        }
-    }
-
-    /*
-     * Ensure the buffer contains the minimum number of readable bytes, throws EOF if enough bytes aren't available
-     */
-    private void prepareReadPrimitive(int minimum) throws IOException
-    {
-        if (buf.remaining() < minimum)
-            readMinimum(minimum);
-    }
-
-    @Override
-    public int skipBytes(int n) throws IOException
-    {
-        int skipped = 0;
-
-        while (skipped < n)
-        {
-            int skippedThisTime = (int)skip(n - skipped);
-            if (skippedThisTime <= 0) break;
-            skipped += skippedThisTime;
-        }
-
-        return skipped;
-    }
-
-    @Override
-    public boolean readBoolean() throws IOException
-    {
-        prepareReadPrimitive(1);
-        return buf.get() != 0;
-    }
-
-    @Override
-    public byte readByte() throws IOException
-    {
-        prepareReadPrimitive(1);
-        return buf.get();
-    }
-
-    @Override
-    public int readUnsignedByte() throws IOException
-    {
-        prepareReadPrimitive(1);
-        return buf.get() & 0xff;
-    }
-
-    @Override
-    public short readShort() throws IOException
-    {
-        prepareReadPrimitive(2);
-        return buf.getShort();
-    }
-
-    @Override
-    public int readUnsignedShort() throws IOException
-    {
-        return readShort() & 0xFFFF;
-    }
-
-    @Override
-    public char readChar() throws IOException
-    {
-        prepareReadPrimitive(2);
-        return buf.getChar();
-    }
-
-    @Override
-    public int readInt() throws IOException
-    {
-        prepareReadPrimitive(4);
-        return buf.getInt();
-    }
-
-    @Override
-    public long readLong() throws IOException
-    {
-        prepareReadPrimitive(8);
-        return buf.getLong();
-    }
-
-    public long readVInt() throws IOException
-    {
-        return VIntCoding.decodeZigZag64(readUnsignedVInt());
-    }
-
-    public long readUnsignedVInt() throws IOException
-    {
-        //If 9 bytes aren't available use the slow path in VIntCoding
-        if (buf.remaining() < 9)
-            return VIntCoding.readUnsignedVInt(this);
-
-        byte firstByte = buf.get();
-
-        //Bail out early if this is one byte, necessary or it fails later
-        if (firstByte >= 0)
-            return firstByte;
-
-        int extraBytes = VIntCoding.numberOfExtraBytesToRead(firstByte);
-
-        int position = buf.position();
-        int extraBits = extraBytes * 8;
-
-        long retval = buf.getLong(position);
-        if (buf.order() == ByteOrder.LITTLE_ENDIAN)
-            retval = Long.reverseBytes(retval);
-        buf.position(position + extraBytes);
-
-        // truncate the bytes we read in excess of those we needed
-        retval >>>= 64 - extraBits;
-        // remove the non-value bits from the first byte
-        firstByte &= VIntCoding.firstByteValueMask(extraBytes);
-        // shift the first byte up to its correct position
-        retval |= (long) firstByte << extraBits;
-        return retval;
-    }
-
-    @Override
-    public float readFloat() throws IOException
-    {
-        prepareReadPrimitive(4);
-        return buf.getFloat();
-    }
-
-    @Override
-    public double readDouble() throws IOException
-    {
-        prepareReadPrimitive(8);
-        return buf.getDouble();
-    }
-
-    @Override
-    public String readLine() throws IOException
-    {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public String readUTF() throws IOException
-    {
-        return DataInputStream.readUTF(this);
+        buffer.flip();
     }
 
     @Override
     public void close() throws IOException
     {
-            rbc.close();
-    }
-
-    @Override
-    public int read() throws IOException
-    {
-        return readUnsignedByte();
+        channel.close();
+        super.close();
     }
 
     @Override
     public int available() throws IOException
     {
-        if (rbc instanceof SeekableByteChannel)
+        if (channel instanceof SeekableByteChannel)
         {
-            SeekableByteChannel sbc = (SeekableByteChannel)rbc;
+            SeekableByteChannel sbc = (SeekableByteChannel) channel;
             long remainder = Math.max(0, sbc.size() - sbc.position());
-            return (remainder > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int)(remainder + buf.remaining());
+            return (remainder > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int)(remainder + buffer.remaining());
         }
-        return buf.remaining();
-    }
-
-    @Override
-    public void reset() throws IOException
-    {
-        throw new IOException("mark/reset not supported");
-    }
-
-    @Override
-    public boolean markSupported()
-    {
-        return false;
+        return buffer.remaining();
     }
 }

src/java/org/apache/cassandra/io/util/RandomAccessReader.java

@@ -19,119 +19,129 @@ package org.apache.cassandra.io.util;
 
 import java.io.*;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import com.google.common.primitives.Ints;
+import com.google.common.util.concurrent.RateLimiter;
 
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.compress.BufferType;
-import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.memory.BufferPool;
 
-public class RandomAccessReader extends AbstractDataInput implements FileDataInput
+public class RandomAccessReader extends RebufferingInputStream implements FileDataInput
 {
+    // The default buffer size when the client doesn't specify it
     public static final int DEFAULT_BUFFER_SIZE = 4096;
 
+    // The maximum buffer size when the limiter is not null, i.e. when throttling
+    // is enabled. This is required to avoid aquiring permits that are too large.
+    public static final int MAX_THROTTLED_BUFFER_SIZE = 1 << 16; // 64k
+
     // the IO channel to the file, we do not own a reference to this due to
     // performance reasons (CASSANDRA-9379) so it's up to the owner of the RAR to
     // ensure that the channel stays open and that it is closed afterwards
     protected final ChannelProxy channel;
 
-    // buffer which will cache file blocks
-    protected ByteBuffer buffer;
+    // optional memory mapped regions for the channel
+    protected final MmappedRegions regions;
 
-    // `bufferOffset` is the offset of the beginning of the buffer
-    // `markedPointer` folds the offset of the last file mark
-    protected long bufferOffset, markedPointer;
+    // An optional limiter that will throttle the amount of data we read
+    protected final RateLimiter limiter;
 
-    // this can be overridden at construction to a value shorter than the true length of the file;
-    // if so, it acts as an imposed limit on reads, rather than a convenience property
+    // the file length, this can be overridden at construction to a value shorter
+    // than the true length of the file; if so, it acts as an imposed limit on reads,
+    // required when opening sstables early not to read past the mark
     private final long fileLength;
 
-    protected RandomAccessReader(ChannelProxy channel, int bufferSize, long overrideLength, BufferType bufferType)
-    {
-        this.channel = channel;
+    // the buffer size for buffered readers
+    protected final int bufferSize;
 
-        if (bufferSize <= 0)
+    // the buffer type for buffered readers
+    private final BufferType bufferType;
+
+    // offset from the beginning of the file
+    protected long bufferOffset;
+
+    // offset of the last file mark
+    protected long markedPointer;
+
+    protected RandomAccessReader(Builder builder)
+    {
+        super(null);
+
+        this.channel = builder.channel;
+        this.regions = builder.regions;
+        this.limiter = builder.limiter;
+        this.fileLength = builder.overrideLength <= 0 ? builder.channel.size() : builder.overrideLength;
+        this.bufferSize = getBufferSize(builder);
+        this.bufferType = builder.bufferType;
+
+        if (builder.bufferSize <= 0)
             throw new IllegalArgumentException("bufferSize must be positive");
 
-        // we can cache file length in read-only mode
-        fileLength = overrideLength <= 0 ? channel.size() : overrideLength;
+        if (builder.initializeBuffers)
+            initializeBuffer();
+    }
+
+    protected int getBufferSize(Builder builder)
+    {
+        if (builder.limiter == null)
+            return builder.bufferSize;
+
+        // limit to ensure more accurate throttling
+        return Math.min(MAX_THROTTLED_BUFFER_SIZE, builder.bufferSize);
+    }
+
+    protected void initializeBuffer()
+    {
+        if (regions == null)
+            buffer = allocateBuffer(bufferSize);
+        else
+            buffer = regions.floor(0).buffer.duplicate();
 
-        buffer = allocateBuffer(getBufferSize(bufferSize), bufferType);
         buffer.limit(0);
     }
 
-    /** The buffer size is typically already page aligned but if that is not the case
-     * make sure that it is a multiple of the page size, 4096.
-     * */
-    protected int getBufferSize(int size)
+    protected ByteBuffer allocateBuffer(int size)
     {
-        if ((size & ~4095) != size)
-        { // should already be a page size multiple but if that's not case round it up
-            size = (size + 4095) & ~4095;
-        }
-        return size;
+        return BufferPool.get(size, bufferType).order(ByteOrder.BIG_ENDIAN);
     }
 
-    protected ByteBuffer allocateBuffer(int size, BufferType bufferType)
+    protected void releaseBuffer()
     {
-        return BufferPool.get(size, bufferType);
-    }
-
-    // A wrapper of the RandomAccessReader that closes the channel when done.
-    // For performance reasons RAR does not increase the reference count of
-    // a channel but assumes the owner will keep it open and close it,
-    // see CASSANDRA-9379, this thin class is just for those cases where we do
-    // not have a shared channel.
-    private static class RandomAccessReaderWithChannel extends RandomAccessReader
-    {
-        @SuppressWarnings("resource")
-        RandomAccessReaderWithChannel(File file)
+        if (buffer != null)
         {
-            super(new ChannelProxy(file), DEFAULT_BUFFER_SIZE, -1L, BufferType.OFF_HEAP);
+            if (regions == null)
+                BufferPool.put(buffer);
+            buffer = null;
         }
-
-        @Override
-        public void close()
-        {
-            try
-            {
-                super.close();
-            }
-            finally
-            {
-                channel.close();
-            }
-        }
-    }
-
-    public static RandomAccessReader open(File file)
-    {
-        return new RandomAccessReaderWithChannel(file);
-    }
-
-    public static RandomAccessReader open(ChannelProxy channel)
-    {
-        return open(channel, DEFAULT_BUFFER_SIZE, -1L);
-    }
-
-    public static RandomAccessReader open(ChannelProxy channel, int bufferSize, long overrideSize)
-    {
-        return new RandomAccessReader(channel, bufferSize, overrideSize, BufferType.OFF_HEAP);
-    }
-
-    public ChannelProxy getChannel()
-    {
-        return channel;
     }
 
     /**
      * Read data from file starting from current currentOffset to populate buffer.
      */
-    protected void reBuffer()
+    public void reBuffer()
+    {
+        if (isEOF())
+            return;
+
+        if (regions == null)
+            reBufferStandard();
+        else
+            reBufferMmap();
+
+        if (limiter != null)
+            limiter.acquire(buffer.remaining());
+
+        assert buffer.order() == ByteOrder.BIG_ENDIAN : "Buffer must have BIG ENDIAN byte ordering";
+    }
+
+    protected void reBufferStandard()
     {
         bufferOffset += buffer.position();
-        buffer.clear();
         assert bufferOffset < fileLength;
 
+        buffer.clear();
         long position = bufferOffset;
         long limit = bufferOffset;
 
@@ -145,15 +155,31 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp
         {
             int n = channel.read(buffer, position);
             if (n < 0)
-                break;
+                throw new FSReadError(new IOException("Unexpected end of file"), channel.filePath());
+
             position += n;
             limit = bufferOffset + buffer.position();
         }
-        if (limit > fileLength)
-            buffer.position((int)(fileLength - bufferOffset));
+
         buffer.flip();
     }
 
+    protected void reBufferMmap()
+    {
+        long position = bufferOffset + buffer.position();
+        assert position < fileLength;
+
+        MmappedRegions.Region region = regions.floor(position);
+        bufferOffset = region.bottom();
+        buffer = region.buffer.duplicate();
+        buffer.position(Ints.checkedCast(position - bufferOffset));
+
+        if (limiter != null && bufferSize < buffer.remaining())
+        { // ensure accurate throttling
+            buffer.limit(buffer.position() + bufferSize);
+        }
+    }
+
     @Override
     public long getFilePointer()
     {
@@ -170,19 +196,23 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp
         return channel.filePath();
     }
 
-    public int getTotalBufferSize()
+    public ChannelProxy getChannel()
     {
-        //This may NPE so we make a ref
-        //https://issues.apache.org/jira/browse/CASSANDRA-7756
-        ByteBuffer ref = buffer;
-        return ref != null ? ref.capacity() : 0;
+        return channel;
     }
 
-    public void reset()
+    @Override
+    public void reset() throws IOException
     {
         seek(markedPointer);
     }
 
+    @Override
+    public boolean markSupported()
+    {
+        return true;
+    }
+
     public long bytesPastMark()
     {
         long bytes = current() - markedPointer;
@@ -215,7 +245,7 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp
      */
     public boolean isEOF()
     {
-        return getFilePointer() == length();
+        return current() == length();
     }
 
     public long bytesRemaining()
@@ -223,6 +253,12 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp
         return length() - getFilePointer();
     }
 
+    @Override
+    public int available() throws IOException
+    {
+        return Ints.saturatedCast(bytesRemaining());
+    }
+
     @Override
     public void close()
     {
@@ -230,16 +266,17 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp
         if (buffer == null)
             return;
 
-
         bufferOffset += buffer.position();
-        BufferPool.put(buffer);
-        buffer = null;
+        releaseBuffer();
+
+        //For performance reasons we don't keep a reference to the file
+        //channel so we don't close it
     }
 
     @Override
     public String toString()
     {
-        return getClass().getSimpleName() + "(" + "filePath='" + channel + "')";
+        return getClass().getSimpleName() + "(filePath='" + channel + "')";
     }
 
     /**
@@ -286,78 +323,51 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp
         assert current() == newPosition;
     }
 
-    // -1 will be returned if there is nothing to read; higher-level methods like readInt
-    // or readFully (from RandomAccessFile) will throw EOFException but this should not
-    public int read()
+    /**
+     * Reads a line of text form the current position in this file. A line is
+     * represented by zero or more characters followed by {@code '\n'}, {@code
+     * '\r'}, {@code "\r\n"} or the end of file marker. The string does not
+     * include the line terminating sequence.
+     * <p/>
+     * Blocks until a line terminating sequence has been read, the end of the
+     * file is reached or an exception is thrown.
+     *
+     * @return the contents of the line or {@code null} if no characters have
+     * been read before the end of the file has been reached.
+     * @throws IOException if this file is closed or another I/O error occurs.
+     */
+    public final String readLine() throws IOException
     {
-        if (buffer == null)
-            throw new AssertionError("Attempted to read from closed RAR");
-
-        if (isEOF())
-            return -1; // required by RandomAccessFile
-
-        if (!buffer.hasRemaining())
-            reBuffer();
-
-        return (int)buffer.get() & 0xff;
-    }
-
-    @Override
-    public int read(byte[] buffer)
-    {
-        return read(buffer, 0, buffer.length);
-    }
-
-    @Override
-    // -1 will be returned if there is nothing to read; higher-level methods like readInt
-    // or readFully (from RandomAccessFile) will throw EOFException but this should not
-    public int read(byte[] buff, int offset, int length)
-    {
-        if (buffer == null)
-            throw new IllegalStateException("Attempted to read from closed RAR");
-
-        if (length == 0)
-            return 0;
-
-        if (isEOF())
-            return -1;
-
-        if (!buffer.hasRemaining())
-            reBuffer();
-
-        int toCopy = Math.min(length, buffer.remaining());
-        buffer.get(buff, offset, toCopy);
-        return toCopy;
-    }
-
-    public ByteBuffer readBytes(int length) throws EOFException
-    {
-        assert length >= 0 : "buffer length should not be negative: " + length;
-
-        if (buffer == null)
-            throw new IllegalStateException("Attempted to read from closed RAR");
-
-        try
+        StringBuilder line = new StringBuilder(80); // Typical line length
+        boolean foundTerminator = false;
+        long unreadPosition = -1;
+        while (true)
         {
-            ByteBuffer result = ByteBuffer.allocate(length);
-            while (result.hasRemaining())
+            int nextByte = read();
+            switch (nextByte)
             {
-                if (isEOF())
-                    throw new EOFException();
-                if (!buffer.hasRemaining())
-                    reBuffer();
-                ByteBufferUtil.put(buffer, result);
+                case -1:
+                    return line.length() != 0 ? line.toString() : null;
+                case (byte) '\r':
+                    if (foundTerminator)
+                    {
+                        seek(unreadPosition);
+                        return line.toString();
+                    }
+                    foundTerminator = true;
+                    /* Have to be able to peek ahead one byte */
+                    unreadPosition = getPosition();
+                    break;
+                case (byte) '\n':
+                    return line.toString();
+                default:
+                    if (foundTerminator)
+                    {
+                        seek(unreadPosition);
+                        return line.toString();
+                    }
+                    line.append((char) nextByte);
             }
-            result.flip();
-            return result;
-        }
-        catch (EOFException e)
-        {
-            throw e;
-        }
-        catch (Exception e)
-        {
-            throw new FSReadError(e, channel.toString());
         }
     }
 
@@ -368,11 +378,142 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp
 
     public long getPosition()
     {
-        return bufferOffset + (buffer == null ? 0 : buffer.position());
+        return current();
     }
 
-    public long getPositionLimit()
+    public static class Builder
     {
-        return length();
+        // The NIO file channel or an empty channel
+        public final ChannelProxy channel;
+
+        // We override the file length when we open sstables early, so that we do not
+        // read past the early mark
+        public long overrideLength;
+
+        // The size of the buffer for buffered readers
+        public int bufferSize;
+
+        // The type of the buffer for buffered readers
+        public BufferType bufferType;
+
+        // The mmap segments for mmap readers
+        public MmappedRegions regions;
+
+        // An optional limiter that will throttle the amount of data we read
+        public RateLimiter limiter;
+
+        public boolean initializeBuffers;
+
+        public Builder(ChannelProxy channel)
+        {
+            this.channel = channel;
+            this.overrideLength = -1L;
+            this.bufferSize = getBufferSize(DEFAULT_BUFFER_SIZE);
+            this.bufferType = BufferType.OFF_HEAP;
+            this.regions = null;
+            this.limiter = null;
+            this.initializeBuffers = true;
+        }
+
+        /** The buffer size is typically already page aligned but if that is not the case
+         * make sure that it is a multiple of the page size, 4096.
+         * */
+        private static int getBufferSize(int size)
+        {
+            if ((size & ~4095) != size)
+            { // should already be a page size multiple but if that's not case round it up
+                size = (size + 4095) & ~4095;
+            }
+            return size;
+        }
+
+        public Builder overrideLength(long overrideLength)
+        {
+            if (overrideLength > channel.size())
+                throw new IllegalArgumentException("overrideLength cannot be more than the file size");
+
+            this.overrideLength = overrideLength;
+            return this;
+        }
+
+        public Builder bufferSize(int bufferSize)
+        {
+            if (bufferSize <= 0)
+                throw new IllegalArgumentException("bufferSize must be positive");
+
+            this.bufferSize = getBufferSize(bufferSize);
+            return this;
+        }
+
+        public Builder bufferType(BufferType bufferType)
+        {
+            this.bufferType = bufferType;
+            return this;
+        }
+
+        public Builder regions(MmappedRegions regions)
+        {
+            this.regions = regions;
+            return this;
+        }
+
+        public Builder limiter(RateLimiter limiter)
+        {
+            this.limiter = limiter;
+            return this;
+        }
+
+        public Builder initializeBuffers(boolean initializeBuffers)
+        {
+            this.initializeBuffers = initializeBuffers;
+            return this;
+        }
+
+        public RandomAccessReader build()
+        {
+            return new RandomAccessReader(this);
+        }
+
+        public RandomAccessReader buildWithChannel()
+        {
+            return new RandomAccessReaderWithOwnChannel(this);
+        }
+    }
+
+    // A wrapper of the RandomAccessReader that closes the channel when done.
+    // For performance reasons RAR does not increase the reference count of
+    // a channel but assumes the owner will keep it open and close it,
+    // see CASSANDRA-9379, this thin class is just for those cases where we do
+    // not have a shared channel.
+    public static class RandomAccessReaderWithOwnChannel extends RandomAccessReader
+    {
+        protected RandomAccessReaderWithOwnChannel(Builder builder)
+        {
+            super(builder);
+        }
+
+        @Override
+        public void close()
+        {
+            try
+            {
+                super.close();
+            }
+            finally
+            {
+                channel.close();
+            }
+        }
+    }
+
+    @SuppressWarnings("resource")
+    public static RandomAccessReader open(File file)
+    {
+        return new Builder(new ChannelProxy(file)).buildWithChannel();
+    }
+
+    public static RandomAccessReader open(ChannelProxy channel)
+    {
+        return new Builder(channel).build();
     }
 }

src/java/org/apache/cassandra/io/util/RebufferingInputStream.java

@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.util;
+
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import net.nicoulaj.compilecommand.annotations.DontInline;
+import net.nicoulaj.compilecommand.annotations.Print;
+import org.apache.cassandra.utils.FastByteOperations;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Rough equivalent of BufferedInputStream and DataInputStream wrapping a ByteBuffer that can be refilled
+ * via rebuffer. Implementations provide this buffer from various channels (socket, file, memory, etc).
+ *
+ * RebufferingInputStream is not thread safe.
+ */
+public abstract class RebufferingInputStream extends InputStream implements DataInputPlus, Closeable
+{
+    protected ByteBuffer buffer;
+
+    protected RebufferingInputStream(ByteBuffer buffer)
+    {
+        Preconditions.checkArgument(buffer == null || buffer.order() == ByteOrder.BIG_ENDIAN, "Buffer must have BIG ENDIAN byte ordering");
+        this.buffer = buffer;
+    }
+
+    /**
+     * Implementations must implement this method to refill the buffer.
+     * They can expect the buffer to be empty when this method is invoked.
+     * @throws IOException
+     */
+    protected abstract void reBuffer() throws IOException;
+
+    @Override
+    public void readFully(byte[] b) throws IOException
+    {
+        readFully(b, 0, b.length);
+    }
+
+    @Override
+    public void readFully(byte[] b, int off, int len) throws IOException
+    {
+        int read = read(b, off, len);
+        if (read < len)
+            throw new EOFException();
+    }
+
+    @Print
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+
+        // avoid int overflow
+        if (off < 0 || off > b.length || len < 0 || len > b.length - off)
+            throw new IndexOutOfBoundsException();
+
+        if (len == 0)
+            return 0;
+
+        int copied = 0;
+        while (copied < len)
+        {
+            int position = buffer.position();
+            int remaining = buffer.limit() - position;
+            if (remaining == 0)
+            {
+                reBuffer();
+                position = buffer.position();
+                remaining = buffer.limit() - position;
+                if (remaining == 0)
+                    return copied == 0 ? -1 : copied;
+            }
+            int toCopy = Math.min(len - copied, remaining);
+            FastByteOperations.copy(buffer, position, b, off + copied, toCopy);
+            buffer.position(position + toCopy);
+            copied += toCopy;
+        }
+
+        return copied;
+    }
+
+    @DontInline
+    protected long readPrimitiveSlowly(int bytes) throws IOException
+    {
+        long result = 0;
+        for (int i = 0; i < bytes; i++)
+            result = (result << 8) | (readByte() & 0xFFL);
+        return result;
+    }
+
+    @Override
+    public int skipBytes(int n) throws IOException
+    {
+        int skipped = 0;
+
+        while (skipped < n)
+        {
+            int skippedThisTime = (int)skip(n - skipped);
+            if (skippedThisTime <= 0) break;
+            skipped += skippedThisTime;
+        }
+
+        return skipped;
+    }
+
+    @Override
+    public boolean readBoolean() throws IOException
+    {
+        return readByte() != 0;
+    }
+
+    @Override
+    public byte readByte() throws IOException
+    {
+        if (!buffer.hasRemaining())
+        {
+            reBuffer();
+            if (!buffer.hasRemaining())
+                throw new EOFException();
+        }
+
+        return buffer.get();
+    }
+
+    @Override
+    public int readUnsignedByte() throws IOException
+    {
+        return readByte() & 0xff;
+    }
+
+    @Override
+    public short readShort() throws IOException
+    {
+        if (buffer.remaining() >= 2)
+            return buffer.getShort();
+        else
+            return (short) readPrimitiveSlowly(2);
+    }
+
+    @Override
+    public int readUnsignedShort() throws IOException
+    {
+        return readShort() & 0xFFFF;
+    }
+
+    @Override
+    public char readChar() throws IOException
+    {
+        if (buffer.remaining() >= 2)
+            return buffer.getChar();
+        else
+            return (char) readPrimitiveSlowly(2);
+    }
+
+    @Override
+    public int readInt() throws IOException
+    {
+        if (buffer.remaining() >= 4)
+            return buffer.getInt();
+        else
+            return (int) readPrimitiveSlowly(4);
+    }
+
+    @Override
+    public long readLong() throws IOException
+    {
+        if (buffer.remaining() >= 8)
+            return buffer.getLong();
+        else
+            return readPrimitiveSlowly(8);
+    }
+
+    public long readVInt() throws IOException
+    {
+        return VIntCoding.decodeZigZag64(readUnsignedVInt());
+    }
+
+    public long readUnsignedVInt() throws IOException
+    {
+        //If 9 bytes aren't available use the slow path in VIntCoding
+        if (buffer.remaining() < 9)
+            return VIntCoding.readUnsignedVInt(this);
+
+        byte firstByte = buffer.get();
+
+        //Bail out early if this is one byte, necessary or it fails later
+        if (firstByte >= 0)
+            return firstByte;
+
+        int extraBytes = VIntCoding.numberOfExtraBytesToRead(firstByte);
+
+        int position = buffer.position();
+        int extraBits = extraBytes * 8;
+
+        long retval = buffer.getLong(position);
+        if (buffer.order() == ByteOrder.LITTLE_ENDIAN)
+            retval = Long.reverseBytes(retval);
+        buffer.position(position + extraBytes);
+
+        // truncate the bytes we read in excess of those we needed
+        retval >>>= 64 - extraBits;
+        // remove the non-value bits from the first byte
+        firstByte &= VIntCoding.firstByteValueMask(extraBytes);
+        // shift the first byte up to its correct position
+        retval |= (long) firstByte << extraBits;
+        return retval;
+    }
+
+    @Override
+    public float readFloat() throws IOException
+    {
+        if (buffer.remaining() >= 4)
+            return buffer.getFloat();
+        else
+            return Float.intBitsToFloat((int)readPrimitiveSlowly(4));
+    }
+
+    @Override
+    public double readDouble() throws IOException
+    {
+        if (buffer.remaining() >= 8)
+            return buffer.getDouble();
+        else
+            return Double.longBitsToDouble(readPrimitiveSlowly(8));
+    }
+
+    @Override
+    public String readLine() throws IOException
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public String readUTF() throws IOException
+    {
+        return DataInputStream.readUTF(this);
+    }
+
+    @Override
+    public int read() throws IOException
+    {
+        try
+        {
+            return readUnsignedByte();
+        }
+        catch (EOFException ex)
+        {
+            return -1;
+        }
+    }
+
+    @Override
+    public void reset() throws IOException
+    {
+        throw new IOException("mark/reset not supported");
+    }
+
+    @Override
+    public boolean markSupported()
+    {
+        return false;
+    }
+}

src/java/org/apache/cassandra/io/util/SegmentedFile.java

@@ -21,23 +21,19 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.File;
 import java.io.IOException;
-import java.nio.MappedByteBuffer;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
 
 import com.google.common.util.concurrent.RateLimiter;
 
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.compress.CompressedSequentialWriter;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.IndexSummary;
 import org.apache.cassandra.io.sstable.IndexSummaryBuilder;
+import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.utils.CLibrary;
-import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.RefCounted;
 import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
 
@@ -79,7 +75,7 @@ public abstract class SegmentedFile extends SharedCloseableImpl
         this.onDiskLength = onDiskLength;
     }
 
-    public SegmentedFile(SegmentedFile copy)
+    protected SegmentedFile(SegmentedFile copy)
     {
         super(copy);
         channel = copy.channel;
@@ -93,7 +89,7 @@ public abstract class SegmentedFile extends SharedCloseableImpl
         return channel.filePath();
     }
 
-    protected static abstract class Cleanup implements RefCounted.Tidy
+    protected static class Cleanup implements RefCounted.Tidy
     {
         final ChannelProxy channel;
         protected Cleanup(ChannelProxy channel)
@@ -116,16 +112,22 @@ public abstract class SegmentedFile extends SharedCloseableImpl
 
     public RandomAccessReader createReader()
     {
-        return RandomAccessReader.open(channel, bufferSize, length);
+        return new RandomAccessReader.Builder(channel)
+               .overrideLength(length)
+               .bufferSize(bufferSize)
+               .build();
     }
 
-    public RandomAccessReader createThrottledReader(RateLimiter limiter)
+    public RandomAccessReader createReader(RateLimiter limiter)
     {
-        assert limiter != null;
-        return ThrottledReader.open(channel, bufferSize, length, limiter);
+        return new RandomAccessReader.Builder(channel)
+               .overrideLength(length)
+               .bufferSize(bufferSize)
+               .limiter(limiter)
+               .build();
     }
 
-    public FileDataInput getSegment(long position)
+    public FileDataInput createReader(long position)
     {
         RandomAccessReader reader = createReader();
         reader.seek(position);
@@ -152,14 +154,6 @@ public abstract class SegmentedFile extends SharedCloseableImpl
         return new CompressedSegmentedFile.Builder(writer);
     }
 
-    /**
-     * @return An Iterator over segments, beginning with the segment containing the given position: each segment must be closed after use.
-     */
-    public Iterator<FileDataInput> iterator(long position)
-    {
-        return new SegmentIterator(position);
-    }
-
     /**
      * Collects potential segmentation points in an underlying file, and builds a SegmentedFile to represent it.
      */
@@ -167,13 +161,6 @@ public abstract class SegmentedFile extends SharedCloseableImpl
     {
         private ChannelProxy channel;
 
-        /**
-         * Adds a position that would be a safe place for a segment boundary in the file. For a block/row based file
-         * format, safe boundaries are block/row edges.
-         * @param boundary The absolute position of the potential boundary in the file.
-         */
-        public abstract void addPotentialBoundary(long boundary);
-
         /**
          * Called after all potential boundaries have been added to apply this Builder to a concrete file on disk.
          * @param channel The channel to the file on disk.
@@ -214,12 +201,12 @@ public abstract class SegmentedFile extends SharedCloseableImpl
             return complete(desc.filenameFor(Component.PRIMARY_INDEX), bufferSize(desc, indexSummary), -1L);
         }
 
-        private int bufferSize(StatsMetadata stats)
+        private static int bufferSize(StatsMetadata stats)
         {
             return bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
         }
 
-        private int bufferSize(Descriptor desc, IndexSummary indexSummary)
+        private static int bufferSize(Descriptor desc, IndexSummary indexSummary)
         {
             File file = new File(desc.filenameFor(Component.PRIMARY_INDEX));
             return bufferSize(file.length() / indexSummary.size());
@@ -267,13 +254,19 @@ public abstract class SegmentedFile extends SharedCloseableImpl
             return (int)Math.min(size, 1 << 16);
         }
 
-        public void serializeBounds(DataOutput out) throws IOException
+        public void serializeBounds(DataOutput out, Version version) throws IOException
         {
+            if (!version.hasBoundaries())
+                return;
+
             out.writeUTF(DatabaseDescriptor.getDiskAccessMode().name());
         }
 
-        public void deserializeBounds(DataInput in) throws IOException
+        public void deserializeBounds(DataInput in, Version version) throws IOException
         {
+            if (!version.hasBoundaries())
+                return;
+
             if (!in.readUTF().equals(DatabaseDescriptor.getDiskAccessMode().name()))
                 throw new IOException("Cannot deserialize SSTable Summary component because the DiskAccessMode was changed!");
         }
@@ -282,6 +275,7 @@ public abstract class SegmentedFile extends SharedCloseableImpl
         {
             if (channel != null)
                 return channel.close(accumulate);
+
             return accumulate;
         }
 
@@ -294,6 +288,10 @@ public abstract class SegmentedFile extends SharedCloseableImpl
         {
             if (channel != null)
             {
+                // This is really fragile, both path and channel.filePath()
+                // must agree, i.e. they both must be absolute or both relative
+                // eventually we should really pass the filePath to the builder
+                // constructor and remove this
                 if (channel.filePath().equals(path))
                     return channel.sharedCopy();
                 else
@@ -305,61 +303,10 @@ public abstract class SegmentedFile extends SharedCloseableImpl
         }
     }
 
-    static final class Segment extends Pair<Long, MappedByteBuffer> implements Comparable<Segment>
-    {
-        public Segment(long offset, MappedByteBuffer segment)
-        {
-            super(offset, segment);
-        }
-
-        public final int compareTo(Segment that)
-        {
-            return (int)Math.signum(this.left - that.left);
-        }
-    }
-
-    /**
-     * A lazy Iterator over segments in forward order from the given position.  It is caller's responsibility
-     * to close the FileDataIntputs when finished.
-     */
-    final class SegmentIterator implements Iterator<FileDataInput>
-    {
-        private long nextpos;
-        public SegmentIterator(long position)
-        {
-            this.nextpos = position;
-        }
-
-        public boolean hasNext()
-        {
-            return nextpos < length;
-        }
-
-        public FileDataInput next()
-        {
-            long position = nextpos;
-            if (position >= length)
-                throw new NoSuchElementException();
-
-            FileDataInput segment = getSegment(nextpos);
-            try
-            {
-                nextpos = nextpos + segment.bytesRemaining();
-            }
-            catch (IOException e)
-            {
-                throw new FSReadError(e, path());
-            }
-            return segment;
-        }
-
-        public void remove() { throw new UnsupportedOperationException(); }
-    }
-
     @Override
     public String toString() {
-        return getClass().getSimpleName() + "(path='" + path() + "'" +
+        return getClass().getSimpleName() + "(path='" + path() + '\'' +
                ", length=" + length +
-               ")";
+               ')';
 }
 }

src/java/org/apache/cassandra/io/util/ThrottledReader.java

@@ -1,48 +0,0 @@
-package org.apache.cassandra.io.util;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-
-import com.google.common.util.concurrent.RateLimiter;
-
-import org.apache.cassandra.io.compress.BufferType;
-
-public class ThrottledReader extends RandomAccessReader
-{
-    private final RateLimiter limiter;
-
-    protected ThrottledReader(ChannelProxy channel, int bufferSize, long overrideLength, RateLimiter limiter)
-    {
-        super(channel, bufferSize, overrideLength, BufferType.OFF_HEAP);
-        this.limiter = limiter;
-    }
-
-    protected void reBuffer()
-    {
-        limiter.acquire(buffer.capacity());
-        super.reBuffer();
-    }
-
-    public static ThrottledReader open(ChannelProxy channel, int bufferSize, long overrideLength, RateLimiter limiter)
-    {
-        return new ThrottledReader(channel, bufferSize, overrideLength, limiter);
-    }
-}

src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java

@@ -55,7 +55,7 @@ public class CompressedStreamWriter extends StreamWriter
     public void write(DataOutputStreamPlus out) throws IOException
     {
         long totalSize = totalSize();
-        try (RandomAccessReader file = sstable.openDataReader(); final ChannelProxy fc = file.getChannel().sharedCopy())
+        try (ChannelProxy fc = sstable.getDataChannel().sharedCopy())
         {
             long progress = 0L;
             // calculate chunks to transfer. we want to send continuous chunks altogether.
@@ -72,13 +72,7 @@ public class CompressedStreamWriter extends StreamWriter
                     final long bytesTransferredFinal = bytesTransferred;
                     final int toTransfer = (int) Math.min(CHUNK_SIZE, length - bytesTransferred);
                     limiter.acquire(toTransfer);
-                    long lastWrite = out.applyToChannel(new Function<WritableByteChannel, Long>()
-                    {
-                        public Long apply(WritableByteChannel wbc)
-                        {
-                            return fc.transferTo(section.left + bytesTransferredFinal, toTransfer, wbc);
-                        }
-                    });
+                    long lastWrite = out.applyToChannel((wbc) -> fc.transferTo(section.left + bytesTransferredFinal, toTransfer, wbc));
                     bytesTransferred += lastWrite;
                     progress += lastWrite;
                     session.progress(sstable.descriptor, ProgressInfo.Direction.OUT, progress, totalSize);

src/java/org/apache/cassandra/utils/ByteBufferUtil.java

@@ -398,9 +398,6 @@ public class ByteBufferUtil
         if (length == 0)
             return EMPTY_BYTE_BUFFER;
 
-        if (in instanceof FileDataInput)
-            return ((FileDataInput) in).readBytes(length);
-
         byte[] buff = new byte[length];
         in.readFully(buff);
         return ByteBuffer.wrap(buff);

src/java/org/apache/cassandra/utils/Throwables.java

@@ -82,29 +82,54 @@ public final class Throwables
     @SuppressWarnings("unchecked")
     public static <E extends Exception> void perform(Stream<DiscreteAction<? extends E>> actions) throws E
     {
-        Throwable fail = null;
-        Iterator<DiscreteAction<? extends E>> iter = actions.iterator();
-        while (iter.hasNext())
+        Throwable fail = perform(null, actions);
+        if (failIfCanCast(fail, null))
+            throw (E) fail;
+    }
+
+    public static Throwable perform(Throwable accumulate, Stream<? extends DiscreteAction<?>> actions)
+    {
+        return perform(accumulate, actions.iterator());
+    }
+
+    public static Throwable perform(Throwable accumulate, Iterator<? extends DiscreteAction<?>> actions)
+    {
+        while (actions.hasNext())
         {
-            DiscreteAction<? extends E> action = iter.next();
+            DiscreteAction<?> action = actions.next();
             try
             {
                 action.perform();
             }
             catch (Throwable t)
             {
-                fail = merge(fail, t);
+                accumulate = merge(accumulate, t);
             }
         }
-
-        if (failIfCanCast(fail, null))
-            throw (E) fail;
+        return accumulate;
     }
 
     @SafeVarargs
     public static void perform(File against, FileOpType opType, DiscreteAction<? extends IOException> ... actions)
     {
-        perform(Arrays.stream(actions).map((action) -> () ->
+        perform(against.getPath(), opType, actions);
+    }
+
+    @SafeVarargs
+    public static void perform(String filePath, FileOpType opType, DiscreteAction<? extends IOException> ... actions)
+    {
+        maybeFail(perform(null, filePath, opType, actions));
+    }
+
+    @SafeVarargs
+    public static Throwable perform(Throwable accumulate, String filePath, FileOpType opType, DiscreteAction<? extends IOException> ... actions)
+    {
+        return perform(accumulate, filePath, opType, Arrays.stream(actions));
+    }
+
+    public static Throwable perform(Throwable accumulate, String filePath, FileOpType opType, Stream<DiscreteAction<? extends IOException>> actions)
+    {
+        return perform(accumulate, actions.map((action) -> () ->
         {
             try
             {
@@ -112,7 +137,7 @@ public final class Throwables
             }
             catch (IOException e)
             {
-                throw (opType == FileOpType.WRITE) ? new FSWriteError(e, against) : new FSReadError(e, against);
+                throw (opType == FileOpType.WRITE) ? new FSWriteError(e, filePath) : new FSReadError(e, filePath);
             }
         }));
     }

src/java/org/apache/cassandra/utils/vint/VIntCoding.java

@@ -67,7 +67,7 @@ public class VIntCoding
             return firstByte;
 
         int size = numberOfExtraBytesToRead(firstByte);
-        long retval = firstByte & firstByteValueMask(size);;
+        long retval = firstByte & firstByteValueMask(size);
         for (int ii = 0; ii < size; ii++)
         {
             byte b = input.readByte();

test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java

@@ -60,8 +60,7 @@ import org.apache.cassandra.db.commitlog.CommitLogSegment;
 import org.apache.cassandra.db.commitlog.CommitLogReplayer.CommitLogReplayException;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.util.ByteBufferDataInput;
-import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -535,13 +534,12 @@ public class CommitLogTest
     {
         ByteBuffer buf = ByteBuffer.allocate(1024);
         CommitLogDescriptor.writeHeader(buf, desc);
-        long length = buf.position();
         // Put some extra data in the stream.
         buf.putDouble(0.1);
         buf.flip();
-        FileDataInput input = new ByteBufferDataInput(buf, "input", 0, 0);
+
+        DataInputBuffer input = new DataInputBuffer(buf, false);
         CommitLogDescriptor read = CommitLogDescriptor.readHeader(input);
-        Assert.assertEquals("Descriptor length", length, input.getFilePointer());
         Assert.assertEquals("Descriptors", desc, read);
     }
 

test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java

@@ -29,6 +29,7 @@ import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.rows.SerializationHelper;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.NIODataInputStream;
+import org.apache.cassandra.io.util.RebufferingInputStream;
 
 /**
  * Utility class for tests needing to examine the commitlog contents.
@@ -60,7 +61,7 @@ public class CommitLogTestReplayer extends CommitLogReplayer
     @Override
     void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
     {
-        NIODataInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size);
+        RebufferingInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size);
         Mutation mutation;
         try
         {

test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java

@@ -17,96 +17,217 @@
  */
 package org.apache.cassandra.hints;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.zip.CRC32;
 
 import org.junit.Test;
 
-import org.apache.cassandra.hints.ChecksummedDataInput;
-import org.apache.cassandra.io.util.AbstractDataInput;
 import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 public class ChecksummedDataInputTest
 {
     @Test
-    public void testThatItWorks() throws IOException
+    public void testReadMethods() throws IOException
     {
-        // fill a bytebuffer with some input
-        DataOutputBuffer out = new DataOutputBuffer();
-        out.write(127);
-        out.write(new byte[]{ 0, 1, 2, 3, 4, 5, 6 });
-        out.writeBoolean(false);
-        out.writeByte(10);
-        out.writeChar('t');
-        out.writeDouble(3.3);
-        out.writeFloat(2.2f);
-        out.writeInt(42);
-        out.writeLong(Long.MAX_VALUE);
-        out.writeShort(Short.MIN_VALUE);
-        out.writeUTF("utf");
-        ByteBuffer buffer = out.buffer();
+        // Make sure this array is bigger than the reader buffer size
+        // so we test updating the crc across buffer boundaries
+        byte[] b = new byte[RandomAccessReader.DEFAULT_BUFFER_SIZE * 2];
+        for (int i = 0; i < b.length; i++)
+            b[i] = (byte)i;
 
-        // calculate resulting CRC
+        ByteBuffer buffer;
+
+        // fill a bytebuffer with some input
+        try (DataOutputBuffer out = new DataOutputBuffer())
+        {
+            out.write(127);
+            out.write(b);
+            out.writeBoolean(false);
+            out.writeByte(10);
+            out.writeChar('t');
+            out.writeDouble(3.3);
+            out.writeFloat(2.2f);
+            out.writeInt(42);
+            out.writeLong(Long.MAX_VALUE);
+            out.writeShort(Short.MIN_VALUE);
+            out.writeUTF("utf");
+            out.writeVInt(67L);
+            out.writeUnsignedVInt(88L);
+            out.writeBytes("abcdefghi");
+
+            buffer = out.buffer();
+        }
+
+        // calculate expected CRC
         CRC32 crc = new CRC32();
         FBUtilities.updateChecksum(crc, buffer);
-        int expectedCRC = (int) crc.getValue();
 
-        ChecksummedDataInput crcInput = ChecksummedDataInput.wrap(new DummyByteBufferDataInput(buffer.duplicate()));
-        crcInput.limit(buffer.remaining());
+        // save the buffer to file to create a RAR
+        File file = File.createTempFile("testReadMethods", "1");
+        file.deleteOnExit();
+        try (SequentialWriter writer = SequentialWriter.open(file))
+        {
+            writer.write(buffer);
+            writer.writeInt((int) crc.getValue());
+            writer.finish();
+        }
 
-        // assert that we read all the right values back
-        assertEquals(127, crcInput.read());
-        byte[] bytes = new byte[7];
-        crcInput.readFully(bytes);
-        assertTrue(Arrays.equals(new byte[]{ 0, 1, 2, 3, 4, 5, 6 }, bytes));
-        assertEquals(false, crcInput.readBoolean());
-        assertEquals(10, crcInput.readByte());
-        assertEquals('t', crcInput.readChar());
-        assertEquals(3.3, crcInput.readDouble());
-        assertEquals(2.2f, crcInput.readFloat());
-        assertEquals(42, crcInput.readInt());
-        assertEquals(Long.MAX_VALUE, crcInput.readLong());
-        assertEquals(Short.MIN_VALUE, crcInput.readShort());
-        assertEquals("utf", crcInput.readUTF());
+        assertTrue(file.exists());
+        assertEquals(buffer.remaining() + 4, file.length());
 
-        // assert that the crc matches, and that we've read exactly as many bytes as expected
-        assertEquals(0, crcInput.bytesRemaining());
-        assertEquals(expectedCRC, crcInput.getCrc());
+        try (ChecksummedDataInput reader = ChecksummedDataInput.open(file))
+        {
+            reader.limit(buffer.remaining() + 4);
+
+            // assert that we read all the right values back
+            assertEquals(127, reader.read());
+            byte[] bytes = new byte[b.length];
+            reader.readFully(bytes);
+            assertTrue(Arrays.equals(bytes, b));
+            assertEquals(false, reader.readBoolean());
+            assertEquals(10, reader.readByte());
+            assertEquals('t', reader.readChar());
+            assertEquals(3.3, reader.readDouble());
+            assertEquals(2.2f, reader.readFloat());
+            assertEquals(42, reader.readInt());
+            assertEquals(Long.MAX_VALUE, reader.readLong());
+            assertEquals(Short.MIN_VALUE, reader.readShort());
+            assertEquals("utf", reader.readUTF());
+            assertEquals(67L, reader.readVInt());
+            assertEquals(88L, reader.readUnsignedVInt());
+            assertEquals("abcdefghi", new String(ByteBufferUtil.read(reader, 9).array(), StandardCharsets.UTF_8));
+
+            // assert that the crc matches, and that we've read exactly as many bytes as expected
+            assertTrue(reader.checkCrc());
+            assertEquals(0, reader.bytesRemaining());
+
+            reader.checkLimit(0);
+        }
     }
 
-    private static final class DummyByteBufferDataInput extends AbstractDataInput
+    @Test
+    public void testResetCrc() throws IOException
     {
-        private final ByteBuffer buffer;
+        CRC32 crc = new CRC32();
+        ByteBuffer buffer;
 
-        DummyByteBufferDataInput(ByteBuffer buffer)
+        // fill a bytebuffer with some input
+        try (DataOutputBuffer out = new DataOutputBuffer())
         {
-            this.buffer = buffer;
+            out.write(127);
+            out.writeBoolean(false);
+            out.writeByte(10);
+            out.writeChar('t');
+
+            buffer = out.buffer();
+            FBUtilities.updateChecksum(crc, buffer);
+            out.writeInt((int) crc.getValue());
+
+            int bufferPos = out.getLength();
+            out.writeDouble(3.3);
+            out.writeFloat(2.2f);
+            out.writeInt(42);
+
+            buffer = out.buffer();
+            buffer.position(bufferPos);
+            crc.reset();
+            FBUtilities.updateChecksum(crc, buffer);
+
+            out.writeInt((int) crc.getValue());
+            buffer = out.buffer();
         }
 
-        public void seek(long position)
+        // save the buffer to file to create a RAR
+        File file = File.createTempFile("testResetCrc", "1");
+        file.deleteOnExit();
+        try (SequentialWriter writer = SequentialWriter.open(file))
         {
-            throw new UnsupportedOperationException();
+            writer.write(buffer);
+            writer.finish();
         }
 
-        public long getPosition()
+        assertTrue(file.exists());
+        assertEquals(buffer.remaining(), file.length());
+
+        try (ChecksummedDataInput reader = ChecksummedDataInput.open(file))
         {
-            throw new UnsupportedOperationException();
+            reader.limit(buffer.remaining());
+
+            // assert that we read all the right values back
+            assertEquals(127, reader.read());
+            assertEquals(false, reader.readBoolean());
+            assertEquals(10, reader.readByte());
+            assertEquals('t', reader.readChar());
+            assertTrue(reader.checkCrc());
+
+            reader.resetCrc();
+            assertEquals(3.3, reader.readDouble());
+            assertEquals(2.2f, reader.readFloat());
+            assertEquals(42, reader.readInt());
+            assertTrue(reader.checkCrc());
+            assertEquals(0, reader.bytesRemaining());
+        }
+    }
+
+    @Test
+    public void testFailedCrc() throws IOException
+    {
+        CRC32 crc = new CRC32();
+        ByteBuffer buffer;
+
+        // fill a bytebuffer with some input
+        try (DataOutputBuffer out = new DataOutputBuffer())
+        {
+            out.write(127);
+            out.writeBoolean(false);
+            out.writeByte(10);
+            out.writeChar('t');
+
+            buffer = out.buffer();
+            FBUtilities.updateChecksum(crc, buffer);
+
+            // update twice so it won't match
+            FBUtilities.updateChecksum(crc, buffer);
+            out.writeInt((int) crc.getValue());
+
+            buffer = out.buffer();
         }
 
-        public long getPositionLimit()
+        // save the buffer to file to create a RAR
+        File file = File.createTempFile("testFailedCrc", "1");
+        file.deleteOnExit();
+        try (SequentialWriter writer = SequentialWriter.open(file))
         {
-            throw new UnsupportedOperationException();
+            writer.write(buffer);
+            writer.finish();
         }
 
-        public int read()
+        assertTrue(file.exists());
+        assertEquals(buffer.remaining(), file.length());
+
+        try (ChecksummedDataInput reader = ChecksummedDataInput.open(file))
         {
-            return buffer.get() & 0xFF;
+            reader.limit(buffer.remaining());
+
+            // assert that we read all the right values back
+            assertEquals(127, reader.read());
+            assertEquals(false, reader.readBoolean());
+            assertEquals(10, reader.readByte());
+            assertEquals('t', reader.readChar());
+            assertFalse(reader.checkCrc());
+            assertEquals(0, reader.bytesRemaining());
         }
     }
 }

test/unit/org/apache/cassandra/io/RandomAccessReaderTest.java

@@ -1,269 +0,0 @@
-package org.apache.cassandra.io;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-import org.apache.cassandra.io.util.ChannelProxy;
-import org.apache.cassandra.io.util.FileMark;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.io.util.SequentialWriter;
-
-public class RandomAccessReaderTest
-{
-    @Test
-    public void testReadFully() throws IOException
-    {
-        testReadImpl(1, 0);
-    }
-
-    @Test
-    public void testReadLarge() throws IOException
-    {
-        testReadImpl(1000, 0);
-    }
-
-    @Test
-    public void testReadLargeWithSkip() throws IOException
-    {
-        testReadImpl(1000, 322);
-    }
-
-    @Test
-    public void testReadBufferSizeNotAligned() throws IOException
-    {
-        testReadImpl(1000, 0, 5122);
-    }
-
-    private void testReadImpl(int numIterations, int skipIterations) throws IOException
-    {
-        testReadImpl(numIterations, skipIterations, RandomAccessReader.DEFAULT_BUFFER_SIZE);
-    }
-
-    private void testReadImpl(int numIterations, int skipIterations, int bufferSize) throws IOException
-    {
-        final File f = File.createTempFile("testReadFully", "1");
-        final String expected = "The quick brown fox jumps over the lazy dog";
-
-        SequentialWriter writer = SequentialWriter.open(f);
-        for (int i = 0; i < numIterations; i++)
-            writer.write(expected.getBytes());
-        writer.finish();
-
-        assert f.exists();
-
-        ChannelProxy channel = new ChannelProxy(f);
-        RandomAccessReader reader = RandomAccessReader.open(channel, bufferSize, -1L);
-        assertEquals(f.getAbsolutePath(), reader.getPath());
-        assertEquals(expected.length() * numIterations, reader.length());
-
-        if (skipIterations > 0)
-        {
-            reader.seek(skipIterations * expected.length());
-        }
-
-        byte[] b = new byte[expected.length()];
-        int n = numIterations - skipIterations;
-        for (int i = 0; i < n; i++)
-        {
-            reader.readFully(b);
-            assertEquals(expected, new String(b));
-        }
-
-        assertTrue(reader.isEOF());
-        assertEquals(0, reader.bytesRemaining());
-
-        reader.close();
-        channel.close();
-    }
-
-    @Test
-    public void testReadBytes() throws IOException
-    {
-        File f = File.createTempFile("testReadBytes", "1");
-        final String expected = "The quick brown fox jumps over the lazy dog";
-
-        SequentialWriter writer = SequentialWriter.open(f);
-        writer.write(expected.getBytes());
-        writer.finish();
-
-        assert f.exists();
-
-        ChannelProxy channel = new ChannelProxy(f);
-        RandomAccessReader reader = RandomAccessReader.open(channel);
-        assertEquals(f.getAbsolutePath(), reader.getPath());
-        assertEquals(expected.length(), reader.length());
-
-        ByteBuffer b = reader.readBytes(expected.length());
-        assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
-
-        assertTrue(reader.isEOF());
-        assertEquals(0, reader.bytesRemaining());
-
-        reader.close();
-        channel.close();
-    }
-
-    @Test
-    public void testReset() throws IOException
-    {
-        File f = File.createTempFile("testMark", "1");
-        final String expected = "The quick brown fox jumps over the lazy dog";
-        final int numIterations = 10;
-
-        SequentialWriter writer = SequentialWriter.open(f);
-        for (int i = 0; i < numIterations; i++)
-            writer.write(expected.getBytes());
-        writer.finish();
-
-        assert f.exists();
-
-        ChannelProxy channel = new ChannelProxy(f);
-        RandomAccessReader reader = RandomAccessReader.open(channel);
-        assertEquals(expected.length() * numIterations, reader.length());
-
-        ByteBuffer b = reader.readBytes(expected.length());
-        assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
-
-        assertFalse(reader.isEOF());
-        assertEquals((numIterations - 1) * expected.length(), reader.bytesRemaining());
-
-        FileMark mark = reader.mark();
-        assertEquals(0, reader.bytesPastMark());
-        assertEquals(0, reader.bytesPastMark(mark));
-
-        for (int i = 0; i < (numIterations - 1); i++)
-        {
-            b = reader.readBytes(expected.length());
-            assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
-        }
-        assertTrue(reader.isEOF());
-        assertEquals(expected.length() * (numIterations -1), reader.bytesPastMark());
-        assertEquals(expected.length() * (numIterations - 1), reader.bytesPastMark(mark));
-
-        reader.reset(mark);
-        assertEquals(0, reader.bytesPastMark());
-        assertEquals(0, reader.bytesPastMark(mark));
-        assertFalse(reader.isEOF());
-        for (int i = 0; i < (numIterations - 1); i++)
-        {
-            b = reader.readBytes(expected.length());
-            assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
-        }
-
-        reader.reset();
-        assertEquals(0, reader.bytesPastMark());
-        assertEquals(0, reader.bytesPastMark(mark));
-        assertFalse(reader.isEOF());
-        for (int i = 0; i < (numIterations - 1); i++)
-        {
-            b = reader.readBytes(expected.length());
-            assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
-        }
-
-        assertTrue(reader.isEOF());
-        reader.close();
-        channel.close();
-    }
-
-    @Test
-    public void testSeekSingleThread() throws IOException, InterruptedException
-    {
-        testSeek(1);
-    }
-
-    @Test
-    public void testSeekMultipleThreads() throws IOException, InterruptedException
-    {
-        testSeek(10);
-    }
-
-    private void testSeek(int numThreads) throws IOException, InterruptedException
-    {
-        final File f = File.createTempFile("testMark", "1");
-        final String[] expected = new String[10];
-        int len = 0;
-        for (int i = 0; i < expected.length; i++)
-        {
-            expected[i] = UUID.randomUUID().toString();
-            len += expected[i].length();
-        }
-        final int totalLength = len;
-
-        SequentialWriter writer = SequentialWriter.open(f);
-        for (int i = 0; i < expected.length; i++)
-            writer.write(expected[i].getBytes());
-        writer.finish();
-
-        assert f.exists();
-
-        final ChannelProxy channel = new ChannelProxy(f);
-
-        final Runnable worker = new Runnable() {
-
-            @Override
-            public void run()
-            {
-                try
-                {
-                    RandomAccessReader reader = RandomAccessReader.open(channel);
-                    assertEquals(totalLength, reader.length());
-
-                    ByteBuffer b = reader.readBytes(expected[0].length());
-                    assertEquals(expected[0], new String(b.array(), Charset.forName("UTF-8")));
-
-                    assertFalse(reader.isEOF());
-                    assertEquals(totalLength - expected[0].length(), reader.bytesRemaining());
-
-                    long filePointer = reader.getFilePointer();
-
-                    for (int i = 1; i < expected.length; i++)
-                    {
-                        b = reader.readBytes(expected[i].length());
-                        assertEquals(expected[i], new String(b.array(), Charset.forName("UTF-8")));
-                    }
-                    assertTrue(reader.isEOF());
-
-                    reader.seek(filePointer);
-                    assertFalse(reader.isEOF());
-                    for (int i = 1; i < expected.length; i++)
-                    {
-                        b = reader.readBytes(expected[i].length());
-                        assertEquals(expected[i], new String(b.array(), Charset.forName("UTF-8")));
-                    }
-
-                    assertTrue(reader.isEOF());
-                    reader.close();
-                }
-                catch (Exception ex)
-                {
-                    ex.printStackTrace();
-                    fail(ex.getMessage());
-                }
-            }
-        };
-
-        if(numThreads == 1)
-        {
-            worker.run();
-            return;
-        }
-
-        ExecutorService executor = Executors.newFixedThreadPool(numThreads);
-        for (int i = 0; i < numThreads; i++)
-            executor.submit(worker);
-
-        executor.shutdown();
-        executor.awaitTermination(1, TimeUnit.MINUTES);
-
-        channel.close();
-    }
-}

test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java

@@ -31,6 +31,7 @@ import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.util.ChannelProxy;
 import org.apache.cassandra.io.util.FileMark;
+import org.apache.cassandra.io.util.MmappedRegions;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.io.util.SequentialWriter;
 import org.apache.cassandra.schema.CompressionParams;
@@ -39,6 +40,8 @@ import org.apache.cassandra.utils.SyncUtil;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
 
 public class CompressedRandomAccessReaderTest
 {
@@ -46,16 +49,24 @@ public class CompressedRandomAccessReaderTest
     public void testResetAndTruncate() throws IOException
     {
         // test reset in current buffer or previous one
-        testResetAndTruncate(File.createTempFile("normal", "1"), false, 10);
-        testResetAndTruncate(File.createTempFile("normal", "2"), false, CompressionParams.DEFAULT_CHUNK_LENGTH);
+        testResetAndTruncate(File.createTempFile("normal", "1"), false, false, 10);
+        testResetAndTruncate(File.createTempFile("normal", "2"), false, false, CompressionParams.DEFAULT_CHUNK_LENGTH);
     }
 
     @Test
     public void testResetAndTruncateCompressed() throws IOException
     {
         // test reset in current buffer or previous one
-        testResetAndTruncate(File.createTempFile("compressed", "1"), true, 10);
-        testResetAndTruncate(File.createTempFile("compressed", "2"), true, CompressionParams.DEFAULT_CHUNK_LENGTH);
+        testResetAndTruncate(File.createTempFile("compressed", "1"), true, false, 10);
+        testResetAndTruncate(File.createTempFile("compressed", "2"), true, false, CompressionParams.DEFAULT_CHUNK_LENGTH);
+    }
+
+    @Test
+    public void testResetAndTruncateCompressedMmap() throws IOException
+    {
+        // test reset in current buffer or previous one
+        testResetAndTruncate(File.createTempFile("compressed_mmap", "1"), true, true, 10);
+        testResetAndTruncate(File.createTempFile("compressed_mmap", "2"), true, true, CompressionParams.DEFAULT_CHUNK_LENGTH);
     }
 
     @Test
@@ -63,87 +74,102 @@ public class CompressedRandomAccessReaderTest
     {
         File f = File.createTempFile("compressed6791_", "3");
         String filename = f.getAbsolutePath();
-        ChannelProxy channel = new ChannelProxy(f);
-        try
+        try(ChannelProxy channel = new ChannelProxy(f))
         {
 
             MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
-            CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(32), sstableMetadataCollector);
+            try(CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(32), sstableMetadataCollector))
+            {
 
-            for (int i = 0; i < 20; i++)
-                writer.write("x".getBytes());
+                for (int i = 0; i < 20; i++)
+                    writer.write("x".getBytes());
 
-            FileMark mark = writer.mark();
-            // write enough garbage to create new chunks:
-            for (int i = 0; i < 40; ++i)
-                writer.write("y".getBytes());
+                FileMark mark = writer.mark();
+                // write enough garbage to create new chunks:
+                for (int i = 0; i < 40; ++i)
+                    writer.write("y".getBytes());
 
-            writer.resetAndTruncate(mark);
+                writer.resetAndTruncate(mark);
 
-            for (int i = 0; i < 20; i++)
-                writer.write("x".getBytes());
-            writer.finish();
+                for (int i = 0; i < 20; i++)
+                    writer.write("x".getBytes());
+                writer.finish();
+            }
 
-            CompressedRandomAccessReader reader = CompressedRandomAccessReader.open(channel, new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32));
-            String res = reader.readLine();
-            assertEquals(res, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
-            assertEquals(40, res.length());
+            try(RandomAccessReader reader = new CompressedRandomAccessReader.Builder(channel,
+                                                                                     new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32))
+                                            .build())
+            {
+                String res = reader.readLine();
+                assertEquals(res, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+                assertEquals(40, res.length());
+            }
         }
         finally
         {
-            // cleanup
-            channel.close();
-
             if (f.exists())
-                f.delete();
+                assertTrue(f.delete());
             File metadata = new File(filename+ ".metadata");
             if (metadata.exists())
                 metadata.delete();
         }
     }
 
-    private void testResetAndTruncate(File f, boolean compressed, int junkSize) throws IOException
+    private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize) throws IOException
     {
         final String filename = f.getAbsolutePath();
-        ChannelProxy channel = new ChannelProxy(f);
-        try
+        try(ChannelProxy channel = new ChannelProxy(f))
         {
             MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null);
-            SequentialWriter writer = compressed
+            try(SequentialWriter writer = compressed
                 ? new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(), sstableMetadataCollector)
-                : SequentialWriter.open(f);
-
-            writer.write("The quick ".getBytes());
-            FileMark mark = writer.mark();
-            writer.write("blue fox jumps over the lazy dog".getBytes());
-
-            // write enough to be sure to change chunk
-            for (int i = 0; i < junkSize; ++i)
+                : SequentialWriter.open(f))
             {
-                writer.write((byte)1);
+                writer.write("The quick ".getBytes());
+                FileMark mark = writer.mark();
+                writer.write("blue fox jumps over the lazy dog".getBytes());
+
+                // write enough to be sure to change chunk
+                for (int i = 0; i < junkSize; ++i)
+                {
+                    writer.write((byte) 1);
+                }
+
+                writer.resetAndTruncate(mark);
+                writer.write("brown fox jumps over the lazy dog".getBytes());
+                writer.finish();
+            }
+            assert f.exists();
+
+            CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) : null;
+            RandomAccessReader.Builder builder = compressed
+                                                 ? new CompressedRandomAccessReader.Builder(channel, compressionMetadata)
+                                                 : new RandomAccessReader.Builder(channel);
+
+            if (usemmap)
+            {
+                if (compressed)
+                    builder.regions(MmappedRegions.map(channel, compressionMetadata));
+                else
+                    builder.regions(MmappedRegions.map(channel, f.length()));
             }
 
-            writer.resetAndTruncate(mark);
-            writer.write("brown fox jumps over the lazy dog".getBytes());
-            writer.finish();
+            try(RandomAccessReader reader = builder.build())
+            {
+                String expected = "The quick brown fox jumps over the lazy dog";
+                assertEquals(expected.length(), reader.length());
+                byte[] b = new byte[expected.length()];
+                reader.readFully(b);
+                assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + '\'';
+            }
 
-            assert f.exists();
-            RandomAccessReader reader = compressed
-                                      ? CompressedRandomAccessReader.open(channel, new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32))
-                                      : RandomAccessReader.open(f);
-            String expected = "The quick brown fox jumps over the lazy dog";
-            assertEquals(expected.length(), reader.length());
-            byte[] b = new byte[expected.length()];
-            reader.readFully(b);
-            assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + "'";
+            if (usemmap)
+                builder.regions.close();
         }
         finally
         {
-            // cleanup
-            channel.close();
-
             if (f.exists())
-                f.delete();
+                assertTrue(f.delete());
             File metadata = new File(filename + ".metadata");
             if (compressed && metadata.exists())
                 metadata.delete();
@@ -161,6 +187,9 @@ public class CompressedRandomAccessReaderTest
         File metadata = new File(file.getPath() + ".meta");
         metadata.deleteOnExit();
 
+        assertTrue(file.createNewFile());
+        assertTrue(metadata.createNewFile());
+
         MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null);
         try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), CompressionParams.snappy(), sstableMetadataCollector))
         {
@@ -168,74 +197,74 @@ public class CompressedRandomAccessReaderTest
             writer.finish();
         }
 
-        ChannelProxy channel = new ChannelProxy(file);
-
-        // open compression metadata and get chunk information
-        CompressionMetadata meta = new CompressionMetadata(metadata.getPath(), file.length(), ChecksumType.CRC32);
-        CompressionMetadata.Chunk chunk = meta.chunkFor(0);
-
-        RandomAccessReader reader = CompressedRandomAccessReader.open(channel, meta);
-        // read and verify compressed data
-        assertEquals(CONTENT, reader.readLine());
-
-        Random random = new Random();
-        RandomAccessFile checksumModifier = null;
-
-        try
+        try(ChannelProxy channel = new ChannelProxy(file))
         {
-            checksumModifier = new RandomAccessFile(file, "rw");
-            byte[] checksum = new byte[4];
+            // open compression metadata and get chunk information
+            CompressionMetadata meta = new CompressionMetadata(metadata.getPath(), file.length(), ChecksumType.CRC32);
+            CompressionMetadata.Chunk chunk = meta.chunkFor(0);
 
-            // seek to the end of the compressed chunk
-            checksumModifier.seek(chunk.length);
-            // read checksum bytes
-            checksumModifier.read(checksum);
-            // seek back to the chunk end
-            checksumModifier.seek(chunk.length);
+            try(RandomAccessReader reader = new CompressedRandomAccessReader.Builder(channel, meta).build())
+            {// read and verify compressed data
+                assertEquals(CONTENT, reader.readLine());
 
-            // lets modify one byte of the checksum on each iteration
-            for (int i = 0; i < checksum.length; i++)
-            {
-                checksumModifier.write(random.nextInt());
-                SyncUtil.sync(checksumModifier); // making sure that change was synced with disk
+                Random random = new Random();
+                RandomAccessFile checksumModifier = null;
 
-                final RandomAccessReader r = CompressedRandomAccessReader.open(channel, meta);
-
-                Throwable exception = null;
                 try
                 {
-                    r.readLine();
+                    checksumModifier = new RandomAccessFile(file, "rw");
+                    byte[] checksum = new byte[4];
+
+                    // seek to the end of the compressed chunk
+                    checksumModifier.seek(chunk.length);
+                    // read checksum bytes
+                    checksumModifier.read(checksum);
+                    // seek back to the chunk end
+                    checksumModifier.seek(chunk.length);
+
+                    // lets modify one byte of the checksum on each iteration
+                    for (int i = 0; i < checksum.length; i++)
+                    {
+                        checksumModifier.write(random.nextInt());
+                        SyncUtil.sync(checksumModifier); // making sure that change was synced with disk
+
+                        try (final RandomAccessReader r = new CompressedRandomAccessReader.Builder(channel, meta).build())
+                        {
+                            Throwable exception = null;
+                            try
+                            {
+                                r.readLine();
+                            }
+                            catch (Throwable t)
+                            {
+                                exception = t;
+                            }
+                            assertNotNull(exception);
+                            assertSame(exception.getClass(), CorruptSSTableException.class);
+                            assertSame(exception.getCause().getClass(), CorruptBlockException.class);
+                        }
+                    }
+
+                    // lets write original checksum and check if we can read data
+                    updateChecksum(checksumModifier, chunk.length, checksum);
+
+                    try (RandomAccessReader cr = new CompressedRandomAccessReader.Builder(channel, meta).build())
+                    {
+                        // read and verify compressed data
+                        assertEquals(CONTENT, cr.readLine());
+                        // close reader
+                    }
                 }
-                catch (Throwable t)
+                finally
                 {
-                    exception = t;
+                    if (checksumModifier != null)
+                        checksumModifier.close();
                 }
-                assertNotNull(exception);
-                assertEquals(exception.getClass(), CorruptSSTableException.class);
-                assertEquals(exception.getCause().getClass(), CorruptBlockException.class);
-
-                r.close();
             }
-
-            // lets write original checksum and check if we can read data
-            updateChecksum(checksumModifier, chunk.length, checksum);
-
-            reader = CompressedRandomAccessReader.open(channel, meta);
-            // read and verify compressed data
-            assertEquals(CONTENT, reader.readLine());
-            // close reader
-            reader.close();
-        }
-        finally
-        {
-            channel.close();
-
-            if (checksumModifier != null)
-                checksumModifier.close();
         }
     }
 
-    private void updateChecksum(RandomAccessFile file, long checksumOffset, byte[] checksum) throws IOException
+    private static void updateChecksum(RandomAccessFile file, long checksumOffset, byte[] checksum) throws IOException
     {
         file.seek(checksumOffset);
         file.write(checksum);

test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java

@@ -117,7 +117,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest
             }
 
             assert f.exists();
-            RandomAccessReader reader = CompressedRandomAccessReader.open(channel, new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32));
+            RandomAccessReader reader = new CompressedRandomAccessReader.Builder(channel, new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32)).build();
             assertEquals(dataPre.length + rawPost.length, reader.length());
             byte[] result = new byte[(int)reader.length()];
 

test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java

@@ -46,6 +46,7 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.MmappedRegions;
 import org.apache.cassandra.io.util.MmappedSegmentedFile;
 import org.apache.cassandra.io.util.SegmentedFile;
 import org.apache.cassandra.schema.CachingParams;
@@ -133,7 +134,7 @@ public class SSTableReaderTest
     @Test
     public void testSpannedIndexPositions() throws IOException
     {
-        MmappedSegmentedFile.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of segments
+        MmappedRegions.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of segments
 
         Keyspace keyspace = Keyspace.open(KEYSPACE1);
         ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");

test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java

@@ -542,14 +542,12 @@ public class BufferedDataOutputStreamTest
 
         ndosp.flush();
 
-        @SuppressWarnings("resource")
-        ByteBufferDataInput bbdi = new ByteBufferDataInput(ByteBuffer.wrap(generated.toByteArray()), "", 0, 0);
-
+        DataInputBuffer in = new DataInputBuffer(generated.toByteArray());
         assertEquals(expectedSize, generated.toByteArray().length);
 
         for (long v : testValues)
         {
-            assertEquals(v, bbdi.readVInt());
+            assertEquals(v, in.readVInt());
         }
     }
 
@@ -574,13 +572,11 @@ public class BufferedDataOutputStreamTest
 
         ndosp.flush();
 
-        @SuppressWarnings("resource")
-        ByteBufferDataInput bbdi = new ByteBufferDataInput(ByteBuffer.wrap(generated.toByteArray()), "", 0, 0);
-
+        DataInputBuffer in = new DataInputBuffer(generated.toByteArray());
         assertEquals(expectedSize, generated.toByteArray().length);
 
         for (long v : testValues)
-            assertEquals(v, bbdi.readUnsignedVInt());
+            assertEquals(v, in.readUnsignedVInt());
     }
 
     @Test

test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java

@@ -25,9 +25,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.channels.ClosedChannelException;
 import java.util.Arrays;
-import java.util.concurrent.Callable;
 
 import static org.apache.cassandra.Util.expectEOF;
 import static org.apache.cassandra.Util.expectException;
@@ -96,7 +94,7 @@ public class BufferedRandomAccessFileTest
 
         // test readBytes(int) method
         r.seek(0);
-        ByteBuffer fileContent = r.readBytes((int) w.length());
+        ByteBuffer fileContent = ByteBufferUtil.read(r, (int) w.length());
         assertEquals(fileContent.limit(), w.length());
         assert ByteBufferUtil.string(fileContent).equals("Hello" + new String(bigData));
 
@@ -204,25 +202,19 @@ public class BufferedRandomAccessFileTest
         final ChannelProxy channel = new ChannelProxy(w.getPath());
         final RandomAccessReader r = RandomAccessReader.open(channel);
 
-        ByteBuffer content = r.readBytes((int) r.length());
+        ByteBuffer content = ByteBufferUtil.read(r, (int) r.length());
 
         // after reading whole file we should be at EOF
         assertEquals(0, ByteBufferUtil.compare(content, data));
         assert r.bytesRemaining() == 0 && r.isEOF();
 
         r.seek(0);
-        content = r.readBytes(10); // reading first 10 bytes
+        content = ByteBufferUtil.read(r, 10); // reading first 10 bytes
         assertEquals(ByteBufferUtil.compare(content, "cccccccccc".getBytes()), 0);
         assertEquals(r.bytesRemaining(), r.length() - content.limit());
 
         // trying to read more than file has right now
-        expectEOF(new Callable<Object>()
-        {
-            public Object call() throws IOException
-            {
-                return r.readBytes((int) r.length() + 10);
-            }
-        });
+        expectEOF(() -> ByteBufferUtil.read(r, (int) r.length() + 10));
 
         w.finish();
         r.close();
@@ -249,23 +241,9 @@ public class BufferedRandomAccessFileTest
         assertEquals(file.bytesRemaining(), file.length() - 20);
 
         // trying to seek past the end of the file should produce EOFException
-        expectException(new Callable<Object>()
-        {
-            public Object call()
-            {
-                file.seek(file.length() + 30);
-                return null;
-            }
-        }, IllegalArgumentException.class);
+        expectException(() -> { file.seek(file.length() + 30); return null; }, IllegalArgumentException.class);
 
-        expectException(new Callable<Object>()
-        {
-            public Object call() throws IOException
-            {
-                file.seek(-1);
-                return null;
-            }
-        }, IllegalArgumentException.class); // throws IllegalArgumentException
+        expectException(() -> { file.seek(-1); return null; }, IllegalArgumentException.class); // throws IllegalArgumentException
 
         file.close();
         channel.close();
@@ -352,16 +330,11 @@ public class BufferedRandomAccessFileTest
             {
                 File file1 = writeTemporaryFile(new byte[16]);
                 try (final ChannelProxy channel = new ChannelProxy(file1);
-                     final RandomAccessReader file = RandomAccessReader.open(channel, bufferSize, -1L))
+                     final RandomAccessReader file = new RandomAccessReader.Builder(channel)
+                                                     .bufferSize(bufferSize)
+                                                     .build())
                 {
-                    expectEOF(new Callable<Object>()
-                    {
-                        public Object call() throws IOException
-                        {
-                            file.readFully(target, offset, 17);
-                            return null;
-                        }
-                    });
+                    expectEOF(() -> { file.readFully(target, offset, 17); return null; });
                 }
             }
 
@@ -370,15 +343,11 @@ public class BufferedRandomAccessFileTest
             {
                 File file1 = writeTemporaryFile(new byte[16]);
                 try (final ChannelProxy channel = new ChannelProxy(file1);
-                     final RandomAccessReader file = RandomAccessReader.open(channel, bufferSize, -1L))
+                     final RandomAccessReader file = new RandomAccessReader.Builder(channel).bufferSize(bufferSize).build())
                 {
-                    expectEOF(new Callable<Object>()
-                    {
-                        public Object call() throws IOException
-                        {
-                            while (true)
-                                file.readFully(target, 0, n);
-                        }
+                    expectEOF(() -> {
+                        while (true)
+                            file.readFully(target, 0, n);
                     });
                 }
             }
@@ -459,30 +428,17 @@ public class BufferedRandomAccessFileTest
 
         r.close(); // closing to test read after close
 
-        expectException(new Callable<Object>()
-        {
-            public Object call()
-            {
-                return r.read();
-            }
-        }, AssertionError.class);
+        expectException(() -> r.read(), NullPointerException.class);
 
         //Used to throw ClosedChannelException, but now that it extends BDOSP it just NPEs on the buffer
         //Writing to a BufferedOutputStream that is closed generates no error
         //Going to allow the NPE to throw to catch as a bug any use after close. Notably it won't throw NPE for a
         //write of a 0 length, but that is kind of a corner case
-        expectException(new Callable<Object>()
-        {
-            public Object call() throws IOException
-            {
-                w.write(generateByteArray(1));
-                return null;
-            }
-        }, NullPointerException.class);
+        expectException(() -> { w.write(generateByteArray(1)); return null; }, NullPointerException.class);
 
         try (RandomAccessReader copy = RandomAccessReader.open(new File(r.getPath())))
         {
-            ByteBuffer contents = copy.readBytes((int) copy.length());
+            ByteBuffer contents = ByteBufferUtil.read(copy, (int) copy.length());
 
             assertEquals(contents.limit(), data.length);
             assertEquals(ByteBufferUtil.compare(contents, data), 0);
@@ -526,7 +482,7 @@ public class BufferedRandomAccessFileTest
         channel.close();
     }
 
-    @Test (expected = AssertionError.class)
+    @Test(expected = AssertionError.class)
     public void testAssertionErrorWhenBytesPastMarkIsNegative() throws IOException
     {
         try (SequentialWriter w = createTempFile("brafAssertionErrorWhenBytesPastMarkIsNegative"))
@@ -565,14 +521,7 @@ public class BufferedRandomAccessFileTest
         assertTrue(copy.bytesRemaining() == 0 && copy.isEOF());
 
         // can't seek past the end of the file for read-only files
-        expectException(new Callable<Object>()
-        {
-            public Object call()
-            {
-                copy.seek(copy.length() + 1);
-                return null;
-            }
-        }, IllegalArgumentException.class);
+        expectException(() -> { copy.seek(copy.length() + 1); return null; }, IllegalArgumentException.class);
 
         copy.seek(0);
         copy.skipBytes(5);
@@ -582,7 +531,7 @@ public class BufferedRandomAccessFileTest
         assertTrue(!copy.isEOF());
 
         copy.seek(0);
-        ByteBuffer contents = copy.readBytes((int) copy.length());
+        ByteBuffer contents = ByteBufferUtil.read(copy, (int) copy.length());
 
         assertEquals(contents.limit(), copy.length());
         assertTrue(ByteBufferUtil.compare(contents, data) == 0);

test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.io;
+package org.apache.cassandra.io.util;
 
 import java.io.File;
 import java.io.IOException;
@@ -49,7 +49,7 @@ public class ChecksummedRandomAccessReaderTest
 
         assert data.exists();
 
-        RandomAccessReader reader = ChecksummedRandomAccessReader.open(data, crc);
+        RandomAccessReader reader = new ChecksummedRandomAccessReader.Builder(data, crc).build();
         byte[] b = new byte[expected.length];
         reader.readFully(b);
 
@@ -75,7 +75,7 @@ public class ChecksummedRandomAccessReaderTest
 
         assert data.exists();
 
-        RandomAccessReader reader = ChecksummedRandomAccessReader.open(data, crc);
+        RandomAccessReader reader = new ChecksummedRandomAccessReader.Builder(data, crc).build();
 
         final int seekPosition = 66000;
         reader.seek(seekPosition);
@@ -114,7 +114,7 @@ public class ChecksummedRandomAccessReaderTest
             dataFile.write((byte) 5);
         }
 
-        RandomAccessReader reader = ChecksummedRandomAccessReader.open(data, crc);
+        RandomAccessReader reader = new ChecksummedRandomAccessReader.Builder(data, crc).build();
         byte[] b = new byte[expected.length];
         reader.readFully(b);
 

test/unit/org/apache/cassandra/io/util/FileSegmentInputStreamTest.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Random;
+
+import com.google.common.primitives.Ints;
+import org.junit.Test;
+
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class FileSegmentInputStreamTest
+{
+    private ByteBuffer allocateBuffer(int size)
+    {
+        ByteBuffer ret = ByteBuffer.allocate(Ints.checkedCast(size));
+        long seed = System.nanoTime();
+        //seed = 365238103404423L;
+        System.out.println("Seed " + seed);
+
+        new Random(seed).nextBytes(ret.array());
+        return ret;
+    }
+
+    @Test
+    public void testRead() throws IOException
+    {
+        testRead(0, 4096, 1024);
+        testRead(1024, 4096, 1024);
+        testRead(4096, 4096, 1024);
+    }
+
+    private void testRead(int offset, int size, int checkInterval) throws IOException
+    {
+        final ByteBuffer buffer = allocateBuffer(size);
+        final String path = buffer.toString();
+
+        FileSegmentInputStream reader = new FileSegmentInputStream(buffer.duplicate(), path, offset);
+        assertEquals(path, reader.getPath());
+
+        for (int i = offset; i < (size + offset); i += checkInterval)
+        {
+            reader.seek(i);
+            assertFalse(reader.isEOF());
+            assertEquals(i, reader.getFilePointer());
+
+            buffer.position(i - offset);
+
+            int remaining = buffer.remaining();
+            assertEquals(remaining, reader.bytesRemaining());
+            byte[] expected = new byte[buffer.remaining()];
+            buffer.get(expected);
+            assertTrue(Arrays.equals(expected, ByteBufferUtil.read(reader, remaining).array()));
+
+            assertTrue(reader.isEOF());
+            assertEquals(0, reader.bytesRemaining());
+            assertEquals(buffer.capacity() + offset, reader.getFilePointer());
+        }
+
+        reader.close();
+        reader.close();
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testMarkNotSupported() throws Exception
+    {
+        FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 0);
+        assertFalse(reader.markSupported());
+        assertEquals(0, reader.bytesPastMark(null));
+        reader.mark();
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testResetNotSupported() throws Exception
+    {
+        FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 0);
+        reader.reset(null);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSeekNegative() throws Exception
+    {
+        FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 0);
+        reader.seek(-1);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSeekBeforeOffset() throws Exception
+    {
+        FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 1024);
+        reader.seek(1023);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSeekPastLength() throws Exception
+    {
+        FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 1024);
+        reader.seek(2049);
+    }
+
+    @Test(expected = EOFException.class)
+    public void testReadBytesTooMany() throws Exception
+    {
+        FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 1024);
+        ByteBufferUtil.read(reader, 2049);
+    }
+}

test/unit/org/apache/cassandra/io/util/MemoryTest.java

@@ -18,8 +18,11 @@
 */
 package org.apache.cassandra.io.util;
 
+import java.io.EOFException;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.util.Arrays;
 import java.util.concurrent.ThreadLocalRandom;
 
 import org.junit.Test;
@@ -27,6 +30,10 @@ import org.junit.Test;
 import junit.framework.Assert;
 import org.apache.cassandra.utils.memory.MemoryUtil;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 public class MemoryTest
 {
 
@@ -45,6 +52,36 @@ public class MemoryTest
         memory.close();
     }
 
+    @Test
+    public void testInputStream() throws IOException
+    {
+        byte[] bytes = new byte[4096];
+        ThreadLocalRandom.current().nextBytes(bytes);
+        final Memory memory = Memory.allocate(bytes.length);
+        memory.setBytes(0, bytes, 0, bytes.length);
+
+        try(MemoryInputStream stream = new MemoryInputStream(memory, 1024))
+        {
+            byte[] bb = new byte[bytes.length];
+            assertEquals(bytes.length, stream.available());
+
+            stream.readFully(bb);
+            assertEquals(0, stream.available());
+
+            assertTrue(Arrays.equals(bytes, bb));
+
+            try
+            {
+                stream.readInt();
+                fail("Expected EOF exception");
+            }
+            catch (EOFException e)
+            {
+                //pass
+            }
+        }
+    }
+
     private static void test(ByteBuffer canon, Memory memory)
     {
         ByteBuffer hollow = MemoryUtil.getHollowDirectByteBuffer();

test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java

@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import com.google.common.primitives.Ints;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.io.compress.CompressedSequentialWriter;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.utils.ChecksumType;
+
+import static junit.framework.Assert.assertNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class MmappedRegionsTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(MmappedRegionsTest.class);
+
+    private static ByteBuffer allocateBuffer(int size)
+    {
+        ByteBuffer ret = ByteBuffer.allocate(Ints.checkedCast(size));
+        long seed = System.nanoTime();
+        //seed = 365238103404423L;
+        logger.info("Seed {}", seed);
+
+        new Random(seed).nextBytes(ret.array());
+        return ret;
+    }
+
+    private static File writeFile(String fileName, ByteBuffer buffer) throws IOException
+    {
+        File ret = File.createTempFile(fileName, "1");
+        ret.deleteOnExit();
+
+        try (SequentialWriter writer = SequentialWriter.open(ret))
+        {
+            writer.write(buffer);
+            writer.finish();
+        }
+
+        assert ret.exists();
+        assert ret.length() >= buffer.capacity();
+        return ret;
+
+    }
+
+    @Test
+    public void testEmpty() throws Exception
+    {
+        ByteBuffer buffer = allocateBuffer(1024);
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testEmpty", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
+        {
+            assertTrue(regions.isEmpty());
+            assertTrue(regions.isValid(channel));
+        }
+    }
+
+    @Test
+    public void testTwoSegments() throws Exception
+    {
+        ByteBuffer buffer = allocateBuffer(2048);
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
+        {
+            regions.extend(1024);
+            for (int i = 0; i < 1024; i++)
+            {
+                MmappedRegions.Region region = regions.floor(i);
+                assertNotNull(region);
+                assertEquals(0, region.bottom());
+                assertEquals(1024, region.top());
+            }
+
+            regions.extend(2048);
+            for (int i = 0; i < 2048; i++)
+            {
+                MmappedRegions.Region region = regions.floor(i);
+                assertNotNull(region);
+                if (i < 1024)
+                {
+                    assertEquals(0, region.bottom());
+                    assertEquals(1024, region.top());
+                }
+                else
+                {
+                    assertEquals(1024, region.bottom());
+                    assertEquals(2048, region.top());
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testSmallSegmentSize() throws Exception
+    {
+        int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+        MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+        ByteBuffer buffer = allocateBuffer(4096);
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
+        {
+            regions.extend(1024);
+            regions.extend(2048);
+            regions.extend(4096);
+
+            final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+            for (int i = 0; i < buffer.capacity(); i++)
+            {
+                MmappedRegions.Region region = regions.floor(i);
+                assertNotNull(region);
+                assertEquals(SIZE * (i / SIZE), region.bottom());
+                assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+            }
+        }
+        finally
+        {
+            MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+        }
+    }
+
+    @Test
+    public void testAllocRegions() throws Exception
+    {
+        int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+        MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+        ByteBuffer buffer = allocateBuffer(MmappedRegions.MAX_SEGMENT_SIZE * MmappedRegions.REGION_ALLOC_SIZE * 3);
+
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
+        {
+            regions.extend(buffer.capacity());
+
+            final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+            for (int i = 0; i < buffer.capacity(); i++)
+            {
+                MmappedRegions.Region region = regions.floor(i);
+                assertNotNull(region);
+                assertEquals(SIZE * (i / SIZE), region.bottom());
+                assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+            }
+        }
+        finally
+        {
+            MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+        }
+    }
+
+    @Test
+    public void testCopy() throws Exception
+    {
+        ByteBuffer buffer = allocateBuffer(128 * 1024);
+
+        MmappedRegions snapshot;
+        ChannelProxy channelCopy;
+
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer));
+            MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4))
+        {
+            // create 3 more segments, one per quater capacity
+            regions.extend(buffer.capacity() / 2);
+            regions.extend(3 * buffer.capacity() / 4);
+            regions.extend(buffer.capacity());
+
+            // make a snapshot
+            snapshot = regions.sharedCopy();
+
+            // keep the channel open
+            channelCopy = channel.sharedCopy();
+        }
+
+        assertFalse(snapshot.isCleanedUp());
+
+        final int SIZE = buffer.capacity() / 4;
+        for (int i = 0; i < buffer.capacity(); i++)
+        {
+            MmappedRegions.Region region = snapshot.floor(i);
+            assertNotNull(region);
+            assertEquals(SIZE * (i / SIZE), region.bottom());
+            assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+
+            // check we can access the buffer
+            assertNotNull(region.buffer.duplicate().getInt());
+        }
+
+        assertNull(snapshot.close(null));
+        assertNull(channelCopy.close(null));
+        assertTrue(snapshot.isCleanedUp());
+    }
+
+    @Test(expected = AssertionError.class)
+    public void testCopyCannotExtend() throws Exception
+    {
+        ByteBuffer buffer = allocateBuffer(128 * 1024);
+
+        MmappedRegions snapshot;
+        ChannelProxy channelCopy;
+
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
+        {
+            regions.extend(buffer.capacity() / 2);
+
+            // make a snapshot
+            snapshot = regions.sharedCopy();
+
+            // keep the channel open
+            channelCopy = channel.sharedCopy();
+        }
+
+        try
+        {
+            snapshot.extend(buffer.capacity());
+        }
+        finally
+        {
+            assertNull(snapshot.close(null));
+            assertNull(channelCopy.close(null));
+        }
+    }
+
+    @Test
+    public void testExtendOutOfOrder() throws Exception
+    {
+        ByteBuffer buffer = allocateBuffer(4096);
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
+        {
+            regions.extend(4096);
+            regions.extend(1024);
+            regions.extend(2048);
+
+            for (int i = 0; i < buffer.capacity(); i++)
+            {
+                MmappedRegions.Region region = regions.floor(i);
+                assertNotNull(region);
+                assertEquals(0, region.bottom());
+                assertEquals(4096, region.top());
+            }
+        }
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testNegativeExtend() throws Exception
+    {
+        ByteBuffer buffer = allocateBuffer(1024);
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer));
+            MmappedRegions regions = MmappedRegions.empty(channel))
+        {
+            regions.extend(-1);
+        }
+    }
+
+    @Test
+    public void testMapForCompressionMetadata() throws Exception
+    {
+        int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+        MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+        ByteBuffer buffer = allocateBuffer(128 * 1024);
+        File f = File.createTempFile("testMapForCompressionMetadata", "1");
+        f.deleteOnExit();
+
+        File cf = File.createTempFile(f.getName() + ".metadata", "1");
+        cf.deleteOnExit();
+
+        MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance))
+                                                     .replayPosition(null);
+        try(SequentialWriter writer = new CompressedSequentialWriter(f,
+                                                                     cf.getAbsolutePath(),
+                                                                     CompressionParams.snappy(),
+                                                                     sstableMetadataCollector))
+        {
+            writer.write(buffer);
+            writer.finish();
+        }
+
+        CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), ChecksumType.CRC32);
+        try(ChannelProxy channel = new ChannelProxy(f);
+            MmappedRegions regions = MmappedRegions.map(channel, metadata))
+        {
+
+            assertFalse(regions.isEmpty());
+            int i = 0;
+            while(i < buffer.capacity())
+            {
+                CompressionMetadata.Chunk chunk = metadata.chunkFor(i);
+
+                MmappedRegions.Region region = regions.floor(chunk.offset);
+                assertNotNull(region);
+
+                ByteBuffer compressedChunk = region.buffer.duplicate();
+                assertNotNull(compressedChunk);
+                assertEquals(chunk.length + 4, compressedChunk.capacity());
+
+                assertEquals(chunk.offset, region.bottom());
+                assertEquals(chunk.offset + chunk.length + 4, region.top());
+
+                i += metadata.chunkLength();
+            }
+        }
+        finally
+        {
+            MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+            metadata.close();
+        }
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testIllegalArgForMap1() throws Exception
+    {
+        ByteBuffer buffer = allocateBuffer(1024);
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer));
+            MmappedRegions regions = MmappedRegions.map(channel, 0))
+        {
+            assertTrue(regions.isEmpty());
+        }
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testIllegalArgForMap2() throws Exception
+    {
+        ByteBuffer buffer = allocateBuffer(1024);
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer));
+            MmappedRegions regions = MmappedRegions.map(channel, -1L))
+        {
+            assertTrue(regions.isEmpty());
+        }
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testIllegalArgForMap3() throws Exception
+    {
+        ByteBuffer buffer = allocateBuffer(1024);
+        try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer));
+            MmappedRegions regions = MmappedRegions.map(channel, null))
+        {
+            assertTrue(regions.isEmpty());
+        }
+    }
+
+}

test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java

@@ -14,7 +14,6 @@ import java.util.ArrayDeque;
 import java.util.Queue;
 import java.util.Random;
 
-import org.apache.cassandra.io.util.NIODataInputStream;
 import org.junit.Test;
 
 import com.google.common.base.Charsets;
@@ -179,18 +178,11 @@ public class NIODataInputStreamTest
         assertFalse(fakeStream.markSupported());
     }
 
-    @SuppressWarnings("resource")
-    @Test(expected = IllegalArgumentException.class)
-    public void testTooSmallBufferSize() throws Exception
-    {
-        new NIODataInputStream(new FakeChannel(), 4);
-    }
-
     @SuppressWarnings("resource")
     @Test(expected = NullPointerException.class)
     public void testNullRBC() throws Exception
     {
-        new NIODataInputStream(null, 8);
+        new NIODataInputStream(null, 9);
     }
 
     @SuppressWarnings("resource")
@@ -769,7 +761,7 @@ public class NIODataInputStreamTest
                 out.writeUnsignedVInt(value);
 
                 buf.position(ii);
-                NIODataInputStream in = new DataInputBuffer(buf, false);
+                RebufferingInputStream in = new DataInputBuffer(buf, false);
 
                 assertEquals(value, in.readUnsignedVInt());
             }
@@ -792,7 +784,7 @@ public class NIODataInputStreamTest
             out.writeUnsignedVInt(value);
 
             buf.position(0);
-            NIODataInputStream in = new DataInputBuffer(buf, false);
+            RebufferingInputStream in = new DataInputBuffer(buf, false);
 
             assertEquals(value, in.readUnsignedVInt());
 
@@ -831,7 +823,7 @@ public class NIODataInputStreamTest
             truncated.put(buf);
             truncated.flip();
 
-            NIODataInputStream in = new DataInputBuffer(truncated, false);
+            RebufferingInputStream in = new DataInputBuffer(truncated, false);
 
             boolean threw = false;
             try

test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java

@@ -0,0 +1,483 @@
+package org.apache.cassandra.io.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.*;
+
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class RandomAccessReaderTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(RandomAccessReaderTest.class);
+
+    private static final class Parameters
+    {
+        public final long fileLength;
+        public final int bufferSize;
+
+        public BufferType bufferType;
+        public int maxSegmentSize;
+        public boolean mmappedRegions;
+        public byte[] expected;
+
+        Parameters(long fileLength, int bufferSize)
+        {
+            this.fileLength = fileLength;
+            this.bufferSize = bufferSize;
+            this.bufferType = BufferType.OFF_HEAP;
+            this.maxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE;
+            this.mmappedRegions = false;
+            this.expected = "The quick brown fox jumps over the lazy dog".getBytes(FileUtils.CHARSET);
+        }
+
+        public Parameters mmappedRegions(boolean mmappedRegions)
+        {
+            this.mmappedRegions = mmappedRegions;
+            return this;
+        }
+
+        public Parameters bufferType(BufferType bufferType)
+        {
+            this.bufferType = bufferType;
+            return this;
+        }
+
+        public Parameters maxSegmentSize(int maxSegmentSize)
+        {
+            this.maxSegmentSize = maxSegmentSize;
+            return this;
+        }
+
+        public Parameters expected(byte[] expected)
+        {
+            this.expected = expected;
+            return this;
+        }
+    }
+
+    @Test
+    public void testBufferedOffHeap() throws IOException
+    {
+        testReadFully(new Parameters(8192, 4096).bufferType(BufferType.OFF_HEAP));
+    }
+
+    @Test
+    public void testBufferedOnHeap() throws IOException
+    {
+        testReadFully(new Parameters(8192, 4096).bufferType(BufferType.ON_HEAP));
+    }
+
+    @Test
+    public void testBigBufferSize() throws IOException
+    {
+        testReadFully(new Parameters(8192, 65536).bufferType(BufferType.ON_HEAP));
+    }
+
+    @Test
+    public void testTinyBufferSize() throws IOException
+    {
+        testReadFully(new Parameters(8192, 16).bufferType(BufferType.ON_HEAP));
+    }
+
+    @Test
+    public void testOneSegment() throws IOException
+    {
+        testReadFully(new Parameters(8192, 4096).mmappedRegions(true));
+    }
+
+    @Test
+    public void testMultipleSegments() throws IOException
+    {
+        testReadFully(new Parameters(8192, 4096).mmappedRegions(true).maxSegmentSize(1024));
+    }
+
+    @Test
+    public void testVeryLarge() throws IOException
+    {
+        final long SIZE = 1L << 32; // 2GB
+        Parameters params = new Parameters(SIZE, 1 << 20); // 1MB
+
+        try(ChannelProxy channel = new ChannelProxy("abc", new FakeFileChannel(SIZE)))
+        {
+            RandomAccessReader.Builder builder = new RandomAccessReader.Builder(channel)
+                                                 .bufferType(params.bufferType)
+                                                 .bufferSize(params.bufferSize);
+
+            try(RandomAccessReader reader = builder.build())
+            {
+                assertEquals(channel.size(), reader.length());
+                assertEquals(channel.size(), reader.bytesRemaining());
+                assertEquals(Integer.MAX_VALUE, reader.available());
+
+                assertEquals(channel.size(), reader.skip(channel.size()));
+
+                assertTrue(reader.isEOF());
+                assertEquals(0, reader.bytesRemaining());
+            }
+        }
+    }
+
+    /** A fake file channel that simply increments the position and doesn't
+     * actually read anything. We use it to simulate very large files, > 2G.
+     */
+    private static final class FakeFileChannel extends FileChannel
+    {
+        private final long size;
+        private long position;
+
+        FakeFileChannel(long size)
+        {
+            this.size = size;
+        }
+
+        public int read(ByteBuffer dst)
+        {
+            int ret = dst.remaining();
+            position += ret;
+            dst.position(dst.limit());
+            return ret;
+        }
+
+        public long read(ByteBuffer[] dsts, int offset, int length)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public int write(ByteBuffer src)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public long write(ByteBuffer[] srcs, int offset, int length)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public long position()
+        {
+            return position;
+        }
+
+        public FileChannel position(long newPosition)
+        {
+            position = newPosition;
+            return this;
+        }
+
+        public long size()
+        {
+            return size;
+        }
+
+        public FileChannel truncate(long size)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public void force(boolean metaData)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public long transferTo(long position, long count, WritableByteChannel target)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public long transferFrom(ReadableByteChannel src, long position, long count)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public int read(ByteBuffer dst, long position)
+        {
+            int ret = dst.remaining();
+            this.position = position + ret;
+            dst.position(dst.limit());
+            return ret;
+        }
+
+        public int write(ByteBuffer src, long position)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public MappedByteBuffer map(MapMode mode, long position, long size)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public FileLock lock(long position, long size, boolean shared)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public FileLock tryLock(long position, long size, boolean shared)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        protected void implCloseChannel()
+        {
+
+        }
+    }
+
+    private static File writeFile(Parameters params) throws IOException
+    {
+        final File f = File.createTempFile("testReadFully", "1");
+        f.deleteOnExit();
+
+        try(SequentialWriter writer = SequentialWriter.open(f))
+        {
+            long numWritten = 0;
+            while (numWritten < params.fileLength)
+            {
+                writer.write(params.expected);
+                numWritten += params.expected.length;
+            }
+
+            writer.finish();
+        }
+
+        assert f.exists();
+        assert f.length() >= params.fileLength;
+        return f;
+    }
+
+    private static void testReadFully(Parameters params) throws IOException
+    {
+        final File f = writeFile(params);
+        try(ChannelProxy channel = new ChannelProxy(f))
+        {
+            RandomAccessReader.Builder builder = new RandomAccessReader.Builder(channel)
+                                                 .bufferType(params.bufferType)
+                                                 .bufferSize(params.bufferSize);
+            if (params.mmappedRegions)
+                builder.regions(MmappedRegions.map(channel, f.length()));
+
+            try(RandomAccessReader reader = builder.build())
+            {
+                assertEquals(f.getAbsolutePath(), reader.getPath());
+                assertEquals(f.length(), reader.length());
+                assertEquals(f.length(), reader.bytesRemaining());
+                assertEquals(Math.min(Integer.MAX_VALUE, f.length()), reader.available());
+
+                byte[] b = new byte[params.expected.length];
+                long numRead = 0;
+                while (numRead < params.fileLength)
+                {
+                    reader.readFully(b);
+                    assertTrue(Arrays.equals(params.expected, b));
+                    numRead += b.length;
+                }
+
+                assertTrue(reader.isEOF());
+                assertEquals(0, reader.bytesRemaining());
+            }
+
+            if (builder.regions != null)
+                assertNull(builder.regions.close(null));
+        }
+    }
+
+    @Test
+    public void testReadBytes() throws IOException
+    {
+        File f = File.createTempFile("testReadBytes", "1");
+        final String expected = "The quick brown fox jumps over the lazy dog";
+
+        try(SequentialWriter writer = SequentialWriter.open(f))
+        {
+            writer.write(expected.getBytes());
+            writer.finish();
+        }
+
+        assert f.exists();
+
+        try(ChannelProxy channel = new ChannelProxy(f);
+            RandomAccessReader reader = new RandomAccessReader.Builder(channel).build())
+        {
+            assertEquals(f.getAbsolutePath(), reader.getPath());
+            assertEquals(expected.length(), reader.length());
+
+            ByteBuffer b = ByteBufferUtil.read(reader, expected.length());
+            assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+
+            assertTrue(reader.isEOF());
+            assertEquals(0, reader.bytesRemaining());
+        }
+    }
+
+    @Test
+    public void testReset() throws IOException
+    {
+        File f = File.createTempFile("testMark", "1");
+        final String expected = "The quick brown fox jumps over the lazy dog";
+        final int numIterations = 10;
+
+        try(SequentialWriter writer = SequentialWriter.open(f))
+        {
+            for (int i = 0; i < numIterations; i++)
+                writer.write(expected.getBytes());
+            writer.finish();
+        }
+
+        assert f.exists();
+
+        try(ChannelProxy channel = new ChannelProxy(f);
+        RandomAccessReader reader = new RandomAccessReader.Builder(channel).build())
+        {
+            assertEquals(expected.length() * numIterations, reader.length());
+
+            ByteBuffer b = ByteBufferUtil.read(reader, expected.length());
+            assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+
+            assertFalse(reader.isEOF());
+            assertEquals((numIterations - 1) * expected.length(), reader.bytesRemaining());
+
+            FileMark mark = reader.mark();
+            assertEquals(0, reader.bytesPastMark());
+            assertEquals(0, reader.bytesPastMark(mark));
+
+            for (int i = 0; i < (numIterations - 1); i++)
+            {
+                b = ByteBufferUtil.read(reader, expected.length());
+                assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+            }
+            assertTrue(reader.isEOF());
+            assertEquals(expected.length() * (numIterations - 1), reader.bytesPastMark());
+            assertEquals(expected.length() * (numIterations - 1), reader.bytesPastMark(mark));
+
+            reader.reset(mark);
+            assertEquals(0, reader.bytesPastMark());
+            assertEquals(0, reader.bytesPastMark(mark));
+            assertFalse(reader.isEOF());
+            for (int i = 0; i < (numIterations - 1); i++)
+            {
+                b = ByteBufferUtil.read(reader, expected.length());
+                assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+            }
+
+            reader.reset();
+            assertEquals(0, reader.bytesPastMark());
+            assertEquals(0, reader.bytesPastMark(mark));
+            assertFalse(reader.isEOF());
+            for (int i = 0; i < (numIterations - 1); i++)
+            {
+                b = ByteBufferUtil.read(reader, expected.length());
+                assertEquals(expected, new String(b.array(), Charset.forName("UTF-8")));
+            }
+
+            assertTrue(reader.isEOF());
+        }
+    }
+
+    @Test
+    public void testSeekSingleThread() throws IOException, InterruptedException
+    {
+        testSeek(1);
+    }
+
+    @Test
+    public void testSeekMultipleThreads() throws IOException, InterruptedException
+    {
+        testSeek(10);
+    }
+
+    private static void testSeek(int numThreads) throws IOException, InterruptedException
+    {
+        final File f = File.createTempFile("testMark", "1");
+        final byte[] expected = new byte[1 << 16];
+
+        long seed = System.nanoTime();
+        //seed = 365238103404423L;
+        logger.info("Seed {}", seed);
+        Random r = new Random(seed);
+        r.nextBytes(expected);
+
+        try(SequentialWriter writer = SequentialWriter.open(f))
+        {
+            writer.write(expected);
+            writer.finish();
+        }
+
+        assert f.exists();
+
+        try(final ChannelProxy channel = new ChannelProxy(f))
+        {
+            final Runnable worker = () ->
+            {
+                try(RandomAccessReader reader = new RandomAccessReader.Builder(channel).build())
+                {
+                    assertEquals(expected.length, reader.length());
+
+                    ByteBuffer b = ByteBufferUtil.read(reader, expected.length);
+                    assertTrue(Arrays.equals(expected, b.array()));
+                    assertTrue(reader.isEOF());
+                    assertEquals(0, reader.bytesRemaining());
+
+                    reader.seek(0);
+                    b = ByteBufferUtil.read(reader, expected.length);
+                    assertTrue(Arrays.equals(expected, b.array()));
+                    assertTrue(reader.isEOF());
+                    assertEquals(0, reader.bytesRemaining());
+
+                    for (int i = 0; i < 10; i++)
+                    {
+                        int pos = r.nextInt(expected.length);
+                        reader.seek(pos);
+                        assertEquals(pos, reader.getPosition());
+
+                        ByteBuffer buf = ByteBuffer.wrap(expected, pos, expected.length - pos)
+                                                   .order(ByteOrder.BIG_ENDIAN);
+
+                        while (reader.bytesRemaining() > 4)
+                            assertEquals(buf.getInt(), reader.readInt());
+                    }
+
+                    reader.close();
+                }
+                catch (Exception ex)
+                {
+                    ex.printStackTrace();
+                    fail(ex.getMessage());
+                }
+            };
+
+            if (numThreads == 1)
+            {
+                worker.run();
+            }
+            else
+            {
+                ExecutorService executor = Executors.newFixedThreadPool(numThreads);
+                for (int i = 0; i < numThreads; i++)
+                    executor.submit(worker);
+
+                executor.shutdown();
+                executor.awaitTermination(1, TimeUnit.MINUTES);
+            }
+        }
+    }
+}