diff --git a/build.xml b/build.xml index 28f08d632f..252b1a89c5 100644 --- a/build.xml +++ b/build.xml @@ -415,8 +415,8 @@ --> - - + + @@ -470,8 +470,8 @@ --> - - + + diff --git a/lib/licenses/ohc-0.3.4.txt b/lib/licenses/ohc-0.4.2.txt similarity index 100% rename from lib/licenses/ohc-0.3.4.txt rename to lib/licenses/ohc-0.4.2.txt diff --git a/lib/ohc-core-0.4.2.jar b/lib/ohc-core-0.4.2.jar new file mode 100644 index 0000000000..019adcd3f8 Binary files /dev/null and b/lib/ohc-core-0.4.2.jar differ diff --git a/lib/ohc-core-0.4.jar b/lib/ohc-core-0.4.jar deleted file mode 100644 index 1b1b9392c1..0000000000 Binary files a/lib/ohc-core-0.4.jar and /dev/null differ diff --git a/lib/ohc-core-j8-0.4.2.jar b/lib/ohc-core-j8-0.4.2.jar new file mode 100644 index 0000000000..583f4aaaff Binary files /dev/null and b/lib/ohc-core-j8-0.4.2.jar differ diff --git a/lib/ohc-core-j8-0.4.jar b/lib/ohc-core-j8-0.4.jar deleted file mode 100644 index f97ddf5b81..0000000000 Binary files a/lib/ohc-core-j8-0.4.jar and /dev/null differ diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java index 8650d477a3..ebd28305fa 100644 --- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java +++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java @@ -68,7 +68,7 @@ public class AutoSavingCache extends InstrumentingCache { try { - NIODataInputStream in = new DataInputBuffer(buf, false); + RebufferingInputStream in = new DataInputBuffer(buf, false); boolean isSentinel = in.readBoolean(); if (isSentinel) return new RowCacheSentinel(in.readLong()); diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index cf8e14d4a1..d54ee8bd83 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -676,7 +676,7 @@ public final class SystemKeyspace { try { - NIODataInputStream in = new DataInputBuffer(bytes, true); + RebufferingInputStream in = new DataInputBuffer(bytes, true); return Pair.create(ReplayPosition.serializer.deserialize(in), in.available() > 0 ? in.readLong() : Long.MIN_VALUE); } catch (IOException e) diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java index 4f50008d3f..70491919b5 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java @@ -47,12 +47,13 @@ import org.apache.cassandra.db.rows.SerializationHelper; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.util.FileSegmentInputStream; +import org.apache.cassandra.io.util.RebufferingInputStream; import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.io.compress.ICompressor; -import org.apache.cassandra.io.util.ByteBufferDataInput; +import org.apache.cassandra.io.util.ChannelProxy; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.NIODataInputStream; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.utils.FBUtilities; @@ -290,8 +291,8 @@ public class CommitLogReplayer public void recover(File file, boolean tolerateTruncation) throws IOException { CommitLogDescriptor desc = CommitLogDescriptor.fromFileName(file.getName()); - RandomAccessReader reader = RandomAccessReader.open(new File(file.getAbsolutePath())); - try + try(ChannelProxy channel = new ChannelProxy(file); + RandomAccessReader reader = RandomAccessReader.open(channel)) { if (desc.version < CommitLogDescriptor.VERSION_21) { @@ -299,7 +300,7 @@ public class CommitLogReplayer return; if (globalPosition.segment == desc.id) reader.seek(globalPosition.position); - replaySyncSection(reader, (int) reader.getPositionLimit(), desc, desc.fileName(), tolerateTruncation); + replaySyncSection(reader, (int) reader.length(), desc, desc.fileName(), tolerateTruncation); return; } @@ -388,7 +389,7 @@ public class CommitLogReplayer if (uncompressedLength > uncompressedBuffer.length) uncompressedBuffer = new byte[(int) (1.2 * uncompressedLength)]; compressedLength = compressor.uncompress(buffer, 0, compressedLength, uncompressedBuffer, 0); - sectionReader = new ByteBufferDataInput(ByteBuffer.wrap(uncompressedBuffer), reader.getPath(), replayPos, 0); + sectionReader = new FileSegmentInputStream(ByteBuffer.wrap(uncompressedBuffer), reader.getPath(), replayPos); errorContext = "compressed section at " + start + " in " + errorContext; } catch (IOException | ArrayIndexOutOfBoundsException e) @@ -403,10 +404,6 @@ public class CommitLogReplayer if (!replaySyncSection(sectionReader, replayEnd, desc, errorContext, tolerateErrorsInSection)) break; } - } - finally - { - FileUtils.closeQuietly(reader); logger.info("Finished reading {}", file); } } @@ -522,7 +519,7 @@ public class CommitLogReplayer { final Mutation mutation; - try (NIODataInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size)) + try (RebufferingInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size)) { mutation = Mutation.serializer.deserialize(bufIn, desc.getMessagingVersion(), diff --git a/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java b/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java index fa727bc0ed..543f14e450 100644 --- a/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java +++ b/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java @@ -17,98 +17,148 @@ */ package org.apache.cassandra.hints; +import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.zip.CRC32; -import org.apache.cassandra.io.util.AbstractDataInput; +import org.apache.cassandra.io.util.ChannelProxy; +import org.apache.cassandra.io.util.FileMark; +import org.apache.cassandra.io.util.RandomAccessReader; /** - * An {@link AbstractDataInput} wrapper that calctulates the CRC in place. + * A {@link RandomAccessReader} wrapper that calctulates the CRC in place. * * Useful for {@link org.apache.cassandra.hints.HintsReader}, for example, where we must verify the CRC, yet don't want - * to allocate an extra byte array just that purpose. + * to allocate an extra byte array just that purpose. The CRC can be embedded in the input stream and checked via checkCrc(). * - * In addition to calculating the CRC, allows to enforce a maximim known size. This is needed + * In addition to calculating the CRC, it allows to enforce a maximim known size. This is needed * so that {@link org.apache.cassandra.db.Mutation.MutationSerializer} doesn't blow up the heap when deserializing a * corrupted sequence by reading a huge corrupted length of bytes via * via {@link org.apache.cassandra.utils.ByteBufferUtil#readWithLength(java.io.DataInput)}. */ -public final class ChecksummedDataInput extends AbstractDataInput +public final class ChecksummedDataInput extends RandomAccessReader.RandomAccessReaderWithOwnChannel { private final CRC32 crc; - private final AbstractDataInput source; - private int limit; + private int crcPosition; + private boolean crcUpdateDisabled; - private ChecksummedDataInput(AbstractDataInput source) + private long limit; + private FileMark limitMark; + + private ChecksummedDataInput(Builder builder) { - this.source = source; + super(builder); crc = new CRC32(); - limit = Integer.MAX_VALUE; + crcPosition = 0; + crcUpdateDisabled = false; + + resetLimit(); } - public static ChecksummedDataInput wrap(AbstractDataInput source) + public static ChecksummedDataInput open(File file) { - return new ChecksummedDataInput(source); + return new Builder(new ChannelProxy(file)).build(); } public void resetCrc() { crc.reset(); + crcPosition = buffer.position(); + } + + public void limit(long newLimit) + { + limit = newLimit; + limitMark = mark(); } public void resetLimit() { - limit = Integer.MAX_VALUE; + limit = Long.MAX_VALUE; + limitMark = null; } - public void limit(int newLimit) + public void checkLimit(int length) throws IOException { - limit = newLimit; + if (limitMark == null) + return; + + if ((bytesPastLimit() + length) > limit) + throw new IOException("Digest mismatch exception"); } - public int bytesRemaining() + public long bytesPastLimit() { - return limit; + assert limitMark != null; + return bytesPastMark(limitMark); } - public int getCrc() + public boolean checkCrc() throws IOException { - return (int) crc.getValue(); - } + try + { + updateCrc(); - public void seek(long position) throws IOException - { - source.seek(position); - } - - public long getPosition() - { - return source.getPosition(); - } - - public long getPositionLimit() - { - return source.getPositionLimit(); - } - - public int read() throws IOException - { - int b = source.read(); - crc.update(b); - limit--; - return b; + // we must diable crc updates in case we rebuffer + // when called source.readInt() + crcUpdateDisabled = true; + return ((int) crc.getValue()) == readInt(); + } + finally + { + crcPosition = buffer.position(); + crcUpdateDisabled = false; + } } @Override - public int read(byte[] buff, int offset, int length) throws IOException + public void readFully(byte[] b) throws IOException { - if (length > limit) - throw new IOException("Digest mismatch exception"); + checkLimit(b.length); + super.readFully(b); + } - int copied = source.read(buff, offset, length); - crc.update(buff, offset, copied); - limit -= copied; - return copied; + @Override + public int read(byte[] b, int off, int len) throws IOException + { + checkLimit(len); + return super.read(b, off, len); + } + + @Override + public void reBuffer() + { + updateCrc(); + super.reBuffer(); + crcPosition = buffer.position(); + } + + private void updateCrc() + { + if (crcPosition == buffer.position() | crcUpdateDisabled) + return; + + assert crcPosition >= 0 && crcPosition < buffer.position(); + + ByteBuffer unprocessed = buffer.duplicate(); + unprocessed.position(crcPosition) + .limit(buffer.position()); + + crc.update(unprocessed); + } + + public final static class Builder extends RandomAccessReader.Builder + { + public Builder(ChannelProxy channel) + { + super(channel); + } + + public ChecksummedDataInput build() + { + return new ChecksummedDataInput(this); + } } } diff --git a/src/java/org/apache/cassandra/hints/HintsReader.java b/src/java/org/apache/cassandra/hints/HintsReader.java index 7d164b454f..bc8365415f 100644 --- a/src/java/org/apache/cassandra/hints/HintsReader.java +++ b/src/java/org/apache/cassandra/hints/HintsReader.java @@ -24,6 +24,7 @@ import java.util.Iterator; import javax.annotation.Nullable; +import com.google.common.primitives.Ints; import com.google.common.util.concurrent.RateLimiter; import org.slf4j.Logger; @@ -32,7 +33,6 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.UnknownColumnFamilyException; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.utils.AbstractIterator; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.CLibrary; @@ -57,25 +57,23 @@ final class HintsReader implements AutoCloseable, Iterable private final HintsDescriptor descriptor; private final File file; - private final RandomAccessReader reader; - private final ChecksummedDataInput crcInput; + private final ChecksummedDataInput input; // we pass the RateLimiter into HintsReader itself because it's cheaper to calculate the size before the hint is deserialized @Nullable private final RateLimiter rateLimiter; - private HintsReader(HintsDescriptor descriptor, File file, RandomAccessReader reader, RateLimiter rateLimiter) + private HintsReader(HintsDescriptor descriptor, File file, ChecksummedDataInput reader, RateLimiter rateLimiter) { this.descriptor = descriptor; this.file = file; - this.reader = reader; - this.crcInput = ChecksummedDataInput.wrap(reader); + this.input = reader; this.rateLimiter = rateLimiter; } static HintsReader open(File file, RateLimiter rateLimiter) { - RandomAccessReader reader = RandomAccessReader.open(file); + ChecksummedDataInput reader = ChecksummedDataInput.open(file); try { HintsDescriptor descriptor = HintsDescriptor.deserialize(reader); @@ -83,7 +81,7 @@ final class HintsReader implements AutoCloseable, Iterable } catch (IOException e) { - reader.close(); + FileUtils.closeQuietly(reader); throw new FSReadError(e, file); } } @@ -95,7 +93,7 @@ final class HintsReader implements AutoCloseable, Iterable public void close() { - FileUtils.closeQuietly(reader); + FileUtils.closeQuietly(input); } public HintsDescriptor descriptor() @@ -105,7 +103,7 @@ final class HintsReader implements AutoCloseable, Iterable void seek(long newPosition) { - reader.seek(newPosition); + input.seek(newPosition); } public Iterator iterator() @@ -138,12 +136,12 @@ final class HintsReader implements AutoCloseable, Iterable @SuppressWarnings("resource") protected Page computeNext() { - CLibrary.trySkipCache(reader.getChannel().getFileDescriptor(), 0, reader.getFilePointer(), reader.getPath()); + CLibrary.trySkipCache(input.getChannel().getFileDescriptor(), 0, input.getFilePointer(), input.getPath()); - if (reader.length() == reader.getFilePointer()) + if (input.length() == input.getFilePointer()) return endOfData(); - return new Page(reader.getFilePointer()); + return new Page(input.getFilePointer()); } } @@ -166,9 +164,9 @@ final class HintsReader implements AutoCloseable, Iterable do { - long position = reader.getFilePointer(); + long position = input.getFilePointer(); - if (reader.length() == position) + if (input.length() == position) return endOfData(); // reached EOF if (position - offset >= PAGE_SIZE) @@ -190,13 +188,13 @@ final class HintsReader implements AutoCloseable, Iterable private Hint computeNextInternal() throws IOException { - crcInput.resetCrc(); - crcInput.resetLimit(); + input.resetCrc(); + input.resetLimit(); - int size = crcInput.readInt(); + int size = input.readInt(); // if we cannot corroborate the size via crc, then we cannot safely skip this hint - if (reader.readInt() != crcInput.getCrc()) + if (!input.checkCrc()) throw new IOException("Digest mismatch exception"); return readHint(size); @@ -206,12 +204,13 @@ final class HintsReader implements AutoCloseable, Iterable { if (rateLimiter != null) rateLimiter.acquire(size); - crcInput.limit(size); + input.limit(size); Hint hint; try { - hint = Hint.serializer.deserialize(crcInput, descriptor.messagingVersion()); + hint = Hint.serializer.deserialize(input, descriptor.messagingVersion()); + input.checkLimit(0); } catch (UnknownColumnFamilyException e) { @@ -219,18 +218,18 @@ final class HintsReader implements AutoCloseable, Iterable descriptor.hostId, e.cfId, descriptor.fileName()); - reader.skipBytes(crcInput.bytesRemaining()); + input.skipBytes(Ints.checkedCast(size - input.bytesPastLimit())); return null; } - if (reader.readInt() == crcInput.getCrc()) + if (input.checkCrc()) return hint; // log a warning and skip the corrupted entry logger.warn("Failed to read a hint for {} - digest mismatch for hint at position {} in file {}", descriptor.hostId, - crcInput.getPosition() - size - 4, + input.getPosition() - size - 4, descriptor.fileName()); return null; } @@ -255,9 +254,9 @@ final class HintsReader implements AutoCloseable, Iterable do { - long position = reader.getFilePointer(); + long position = input.getFilePointer(); - if (reader.length() == position) + if (input.length() == position) return endOfData(); // reached EOF if (position - offset >= PAGE_SIZE) @@ -279,13 +278,13 @@ final class HintsReader implements AutoCloseable, Iterable private ByteBuffer computeNextInternal() throws IOException { - crcInput.resetCrc(); - crcInput.resetLimit(); + input.resetCrc(); + input.resetLimit(); - int size = crcInput.readInt(); + int size = input.readInt(); // if we cannot corroborate the size via crc, then we cannot safely skip this hint - if (reader.readInt() != crcInput.getCrc()) + if (!input.checkCrc()) throw new IOException("Digest mismatch exception"); return readBuffer(size); @@ -295,16 +294,16 @@ final class HintsReader implements AutoCloseable, Iterable { if (rateLimiter != null) rateLimiter.acquire(size); - crcInput.limit(size); + input.limit(size); - ByteBuffer buffer = ByteBufferUtil.read(crcInput, size); - if (reader.readInt() == crcInput.getCrc()) + ByteBuffer buffer = ByteBufferUtil.read(input, size); + if (input.checkCrc()) return buffer; // log a warning and skip the corrupted entry logger.warn("Failed to read a hint for {} - digest mismatch for hint at position {} in file {}", descriptor.hostId, - crcInput.getPosition() - size - 4, + input.getPosition() - size - 4, descriptor.fileName()); return null; } diff --git a/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java b/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java index c38f4d2dad..0242871b2b 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java +++ b/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java @@ -19,11 +19,7 @@ package org.apache.cassandra.io.compress; import java.io.*; import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; -import java.util.Map; -import java.util.TreeMap; import java.util.concurrent.ThreadLocalRandom; -import java.util.zip.Adler32; import java.util.zip.Checksum; import com.google.common.primitives.Ints; @@ -31,7 +27,6 @@ import com.google.common.primitives.Ints; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.sstable.CorruptSSTableException; import org.apache.cassandra.io.util.*; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.memory.BufferPool; /** @@ -40,18 +35,6 @@ import org.apache.cassandra.utils.memory.BufferPool; */ public class CompressedRandomAccessReader extends RandomAccessReader { - public static CompressedRandomAccessReader open(ChannelProxy channel, CompressionMetadata metadata) - { - return new CompressedRandomAccessReader(channel, metadata, null); - } - - public static CompressedRandomAccessReader open(ICompressedFile file) - { - return new CompressedRandomAccessReader(file.channel(), file.getMetadata(), file); - } - - private final TreeMap chunkSegments; - private final CompressionMetadata metadata; // we read the raw compressed bytes into this buffer, then move the uncompressed ones into super.buffer. @@ -63,39 +46,59 @@ public class CompressedRandomAccessReader extends RandomAccessReader // raw checksum bytes private ByteBuffer checksumBytes; - protected CompressedRandomAccessReader(ChannelProxy channel, CompressionMetadata metadata, ICompressedFile file) + protected CompressedRandomAccessReader(Builder builder) { - super(channel, metadata.chunkLength(), metadata.compressedFileLength, metadata.compressor().preferredBufferType()); - this.metadata = metadata; - checksum = metadata.checksumType.newInstance(); + super(builder.initializeBuffers(false)); + this.metadata = builder.metadata; + this.checksum = metadata.checksumType.newInstance(); - chunkSegments = file == null ? null : file.chunkSegments(); - if (chunkSegments == null) + initializeBuffer(); + } + + @Override + protected int getBufferSize(RandomAccessReader.Builder builder) + { + // this is the chunk data length, throttling is OK with this + return builder.bufferSize; + } + + @Override + protected void initializeBuffer() + { + buffer = allocateBuffer(bufferSize); + buffer.limit(0); + + if (regions == null) { - compressed = allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength()), metadata.compressor().preferredBufferType()); + compressed = allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength())); checksumBytes = ByteBuffer.wrap(new byte[4]); } } - protected int getBufferSize(int size) - { - assert Integer.bitCount(size) == 1; //must be a power of two - return size; - } - @Override - public void close() + protected void releaseBuffer() { - super.close(); - - if (compressed != null) + try { - BufferPool.put(compressed); - compressed = null; + if (buffer != null) + { + BufferPool.put(buffer); + buffer = null; + } + } + finally + { + // this will always be null if using mmap access mode (unlike in parent, where buffer is set to a region) + if (compressed != null) + { + BufferPool.put(compressed); + compressed = null; + } } } - private void reBufferStandard() + @Override + protected void reBufferStandard() { try { @@ -105,13 +108,19 @@ public class CompressedRandomAccessReader extends RandomAccessReader CompressionMetadata.Chunk chunk = metadata.chunkFor(position); if (compressed.capacity() < chunk.length) - compressed = allocateBuffer(chunk.length, metadata.compressor().preferredBufferType()); + { + BufferPool.put(compressed); + compressed = allocateBuffer(chunk.length); + } else + { compressed.clear(); - compressed.limit(chunk.length); + } + compressed.limit(chunk.length); if (channel.read(compressed, chunk.offset) != chunk.length) throw new CorruptBlockException(getPath(), chunk); + compressed.flip(); buffer.clear(); @@ -158,7 +167,8 @@ public class CompressedRandomAccessReader extends RandomAccessReader } } - private void reBufferMmap() + @Override + protected void reBufferMmap() { try { @@ -167,10 +177,10 @@ public class CompressedRandomAccessReader extends RandomAccessReader CompressionMetadata.Chunk chunk = metadata.chunkFor(position); - Map.Entry entry = chunkSegments.floorEntry(chunk.offset); - long segmentOffset = entry.getKey(); + MmappedRegions.Region region = regions.floor(chunk.offset); + long segmentOffset = region.bottom(); int chunkOffset = Ints.checkedCast(chunk.offset - segmentOffset); - ByteBuffer compressedChunk = entry.getValue().duplicate(); // TODO: change to slice(chunkOffset) when we upgrade LZ4-java + ByteBuffer compressedChunk = region.buffer.duplicate(); // TODO: change to slice(chunkOffset) when we upgrade LZ4-java compressedChunk.position(chunkOffset).limit(chunkOffset + chunk.length); @@ -218,19 +228,6 @@ public class CompressedRandomAccessReader extends RandomAccessReader } - @Override - protected void reBuffer() - { - if (chunkSegments != null) - { - reBufferMmap(); - } - else - { - reBufferStandard(); - } - } - private int checksum(CompressionMetadata.Chunk chunk) throws IOException { long position = chunk.offset + chunk.length; @@ -240,11 +237,6 @@ public class CompressedRandomAccessReader extends RandomAccessReader return checksumBytes.getInt(0); } - public int getTotalBufferSize() - { - return super.getTotalBufferSize() + (chunkSegments != null ? 0 : compressed.capacity()); - } - @Override public long length() { @@ -256,4 +248,39 @@ public class CompressedRandomAccessReader extends RandomAccessReader { return String.format("%s - chunk length %d, data length %d.", getPath(), metadata.chunkLength(), metadata.dataLength); } + + public final static class Builder extends RandomAccessReader.Builder + { + private final CompressionMetadata metadata; + + public Builder(ICompressedFile file) + { + super(file.channel()); + this.metadata = applyMetadata(file.getMetadata()); + this.regions = file.regions(); + } + + public Builder(ChannelProxy channel, CompressionMetadata metadata) + { + super(channel); + this.metadata = applyMetadata(metadata); + } + + private CompressionMetadata applyMetadata(CompressionMetadata metadata) + { + this.overrideLength = metadata.compressedFileLength; + this.bufferSize = metadata.chunkLength(); + this.bufferType = metadata.compressor().preferredBufferType(); + + assert Integer.bitCount(this.bufferSize) == 1; //must be a power of two + + return metadata; + } + + @Override + public RandomAccessReader build() + { + return new CompressedRandomAccessReader(this); + } + } } diff --git a/src/java/org/apache/cassandra/io/compress/CompressedThrottledReader.java b/src/java/org/apache/cassandra/io/compress/CompressedThrottledReader.java deleted file mode 100644 index ea5edafb44..0000000000 --- a/src/java/org/apache/cassandra/io/compress/CompressedThrottledReader.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.apache.cassandra.io.compress; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -import com.google.common.util.concurrent.RateLimiter; - -import org.apache.cassandra.io.util.ChannelProxy; -import org.apache.cassandra.io.util.ICompressedFile; - -public class CompressedThrottledReader extends CompressedRandomAccessReader -{ - private final RateLimiter limiter; - - public CompressedThrottledReader(ChannelProxy channel, CompressionMetadata metadata, ICompressedFile file, RateLimiter limiter) - { - super(channel, metadata, file); - this.limiter = limiter; - } - - protected void reBuffer() - { - limiter.acquire(buffer.capacity()); - super.reBuffer(); - } - - public static CompressedThrottledReader open(ICompressedFile file, RateLimiter limiter) - { - return new CompressedThrottledReader(file.channel(), file.getMetadata(), file, limiter); - } -} diff --git a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java index f5d8f7e09c..1681b0c8ce 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java +++ b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java @@ -92,7 +92,7 @@ public class CompressionMetadata } @VisibleForTesting - CompressionMetadata(String indexFilePath, long compressedLength, ChecksumType checksumType) + public CompressionMetadata(String indexFilePath, long compressedLength, ChecksumType checksumType) { this.indexFilePath = indexFilePath; this.checksumType = checksumType; diff --git a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java index 124720ec2f..6f1e2f4292 100644 --- a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java @@ -28,7 +28,6 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.CloseableIterator; diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index 522f7a1fb9..5d8ab50c2d 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -828,8 +828,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted segments = ifile.iterator(0); int i = 0; int summaryEntriesChecked = 0; int expectedIndexInterval = getMinIndexInterval(); - while (segments.hasNext()) + String path = null; + try (FileDataInput in = ifile.createReader(0)) { - String path = null; - try (FileDataInput in = segments.next()) + path = in.getPath(); + while (!in.isEOF()) { - path = in.getPath(); - while (!in.isEOF()) + ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); + if (i % expectedIndexInterval == 0) { - ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); - if (i % expectedIndexInterval == 0) - { - ByteBuffer summaryKey = ByteBuffer.wrap(indexSummary.getKey(i / expectedIndexInterval)); - if (!summaryKey.equals(indexKey)) - return false; - summaryEntriesChecked++; + ByteBuffer summaryKey = ByteBuffer.wrap(indexSummary.getKey(i / expectedIndexInterval)); + if (!summaryKey.equals(indexKey)) + return false; + summaryEntriesChecked++; - if (summaryEntriesChecked == Downsampling.BASE_SAMPLING_LEVEL) - return true; - } - RowIndexEntry.Serializer.skip(in); - i++; + if (summaryEntriesChecked == Downsampling.BASE_SAMPLING_LEVEL) + return true; } + RowIndexEntry.Serializer.skip(in); + i++; } - catch (IOException e) - { - markSuspect(); - throw new CorruptSSTableException(e, path); - } + } + catch (IOException e) + { + markSuspect(); + throw new CorruptSSTableException(e, path); } return true; @@ -972,8 +966,8 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted segments = ifile.iterator(sampledPosition); - while (segments.hasNext()) + String path = null; + try (FileDataInput in = ifile.createReader(sampledPosition)) { - String path = null; - try (FileDataInput in = segments.next();) + path = in.getPath(); + while (!in.isEOF()) { - path = in.getPath(); - while (!in.isEOF()) - { - ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); - DecoratedKey indexDecoratedKey = decorateKey(indexKey); - if (indexDecoratedKey.compareTo(token) > 0) - return indexDecoratedKey; + ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); + DecoratedKey indexDecoratedKey = decorateKey(indexKey); + if (indexDecoratedKey.compareTo(token) > 0) + return indexDecoratedKey; - RowIndexEntry.Serializer.skip(in); - } - } - catch (IOException e) - { - markSuspect(); - throw new CorruptSSTableException(e, path); + RowIndexEntry.Serializer.skip(in); } } + catch (IOException e) + { + markSuspect(); + throw new CorruptSSTableException(e, path); + } return null; } @@ -1744,11 +1734,9 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted segments = ifile.iterator(sampledPosition); - while (segments.hasNext()) + String path = null; + try (FileDataInput in = ifile.createReader(sampledPosition)) { - String path = null; - try (FileDataInput in = segments.next()) + path = in.getPath(); + while (!in.isEOF()) { - path = in.getPath(); - while (!in.isEOF()) + i++; + + ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); + + boolean opSatisfied; // did we find an appropriate position for the op requested + boolean exactMatch; // is the current position an exact match for the key, suitable for caching + + // Compare raw keys if possible for performance, otherwise compare decorated keys. + if (op == Operator.EQ && i <= effectiveInterval) { - i++; - - ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); - - boolean opSatisfied; // did we find an appropriate position for the op requested - boolean exactMatch; // is the current position an exact match for the key, suitable for caching - - // Compare raw keys if possible for performance, otherwise compare decorated keys. - if (op == Operator.EQ && i <= effectiveInterval) - { - opSatisfied = exactMatch = indexKey.equals(((DecoratedKey) key).getKey()); - } - else - { - DecoratedKey indexDecoratedKey = decorateKey(indexKey); - int comparison = indexDecoratedKey.compareTo(key); - int v = op.apply(comparison); - opSatisfied = (v == 0); - exactMatch = (comparison == 0); - if (v < 0) - { - Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.generation); - return null; - } - } - - if (opSatisfied) - { - // read data position from index entry - RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in); - if (exactMatch && updateCacheAndStats) - { - assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key - DecoratedKey decoratedKey = (DecoratedKey)key; - - if (logger.isTraceEnabled()) - { - // expensive sanity check! see CASSANDRA-4687 - try (FileDataInput fdi = dfile.getSegment(indexEntry.position)) - { - DecoratedKey keyInDisk = decorateKey(ByteBufferUtil.readWithShortLength(fdi)); - if (!keyInDisk.equals(key)) - throw new AssertionError(String.format("%s != %s in %s", keyInDisk, key, fdi.getPath())); - } - } - - // store exact match for the key - cacheKey(decoratedKey, indexEntry); - } - if (op == Operator.EQ && updateCacheAndStats) - bloomFilterTracker.addTruePositive(); - Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndex().size(), descriptor.generation); - return indexEntry; - } - - RowIndexEntry.Serializer.skip(in); + opSatisfied = exactMatch = indexKey.equals(((DecoratedKey) key).getKey()); } + else + { + DecoratedKey indexDecoratedKey = decorateKey(indexKey); + int comparison = indexDecoratedKey.compareTo(key); + int v = op.apply(comparison); + opSatisfied = (v == 0); + exactMatch = (comparison == 0); + if (v < 0) + { + Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.generation); + return null; + } + } + + if (opSatisfied) + { + // read data position from index entry + RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in); + if (exactMatch && updateCacheAndStats) + { + assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key + DecoratedKey decoratedKey = (DecoratedKey)key; + + if (logger.isTraceEnabled()) + { + // expensive sanity check! see CASSANDRA-4687 + try (FileDataInput fdi = dfile.createReader(indexEntry.position)) + { + DecoratedKey keyInDisk = decorateKey(ByteBufferUtil.readWithShortLength(fdi)); + if (!keyInDisk.equals(key)) + throw new AssertionError(String.format("%s != %s in %s", keyInDisk, key, fdi.getPath())); + } + } + + // store exact match for the key + cacheKey(decoratedKey, indexEntry); + } + if (op == Operator.EQ && updateCacheAndStats) + bloomFilterTracker.addTruePositive(); + Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndex().size(), descriptor.generation); + return indexEntry; + } + + RowIndexEntry.Serializer.skip(in); } - catch (IOException e) - { - markSuspect(); - throw new CorruptSSTableException(e, path); - } + } + catch (IOException e) + { + markSuspect(); + throw new CorruptSSTableException(e, path); } if (op == SSTableReader.Operator.EQ && updateCacheAndStats) diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java index 77bf3d65c7..38dab9a63b 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java @@ -121,7 +121,6 @@ public class BigTableWriter extends SSTableWriter if (logger.isTraceEnabled()) logger.trace("wrote {} at {}", decoratedKey, dataEnd); iwriter.append(decoratedKey, index, dataEnd); - dbuilder.addPotentialBoundary(dataEnd); } /** @@ -420,7 +419,6 @@ public class BigTableWriter extends SSTableWriter logger.trace("wrote index entry: {} at {}", indexEntry, indexStart); summary.maybeAddEntry(key, indexStart, indexEnd, dataEnd); - builder.addPotentialBoundary(indexStart); } /** diff --git a/src/java/org/apache/cassandra/io/util/AbstractDataInput.java b/src/java/org/apache/cassandra/io/util/AbstractDataInput.java deleted file mode 100644 index 29ce2c312c..0000000000 --- a/src/java/org/apache/cassandra/io/util/AbstractDataInput.java +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.util; - -import java.io.*; - -public abstract class AbstractDataInput extends InputStream implements DataInputPlus -{ - public abstract void seek(long position) throws IOException; - public abstract long getPosition(); - public abstract long getPositionLimit(); - - public int skipBytes(int n) throws IOException - { - if (n <= 0) - return 0; - long oldPosition = getPosition(); - seek(Math.min(getPositionLimit(), oldPosition + n)); - long skipped = getPosition() - oldPosition; - assert skipped >= 0 && skipped <= n; - return (int) skipped; - } - - /** - * Reads a boolean from the current position in this file. Blocks until one - * byte has been read, the end of the file is reached or an exception is - * thrown. - * - * @return the next boolean value from this file. - * @throws java.io.EOFException - * if the end of this file is detected. - * @throws java.io.IOException - * if this file is closed or another I/O error occurs. - */ - public final boolean readBoolean() throws IOException { - int temp = this.read(); - if (temp < 0) { - throw new EOFException(); - } - return temp != 0; - } - - /** - * Reads an 8-bit byte from the current position in this file. Blocks until - * one byte has been read, the end of the file is reached or an exception is - * thrown. - * - * @return the next signed 8-bit byte value from this file. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public final byte readByte() throws IOException { - int temp = this.read(); - if (temp < 0) { - throw new EOFException(); - } - return (byte) temp; - } - - /** - * Reads a 16-bit character from the current position in this file. Blocks until - * two bytes have been read, the end of the file is reached or an exception is - * thrown. - * - * @return the next char value from this file. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public final char readChar() throws IOException { - int ch1 = this.read(); - int ch2 = this.read(); - if ((ch1 | ch2) < 0) - throw new EOFException(); - return (char)((ch1 << 8) + (ch2 << 0)); - } - - /** - * Reads a 64-bit double from the current position in this file. Blocks - * until eight bytes have been read, the end of the file is reached or an - * exception is thrown. - * - * @return the next double value from this file. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public final double readDouble() throws IOException { - return Double.longBitsToDouble(readLong()); - } - - /** - * Reads a 32-bit float from the current position in this file. Blocks - * until four bytes have been read, the end of the file is reached or an - * exception is thrown. - * - * @return the next float value from this file. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public final float readFloat() throws IOException { - return Float.intBitsToFloat(readInt()); - } - - /** - * Reads bytes from this file into {@code buffer}. Blocks until {@code - * buffer.length} number of bytes have been read, the end of the file is - * reached or an exception is thrown. - * - * @param buffer - * the buffer to read bytes into. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - * @throws NullPointerException - * if {@code buffer} is {@code null}. - */ - public void readFully(byte[] buffer) throws IOException - { - readFully(buffer, 0, buffer.length); - } - - /** - * Read bytes from this file into {@code buffer} starting at offset {@code - * offset}. This method blocks until {@code count} number of bytes have been - * read. - * - * @param buffer - * the buffer to read bytes into. - * @param offset - * the initial position in {@code buffer} to store the bytes read - * from this file. - * @param count - * the maximum number of bytes to store in {@code buffer}. - * @throws EOFException - * if the end of this file is detected. - * @throws IndexOutOfBoundsException - * if {@code offset < 0} or {@code count < 0}, or if {@code - * offset + count} is greater than the length of {@code buffer}. - * @throws IOException - * if this file is closed or another I/O error occurs. - * @throws NullPointerException - * if {@code buffer} is {@code null}. - */ - public void readFully(byte[] buffer, int offset, int count) throws IOException - { - if (buffer == null) { - throw new NullPointerException(); - } - // avoid int overflow - if (offset < 0 || offset > buffer.length || count < 0 - || count > buffer.length - offset) { - throw new IndexOutOfBoundsException(); - } - while (count > 0) { - int result = read(buffer, offset, count); - if (result < 0) { - throw new EOFException(); - } - offset += result; - count -= result; - } - } - - /** - * Reads a 32-bit integer from the current position in this file. Blocks - * until four bytes have been read, the end of the file is reached or an - * exception is thrown. - * - * @return the next int value from this file. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public int readInt() throws IOException { - int ch1 = this.read(); - int ch2 = this.read(); - int ch3 = this.read(); - int ch4 = this.read(); - if ((ch1 | ch2 | ch3 | ch4) < 0) - throw new EOFException(); - return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0)); - } - - /** - * Reads a line of text form the current position in this file. A line is - * represented by zero or more characters followed by {@code '\n'}, {@code - * '\r'}, {@code "\r\n"} or the end of file marker. The string does not - * include the line terminating sequence. - *

- * Blocks until a line terminating sequence has been read, the end of the - * file is reached or an exception is thrown. - * - * @return the contents of the line or {@code null} if no characters have - * been read before the end of the file has been reached. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public final String readLine() throws IOException { - StringBuilder line = new StringBuilder(80); // Typical line length - boolean foundTerminator = false; - long unreadPosition = -1; - while (true) { - int nextByte = read(); - switch (nextByte) { - case -1: - return line.length() != 0 ? line.toString() : null; - case (byte) '\r': - if (foundTerminator) { - seek(unreadPosition); - return line.toString(); - } - foundTerminator = true; - /* Have to be able to peek ahead one byte */ - unreadPosition = getPosition(); - break; - case (byte) '\n': - return line.toString(); - default: - if (foundTerminator) { - seek(unreadPosition); - return line.toString(); - } - line.append((char) nextByte); - } - } - } - - /** - * Reads a 64-bit long from the current position in this file. Blocks until - * eight bytes have been read, the end of the file is reached or an - * exception is thrown. - * - * @return the next long value from this file. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public long readLong() throws IOException { - return ((long)(readInt()) << 32) + (readInt() & 0xFFFFFFFFL); - } - - /** - * Reads a 16-bit short from the current position in this file. Blocks until - * two bytes have been read, the end of the file is reached or an exception - * is thrown. - * - * @return the next short value from this file. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public short readShort() throws IOException { - int ch1 = this.read(); - int ch2 = this.read(); - if ((ch1 | ch2) < 0) - throw new EOFException(); - return (short)((ch1 << 8) + (ch2 << 0)); - } - - /** - * Reads an unsigned 8-bit byte from the current position in this file and - * returns it as an integer. Blocks until one byte has been read, the end of - * the file is reached or an exception is thrown. - * - * @return the next unsigned byte value from this file as an int. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public final int readUnsignedByte() throws IOException { - int temp = this.read(); - if (temp < 0) { - throw new EOFException(); - } - return temp; - } - - /** - * Reads an unsigned 16-bit short from the current position in this file and - * returns it as an integer. Blocks until two bytes have been read, the end of - * the file is reached or an exception is thrown. - * - * @return the next unsigned short value from this file as an int. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - */ - public int readUnsignedShort() throws IOException { - int ch1 = this.read(); - int ch2 = this.read(); - if ((ch1 | ch2) < 0) - throw new EOFException(); - return (ch1 << 8) + (ch2 << 0); - } - - /** - * Reads a string that is encoded in {@link java.io.DataInput modified UTF-8} from - * this file. The number of bytes that must be read for the complete string - * is determined by the first two bytes read from the file. Blocks until all - * required bytes have been read, the end of the file is reached or an - * exception is thrown. - * - * @return the next string encoded in {@link java.io.DataInput modified UTF-8} from - * this file. - * @throws EOFException - * if the end of this file is detected. - * @throws IOException - * if this file is closed or another I/O error occurs. - * @throws java.io.UTFDataFormatException - * if the bytes read cannot be decoded into a character string. - */ - public final String readUTF() throws IOException { - return DataInputStream.readUTF(this); - } -} diff --git a/src/java/org/apache/cassandra/io/util/BufferedSegmentedFile.java b/src/java/org/apache/cassandra/io/util/BufferedSegmentedFile.java index 744e828a06..090c5bd198 100644 --- a/src/java/org/apache/cassandra/io/util/BufferedSegmentedFile.java +++ b/src/java/org/apache/cassandra/io/util/BufferedSegmentedFile.java @@ -29,25 +29,8 @@ public class BufferedSegmentedFile extends SegmentedFile super(copy); } - private static class Cleanup extends SegmentedFile.Cleanup - { - protected Cleanup(ChannelProxy channel) - { - super(channel); - } - public void tidy() - { - super.tidy(); - } - } - public static class Builder extends SegmentedFile.Builder { - public void addPotentialBoundary(long boundary) - { - // only one segment in a standard-io file - } - public SegmentedFile complete(ChannelProxy channel, int bufferSize, long overrideLength) { long length = overrideLength > 0 ? overrideLength : channel.size(); @@ -55,13 +38,6 @@ public class BufferedSegmentedFile extends SegmentedFile } } - public FileDataInput getSegment(long position) - { - RandomAccessReader reader = RandomAccessReader.open(channel, bufferSize, -1L); - reader.seek(position); - return reader; - } - public BufferedSegmentedFile sharedCopy() { return new BufferedSegmentedFile(this); diff --git a/src/java/org/apache/cassandra/io/util/ByteBufferDataInput.java b/src/java/org/apache/cassandra/io/util/ByteBufferDataInput.java deleted file mode 100644 index bf926e90ab..0000000000 --- a/src/java/org/apache/cassandra/io/util/ByteBufferDataInput.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.util; - -import java.io.*; -import java.nio.ByteBuffer; - -import org.apache.cassandra.utils.ByteBufferUtil; - -public class ByteBufferDataInput extends AbstractDataInput implements FileDataInput, DataInput -{ - private final ByteBuffer buffer; - private final String filename; - private final long segmentOffset; - private int position; - - public ByteBufferDataInput(ByteBuffer buffer, String filename, long segmentOffset, int position) - { - assert buffer != null; - this.buffer = buffer; - this.filename = filename; - this.segmentOffset = segmentOffset; - this.position = position; - } - - // Only use when we know the seek in within the mapped segment. Throws an - // IOException otherwise. - public void seek(long pos) throws IOException - { - long inSegmentPos = pos - segmentOffset; - if (inSegmentPos < 0 || inSegmentPos > buffer.capacity()) - throw new IOException(String.format("Seek position %d is not within mmap segment (seg offs: %d, length: %d)", pos, segmentOffset, buffer.capacity())); - - position = (int) inSegmentPos; - } - - public long getFilePointer() - { - return segmentOffset + position; - } - - public long getPosition() - { - return segmentOffset + position; - } - - public long getPositionLimit() - { - return segmentOffset + buffer.capacity(); - } - - @Override - public boolean markSupported() - { - return false; - } - - public void reset(FileMark mark) throws IOException - { - assert mark instanceof MappedFileDataInputMark; - position = ((MappedFileDataInputMark) mark).position; - } - - public FileMark mark() - { - return new MappedFileDataInputMark(position); - } - - public long bytesPastMark(FileMark mark) - { - assert mark instanceof MappedFileDataInputMark; - assert position >= ((MappedFileDataInputMark) mark).position; - return position - ((MappedFileDataInputMark) mark).position; - } - - public boolean isEOF() throws IOException - { - return position == buffer.capacity(); - } - - public long bytesRemaining() throws IOException - { - return buffer.capacity() - position; - } - - public String getPath() - { - return filename; - } - - public int read() throws IOException - { - if (isEOF()) - return -1; - return buffer.get(position++) & 0xFF; - } - - /** - * Does the same thing as readFully do but without copying data (thread safe) - * @param length length of the bytes to read - * @return buffer with portion of file content - * @throws IOException on any fail of I/O operation - */ - public ByteBuffer readBytes(int length) throws IOException - { - int remaining = buffer.remaining() - position; - if (length > remaining) - throw new IOException(String.format("mmap segment underflow; remaining is %d but %d requested", - remaining, length)); - - if (length == 0) - return ByteBufferUtil.EMPTY_BYTE_BUFFER; - - ByteBuffer bytes = buffer.duplicate(); - bytes.position(buffer.position() + position).limit(buffer.position() + position + length); - position += length; - - // we have to copy the data in case we unreference the underlying sstable. See CASSANDRA-3179 - ByteBuffer clone = ByteBuffer.allocate(bytes.remaining()); - clone.put(bytes); - clone.flip(); - return clone; - } - - @Override - public final void readFully(byte[] bytes) throws IOException - { - ByteBufferUtil.arrayCopy(buffer, buffer.position() + position, bytes, 0, bytes.length); - position += bytes.length; - } - - @Override - public final void readFully(byte[] bytes, int offset, int count) throws IOException - { - ByteBufferUtil.arrayCopy(buffer, buffer.position() + position, bytes, offset, count); - position += count; - } - - private static class MappedFileDataInputMark implements FileMark - { - int position; - - MappedFileDataInputMark(int position) - { - this.position = position; - } - } - - @Override - public String toString() { - return getClass().getSimpleName() + "(" + - "filename='" + filename + "'" + - ", position=" + position + - ")"; - } -} diff --git a/src/java/org/apache/cassandra/io/util/ChannelProxy.java b/src/java/org/apache/cassandra/io/util/ChannelProxy.java index 79954a5d78..f8661606b9 100644 --- a/src/java/org/apache/cassandra/io/util/ChannelProxy.java +++ b/src/java/org/apache/cassandra/io/util/ChannelProxy.java @@ -63,7 +63,7 @@ public final class ChannelProxy extends SharedCloseableImpl public ChannelProxy(File file) { - this(file.getAbsolutePath(), openChannel(file)); + this(file.getPath(), openChannel(file)); } public ChannelProxy(String filePath, FileChannel channel) @@ -87,7 +87,7 @@ public final class ChannelProxy extends SharedCloseableImpl final String filePath; final FileChannel channel; - protected Cleanup(String filePath, FileChannel channel) + Cleanup(String filePath, FileChannel channel) { this.filePath = filePath; this.channel = channel; diff --git a/src/java/org/apache/cassandra/io/util/ChecksummedRandomAccessReader.java b/src/java/org/apache/cassandra/io/util/ChecksummedRandomAccessReader.java index 98869a19b5..30f1e0c1fa 100644 --- a/src/java/org/apache/cassandra/io/util/ChecksummedRandomAccessReader.java +++ b/src/java/org/apache/cassandra/io/util/ChecksummedRandomAccessReader.java @@ -23,43 +23,33 @@ import java.util.zip.CRC32; import org.apache.cassandra.io.compress.BufferType; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Throwables; public class ChecksummedRandomAccessReader extends RandomAccessReader { @SuppressWarnings("serial") public static class CorruptFileException extends RuntimeException { - public final File file; + public final String filePath; - public CorruptFileException(Exception cause, File file) + public CorruptFileException(Exception cause, String filePath) { super(cause); - this.file = file; + this.filePath = filePath; } } private final DataIntegrityMetadata.ChecksumValidator validator; - private final File file; - protected ChecksummedRandomAccessReader(File file, ChannelProxy channel, DataIntegrityMetadata.ChecksumValidator validator) + private ChecksummedRandomAccessReader(Builder builder) { - super(channel, validator.chunkSize, -1, BufferType.ON_HEAP); - this.validator = validator; - this.file = file; + super(builder); + this.validator = builder.validator; } @SuppressWarnings("resource") - public static ChecksummedRandomAccessReader open(File file, File crcFile) throws IOException - { - ChannelProxy channel = new ChannelProxy(file); - RandomAccessReader crcReader = RandomAccessReader.open(crcFile); - DataIntegrityMetadata.ChecksumValidator validator = - new DataIntegrityMetadata.ChecksumValidator(new CRC32(), crcReader, file.getPath()); - return new ChecksummedRandomAccessReader(file, channel, validator); - } - @Override - protected void reBuffer() + protected void reBufferStandard() { long desiredPosition = current(); // align with buffer size, as checksums were computed in chunks of buffer size each. @@ -84,12 +74,18 @@ public class ChecksummedRandomAccessReader extends RandomAccessReader } catch (IOException e) { - throw new CorruptFileException(e, file); + throw new CorruptFileException(e, channel.filePath()); } buffer.position((int) (desiredPosition - bufferOffset)); } + @Override + protected void reBufferMmap() + { + throw new AssertionError("Unsupported operation"); + } + @Override public void seek(long newPosition) { @@ -100,14 +96,32 @@ public class ChecksummedRandomAccessReader extends RandomAccessReader @Override public void close() { - try + Throwables.perform(channel.filePath(), Throwables.FileOpType.READ, + super::close, + validator::close, + channel::close); + } + + public static final class Builder extends RandomAccessReader.Builder + { + private final DataIntegrityMetadata.ChecksumValidator validator; + + @SuppressWarnings("resource") + public Builder(File file, File crcFile) throws IOException { - super.close(); + super(new ChannelProxy(file)); + this.validator = new DataIntegrityMetadata.ChecksumValidator(new CRC32(), + RandomAccessReader.open(crcFile), + file.getPath()); + + super.bufferSize(validator.chunkSize) + .bufferType(BufferType.ON_HEAP); } - finally + + @Override + public RandomAccessReader build() { - channel.close(); - validator.close(); + return new ChecksummedRandomAccessReader(this); } } } diff --git a/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java b/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java index 95c61c151a..16f791a43d 100644 --- a/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java +++ b/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java @@ -17,44 +17,49 @@ */ package org.apache.cassandra.io.util; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.util.TreeMap; - import com.google.common.util.concurrent.RateLimiter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.compress.CompressedRandomAccessReader; import org.apache.cassandra.io.compress.CompressedSequentialWriter; -import org.apache.cassandra.io.compress.CompressedThrottledReader; import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.concurrent.Ref; public class CompressedSegmentedFile extends SegmentedFile implements ICompressedFile { - public final CompressionMetadata metadata; + private static final Logger logger = LoggerFactory.getLogger(CompressedSegmentedFile.class); private static final boolean useMmap = DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap; - private static int MAX_SEGMENT_SIZE = Integer.MAX_VALUE; - private final TreeMap chunkSegments; + + public final CompressionMetadata metadata; + private final MmappedRegions regions; public CompressedSegmentedFile(ChannelProxy channel, int bufferSize, CompressionMetadata metadata) { - this(channel, bufferSize, metadata, createMappedSegments(channel, metadata)); + this(channel, + bufferSize, + metadata, + useMmap + ? MmappedRegions.map(channel, metadata) + : null); } - public CompressedSegmentedFile(ChannelProxy channel, int bufferSize, CompressionMetadata metadata, TreeMap chunkSegments) + public CompressedSegmentedFile(ChannelProxy channel, int bufferSize, CompressionMetadata metadata, MmappedRegions regions) { - super(new Cleanup(channel, metadata, chunkSegments), channel, bufferSize, metadata.dataLength, metadata.compressedFileLength); + super(new Cleanup(channel, metadata, regions), channel, bufferSize, metadata.dataLength, metadata.compressedFileLength); this.metadata = metadata; - this.chunkSegments = chunkSegments; + this.regions = regions; } private CompressedSegmentedFile(CompressedSegmentedFile copy) { super(copy); this.metadata = copy.metadata; - this.chunkSegments = copy.chunkSegments; + this.regions = copy.regions; } public ChannelProxy channel() @@ -62,60 +67,36 @@ public class CompressedSegmentedFile extends SegmentedFile implements ICompresse return channel; } - public TreeMap chunkSegments() + public MmappedRegions regions() { - return chunkSegments; - } - - static TreeMap createMappedSegments(ChannelProxy channel, CompressionMetadata metadata) - { - if (!useMmap) - return null; - TreeMap chunkSegments = new TreeMap<>(); - long offset = 0; - long lastSegmentOffset = 0; - long segmentSize = 0; - - while (offset < metadata.dataLength) - { - CompressionMetadata.Chunk chunk = metadata.chunkFor(offset); - - //Reached a new mmap boundary - if (segmentSize + chunk.length + 4 > MAX_SEGMENT_SIZE) - { - chunkSegments.put(lastSegmentOffset, channel.map(FileChannel.MapMode.READ_ONLY, lastSegmentOffset, segmentSize)); - lastSegmentOffset += segmentSize; - segmentSize = 0; - } - - segmentSize += chunk.length + 4; //checksum - offset += metadata.chunkLength(); - } - - if (segmentSize > 0) - chunkSegments.put(lastSegmentOffset, channel.map(FileChannel.MapMode.READ_ONLY, lastSegmentOffset, segmentSize)); - return chunkSegments; + return regions; } private static final class Cleanup extends SegmentedFile.Cleanup { final CompressionMetadata metadata; - final TreeMap chunkSegments; - protected Cleanup(ChannelProxy channel, CompressionMetadata metadata, TreeMap chunkSegments) + private final MmappedRegions regions; + + protected Cleanup(ChannelProxy channel, CompressionMetadata metadata, MmappedRegions regions) { super(channel); this.metadata = metadata; - this.chunkSegments = chunkSegments; + this.regions = regions; } public void tidy() { - super.tidy(); - metadata.close(); - if (chunkSegments != null) + Throwable err = regions == null ? null : regions.close(null); + if (err != null) { - for (MappedByteBuffer segment : chunkSegments.values()) - FileUtils.clean(segment); + JVMStabilityInspector.inspectThrowable(err); + + // This is not supposed to happen + logger.error("Error while closing mmapped regions", err); } + + metadata.close(); + + super.tidy(); } } @@ -132,17 +113,12 @@ public class CompressedSegmentedFile extends SegmentedFile implements ICompresse public static class Builder extends SegmentedFile.Builder { - protected final CompressedSequentialWriter writer; + final CompressedSequentialWriter writer; public Builder(CompressedSequentialWriter writer) { this.writer = writer; } - public void addPotentialBoundary(long boundary) - { - // only one segment in a standard-io file - } - protected CompressionMetadata metadata(String path, long overrideLength) { if (writer == null) @@ -166,12 +142,12 @@ public class CompressedSegmentedFile extends SegmentedFile implements ICompresse public RandomAccessReader createReader() { - return CompressedRandomAccessReader.open(this); + return new CompressedRandomAccessReader.Builder(this).build(); } - public RandomAccessReader createThrottledReader(RateLimiter limiter) + public RandomAccessReader createReader(RateLimiter limiter) { - return CompressedThrottledReader.open(this, limiter); + return new CompressedRandomAccessReader.Builder(this).limiter(limiter).build(); } public CompressionMetadata getMetadata() diff --git a/src/java/org/apache/cassandra/io/util/DataInputBuffer.java b/src/java/org/apache/cassandra/io/util/DataInputBuffer.java index 63091d034e..a68dcc25ff 100644 --- a/src/java/org/apache/cassandra/io/util/DataInputBuffer.java +++ b/src/java/org/apache/cassandra/io/util/DataInputBuffer.java @@ -21,13 +21,10 @@ import java.io.IOException; import java.nio.ByteBuffer; /** - * Input stream around a fixed ByteBuffer. Necessary to have this derived class to avoid NIODataInputStream's - * shuffling of bytes behavior in readNext() - * + * Input stream around a single ByteBuffer. */ -public class DataInputBuffer extends NIODataInputStream +public class DataInputBuffer extends RebufferingInputStream { - private static ByteBuffer slice(byte[] buffer, int offset, int length) { ByteBuffer buf = ByteBuffer.wrap(buffer); @@ -41,13 +38,12 @@ public class DataInputBuffer extends NIODataInputStream } /** - * - * @param buf + * @param buffer * @param duplicate Whether or not to duplicate the buffer to ensure thread safety */ - public DataInputBuffer(ByteBuffer buf, boolean duplicate) + public DataInputBuffer(ByteBuffer buffer, boolean duplicate) { - super(buf, duplicate); + super(duplicate ? buffer.duplicate() : buffer); } public DataInputBuffer(byte[] buffer, int offset, int length) @@ -61,8 +57,14 @@ public class DataInputBuffer extends NIODataInputStream } @Override - protected int readNext() throws IOException + protected void reBuffer() throws IOException { - return -1; + //nope, we don't rebuffer, we are done! + } + + @Override + public int available() throws IOException + { + return buffer.remaining(); } } diff --git a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java index 70cd8604b8..01c004911b 100644 --- a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java +++ b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java @@ -34,6 +34,7 @@ import com.google.common.base.Charsets; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.utils.Throwables; public class DataIntegrityMetadata { @@ -138,14 +139,8 @@ public class DataIntegrityMetadata public void close() { - try - { - this.digestReader.close(); - } - finally - { - this.dataReader.close(); - } + Throwables.perform(digestReader::close, + dataReader::close); } } diff --git a/src/java/org/apache/cassandra/io/util/FileDataInput.java b/src/java/org/apache/cassandra/io/util/FileDataInput.java index b63b750702..f56193bb1c 100644 --- a/src/java/org/apache/cassandra/io/util/FileDataInput.java +++ b/src/java/org/apache/cassandra/io/util/FileDataInput.java @@ -19,31 +19,22 @@ package org.apache.cassandra.io.util; import java.io.Closeable; import java.io.IOException; -import java.nio.ByteBuffer; public interface FileDataInput extends DataInputPlus, Closeable { - public String getPath(); + String getPath(); - public boolean isEOF() throws IOException; + boolean isEOF() throws IOException; - public long bytesRemaining() throws IOException; + long bytesRemaining() throws IOException; - public void seek(long pos) throws IOException; + void seek(long pos) throws IOException; - public FileMark mark(); + FileMark mark(); - public void reset(FileMark mark) throws IOException; + void reset(FileMark mark) throws IOException; - public long bytesPastMark(FileMark mark); + long bytesPastMark(FileMark mark); - public long getFilePointer(); - - /** - * Read length bytes from current file position - * @param length length of the bytes to read - * @return buffer with bytes read - * @throws IOException if any I/O operation failed - */ - public ByteBuffer readBytes(int length) throws IOException; + long getFilePointer(); } diff --git a/src/java/org/apache/cassandra/io/util/FileSegmentInputStream.java b/src/java/org/apache/cassandra/io/util/FileSegmentInputStream.java new file mode 100644 index 0000000000..425c7d6f35 --- /dev/null +++ b/src/java/org/apache/cassandra/io/util/FileSegmentInputStream.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.util; + +import java.nio.ByteBuffer; + +/** + * This is the same as DataInputBuffer, i.e. a stream for a fixed byte buffer, + * except that we also implement FileDataInput by using an offset and a file path. + */ +public class FileSegmentInputStream extends DataInputBuffer implements FileDataInput +{ + private final String filePath; + private final long offset; + + public FileSegmentInputStream(ByteBuffer buffer, String filePath, long offset) + { + super(buffer, false); + this.filePath = filePath; + this.offset = offset; + } + + public String getPath() + { + return filePath; + } + + private long size() + { + return offset + buffer.capacity(); + } + + public boolean isEOF() + { + return !buffer.hasRemaining(); + } + + public long bytesRemaining() + { + return buffer.remaining(); + } + + public void seek(long pos) + { + if (pos < 0 || pos > size()) + throw new IllegalArgumentException(String.format("Unable to seek to position %d in %s (%d bytes) in partial mode", + pos, + getPath(), + size())); + + + buffer.position((int) (pos - offset)); + } + + @Override + public boolean markSupported() + { + return false; + } + + public FileMark mark() + { + throw new UnsupportedOperationException(); + } + + public void reset(FileMark mark) + { + throw new UnsupportedOperationException(); + } + + public long bytesPastMark(FileMark mark) + { + return 0; + } + + public long getFilePointer() + { + return offset + buffer.position(); + } +} diff --git a/src/java/org/apache/cassandra/io/util/ICompressedFile.java b/src/java/org/apache/cassandra/io/util/ICompressedFile.java index ce7b22c32d..43d37dc3ce 100644 --- a/src/java/org/apache/cassandra/io/util/ICompressedFile.java +++ b/src/java/org/apache/cassandra/io/util/ICompressedFile.java @@ -17,14 +17,11 @@ */ package org.apache.cassandra.io.util; -import java.nio.MappedByteBuffer; -import java.util.TreeMap; - import org.apache.cassandra.io.compress.CompressionMetadata; public interface ICompressedFile { - public ChannelProxy channel(); - public CompressionMetadata getMetadata(); - public TreeMap chunkSegments(); + ChannelProxy channel(); + CompressionMetadata getMetadata(); + MmappedRegions regions(); } diff --git a/src/java/org/apache/cassandra/io/util/MemoryInputStream.java b/src/java/org/apache/cassandra/io/util/MemoryInputStream.java index 45261e0eed..e009528819 100644 --- a/src/java/org/apache/cassandra/io/util/MemoryInputStream.java +++ b/src/java/org/apache/cassandra/io/util/MemoryInputStream.java @@ -19,50 +19,58 @@ package org.apache.cassandra.io.util; import java.io.DataInput; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; -public class MemoryInputStream extends AbstractDataInput implements DataInput +import com.google.common.annotations.VisibleForTesting; +import com.google.common.primitives.Ints; + +import org.apache.cassandra.utils.memory.MemoryUtil; + +public class MemoryInputStream extends RebufferingInputStream implements DataInput { private final Memory mem; - private int position = 0; + private final int bufferSize; + private long offset; + public MemoryInputStream(Memory mem) { + this(mem, Ints.saturatedCast(mem.size)); + } + + @VisibleForTesting + public MemoryInputStream(Memory mem, int bufferSize) + { + super(getByteBuffer(mem.peer, bufferSize)); this.mem = mem; + this.bufferSize = bufferSize; + this.offset = mem.peer + bufferSize; } - public int read() throws IOException + @Override + protected void reBuffer() throws IOException { - return mem.getByte(position++) & 0xFF; + if (offset - mem.peer >= mem.size()) + return; + + buffer = getByteBuffer(offset, Math.min(bufferSize, Ints.saturatedCast(memRemaining()))); + offset += buffer.capacity(); } - public void readFully(byte[] buffer, int offset, int count) throws IOException + @Override + public int available() { - mem.getBytes(position, buffer, offset, count); - position += count; + return Ints.saturatedCast(buffer.remaining() + memRemaining()); } - public void seek(long pos) + private long memRemaining() { - position = (int) pos; + return mem.size + mem.peer - offset; } - public long getPosition() + private static ByteBuffer getByteBuffer(long offset, int length) { - return position; - } - - public long getPositionLimit() - { - return mem.size(); - } - - protected long length() - { - return mem.size(); - } - - public void close() - { - // do nothing. + return MemoryUtil.getByteBuffer(offset, length).order(ByteOrder.BIG_ENDIAN); } } diff --git a/src/java/org/apache/cassandra/io/util/MmappedRegions.java b/src/java/org/apache/cassandra/io/util/MmappedRegions.java new file mode 100644 index 0000000000..8f6cd92eb5 --- /dev/null +++ b/src/java/org/apache/cassandra/io/util/MmappedRegions.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.util; + +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Arrays; + +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.concurrent.RefCounted; +import org.apache.cassandra.utils.concurrent.SharedCloseableImpl; + +import static java.util.stream.Stream.of; +import static org.apache.cassandra.utils.Throwables.perform; + +public class MmappedRegions extends SharedCloseableImpl +{ + /** In a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size */ + public static int MAX_SEGMENT_SIZE = Integer.MAX_VALUE; + + /** When we need to grow the arrays, we add this number of region slots */ + static final int REGION_ALLOC_SIZE = 15; + + /** The original state, which is shared with the tidier and + * contains all the regions mapped so far. It also + * does the actual mapping. */ + private final State state; + + /** A copy of the latest state. We update this each time the original state is + * updated and we share this with copies. If we are a copy, then this + * is null. Copies can only access existing regions, they cannot create + * new ones. This is for thread safety and because MmappedRegions is + * reference counted, only the original state will be cleaned-up, + * therefore only the original state should create new mapped regions. + */ + private volatile State copy; + + private MmappedRegions(ChannelProxy channel, CompressionMetadata metadata, long length) + { + this(new State(channel), metadata, length); + } + + private MmappedRegions(State state, CompressionMetadata metadata, long length) + { + super(new Tidier(state)); + + this.state = state; + + if (metadata != null) + { + assert length == 0 : "expected no length with metadata"; + updateState(metadata); + } + else if (length > 0) + { + updateState(length); + } + + this.copy = new State(state); + } + + private MmappedRegions(MmappedRegions original) + { + super(original); + this.state = original.copy; + } + + public static MmappedRegions empty(ChannelProxy channel) + { + return new MmappedRegions(channel, null, 0); + } + + public static MmappedRegions map(ChannelProxy channel, CompressionMetadata metadata) + { + if (metadata == null) + throw new IllegalArgumentException("metadata cannot be null"); + + return new MmappedRegions(channel, metadata, 0); + } + + public static MmappedRegions map(ChannelProxy channel, long length) + { + if (length <= 0) + throw new IllegalArgumentException("Length must be positive"); + + return new MmappedRegions(channel, null, length); + } + + /** + * @return a snapshot of the memory mapped regions. The snapshot can + * only use existing regions, it cannot create new ones. + */ + public MmappedRegions sharedCopy() + { + return new MmappedRegions(this); + } + + private boolean isCopy() + { + return copy == null; + } + + public void extend(long length) + { + if (length < 0) + throw new IllegalArgumentException("Length must not be negative"); + + assert !isCopy() : "Copies cannot be extended"; + + if (length <= state.length) + return; + + updateState(length); + copy = new State(state); + } + + private void updateState(long length) + { + state.length = length; + long pos = state.getPosition(); + while (pos < length) + { + long size = Math.min(MAX_SEGMENT_SIZE, length - pos); + state.add(pos, size); + pos += size; + } + } + + private void updateState(CompressionMetadata metadata) + { + long offset = 0; + long lastSegmentOffset = 0; + long segmentSize = 0; + + while (offset < metadata.dataLength) + { + CompressionMetadata.Chunk chunk = metadata.chunkFor(offset); + + //Reached a new mmap boundary + if (segmentSize + chunk.length + 4 > MAX_SEGMENT_SIZE) + { + if (segmentSize > 0) + { + state.add(lastSegmentOffset, segmentSize); + lastSegmentOffset += segmentSize; + segmentSize = 0; + } + } + + segmentSize += chunk.length + 4; //checksum + offset += metadata.chunkLength(); + } + + if (segmentSize > 0) + state.add(lastSegmentOffset, segmentSize); + + state.length = lastSegmentOffset + segmentSize; + } + + public boolean isValid(ChannelProxy channel) + { + return state.isValid(channel); + } + + public boolean isEmpty() + { + return state.isEmpty(); + } + + public Region floor(long position) + { + assert !isCleanedUp() : "Attempted to use closed region"; + return state.floor(position); + } + + public static final class Region + { + public final long offset; + public final ByteBuffer buffer; + + public Region(long offset, ByteBuffer buffer) + { + this.offset = offset; + this.buffer = buffer; + } + + public long bottom() + { + return offset; + } + + public long top() + { + return offset + buffer.capacity(); + } + } + + private static final class State + { + /** The file channel */ + private final ChannelProxy channel; + + /** An array of region buffers, synchronized with offsets */ + private ByteBuffer[] buffers; + + /** An array of region offsets, synchronized with buffers */ + private long[] offsets; + + /** The maximum file length we have mapped */ + private long length; + + /** The index to the last region added */ + private int last; + + private State(ChannelProxy channel) + { + this.channel = channel.sharedCopy(); + this.buffers = new ByteBuffer[REGION_ALLOC_SIZE]; + this.offsets = new long[REGION_ALLOC_SIZE]; + this.length = 0; + this.last = -1; + } + + private State(State original) + { + this.channel = original.channel; + this.buffers = original.buffers; + this.offsets = original.offsets; + this.length = original.length; + this.last = original.last; + } + + private boolean isEmpty() + { + return last < 0; + } + + private boolean isValid(ChannelProxy channel) + { + return this.channel.filePath().equals(channel.filePath()); + } + + private Region floor(long position) + { + assert 0 <= position && position < length : String.format("%d >= %d", position, length); + + int idx = Arrays.binarySearch(offsets, 0, last +1, position); + assert idx != -1 : String.format("Bad position %d for regions %s, last %d in %s", position, Arrays.toString(offsets), last, channel); + if (idx < 0) + idx = -(idx + 2); // round down to entry at insertion point + + return new Region(offsets[idx], buffers[idx]); + } + + private long getPosition() + { + return last < 0 ? 0 : offsets[last] + buffers[last].capacity(); + } + + private void add(long pos, long size) + { + ByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, pos, size); + + ++last; + + if (last == offsets.length) + { + offsets = Arrays.copyOf(offsets, offsets.length + REGION_ALLOC_SIZE); + buffers = Arrays.copyOf(buffers, buffers.length + REGION_ALLOC_SIZE); + } + + offsets[last] = pos; + buffers[last] = buffer; + } + + private Throwable close(Throwable accumulate) + { + accumulate = channel.close(accumulate); + + /* + * Try forcing the unmapping of segments using undocumented unsafe sun APIs. + * If this fails (non Sun JVM), we'll have to wait for the GC to finalize the mapping. + * If this works and a thread tries to access any segment, hell will unleash on earth. + */ + if (!FileUtils.isCleanerAvailable()) + return accumulate; + + return perform(accumulate, channel.filePath(), Throwables.FileOpType.READ, + of(buffers) + .map((buffer) -> + () -> + { + if (buffer != null) + FileUtils.clean(buffer); + })); + } + } + + public static final class Tidier implements RefCounted.Tidy + { + final State state; + + Tidier(State state) + { + this.state = state; + } + + public String name() + { + return state.channel.filePath(); + } + + public void tidy() + { + try + { + Throwables.maybeFail(state.close(null)); + } + catch (Exception e) + { + throw new FSReadError(e, state.channel.filePath()); + } + } + } + +} diff --git a/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java b/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java index 879ca6f882..5f56ff64a1 100644 --- a/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java +++ b/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java @@ -18,41 +18,31 @@ package org.apache.cassandra.io.util; import java.io.*; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.google.common.util.concurrent.RateLimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.utils.JVMStabilityInspector; public class MmappedSegmentedFile extends SegmentedFile { private static final Logger logger = LoggerFactory.getLogger(MmappedSegmentedFile.class); - // in a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size to stay sane. - public static long MAX_SEGMENT_SIZE = Integer.MAX_VALUE; + private final MmappedRegions regions; - /** - * Sorted array of segment offsets and MappedByteBuffers for segments. If mmap is completely disabled, or if the - * segment would be too long to mmap, the value for an offset will be null, indicating that we need to fall back - * to a RandomAccessFile. - */ - private final Segment[] segments; - - public MmappedSegmentedFile(ChannelProxy channel, int bufferSize, long length, Segment[] segments) + public MmappedSegmentedFile(ChannelProxy channel, int bufferSize, long length, MmappedRegions regions) { - super(new Cleanup(channel, segments), channel, bufferSize, length); - this.segments = segments; + super(new Cleanup(channel, regions), channel, bufferSize, length); + this.regions = regions; } private MmappedSegmentedFile(MmappedSegmentedFile copy) { super(copy); - this.segments = copy.segments; + this.regions = copy.regions; } public MmappedSegmentedFile sharedCopy() @@ -60,78 +50,46 @@ public class MmappedSegmentedFile extends SegmentedFile return new MmappedSegmentedFile(this); } - /** - * @return The segment entry for the given position. - */ - private Segment floor(long position) + public RandomAccessReader createReader() { - assert 0 <= position && position < length: String.format("%d >= %d in %s", position, length, path()); - Segment seg = new Segment(position, null); - int idx = Arrays.binarySearch(segments, seg); - assert idx != -1 : String.format("Bad position %d for segments %s in %s", position, Arrays.toString(segments), path()); - if (idx < 0) - // round down to entry at insertion point - idx = -(idx + 2); - return segments[idx]; + return new RandomAccessReader.Builder(channel) + .overrideLength(length) + .regions(regions) + .build(); } - /** - * @return The segment containing the given position: must be closed after use. - */ - public FileDataInput getSegment(long position) + public RandomAccessReader createReader(RateLimiter limiter) { - Segment segment = floor(position); - if (segment.right != null) - { - // segment is mmap'd - return new ByteBufferDataInput(segment.right, path(), segment.left, (int) (position - segment.left)); - } - - // we can have single cells or partitions larger than 2Gb, which is our maximum addressable range in a single segment; - // in this case we open as a normal random access reader - // FIXME: brafs are unbounded, so this segment will cover the rest of the file, rather than just the row - RandomAccessReader file = RandomAccessReader.open(channel, bufferSize, -1L); - file.seek(position); - return file; + return new RandomAccessReader.Builder(channel) + .overrideLength(length) + .bufferSize(bufferSize) + .regions(regions) + .limiter(limiter) + .build(); } private static final class Cleanup extends SegmentedFile.Cleanup { - final Segment[] segments; - protected Cleanup(ChannelProxy channel, Segment[] segments) + private final MmappedRegions regions; + + Cleanup(ChannelProxy channel, MmappedRegions regions) { super(channel); - this.segments = segments; + this.regions = regions; } public void tidy() { - super.tidy(); - - if (!FileUtils.isCleanerAvailable()) - return; - - /* - * Try forcing the unmapping of segments using undocumented unsafe sun APIs. - * If this fails (non Sun JVM), we'll have to wait for the GC to finalize the mapping. - * If this works and a thread tries to access any segment, hell will unleash on earth. - */ - try + Throwable err = regions.close(null); + if (err != null) { - for (Segment segment : segments) - { - if (segment.right == null) - continue; - FileUtils.clean(segment.right); - } - logger.debug("All segments have been unmapped successfully"); - } - catch (Exception e) - { - JVMStabilityInspector.inspectThrowable(e); + JVMStabilityInspector.inspectThrowable(err); + // This is not supposed to happen - logger.error("Error while unmapping segments", e); + logger.error("Error while closing mmapped regions", err); } + + super.tidy(); } } @@ -140,104 +98,64 @@ public class MmappedSegmentedFile extends SegmentedFile */ static class Builder extends SegmentedFile.Builder { - // planned segment boundaries - private List boundaries; + private MmappedRegions regions; - // offset of the open segment (first segment begins at 0). - private long currentStart = 0; - - // current length of the open segment. - // used to allow merging multiple too-large-to-mmap segments, into a single buffered segment. - private long currentSize = 0; - - public Builder() + Builder() { super(); - boundaries = new ArrayList<>(); - boundaries.add(0L); - } - - public void addPotentialBoundary(long boundary) - { - if (boundary - currentStart <= MAX_SEGMENT_SIZE) - { - // boundary fits into current segment: expand it - currentSize = boundary - currentStart; - return; - } - - // close the current segment to try and make room for the boundary - if (currentSize > 0) - { - currentStart += currentSize; - boundaries.add(currentStart); - } - currentSize = boundary - currentStart; - - // if we couldn't make room, the boundary needs its own segment - if (currentSize > MAX_SEGMENT_SIZE) - { - currentStart = boundary; - boundaries.add(currentStart); - currentSize = 0; - } } public SegmentedFile complete(ChannelProxy channel, int bufferSize, long overrideLength) { long length = overrideLength > 0 ? overrideLength : channel.size(); - // create the segments - return new MmappedSegmentedFile(channel, bufferSize, length, createSegments(channel, length)); + updateRegions(channel, length); + + return new MmappedSegmentedFile(channel, bufferSize, length, regions.sharedCopy()); } - private Segment[] createSegments(ChannelProxy channel, long length) + private void updateRegions(ChannelProxy channel, long length) { - // if we're early finishing a range that doesn't span multiple segments, but the finished file now does, - // we remove these from the end (we loop incase somehow this spans multiple segments, but that would - // be a loco dataset - while (length < boundaries.get(boundaries.size() - 1)) - boundaries.remove(boundaries.size() -1); - - // add a sentinel value == length - List boundaries = new ArrayList<>(this.boundaries); - if (length != boundaries.get(boundaries.size() - 1)) - boundaries.add(length); - - int segcount = boundaries.size() - 1; - Segment[] segments = new Segment[segcount]; - for (int i = 0; i < segcount; i++) + if (regions != null && !regions.isValid(channel)) { - long start = boundaries.get(i); - long size = boundaries.get(i + 1) - start; - MappedByteBuffer segment = size <= MAX_SEGMENT_SIZE - ? channel.map(FileChannel.MapMode.READ_ONLY, start, size) - : null; - segments[i] = new Segment(start, segment); + Throwable err = regions.close(null); + if (err != null) + logger.error("Failed to close mapped regions", err); + + regions = null; } - return segments; + + if (regions == null) + regions = MmappedRegions.map(channel, length); + else + regions.extend(length); } @Override - public void serializeBounds(DataOutput out) throws IOException + public void serializeBounds(DataOutput out, Version version) throws IOException { - super.serializeBounds(out); - out.writeInt(boundaries.size()); - for (long position: boundaries) - out.writeLong(position); + if (!version.hasBoundaries()) + return; + + super.serializeBounds(out, version); + out.writeInt(0); } @Override - public void deserializeBounds(DataInput in) throws IOException + public void deserializeBounds(DataInput in, Version version) throws IOException { - super.deserializeBounds(in); + if (!version.hasBoundaries()) + return; - int size = in.readInt(); - List temp = new ArrayList<>(size); - - for (int i = 0; i < size; i++) - temp.add(in.readLong()); + super.deserializeBounds(in, version); + in.skipBytes(in.readInt() * TypeSizes.sizeof(0L)); + } - boundaries = temp; + @Override + public Throwable close(Throwable accumulate) + { + return super.close(regions == null + ? accumulate + : regions.close(accumulate)); } } } diff --git a/src/java/org/apache/cassandra/io/util/NIODataInputStream.java b/src/java/org/apache/cassandra/io/util/NIODataInputStream.java index f6c939a783..e599a696e8 100644 --- a/src/java/org/apache/cassandra/io/util/NIODataInputStream.java +++ b/src/java/org/apache/cassandra/io/util/NIODataInputStream.java @@ -18,17 +18,11 @@ package org.apache.cassandra.io.util; import java.io.Closeable; -import java.io.DataInputStream; -import java.io.EOFException; import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.channels.ReadableByteChannel; import java.nio.channels.SeekableByteChannel; -import org.apache.cassandra.utils.vint.VIntCoding; - import com.google.common.base.Preconditions; /** @@ -43,357 +37,59 @@ import com.google.common.base.Preconditions; * * NIODataInputStream is not thread safe. */ -public class NIODataInputStream extends InputStream implements DataInputPlus, Closeable +public class NIODataInputStream extends RebufferingInputStream { - private final ReadableByteChannel rbc; - private final ByteBuffer buf; + protected final ReadableByteChannel channel; - /* - * Used when wrapping a fixed buffer of data instead of a channel. Should never attempt - * to read from it. - */ - private static final ReadableByteChannel emptyReadableByteChannel = new ReadableByteChannel() + private static ByteBuffer makeBuffer(int bufferSize) { + ByteBuffer buffer = ByteBuffer.allocateDirect(bufferSize); + buffer.position(0); + buffer.limit(0); - @Override - public boolean isOpen() - { - return true; - } - - @Override - public void close() throws IOException - { - } - - @Override - public int read(ByteBuffer dst) throws IOException - { - throw new AssertionError(); - } - - }; - - public NIODataInputStream(ReadableByteChannel rbc, int bufferSize) - { - Preconditions.checkNotNull(rbc); - Preconditions.checkArgument(bufferSize >= 9, "Buffer size must be large enough to accomadate a varint"); - this.rbc = rbc; - buf = ByteBuffer.allocateDirect(bufferSize); - buf.position(0); - buf.limit(0); + return buffer; } - protected NIODataInputStream(ByteBuffer buf, boolean duplicate) + public NIODataInputStream(ReadableByteChannel channel, ByteBuffer buffer) { - Preconditions.checkNotNull(buf); - if (duplicate) - this.buf = buf.duplicate(); - else - this.buf = buf; + super(buffer); - this.rbc = emptyReadableByteChannel; + Preconditions.checkNotNull(channel); + this.channel = channel; } - /* - * The decision to duplicate or not really needs to conscious since it a real impact - * in terms of thread safety so don't expose this constructor with an implicit default. - */ - protected NIODataInputStream(ByteBuffer buf) + public NIODataInputStream(ReadableByteChannel channel, int bufferSize) { - this(buf, false); + this(channel, makeBuffer(bufferSize)); } @Override - public void readFully(byte[] b) throws IOException + protected void reBuffer() throws IOException { - readFully(b, 0, b.length); - } + Preconditions.checkState(buffer.remaining() == 0); + buffer.clear(); + while ((channel.read(buffer)) == 0) {} - @Override - public void readFully(byte[] b, int off, int len) throws IOException - { - int copied = 0; - while (copied < len) - { - int read = read(b, off + copied, len - copied); - if (read < 0) - throw new EOFException(); - copied += read; - } - } - - @Override - public int read(byte b[], int off, int len) throws IOException { - if (b == null) - throw new NullPointerException(); - - // avoid int overflow - if (off < 0 || off > b.length || len < 0 - || len > b.length - off) - throw new IndexOutOfBoundsException(); - - if (len == 0) - return 0; - - int copied = 0; - while (copied < len) - { - if (buf.hasRemaining()) - { - int toCopy = Math.min(len - copied, buf.remaining()); - buf.get(b, off + copied, toCopy); - copied += toCopy; - } - else - { - int read = readNext(); - if (read < 0 && copied == 0) return -1; - if (read <= 0) return copied; - } - } - - return copied; - } - - /* - * Refill the buffer, preserving any unread bytes remaining in the buffer - */ - protected int readNext() throws IOException - { - Preconditions.checkState(buf.remaining() != buf.capacity()); - assert(buf.remaining() < 9); - - /* - * If there is data already at the start of the buffer, move the position to the end - * If there is data but not at the start, move it to the start - * Otherwise move the position to 0 so writes start at the beginning of the buffer - * - * We go to the trouble of shuffling the bytes remaining for cases where the buffer isn't fully drained - * while retrieving a multi-byte value while the position is in the middle. - */ - if (buf.position() == 0 && buf.hasRemaining()) - { - buf.position(buf.limit()); - } - else if (buf.hasRemaining()) - { - //FastByteOperations.copy failed to do the copy so inline a simple one here - int position = buf.position(); - int remaining = buf.remaining(); - buf.clear(); - for (int ii = 0; ii < remaining; ii++) - buf.put(buf.get(position + ii)); - } - else - { - buf.position(0); - } - - buf.limit(buf.capacity()); - - int read = 0; - while ((read = rbc.read(buf)) == 0) {} - - buf.flip(); - - return read; - } - - /* - * Read the minimum number of bytes and throw EOF if the minimum could not be read - */ - private void readMinimum(int minimum) throws IOException - { - assert(buf.remaining() < 8); - while (buf.remaining() < minimum) - { - int read = readNext(); - if (read == -1) - { - //DataInputStream consumes the bytes even if it doesn't get the entire value, match the behavior here - buf.position(0); - buf.limit(0); - throw new EOFException(); - } - } - } - - /* - * Ensure the buffer contains the minimum number of readable bytes, throws EOF if enough bytes aren't available - */ - private void prepareReadPrimitive(int minimum) throws IOException - { - if (buf.remaining() < minimum) - readMinimum(minimum); - } - - @Override - public int skipBytes(int n) throws IOException - { - int skipped = 0; - - while (skipped < n) - { - int skippedThisTime = (int)skip(n - skipped); - if (skippedThisTime <= 0) break; - skipped += skippedThisTime; - } - - return skipped; - } - - @Override - public boolean readBoolean() throws IOException - { - prepareReadPrimitive(1); - return buf.get() != 0; - } - - @Override - public byte readByte() throws IOException - { - prepareReadPrimitive(1); - return buf.get(); - } - - @Override - public int readUnsignedByte() throws IOException - { - prepareReadPrimitive(1); - return buf.get() & 0xff; - } - - @Override - public short readShort() throws IOException - { - prepareReadPrimitive(2); - return buf.getShort(); - } - - @Override - public int readUnsignedShort() throws IOException - { - return readShort() & 0xFFFF; - } - - @Override - public char readChar() throws IOException - { - prepareReadPrimitive(2); - return buf.getChar(); - } - - @Override - public int readInt() throws IOException - { - prepareReadPrimitive(4); - return buf.getInt(); - } - - @Override - public long readLong() throws IOException - { - prepareReadPrimitive(8); - return buf.getLong(); - } - - public long readVInt() throws IOException - { - return VIntCoding.decodeZigZag64(readUnsignedVInt()); - } - - public long readUnsignedVInt() throws IOException - { - //If 9 bytes aren't available use the slow path in VIntCoding - if (buf.remaining() < 9) - return VIntCoding.readUnsignedVInt(this); - - byte firstByte = buf.get(); - - //Bail out early if this is one byte, necessary or it fails later - if (firstByte >= 0) - return firstByte; - - int extraBytes = VIntCoding.numberOfExtraBytesToRead(firstByte); - - int position = buf.position(); - int extraBits = extraBytes * 8; - - long retval = buf.getLong(position); - if (buf.order() == ByteOrder.LITTLE_ENDIAN) - retval = Long.reverseBytes(retval); - buf.position(position + extraBytes); - - // truncate the bytes we read in excess of those we needed - retval >>>= 64 - extraBits; - // remove the non-value bits from the first byte - firstByte &= VIntCoding.firstByteValueMask(extraBytes); - // shift the first byte up to its correct position - retval |= (long) firstByte << extraBits; - return retval; - } - - @Override - public float readFloat() throws IOException - { - prepareReadPrimitive(4); - return buf.getFloat(); - } - - @Override - public double readDouble() throws IOException - { - prepareReadPrimitive(8); - return buf.getDouble(); - } - - @Override - public String readLine() throws IOException - { - throw new UnsupportedOperationException(); - } - - @Override - public String readUTF() throws IOException - { - return DataInputStream.readUTF(this); + buffer.flip(); } @Override public void close() throws IOException { - rbc.close(); - } - - @Override - public int read() throws IOException - { - return readUnsignedByte(); + channel.close(); + super.close(); } @Override public int available() throws IOException { - if (rbc instanceof SeekableByteChannel) + if (channel instanceof SeekableByteChannel) { - SeekableByteChannel sbc = (SeekableByteChannel)rbc; + SeekableByteChannel sbc = (SeekableByteChannel) channel; long remainder = Math.max(0, sbc.size() - sbc.position()); - return (remainder > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int)(remainder + buf.remaining()); + return (remainder > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int)(remainder + buffer.remaining()); } - return buf.remaining(); - } - - @Override - public void reset() throws IOException - { - throw new IOException("mark/reset not supported"); - } - - @Override - public boolean markSupported() - { - return false; + return buffer.remaining(); } } diff --git a/src/java/org/apache/cassandra/io/util/RandomAccessReader.java b/src/java/org/apache/cassandra/io/util/RandomAccessReader.java index b13d154605..e9b0ee4657 100644 --- a/src/java/org/apache/cassandra/io/util/RandomAccessReader.java +++ b/src/java/org/apache/cassandra/io/util/RandomAccessReader.java @@ -19,119 +19,129 @@ package org.apache.cassandra.io.util; import java.io.*; import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.RateLimiter; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.compress.BufferType; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.memory.BufferPool; -public class RandomAccessReader extends AbstractDataInput implements FileDataInput +public class RandomAccessReader extends RebufferingInputStream implements FileDataInput { + // The default buffer size when the client doesn't specify it public static final int DEFAULT_BUFFER_SIZE = 4096; + // The maximum buffer size when the limiter is not null, i.e. when throttling + // is enabled. This is required to avoid aquiring permits that are too large. + public static final int MAX_THROTTLED_BUFFER_SIZE = 1 << 16; // 64k + // the IO channel to the file, we do not own a reference to this due to // performance reasons (CASSANDRA-9379) so it's up to the owner of the RAR to // ensure that the channel stays open and that it is closed afterwards protected final ChannelProxy channel; - // buffer which will cache file blocks - protected ByteBuffer buffer; + // optional memory mapped regions for the channel + protected final MmappedRegions regions; - // `bufferOffset` is the offset of the beginning of the buffer - // `markedPointer` folds the offset of the last file mark - protected long bufferOffset, markedPointer; + // An optional limiter that will throttle the amount of data we read + protected final RateLimiter limiter; - // this can be overridden at construction to a value shorter than the true length of the file; - // if so, it acts as an imposed limit on reads, rather than a convenience property + // the file length, this can be overridden at construction to a value shorter + // than the true length of the file; if so, it acts as an imposed limit on reads, + // required when opening sstables early not to read past the mark private final long fileLength; - protected RandomAccessReader(ChannelProxy channel, int bufferSize, long overrideLength, BufferType bufferType) - { - this.channel = channel; + // the buffer size for buffered readers + protected final int bufferSize; - if (bufferSize <= 0) + // the buffer type for buffered readers + private final BufferType bufferType; + + // offset from the beginning of the file + protected long bufferOffset; + + // offset of the last file mark + protected long markedPointer; + + protected RandomAccessReader(Builder builder) + { + super(null); + + this.channel = builder.channel; + this.regions = builder.regions; + this.limiter = builder.limiter; + this.fileLength = builder.overrideLength <= 0 ? builder.channel.size() : builder.overrideLength; + this.bufferSize = getBufferSize(builder); + this.bufferType = builder.bufferType; + + if (builder.bufferSize <= 0) throw new IllegalArgumentException("bufferSize must be positive"); - // we can cache file length in read-only mode - fileLength = overrideLength <= 0 ? channel.size() : overrideLength; + if (builder.initializeBuffers) + initializeBuffer(); + } + + protected int getBufferSize(Builder builder) + { + if (builder.limiter == null) + return builder.bufferSize; + + // limit to ensure more accurate throttling + return Math.min(MAX_THROTTLED_BUFFER_SIZE, builder.bufferSize); + } + + protected void initializeBuffer() + { + if (regions == null) + buffer = allocateBuffer(bufferSize); + else + buffer = regions.floor(0).buffer.duplicate(); - buffer = allocateBuffer(getBufferSize(bufferSize), bufferType); buffer.limit(0); } - /** The buffer size is typically already page aligned but if that is not the case - * make sure that it is a multiple of the page size, 4096. - * */ - protected int getBufferSize(int size) + protected ByteBuffer allocateBuffer(int size) { - if ((size & ~4095) != size) - { // should already be a page size multiple but if that's not case round it up - size = (size + 4095) & ~4095; - } - return size; + return BufferPool.get(size, bufferType).order(ByteOrder.BIG_ENDIAN); } - protected ByteBuffer allocateBuffer(int size, BufferType bufferType) + protected void releaseBuffer() { - return BufferPool.get(size, bufferType); - } - - // A wrapper of the RandomAccessReader that closes the channel when done. - // For performance reasons RAR does not increase the reference count of - // a channel but assumes the owner will keep it open and close it, - // see CASSANDRA-9379, this thin class is just for those cases where we do - // not have a shared channel. - private static class RandomAccessReaderWithChannel extends RandomAccessReader - { - @SuppressWarnings("resource") - RandomAccessReaderWithChannel(File file) + if (buffer != null) { - super(new ChannelProxy(file), DEFAULT_BUFFER_SIZE, -1L, BufferType.OFF_HEAP); + if (regions == null) + BufferPool.put(buffer); + buffer = null; } - - @Override - public void close() - { - try - { - super.close(); - } - finally - { - channel.close(); - } - } - } - - public static RandomAccessReader open(File file) - { - return new RandomAccessReaderWithChannel(file); - } - - public static RandomAccessReader open(ChannelProxy channel) - { - return open(channel, DEFAULT_BUFFER_SIZE, -1L); - } - - public static RandomAccessReader open(ChannelProxy channel, int bufferSize, long overrideSize) - { - return new RandomAccessReader(channel, bufferSize, overrideSize, BufferType.OFF_HEAP); - } - - public ChannelProxy getChannel() - { - return channel; } /** * Read data from file starting from current currentOffset to populate buffer. */ - protected void reBuffer() + public void reBuffer() + { + if (isEOF()) + return; + + if (regions == null) + reBufferStandard(); + else + reBufferMmap(); + + if (limiter != null) + limiter.acquire(buffer.remaining()); + + assert buffer.order() == ByteOrder.BIG_ENDIAN : "Buffer must have BIG ENDIAN byte ordering"; + } + + protected void reBufferStandard() { bufferOffset += buffer.position(); - buffer.clear(); assert bufferOffset < fileLength; + buffer.clear(); long position = bufferOffset; long limit = bufferOffset; @@ -145,15 +155,31 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp { int n = channel.read(buffer, position); if (n < 0) - break; + throw new FSReadError(new IOException("Unexpected end of file"), channel.filePath()); + position += n; limit = bufferOffset + buffer.position(); } - if (limit > fileLength) - buffer.position((int)(fileLength - bufferOffset)); + buffer.flip(); } + protected void reBufferMmap() + { + long position = bufferOffset + buffer.position(); + assert position < fileLength; + + MmappedRegions.Region region = regions.floor(position); + bufferOffset = region.bottom(); + buffer = region.buffer.duplicate(); + buffer.position(Ints.checkedCast(position - bufferOffset)); + + if (limiter != null && bufferSize < buffer.remaining()) + { // ensure accurate throttling + buffer.limit(buffer.position() + bufferSize); + } + } + @Override public long getFilePointer() { @@ -170,19 +196,23 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp return channel.filePath(); } - public int getTotalBufferSize() + public ChannelProxy getChannel() { - //This may NPE so we make a ref - //https://issues.apache.org/jira/browse/CASSANDRA-7756 - ByteBuffer ref = buffer; - return ref != null ? ref.capacity() : 0; + return channel; } - public void reset() + @Override + public void reset() throws IOException { seek(markedPointer); } + @Override + public boolean markSupported() + { + return true; + } + public long bytesPastMark() { long bytes = current() - markedPointer; @@ -215,7 +245,7 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp */ public boolean isEOF() { - return getFilePointer() == length(); + return current() == length(); } public long bytesRemaining() @@ -223,6 +253,12 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp return length() - getFilePointer(); } + @Override + public int available() throws IOException + { + return Ints.saturatedCast(bytesRemaining()); + } + @Override public void close() { @@ -230,16 +266,17 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp if (buffer == null) return; - bufferOffset += buffer.position(); - BufferPool.put(buffer); - buffer = null; + releaseBuffer(); + + //For performance reasons we don't keep a reference to the file + //channel so we don't close it } @Override public String toString() { - return getClass().getSimpleName() + "(" + "filePath='" + channel + "')"; + return getClass().getSimpleName() + "(filePath='" + channel + "')"; } /** @@ -286,78 +323,51 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp assert current() == newPosition; } - // -1 will be returned if there is nothing to read; higher-level methods like readInt - // or readFully (from RandomAccessFile) will throw EOFException but this should not - public int read() + /** + * Reads a line of text form the current position in this file. A line is + * represented by zero or more characters followed by {@code '\n'}, {@code + * '\r'}, {@code "\r\n"} or the end of file marker. The string does not + * include the line terminating sequence. + *

+ * Blocks until a line terminating sequence has been read, the end of the + * file is reached or an exception is thrown. + * + * @return the contents of the line or {@code null} if no characters have + * been read before the end of the file has been reached. + * @throws IOException if this file is closed or another I/O error occurs. + */ + public final String readLine() throws IOException { - if (buffer == null) - throw new AssertionError("Attempted to read from closed RAR"); - - if (isEOF()) - return -1; // required by RandomAccessFile - - if (!buffer.hasRemaining()) - reBuffer(); - - return (int)buffer.get() & 0xff; - } - - @Override - public int read(byte[] buffer) - { - return read(buffer, 0, buffer.length); - } - - @Override - // -1 will be returned if there is nothing to read; higher-level methods like readInt - // or readFully (from RandomAccessFile) will throw EOFException but this should not - public int read(byte[] buff, int offset, int length) - { - if (buffer == null) - throw new IllegalStateException("Attempted to read from closed RAR"); - - if (length == 0) - return 0; - - if (isEOF()) - return -1; - - if (!buffer.hasRemaining()) - reBuffer(); - - int toCopy = Math.min(length, buffer.remaining()); - buffer.get(buff, offset, toCopy); - return toCopy; - } - - public ByteBuffer readBytes(int length) throws EOFException - { - assert length >= 0 : "buffer length should not be negative: " + length; - - if (buffer == null) - throw new IllegalStateException("Attempted to read from closed RAR"); - - try + StringBuilder line = new StringBuilder(80); // Typical line length + boolean foundTerminator = false; + long unreadPosition = -1; + while (true) { - ByteBuffer result = ByteBuffer.allocate(length); - while (result.hasRemaining()) + int nextByte = read(); + switch (nextByte) { - if (isEOF()) - throw new EOFException(); - if (!buffer.hasRemaining()) - reBuffer(); - ByteBufferUtil.put(buffer, result); + case -1: + return line.length() != 0 ? line.toString() : null; + case (byte) '\r': + if (foundTerminator) + { + seek(unreadPosition); + return line.toString(); + } + foundTerminator = true; + /* Have to be able to peek ahead one byte */ + unreadPosition = getPosition(); + break; + case (byte) '\n': + return line.toString(); + default: + if (foundTerminator) + { + seek(unreadPosition); + return line.toString(); + } + line.append((char) nextByte); } - result.flip(); - return result; - } - catch (EOFException e) - { - throw e; - } - catch (Exception e) - { - throw new FSReadError(e, channel.toString()); } } @@ -368,11 +378,142 @@ public class RandomAccessReader extends AbstractDataInput implements FileDataInp public long getPosition() { - return bufferOffset + (buffer == null ? 0 : buffer.position()); + return current(); } - public long getPositionLimit() + public static class Builder { - return length(); + // The NIO file channel or an empty channel + public final ChannelProxy channel; + + // We override the file length when we open sstables early, so that we do not + // read past the early mark + public long overrideLength; + + // The size of the buffer for buffered readers + public int bufferSize; + + // The type of the buffer for buffered readers + public BufferType bufferType; + + // The mmap segments for mmap readers + public MmappedRegions regions; + + // An optional limiter that will throttle the amount of data we read + public RateLimiter limiter; + + public boolean initializeBuffers; + + public Builder(ChannelProxy channel) + { + this.channel = channel; + this.overrideLength = -1L; + this.bufferSize = getBufferSize(DEFAULT_BUFFER_SIZE); + this.bufferType = BufferType.OFF_HEAP; + this.regions = null; + this.limiter = null; + this.initializeBuffers = true; + } + + /** The buffer size is typically already page aligned but if that is not the case + * make sure that it is a multiple of the page size, 4096. + * */ + private static int getBufferSize(int size) + { + if ((size & ~4095) != size) + { // should already be a page size multiple but if that's not case round it up + size = (size + 4095) & ~4095; + } + return size; + } + + public Builder overrideLength(long overrideLength) + { + if (overrideLength > channel.size()) + throw new IllegalArgumentException("overrideLength cannot be more than the file size"); + + this.overrideLength = overrideLength; + return this; + } + + public Builder bufferSize(int bufferSize) + { + if (bufferSize <= 0) + throw new IllegalArgumentException("bufferSize must be positive"); + + this.bufferSize = getBufferSize(bufferSize); + return this; + } + + public Builder bufferType(BufferType bufferType) + { + this.bufferType = bufferType; + return this; + } + + public Builder regions(MmappedRegions regions) + { + this.regions = regions; + return this; + } + + public Builder limiter(RateLimiter limiter) + { + this.limiter = limiter; + return this; + } + + public Builder initializeBuffers(boolean initializeBuffers) + { + this.initializeBuffers = initializeBuffers; + return this; + } + + public RandomAccessReader build() + { + return new RandomAccessReader(this); + } + + public RandomAccessReader buildWithChannel() + { + return new RandomAccessReaderWithOwnChannel(this); + } + } + + // A wrapper of the RandomAccessReader that closes the channel when done. + // For performance reasons RAR does not increase the reference count of + // a channel but assumes the owner will keep it open and close it, + // see CASSANDRA-9379, this thin class is just for those cases where we do + // not have a shared channel. + public static class RandomAccessReaderWithOwnChannel extends RandomAccessReader + { + protected RandomAccessReaderWithOwnChannel(Builder builder) + { + super(builder); + } + + @Override + public void close() + { + try + { + super.close(); + } + finally + { + channel.close(); + } + } + } + + @SuppressWarnings("resource") + public static RandomAccessReader open(File file) + { + return new Builder(new ChannelProxy(file)).buildWithChannel(); + } + + public static RandomAccessReader open(ChannelProxy channel) + { + return new Builder(channel).build(); } } diff --git a/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java new file mode 100644 index 0000000000..7d64f3d2ba --- /dev/null +++ b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.util; + +import java.io.Closeable; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import net.nicoulaj.compilecommand.annotations.DontInline; +import net.nicoulaj.compilecommand.annotations.Print; +import org.apache.cassandra.utils.FastByteOperations; +import org.apache.cassandra.utils.vint.VIntCoding; + +import com.google.common.base.Preconditions; + +/** + * Rough equivalent of BufferedInputStream and DataInputStream wrapping a ByteBuffer that can be refilled + * via rebuffer. Implementations provide this buffer from various channels (socket, file, memory, etc). + * + * RebufferingInputStream is not thread safe. + */ +public abstract class RebufferingInputStream extends InputStream implements DataInputPlus, Closeable +{ + protected ByteBuffer buffer; + + protected RebufferingInputStream(ByteBuffer buffer) + { + Preconditions.checkArgument(buffer == null || buffer.order() == ByteOrder.BIG_ENDIAN, "Buffer must have BIG ENDIAN byte ordering"); + this.buffer = buffer; + } + + /** + * Implementations must implement this method to refill the buffer. + * They can expect the buffer to be empty when this method is invoked. + * @throws IOException + */ + protected abstract void reBuffer() throws IOException; + + @Override + public void readFully(byte[] b) throws IOException + { + readFully(b, 0, b.length); + } + + @Override + public void readFully(byte[] b, int off, int len) throws IOException + { + int read = read(b, off, len); + if (read < len) + throw new EOFException(); + } + + @Print + @Override + public int read(byte[] b, int off, int len) throws IOException { + + // avoid int overflow + if (off < 0 || off > b.length || len < 0 || len > b.length - off) + throw new IndexOutOfBoundsException(); + + if (len == 0) + return 0; + + int copied = 0; + while (copied < len) + { + int position = buffer.position(); + int remaining = buffer.limit() - position; + if (remaining == 0) + { + reBuffer(); + position = buffer.position(); + remaining = buffer.limit() - position; + if (remaining == 0) + return copied == 0 ? -1 : copied; + } + int toCopy = Math.min(len - copied, remaining); + FastByteOperations.copy(buffer, position, b, off + copied, toCopy); + buffer.position(position + toCopy); + copied += toCopy; + } + + return copied; + } + + @DontInline + protected long readPrimitiveSlowly(int bytes) throws IOException + { + long result = 0; + for (int i = 0; i < bytes; i++) + result = (result << 8) | (readByte() & 0xFFL); + return result; + } + + @Override + public int skipBytes(int n) throws IOException + { + int skipped = 0; + + while (skipped < n) + { + int skippedThisTime = (int)skip(n - skipped); + if (skippedThisTime <= 0) break; + skipped += skippedThisTime; + } + + return skipped; + } + + @Override + public boolean readBoolean() throws IOException + { + return readByte() != 0; + } + + @Override + public byte readByte() throws IOException + { + if (!buffer.hasRemaining()) + { + reBuffer(); + if (!buffer.hasRemaining()) + throw new EOFException(); + } + + return buffer.get(); + } + + @Override + public int readUnsignedByte() throws IOException + { + return readByte() & 0xff; + } + + @Override + public short readShort() throws IOException + { + if (buffer.remaining() >= 2) + return buffer.getShort(); + else + return (short) readPrimitiveSlowly(2); + } + + @Override + public int readUnsignedShort() throws IOException + { + return readShort() & 0xFFFF; + } + + @Override + public char readChar() throws IOException + { + if (buffer.remaining() >= 2) + return buffer.getChar(); + else + return (char) readPrimitiveSlowly(2); + } + + @Override + public int readInt() throws IOException + { + if (buffer.remaining() >= 4) + return buffer.getInt(); + else + return (int) readPrimitiveSlowly(4); + } + + @Override + public long readLong() throws IOException + { + if (buffer.remaining() >= 8) + return buffer.getLong(); + else + return readPrimitiveSlowly(8); + } + + public long readVInt() throws IOException + { + return VIntCoding.decodeZigZag64(readUnsignedVInt()); + } + + public long readUnsignedVInt() throws IOException + { + //If 9 bytes aren't available use the slow path in VIntCoding + if (buffer.remaining() < 9) + return VIntCoding.readUnsignedVInt(this); + + byte firstByte = buffer.get(); + + //Bail out early if this is one byte, necessary or it fails later + if (firstByte >= 0) + return firstByte; + + int extraBytes = VIntCoding.numberOfExtraBytesToRead(firstByte); + + int position = buffer.position(); + int extraBits = extraBytes * 8; + + long retval = buffer.getLong(position); + if (buffer.order() == ByteOrder.LITTLE_ENDIAN) + retval = Long.reverseBytes(retval); + buffer.position(position + extraBytes); + + // truncate the bytes we read in excess of those we needed + retval >>>= 64 - extraBits; + // remove the non-value bits from the first byte + firstByte &= VIntCoding.firstByteValueMask(extraBytes); + // shift the first byte up to its correct position + retval |= (long) firstByte << extraBits; + return retval; + } + + @Override + public float readFloat() throws IOException + { + if (buffer.remaining() >= 4) + return buffer.getFloat(); + else + return Float.intBitsToFloat((int)readPrimitiveSlowly(4)); + } + + @Override + public double readDouble() throws IOException + { + if (buffer.remaining() >= 8) + return buffer.getDouble(); + else + return Double.longBitsToDouble(readPrimitiveSlowly(8)); + } + + @Override + public String readLine() throws IOException + { + throw new UnsupportedOperationException(); + } + + @Override + public String readUTF() throws IOException + { + return DataInputStream.readUTF(this); + } + + @Override + public int read() throws IOException + { + try + { + return readUnsignedByte(); + } + catch (EOFException ex) + { + return -1; + } + } + + @Override + public void reset() throws IOException + { + throw new IOException("mark/reset not supported"); + } + + @Override + public boolean markSupported() + { + return false; + } +} diff --git a/src/java/org/apache/cassandra/io/util/SegmentedFile.java b/src/java/org/apache/cassandra/io/util/SegmentedFile.java index e586682da4..c2a2374e25 100644 --- a/src/java/org/apache/cassandra/io/util/SegmentedFile.java +++ b/src/java/org/apache/cassandra/io/util/SegmentedFile.java @@ -21,23 +21,19 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.File; import java.io.IOException; -import java.nio.MappedByteBuffer; -import java.util.Iterator; -import java.util.NoSuchElementException; import com.google.common.util.concurrent.RateLimiter; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.compress.CompressedSequentialWriter; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.IndexSummary; import org.apache.cassandra.io.sstable.IndexSummaryBuilder; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.utils.CLibrary; -import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.RefCounted; import org.apache.cassandra.utils.concurrent.SharedCloseableImpl; @@ -79,7 +75,7 @@ public abstract class SegmentedFile extends SharedCloseableImpl this.onDiskLength = onDiskLength; } - public SegmentedFile(SegmentedFile copy) + protected SegmentedFile(SegmentedFile copy) { super(copy); channel = copy.channel; @@ -93,7 +89,7 @@ public abstract class SegmentedFile extends SharedCloseableImpl return channel.filePath(); } - protected static abstract class Cleanup implements RefCounted.Tidy + protected static class Cleanup implements RefCounted.Tidy { final ChannelProxy channel; protected Cleanup(ChannelProxy channel) @@ -116,16 +112,22 @@ public abstract class SegmentedFile extends SharedCloseableImpl public RandomAccessReader createReader() { - return RandomAccessReader.open(channel, bufferSize, length); + return new RandomAccessReader.Builder(channel) + .overrideLength(length) + .bufferSize(bufferSize) + .build(); } - public RandomAccessReader createThrottledReader(RateLimiter limiter) + public RandomAccessReader createReader(RateLimiter limiter) { - assert limiter != null; - return ThrottledReader.open(channel, bufferSize, length, limiter); + return new RandomAccessReader.Builder(channel) + .overrideLength(length) + .bufferSize(bufferSize) + .limiter(limiter) + .build(); } - public FileDataInput getSegment(long position) + public FileDataInput createReader(long position) { RandomAccessReader reader = createReader(); reader.seek(position); @@ -152,14 +154,6 @@ public abstract class SegmentedFile extends SharedCloseableImpl return new CompressedSegmentedFile.Builder(writer); } - /** - * @return An Iterator over segments, beginning with the segment containing the given position: each segment must be closed after use. - */ - public Iterator iterator(long position) - { - return new SegmentIterator(position); - } - /** * Collects potential segmentation points in an underlying file, and builds a SegmentedFile to represent it. */ @@ -167,13 +161,6 @@ public abstract class SegmentedFile extends SharedCloseableImpl { private ChannelProxy channel; - /** - * Adds a position that would be a safe place for a segment boundary in the file. For a block/row based file - * format, safe boundaries are block/row edges. - * @param boundary The absolute position of the potential boundary in the file. - */ - public abstract void addPotentialBoundary(long boundary); - /** * Called after all potential boundaries have been added to apply this Builder to a concrete file on disk. * @param channel The channel to the file on disk. @@ -214,12 +201,12 @@ public abstract class SegmentedFile extends SharedCloseableImpl return complete(desc.filenameFor(Component.PRIMARY_INDEX), bufferSize(desc, indexSummary), -1L); } - private int bufferSize(StatsMetadata stats) + private static int bufferSize(StatsMetadata stats) { return bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); } - private int bufferSize(Descriptor desc, IndexSummary indexSummary) + private static int bufferSize(Descriptor desc, IndexSummary indexSummary) { File file = new File(desc.filenameFor(Component.PRIMARY_INDEX)); return bufferSize(file.length() / indexSummary.size()); @@ -267,13 +254,19 @@ public abstract class SegmentedFile extends SharedCloseableImpl return (int)Math.min(size, 1 << 16); } - public void serializeBounds(DataOutput out) throws IOException + public void serializeBounds(DataOutput out, Version version) throws IOException { + if (!version.hasBoundaries()) + return; + out.writeUTF(DatabaseDescriptor.getDiskAccessMode().name()); } - public void deserializeBounds(DataInput in) throws IOException + public void deserializeBounds(DataInput in, Version version) throws IOException { + if (!version.hasBoundaries()) + return; + if (!in.readUTF().equals(DatabaseDescriptor.getDiskAccessMode().name())) throw new IOException("Cannot deserialize SSTable Summary component because the DiskAccessMode was changed!"); } @@ -282,6 +275,7 @@ public abstract class SegmentedFile extends SharedCloseableImpl { if (channel != null) return channel.close(accumulate); + return accumulate; } @@ -294,6 +288,10 @@ public abstract class SegmentedFile extends SharedCloseableImpl { if (channel != null) { + // This is really fragile, both path and channel.filePath() + // must agree, i.e. they both must be absolute or both relative + // eventually we should really pass the filePath to the builder + // constructor and remove this if (channel.filePath().equals(path)) return channel.sharedCopy(); else @@ -305,61 +303,10 @@ public abstract class SegmentedFile extends SharedCloseableImpl } } - static final class Segment extends Pair implements Comparable - { - public Segment(long offset, MappedByteBuffer segment) - { - super(offset, segment); - } - - public final int compareTo(Segment that) - { - return (int)Math.signum(this.left - that.left); - } - } - - /** - * A lazy Iterator over segments in forward order from the given position. It is caller's responsibility - * to close the FileDataIntputs when finished. - */ - final class SegmentIterator implements Iterator - { - private long nextpos; - public SegmentIterator(long position) - { - this.nextpos = position; - } - - public boolean hasNext() - { - return nextpos < length; - } - - public FileDataInput next() - { - long position = nextpos; - if (position >= length) - throw new NoSuchElementException(); - - FileDataInput segment = getSegment(nextpos); - try - { - nextpos = nextpos + segment.bytesRemaining(); - } - catch (IOException e) - { - throw new FSReadError(e, path()); - } - return segment; - } - - public void remove() { throw new UnsupportedOperationException(); } - } - @Override public String toString() { - return getClass().getSimpleName() + "(path='" + path() + "'" + + return getClass().getSimpleName() + "(path='" + path() + '\'' + ", length=" + length + - ")"; + ')'; } } diff --git a/src/java/org/apache/cassandra/io/util/ThrottledReader.java b/src/java/org/apache/cassandra/io/util/ThrottledReader.java deleted file mode 100644 index 024d38f0db..0000000000 --- a/src/java/org/apache/cassandra/io/util/ThrottledReader.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.apache.cassandra.io.util; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import com.google.common.util.concurrent.RateLimiter; - -import org.apache.cassandra.io.compress.BufferType; - -public class ThrottledReader extends RandomAccessReader -{ - private final RateLimiter limiter; - - protected ThrottledReader(ChannelProxy channel, int bufferSize, long overrideLength, RateLimiter limiter) - { - super(channel, bufferSize, overrideLength, BufferType.OFF_HEAP); - this.limiter = limiter; - } - - protected void reBuffer() - { - limiter.acquire(buffer.capacity()); - super.reBuffer(); - } - - public static ThrottledReader open(ChannelProxy channel, int bufferSize, long overrideLength, RateLimiter limiter) - { - return new ThrottledReader(channel, bufferSize, overrideLength, limiter); - } -} diff --git a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java index 6b247073be..adbd091284 100644 --- a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java +++ b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java @@ -55,7 +55,7 @@ public class CompressedStreamWriter extends StreamWriter public void write(DataOutputStreamPlus out) throws IOException { long totalSize = totalSize(); - try (RandomAccessReader file = sstable.openDataReader(); final ChannelProxy fc = file.getChannel().sharedCopy()) + try (ChannelProxy fc = sstable.getDataChannel().sharedCopy()) { long progress = 0L; // calculate chunks to transfer. we want to send continuous chunks altogether. @@ -72,13 +72,7 @@ public class CompressedStreamWriter extends StreamWriter final long bytesTransferredFinal = bytesTransferred; final int toTransfer = (int) Math.min(CHUNK_SIZE, length - bytesTransferred); limiter.acquire(toTransfer); - long lastWrite = out.applyToChannel(new Function() - { - public Long apply(WritableByteChannel wbc) - { - return fc.transferTo(section.left + bytesTransferredFinal, toTransfer, wbc); - } - }); + long lastWrite = out.applyToChannel((wbc) -> fc.transferTo(section.left + bytesTransferredFinal, toTransfer, wbc)); bytesTransferred += lastWrite; progress += lastWrite; session.progress(sstable.descriptor, ProgressInfo.Direction.OUT, progress, totalSize); diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java index abc2a37bd5..a05c3c806e 100644 --- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java +++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java @@ -398,9 +398,6 @@ public class ByteBufferUtil if (length == 0) return EMPTY_BYTE_BUFFER; - if (in instanceof FileDataInput) - return ((FileDataInput) in).readBytes(length); - byte[] buff = new byte[length]; in.readFully(buff); return ByteBuffer.wrap(buff); diff --git a/src/java/org/apache/cassandra/utils/Throwables.java b/src/java/org/apache/cassandra/utils/Throwables.java index d6ce7b4535..a5170ffc61 100644 --- a/src/java/org/apache/cassandra/utils/Throwables.java +++ b/src/java/org/apache/cassandra/utils/Throwables.java @@ -82,29 +82,54 @@ public final class Throwables @SuppressWarnings("unchecked") public static void perform(Stream> actions) throws E { - Throwable fail = null; - Iterator> iter = actions.iterator(); - while (iter.hasNext()) + Throwable fail = perform(null, actions); + if (failIfCanCast(fail, null)) + throw (E) fail; + } + + public static Throwable perform(Throwable accumulate, Stream> actions) + { + return perform(accumulate, actions.iterator()); + } + + public static Throwable perform(Throwable accumulate, Iterator> actions) + { + while (actions.hasNext()) { - DiscreteAction action = iter.next(); + DiscreteAction action = actions.next(); try { action.perform(); } catch (Throwable t) { - fail = merge(fail, t); + accumulate = merge(accumulate, t); } } - - if (failIfCanCast(fail, null)) - throw (E) fail; + return accumulate; } @SafeVarargs public static void perform(File against, FileOpType opType, DiscreteAction ... actions) { - perform(Arrays.stream(actions).map((action) -> () -> + perform(against.getPath(), opType, actions); + } + + @SafeVarargs + public static void perform(String filePath, FileOpType opType, DiscreteAction ... actions) + { + maybeFail(perform(null, filePath, opType, actions)); + } + + @SafeVarargs + public static Throwable perform(Throwable accumulate, String filePath, FileOpType opType, DiscreteAction ... actions) + { + return perform(accumulate, filePath, opType, Arrays.stream(actions)); + } + + public static Throwable perform(Throwable accumulate, String filePath, FileOpType opType, Stream> actions) + { + return perform(accumulate, actions.map((action) -> () -> { try { @@ -112,7 +137,7 @@ public final class Throwables } catch (IOException e) { - throw (opType == FileOpType.WRITE) ? new FSWriteError(e, against) : new FSReadError(e, against); + throw (opType == FileOpType.WRITE) ? new FSWriteError(e, filePath) : new FSReadError(e, filePath); } })); } diff --git a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java index 0ac4124cca..daf5006b94 100644 --- a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java +++ b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java @@ -67,7 +67,7 @@ public class VIntCoding return firstByte; int size = numberOfExtraBytesToRead(firstByte); - long retval = firstByte & firstByteValueMask(size);; + long retval = firstByte & firstByteValueMask(size); for (int ii = 0; ii < size; ii++) { byte b = input.readByte(); diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java index fcdab62d99..555cddaddb 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java @@ -60,8 +60,7 @@ import org.apache.cassandra.db.commitlog.CommitLogSegment; import org.apache.cassandra.db.commitlog.CommitLogReplayer.CommitLogReplayException; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.util.ByteBufferDataInput; -import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.ByteBufferUtil; @@ -535,13 +534,12 @@ public class CommitLogTest { ByteBuffer buf = ByteBuffer.allocate(1024); CommitLogDescriptor.writeHeader(buf, desc); - long length = buf.position(); // Put some extra data in the stream. buf.putDouble(0.1); buf.flip(); - FileDataInput input = new ByteBufferDataInput(buf, "input", 0, 0); + + DataInputBuffer input = new DataInputBuffer(buf, false); CommitLogDescriptor read = CommitLogDescriptor.readHeader(input); - Assert.assertEquals("Descriptor length", length, input.getFilePointer()); Assert.assertEquals("Descriptors", desc, read); } diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java index 1ab9ca7295..fea83c1ce5 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java @@ -29,6 +29,7 @@ import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.rows.SerializationHelper; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.NIODataInputStream; +import org.apache.cassandra.io.util.RebufferingInputStream; /** * Utility class for tests needing to examine the commitlog contents. @@ -60,7 +61,7 @@ public class CommitLogTestReplayer extends CommitLogReplayer @Override void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc) { - NIODataInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size); + RebufferingInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size); Mutation mutation; try { diff --git a/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java b/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java index e431924f32..323a12d603 100644 --- a/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java +++ b/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java @@ -17,96 +17,217 @@ */ package org.apache.cassandra.hints; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.zip.CRC32; import org.junit.Test; -import org.apache.cassandra.hints.ChecksummedDataInput; -import org.apache.cassandra.io.util.AbstractDataInput; import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertFalse; public class ChecksummedDataInputTest { @Test - public void testThatItWorks() throws IOException + public void testReadMethods() throws IOException { - // fill a bytebuffer with some input - DataOutputBuffer out = new DataOutputBuffer(); - out.write(127); - out.write(new byte[]{ 0, 1, 2, 3, 4, 5, 6 }); - out.writeBoolean(false); - out.writeByte(10); - out.writeChar('t'); - out.writeDouble(3.3); - out.writeFloat(2.2f); - out.writeInt(42); - out.writeLong(Long.MAX_VALUE); - out.writeShort(Short.MIN_VALUE); - out.writeUTF("utf"); - ByteBuffer buffer = out.buffer(); + // Make sure this array is bigger than the reader buffer size + // so we test updating the crc across buffer boundaries + byte[] b = new byte[RandomAccessReader.DEFAULT_BUFFER_SIZE * 2]; + for (int i = 0; i < b.length; i++) + b[i] = (byte)i; - // calculate resulting CRC + ByteBuffer buffer; + + // fill a bytebuffer with some input + try (DataOutputBuffer out = new DataOutputBuffer()) + { + out.write(127); + out.write(b); + out.writeBoolean(false); + out.writeByte(10); + out.writeChar('t'); + out.writeDouble(3.3); + out.writeFloat(2.2f); + out.writeInt(42); + out.writeLong(Long.MAX_VALUE); + out.writeShort(Short.MIN_VALUE); + out.writeUTF("utf"); + out.writeVInt(67L); + out.writeUnsignedVInt(88L); + out.writeBytes("abcdefghi"); + + buffer = out.buffer(); + } + + // calculate expected CRC CRC32 crc = new CRC32(); FBUtilities.updateChecksum(crc, buffer); - int expectedCRC = (int) crc.getValue(); - ChecksummedDataInput crcInput = ChecksummedDataInput.wrap(new DummyByteBufferDataInput(buffer.duplicate())); - crcInput.limit(buffer.remaining()); + // save the buffer to file to create a RAR + File file = File.createTempFile("testReadMethods", "1"); + file.deleteOnExit(); + try (SequentialWriter writer = SequentialWriter.open(file)) + { + writer.write(buffer); + writer.writeInt((int) crc.getValue()); + writer.finish(); + } - // assert that we read all the right values back - assertEquals(127, crcInput.read()); - byte[] bytes = new byte[7]; - crcInput.readFully(bytes); - assertTrue(Arrays.equals(new byte[]{ 0, 1, 2, 3, 4, 5, 6 }, bytes)); - assertEquals(false, crcInput.readBoolean()); - assertEquals(10, crcInput.readByte()); - assertEquals('t', crcInput.readChar()); - assertEquals(3.3, crcInput.readDouble()); - assertEquals(2.2f, crcInput.readFloat()); - assertEquals(42, crcInput.readInt()); - assertEquals(Long.MAX_VALUE, crcInput.readLong()); - assertEquals(Short.MIN_VALUE, crcInput.readShort()); - assertEquals("utf", crcInput.readUTF()); + assertTrue(file.exists()); + assertEquals(buffer.remaining() + 4, file.length()); - // assert that the crc matches, and that we've read exactly as many bytes as expected - assertEquals(0, crcInput.bytesRemaining()); - assertEquals(expectedCRC, crcInput.getCrc()); + try (ChecksummedDataInput reader = ChecksummedDataInput.open(file)) + { + reader.limit(buffer.remaining() + 4); + + // assert that we read all the right values back + assertEquals(127, reader.read()); + byte[] bytes = new byte[b.length]; + reader.readFully(bytes); + assertTrue(Arrays.equals(bytes, b)); + assertEquals(false, reader.readBoolean()); + assertEquals(10, reader.readByte()); + assertEquals('t', reader.readChar()); + assertEquals(3.3, reader.readDouble()); + assertEquals(2.2f, reader.readFloat()); + assertEquals(42, reader.readInt()); + assertEquals(Long.MAX_VALUE, reader.readLong()); + assertEquals(Short.MIN_VALUE, reader.readShort()); + assertEquals("utf", reader.readUTF()); + assertEquals(67L, reader.readVInt()); + assertEquals(88L, reader.readUnsignedVInt()); + assertEquals("abcdefghi", new String(ByteBufferUtil.read(reader, 9).array(), StandardCharsets.UTF_8)); + + // assert that the crc matches, and that we've read exactly as many bytes as expected + assertTrue(reader.checkCrc()); + assertEquals(0, reader.bytesRemaining()); + + reader.checkLimit(0); + } } - private static final class DummyByteBufferDataInput extends AbstractDataInput + @Test + public void testResetCrc() throws IOException { - private final ByteBuffer buffer; + CRC32 crc = new CRC32(); + ByteBuffer buffer; - DummyByteBufferDataInput(ByteBuffer buffer) + // fill a bytebuffer with some input + try (DataOutputBuffer out = new DataOutputBuffer()) { - this.buffer = buffer; + out.write(127); + out.writeBoolean(false); + out.writeByte(10); + out.writeChar('t'); + + buffer = out.buffer(); + FBUtilities.updateChecksum(crc, buffer); + out.writeInt((int) crc.getValue()); + + int bufferPos = out.getLength(); + out.writeDouble(3.3); + out.writeFloat(2.2f); + out.writeInt(42); + + buffer = out.buffer(); + buffer.position(bufferPos); + crc.reset(); + FBUtilities.updateChecksum(crc, buffer); + + out.writeInt((int) crc.getValue()); + buffer = out.buffer(); } - public void seek(long position) + // save the buffer to file to create a RAR + File file = File.createTempFile("testResetCrc", "1"); + file.deleteOnExit(); + try (SequentialWriter writer = SequentialWriter.open(file)) { - throw new UnsupportedOperationException(); + writer.write(buffer); + writer.finish(); } - public long getPosition() + assertTrue(file.exists()); + assertEquals(buffer.remaining(), file.length()); + + try (ChecksummedDataInput reader = ChecksummedDataInput.open(file)) { - throw new UnsupportedOperationException(); + reader.limit(buffer.remaining()); + + // assert that we read all the right values back + assertEquals(127, reader.read()); + assertEquals(false, reader.readBoolean()); + assertEquals(10, reader.readByte()); + assertEquals('t', reader.readChar()); + assertTrue(reader.checkCrc()); + + reader.resetCrc(); + assertEquals(3.3, reader.readDouble()); + assertEquals(2.2f, reader.readFloat()); + assertEquals(42, reader.readInt()); + assertTrue(reader.checkCrc()); + assertEquals(0, reader.bytesRemaining()); + } + } + + @Test + public void testFailedCrc() throws IOException + { + CRC32 crc = new CRC32(); + ByteBuffer buffer; + + // fill a bytebuffer with some input + try (DataOutputBuffer out = new DataOutputBuffer()) + { + out.write(127); + out.writeBoolean(false); + out.writeByte(10); + out.writeChar('t'); + + buffer = out.buffer(); + FBUtilities.updateChecksum(crc, buffer); + + // update twice so it won't match + FBUtilities.updateChecksum(crc, buffer); + out.writeInt((int) crc.getValue()); + + buffer = out.buffer(); } - public long getPositionLimit() + // save the buffer to file to create a RAR + File file = File.createTempFile("testFailedCrc", "1"); + file.deleteOnExit(); + try (SequentialWriter writer = SequentialWriter.open(file)) { - throw new UnsupportedOperationException(); + writer.write(buffer); + writer.finish(); } - public int read() + assertTrue(file.exists()); + assertEquals(buffer.remaining(), file.length()); + + try (ChecksummedDataInput reader = ChecksummedDataInput.open(file)) { - return buffer.get() & 0xFF; + reader.limit(buffer.remaining()); + + // assert that we read all the right values back + assertEquals(127, reader.read()); + assertEquals(false, reader.readBoolean()); + assertEquals(10, reader.readByte()); + assertEquals('t', reader.readChar()); + assertFalse(reader.checkCrc()); + assertEquals(0, reader.bytesRemaining()); } } } diff --git a/test/unit/org/apache/cassandra/io/RandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/RandomAccessReaderTest.java deleted file mode 100644 index edbd603609..0000000000 --- a/test/unit/org/apache/cassandra/io/RandomAccessReaderTest.java +++ /dev/null @@ -1,269 +0,0 @@ -package org.apache.cassandra.io; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import org.junit.Test; - -import static org.junit.Assert.*; -import org.apache.cassandra.io.util.ChannelProxy; -import org.apache.cassandra.io.util.FileMark; -import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.io.util.SequentialWriter; - -public class RandomAccessReaderTest -{ - @Test - public void testReadFully() throws IOException - { - testReadImpl(1, 0); - } - - @Test - public void testReadLarge() throws IOException - { - testReadImpl(1000, 0); - } - - @Test - public void testReadLargeWithSkip() throws IOException - { - testReadImpl(1000, 322); - } - - @Test - public void testReadBufferSizeNotAligned() throws IOException - { - testReadImpl(1000, 0, 5122); - } - - private void testReadImpl(int numIterations, int skipIterations) throws IOException - { - testReadImpl(numIterations, skipIterations, RandomAccessReader.DEFAULT_BUFFER_SIZE); - } - - private void testReadImpl(int numIterations, int skipIterations, int bufferSize) throws IOException - { - final File f = File.createTempFile("testReadFully", "1"); - final String expected = "The quick brown fox jumps over the lazy dog"; - - SequentialWriter writer = SequentialWriter.open(f); - for (int i = 0; i < numIterations; i++) - writer.write(expected.getBytes()); - writer.finish(); - - assert f.exists(); - - ChannelProxy channel = new ChannelProxy(f); - RandomAccessReader reader = RandomAccessReader.open(channel, bufferSize, -1L); - assertEquals(f.getAbsolutePath(), reader.getPath()); - assertEquals(expected.length() * numIterations, reader.length()); - - if (skipIterations > 0) - { - reader.seek(skipIterations * expected.length()); - } - - byte[] b = new byte[expected.length()]; - int n = numIterations - skipIterations; - for (int i = 0; i < n; i++) - { - reader.readFully(b); - assertEquals(expected, new String(b)); - } - - assertTrue(reader.isEOF()); - assertEquals(0, reader.bytesRemaining()); - - reader.close(); - channel.close(); - } - - @Test - public void testReadBytes() throws IOException - { - File f = File.createTempFile("testReadBytes", "1"); - final String expected = "The quick brown fox jumps over the lazy dog"; - - SequentialWriter writer = SequentialWriter.open(f); - writer.write(expected.getBytes()); - writer.finish(); - - assert f.exists(); - - ChannelProxy channel = new ChannelProxy(f); - RandomAccessReader reader = RandomAccessReader.open(channel); - assertEquals(f.getAbsolutePath(), reader.getPath()); - assertEquals(expected.length(), reader.length()); - - ByteBuffer b = reader.readBytes(expected.length()); - assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); - - assertTrue(reader.isEOF()); - assertEquals(0, reader.bytesRemaining()); - - reader.close(); - channel.close(); - } - - @Test - public void testReset() throws IOException - { - File f = File.createTempFile("testMark", "1"); - final String expected = "The quick brown fox jumps over the lazy dog"; - final int numIterations = 10; - - SequentialWriter writer = SequentialWriter.open(f); - for (int i = 0; i < numIterations; i++) - writer.write(expected.getBytes()); - writer.finish(); - - assert f.exists(); - - ChannelProxy channel = new ChannelProxy(f); - RandomAccessReader reader = RandomAccessReader.open(channel); - assertEquals(expected.length() * numIterations, reader.length()); - - ByteBuffer b = reader.readBytes(expected.length()); - assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); - - assertFalse(reader.isEOF()); - assertEquals((numIterations - 1) * expected.length(), reader.bytesRemaining()); - - FileMark mark = reader.mark(); - assertEquals(0, reader.bytesPastMark()); - assertEquals(0, reader.bytesPastMark(mark)); - - for (int i = 0; i < (numIterations - 1); i++) - { - b = reader.readBytes(expected.length()); - assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); - } - assertTrue(reader.isEOF()); - assertEquals(expected.length() * (numIterations -1), reader.bytesPastMark()); - assertEquals(expected.length() * (numIterations - 1), reader.bytesPastMark(mark)); - - reader.reset(mark); - assertEquals(0, reader.bytesPastMark()); - assertEquals(0, reader.bytesPastMark(mark)); - assertFalse(reader.isEOF()); - for (int i = 0; i < (numIterations - 1); i++) - { - b = reader.readBytes(expected.length()); - assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); - } - - reader.reset(); - assertEquals(0, reader.bytesPastMark()); - assertEquals(0, reader.bytesPastMark(mark)); - assertFalse(reader.isEOF()); - for (int i = 0; i < (numIterations - 1); i++) - { - b = reader.readBytes(expected.length()); - assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); - } - - assertTrue(reader.isEOF()); - reader.close(); - channel.close(); - } - - @Test - public void testSeekSingleThread() throws IOException, InterruptedException - { - testSeek(1); - } - - @Test - public void testSeekMultipleThreads() throws IOException, InterruptedException - { - testSeek(10); - } - - private void testSeek(int numThreads) throws IOException, InterruptedException - { - final File f = File.createTempFile("testMark", "1"); - final String[] expected = new String[10]; - int len = 0; - for (int i = 0; i < expected.length; i++) - { - expected[i] = UUID.randomUUID().toString(); - len += expected[i].length(); - } - final int totalLength = len; - - SequentialWriter writer = SequentialWriter.open(f); - for (int i = 0; i < expected.length; i++) - writer.write(expected[i].getBytes()); - writer.finish(); - - assert f.exists(); - - final ChannelProxy channel = new ChannelProxy(f); - - final Runnable worker = new Runnable() { - - @Override - public void run() - { - try - { - RandomAccessReader reader = RandomAccessReader.open(channel); - assertEquals(totalLength, reader.length()); - - ByteBuffer b = reader.readBytes(expected[0].length()); - assertEquals(expected[0], new String(b.array(), Charset.forName("UTF-8"))); - - assertFalse(reader.isEOF()); - assertEquals(totalLength - expected[0].length(), reader.bytesRemaining()); - - long filePointer = reader.getFilePointer(); - - for (int i = 1; i < expected.length; i++) - { - b = reader.readBytes(expected[i].length()); - assertEquals(expected[i], new String(b.array(), Charset.forName("UTF-8"))); - } - assertTrue(reader.isEOF()); - - reader.seek(filePointer); - assertFalse(reader.isEOF()); - for (int i = 1; i < expected.length; i++) - { - b = reader.readBytes(expected[i].length()); - assertEquals(expected[i], new String(b.array(), Charset.forName("UTF-8"))); - } - - assertTrue(reader.isEOF()); - reader.close(); - } - catch (Exception ex) - { - ex.printStackTrace(); - fail(ex.getMessage()); - } - } - }; - - if(numThreads == 1) - { - worker.run(); - return; - } - - ExecutorService executor = Executors.newFixedThreadPool(numThreads); - for (int i = 0; i < numThreads; i++) - executor.submit(worker); - - executor.shutdown(); - executor.awaitTermination(1, TimeUnit.MINUTES); - - channel.close(); - } -} diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java index 8f94cf2841..9154d79cc4 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java @@ -31,6 +31,7 @@ import org.apache.cassandra.io.sstable.CorruptSSTableException; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.util.ChannelProxy; import org.apache.cassandra.io.util.FileMark; +import org.apache.cassandra.io.util.MmappedRegions; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.io.util.SequentialWriter; import org.apache.cassandra.schema.CompressionParams; @@ -39,6 +40,8 @@ import org.apache.cassandra.utils.SyncUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; public class CompressedRandomAccessReaderTest { @@ -46,16 +49,24 @@ public class CompressedRandomAccessReaderTest public void testResetAndTruncate() throws IOException { // test reset in current buffer or previous one - testResetAndTruncate(File.createTempFile("normal", "1"), false, 10); - testResetAndTruncate(File.createTempFile("normal", "2"), false, CompressionParams.DEFAULT_CHUNK_LENGTH); + testResetAndTruncate(File.createTempFile("normal", "1"), false, false, 10); + testResetAndTruncate(File.createTempFile("normal", "2"), false, false, CompressionParams.DEFAULT_CHUNK_LENGTH); } @Test public void testResetAndTruncateCompressed() throws IOException { // test reset in current buffer or previous one - testResetAndTruncate(File.createTempFile("compressed", "1"), true, 10); - testResetAndTruncate(File.createTempFile("compressed", "2"), true, CompressionParams.DEFAULT_CHUNK_LENGTH); + testResetAndTruncate(File.createTempFile("compressed", "1"), true, false, 10); + testResetAndTruncate(File.createTempFile("compressed", "2"), true, false, CompressionParams.DEFAULT_CHUNK_LENGTH); + } + + @Test + public void testResetAndTruncateCompressedMmap() throws IOException + { + // test reset in current buffer or previous one + testResetAndTruncate(File.createTempFile("compressed_mmap", "1"), true, true, 10); + testResetAndTruncate(File.createTempFile("compressed_mmap", "2"), true, true, CompressionParams.DEFAULT_CHUNK_LENGTH); } @Test @@ -63,87 +74,102 @@ public class CompressedRandomAccessReaderTest { File f = File.createTempFile("compressed6791_", "3"); String filename = f.getAbsolutePath(); - ChannelProxy channel = new ChannelProxy(f); - try + try(ChannelProxy channel = new ChannelProxy(f)) { MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); - CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(32), sstableMetadataCollector); + try(CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(32), sstableMetadataCollector)) + { - for (int i = 0; i < 20; i++) - writer.write("x".getBytes()); + for (int i = 0; i < 20; i++) + writer.write("x".getBytes()); - FileMark mark = writer.mark(); - // write enough garbage to create new chunks: - for (int i = 0; i < 40; ++i) - writer.write("y".getBytes()); + FileMark mark = writer.mark(); + // write enough garbage to create new chunks: + for (int i = 0; i < 40; ++i) + writer.write("y".getBytes()); - writer.resetAndTruncate(mark); + writer.resetAndTruncate(mark); - for (int i = 0; i < 20; i++) - writer.write("x".getBytes()); - writer.finish(); + for (int i = 0; i < 20; i++) + writer.write("x".getBytes()); + writer.finish(); + } - CompressedRandomAccessReader reader = CompressedRandomAccessReader.open(channel, new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32)); - String res = reader.readLine(); - assertEquals(res, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); - assertEquals(40, res.length()); + try(RandomAccessReader reader = new CompressedRandomAccessReader.Builder(channel, + new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32)) + .build()) + { + String res = reader.readLine(); + assertEquals(res, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + assertEquals(40, res.length()); + } } finally { - // cleanup - channel.close(); - if (f.exists()) - f.delete(); + assertTrue(f.delete()); File metadata = new File(filename+ ".metadata"); if (metadata.exists()) metadata.delete(); } } - private void testResetAndTruncate(File f, boolean compressed, int junkSize) throws IOException + private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize) throws IOException { final String filename = f.getAbsolutePath(); - ChannelProxy channel = new ChannelProxy(f); - try + try(ChannelProxy channel = new ChannelProxy(f)) { MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null); - SequentialWriter writer = compressed + try(SequentialWriter writer = compressed ? new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(), sstableMetadataCollector) - : SequentialWriter.open(f); - - writer.write("The quick ".getBytes()); - FileMark mark = writer.mark(); - writer.write("blue fox jumps over the lazy dog".getBytes()); - - // write enough to be sure to change chunk - for (int i = 0; i < junkSize; ++i) + : SequentialWriter.open(f)) { - writer.write((byte)1); + writer.write("The quick ".getBytes()); + FileMark mark = writer.mark(); + writer.write("blue fox jumps over the lazy dog".getBytes()); + + // write enough to be sure to change chunk + for (int i = 0; i < junkSize; ++i) + { + writer.write((byte) 1); + } + + writer.resetAndTruncate(mark); + writer.write("brown fox jumps over the lazy dog".getBytes()); + writer.finish(); + } + assert f.exists(); + + CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) : null; + RandomAccessReader.Builder builder = compressed + ? new CompressedRandomAccessReader.Builder(channel, compressionMetadata) + : new RandomAccessReader.Builder(channel); + + if (usemmap) + { + if (compressed) + builder.regions(MmappedRegions.map(channel, compressionMetadata)); + else + builder.regions(MmappedRegions.map(channel, f.length())); } - writer.resetAndTruncate(mark); - writer.write("brown fox jumps over the lazy dog".getBytes()); - writer.finish(); + try(RandomAccessReader reader = builder.build()) + { + String expected = "The quick brown fox jumps over the lazy dog"; + assertEquals(expected.length(), reader.length()); + byte[] b = new byte[expected.length()]; + reader.readFully(b); + assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + '\''; + } - assert f.exists(); - RandomAccessReader reader = compressed - ? CompressedRandomAccessReader.open(channel, new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32)) - : RandomAccessReader.open(f); - String expected = "The quick brown fox jumps over the lazy dog"; - assertEquals(expected.length(), reader.length()); - byte[] b = new byte[expected.length()]; - reader.readFully(b); - assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + "'"; + if (usemmap) + builder.regions.close(); } finally { - // cleanup - channel.close(); - if (f.exists()) - f.delete(); + assertTrue(f.delete()); File metadata = new File(filename + ".metadata"); if (compressed && metadata.exists()) metadata.delete(); @@ -161,6 +187,9 @@ public class CompressedRandomAccessReaderTest File metadata = new File(file.getPath() + ".meta"); metadata.deleteOnExit(); + assertTrue(file.createNewFile()); + assertTrue(metadata.createNewFile()); + MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null); try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), CompressionParams.snappy(), sstableMetadataCollector)) { @@ -168,74 +197,74 @@ public class CompressedRandomAccessReaderTest writer.finish(); } - ChannelProxy channel = new ChannelProxy(file); - - // open compression metadata and get chunk information - CompressionMetadata meta = new CompressionMetadata(metadata.getPath(), file.length(), ChecksumType.CRC32); - CompressionMetadata.Chunk chunk = meta.chunkFor(0); - - RandomAccessReader reader = CompressedRandomAccessReader.open(channel, meta); - // read and verify compressed data - assertEquals(CONTENT, reader.readLine()); - - Random random = new Random(); - RandomAccessFile checksumModifier = null; - - try + try(ChannelProxy channel = new ChannelProxy(file)) { - checksumModifier = new RandomAccessFile(file, "rw"); - byte[] checksum = new byte[4]; + // open compression metadata and get chunk information + CompressionMetadata meta = new CompressionMetadata(metadata.getPath(), file.length(), ChecksumType.CRC32); + CompressionMetadata.Chunk chunk = meta.chunkFor(0); - // seek to the end of the compressed chunk - checksumModifier.seek(chunk.length); - // read checksum bytes - checksumModifier.read(checksum); - // seek back to the chunk end - checksumModifier.seek(chunk.length); + try(RandomAccessReader reader = new CompressedRandomAccessReader.Builder(channel, meta).build()) + {// read and verify compressed data + assertEquals(CONTENT, reader.readLine()); - // lets modify one byte of the checksum on each iteration - for (int i = 0; i < checksum.length; i++) - { - checksumModifier.write(random.nextInt()); - SyncUtil.sync(checksumModifier); // making sure that change was synced with disk + Random random = new Random(); + RandomAccessFile checksumModifier = null; - final RandomAccessReader r = CompressedRandomAccessReader.open(channel, meta); - - Throwable exception = null; try { - r.readLine(); + checksumModifier = new RandomAccessFile(file, "rw"); + byte[] checksum = new byte[4]; + + // seek to the end of the compressed chunk + checksumModifier.seek(chunk.length); + // read checksum bytes + checksumModifier.read(checksum); + // seek back to the chunk end + checksumModifier.seek(chunk.length); + + // lets modify one byte of the checksum on each iteration + for (int i = 0; i < checksum.length; i++) + { + checksumModifier.write(random.nextInt()); + SyncUtil.sync(checksumModifier); // making sure that change was synced with disk + + try (final RandomAccessReader r = new CompressedRandomAccessReader.Builder(channel, meta).build()) + { + Throwable exception = null; + try + { + r.readLine(); + } + catch (Throwable t) + { + exception = t; + } + assertNotNull(exception); + assertSame(exception.getClass(), CorruptSSTableException.class); + assertSame(exception.getCause().getClass(), CorruptBlockException.class); + } + } + + // lets write original checksum and check if we can read data + updateChecksum(checksumModifier, chunk.length, checksum); + + try (RandomAccessReader cr = new CompressedRandomAccessReader.Builder(channel, meta).build()) + { + // read and verify compressed data + assertEquals(CONTENT, cr.readLine()); + // close reader + } } - catch (Throwable t) + finally { - exception = t; + if (checksumModifier != null) + checksumModifier.close(); } - assertNotNull(exception); - assertEquals(exception.getClass(), CorruptSSTableException.class); - assertEquals(exception.getCause().getClass(), CorruptBlockException.class); - - r.close(); } - - // lets write original checksum and check if we can read data - updateChecksum(checksumModifier, chunk.length, checksum); - - reader = CompressedRandomAccessReader.open(channel, meta); - // read and verify compressed data - assertEquals(CONTENT, reader.readLine()); - // close reader - reader.close(); - } - finally - { - channel.close(); - - if (checksumModifier != null) - checksumModifier.close(); } } - private void updateChecksum(RandomAccessFile file, long checksumOffset, byte[] checksum) throws IOException + private static void updateChecksum(RandomAccessFile file, long checksumOffset, byte[] checksum) throws IOException { file.seek(checksumOffset); file.write(checksum); diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java index ca157222f6..1bdc591061 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java @@ -117,7 +117,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest } assert f.exists(); - RandomAccessReader reader = CompressedRandomAccessReader.open(channel, new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32)); + RandomAccessReader reader = new CompressedRandomAccessReader.Builder(channel, new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32)).build(); assertEquals(dataPre.length + rawPost.length, reader.length()); byte[] result = new byte[(int)reader.length()]; diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java index 793348adc9..c7f3c361b4 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java @@ -46,6 +46,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.Index; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.MmappedRegions; import org.apache.cassandra.io.util.MmappedSegmentedFile; import org.apache.cassandra.io.util.SegmentedFile; import org.apache.cassandra.schema.CachingParams; @@ -133,7 +134,7 @@ public class SSTableReaderTest @Test public void testSpannedIndexPositions() throws IOException { - MmappedSegmentedFile.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of segments + MmappedRegions.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of segments Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); diff --git a/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java b/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java index b875a6a2c8..4a72281c19 100644 --- a/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java +++ b/test/unit/org/apache/cassandra/io/util/BufferedDataOutputStreamTest.java @@ -542,14 +542,12 @@ public class BufferedDataOutputStreamTest ndosp.flush(); - @SuppressWarnings("resource") - ByteBufferDataInput bbdi = new ByteBufferDataInput(ByteBuffer.wrap(generated.toByteArray()), "", 0, 0); - + DataInputBuffer in = new DataInputBuffer(generated.toByteArray()); assertEquals(expectedSize, generated.toByteArray().length); for (long v : testValues) { - assertEquals(v, bbdi.readVInt()); + assertEquals(v, in.readVInt()); } } @@ -574,13 +572,11 @@ public class BufferedDataOutputStreamTest ndosp.flush(); - @SuppressWarnings("resource") - ByteBufferDataInput bbdi = new ByteBufferDataInput(ByteBuffer.wrap(generated.toByteArray()), "", 0, 0); - + DataInputBuffer in = new DataInputBuffer(generated.toByteArray()); assertEquals(expectedSize, generated.toByteArray().length); for (long v : testValues) - assertEquals(v, bbdi.readUnsignedVInt()); + assertEquals(v, in.readUnsignedVInt()); } @Test diff --git a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java index e051c000c8..364ea7129a 100644 --- a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java +++ b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java @@ -25,9 +25,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.ClosedChannelException; import java.util.Arrays; -import java.util.concurrent.Callable; import static org.apache.cassandra.Util.expectEOF; import static org.apache.cassandra.Util.expectException; @@ -96,7 +94,7 @@ public class BufferedRandomAccessFileTest // test readBytes(int) method r.seek(0); - ByteBuffer fileContent = r.readBytes((int) w.length()); + ByteBuffer fileContent = ByteBufferUtil.read(r, (int) w.length()); assertEquals(fileContent.limit(), w.length()); assert ByteBufferUtil.string(fileContent).equals("Hello" + new String(bigData)); @@ -204,25 +202,19 @@ public class BufferedRandomAccessFileTest final ChannelProxy channel = new ChannelProxy(w.getPath()); final RandomAccessReader r = RandomAccessReader.open(channel); - ByteBuffer content = r.readBytes((int) r.length()); + ByteBuffer content = ByteBufferUtil.read(r, (int) r.length()); // after reading whole file we should be at EOF assertEquals(0, ByteBufferUtil.compare(content, data)); assert r.bytesRemaining() == 0 && r.isEOF(); r.seek(0); - content = r.readBytes(10); // reading first 10 bytes + content = ByteBufferUtil.read(r, 10); // reading first 10 bytes assertEquals(ByteBufferUtil.compare(content, "cccccccccc".getBytes()), 0); assertEquals(r.bytesRemaining(), r.length() - content.limit()); // trying to read more than file has right now - expectEOF(new Callable() - { - public Object call() throws IOException - { - return r.readBytes((int) r.length() + 10); - } - }); + expectEOF(() -> ByteBufferUtil.read(r, (int) r.length() + 10)); w.finish(); r.close(); @@ -249,23 +241,9 @@ public class BufferedRandomAccessFileTest assertEquals(file.bytesRemaining(), file.length() - 20); // trying to seek past the end of the file should produce EOFException - expectException(new Callable() - { - public Object call() - { - file.seek(file.length() + 30); - return null; - } - }, IllegalArgumentException.class); + expectException(() -> { file.seek(file.length() + 30); return null; }, IllegalArgumentException.class); - expectException(new Callable() - { - public Object call() throws IOException - { - file.seek(-1); - return null; - } - }, IllegalArgumentException.class); // throws IllegalArgumentException + expectException(() -> { file.seek(-1); return null; }, IllegalArgumentException.class); // throws IllegalArgumentException file.close(); channel.close(); @@ -352,16 +330,11 @@ public class BufferedRandomAccessFileTest { File file1 = writeTemporaryFile(new byte[16]); try (final ChannelProxy channel = new ChannelProxy(file1); - final RandomAccessReader file = RandomAccessReader.open(channel, bufferSize, -1L)) + final RandomAccessReader file = new RandomAccessReader.Builder(channel) + .bufferSize(bufferSize) + .build()) { - expectEOF(new Callable() - { - public Object call() throws IOException - { - file.readFully(target, offset, 17); - return null; - } - }); + expectEOF(() -> { file.readFully(target, offset, 17); return null; }); } } @@ -370,15 +343,11 @@ public class BufferedRandomAccessFileTest { File file1 = writeTemporaryFile(new byte[16]); try (final ChannelProxy channel = new ChannelProxy(file1); - final RandomAccessReader file = RandomAccessReader.open(channel, bufferSize, -1L)) + final RandomAccessReader file = new RandomAccessReader.Builder(channel).bufferSize(bufferSize).build()) { - expectEOF(new Callable() - { - public Object call() throws IOException - { - while (true) - file.readFully(target, 0, n); - } + expectEOF(() -> { + while (true) + file.readFully(target, 0, n); }); } } @@ -459,30 +428,17 @@ public class BufferedRandomAccessFileTest r.close(); // closing to test read after close - expectException(new Callable() - { - public Object call() - { - return r.read(); - } - }, AssertionError.class); + expectException(() -> r.read(), NullPointerException.class); //Used to throw ClosedChannelException, but now that it extends BDOSP it just NPEs on the buffer //Writing to a BufferedOutputStream that is closed generates no error //Going to allow the NPE to throw to catch as a bug any use after close. Notably it won't throw NPE for a //write of a 0 length, but that is kind of a corner case - expectException(new Callable() - { - public Object call() throws IOException - { - w.write(generateByteArray(1)); - return null; - } - }, NullPointerException.class); + expectException(() -> { w.write(generateByteArray(1)); return null; }, NullPointerException.class); try (RandomAccessReader copy = RandomAccessReader.open(new File(r.getPath()))) { - ByteBuffer contents = copy.readBytes((int) copy.length()); + ByteBuffer contents = ByteBufferUtil.read(copy, (int) copy.length()); assertEquals(contents.limit(), data.length); assertEquals(ByteBufferUtil.compare(contents, data), 0); @@ -526,7 +482,7 @@ public class BufferedRandomAccessFileTest channel.close(); } - @Test (expected = AssertionError.class) + @Test(expected = AssertionError.class) public void testAssertionErrorWhenBytesPastMarkIsNegative() throws IOException { try (SequentialWriter w = createTempFile("brafAssertionErrorWhenBytesPastMarkIsNegative")) @@ -565,14 +521,7 @@ public class BufferedRandomAccessFileTest assertTrue(copy.bytesRemaining() == 0 && copy.isEOF()); // can't seek past the end of the file for read-only files - expectException(new Callable() - { - public Object call() - { - copy.seek(copy.length() + 1); - return null; - } - }, IllegalArgumentException.class); + expectException(() -> { copy.seek(copy.length() + 1); return null; }, IllegalArgumentException.class); copy.seek(0); copy.skipBytes(5); @@ -582,7 +531,7 @@ public class BufferedRandomAccessFileTest assertTrue(!copy.isEOF()); copy.seek(0); - ByteBuffer contents = copy.readBytes((int) copy.length()); + ByteBuffer contents = ByteBufferUtil.read(copy, (int) copy.length()); assertEquals(contents.limit(), copy.length()); assertTrue(ByteBufferUtil.compare(contents, data) == 0); diff --git a/test/unit/org/apache/cassandra/io/ChecksummedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java similarity index 92% rename from test/unit/org/apache/cassandra/io/ChecksummedRandomAccessReaderTest.java rename to test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java index c1e43c94e2..57428afeaf 100644 --- a/test/unit/org/apache/cassandra/io/ChecksummedRandomAccessReaderTest.java +++ b/test/unit/org/apache/cassandra/io/util/ChecksummedRandomAccessReaderTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.io; +package org.apache.cassandra.io.util; import java.io.File; import java.io.IOException; @@ -49,7 +49,7 @@ public class ChecksummedRandomAccessReaderTest assert data.exists(); - RandomAccessReader reader = ChecksummedRandomAccessReader.open(data, crc); + RandomAccessReader reader = new ChecksummedRandomAccessReader.Builder(data, crc).build(); byte[] b = new byte[expected.length]; reader.readFully(b); @@ -75,7 +75,7 @@ public class ChecksummedRandomAccessReaderTest assert data.exists(); - RandomAccessReader reader = ChecksummedRandomAccessReader.open(data, crc); + RandomAccessReader reader = new ChecksummedRandomAccessReader.Builder(data, crc).build(); final int seekPosition = 66000; reader.seek(seekPosition); @@ -114,7 +114,7 @@ public class ChecksummedRandomAccessReaderTest dataFile.write((byte) 5); } - RandomAccessReader reader = ChecksummedRandomAccessReader.open(data, crc); + RandomAccessReader reader = new ChecksummedRandomAccessReader.Builder(data, crc).build(); byte[] b = new byte[expected.length]; reader.readFully(b); diff --git a/test/unit/org/apache/cassandra/io/util/FileSegmentInputStreamTest.java b/test/unit/org/apache/cassandra/io/util/FileSegmentInputStreamTest.java new file mode 100644 index 0000000000..fcee9b7659 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/util/FileSegmentInputStreamTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.util; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Random; + +import com.google.common.primitives.Ints; +import org.junit.Test; + +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class FileSegmentInputStreamTest +{ + private ByteBuffer allocateBuffer(int size) + { + ByteBuffer ret = ByteBuffer.allocate(Ints.checkedCast(size)); + long seed = System.nanoTime(); + //seed = 365238103404423L; + System.out.println("Seed " + seed); + + new Random(seed).nextBytes(ret.array()); + return ret; + } + + @Test + public void testRead() throws IOException + { + testRead(0, 4096, 1024); + testRead(1024, 4096, 1024); + testRead(4096, 4096, 1024); + } + + private void testRead(int offset, int size, int checkInterval) throws IOException + { + final ByteBuffer buffer = allocateBuffer(size); + final String path = buffer.toString(); + + FileSegmentInputStream reader = new FileSegmentInputStream(buffer.duplicate(), path, offset); + assertEquals(path, reader.getPath()); + + for (int i = offset; i < (size + offset); i += checkInterval) + { + reader.seek(i); + assertFalse(reader.isEOF()); + assertEquals(i, reader.getFilePointer()); + + buffer.position(i - offset); + + int remaining = buffer.remaining(); + assertEquals(remaining, reader.bytesRemaining()); + byte[] expected = new byte[buffer.remaining()]; + buffer.get(expected); + assertTrue(Arrays.equals(expected, ByteBufferUtil.read(reader, remaining).array())); + + assertTrue(reader.isEOF()); + assertEquals(0, reader.bytesRemaining()); + assertEquals(buffer.capacity() + offset, reader.getFilePointer()); + } + + reader.close(); + reader.close(); + } + + @Test(expected = UnsupportedOperationException.class) + public void testMarkNotSupported() throws Exception + { + FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 0); + assertFalse(reader.markSupported()); + assertEquals(0, reader.bytesPastMark(null)); + reader.mark(); + } + + @Test(expected = UnsupportedOperationException.class) + public void testResetNotSupported() throws Exception + { + FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 0); + reader.reset(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testSeekNegative() throws Exception + { + FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 0); + reader.seek(-1); + } + + @Test(expected = IllegalArgumentException.class) + public void testSeekBeforeOffset() throws Exception + { + FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 1024); + reader.seek(1023); + } + + @Test(expected = IllegalArgumentException.class) + public void testSeekPastLength() throws Exception + { + FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 1024); + reader.seek(2049); + } + + @Test(expected = EOFException.class) + public void testReadBytesTooMany() throws Exception + { + FileSegmentInputStream reader = new FileSegmentInputStream(allocateBuffer(1024), "", 1024); + ByteBufferUtil.read(reader, 2049); + } +} diff --git a/test/unit/org/apache/cassandra/io/util/MemoryTest.java b/test/unit/org/apache/cassandra/io/util/MemoryTest.java index 9be69aceee..81dee7ee3b 100644 --- a/test/unit/org/apache/cassandra/io/util/MemoryTest.java +++ b/test/unit/org/apache/cassandra/io/util/MemoryTest.java @@ -18,8 +18,11 @@ */ package org.apache.cassandra.io.util; +import java.io.EOFException; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.Arrays; import java.util.concurrent.ThreadLocalRandom; import org.junit.Test; @@ -27,6 +30,10 @@ import org.junit.Test; import junit.framework.Assert; import org.apache.cassandra.utils.memory.MemoryUtil; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class MemoryTest { @@ -45,6 +52,36 @@ public class MemoryTest memory.close(); } + @Test + public void testInputStream() throws IOException + { + byte[] bytes = new byte[4096]; + ThreadLocalRandom.current().nextBytes(bytes); + final Memory memory = Memory.allocate(bytes.length); + memory.setBytes(0, bytes, 0, bytes.length); + + try(MemoryInputStream stream = new MemoryInputStream(memory, 1024)) + { + byte[] bb = new byte[bytes.length]; + assertEquals(bytes.length, stream.available()); + + stream.readFully(bb); + assertEquals(0, stream.available()); + + assertTrue(Arrays.equals(bytes, bb)); + + try + { + stream.readInt(); + fail("Expected EOF exception"); + } + catch (EOFException e) + { + //pass + } + } + } + private static void test(ByteBuffer canon, Memory memory) { ByteBuffer hollow = MemoryUtil.getHollowDirectByteBuffer(); diff --git a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java new file mode 100644 index 0000000000..9df3fed468 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java @@ -0,0 +1,375 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.util; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Random; + +import com.google.common.primitives.Ints; +import org.junit.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.io.compress.CompressedSequentialWriter; +import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.schema.CompressionParams; +import org.apache.cassandra.utils.ChecksumType; + +import static junit.framework.Assert.assertNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class MmappedRegionsTest +{ + private static final Logger logger = LoggerFactory.getLogger(MmappedRegionsTest.class); + + private static ByteBuffer allocateBuffer(int size) + { + ByteBuffer ret = ByteBuffer.allocate(Ints.checkedCast(size)); + long seed = System.nanoTime(); + //seed = 365238103404423L; + logger.info("Seed {}", seed); + + new Random(seed).nextBytes(ret.array()); + return ret; + } + + private static File writeFile(String fileName, ByteBuffer buffer) throws IOException + { + File ret = File.createTempFile(fileName, "1"); + ret.deleteOnExit(); + + try (SequentialWriter writer = SequentialWriter.open(ret)) + { + writer.write(buffer); + writer.finish(); + } + + assert ret.exists(); + assert ret.length() >= buffer.capacity(); + return ret; + + } + + @Test + public void testEmpty() throws Exception + { + ByteBuffer buffer = allocateBuffer(1024); + try(ChannelProxy channel = new ChannelProxy(writeFile("testEmpty", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) + { + assertTrue(regions.isEmpty()); + assertTrue(regions.isValid(channel)); + } + } + + @Test + public void testTwoSegments() throws Exception + { + ByteBuffer buffer = allocateBuffer(2048); + try(ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) + { + regions.extend(1024); + for (int i = 0; i < 1024; i++) + { + MmappedRegions.Region region = regions.floor(i); + assertNotNull(region); + assertEquals(0, region.bottom()); + assertEquals(1024, region.top()); + } + + regions.extend(2048); + for (int i = 0; i < 2048; i++) + { + MmappedRegions.Region region = regions.floor(i); + assertNotNull(region); + if (i < 1024) + { + assertEquals(0, region.bottom()); + assertEquals(1024, region.top()); + } + else + { + assertEquals(1024, region.bottom()); + assertEquals(2048, region.top()); + } + } + } + } + + @Test + public void testSmallSegmentSize() throws Exception + { + int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE; + MmappedRegions.MAX_SEGMENT_SIZE = 1024; + + ByteBuffer buffer = allocateBuffer(4096); + try(ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) + { + regions.extend(1024); + regions.extend(2048); + regions.extend(4096); + + final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE; + for (int i = 0; i < buffer.capacity(); i++) + { + MmappedRegions.Region region = regions.floor(i); + assertNotNull(region); + assertEquals(SIZE * (i / SIZE), region.bottom()); + assertEquals(SIZE + (SIZE * (i / SIZE)), region.top()); + } + } + finally + { + MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE; + } + } + + @Test + public void testAllocRegions() throws Exception + { + int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE; + MmappedRegions.MAX_SEGMENT_SIZE = 1024; + + ByteBuffer buffer = allocateBuffer(MmappedRegions.MAX_SEGMENT_SIZE * MmappedRegions.REGION_ALLOC_SIZE * 3); + + try(ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) + { + regions.extend(buffer.capacity()); + + final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE; + for (int i = 0; i < buffer.capacity(); i++) + { + MmappedRegions.Region region = regions.floor(i); + assertNotNull(region); + assertEquals(SIZE * (i / SIZE), region.bottom()); + assertEquals(SIZE + (SIZE * (i / SIZE)), region.top()); + } + } + finally + { + MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE; + } + } + + @Test + public void testCopy() throws Exception + { + ByteBuffer buffer = allocateBuffer(128 * 1024); + + MmappedRegions snapshot; + ChannelProxy channelCopy; + + try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer)); + MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4)) + { + // create 3 more segments, one per quater capacity + regions.extend(buffer.capacity() / 2); + regions.extend(3 * buffer.capacity() / 4); + regions.extend(buffer.capacity()); + + // make a snapshot + snapshot = regions.sharedCopy(); + + // keep the channel open + channelCopy = channel.sharedCopy(); + } + + assertFalse(snapshot.isCleanedUp()); + + final int SIZE = buffer.capacity() / 4; + for (int i = 0; i < buffer.capacity(); i++) + { + MmappedRegions.Region region = snapshot.floor(i); + assertNotNull(region); + assertEquals(SIZE * (i / SIZE), region.bottom()); + assertEquals(SIZE + (SIZE * (i / SIZE)), region.top()); + + // check we can access the buffer + assertNotNull(region.buffer.duplicate().getInt()); + } + + assertNull(snapshot.close(null)); + assertNull(channelCopy.close(null)); + assertTrue(snapshot.isCleanedUp()); + } + + @Test(expected = AssertionError.class) + public void testCopyCannotExtend() throws Exception + { + ByteBuffer buffer = allocateBuffer(128 * 1024); + + MmappedRegions snapshot; + ChannelProxy channelCopy; + + try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) + { + regions.extend(buffer.capacity() / 2); + + // make a snapshot + snapshot = regions.sharedCopy(); + + // keep the channel open + channelCopy = channel.sharedCopy(); + } + + try + { + snapshot.extend(buffer.capacity()); + } + finally + { + assertNull(snapshot.close(null)); + assertNull(channelCopy.close(null)); + } + } + + @Test + public void testExtendOutOfOrder() throws Exception + { + ByteBuffer buffer = allocateBuffer(4096); + try(ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) + { + regions.extend(4096); + regions.extend(1024); + regions.extend(2048); + + for (int i = 0; i < buffer.capacity(); i++) + { + MmappedRegions.Region region = regions.floor(i); + assertNotNull(region); + assertEquals(0, region.bottom()); + assertEquals(4096, region.top()); + } + } + } + + @Test(expected = IllegalArgumentException.class) + public void testNegativeExtend() throws Exception + { + ByteBuffer buffer = allocateBuffer(1024); + try(ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) + { + regions.extend(-1); + } + } + + @Test + public void testMapForCompressionMetadata() throws Exception + { + int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE; + MmappedRegions.MAX_SEGMENT_SIZE = 1024; + + ByteBuffer buffer = allocateBuffer(128 * 1024); + File f = File.createTempFile("testMapForCompressionMetadata", "1"); + f.deleteOnExit(); + + File cf = File.createTempFile(f.getName() + ".metadata", "1"); + cf.deleteOnExit(); + + MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)) + .replayPosition(null); + try(SequentialWriter writer = new CompressedSequentialWriter(f, + cf.getAbsolutePath(), + CompressionParams.snappy(), + sstableMetadataCollector)) + { + writer.write(buffer); + writer.finish(); + } + + CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), ChecksumType.CRC32); + try(ChannelProxy channel = new ChannelProxy(f); + MmappedRegions regions = MmappedRegions.map(channel, metadata)) + { + + assertFalse(regions.isEmpty()); + int i = 0; + while(i < buffer.capacity()) + { + CompressionMetadata.Chunk chunk = metadata.chunkFor(i); + + MmappedRegions.Region region = regions.floor(chunk.offset); + assertNotNull(region); + + ByteBuffer compressedChunk = region.buffer.duplicate(); + assertNotNull(compressedChunk); + assertEquals(chunk.length + 4, compressedChunk.capacity()); + + assertEquals(chunk.offset, region.bottom()); + assertEquals(chunk.offset + chunk.length + 4, region.top()); + + i += metadata.chunkLength(); + } + } + finally + { + MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE; + metadata.close(); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testIllegalArgForMap1() throws Exception + { + ByteBuffer buffer = allocateBuffer(1024); + try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer)); + MmappedRegions regions = MmappedRegions.map(channel, 0)) + { + assertTrue(regions.isEmpty()); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testIllegalArgForMap2() throws Exception + { + ByteBuffer buffer = allocateBuffer(1024); + try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer)); + MmappedRegions regions = MmappedRegions.map(channel, -1L)) + { + assertTrue(regions.isEmpty()); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testIllegalArgForMap3() throws Exception + { + ByteBuffer buffer = allocateBuffer(1024); + try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer)); + MmappedRegions regions = MmappedRegions.map(channel, null)) + { + assertTrue(regions.isEmpty()); + } + } + +} diff --git a/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java b/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java index 3aad7e9555..3ebbc67fe0 100644 --- a/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java +++ b/test/unit/org/apache/cassandra/io/util/NIODataInputStreamTest.java @@ -14,7 +14,6 @@ import java.util.ArrayDeque; import java.util.Queue; import java.util.Random; -import org.apache.cassandra.io.util.NIODataInputStream; import org.junit.Test; import com.google.common.base.Charsets; @@ -179,18 +178,11 @@ public class NIODataInputStreamTest assertFalse(fakeStream.markSupported()); } - @SuppressWarnings("resource") - @Test(expected = IllegalArgumentException.class) - public void testTooSmallBufferSize() throws Exception - { - new NIODataInputStream(new FakeChannel(), 4); - } - @SuppressWarnings("resource") @Test(expected = NullPointerException.class) public void testNullRBC() throws Exception { - new NIODataInputStream(null, 8); + new NIODataInputStream(null, 9); } @SuppressWarnings("resource") @@ -769,7 +761,7 @@ public class NIODataInputStreamTest out.writeUnsignedVInt(value); buf.position(ii); - NIODataInputStream in = new DataInputBuffer(buf, false); + RebufferingInputStream in = new DataInputBuffer(buf, false); assertEquals(value, in.readUnsignedVInt()); } @@ -792,7 +784,7 @@ public class NIODataInputStreamTest out.writeUnsignedVInt(value); buf.position(0); - NIODataInputStream in = new DataInputBuffer(buf, false); + RebufferingInputStream in = new DataInputBuffer(buf, false); assertEquals(value, in.readUnsignedVInt()); @@ -831,7 +823,7 @@ public class NIODataInputStreamTest truncated.put(buf); truncated.flip(); - NIODataInputStream in = new DataInputBuffer(truncated, false); + RebufferingInputStream in = new DataInputBuffer(truncated, false); boolean threw = false; try diff --git a/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java new file mode 100644 index 0000000000..f0d4383c20 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java @@ -0,0 +1,483 @@ +package org.apache.cassandra.io.util; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.junit.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.*; + +import org.apache.cassandra.io.compress.BufferType; +import org.apache.cassandra.utils.ByteBufferUtil; + +public class RandomAccessReaderTest +{ + private static final Logger logger = LoggerFactory.getLogger(RandomAccessReaderTest.class); + + private static final class Parameters + { + public final long fileLength; + public final int bufferSize; + + public BufferType bufferType; + public int maxSegmentSize; + public boolean mmappedRegions; + public byte[] expected; + + Parameters(long fileLength, int bufferSize) + { + this.fileLength = fileLength; + this.bufferSize = bufferSize; + this.bufferType = BufferType.OFF_HEAP; + this.maxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE; + this.mmappedRegions = false; + this.expected = "The quick brown fox jumps over the lazy dog".getBytes(FileUtils.CHARSET); + } + + public Parameters mmappedRegions(boolean mmappedRegions) + { + this.mmappedRegions = mmappedRegions; + return this; + } + + public Parameters bufferType(BufferType bufferType) + { + this.bufferType = bufferType; + return this; + } + + public Parameters maxSegmentSize(int maxSegmentSize) + { + this.maxSegmentSize = maxSegmentSize; + return this; + } + + public Parameters expected(byte[] expected) + { + this.expected = expected; + return this; + } + } + + @Test + public void testBufferedOffHeap() throws IOException + { + testReadFully(new Parameters(8192, 4096).bufferType(BufferType.OFF_HEAP)); + } + + @Test + public void testBufferedOnHeap() throws IOException + { + testReadFully(new Parameters(8192, 4096).bufferType(BufferType.ON_HEAP)); + } + + @Test + public void testBigBufferSize() throws IOException + { + testReadFully(new Parameters(8192, 65536).bufferType(BufferType.ON_HEAP)); + } + + @Test + public void testTinyBufferSize() throws IOException + { + testReadFully(new Parameters(8192, 16).bufferType(BufferType.ON_HEAP)); + } + + @Test + public void testOneSegment() throws IOException + { + testReadFully(new Parameters(8192, 4096).mmappedRegions(true)); + } + + @Test + public void testMultipleSegments() throws IOException + { + testReadFully(new Parameters(8192, 4096).mmappedRegions(true).maxSegmentSize(1024)); + } + + @Test + public void testVeryLarge() throws IOException + { + final long SIZE = 1L << 32; // 2GB + Parameters params = new Parameters(SIZE, 1 << 20); // 1MB + + try(ChannelProxy channel = new ChannelProxy("abc", new FakeFileChannel(SIZE))) + { + RandomAccessReader.Builder builder = new RandomAccessReader.Builder(channel) + .bufferType(params.bufferType) + .bufferSize(params.bufferSize); + + try(RandomAccessReader reader = builder.build()) + { + assertEquals(channel.size(), reader.length()); + assertEquals(channel.size(), reader.bytesRemaining()); + assertEquals(Integer.MAX_VALUE, reader.available()); + + assertEquals(channel.size(), reader.skip(channel.size())); + + assertTrue(reader.isEOF()); + assertEquals(0, reader.bytesRemaining()); + } + } + } + + /** A fake file channel that simply increments the position and doesn't + * actually read anything. We use it to simulate very large files, > 2G. + */ + private static final class FakeFileChannel extends FileChannel + { + private final long size; + private long position; + + FakeFileChannel(long size) + { + this.size = size; + } + + public int read(ByteBuffer dst) + { + int ret = dst.remaining(); + position += ret; + dst.position(dst.limit()); + return ret; + } + + public long read(ByteBuffer[] dsts, int offset, int length) + { + throw new UnsupportedOperationException(); + } + + public int write(ByteBuffer src) + { + throw new UnsupportedOperationException(); + } + + public long write(ByteBuffer[] srcs, int offset, int length) + { + throw new UnsupportedOperationException(); + } + + public long position() + { + return position; + } + + public FileChannel position(long newPosition) + { + position = newPosition; + return this; + } + + public long size() + { + return size; + } + + public FileChannel truncate(long size) + { + throw new UnsupportedOperationException(); + } + + public void force(boolean metaData) + { + throw new UnsupportedOperationException(); + } + + public long transferTo(long position, long count, WritableByteChannel target) + { + throw new UnsupportedOperationException(); + } + + public long transferFrom(ReadableByteChannel src, long position, long count) + { + throw new UnsupportedOperationException(); + } + + public int read(ByteBuffer dst, long position) + { + int ret = dst.remaining(); + this.position = position + ret; + dst.position(dst.limit()); + return ret; + } + + public int write(ByteBuffer src, long position) + { + throw new UnsupportedOperationException(); + } + + public MappedByteBuffer map(MapMode mode, long position, long size) + { + throw new UnsupportedOperationException(); + } + + public FileLock lock(long position, long size, boolean shared) + { + throw new UnsupportedOperationException(); + } + + public FileLock tryLock(long position, long size, boolean shared) + { + throw new UnsupportedOperationException(); + } + + protected void implCloseChannel() + { + + } + } + + private static File writeFile(Parameters params) throws IOException + { + final File f = File.createTempFile("testReadFully", "1"); + f.deleteOnExit(); + + try(SequentialWriter writer = SequentialWriter.open(f)) + { + long numWritten = 0; + while (numWritten < params.fileLength) + { + writer.write(params.expected); + numWritten += params.expected.length; + } + + writer.finish(); + } + + assert f.exists(); + assert f.length() >= params.fileLength; + return f; + } + + private static void testReadFully(Parameters params) throws IOException + { + final File f = writeFile(params); + try(ChannelProxy channel = new ChannelProxy(f)) + { + RandomAccessReader.Builder builder = new RandomAccessReader.Builder(channel) + .bufferType(params.bufferType) + .bufferSize(params.bufferSize); + if (params.mmappedRegions) + builder.regions(MmappedRegions.map(channel, f.length())); + + try(RandomAccessReader reader = builder.build()) + { + assertEquals(f.getAbsolutePath(), reader.getPath()); + assertEquals(f.length(), reader.length()); + assertEquals(f.length(), reader.bytesRemaining()); + assertEquals(Math.min(Integer.MAX_VALUE, f.length()), reader.available()); + + byte[] b = new byte[params.expected.length]; + long numRead = 0; + while (numRead < params.fileLength) + { + reader.readFully(b); + assertTrue(Arrays.equals(params.expected, b)); + numRead += b.length; + } + + assertTrue(reader.isEOF()); + assertEquals(0, reader.bytesRemaining()); + } + + if (builder.regions != null) + assertNull(builder.regions.close(null)); + } + } + + @Test + public void testReadBytes() throws IOException + { + File f = File.createTempFile("testReadBytes", "1"); + final String expected = "The quick brown fox jumps over the lazy dog"; + + try(SequentialWriter writer = SequentialWriter.open(f)) + { + writer.write(expected.getBytes()); + writer.finish(); + } + + assert f.exists(); + + try(ChannelProxy channel = new ChannelProxy(f); + RandomAccessReader reader = new RandomAccessReader.Builder(channel).build()) + { + assertEquals(f.getAbsolutePath(), reader.getPath()); + assertEquals(expected.length(), reader.length()); + + ByteBuffer b = ByteBufferUtil.read(reader, expected.length()); + assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); + + assertTrue(reader.isEOF()); + assertEquals(0, reader.bytesRemaining()); + } + } + + @Test + public void testReset() throws IOException + { + File f = File.createTempFile("testMark", "1"); + final String expected = "The quick brown fox jumps over the lazy dog"; + final int numIterations = 10; + + try(SequentialWriter writer = SequentialWriter.open(f)) + { + for (int i = 0; i < numIterations; i++) + writer.write(expected.getBytes()); + writer.finish(); + } + + assert f.exists(); + + try(ChannelProxy channel = new ChannelProxy(f); + RandomAccessReader reader = new RandomAccessReader.Builder(channel).build()) + { + assertEquals(expected.length() * numIterations, reader.length()); + + ByteBuffer b = ByteBufferUtil.read(reader, expected.length()); + assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); + + assertFalse(reader.isEOF()); + assertEquals((numIterations - 1) * expected.length(), reader.bytesRemaining()); + + FileMark mark = reader.mark(); + assertEquals(0, reader.bytesPastMark()); + assertEquals(0, reader.bytesPastMark(mark)); + + for (int i = 0; i < (numIterations - 1); i++) + { + b = ByteBufferUtil.read(reader, expected.length()); + assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); + } + assertTrue(reader.isEOF()); + assertEquals(expected.length() * (numIterations - 1), reader.bytesPastMark()); + assertEquals(expected.length() * (numIterations - 1), reader.bytesPastMark(mark)); + + reader.reset(mark); + assertEquals(0, reader.bytesPastMark()); + assertEquals(0, reader.bytesPastMark(mark)); + assertFalse(reader.isEOF()); + for (int i = 0; i < (numIterations - 1); i++) + { + b = ByteBufferUtil.read(reader, expected.length()); + assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); + } + + reader.reset(); + assertEquals(0, reader.bytesPastMark()); + assertEquals(0, reader.bytesPastMark(mark)); + assertFalse(reader.isEOF()); + for (int i = 0; i < (numIterations - 1); i++) + { + b = ByteBufferUtil.read(reader, expected.length()); + assertEquals(expected, new String(b.array(), Charset.forName("UTF-8"))); + } + + assertTrue(reader.isEOF()); + } + } + + @Test + public void testSeekSingleThread() throws IOException, InterruptedException + { + testSeek(1); + } + + @Test + public void testSeekMultipleThreads() throws IOException, InterruptedException + { + testSeek(10); + } + + private static void testSeek(int numThreads) throws IOException, InterruptedException + { + final File f = File.createTempFile("testMark", "1"); + final byte[] expected = new byte[1 << 16]; + + long seed = System.nanoTime(); + //seed = 365238103404423L; + logger.info("Seed {}", seed); + Random r = new Random(seed); + r.nextBytes(expected); + + try(SequentialWriter writer = SequentialWriter.open(f)) + { + writer.write(expected); + writer.finish(); + } + + assert f.exists(); + + try(final ChannelProxy channel = new ChannelProxy(f)) + { + final Runnable worker = () -> + { + try(RandomAccessReader reader = new RandomAccessReader.Builder(channel).build()) + { + assertEquals(expected.length, reader.length()); + + ByteBuffer b = ByteBufferUtil.read(reader, expected.length); + assertTrue(Arrays.equals(expected, b.array())); + assertTrue(reader.isEOF()); + assertEquals(0, reader.bytesRemaining()); + + reader.seek(0); + b = ByteBufferUtil.read(reader, expected.length); + assertTrue(Arrays.equals(expected, b.array())); + assertTrue(reader.isEOF()); + assertEquals(0, reader.bytesRemaining()); + + for (int i = 0; i < 10; i++) + { + int pos = r.nextInt(expected.length); + reader.seek(pos); + assertEquals(pos, reader.getPosition()); + + ByteBuffer buf = ByteBuffer.wrap(expected, pos, expected.length - pos) + .order(ByteOrder.BIG_ENDIAN); + + while (reader.bytesRemaining() > 4) + assertEquals(buf.getInt(), reader.readInt()); + } + + reader.close(); + } + catch (Exception ex) + { + ex.printStackTrace(); + fail(ex.getMessage()); + } + }; + + if (numThreads == 1) + { + worker.run(); + } + else + { + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + for (int i = 0; i < numThreads; i++) + executor.submit(worker); + + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.MINUTES); + } + } + } +}