diff --git a/CHANGES.txt b/CHANGES.txt index cac380a6da..07b0630a81 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Adds the ability to use uncompressed chunks in compressed files (CASSANDRA-10520) * Don't flush sstables when streaming for incremental repair (CASSANDRA-13226) * Remove unused method (CASSANDRA-13227) * Fix minor bugs related to #9143 (CASSANDRA-13217) diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java index 973c94c054..b4ea61f234 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java +++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java @@ -60,6 +60,8 @@ public class CompressedSequentialWriter extends SequentialWriter private final ByteBuffer crcCheckBuffer = ByteBuffer.allocate(4); private final Optional digestFile; + private final int maxCompressedLength; + /** * Create CompressedSequentialWriter without digest file. * @@ -90,6 +92,8 @@ public class CompressedSequentialWriter extends SequentialWriter // buffer for compression should be the same size as buffer itself compressed = compressor.preferredBufferType().allocate(compressor.initialCompressedBufferLength(buffer.capacity())); + maxCompressedLength = parameters.maxCompressedLength(); + /* Index File (-CompressionInfo.db component) and it's header */ metadataWriter = CompressionMetadata.Writer.open(parameters, offsetsPath); @@ -146,6 +150,12 @@ public class CompressedSequentialWriter extends SequentialWriter int compressedLength = compressed.position(); uncompressedSize += buffer.position(); + ByteBuffer toWrite = compressed; + if (compressedLength > maxCompressedLength) + { + toWrite = buffer; + compressedLength = buffer.position(); + } compressedSize += compressedLength; try @@ -155,18 +165,20 @@ public class CompressedSequentialWriter extends SequentialWriter chunkCount++; // write out the compressed data - compressed.flip(); - channel.write(compressed); + toWrite.flip(); + channel.write(toWrite); // write corresponding checksum - compressed.rewind(); - crcMetadata.appendDirect(compressed, true); + toWrite.rewind(); + crcMetadata.appendDirect(toWrite, true); lastFlushOffset += compressedLength + 4; } catch (IOException e) { throw new FSWriteError(e, getPath()); } + if (toWrite == buffer) + buffer.position(compressedLength); // next chunk should be written right after current + length of the checksum (int) chunkOffset += compressedLength + 4; @@ -228,7 +240,10 @@ public class CompressedSequentialWriter extends SequentialWriter // Repopulate buffer from compressed data buffer.clear(); compressed.flip(); - compressor.uncompress(compressed, buffer); + if (chunkSize <= maxCompressedLength) + compressor.uncompress(compressed, buffer); + else + buffer.put(compressed); } catch (IOException e) { diff --git a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java index be264bf23a..a591bb570a 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java +++ b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java @@ -84,12 +84,22 @@ public class CompressionMetadata */ public static CompressionMetadata create(String dataFilePath) { - Descriptor desc = Descriptor.fromFilename(dataFilePath); - return new CompressionMetadata(desc.filenameFor(Component.COMPRESSION_INFO), new File(dataFilePath).length()); + return createWithLength(dataFilePath, new File(dataFilePath).length()); + } + + public static CompressionMetadata createWithLength(String dataFilePath, long compressedLength) + { + return new CompressionMetadata(Descriptor.fromFilename(dataFilePath), compressedLength); } @VisibleForTesting - public CompressionMetadata(String indexFilePath, long compressedLength) + public CompressionMetadata(Descriptor desc, long compressedLength) + { + this(desc.filenameFor(Component.COMPRESSION_INFO), compressedLength, desc.version.hasMaxCompressedLength()); + } + + @VisibleForTesting + public CompressionMetadata(String indexFilePath, long compressedLength, boolean hasMaxCompressedSize) { this.indexFilePath = indexFilePath; @@ -105,9 +115,12 @@ public class CompressionMetadata options.put(key, value); } int chunkLength = stream.readInt(); + int maxCompressedSize = Integer.MAX_VALUE; + if (hasMaxCompressedSize) + maxCompressedSize = stream.readInt(); try { - parameters = new CompressionParams(compressorName, chunkLength, options); + parameters = new CompressionParams(compressorName, chunkLength, maxCompressedSize, options); } catch (ConfigurationException e) { @@ -150,6 +163,11 @@ public class CompressionMetadata return parameters.chunkLength(); } + public int maxCompressedLength() + { + return parameters.maxCompressedLength(); + } + /** * Returns the amount of memory in bytes used off heap. * @return the amount of memory in bytes used off heap @@ -350,6 +368,7 @@ public class CompressionMetadata // store the length of the chunk out.writeInt(parameters.chunkLength()); + out.writeInt(parameters.maxCompressedLength()); // store position and reserve a place for uncompressed data length and chunks count out.writeLong(dataLength); out.writeInt(chunks); diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java index a07e48fb83..a7d1abffbb 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@ -51,6 +51,8 @@ public abstract class Version public abstract boolean hasCommitLogIntervals(); + public abstract boolean hasMaxCompressedLength(); + public abstract boolean hasPendingRepair(); public String getVersion() diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index cad192af18..9b0b5c5744 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -112,7 +112,7 @@ public class BigFormat implements SSTableFormat // we always incremented the major version. static class BigVersion extends Version { - public static final String current_version = "md"; + public static final String current_version = "na"; public static final String earliest_supported_version = "ma"; // ma (3.0.0): swap bf hash order @@ -120,6 +120,8 @@ public class BigFormat implements SSTableFormat // mb (3.0.7, 3.7): commit log lower bound included // mc (3.0.8, 3.9): commit log intervals included // md (3.0.9, 3.10): pending repair session included + + // na (4.0.0): uncompressed chunks // // NOTE: when adding a new version, please add that to LegacySSTableTest, too. @@ -127,6 +129,7 @@ public class BigFormat implements SSTableFormat public final int correspondingMessagingVersion; private final boolean hasCommitLogLowerBound; private final boolean hasCommitLogIntervals; + public final boolean hasMaxCompressedLength; private final boolean hasPendingRepair; BigVersion(String version) @@ -138,6 +141,7 @@ public class BigFormat implements SSTableFormat hasCommitLogLowerBound = version.compareTo("mb") >= 0; hasCommitLogIntervals = version.compareTo("mc") >= 0; + hasMaxCompressedLength = version.compareTo("na") >= 0; hasPendingRepair = version.compareTo("md") >= 0; } @@ -181,5 +185,11 @@ public class BigFormat implements SSTableFormat { return isCompatible() && version.charAt(0) == current_version.charAt(0); } + + @Override + public boolean hasMaxCompressedLength() + { + return hasMaxCompressedLength; + } } } diff --git a/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java b/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java index 219f0ebb3c..15f9fa02b4 100644 --- a/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java +++ b/src/java/org/apache/cassandra/io/util/CompressedChunkReader.java @@ -34,11 +34,13 @@ import org.apache.cassandra.utils.ChecksumType; public abstract class CompressedChunkReader extends AbstractReaderFileProxy implements ChunkReader { final CompressionMetadata metadata; + final int maxCompressedLength; protected CompressedChunkReader(ChannelProxy channel, CompressionMetadata metadata) { super(channel, metadata.dataLength); this.metadata = metadata; + this.maxCompressedLength = metadata.maxCompressedLength(); assert Integer.bitCount(metadata.chunkLength()) == 1; //must be a power of two } @@ -48,6 +50,11 @@ public abstract class CompressedChunkReader extends AbstractReaderFileProxy impl return metadata.parameters.getCrcCheckChance(); } + public boolean maybeCheckCrc() + { + return metadata.parameters.maybeCheckCrc(); + } + @Override public String toString() { @@ -90,7 +97,8 @@ public abstract class CompressedChunkReader extends AbstractReaderFileProxy impl public ByteBuffer allocateBuffer() { - return allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength())); + return allocateBuffer(Math.min(maxCompressedLength, + metadata.compressor().initialCompressedBufferLength(metadata.chunkLength()))); } public ByteBuffer allocateBuffer(int size) @@ -108,54 +116,58 @@ public abstract class CompressedChunkReader extends AbstractReaderFileProxy impl assert position <= fileLength; CompressionMetadata.Chunk chunk = metadata.chunkFor(position); - ByteBuffer compressed = compressedHolder.get(); - - if (compressed.capacity() < chunk.length) + if (chunk.length <= maxCompressedLength) { - compressed = allocateBuffer(chunk.length); - compressedHolder.set(compressed); + ByteBuffer compressed = compressedHolder.get(); + assert compressed.capacity() >= chunk.length; + compressed.clear().limit(chunk.length); + if (channel.read(compressed, chunk.offset) != chunk.length) + throw new CorruptBlockException(channel.filePath(), chunk); + + compressed.flip(); + uncompressed.clear(); + + try + { + metadata.compressor().uncompress(compressed, uncompressed); + } + catch (IOException e) + { + throw new CorruptBlockException(channel.filePath(), chunk, e); + } + maybeCheckCrc(chunk, compressed); } else { - compressed.clear(); - } - - compressed.limit(chunk.length); - if (channel.read(compressed, chunk.offset) != chunk.length) - throw new CorruptBlockException(channel.filePath(), chunk); - - compressed.flip(); - uncompressed.clear(); - - try - { - metadata.compressor().uncompress(compressed, uncompressed); - } - catch (IOException e) - { - throw new CorruptBlockException(channel.filePath(), chunk, e); - } - finally - { - uncompressed.flip(); - } - - if (getCrcCheckChance() > ThreadLocalRandom.current().nextDouble()) - { - compressed.rewind(); - int checksum = (int) ChecksumType.CRC32.of(compressed); - - compressed.clear().limit(Integer.BYTES); - if (channel.read(compressed, chunk.offset + chunk.length) != Integer.BYTES - || compressed.getInt(0) != checksum) + uncompressed.position(0).limit(chunk.length); + if (channel.read(uncompressed, chunk.offset) != chunk.length) throw new CorruptBlockException(channel.filePath(), chunk); + maybeCheckCrc(chunk, uncompressed); } + uncompressed.flip(); } catch (CorruptBlockException e) { + // Make sure reader does not see stale data. + uncompressed.position(0).limit(0); throw new CorruptSSTableException(e, channel.filePath()); } } + + void maybeCheckCrc(CompressionMetadata.Chunk chunk, ByteBuffer content) throws CorruptBlockException + { + if (metadata.parameters.maybeCheckCrc()) + { + content.flip(); + int checksum = (int) ChecksumType.CRC32.of(content); + + ByteBuffer scratch = compressedHolder.get(); // This may match content. That's ok, we no longer need it. + scratch.clear().limit(Integer.BYTES); + if (channel.read(scratch, chunk.offset + chunk.length) != Integer.BYTES + || scratch.getInt(0) != checksum) + throw new CorruptBlockException(channel.filePath(), chunk); + } + } } public static class Mmap extends CompressedChunkReader @@ -190,18 +202,18 @@ public abstract class CompressedChunkReader extends AbstractReaderFileProxy impl try { - metadata.compressor().uncompress(compressedChunk, uncompressed); + if (chunk.length <= maxCompressedLength) + metadata.compressor().uncompress(compressedChunk, uncompressed); + else + uncompressed.put(compressedChunk); } catch (IOException e) { throw new CorruptBlockException(channel.filePath(), chunk, e); } - finally - { - uncompressed.flip(); - } + uncompressed.flip(); - if (getCrcCheckChance() > ThreadLocalRandom.current().nextDouble()) + if (maybeCheckCrc()) { compressedChunk.position(chunkOffset).limit(chunkOffset + chunk.length); @@ -214,6 +226,8 @@ public abstract class CompressedChunkReader extends AbstractReaderFileProxy impl } catch (CorruptBlockException e) { + // Make sure reader does not see stale data. + uncompressed.position(0).limit(0); throw new CorruptSSTableException(e, channel.filePath()); } diff --git a/src/java/org/apache/cassandra/schema/CompressionParams.java b/src/java/org/apache/cassandra/schema/CompressionParams.java index f48a68835b..a319311395 100644 --- a/src/java/org/apache/cassandra/schema/CompressionParams.java +++ b/src/java/org/apache/cassandra/schema/CompressionParams.java @@ -23,10 +23,13 @@ import java.lang.reflect.Method; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; import com.google.common.collect.ImmutableMap; + import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +40,7 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.compress.*; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.streaming.messages.StreamMessage; import static java.lang.String.format; @@ -50,14 +54,18 @@ public final class CompressionParams private static volatile boolean hasLoggedCrcCheckChanceWarning; public static final int DEFAULT_CHUNK_LENGTH = 65536; + public static final double DEFAULT_MIN_COMPRESS_RATIO = 1.1; public static final IVersionedSerializer serializer = new Serializer(); public static final String CLASS = "class"; public static final String CHUNK_LENGTH_IN_KB = "chunk_length_in_kb"; public static final String ENABLED = "enabled"; + public static final String MIN_COMPRESS_RATIO = "min_compress_ratio"; public static final CompressionParams DEFAULT = new CompressionParams(LZ4Compressor.create(Collections.emptyMap()), DEFAULT_CHUNK_LENGTH, + calcMaxCompressedLength(DEFAULT_CHUNK_LENGTH, DEFAULT_MIN_COMPRESS_RATIO), + DEFAULT_MIN_COMPRESS_RATIO, Collections.emptyMap()); private static final String CRC_CHECK_CHANCE_WARNING = "The option crc_check_chance was deprecated as a compression option. " + @@ -68,7 +76,9 @@ public final class CompressionParams @Deprecated public static final String CRC_CHECK_CHANCE = "crc_check_chance"; private final ICompressor sstableCompressor; - private final Integer chunkLength; + private final int chunkLength; + private final int maxCompressedLength; // In content we store max length to avoid rounding errors causing compress/decompress mismatch. + private final double minCompressRatio; // In configuration we store min ratio, the input parameter. private final ImmutableMap otherOptions; // Unrecognized options, can be used by the compressor private volatile double crcCheckChance = 1.0; @@ -94,9 +104,10 @@ public final class CompressionParams sstableCompressionClass = removeSstableCompressionClass(options); } - Integer chunkLength = removeChunkLength(options); + int chunkLength = removeChunkLength(options); + double minCompressRatio = removeMinCompressRatio(options); - CompressionParams cp = new CompressionParams(sstableCompressionClass, chunkLength, options); + CompressionParams cp = new CompressionParams(sstableCompressionClass, options, chunkLength, minCompressRatio); cp.validate(); return cp; @@ -109,54 +120,83 @@ public final class CompressionParams public static CompressionParams noCompression() { - return new CompressionParams((ICompressor) null, DEFAULT_CHUNK_LENGTH, Collections.emptyMap()); + return new CompressionParams((ICompressor) null, DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, 0.0, Collections.emptyMap()); } public static CompressionParams snappy() { - return snappy(null); + return snappy(DEFAULT_CHUNK_LENGTH); } - public static CompressionParams snappy(Integer chunkLength) + public static CompressionParams snappy(int chunkLength) { - return new CompressionParams(SnappyCompressor.instance, chunkLength, Collections.emptyMap()); + return snappy(chunkLength, DEFAULT_MIN_COMPRESS_RATIO); + } + + public static CompressionParams snappy(int chunkLength, double minCompressRatio) + { + return new CompressionParams(SnappyCompressor.instance, chunkLength, calcMaxCompressedLength(chunkLength, minCompressRatio), minCompressRatio, Collections.emptyMap()); } public static CompressionParams deflate() { - return deflate(null); + return deflate(DEFAULT_CHUNK_LENGTH); } - public static CompressionParams deflate(Integer chunkLength) + public static CompressionParams deflate(int chunkLength) { - return new CompressionParams(DeflateCompressor.instance, chunkLength, Collections.emptyMap()); + return new CompressionParams(DeflateCompressor.instance, chunkLength, Integer.MAX_VALUE, 0.0, Collections.emptyMap()); } public static CompressionParams lz4() { - return lz4(null); + return lz4(DEFAULT_CHUNK_LENGTH); } - public static CompressionParams lz4(Integer chunkLength) + public static CompressionParams lz4(int chunkLength) { - return new CompressionParams(LZ4Compressor.create(Collections.emptyMap()), chunkLength, Collections.emptyMap()); + return lz4(chunkLength, Integer.MAX_VALUE); } - public CompressionParams(String sstableCompressorClass, Integer chunkLength, Map otherOptions) throws ConfigurationException + public static CompressionParams lz4(int chunkLength, int maxCompressedLength) { - this(createCompressor(parseCompressorClass(sstableCompressorClass), otherOptions), chunkLength, otherOptions); + return new CompressionParams(LZ4Compressor.create(Collections.emptyMap()), chunkLength, maxCompressedLength, calcMinCompressRatio(chunkLength, maxCompressedLength), Collections.emptyMap()); } - private CompressionParams(ICompressor sstableCompressor, Integer chunkLength, Map otherOptions) throws ConfigurationException + public CompressionParams(String sstableCompressorClass, Map otherOptions, int chunkLength, double minCompressRatio) throws ConfigurationException + { + this(createCompressor(parseCompressorClass(sstableCompressorClass), otherOptions), chunkLength, calcMaxCompressedLength(chunkLength, minCompressRatio), minCompressRatio, otherOptions); + } + + static int calcMaxCompressedLength(int chunkLength, double minCompressRatio) + { + return (int) Math.ceil(Math.min(chunkLength / minCompressRatio, Integer.MAX_VALUE)); + } + + public CompressionParams(String sstableCompressorClass, int chunkLength, int maxCompressedLength, Map otherOptions) throws ConfigurationException + { + this(createCompressor(parseCompressorClass(sstableCompressorClass), otherOptions), chunkLength, maxCompressedLength, calcMinCompressRatio(chunkLength, maxCompressedLength), otherOptions); + } + + static double calcMinCompressRatio(int chunkLength, int maxCompressedLength) + { + if (maxCompressedLength == Integer.MAX_VALUE) + return 0; + return chunkLength * 1.0 / maxCompressedLength; + } + + private CompressionParams(ICompressor sstableCompressor, int chunkLength, int maxCompressedLength, double minCompressRatio, Map otherOptions) throws ConfigurationException { this.sstableCompressor = sstableCompressor; this.chunkLength = chunkLength; this.otherOptions = ImmutableMap.copyOf(otherOptions); + this.minCompressRatio = minCompressRatio; + this.maxCompressedLength = maxCompressedLength; } public CompressionParams copy() { - return new CompressionParams(sstableCompressor, chunkLength, otherOptions); + return new CompressionParams(sstableCompressor, chunkLength, maxCompressedLength, minCompressRatio, otherOptions); } /** @@ -184,7 +224,12 @@ public final class CompressionParams public int chunkLength() { - return chunkLength == null ? DEFAULT_CHUNK_LENGTH : chunkLength; + return chunkLength; + } + + public int maxCompressedLength() + { + return maxCompressedLength; } private static Class parseCompressorClass(String className) throws ConfigurationException @@ -312,7 +357,7 @@ public final class CompressionParams * @param options the options * @return the chunk length value */ - private static Integer removeChunkLength(Map options) + private static int removeChunkLength(Map options) { if (options.containsKey(CHUNK_LENGTH_IN_KB)) { @@ -339,7 +384,23 @@ public final class CompressionParams return parseChunkLength(options.remove(CHUNK_LENGTH_KB)); } - return null; + return DEFAULT_CHUNK_LENGTH; + } + + /** + * Removes the min compress ratio option from the specified set of option. + * + * @param options the options + * @return the min compress ratio, used to calculate max chunk size to write compressed + */ + private static double removeMinCompressRatio(Map options) + { + String ratio = options.remove(MIN_COMPRESS_RATIO); + if (ratio != null) + { + return Double.parseDouble(ratio); + } + return DEFAULT_MIN_COMPRESS_RATIO; } /** @@ -420,25 +481,14 @@ public final class CompressionParams public void validate() throws ConfigurationException { // if chunk length was not set (chunkLength == null), this is fine, default will be used - if (chunkLength != null) - { - if (chunkLength <= 0) - throw new ConfigurationException("Invalid negative or null " + CHUNK_LENGTH_IN_KB); + if (chunkLength <= 0) + throw new ConfigurationException("Invalid negative or null " + CHUNK_LENGTH_IN_KB); - int c = chunkLength; - boolean found = false; - while (c != 0) - { - if ((c & 0x01) != 0) - { - if (found) - throw new ConfigurationException(CHUNK_LENGTH_IN_KB + " must be a power of 2"); - else - found = true; - } - c >>= 1; - } - } + if ((chunkLength & (chunkLength - 1)) != 0) + throw new ConfigurationException(CHUNK_LENGTH_IN_KB + " must be a power of 2"); + + if (maxCompressedLength < 0) + throw new ConfigurationException("Invalid negative " + MIN_COMPRESS_RATIO); } public Map asMap() @@ -449,6 +499,7 @@ public final class CompressionParams Map options = new HashMap<>(otherOptions); options.put(CLASS, sstableCompressor.getClass().getName()); options.put(CHUNK_LENGTH_IN_KB, chunkLengthInKB()); + options.put(MIN_COMPRESS_RATIO, String.valueOf(minCompressRatio)); return options; } @@ -468,6 +519,12 @@ public final class CompressionParams return crcCheckChance; } + public boolean maybeCheckCrc() + { + double checkChance = getCrcCheckChance(); + return checkChance > 0d && checkChance > ThreadLocalRandom.current().nextDouble(); + } + @Override public boolean equals(Object obj) { @@ -510,6 +567,11 @@ public final class CompressionParams out.writeUTF(entry.getValue()); } out.writeInt(parameters.chunkLength()); + if (version >= StreamMessage.VERSION_40) + out.writeInt(parameters.maxCompressedLength); + else + if (parameters.maxCompressedLength != Integer.MAX_VALUE) + throw new UnsupportedOperationException("Cannot stream SSTables with uncompressed chunks to pre-4.0 nodes."); } public CompressionParams deserialize(DataInputPlus in, int version) throws IOException @@ -524,10 +586,14 @@ public final class CompressionParams options.put(key, value); } int chunkLength = in.readInt(); + int minCompressRatio = Integer.MAX_VALUE; // Earlier Cassandra cannot use uncompressed chunks. + if (version >= StreamMessage.VERSION_40) + minCompressRatio = in.readInt(); + CompressionParams parameters; try { - parameters = new CompressionParams(compressorName, chunkLength, options); + parameters = new CompressionParams(compressorName, chunkLength, minCompressRatio, options); } catch (ConfigurationException e) { @@ -546,6 +612,8 @@ public final class CompressionParams size += TypeSizes.sizeof(entry.getValue()); } size += TypeSizes.sizeof(parameters.chunkLength()); + if (version >= StreamMessage.VERSION_40) + size += TypeSizes.sizeof(parameters.maxCompressedLength()); return size; } } diff --git a/src/java/org/apache/cassandra/service/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/AbstractReadExecutor.java index 6e0f45bb79..7a82187186 100644 --- a/src/java/org/apache/cassandra/service/AbstractReadExecutor.java +++ b/src/java/org/apache/cassandra/service/AbstractReadExecutor.java @@ -148,12 +148,16 @@ public abstract class AbstractReadExecutor private static ReadRepairDecision newReadRepairDecision(TableMetadata metadata) { - double chance = ThreadLocalRandom.current().nextDouble(); - if (metadata.params.readRepairChance > chance) - return ReadRepairDecision.GLOBAL; + if (metadata.params.readRepairChance > 0d || + metadata.params.dcLocalReadRepairChance > 0) + { + double chance = ThreadLocalRandom.current().nextDouble(); + if (metadata.params.readRepairChance > chance) + return ReadRepairDecision.GLOBAL; - if (metadata.params.dcLocalReadRepairChance > chance) - return ReadRepairDecision.DC_LOCAL; + if (metadata.params.dcLocalReadRepairChance > chance) + return ReadRepairDecision.DC_LOCAL; + } return ReadRepairDecision.NONE; } diff --git a/src/java/org/apache/cassandra/streaming/compress/CompressedInputStream.java b/src/java/org/apache/cassandra/streaming/compress/CompressedInputStream.java index 81abefa043..003db6190f 100644 --- a/src/java/org/apache/cassandra/streaming/compress/CompressedInputStream.java +++ b/src/java/org/apache/cassandra/streaming/compress/CompressedInputStream.java @@ -160,11 +160,18 @@ public class CompressedInputStream extends InputStream private void decompress(byte[] compressed) throws IOException { // uncompress - validBufferBytes = info.parameters.getSstableCompressor().uncompress(compressed, 0, compressed.length - checksumBytes.length, buffer, 0); + if (compressed.length - checksumBytes.length < info.parameters.maxCompressedLength()) + validBufferBytes = info.parameters.getSstableCompressor().uncompress(compressed, 0, compressed.length - checksumBytes.length, buffer, 0); + else + { + validBufferBytes = compressed.length - checksumBytes.length; + System.arraycopy(compressed, 0, buffer, 0, validBufferBytes); + } totalCompressedBytesRead += compressed.length; // validate crc randomly - if (this.crcCheckChanceSupplier.get() > ThreadLocalRandom.current().nextDouble()) + double crcCheckChance = this.crcCheckChanceSupplier.get(); + if (crcCheckChance > 0d && crcCheckChance > ThreadLocalRandom.current().nextDouble()) { int checksum = (int) checksumType.of(compressed, 0, compressed.length - checksumBytes.length); diff --git a/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java b/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java index a37420b4ba..f449982f5d 100644 --- a/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java +++ b/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java @@ -221,7 +221,7 @@ public class FileMessageHeader List> sections = new ArrayList<>(count); for (int k = 0; k < count; k++) sections.add(Pair.create(in.readLong(), in.readLong())); - CompressionInfo compressionInfo = CompressionInfo.serializer.deserialize(in, MessagingService.current_version); + CompressionInfo compressionInfo = CompressionInfo.serializer.deserialize(in, version); long repairedAt = in.readLong(); int sstableLevel = in.readInt(); SerializationHeader.Component header = SerializationHeader.serializer.deserialize(sstableVersion, in); @@ -246,8 +246,7 @@ public class FileMessageHeader size += CompressionInfo.serializer.serializedSize(header.compressionInfo, version); size += TypeSizes.sizeof(header.sstableLevel); - if (version >= StreamMessage.VERSION_30) - size += SerializationHeader.serializer.serializedSize(header.version, header.header); + size += SerializationHeader.serializer.serializedSize(header.version, header.header); return size; } diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java index 3ce195808f..48def6468a 100644 --- a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java @@ -33,8 +33,8 @@ import org.apache.cassandra.streaming.StreamSession; public abstract class StreamMessage { /** Streaming protocol version */ - public static final int VERSION_30 = 4; - public static final int CURRENT_VERSION = VERSION_30; + public static final int VERSION_40 = 5; + public static final int CURRENT_VERSION = VERSION_40; private transient volatile boolean sent = false; diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db new file mode 100644 index 0000000000..6bf2986818 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db new file mode 100644 index 0000000000..f92f905b40 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 new file mode 100644 index 0000000000..37f68ab9cb --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 @@ -0,0 +1 @@ +1337396261 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db new file mode 100644 index 0000000000..b173ddad92 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db new file mode 100644 index 0000000000..1fd8bac109 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt new file mode 100644 index 0000000000..ae620de6f0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt @@ -0,0 +1,8 @@ +Summary.db +Filter.db +Index.db +CompressionInfo.db +Data.db +TOC.txt +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-CompressionInfo.db new file mode 100644 index 0000000000..c96fb7dfd3 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Data.db new file mode 100644 index 0000000000..5ecab703ba Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Digest.crc32 new file mode 100644 index 0000000000..e94e369305 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Digest.crc32 @@ -0,0 +1 @@ +2666613329 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Partitions.db new file mode 100644 index 0000000000..aded0e1f0b Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Rows.db new file mode 100644 index 0000000000..44803b414f Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Rows.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Statistics.db new file mode 100644 index 0000000000..9e8a4b1b43 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-TOC.txt new file mode 100644 index 0000000000..c20f4a8ad0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Filter.db +Statistics.db +CompressionInfo.db +Partitions.db +TOC.txt +Rows.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db new file mode 100644 index 0000000000..1e69fcd45e Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db new file mode 100644 index 0000000000..f4fb8f2e65 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 new file mode 100644 index 0000000000..ce2e2f8f3b --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 @@ -0,0 +1 @@ +2454456604 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db new file mode 100644 index 0000000000..3f4635093d Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db new file mode 100644 index 0000000000..b15e758f15 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt new file mode 100644 index 0000000000..ae620de6f0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt @@ -0,0 +1,8 @@ +Summary.db +Filter.db +Index.db +CompressionInfo.db +Data.db +TOC.txt +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-CompressionInfo.db new file mode 100644 index 0000000000..9c013f7045 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Data.db new file mode 100644 index 0000000000..1f5403379e Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Digest.crc32 new file mode 100644 index 0000000000..e57848f5c9 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Digest.crc32 @@ -0,0 +1 @@ +874495544 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Partitions.db new file mode 100644 index 0000000000..aded0e1f0b Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Rows.db new file mode 100644 index 0000000000..6992a68f88 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Rows.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Statistics.db new file mode 100644 index 0000000000..05a82511a7 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-TOC.txt new file mode 100644 index 0000000000..c20f4a8ad0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Filter.db +Statistics.db +CompressionInfo.db +Partitions.db +TOC.txt +Rows.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db new file mode 100644 index 0000000000..7fd6a6c31a Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db new file mode 100644 index 0000000000..5e9425e22f Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 new file mode 100644 index 0000000000..0a20edb04c --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 @@ -0,0 +1 @@ +41575897 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db new file mode 100644 index 0000000000..2474c3e883 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db new file mode 100644 index 0000000000..3df7cf3595 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt new file mode 100644 index 0000000000..ae620de6f0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt @@ -0,0 +1,8 @@ +Summary.db +Filter.db +Index.db +CompressionInfo.db +Data.db +TOC.txt +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-CompressionInfo.db new file mode 100644 index 0000000000..a8ab57274e Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Data.db new file mode 100644 index 0000000000..3632c25b39 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Digest.crc32 new file mode 100644 index 0000000000..a199ec317a --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Digest.crc32 @@ -0,0 +1 @@ +2912620103 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Partitions.db new file mode 100644 index 0000000000..aded0e1f0b Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Rows.db new file mode 100644 index 0000000000..b0aea85181 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Rows.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Statistics.db new file mode 100644 index 0000000000..3b51c1b077 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-TOC.txt new file mode 100644 index 0000000000..c20f4a8ad0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Filter.db +Statistics.db +CompressionInfo.db +Partitions.db +TOC.txt +Rows.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db new file mode 100644 index 0000000000..fd7c663ee8 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db new file mode 100644 index 0000000000..79f591957d Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 new file mode 100644 index 0000000000..7c2e59ab45 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 @@ -0,0 +1 @@ +4050499673 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db new file mode 100644 index 0000000000..6c9cb96f18 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db new file mode 100644 index 0000000000..68b66570f5 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt new file mode 100644 index 0000000000..ae620de6f0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt @@ -0,0 +1,8 @@ +Summary.db +Filter.db +Index.db +CompressionInfo.db +Data.db +TOC.txt +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-CompressionInfo.db new file mode 100644 index 0000000000..b8a1dac507 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Data.db new file mode 100644 index 0000000000..f1c5c62240 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Digest.crc32 new file mode 100644 index 0000000000..7f68c2c8f4 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Digest.crc32 @@ -0,0 +1 @@ +1995270006 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Partitions.db new file mode 100644 index 0000000000..aded0e1f0b Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Rows.db new file mode 100644 index 0000000000..e63ee20c95 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Rows.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Statistics.db new file mode 100644 index 0000000000..e4fcc94ef2 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-TOC.txt new file mode 100644 index 0000000000..c20f4a8ad0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Filter.db +Statistics.db +CompressionInfo.db +Partitions.db +TOC.txt +Rows.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db new file mode 100644 index 0000000000..2640f953d2 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db new file mode 100644 index 0000000000..089aa6c386 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 new file mode 100644 index 0000000000..151fc22029 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 @@ -0,0 +1 @@ +243065670 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Index.db new file mode 100644 index 0000000000..b3094bffba Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db new file mode 100644 index 0000000000..404a3f81cb Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt new file mode 100644 index 0000000000..ae620de6f0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt @@ -0,0 +1,8 @@ +Summary.db +Filter.db +Index.db +CompressionInfo.db +Data.db +TOC.txt +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-CompressionInfo.db new file mode 100644 index 0000000000..0b7faea4a8 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Data.db new file mode 100644 index 0000000000..277996bd4c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Digest.crc32 new file mode 100644 index 0000000000..654f52bb12 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Digest.crc32 @@ -0,0 +1 @@ +4102718625 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Partitions.db new file mode 100644 index 0000000000..f297888fd9 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Rows.db new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Statistics.db new file mode 100644 index 0000000000..62b5d3f7c4 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-TOC.txt new file mode 100644 index 0000000000..c20f4a8ad0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Filter.db +Statistics.db +CompressionInfo.db +Partitions.db +TOC.txt +Rows.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-CompressionInfo.db new file mode 100644 index 0000000000..63039e6c7a Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db new file mode 100644 index 0000000000..5970d11861 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 new file mode 100644 index 0000000000..e85ffd9202 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 @@ -0,0 +1 @@ +576034138 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Index.db new file mode 100644 index 0000000000..56f29df1cb Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db new file mode 100644 index 0000000000..f63efeccd0 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt new file mode 100644 index 0000000000..ae620de6f0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt @@ -0,0 +1,8 @@ +Summary.db +Filter.db +Index.db +CompressionInfo.db +Data.db +TOC.txt +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-CompressionInfo.db new file mode 100644 index 0000000000..adb7fc4579 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Data.db new file mode 100644 index 0000000000..68f29ba3d1 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Digest.crc32 new file mode 100644 index 0000000000..bf775523a2 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Digest.crc32 @@ -0,0 +1 @@ +380992464 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Partitions.db new file mode 100644 index 0000000000..e4f9ea9605 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Rows.db new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Statistics.db new file mode 100644 index 0000000000..b853cfb74c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-TOC.txt new file mode 100644 index 0000000000..c20f4a8ad0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Filter.db +Statistics.db +CompressionInfo.db +Partitions.db +TOC.txt +Rows.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db new file mode 100644 index 0000000000..6726d74e0e Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db new file mode 100644 index 0000000000..28234abda9 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 new file mode 100644 index 0000000000..955bc497c4 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 @@ -0,0 +1 @@ +381306786 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Index.db new file mode 100644 index 0000000000..59e65cab85 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db new file mode 100644 index 0000000000..d65a3a4530 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt new file mode 100644 index 0000000000..ae620de6f0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt @@ -0,0 +1,8 @@ +Summary.db +Filter.db +Index.db +CompressionInfo.db +Data.db +TOC.txt +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-CompressionInfo.db new file mode 100644 index 0000000000..0d9c077a99 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Data.db new file mode 100644 index 0000000000..1489bab6d9 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Digest.crc32 new file mode 100644 index 0000000000..a804901b41 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Digest.crc32 @@ -0,0 +1 @@ +163579974 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Partitions.db new file mode 100644 index 0000000000..1eed5ad232 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Rows.db new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Statistics.db new file mode 100644 index 0000000000..171655ce40 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-TOC.txt new file mode 100644 index 0000000000..c20f4a8ad0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Filter.db +Statistics.db +CompressionInfo.db +Partitions.db +TOC.txt +Rows.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-CompressionInfo.db new file mode 100644 index 0000000000..edf78d5dcd Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db new file mode 100644 index 0000000000..ce4965c1a9 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 new file mode 100644 index 0000000000..06f147ffdb --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 @@ -0,0 +1 @@ +2161889354 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Index.db new file mode 100644 index 0000000000..d094f73e21 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db new file mode 100644 index 0000000000..6b2d88b91c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Summary.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt new file mode 100644 index 0000000000..ae620de6f0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt @@ -0,0 +1,8 @@ +Summary.db +Filter.db +Index.db +CompressionInfo.db +Data.db +TOC.txt +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-CompressionInfo.db new file mode 100644 index 0000000000..56c95a8a36 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Data.db new file mode 100644 index 0000000000..2977f11779 Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Digest.crc32 new file mode 100644 index 0000000000..02bf600438 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Digest.crc32 @@ -0,0 +1 @@ +1528982319 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Filter.db new file mode 100644 index 0000000000..2e1d5d29ca Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Filter.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Partitions.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Partitions.db new file mode 100644 index 0000000000..05e27b4add Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Rows.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Rows.db new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Statistics.db new file mode 100644 index 0000000000..08d8f3ee7d Binary files /dev/null and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-TOC.txt new file mode 100644 index 0000000000..c20f4a8ad0 --- /dev/null +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Filter.db +Statistics.db +CompressionInfo.db +Partitions.db +TOC.txt +Rows.db +Digest.crc32 diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java index c2efb6a9f0..d8f21e881d 100644 --- a/test/unit/org/apache/cassandra/SchemaLoader.java +++ b/test/unit/org/apache/cassandra/SchemaLoader.java @@ -726,7 +726,7 @@ public static TableMetadata.Builder clusteringSASICFMD(String ksName, String cfN public static CompressionParams getCompressionParameters(Integer chunkSize) { if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"))) - return CompressionParams.snappy(chunkSize); + return chunkSize != null ? CompressionParams.snappy(chunkSize) : CompressionParams.snappy(); return CompressionParams.noCompression(); } diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java index 88c6a3f000..8dc6fedb89 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java @@ -328,7 +328,7 @@ public class AlterTest extends CQLTester SchemaKeyspace.TABLES), KEYSPACE, currentTable()), - row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor"))); + row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "1.1"))); execute("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };"); @@ -337,7 +337,7 @@ public class AlterTest extends CQLTester SchemaKeyspace.TABLES), KEYSPACE, currentTable()), - row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"))); + row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "1.1"))); execute("ALTER TABLE %s WITH compression = { 'sstable_compression' : 'LZ4Compressor', 'chunk_length_kb' : 64 };"); @@ -346,7 +346,25 @@ public class AlterTest extends CQLTester SchemaKeyspace.TABLES), KEYSPACE, currentTable()), - row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor"))); + row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "1.1"))); + + execute("ALTER TABLE %s WITH compression = { 'sstable_compression' : 'LZ4Compressor', 'min_compress_ratio' : 2 };"); + + assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", + SchemaConstants.SCHEMA_KEYSPACE_NAME, + SchemaKeyspace.TABLES), + KEYSPACE, + currentTable()), + row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "2.0"))); + + execute("ALTER TABLE %s WITH compression = { 'sstable_compression' : 'LZ4Compressor', 'min_compress_ratio' : 0 };"); + + assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", + SchemaConstants.SCHEMA_KEYSPACE_NAME, + SchemaKeyspace.TABLES), + KEYSPACE, + currentTable()), + row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "0.0"))); execute("ALTER TABLE %s WITH compression = { 'sstable_compression' : '', 'chunk_length_kb' : 32 };"); @@ -381,6 +399,9 @@ public class AlterTest extends CQLTester assertThrowsConfigurationException("The 'chunk_length_kb' option must not be used if the chunk length is already specified by the 'chunk_length_in_kb' option", "ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_kb' : 32 , 'chunk_length_in_kb' : 32 };"); + + assertThrowsConfigurationException("Invalid negative min_compress_ratio", + "ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : -1 };"); } private void assertThrowsConfigurationException(String errorMsg, String alterStmt) throws Throwable diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java index 1a8338f8d9..9c8b12a768 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java @@ -722,7 +722,7 @@ public class CreateTest extends CQLTester SchemaKeyspace.TABLES), KEYSPACE, currentTable()), - row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor"))); + row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "1.1"))); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };"); @@ -732,7 +732,7 @@ public class CreateTest extends CQLTester SchemaKeyspace.TABLES), KEYSPACE, currentTable()), - row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"))); + row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "1.1"))); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32, 'enabled' : true };"); @@ -742,7 +742,7 @@ public class CreateTest extends CQLTester SchemaKeyspace.TABLES), KEYSPACE, currentTable()), - row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"))); + row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "1.1"))); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'chunk_length_kb' : 32 };"); @@ -752,7 +752,27 @@ public class CreateTest extends CQLTester SchemaKeyspace.TABLES), KEYSPACE, currentTable()), - row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"))); + row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "1.1"))); + + createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 2 };"); + + assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", + SchemaConstants.SCHEMA_KEYSPACE_NAME, + SchemaKeyspace.TABLES), + KEYSPACE, + currentTable()), + row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "2.0"))); + + createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 0 };"); + + assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", + SchemaConstants.SCHEMA_KEYSPACE_NAME, + SchemaKeyspace.TABLES), + KEYSPACE, + currentTable()), + row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "0.0"))); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'sstable_compression' : '', 'chunk_length_kb' : 32 };"); @@ -798,6 +818,18 @@ public class CreateTest extends CQLTester "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_kb' : 32 , 'chunk_length_in_kb' : 32 };"); + assertThrowsConfigurationException("chunk_length_in_kb must be a power of 2", + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 31 };"); + + assertThrowsConfigurationException("Invalid negative or null chunk_length_in_kb", + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : -1 };"); + + assertThrowsConfigurationException("Invalid negative min_compress_ratio", + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH compression = { 'class' : 'SnappyCompressor', 'min_compress_ratio' : -1 };"); + assertThrowsConfigurationException("Unknown compression options unknownOption", "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'class' : 'SnappyCompressor', 'unknownOption' : 32 };"); diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreCQLHelperTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreCQLHelperTest.java index 59f9db33f8..6d680cec84 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreCQLHelperTest.java @@ -311,7 +311,7 @@ public class ColumnFamilyStoreCQLHelperTest extends CQLTester .bloomFilterFpChance(1.0) .comment("comment") .compaction(CompactionParams.lcs(Collections.singletonMap("sstable_size_in_mb", "1"))) - .compression(CompressionParams.lz4(1 << 16)) + .compression(CompressionParams.lz4(1 << 16, 1 << 15)) .dcLocalReadRepairChance(0.2) .crcCheckChance(0.3) .defaultTimeToLive(4) @@ -343,7 +343,7 @@ public class ColumnFamilyStoreCQLHelperTest extends CQLTester "\tAND comment = 'comment'\n" + "\tAND caching = { 'keys': 'ALL', 'rows_per_partition': 'NONE' }\n" + "\tAND compaction = { 'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'sstable_size_in_mb': '1' }\n" + - "\tAND compression = { 'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor' }\n" + + "\tAND compression = { 'chunk_length_in_kb': '64', 'min_compress_ratio': '2.0', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor' }\n" + "\tAND cdc = false\n" + "\tAND extensions = { 'ext1': 0x76616c31 };" )); diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java index ceba912636..06dd27883b 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java @@ -54,24 +54,40 @@ public class CompressedRandomAccessReaderTest public void testResetAndTruncate() throws IOException { // test reset in current buffer or previous one - testResetAndTruncate(File.createTempFile("normal", "1"), false, false, 10); - testResetAndTruncate(File.createTempFile("normal", "2"), false, false, CompressionParams.DEFAULT_CHUNK_LENGTH); + testResetAndTruncate(File.createTempFile("normal", "1"), false, false, 10, 0); + testResetAndTruncate(File.createTempFile("normal", "2"), false, false, CompressionParams.DEFAULT_CHUNK_LENGTH, 0); } @Test public void testResetAndTruncateCompressed() throws IOException { // test reset in current buffer or previous one - testResetAndTruncate(File.createTempFile("compressed", "1"), true, false, 10); - testResetAndTruncate(File.createTempFile("compressed", "2"), true, false, CompressionParams.DEFAULT_CHUNK_LENGTH); + testResetAndTruncate(File.createTempFile("compressed", "1"), true, false, 10, 0); + testResetAndTruncate(File.createTempFile("compressed", "2"), true, false, CompressionParams.DEFAULT_CHUNK_LENGTH, 0); } @Test public void testResetAndTruncateCompressedMmap() throws IOException { // test reset in current buffer or previous one - testResetAndTruncate(File.createTempFile("compressed_mmap", "1"), true, true, 10); - testResetAndTruncate(File.createTempFile("compressed_mmap", "2"), true, true, CompressionParams.DEFAULT_CHUNK_LENGTH); + testResetAndTruncate(File.createTempFile("compressed_mmap", "1"), true, true, 10, 0); + testResetAndTruncate(File.createTempFile("compressed_mmap", "2"), true, true, CompressionParams.DEFAULT_CHUNK_LENGTH, 0); + } + + @Test + public void testResetAndTruncateCompressedUncompressedChunks() throws IOException + { + // test reset in current buffer or previous one + testResetAndTruncate(File.createTempFile("compressed_uchunks", "1"), true, false, 10, 3); + testResetAndTruncate(File.createTempFile("compressed_uchunks", "2"), true, false, CompressionParams.DEFAULT_CHUNK_LENGTH, 3); + } + + @Test + public void testResetAndTruncateCompressedUncompressedChunksMmap() throws IOException + { + // test reset in current buffer or previous one + testResetAndTruncate(File.createTempFile("compressed_uchunks_mmap", "1"), true, true, 10, 3); + testResetAndTruncate(File.createTempFile("compressed_uchunks_mmap", "2"), true, true, CompressionParams.DEFAULT_CHUNK_LENGTH, 3); } @Test @@ -102,7 +118,7 @@ public class CompressedRandomAccessReaderTest } try (FileHandle.Builder builder = new FileHandle.Builder(filename) - .withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length())); + .withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true)); FileHandle fh = builder.complete(); RandomAccessReader reader = fh.createReader()) { @@ -120,7 +136,7 @@ public class CompressedRandomAccessReaderTest } } - private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize) throws IOException + private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize, double minCompressRatio) throws IOException { final String filename = f.getAbsolutePath(); MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); @@ -146,7 +162,7 @@ public class CompressedRandomAccessReaderTest } assert f.exists(); - CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length()) : null; + CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), true) : null; try (FileHandle.Builder builder = new FileHandle.Builder(filename).mmapped(usemmap).withCompressionMetadata(compressionMetadata); FileHandle fh = builder.complete(); RandomAccessReader reader = fh.createReader()) @@ -194,7 +210,7 @@ public class CompressedRandomAccessReaderTest } // open compression metadata and get chunk information - CompressionMetadata meta = new CompressionMetadata(metadata.getPath(), file.length()); + CompressionMetadata meta = new CompressionMetadata(metadata.getPath(), file.length(), true); CompressionMetadata.Chunk chunk = meta.chunkFor(0); try (FileHandle.Builder builder = new FileHandle.Builder(file.getPath()).withCompressionMetadata(meta); diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java index b5bf161c5f..dab2013c2d 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java @@ -55,13 +55,23 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest private void runTests(String testName) throws IOException { // Test small < 1 chunk data set - testWrite(File.createTempFile(testName + "_small", "1"), 25); + testWrite(File.createTempFile(testName + "_small", "1"), 25, false); // Test to confirm pipeline w/chunk-aligned data writes works - testWrite(File.createTempFile(testName + "_chunkAligned", "1"), CompressionParams.DEFAULT_CHUNK_LENGTH); + testWrite(File.createTempFile(testName + "_chunkAligned", "1"), CompressionParams.DEFAULT_CHUNK_LENGTH, false); // Test to confirm pipeline on non-chunk boundaries works - testWrite(File.createTempFile(testName + "_large", "1"), CompressionParams.DEFAULT_CHUNK_LENGTH * 3 + 100); + testWrite(File.createTempFile(testName + "_large", "1"), CompressionParams.DEFAULT_CHUNK_LENGTH * 3 + 100, false); + + // Test small < 1 chunk data set + testWrite(File.createTempFile(testName + "_small", "2"), 25, true); + + // Test to confirm pipeline w/chunk-aligned data writes works + testWrite(File.createTempFile(testName + "_chunkAligned", "2"), CompressionParams.DEFAULT_CHUNK_LENGTH, true); + + // Test to confirm pipeline on non-chunk boundaries works + testWrite(File.createTempFile(testName + "_large", "2"), CompressionParams.DEFAULT_CHUNK_LENGTH * 3 + 100, true); + } @Test @@ -85,7 +95,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest runTests("Snappy"); } - private void testWrite(File f, int bytesToTest) throws IOException + private void testWrite(File f, int bytesToTest, boolean useMemmap) throws IOException { final String filename = f.getAbsolutePath(); MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Collections.singletonList(BytesType.instance))); @@ -120,7 +130,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest } assert f.exists(); - try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length())); + try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true)); FileHandle fh = builder.complete(); RandomAccessReader reader = fh.createReader()) { @@ -215,6 +225,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest private static class TestableCSW extends TestableSW { final File offsetsFile; + static final int MAX_COMPRESSED = BUFFER_SIZE * 10; // Always compress for this test. private TestableCSW() throws IOException { @@ -226,7 +237,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest { this(file, offsetsFile, new CompressedSequentialWriter(file, offsetsFile.getPath(), null, SequentialWriterOption.DEFAULT, - CompressionParams.lz4(BUFFER_SIZE), + CompressionParams.lz4(BUFFER_SIZE, MAX_COMPRESSED), new MetadataCollector(new ClusteringComparator(UTF8Type.instance)))); } @@ -255,6 +266,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest Assert.assertTrue(offsets.readUTF().endsWith("LZ4Compressor")); Assert.assertEquals(0, offsets.readInt()); Assert.assertEquals(BUFFER_SIZE, offsets.readInt()); + Assert.assertEquals(MAX_COMPRESSED, offsets.readInt()); Assert.assertEquals(fullContents.length, offsets.readLong()); Assert.assertEquals(2, offsets.readInt()); Assert.assertEquals(0, offsets.readLong()); diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 8996f2aa06..3a963b1be8 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -71,7 +71,7 @@ public class LegacySSTableTest * See {@link #testGenerateSstables()} to generate sstables. * Take care on commit as you need to add the sstable files using {@code git add -f} */ - public static final String[] legacyVersions = {"mc", "mb", "ma"}; + public static final String[] legacyVersions = {"na", "mc", "mb", "ma"}; // 1200 chars static final String longString = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + diff --git a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java index 5051230d5b..cffb49d0f7 100644 --- a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java +++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java @@ -313,7 +313,7 @@ public class MmappedRegionsTest writer.finish(); } - CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length()); + CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), true); try(ChannelProxy channel = new ChannelProxy(f); MmappedRegions regions = MmappedRegions.map(channel, metadata)) { diff --git a/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java index f3d0b52224..19e28fddf7 100644 --- a/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java +++ b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java @@ -32,6 +32,8 @@ import org.apache.cassandra.io.util.SequentialWriterOption; import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.streaming.compress.CompressedInputStream; import org.apache.cassandra.streaming.compress.CompressionInfo; @@ -54,17 +56,17 @@ public class CompressedInputStreamTest @Test public void testCompressedRead() throws Exception { - testCompressedReadWith(new long[]{0L}, false, false); - testCompressedReadWith(new long[]{1L}, false, false); - testCompressedReadWith(new long[]{100L}, false, false); + testCompressedReadWith(new long[]{0L}, false, false, 0); + testCompressedReadWith(new long[]{1L}, false, false, 0); + testCompressedReadWith(new long[]{100L}, false, false, 0); - testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, false); + testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, false, 0); } @Test(expected = EOFException.class) public void testTruncatedRead() throws Exception { - testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, true, false); + testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, true, false, 0); } /** @@ -73,14 +75,36 @@ public class CompressedInputStreamTest @Test(timeout = 30000) public void testException() throws Exception { - testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, true); + testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, true, 0); + } + + @Test + public void testCompressedReadUncompressedChunks() throws Exception + { + testCompressedReadWith(new long[]{0L}, false, false, 3); + testCompressedReadWith(new long[]{1L}, false, false, 3); + testCompressedReadWith(new long[]{100L}, false, false, 3); + + testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, false, 3); + } + + @Test(expected = EOFException.class) + public void testTruncatedReadUncompressedChunks() throws Exception + { + testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, true, false, 3); + } + + @Test(timeout = 30000) + public void testCorruptedReadUncompressedChunks() throws Exception + { + testCompressedReadWith(new long[]{1L, 122L, 123L, 124L, 456L}, false, true, 3); } /** * @param valuesToCheck array of longs of range(0-999) * @throws Exception */ - private void testCompressedReadWith(long[] valuesToCheck, boolean testTruncate, boolean testException) throws Exception + private void testCompressedReadWith(long[] valuesToCheck, boolean testTruncate, boolean testException, double minCompressRatio) throws Exception { assert valuesToCheck != null && valuesToCheck.length > 0; @@ -89,7 +113,7 @@ public class CompressedInputStreamTest Descriptor desc = new Descriptor(parentDir, "ks", "cf", 1); File tmp = new File(desc.filenameFor(Component.DATA)); MetadataCollector collector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); - CompressionParams param = CompressionParams.snappy(32); + CompressionParams param = CompressionParams.snappy(32, minCompressRatio); Map index = new HashMap(); try (CompressedSequentialWriter writer = new CompressedSequentialWriter(tmp, desc.filenameFor(Component.COMPRESSION_INFO),