diff --git a/CHANGES.txt b/CHANGES.txt index 0efa8b91d4..bea407e08e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,7 @@ 4.0 + * Checksum sstable metadata (CASSANDRA-13321, CASSANDRA-13593) * Add result set metadata to prepared statement MD5 hash calculation (CASSANDRA-10786) * Refactor GcCompactionTest to avoid boxing (CASSANDRA-13941) - * Checksum sstable metadata (CASSANDRA-13321) * Expose recent histograms in JmxHistograms (CASSANDRA-13642) * Fix buffer length comparison when decompressing in netty-based streaming (CASSANDRA-13899) * Properly close StreamCompressionInputStream to release any ByteBuf (CASSANDRA-13906) diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java index f17157cb13..9499ff3fe2 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java @@ -19,11 +19,9 @@ package org.apache.cassandra.io.sstable.metadata; import java.io.*; import java.util.*; +import java.util.zip.CRC32; import com.google.common.collect.Lists; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,11 +39,13 @@ import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.utils.FBUtilities; +import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt; + /** - * Metadata serializer for SSTables {@code version >= 'k'}. + * Metadata serializer for SSTables {@code version >= 'na'}. * *
- * File format := | number of components (4 bytes) | toc | component1 | c1 hash | component2 | c2 hash | ... |
+ * File format := | number of components (4 bytes) | crc | toc | crc | component1 | c1 crc | component2 | c2 crc | ... |
  * toc         := | component type (4 bytes) | position of component |
  * 
* @@ -54,31 +54,40 @@ import org.apache.cassandra.utils.FBUtilities; public class MetadataSerializer implements IMetadataSerializer { private static final Logger logger = LoggerFactory.getLogger(MetadataSerializer.class); - private static final HashFunction hashFunction = Hashing.md5(); + + private static final int CHECKSUM_LENGTH = 4; // CRC32 public void serialize(Map components, DataOutputPlus out, Version version) throws IOException { boolean checksum = version.hasMetadataChecksum(); + CRC32 crc = new CRC32(); // sort components by type List sortedComponents = Lists.newArrayList(components.values()); Collections.sort(sortedComponents); // write number of component out.writeInt(components.size()); + updateChecksumInt(crc, components.size()); + maybeWriteChecksum(crc, out, version); + // build and write toc - int lastPosition = 4 + (8 * sortedComponents.size()); + int lastPosition = 4 + (8 * sortedComponents.size()) + (checksum ? 2 * CHECKSUM_LENGTH : 0); Map sizes = new EnumMap<>(MetadataType.class); for (MetadataComponent component : sortedComponents) { MetadataType type = component.getType(); // serialize type out.writeInt(type.ordinal()); + updateChecksumInt(crc, type.ordinal()); // serialize position out.writeInt(lastPosition); + updateChecksumInt(crc, lastPosition); int size = type.serializer.serializedSize(version, component); - lastPosition += size + (checksum ? 8 : 0); // checksum is long + lastPosition += size + (checksum ? CHECKSUM_LENGTH : 0); sizes.put(type, size); } + maybeWriteChecksum(crc, out, version); + // serialize components for (MetadataComponent component : sortedComponents) { @@ -89,11 +98,18 @@ public class MetadataSerializer implements IMetadataSerializer bytes = dob.getData(); } out.write(bytes); - if (checksum) - out.writeLong(hashFunction.hashBytes(bytes).asLong()); + + crc.reset(); crc.update(bytes); + maybeWriteChecksum(crc, out, version); } } + private static void maybeWriteChecksum(CRC32 crc, DataOutputPlus out, Version version) throws IOException + { + if (version.hasMetadataChecksum()) + out.writeInt((int) crc.getValue()); + } + public Map deserialize( Descriptor descriptor, EnumSet types) throws IOException { Map components; @@ -120,66 +136,87 @@ public class MetadataSerializer implements IMetadataSerializer return deserialize(descriptor, EnumSet.of(type)).get(type); } - public Map deserialize(Descriptor descriptor, FileDataInput in, EnumSet types) throws IOException + public Map deserialize(Descriptor descriptor, + FileDataInput in, + EnumSet selectedTypes) + throws IOException { - int totalSize = (int) in.bytesRemaining(); + boolean isChecksummed = descriptor.version.hasMetadataChecksum(); + CRC32 crc = new CRC32(); + + /* + * Read TOC + */ + + int length = (int) in.bytesRemaining(); + + int count = in.readInt(); + updateChecksumInt(crc, count); + maybeValidateChecksum(crc, in, descriptor); + + int[] ordinals = new int[count]; + int[] offsets = new int[count]; + int[] lengths = new int[count]; + + for (int i = 0; i < count; i++) + { + ordinals[i] = in.readInt(); + updateChecksumInt(crc, ordinals[i]); + + offsets[i] = in.readInt(); + updateChecksumInt(crc, offsets[i]); + } + maybeValidateChecksum(crc, in, descriptor); + + lengths[count - 1] = length - offsets[count - 1]; + for (int i = 0; i < count - 1; i++) + lengths[i] = offsets[i + 1] - offsets[i]; + + /* + * Read components + */ + + MetadataType[] allMetadataTypes = MetadataType.values(); + Map components = new EnumMap<>(MetadataType.class); - // read number of components - int numComponents = in.readInt(); - // read toc - Map toc = new EnumMap<>(MetadataType.class); - MetadataType[] values = MetadataType.values(); - Map lengths = new EnumMap<>(MetadataType.class); - int start = 0; - MetadataType lastType = null; - for (int i = 0; i < numComponents; i++) - { - int metadataTypeId = in.readInt(); - int position = in.readInt(); - toc.put(values[metadataTypeId], position); - if (lastType != null) - lengths.put(lastType, position - start); - start = position; - lastType = values[metadataTypeId]; - } - lengths.put(lastType, totalSize - start); - for (MetadataType type : types) + for (int i = 0; i < count; i++) { - Integer offset = toc.get(type); - if (offset != null) + MetadataType type = allMetadataTypes[ordinals[i]]; + + if (!selectedTypes.contains(type)) { - in.seek(offset); - - if (descriptor.version.hasMetadataChecksum()) - { - int size = lengths.get(type) - 8; // 8 bytes checksum - byte[] bytes = new byte[size]; - in.readFully(bytes); - MetadataComponent component; - try (DataInputBuffer dib = new DataInputBuffer(bytes)) - { - component = type.serializer.deserialize(descriptor.version, dib); - } - long writtenChecksum = in.readLong(); - if (writtenChecksum != hashFunction.hashBytes(bytes).asLong()) - { - String filename = descriptor.filenameFor(Component.STATS); - throw new CorruptSSTableException(new IOException("Checksums do not match for " + filename), filename); - } - components.put(type, component); - } - else - { - MetadataComponent component = type.serializer.deserialize(descriptor.version, in); - components.put(type, component); - } - + in.skipBytes(lengths[i]); + continue; } + + byte[] buffer = new byte[isChecksummed ? lengths[i] - CHECKSUM_LENGTH : lengths[i]]; + in.readFully(buffer); + + crc.reset(); crc.update(buffer); + maybeValidateChecksum(crc, in, descriptor); + + components.put(type, type.serializer.deserialize(descriptor.version, new DataInputBuffer(buffer))); } + return components; } + private static void maybeValidateChecksum(CRC32 crc, FileDataInput in, Descriptor descriptor) throws IOException + { + if (!descriptor.version.hasMetadataChecksum()) + return; + + int actualChecksum = (int) crc.getValue(); + int expectedChecksum = in.readInt(); + + if (actualChecksum != expectedChecksum) + { + String filename = descriptor.filenameFor(Component.STATS); + throw new CorruptSSTableException(new IOException("Checksums do not match for " + filename), filename); + } + } + public void mutateLevel(Descriptor descriptor, int newLevel) throws IOException { logger.trace("Mutating {} to level {}", descriptor.filenameFor(Component.STATS), newLevel); diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db index c7fc204d4a..935d8630bf 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db index 36f9800814..9982578898 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 index 9adb8f59fd..1e6cbfc56c 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 @@ -1 +1 @@ -829376603 \ No newline at end of file +4285275084 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db index d3b366dd18..25b063cf25 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db index f750d82823..59969c279d 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt index 0251bd9110..734a80d3cd 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db -CompressionInfo.db -Statistics.db -Summary.db TOC.txt -Digest.crc32 Filter.db +CompressionInfo.db +Summary.db +Data.db +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db index 1ef9177bde..6d3ac8edf2 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db index 1d130d4c01..fd6aebb5cb 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 index a9900dada9..239b37240a 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 @@ -1 +1 @@ -818525950 \ No newline at end of file +3704038982 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db index e04d3dbda9..2e209512e5 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db index f2da58dcf5..cd90137dcf 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt index 0251bd9110..734a80d3cd 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db -CompressionInfo.db -Statistics.db -Summary.db TOC.txt -Digest.crc32 Filter.db +CompressionInfo.db +Summary.db +Data.db +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db index 449f50794c..d6a1ff866f 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db index 55be94d288..03babd4bac 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 index 0608cacb4c..7bd9e39838 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 @@ -1 +1 @@ -1278737554 \ No newline at end of file +1093063834 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db index 646d4d416b..5ad5400ad1 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db index 569221411f..9347a2c7d7 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt index 0251bd9110..734a80d3cd 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db -CompressionInfo.db -Statistics.db -Summary.db TOC.txt -Digest.crc32 Filter.db +CompressionInfo.db +Summary.db +Data.db +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db index 405b9fb381..1b1f2dad67 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db index 1f4e26a4ec..1976981ac7 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 index 3ae0adf46d..22ed33ba43 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 @@ -1 +1 @@ -2193718788 \ No newline at end of file +2742231118 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db index 041117e33b..022bc3da0a 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db index e899252d5b..d393159b43 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt index 0251bd9110..734a80d3cd 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db -CompressionInfo.db -Statistics.db -Summary.db TOC.txt -Digest.crc32 Filter.db +CompressionInfo.db +Summary.db +Data.db +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db index cd52dab59a..414ff070a5 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 index 1275575bff..1dfec4f889 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 @@ -1 +1 @@ -3074491452 \ No newline at end of file +2354437953 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db index 4205870c6b..33f0516456 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt index 0251bd9110..734a80d3cd 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db -CompressionInfo.db -Statistics.db -Summary.db TOC.txt -Digest.crc32 Filter.db +CompressionInfo.db +Summary.db +Data.db +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db index 8adbf543b9..b9b836f818 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 index 138bc9c2d9..669bbcf624 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 @@ -1 +1 @@ -1825853914 \ No newline at end of file +1732739494 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db index 7e2561351d..b112e583e0 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt index 0251bd9110..734a80d3cd 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db -CompressionInfo.db -Statistics.db -Summary.db TOC.txt -Digest.crc32 Filter.db +CompressionInfo.db +Summary.db +Data.db +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db index f3e6359e3b..ed0dadb825 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 index 2ad4fa0d59..1480f5db63 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 @@ -1 +1 @@ -3361465012 \ No newline at end of file +2023874595 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db index 04cacf9835..124dacdd5c 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt index 0251bd9110..734a80d3cd 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db -CompressionInfo.db -Statistics.db -Summary.db TOC.txt -Digest.crc32 Filter.db +CompressionInfo.db +Summary.db +Data.db +Statistics.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db index 72772f06d0..366315ff23 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 index b44a7339ea..839639c3bd 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 @@ -1 +1 @@ -2410392183 \ No newline at end of file +138332031 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db index c431a5913a..613f2ad10f 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt index 0251bd9110..734a80d3cd 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db -CompressionInfo.db -Statistics.db -Summary.db TOC.txt -Digest.crc32 Filter.db +CompressionInfo.db +Summary.db +Data.db +Statistics.db +Digest.crc32 diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index 95d2a7747c..7d8150c1a2 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -98,8 +98,13 @@ public class MetadataSerializerTest String partitioner = RandomPartitioner.class.getCanonicalName(); double bfFpChance = 0.1; - Map originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, null, SerializationHeader.make(cfm, Collections.emptyList())); - return originalMetadata; + return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, SerializationHeader.make(cfm, Collections.emptyList())); + } + + @Test + public void testMaReadMa() throws IOException + { + testOldReadsNew("ma", "ma"); } @Test @@ -114,6 +119,12 @@ public class MetadataSerializerTest testOldReadsNew("ma", "mc"); } + @Test + public void testMbReadMb() throws IOException + { + testOldReadsNew("mb", "mb"); + } + @Test public void testMbReadMc() throws IOException { @@ -121,9 +132,15 @@ public class MetadataSerializerTest } @Test - public void testNaReadMc() throws IOException + public void testMcReadMc() throws IOException { - testOldReadsNew("mc", "na"); + testOldReadsNew("mc", "mc"); + } + + @Test + public void testNaReadNa() throws IOException + { + testOldReadsNew("na", "na"); } public void testOldReadsNew(String oldV, String newV) throws IOException @@ -146,11 +163,9 @@ public class MetadataSerializerTest for (MetadataType type : MetadataType.values()) { assertEquals(deserializedLa.get(type), deserializedLb.get(type)); - if (!originalMetadata.get(type).equals(deserializedLb.get(type))) - { - // Currently only STATS can be different. Change if no longer the case - assertEquals(MetadataType.STATS, type); - } + + if (MetadataType.STATS != type) + assertEquals(originalMetadata.get(type), deserializedLb.get(type)); } } }