CFS.loadNewSSTables() broken for pre-3.0 sstables

patch by Robert Stupp; reviewed by Blake Eggleston for CASSANDRA-10237

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -64,6 +64,7 @@ import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.*;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.metrics.TableMetrics;
@@ -724,7 +725,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                                                descriptor.ksname,
                                                descriptor.cfname,
                                                fileIndexGenerator.incrementAndGet(),
-                                               descriptor.formatType);
+                                               descriptor.formatType,
+                                               descriptor.digestComponent);
             }
             while (new File(newDescriptor.filenameFor(Component.DATA)).exists());
 
@@ -808,7 +810,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                                          keyspace.getName(),
                                          name,
                                          fileIndexGenerator.incrementAndGet(),
-                                         format);
+                                         format,
+                                         Component.digestFor(BigFormat.latestVersion.uncompressedChecksumType()));
         return desc.filenameFor(Component.DATA);
     }
 

src/java/org/apache/cassandra/db/Directories.java

@@ -695,6 +695,24 @@ public class Directories
                             previous = new HashSet<>();
                             components.put(pair.left, previous);
                         }
+                        else if (pair.right.type == Component.Type.DIGEST)
+                        {
+                            if (pair.right != pair.left.digestComponent)
+                            {
+                                // Need to update the DIGEST component as it might be set to another
+                                // digest type as a guess. This may happen if the first component is
+                                // not the DIGEST (but the Data component for example), so the digest
+                                // type is _guessed_ from the Version.
+                                // Although the Version explicitly defines the digest type, it doesn't
+                                // seem to be true under all circumstances. Generated sstables from a
+                                // post 2.1.8 snapshot produced Digest.sha1 files although Version
+                                // defines Adler32.
+                                // TL;DR this piece of code updates the digest component to be "correct".
+                                components.remove(pair.left);
+                                Descriptor updated = pair.left.withDigestComponent(pair.right);
+                                components.put(updated, previous);
+                            }
+                        }
                         previous.add(pair.right);
                         nbFiles++;
                         return false;

src/java/org/apache/cassandra/db/SerializationHeader.java

@@ -413,7 +413,7 @@ public class SerializationHeader
         }
 
         // For SSTables
-        public void serialize(Component header, DataOutputPlus out) throws IOException
+        public void serialize(Version version, Component header, DataOutputPlus out) throws IOException
         {
             EncodingStats.serializer.serialize(header.stats, out);
 
@@ -447,7 +447,7 @@ public class SerializationHeader
         }
 
         // For SSTables
-        public int serializedSize(Component header)
+        public int serializedSize(Version version, Component header)
         {
             int size = EncodingStats.serializer.serializedSize(header.stats);
 

src/java/org/apache/cassandra/db/compaction/Verifier.java

@@ -91,12 +91,13 @@ public class Verifier implements Closeable
         outputHandler.output(String.format("Checking computed hash of %s ", sstable));
 
 
-        // Verify will use the adler32 Digest files, which works for both compressed and uncompressed sstables
+        // Verify will use the Digest files, which works for both compressed and uncompressed sstables
         try
         {
             validator = null;
 
-            if (new File(sstable.descriptor.filenameFor(Component.DIGEST)).exists())
+            if (sstable.descriptor.digestComponent != null &&
+                new File(sstable.descriptor.filenameFor(sstable.descriptor.digestComponent)).exists())
             {
                 validator = DataIntegrityMetadata.fileDigestValidator(sstable.descriptor);
                 validator.validate();

src/java/org/apache/cassandra/io/sstable/Component.java

@@ -22,6 +22,7 @@ import java.util.EnumSet;
 
 import com.google.common.base.Objects;
 
+import org.apache.cassandra.utils.ChecksumType;
 import org.apache.cassandra.utils.Pair;
 
 /**
@@ -34,6 +35,7 @@ public class Component
     public static final char separator = '-';
 
     final static EnumSet<Type> TYPES = EnumSet.allOf(Type.class);
+
     public enum Type
     {
         // the base data for an sstable: the remaining components can be regenerated
@@ -48,7 +50,7 @@ public class Component
         // statistical metadata about the content of the sstable
         STATS("Statistics.db"),
         // holds adler32 checksum of the data file
-        DIGEST(new String[] { "Digest.crc32", "Digest.adler32" }),
+        DIGEST("Digest.crc32", "Digest.adler32", "Digest.sha1"),
         // holds the CRC32 for chunks in an a uncompressed file.
         CRC("CRC.db"),
         // holds SSTable Index Summary (sampling of Index component)
@@ -57,14 +59,14 @@ public class Component
         TOC("TOC.txt"),
         // custom component, used by e.g. custom compaction strategy
         CUSTOM(new String[] { null });
-
+        
         final String[] repr;
         Type(String repr)
         {
             this(new String[] { repr });
         }
 
-        Type(String[] repr)
+        Type(String... repr)
         {
             this.repr = repr;
         }
@@ -85,11 +87,28 @@ public class Component
     public final static Component FILTER = new Component(Type.FILTER);
     public final static Component COMPRESSION_INFO = new Component(Type.COMPRESSION_INFO);
     public final static Component STATS = new Component(Type.STATS);
-    public final static Component DIGEST = new Component(Type.DIGEST);
+    private static final String digestCrc32 = "Digest.crc32";
+    private static final String digestAdler32 = "Digest.adler32";
+    private static final String digestSha1 = "Digest.sha1";
+    public final static Component DIGEST_CRC32 = new Component(Type.DIGEST, digestCrc32);
+    public final static Component DIGEST_ADLER32 = new Component(Type.DIGEST, digestAdler32);
+    public final static Component DIGEST_SHA1 = new Component(Type.DIGEST, digestSha1);
     public final static Component CRC = new Component(Type.CRC);
     public final static Component SUMMARY = new Component(Type.SUMMARY);
     public final static Component TOC = new Component(Type.TOC);
 
+    public static Component digestFor(ChecksumType checksumType)
+    {
+        switch (checksumType)
+        {
+            case Adler32:
+                return DIGEST_ADLER32;
+            case CRC32:
+                return DIGEST_CRC32;
+        }
+        throw new AssertionError();
+    }
+
     public final Type type;
     public final String name;
     public final int hashCode;
@@ -97,6 +116,7 @@ public class Component
     public Component(Type type)
     {
         this(type, type.repr[0]);
+        assert type.repr.length == 1;
         assert type != Type.CUSTOM;
     }
 
@@ -138,7 +158,14 @@ public class Component
             case FILTER:            component = Component.FILTER;                       break;
             case COMPRESSION_INFO:  component = Component.COMPRESSION_INFO;             break;
             case STATS:             component = Component.STATS;                        break;
-            case DIGEST:            component = Component.DIGEST;                       break;
+            case DIGEST:            switch (path.right)
+                                    {
+                                        case digestCrc32:   component = Component.DIGEST_CRC32;     break;
+                                        case digestAdler32: component = Component.DIGEST_ADLER32;   break;
+                                        case digestSha1:    component = Component.DIGEST_SHA1;      break;
+                                        default:            throw new IllegalArgumentException("Invalid digest component " + path.right);
+                                    }
+                                    break;
             case CRC:               component = Component.CRC;                          break;
             case SUMMARY:           component = Component.SUMMARY;                      break;
             case TOC:               component = Component.TOC;                          break;

src/java/org/apache/cassandra/io/sstable/Descriptor.java

@@ -21,6 +21,7 @@ import java.io.File;
 import java.util.*;
 import java.util.regex.Pattern;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.CharMatcher;
 import com.google.common.base.Objects;
 
@@ -28,6 +29,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer;
 import org.apache.cassandra.io.sstable.metadata.LegacyMetadataSerializer;
 import org.apache.cassandra.io.sstable.metadata.MetadataSerializer;
@@ -52,27 +54,34 @@ public class Descriptor
     public final String cfname;
     public final int generation;
     public final SSTableFormat.Type formatType;
+    /** digest component - might be {@code null} for old, legacy sstables */
+    public final Component digestComponent;
     private final int hashCode;
 
     /**
      * A descriptor that assumes CURRENT_VERSION.
      */
+    @VisibleForTesting
     public Descriptor(File directory, String ksname, String cfname, int generation)
     {
-        this(DatabaseDescriptor.getSSTableFormat().info.getLatestVersion(), directory, ksname, cfname, generation, DatabaseDescriptor.getSSTableFormat());
+        this(DatabaseDescriptor.getSSTableFormat().info.getLatestVersion(), directory, ksname, cfname, generation, DatabaseDescriptor.getSSTableFormat(), null);
     }
 
+    /**
+     * Constructor for sstable writers only.
+     */
     public Descriptor(File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
     {
-        this(formatType.info.getLatestVersion(), directory, ksname, cfname, generation, formatType);
+        this(formatType.info.getLatestVersion(), directory, ksname, cfname, generation, formatType, Component.digestFor(BigFormat.latestVersion.uncompressedChecksumType()));
     }
 
+    @VisibleForTesting
     public Descriptor(String version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
     {
-        this(formatType.info.getVersion(version), directory, ksname, cfname, generation, formatType);
+        this(formatType.info.getVersion(version), directory, ksname, cfname, generation, formatType, Component.digestFor(BigFormat.latestVersion.uncompressedChecksumType()));
     }
 
-    public Descriptor(Version version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
+    public Descriptor(Version version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType, Component digestComponent)
     {
         assert version != null && directory != null && ksname != null && cfname != null && formatType.info.getLatestVersion().getClass().equals(version.getClass());
         this.version = version;
@@ -81,18 +90,24 @@ public class Descriptor
         this.cfname = cfname;
         this.generation = generation;
         this.formatType = formatType;
+        this.digestComponent = digestComponent;
 
         hashCode = Objects.hashCode(version, directory, generation, ksname, cfname, formatType);
     }
 
     public Descriptor withGeneration(int newGeneration)
     {
-        return new Descriptor(version, directory, ksname, cfname, newGeneration, formatType);
+        return new Descriptor(version, directory, ksname, cfname, newGeneration, formatType, digestComponent);
     }
 
     public Descriptor withFormatType(SSTableFormat.Type newType)
     {
-        return new Descriptor(newType.info.getLatestVersion(), directory, ksname, cfname, generation, newType);
+        return new Descriptor(newType.info.getLatestVersion(), directory, ksname, cfname, generation, newType, digestComponent);
+    }
+
+    public Descriptor withDigestComponent(Component newDigestComponent)
+    {
+        return new Descriptor(version, directory, ksname, cfname, generation, formatType, newDigestComponent);
     }
 
     public String tmpFilenameFor(Component component)
@@ -102,7 +117,7 @@ public class Descriptor
 
     public String filenameFor(Component component)
     {
-        return filenameFor(component.name());
+        return baseFilename() + separator + component.name();
     }
 
     public String baseFilename()
@@ -139,16 +154,6 @@ public class Descriptor
         return formatType.info;
     }
 
-    /**
-     * @param suffix A component suffix, such as 'Data.db'/'Index.db'/etc
-     * @return A filename for this descriptor with the given suffix.
-     */
-    public String filenameFor(String suffix)
-    {
-        return baseFilename() + separator + suffix;
-    }
-
-
     /** Return any temporary files found in the directory */
     public List<File> getTemporaryFiles()
     {
@@ -299,7 +304,10 @@ public class Descriptor
         }
         assert tokenStack.isEmpty() : "Invalid file name " + name + " in " + directory;
 
-        return Pair.create(new Descriptor(version, parentDirectory, ksname, cfname, generation, fmt), component);
+        return Pair.create(new Descriptor(version, parentDirectory, ksname, cfname, generation, fmt,
+                                          // _assume_ version from version
+                                          Component.digestFor(version.uncompressedChecksumType())),
+                           component);
     }
 
     public IMetadataSerializer getMetadataSerializer()

src/java/org/apache/cassandra/io/sstable/SSTable.java

@@ -23,7 +23,6 @@ import java.nio.charset.Charset;
 import java.util.*;
 import java.util.concurrent.CopyOnWriteArraySet;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicates;
 import com.google.common.collect.Collections2;
 import com.google.common.collect.Sets;
@@ -220,9 +219,17 @@ public abstract class SSTable
         Set<Component> components = Sets.newHashSetWithExpectedSize(knownTypes.size());
         for (Component.Type componentType : knownTypes)
         {
-            Component component = new Component(componentType);
-            if (new File(desc.filenameFor(component)).exists())
-                components.add(component);
+            if (componentType == Component.Type.DIGEST)
+            {
+                if (desc.digestComponent != null && new File(desc.filenameFor(desc.digestComponent)).exists())
+                    components.add(desc.digestComponent);
+            }
+            else
+            {
+                Component component = new Component(componentType);
+                if (new File(desc.filenameFor(component)).exists())
+                    components.add(component);
+            }
         }
         return components;
     }

src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java

@@ -35,6 +35,7 @@ import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
 import org.apache.cassandra.io.sstable.metadata.MetadataType;
@@ -131,7 +132,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional
                 Component.STATS,
                 Component.SUMMARY,
                 Component.TOC,
-                Component.DIGEST));
+                Component.digestFor(BigFormat.latestVersion.uncompressedChecksumType())));
 
         if (metadata.params.bloomFilterFpChance < 1.0)
             components.add(Component.FILTER);

src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java

@@ -40,7 +40,7 @@ import org.apache.cassandra.utils.ChecksumType;
 public class BigFormat implements SSTableFormat
 {
     public static final BigFormat instance = new BigFormat();
-    public static final BigVersion latestVersion = new BigVersion(BigVersion.current_version);
+    public static final Version latestVersion = new BigVersion(BigVersion.current_version);
     private static final SSTableReader.Factory readerFactory = new ReaderFactory();
     private static final SSTableWriter.Factory writerFactory = new WriterFactory();
 
@@ -123,6 +123,8 @@ public class BigFormat implements SSTableFormat
         // la (2.2.0): new file name format
         // ma (3.0.0): swap bf hash order
         //             store rows natively
+        //
+        // NOTE: when adding a new version, please add that to LegacySSTableTest, too.
 
         private final boolean isLatestVersion;
         private final boolean hasSamplingLevel;

src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java

@@ -341,7 +341,7 @@ public class BigTableWriter extends SSTableWriter
         File file = new File(desc.filenameFor(Component.STATS));
         try (SequentialWriter out = SequentialWriter.open(file))
         {
-            desc.getMetadataSerializer().serialize(components, out);
+            desc.getMetadataSerializer().serialize(components, out, desc.version);
             out.setDescriptor(desc).finish();
         }
         catch (IOException e)

src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java

@@ -72,13 +72,13 @@ public class CompactionMetadata extends MetadataComponent
 
     public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
     {
-        public int serializedSize(CompactionMetadata component) throws IOException
+        public int serializedSize(Version version, CompactionMetadata component) throws IOException
         {
             byte[] serializedCardinality = component.cardinalityEstimator.getBytes();
             return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length;
         }
 
-        public void serialize(CompactionMetadata component, DataOutputPlus out) throws IOException
+        public void serialize(Version version, CompactionMetadata component, DataOutputPlus out) throws IOException
         {
             ByteBufferUtil.writeWithLength(component.cardinalityEstimator.getBytes(), out);
         }

src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java

@@ -31,21 +31,26 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
     /**
      * Calculate and return serialized size.
      *
+     *
+     *
+     * @param version
      * @param component MetadataComponent to calculate serialized size
      * @return serialized size of this component
      * @throws IOException
      */
-    int serializedSize(T component) throws IOException;
+    int serializedSize(Version version, T component) throws IOException;
 
     /**
      * Serialize metadata component to given output.
      *
      *
+     *
+     * @param version
      * @param component MetadataComponent to serialize
      * @param out  serialize destination
      * @throws IOException
      */
-    void serialize(T component, DataOutputPlus out) throws IOException;
+    void serialize(Version version, T component, DataOutputPlus out) throws IOException;
 
     /**
      * Deserialize metadata component from given input.

src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java

@@ -22,6 +22,7 @@ import java.util.EnumSet;
 import java.util.Map;
 
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.util.DataOutputPlus;
 
 /**
@@ -35,9 +36,10 @@ public interface IMetadataSerializer
      *
      * @param components Metadata components to serialize
      * @param out
+     * @param version
      * @throws IOException
      */
-    void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException;
+    void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out, Version version) throws IOException;
 
     /**
      * Deserialize specified metadata components from given descriptor.

src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java

@@ -27,6 +27,7 @@ import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.commitlog.ReplayPosition;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.service.ActiveRepairService;
@@ -44,7 +45,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
      * Legacy serialization is only used for SSTable level reset.
      */
     @Override
-    public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+    public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out, Version version) throws IOException
     {
         ValidationMetadata validation = (ValidationMetadata) components.get(MetadataType.VALIDATION);
         StatsMetadata stats = (StatsMetadata) components.get(MetadataType.STATS);

src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java

@@ -27,6 +27,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.io.util.FileDataInput;
@@ -49,7 +50,7 @@ public class MetadataSerializer implements IMetadataSerializer
 {
     private static final Logger logger = LoggerFactory.getLogger(MetadataSerializer.class);
 
-    public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+    public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out, Version version) throws IOException
     {
         // sort components by type
         List<MetadataComponent> sortedComponents = Lists.newArrayList(components.values());
@@ -66,12 +67,12 @@ public class MetadataSerializer implements IMetadataSerializer
             out.writeInt(type.ordinal());
             // serialize position
             out.writeInt(lastPosition);
-            lastPosition += type.serializer.serializedSize(component);
+            lastPosition += type.serializer.serializedSize(version, component);
         }
         // serialize components
         for (MetadataComponent component : sortedComponents)
         {
-            component.getType().serializer.serialize(component, out);
+            component.getType().serializer.serialize(version, component, out);
         }
     }
 
@@ -115,14 +116,13 @@ public class MetadataSerializer implements IMetadataSerializer
         }
         for (MetadataType type : types)
         {
-            MetadataComponent component = null;
             Integer offset = toc.get(type);
             if (offset != null)
             {
                 in.seek(offset);
-                component = type.serializer.deserialize(descriptor.version, in);
+                MetadataComponent component = type.serializer.deserialize(descriptor.version, in);
+                components.put(type, component);
             }
-            components.put(type, component);
         }
         return components;
     }
@@ -152,7 +152,7 @@ public class MetadataSerializer implements IMetadataSerializer
         String filePath = descriptor.tmpFilenameFor(Component.STATS);
         try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(filePath)))
         {
-            serialize(currentComponents, out);
+            serialize(currentComponents, out, descriptor.version);
             out.flush();
         }
         // we cant move a file on top of another file in windows:

src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java

@@ -227,13 +227,16 @@ public class StatsMetadata extends MetadataComponent
 
     public static class StatsMetadataSerializer implements IMetadataComponentSerializer<StatsMetadata>
     {
-        public int serializedSize(StatsMetadata component) throws IOException
+        public int serializedSize(Version version, StatsMetadata component) throws IOException
         {
             int size = 0;
             size += EstimatedHistogram.serializer.serializedSize(component.estimatedPartitionSize);
             size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount);
             size += ReplayPosition.serializer.serializedSize(component.replayPosition);
-            size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
+            if (version.storeRows())
+                size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
+            else
+                size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
             size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime);
             size += TypeSizes.sizeof(component.sstableLevel);
             // min column names
@@ -245,21 +248,26 @@ public class StatsMetadata extends MetadataComponent
             for (ByteBuffer value : component.maxClusteringValues)
                 size += 2 + value.remaining(); // with short length
             size += TypeSizes.sizeof(component.hasLegacyCounterShards);
-            size += 8 + 8; // totalColumnsSet, totalRows
+            if (version.storeRows())
+                size += 8 + 8; // totalColumnsSet, totalRows
             return size;
         }
 
-        public void serialize(StatsMetadata component, DataOutputPlus out) throws IOException
+        public void serialize(Version version, StatsMetadata component, DataOutputPlus out) throws IOException
         {
             EstimatedHistogram.serializer.serialize(component.estimatedPartitionSize, out);
             EstimatedHistogram.serializer.serialize(component.estimatedColumnCount, out);
             ReplayPosition.serializer.serialize(component.replayPosition, out);
             out.writeLong(component.minTimestamp);
             out.writeLong(component.maxTimestamp);
-            out.writeInt(component.minLocalDeletionTime);
+            if (version.storeRows())
+                out.writeInt(component.minLocalDeletionTime);
             out.writeInt(component.maxLocalDeletionTime);
-            out.writeInt(component.minTTL);
-            out.writeInt(component.maxTTL);
+            if (version.storeRows())
+            {
+                out.writeInt(component.minTTL);
+                out.writeInt(component.maxTTL);
+            }
             out.writeDouble(component.compressionRatio);
             StreamingHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out);
             out.writeInt(component.sstableLevel);
@@ -272,8 +280,11 @@ public class StatsMetadata extends MetadataComponent
                 ByteBufferUtil.writeWithShortLength(value, out);
             out.writeBoolean(component.hasLegacyCounterShards);
 
-            out.writeLong(component.totalColumnsSet);
-            out.writeLong(component.totalRows);
+            if (version.storeRows())
+            {
+                out.writeLong(component.totalColumnsSet);
+                out.writeLong(component.totalRows);
+            }
         }
 
         public StatsMetadata deserialize(Version version, DataInputPlus in) throws IOException

src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java

@@ -71,12 +71,12 @@ public class ValidationMetadata extends MetadataComponent
 
     public static class ValidationMetadataSerializer implements IMetadataComponentSerializer<ValidationMetadata>
     {
-        public int serializedSize(ValidationMetadata component) throws IOException
+        public int serializedSize(Version version, ValidationMetadata component) throws IOException
         {
             return TypeSizes.sizeof(component.partitioner) + 8;
         }
 
-        public void serialize(ValidationMetadata component, DataOutputPlus out) throws IOException
+        public void serialize(Version version, ValidationMetadata component, DataOutputPlus out) throws IOException
         {
             out.writeUTF(component.partitioner);
             out.writeDouble(component.bloomFilterFPChance);

src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java

@@ -110,7 +110,7 @@ public class DataIntegrityMetadata
         {
             this.descriptor = descriptor;
             checksum = descriptor.version.uncompressedChecksumType().newInstance();
-            digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DIGEST)));
+            digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.digestFor(descriptor.version.uncompressedChecksumType()))));
             dataReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DATA)));
             try
             {
@@ -206,7 +206,9 @@ public class DataIntegrityMetadata
 
         public void writeFullChecksum(Descriptor descriptor)
         {
-            File outFile = new File(descriptor.filenameFor(Component.DIGEST));
+            if (descriptor.digestComponent == null)
+                throw new NullPointerException("Null digest component for " + descriptor.ksname + '.' + descriptor.cfname + " file " + descriptor.baseFilename());
+            File outFile = new File(descriptor.filenameFor(descriptor.digestComponent));
             try (BufferedWriter out =Files.newBufferedWriter(outFile.toPath(), Charsets.UTF_8))
             {
                 out.write(String.valueOf(fullChecksum.getValue()));

src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java

@@ -23,7 +23,6 @@ import java.util.List;
 import java.util.UUID;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
@@ -162,7 +161,7 @@ public class FileMessageHeader
             out.writeInt(header.sstableLevel);
 
             if (version >= StreamMessage.VERSION_30)
-                SerializationHeader.serializer.serialize(header.header, out);
+                SerializationHeader.serializer.serialize(header.version, header.header, out);
         }
 
         public FileMessageHeader deserialize(DataInputPlus in, int version) throws IOException
@@ -211,7 +210,7 @@ public class FileMessageHeader
             size += TypeSizes.sizeof(header.sstableLevel);
 
             if (version >= StreamMessage.VERSION_30)
-                size += SerializationHeader.serializer.serializedSize(header.header);
+                size += SerializationHeader.serializer.serializedSize(header.version, header.header);
 
             return size;
         }

test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust/legacy_tables-legacy_jb_clust-jb-1-TOC.txt

@@ -0,0 +1,7 @@
+CompressionInfo.db
+Statistics.db
+Filter.db
+Data.db
+TOC.txt
+Index.db
+Summary.db

test/data/legacy-sstables/jb/legacy_tables/legacy_jb_clust_counter/legacy_tables-legacy_jb_clust_counter-jb-1-TOC.txt

@@ -0,0 +1,7 @@
+CompressionInfo.db
+Statistics.db
+Filter.db
+Data.db
+TOC.txt
+Index.db
+Summary.db

test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple/legacy_tables-legacy_jb_simple-jb-1-TOC.txt

@@ -0,0 +1,7 @@
+CompressionInfo.db
+Statistics.db
+Filter.db
+Data.db
+TOC.txt
+Index.db
+Summary.db

test/data/legacy-sstables/jb/legacy_tables/legacy_jb_simple_counter/legacy_tables-legacy_jb_simple_counter-jb-1-TOC.txt

@@ -0,0 +1,7 @@
+CompressionInfo.db
+Statistics.db
+Filter.db
+Data.db
+TOC.txt
+Index.db
+Summary.db

test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-Digest.sha1

@@ -0,0 +1 @@
+4293822635

test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust/legacy_tables-legacy_ka_clust-ka-1-TOC.txt

@@ -0,0 +1,8 @@
+Index.db
+Digest.sha1
+CompressionInfo.db
+Data.db
+Statistics.db
+Summary.db
+TOC.txt
+Filter.db

test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-Digest.sha1

@@ -0,0 +1 @@
+2539906592

test/data/legacy-sstables/ka/legacy_tables/legacy_ka_clust_counter/legacy_tables-legacy_ka_clust_counter-ka-1-TOC.txt

@@ -0,0 +1,8 @@
+Index.db
+Digest.sha1
+CompressionInfo.db
+Data.db
+Statistics.db
+Summary.db
+TOC.txt
+Filter.db

test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-Digest.sha1

@@ -0,0 +1 @@
+2802392853

test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple/legacy_tables-legacy_ka_simple-ka-1-TOC.txt

@@ -0,0 +1,8 @@
+Index.db
+Digest.sha1
+CompressionInfo.db
+Data.db
+Statistics.db
+Summary.db
+TOC.txt
+Filter.db

test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-Digest.sha1

@@ -0,0 +1 @@
+3671794375

test/data/legacy-sstables/ka/legacy_tables/legacy_ka_simple_counter/legacy_tables-legacy_ka_simple_counter-ka-1-TOC.txt

@@ -0,0 +1,8 @@
+Index.db
+Digest.sha1
+CompressionInfo.db
+Data.db
+Statistics.db
+Summary.db
+TOC.txt
+Filter.db

test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-Digest.adler32

@@ -0,0 +1 @@
+1633775217

test/data/legacy-sstables/la/legacy_tables/legacy_la_clust/la-1-big-TOC.txt

@@ -0,0 +1,8 @@
+CompressionInfo.db
+Digest.adler32
+Filter.db
+Summary.db
+Data.db
+Statistics.db
+TOC.txt
+Index.db

test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-Digest.adler32

@@ -0,0 +1 @@
+287946299

test/data/legacy-sstables/la/legacy_tables/legacy_la_clust_counter/la-1-big-TOC.txt

@@ -0,0 +1,8 @@
+CompressionInfo.db
+Digest.adler32
+Filter.db
+Summary.db
+Data.db
+Statistics.db
+TOC.txt
+Index.db

test/data/legacy-sstables/la/legacy_tables/legacy_la_simple/la-1-big-Digest.adler32

@@ -0,0 +1 @@
+4239203875