2.1 format sstable filenames with tmp are not handled by 3.0

patch by stefania; reviewed by yukim for CASSANDRA-10006

CHANGES.txt

@@ -12,6 +12,7 @@
  * Bytecode inspection for Java-UDFs (CASSANDRA-9890)
  * Use byte to serialize MT hash length (CASSANDRA-9792)
  * Replace usage of Adler32 with CRC32 (CASSANDRA-8684)
+ * Fix migration to new format from 2.1 SSTable (CASSANDRA-10006)
 Merged from 2.2:
  * Add checksum to saved cache files (CASSANDRA-9265)
  * Log warning when using an aggregate without partition key (CASSANDRA-9737)

src/java/org/apache/cassandra/db/SystemKeyspace.java

@@ -1307,7 +1307,7 @@ public final class SystemKeyspace
     }
 
     /**
-     * Check data directories for old files that can be removed when migrating from 2.2 to 3.0,
+     * Check data directories for old files that can be removed when migrating from 2.1 or 2.2 to 3.0,
      * these checks can be removed in 4.0, see CASSANDRA-7066
      */
     public static void migrateDataDirs()
@@ -1323,13 +1323,13 @@ public final class SystemKeyspace
             {
                 for (File cfdir : ksdir.listFiles((d, n) -> d.isDirectory()))
                 {
-                    if (Descriptor.isLegacyFile(cfdir.getName()))
+                    if (Descriptor.isLegacyFile(cfdir))
                     {
                         FileUtils.deleteRecursive(cfdir);
                     }
                     else
                     {
-                        FileUtils.delete(cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(n)));
+                        FileUtils.delete(cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(new File(d, n))));
                     }
                 }
             }

src/java/org/apache/cassandra/io/sstable/Descriptor.java

@@ -19,6 +19,7 @@ package org.apache.cassandra.io.sstable;
 
 import java.io.File;
 import java.util.*;
+import java.util.regex.Pattern;
 
 import com.google.common.base.CharMatcher;
 import com.google.common.base.Objects;
@@ -30,7 +31,6 @@ import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer;
 import org.apache.cassandra.io.sstable.metadata.LegacyMetadataSerializer;
 import org.apache.cassandra.io.sstable.metadata.MetadataSerializer;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.utils.Pair;
 
 import static org.apache.cassandra.io.sstable.Component.separator;
@@ -163,20 +163,29 @@ public class Descriptor
     }
 
     /**
-     *  Files obsoleted by CASSANDRA-7066 :
-     *  - temporary files used to start with either tmp or tmplink
-     *  - system.compactions_in_progress sstable files
+     *  Files obsoleted by CASSANDRA-7066 : temporary files and compactions_in_progress. We support
+     *  versions 2.1 (ka) and 2.2 (la).
+     *  Temporary files have tmp- or tmplink- at the beginning for 2.2 sstables or after ks-cf- for 2.1 sstables
      */
-    public static boolean isLegacyFile(String fileName)
+
+    private final static String LEGACY_COMP_IN_PROG_REGEX_STR = "^compactions_in_progress(\\-[\\d,a-f]{32})?$";
+    private final static Pattern LEGACY_COMP_IN_PROG_REGEX = Pattern.compile(LEGACY_COMP_IN_PROG_REGEX_STR);
+    private final static String LEGACY_TMP_REGEX_STR = "^((.*)\\-(.*)\\-)?tmp(link)?\\-(la|ka)\\-(\\d)*\\-(.*)$";
+    private final static Pattern LEGACY_TMP_REGEX = Pattern.compile(LEGACY_TMP_REGEX_STR);
+
+    public static boolean isLegacyFile(File file)
     {
-        return fileName.startsWith("compactions_in_progress") ||
-               fileName.startsWith("tmp") ||
-               fileName.startsWith("tmplink");
+        if (file.isDirectory())
+            return file.getParentFile() != null &&
+                   file.getParentFile().getName().equalsIgnoreCase("system") &&
+                   LEGACY_COMP_IN_PROG_REGEX.matcher(file.getName()).matches();
+        else
+            return LEGACY_TMP_REGEX.matcher(file.getName()).matches();
     }
 
     public static boolean isValidFile(String fileName)
     {
-        return fileName.endsWith(".db") && !isLegacyFile(fileName);
+        return fileName.endsWith(".db") && !LEGACY_TMP_REGEX.matcher(fileName).matches();
     }
 
     /**

test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Digest.sha1

@@ -0,0 +1 @@
+3043896114

test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-TOC.txt

@@ -0,0 +1,8 @@
+Data.db
+TOC.txt
+Filter.db
+Statistics.db
+Summary.db
+Index.db
+Digest.sha1
+CompressionInfo.db

test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Digest.sha1

@@ -0,0 +1 @@
+4283441474

test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-TOC.txt

@@ -0,0 +1,8 @@
+Data.db
+TOC.txt
+Filter.db
+Statistics.db
+Summary.db
+Index.db
+Digest.sha1
+CompressionInfo.db

test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java

@@ -153,9 +153,20 @@ public class SystemKeyspaceTest
     }
 
     @Test
-    public void testMigrateDataDirs() throws IOException
+    public void testMigrateDataDirs_2_1() throws IOException
     {
-        Path migrationSSTableRoot = Paths.get(System.getProperty(MIGRATION_SSTABLES_ROOT), "2.2");
+        testMigrateDataDirs("2.1");
+    }
+
+    @Test
+    public void testMigrateDataDirs_2_2() throws IOException
+    {
+        testMigrateDataDirs("2.2");
+    }
+
+    private void testMigrateDataDirs(String version) throws IOException
+    {
+        Path migrationSSTableRoot = Paths.get(System.getProperty(MIGRATION_SSTABLES_ROOT), version);
         Path dataDir = Paths.get(DatabaseDescriptor.getAllDataFileLocations()[0]);
 
         FileUtils.copyDirectory(migrationSSTableRoot.toFile(), dataDir.toFile());
@@ -178,13 +189,13 @@ public class SystemKeyspaceTest
             {
                 for (File cfdir : ksdir.listFiles((d, n) -> d.isDirectory()))
                 {
-                    if (Descriptor.isLegacyFile(cfdir.getName()))
+                    if (Descriptor.isLegacyFile(cfdir))
                     {
                         ret++;
                     }
                     else
                     {
-                        File[] legacyFiles = cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(n));
+                        File[] legacyFiles = cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(new File(d, n)));
                         ret += legacyFiles.length;
                     }
                 }