From 76ca69717c3072996095057115133ca3b6ca25cd Mon Sep 17 00:00:00 2001 From: Stefania Alborghetti Date: Mon, 10 Aug 2015 20:21:47 -0500 Subject: [PATCH] 2.1 format sstable filenames with tmp are not handled by 3.0 patch by stefania; reviewed by yukim for CASSANDRA-10006 --- CHANGES.txt | 1 + .../apache/cassandra/db/SystemKeyspace.java | 6 ++-- .../cassandra/io/sstable/Descriptor.java | 27 ++++++++++++------ ...ctions_in_progress-ka-1-CompressionInfo.db | Bin 0 -> 43 bytes ...ystem-compactions_in_progress-ka-1-Data.db | Bin 0 -> 146 bytes ...m-compactions_in_progress-ka-1-Digest.sha1 | 1 + ...tem-compactions_in_progress-ka-1-Filter.db | Bin 0 -> 16 bytes ...stem-compactions_in_progress-ka-1-Index.db | Bin 0 -> 30 bytes ...compactions_in_progress-ka-1-Statistics.db | Bin 0 -> 4450 bytes ...em-compactions_in_progress-ka-1-Summary.db | Bin 0 -> 116 bytes ...ystem-compactions_in_progress-ka-1-TOC.txt | 8 ++++++ .../test-foo-ka-3-CompressionInfo.db | Bin 0 -> 43 bytes .../test-foo-ka-3-Data.db | Bin 0 -> 141 bytes .../test-foo-ka-3-Digest.sha1 | 1 + .../test-foo-ka-3-Filter.db | Bin 0 -> 176 bytes .../test-foo-ka-3-Index.db | Bin 0 -> 90 bytes .../test-foo-ka-3-Statistics.db | Bin 0 -> 4458 bytes .../test-foo-ka-3-Summary.db | Bin 0 -> 80 bytes .../test-foo-ka-3-TOC.txt | 8 ++++++ .../test-foo-tmp-ka-4-Data.db | Bin 0 -> 141 bytes .../test-foo-tmp-ka-4-Index.db | Bin 0 -> 90 bytes .../test-foo-tmplink-ka-4-Data.db | Bin 0 -> 141 bytes .../test-foo-tmplink-ka-4-Index.db | Bin 0 -> 90 bytes .../cassandra/db/SystemKeyspaceTest.java | 19 +++++++++--- 24 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-CompressionInfo.db create mode 100644 test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Data.db create mode 100644 test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Digest.sha1 create mode 100644 test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Filter.db create mode 100644 test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Index.db create mode 100644 test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Statistics.db create mode 100644 test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-Summary.db create mode 100644 test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-TOC.txt create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-CompressionInfo.db create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Data.db create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Digest.sha1 create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Filter.db create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Index.db create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Statistics.db create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Summary.db create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-TOC.txt create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Data.db create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Index.db create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Data.db create mode 100644 test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Index.db diff --git a/CHANGES.txt b/CHANGES.txt index eacc1102e2..b23c2d15a0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,6 +12,7 @@ * Bytecode inspection for Java-UDFs (CASSANDRA-9890) * Use byte to serialize MT hash length (CASSANDRA-9792) * Replace usage of Adler32 with CRC32 (CASSANDRA-8684) + * Fix migration to new format from 2.1 SSTable (CASSANDRA-10006) Merged from 2.2: * Add checksum to saved cache files (CASSANDRA-9265) * Log warning when using an aggregate without partition key (CASSANDRA-9737) diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index bc0be65141..d24b18be5b 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -1307,7 +1307,7 @@ public final class SystemKeyspace } /** - * Check data directories for old files that can be removed when migrating from 2.2 to 3.0, + * Check data directories for old files that can be removed when migrating from 2.1 or 2.2 to 3.0, * these checks can be removed in 4.0, see CASSANDRA-7066 */ public static void migrateDataDirs() @@ -1323,13 +1323,13 @@ public final class SystemKeyspace { for (File cfdir : ksdir.listFiles((d, n) -> d.isDirectory())) { - if (Descriptor.isLegacyFile(cfdir.getName())) + if (Descriptor.isLegacyFile(cfdir)) { FileUtils.deleteRecursive(cfdir); } else { - FileUtils.delete(cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(n))); + FileUtils.delete(cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(new File(d, n)))); } } } diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java index 519f14eafa..38829df7da 100644 --- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java +++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java @@ -19,6 +19,7 @@ package org.apache.cassandra.io.sstable; import java.io.File; import java.util.*; +import java.util.regex.Pattern; import com.google.common.base.CharMatcher; import com.google.common.base.Objects; @@ -30,7 +31,6 @@ import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer; import org.apache.cassandra.io.sstable.metadata.LegacyMetadataSerializer; import org.apache.cassandra.io.sstable.metadata.MetadataSerializer; -import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.Pair; import static org.apache.cassandra.io.sstable.Component.separator; @@ -163,20 +163,29 @@ public class Descriptor } /** - * Files obsoleted by CASSANDRA-7066 : - * - temporary files used to start with either tmp or tmplink - * - system.compactions_in_progress sstable files + * Files obsoleted by CASSANDRA-7066 : temporary files and compactions_in_progress. We support + * versions 2.1 (ka) and 2.2 (la). + * Temporary files have tmp- or tmplink- at the beginning for 2.2 sstables or after ks-cf- for 2.1 sstables */ - public static boolean isLegacyFile(String fileName) + + private final static String LEGACY_COMP_IN_PROG_REGEX_STR = "^compactions_in_progress(\\-[\\d,a-f]{32})?$"; + private final static Pattern LEGACY_COMP_IN_PROG_REGEX = Pattern.compile(LEGACY_COMP_IN_PROG_REGEX_STR); + private final static String LEGACY_TMP_REGEX_STR = "^((.*)\\-(.*)\\-)?tmp(link)?\\-(la|ka)\\-(\\d)*\\-(.*)$"; + private final static Pattern LEGACY_TMP_REGEX = Pattern.compile(LEGACY_TMP_REGEX_STR); + + public static boolean isLegacyFile(File file) { - return fileName.startsWith("compactions_in_progress") || - fileName.startsWith("tmp") || - fileName.startsWith("tmplink"); + if (file.isDirectory()) + return file.getParentFile() != null && + file.getParentFile().getName().equalsIgnoreCase("system") && + LEGACY_COMP_IN_PROG_REGEX.matcher(file.getName()).matches(); + else + return LEGACY_TMP_REGEX.matcher(file.getName()).matches(); } public static boolean isValidFile(String fileName) { - return fileName.endsWith(".db") && !isLegacyFile(fileName); + return fileName.endsWith(".db") && !LEGACY_TMP_REGEX.matcher(fileName).matches(); } /** diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-CompressionInfo.db b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-CompressionInfo.db new file mode 100644 index 0000000000000000000000000000000000000000..d9446df698830392b8b7ac8663448e5af83649d3 GIT binary patch literal 43 fcmZSJ^@%cZ&d)6So7xAa+7|NsC01_nk3L1qqyLROi(Oy(Xhco=@N ziZBQ!=jW8>=A|X(X697J=OyN*Dl>dzWKPS^XW(RD%gie%Eh+vlz{3#C7^V=)NxmS6dIBrvCB$g!-=ylW2I1jjZ-((rnn9XV`Q^tiAijf zA6qP@ksp_N`)mI4?|rvt&-e9wzRx|+^ZoHWpPe8G zGU0a_B<+~A&6PG$+Kj@OY|~1OVYMMO-Ka@5(OF=&fCy3?)PPfFfN~uRz#vbrl38!=k@x5ai3ju z-ER>(s0BSF96h28tw}*gA4X5FLnrh_FAqfLI2J1@1qK4CTF%YiW*x?D!C)5r5uGv* zeHiB_vps^in1t(Dyn&Xb+=2FTInICi{aX-MTq^{7e5e3>CE3B=T}E)PE`P8x4dHT}Q?o(F*kTN1$`cX7PKI&i!ri>^TXDQz#ovtsc676w@%p7~A<-Wp#CKG-=C zTzwV$r?ndTllk=dLfbQK>@M(@VJE>i8^%$KBQL@GRot(9K;yEYT$mrSnx)<}F3ze&d8;4&uZ&eYH!`l&(ytwYhgY4rKs=Gcpq6@6SaSID9q>n%FY8cKJe6@ zH`GJN>(5b-e4IR)I{aSx8R|*PHxGt?#B7w`iW=(F88hhmXB7#T==x{&eM-9iMWglkWpcjQc6w?avCPb@ zdx_=4@2cadJ32Pe@lkv$DyH8L&(Z$vF{Hg$$=ST~#6Iyyn@q&M%L?Dq=T|;ay`j&q zy05QNllFZL1*_@zAn>HNRzu>!*W1-}yoP^@jVmDWvGZ-AG#`kHeA}cXajoT5?Rw(b zhbm9g`&n$86h-HAR-&%Fh_ugNQQh1|T>7lFuAI33*zrnwzt)DZmgqlmrrv(*=XBz@ z%r8H-{jreatm8ZQ?;K1d=QyhW%DG&>qu|=ZJ3)%Mb1v+lVx&;T*@AZGT-YD(oD2HH zopXglf4Fn52-+X+oa=vbE*7uK4=EM`ZtN5kgIdLC{VV7G^%m(54XyP=(X0(}M~<8( WZ7k;DnnwNBp5AwTAA1gTSAGFY$<0kH N0BV7m&cFZ`0RVbA8#Dj_ literal 0 HcmV?d00001 diff --git a/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-TOC.txt b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-TOC.txt new file mode 100644 index 0000000000..7dc893035d --- /dev/null +++ b/test/data/migration-sstables/2.1/system/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b/system-compactions_in_progress-ka-1-TOC.txt @@ -0,0 +1,8 @@ +Data.db +TOC.txt +Filter.db +Statistics.db +Summary.db +Index.db +Digest.sha1 +CompressionInfo.db diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-CompressionInfo.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-CompressionInfo.db new file mode 100644 index 0000000000000000000000000000000000000000..b867db835334606772a540ff901230281e7604a5 GIT binary patch literal 43 fcmZSJ^@%cZ&d)6sJ)P$5D9 literal 0 HcmV?d00001 diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Digest.sha1 b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Digest.sha1 new file mode 100644 index 0000000000..2f4daa930c --- /dev/null +++ b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Digest.sha1 @@ -0,0 +1 @@ +4283441474 \ No newline at end of file diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Filter.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..a5bdd8e7518d35ef5caa1bdd65f21a48e19717ad GIT binary patch literal 176 zcmY+7I}U&_3_~9kNQ^A(J__Pw938iA1o=`l-QNOg literal 0 HcmV?d00001 diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Index.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Index.db new file mode 100644 index 0000000000000000000000000000000000000000..5d7131519853889502e9da513b2468d11f72e3e2 GIT binary patch literal 90 rcmZQzVPIfj1!4pMi7-OB4qzH2!UPp*fQx`cLEr$G1o4=`EU0DxJM{sj literal 0 HcmV?d00001 diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Statistics.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Statistics.db new file mode 100644 index 0000000000000000000000000000000000000000..aeb2bb8d0521f42332dd406f2a53c13f7961f050 GIT binary patch literal 4458 zcmeI$|4++rmnEU)5opC+v>Vf3 zJ)egrzj4{dTRVp8%A1X8tv znZM0?%x!^SKJXJdc?J3u&QIR<1act}*RyaBEhe`?dzlR9zwA*f(MNU>O&E0CP->BX7i@#703#^CrV&c_k<@9y*XL7j z|LN67U2vx|n)-a=i3!vXN-T$|zg_ECOw3RAO0A<#TeOI-f4(xlhOU3!UZkSyU&ye& zXdu@My%*-?5sPNtdX!i;@u4P`y1#!HeLwQ=g~jyy!)=OB-vZKJS#mk=DzQi0nIbJE2>PIov8q$7*A%83VKJYuItJjh|u%%B!-`B*?(Xsg?pO$0`ru%`& zh<8mYlIuEO*Y6-+da~*QeV(>&0l}*A2Mfh*6}lVXb&cL<^s9KQFZOPa6nvpE*KBjo(qP9bM3ic zJY0J&*Pe?^i2rNP#mA{FLB*VJ%N|Z1&^_?7$SR$CcvEiTUg-x8-Sdrqi`2bz(nYNP E0Sb9kJpcdz literal 0 HcmV?d00001 diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Summary.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-ka-3-Summary.db new file mode 100644 index 0000000000000000000000000000000000000000..602ec06f74e8fb36fe02f037af28d3823a9d629f GIT binary patch literal 80 zcmZQzU}#`qU|sJ)P$5D9 literal 0 HcmV?d00001 diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Index.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmp-ka-4-Index.db new file mode 100644 index 0000000000000000000000000000000000000000..5d7131519853889502e9da513b2468d11f72e3e2 GIT binary patch literal 90 rcmZQzVPIfj1!4pMi7-OB4qzH2!UPp*fQx`cLEr$G1o4=`EU0DxJM{sj literal 0 HcmV?d00001 diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Data.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..f14d86d3ecad3a48701a8af891abd5e0c6556233 GIT binary patch literal 141 zcmZ=@WMDYVz{0@5z*_(R|NjOCMg~D<4u(QjnY)a|6~R0V32Y3^Wr;bO46LFIjjSH2 zIXU?Z3=RzPj1CM;F?aqwobtqhfkl*o6DY(461vMErTrT&#Nxof?91p^c^pVX6fuMK S{r7vE1Xk1lGqPs2d>sJ)P$5D9 literal 0 HcmV?d00001 diff --git a/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Index.db b/test/data/migration-sstables/2.1/test/foo-0094ac203e7411e59149ef9f87394ca6/test-foo-tmplink-ka-4-Index.db new file mode 100644 index 0000000000000000000000000000000000000000..5d7131519853889502e9da513b2468d11f72e3e2 GIT binary patch literal 90 rcmZQzVPIfj1!4pMi7-OB4qzH2!UPp*fQx`cLEr$G1o4=`EU0DxJM{sj literal 0 HcmV?d00001 diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java index d58985c576..9fc3503d97 100644 --- a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java @@ -153,9 +153,20 @@ public class SystemKeyspaceTest } @Test - public void testMigrateDataDirs() throws IOException + public void testMigrateDataDirs_2_1() throws IOException { - Path migrationSSTableRoot = Paths.get(System.getProperty(MIGRATION_SSTABLES_ROOT), "2.2"); + testMigrateDataDirs("2.1"); + } + + @Test + public void testMigrateDataDirs_2_2() throws IOException + { + testMigrateDataDirs("2.2"); + } + + private void testMigrateDataDirs(String version) throws IOException + { + Path migrationSSTableRoot = Paths.get(System.getProperty(MIGRATION_SSTABLES_ROOT), version); Path dataDir = Paths.get(DatabaseDescriptor.getAllDataFileLocations()[0]); FileUtils.copyDirectory(migrationSSTableRoot.toFile(), dataDir.toFile()); @@ -178,13 +189,13 @@ public class SystemKeyspaceTest { for (File cfdir : ksdir.listFiles((d, n) -> d.isDirectory())) { - if (Descriptor.isLegacyFile(cfdir.getName())) + if (Descriptor.isLegacyFile(cfdir)) { ret++; } else { - File[] legacyFiles = cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(n)); + File[] legacyFiles = cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(new File(d, n))); ret += legacyFiles.length; } }