From 12eb0571e2e65537ee17fbdab4859b429ad2189b Mon Sep 17 00:00:00 2001 From: Gary Dusbabek Date: Wed, 14 Apr 2010 19:51:55 +0000 Subject: [PATCH] implement a legacy sstable test. Patch by Stu Hood, reviewed by Gary Dusbabek. CASSANDRA-767 git-svn-id: https://svn.apache.org/repos/asf/cassandra/trunk@934149 13f79535-47bb-0310-9956-ffa450edef68 --- build.xml | 2 + .../b/Keyspace1/Standard1-b-0-Data.db | Bin 0 -> 39600 bytes .../b/Keyspace1/Standard1-b-0-Filter.db | Bin 0 -> 1773 bytes .../b/Keyspace1/Standard1-b-0-Index.db | Bin 0 -> 11700 bytes .../io/sstable/LegacySSTableTest.java | 113 ++++++++++++++++++ .../cassandra/io/sstable/SSTableUtils.java | 30 +++-- 6 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db create mode 100644 test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Filter.db create mode 100644 test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db create mode 100644 test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java diff --git a/build.xml b/build.xml index 1a09910570..d2fc678539 100644 --- a/build.xml +++ b/build.xml @@ -39,6 +39,7 @@ + @@ -369,6 +370,7 @@ + diff --git a/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..962ed01a99a8b82ab59c49174069a3135bac312e GIT binary patch literal 39600 zcma*rJE~>p0R`cPGm#6PfbQP^)0uNmqQPJZV<~9x_+2%F3Kx9Y%Ie4G`?ueI{P^*= zfBf%{KY#xE{p+7U|MTnj|Ni{*>yKYwKfnI{*I)noV*UL~hW&2#Tl+NoXy0aE?Pzvr zXR}kgnqAu6?AD%UkM=fuvES`C7`FWe!?xdG*!CL?+kS&#+ix&z`wfO|zrnEWHyF14 z2K#(J?Kc><{RYFf-(cAG8w}figJIimFl_q`hHbyWu<{RYFf-(cAG8w}figJIimu+R5tzrnEWHyF142E(@BVA%E>4BLK#VcTyoZ2Jv{ zZNI^=?Kjxx`?lX;*!CL?+kS&#+ix&z`wfO|zrnEWHyF142E(@BVA%E>?DMo^%5Dj z{bnzDS}&1d+i&)gr}YvUw*6)=d0H=#VcT!^lBe|&8MggqFL_!okzw0!_L8Ud5*fDr zW-obKFOgx}Z}yU>^%5Dj{bnzDS}&1d+i&)gr}YvUw*6)=d0H=#VcT!^lBe|&8Mggq zFL_!okzw0!_L8Ud5*fDrW-obKFOgx}Z}yU>^%5Dj{bnzDS}&1d+i&)gr}YvUw*6)= zd0H=#VcT!^lBe|&8MggqFL_!okzw0!_L8Ud5*fDrW-obKFOgx}Z}yU>^%5Dj{bnzD zS}&1d+i&)gr}YvUw*6)=d0H=#VcT!^lBe|&8MggqFL_!okzw0!_L8Ud5*fDrW-obK zFOgx}Z}yU>^%5Dj{bnzDS}&1d+i&)gr}YvUw*6)=d0H=#VcT!^lBe|&8MggqFL_%p zkzvzsGHmK4GHm)yhE2UhhE2c8u&I~Gu<17$HuVx2HvK01ysekWu@!*~kzw0!_L9+h zi45C*vzLt4OJvygo4sVTULwP`-|Qu$^%5Dj{bny2t(VBK?KgYLXuU*+ZNJ$|M(ZUq zZ2Qe#GFmT@VcT!^lF@pJ4BLLQmyFg+WZ3qby=1grBEz=d>?Nc15*fDrW-l47m&mZ~ zH+#uwy+np>zu8Mh>m@R5`^{c5S}&1d+i&)g(Rzst+kUf`jMhtJ*!G*fWVBu)!?xe- zC8PBc8MggqFBz?u$gu4?Nc15*fDr zW-pnom&mZ`HyJkd5*arACc~y)BEzQNWZ2Y8WZ3kZ44Zn144ZzFeP-(=GHm?O1H5*fDr zW-pnom&mZ~H+#u!y+np>zu8M>>m@R5`^{c5TQ8Ae+i&)g*?Nf#+kUf`%+^a}*!G*f zWVT)+!?xe-CA0Ms8MggqFPW{E$gu4?N!95*arACc~y)BEzQNWZ2Y8WZ3kZ44Zn144ZzFVN)-WVbgE2&uYCyhHbyuOIGV8 zGHmN4BLLQm#o%HWZ3qby=1jsBEz=d z>?N!95*fDrW-nQ-m&mZ~H+#uyy+np>zu8Mx>m@R5`^{dmS}&1d+i&)g)q05x+kUf` ztkz3p*!G*fWVK!*!?xe-C9Cxk8MggqFIlaZ$gu4N4BLLQm#o%HWZ3qb zy=1puBEzQNWZ2Y8WZ3kZ44Zn144ZzFVN)-WVbgCiZ0aR4Z2C?1*{zqzu?OPP5*fDrW-r;Tm&mZ~H+#u$y+np>zu8N6>m@R5`^{dmTQ8Ae+i&)g z-Fk@(+kUf`?AA+U*!G*fWVc=-!?xe-CA;+!8MggqFWIe^$gu4s1WZ3kZ44Zn144ZzFVN)-WVbgCiZ0aR4Z2C=xO}#{hO~1)Lr}YvUw*6)=Ijxt- zus1WZ3qbz2vlBBEz=d>?No55*fDrW-mFdm&mZ~H+#uxy+np>zu8Mp>m@R5`^{c* zS}&1d+i&)g(|U;v+kUf`oYqTZ*!G*fZNJ$|ZtEp7Z2Qe#a$7HvVcT!^ zlG}QT4BLLQm)zD%WZ3qbz2vrDBEz=d>?ODL5*fDrW-qy|m&mZ~H+#u#y+np>zu8M} z>m@R5`^{c*TQ8Ae+i&)g+j@x%+kUf`+}2BE*!G*fZNJ$|ZtEp7Z2Rrr F{{x-x)ZzdD literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Filter.db b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..64d8c5d5d17fe7356d3533e8e08ea8008f1bd484 GIT binary patch literal 1773 zcmVeb&~=B0C;c!007jmB^(Avj0NdnksZBbv3f3zF@Q7z z4GQ$bXg0^ft_!Ek<-q3o+`9mqRMnPXxl#mp`j`+Kw)Fh1(?w7LmWrq`g%n2v^CTmJ z{SMC%^B@>$1E$CzAy1FgA_mb6PeCdPv#Mynm;+2u85GKJ7B~bb77~#F$#OcSDH#C) zQ4Fb(H=}EU0xdeiXNz9i&!c4!kOo&v20{*!rI;8lqmmXw-ZBEDlZ_dMt(u)#t1HQE z)5fd>Vo;lV^8^GdMqBX{vIW>HFCa#3hz$ut2DB*v7m;4H3nhU~un8bojbtNCivskH z@UD?h!QhS@-clmM3#^2=ZiXQ68Pq$$K*XqmAi<2l9K;+Nc!1Cfx(rHo8XdYyU@9a` zkS?J81$~^sT7i;QC=gl+PNhIsfS8;Cw{bxLvZb11)p7g*_e0aF_ilu6%8Fqa4nk_D zJU%25JEcaj(T}SdkRyd?FiS-EDKJS&gYJkj>k%`qVFEhf6+{z^Gy*(`pQOD7=ItC= z96gfs9MNY0Z5crX9U*{$DuRp>jfsoGfaNV$h}THGNsjVFvfs@-s7|k~P=n6k@o<38 zuL+)52?l{UL;=BaCj(ex=TuJVTs|dWh^)jw39^zlOBxX)fH6=R1B6IoLCNSsGXA8q zfUXl1n??(^P!Yo$j={G+fg&(oLXigz#YnINK?G(HK24#|6T;$FKqAdl2Tl?xGOvS2 z^}1AD83J4e3p#`wDS;9w@m2|-sma&Ofos*o4s(V%?kBZ=oj1+{VXb?%nPJKph z2yqp~h7BkcbvuJn;;vLud0i1nXMzB%s0})W6uOgQAaB~TQVBt^23ApM4=y$MfLt_0 zwCfTv=r!md=Gb5{Kv3yGp=$^gU&0QjkHA~E7;;%g4z1%NQYE1%b%3UviYb5?P!!>y zD9vHoB?>gK5E$?ac;YBAl*R}kDDW%X@}{Z4Ey@{{u2E9fY*uMihXJ#Ns6cW7KYC=I zaLTK+v4}dM(eO7A~4${N5+j9IG@K1&}d&JvW-2V znE?n~H1Bbkfdq)&=%^sVDARI>1+M{si7OGYa&S`!csPo6gV^`d$E8q02)PC-lIQ_a z!+HpiM&yOxJ>bHOgHb>@)su02Om%<$2oMrM0dHuhxpXkEMSz5eV1NY)m_j*VU=-CE z<$)PY2n+~G&`8D?;bIRXAPX!&EERH))3y1XWDst!C<-|YHLQ>X;^Y8{jk9nv+(0yV zkO5#4Rv<{M1B;{`a{@f29AHYE;F1dD#LEQ4rJx9WyaYhnsA@kxr0!T1Cu7@V1bpR=zcmzTLa2blNzNE*5 zcK-sK0Ad~&FrftQV5K*Us)Ti@CT8*)dzf9)HXqYEE#~ls3k%mx`ALu)VVP$B)LtXgSI1^ zK|nwV4P?|D2rgY zKu?@&n7)lTIfg4)CV(I^RQT8I6uLPff(9W@NtHNFHNlVqioY`;oT~guMx>T98fGZ75Ralt4B?ckk!v-OU zn3$m?3ke$@U0A?_D6JDu{?Z^|3|QGf8xJI5Oqalc7^^KQ8ZF@P#9#>51{l=SWO%Z= zhJ&(+JHQVzT8Jhh*8&9fYfOkt2QqdCx*k-|Z`M`qjs(=OGwE_#i8vqu}iU12{r2`9Q z6#@%p)c^}-wF3)g4F%Y4L3|vrAU+B#h%W{f#Mc1};yZx_@gu;3*$Kdc*;&AX*`>gO z*$n|QEtuU6ESNn8ESQr7ESQr6ESOUcESS>-ESS>^ESNI^ESQ@DESQ@YV21^BD}e=b zTYv>~`+)^>r+@_sX~2Sn0$@QxHLxI|4Ooyc2rQTv8;RK*ZsNQQV8Og1V8OgvV8Of& zV8Og$V8Q%&V8Q%MV8Q$nV8Q%)V8Q&Z0O2Oi9|aaHNCXxv$OaZHC<7KOXap84=m8ci z7zY+4CIbr+bAbhk6#>FcOl$@gB=!Ld5+{KL-=_i#zRw31d|w4D_`Vfb@cjU=U||GU zurNJ9xQPo3fdva|fCUTNfdvbPfCY=wyJJyMP5tM}Y;)5`hKFvVjH5%76vS8i57N zdIE%-xNIC)usj)9usj!7u)G3Tu)G;qu)GgguzV6&kdg{4NXZ8lq*Mh6H!-CZSdcOR zELaf%7OY4I7OW@)7ObcN7OZFo7OWTo7Oac|7Oad$Vzz{vxUv{nu(A$Vu(A_auyO=g zuqpvquqq2!u&NYTu&M!Au&NtauxczoxQVGrz=G5qU_oj*upqSwSdiKaEJ&RI7OYMI z7Oc(#7ObuW7OZXw5N_h?eqh1sDPX~mX~2RX3xEYbRs#!uYy%ejI0!6Q6ALU@lL0JP zQxqWF#5J|Rf;An$f;Gdyg0y&GL0TrTAgu&gkX8>YNb3R?q>Ta#)+PoBH*sw?uwZQ& zuwZQ?uwZQuuwd;tuwY#>uwY#-uwY#UuwY#?uwY$ZfN&GnO#%ykN(C1Dln*TUsR~%| zQ!B9GrvYF=dIVUIo(?QXF9a5(*8~VRF})pFkUj(~SRV&0SRVxztS<%@tgizWtnUOC ztRDduY)AkWY{&`_ZsLYgV8Mn4V8MoNV8MnlV8O;DV8O;5V8O<6V8O;FV8O;-V8OymB50G7GOa}Kd>NU3Rtiy4OpXd|#7+5;?zjspv}CI<*NaceHHU~2`iU~4n5U~3<+VCy8XU|TA%U|T-0 zU|SWiU|TD&VB0`|a1(>S16Z&<9aylv5LmFi23WAY9ayk^2w0FA2P_D#O#llri-851 zbpgUn%d+LD&d%A!Hdq#l; zdlP{Ld$WNBd&>fZo4B_TSg^MTSg>~-Sdg6zEXd9U7Gzfd3$mMm1=)STg6v6P!Oy7y z!cF`+A6W2n6|msvR$#%;1HgiP5n#c-bYQ{0LSVtZ8eqY`c3{E2p#b3~?vDc&?2iHq z_7?*S_SXRm_ICmc_KyGyauR?AIa$DhoKj#xPD3PSYq*Iy-N1sJF<`-gBw)dT9ALqL za$v!MCSbvVUSPq231Gp&6kx%@ya3@Q9;^fw9Bcs=9P9@c9Gn6c97+Qg94Y`79I6Hu z9BKm=92x``}4{lJ2gQ^0~>(triO6a)x2 z@t0~~!7pvVf?o!K1*c+x1*bBA1*eLD1*d9(1*bZI1*e9A1qJZ|!c8p51QryO01FE0 zfdvI!z=DENV8Q7`V8Q8ZV8Q7!V8Q7|V8Q90NX)iy6Hku=3(h113(n*M3(iyk3(hnH z3(oWb3(iae3w})n7W|qIEcmr5K)8v&wgL-&9RL;-Mt}u{>A-@*LSR8*4X~iF9avB} z1S~ik2P`-n4G?bP*D@ zCZ10M7M#xk7Mw2!7MyPa7M$+|7Mz~|78Iob3ySi91x1y>f})lH;U*UK0}G0#fCay$ z0SkUB02cgK4J`Pr4OsBoAh6&F;EGRAm78ExE3yOPy1;yjQf=kK3f=ju;f=d;^ zf=kW7f=hh?!cDw12`u~aBfx^o>A-@^g}{Q#H37m+ zyxa~fxI6?bD2W3Wlth6ACB?vkk~&~PNhh$NWCU1nB>`A)B`ZLq?1y{O( z1y{y^1y_@R1y^%`1y{>~1y`GZ1y_541y?5`G26pUyp{qixRwVjxK;@)xYhzJxYiFW zxHbhWC`|(vlokLBN~?hdrELMiO)MP*7F>@77F^E&7F;g^7F@3d7F_QD7F-_&7Tkyj z7Tm}L7ThQa5N_g)dSJngE?~ipQDDK%L}0-EabQ7Na)59X z%W{DQWfj1JvSwgGSs$>VY!X=TM=G%3k9=UkA63ADKU#qWe+&c&H}O^kSa2&HSa7Ql zSa7QbSa7QySa53ySa3TISa3TEEVx|^EVx}4Al$^;oxp;Hm8p@K%y1Jc^MM7GRltJER$xKp0I=X;1X%De z9a!+N5LocA23YX09a!*iC_uQ0kK%v@kD|bWN5#N`M|Hr0N1eceMUv;7byt9J6RSso1N01Mu=0}I{_0SoHmfCcr@0O2Oq7Xu6G>wpFIoxpj#FenS7d?*JNd}sm|eCP!he3$?h zd`tlre9QwDe5?c(d~692ZsNy&V8O>JU_nC~u%Mv;SkO=nENEy07Bmb33qHjH3qEB4 z3qBPE2siOlEwJEI2e9DNFtFfrJh0$%Ca~ah39#UEJ+R<&7qH;-D6rs5Vt{ZHzhnao zzLWtAzBB?0zVrYKzKjD48k2zqjk&;r#tL9TV>7Uzu`fWliH(!Mg0HE-g0K0&g0EG; cg0HQ>g0BO>g8w4Gg8$Ni1^*QS3;wJ5AE{UVD*ylh literal 0 HcmV?d00001 diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java new file mode 100644 index 0000000000..d37585c3d1 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -0,0 +1,113 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +package org.apache.cassandra.io.sstable; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +import org.apache.cassandra.CleanupHelper; +import org.apache.cassandra.io.util.BufferedRandomAccessFile; +import org.apache.cassandra.db.DecoratedKey; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; + +/** + * Tests backwards compatibility for SSTables. Requires that older SSTables match up with the existing config file, + * and currently only tests specific cases for specific upgrades. + */ +public class LegacySSTableTest extends CleanupHelper +{ + public static final String LEGACY_SSTABLE_PROP = "legacy-sstable-root"; + public static final String KSNAME = "Keyspace1"; + public static final String CFNAME = "Standard1"; + + public static SortedMap TEST_DATA; + public static File LEGACY_SSTABLE_ROOT; + + @BeforeClass + public static void beforeClass() + { + String scp = System.getProperty(LEGACY_SSTABLE_PROP); + assert scp != null; + LEGACY_SSTABLE_ROOT = new File(scp); + assert LEGACY_SSTABLE_ROOT.isDirectory(); + + TEST_DATA = new TreeMap(); + for ( int i = 100; i < 1000; ++i ) + { + TEST_DATA.put(Integer.toString(i), ("Avinash Lakshman is a good man: " + i).getBytes()); + } + } + + /** + * Get a descriptor for the legacy sstable at the given version. + */ + protected SSTable.Descriptor getDescriptor(String ver) throws IOException + { + File directory = new File(LEGACY_SSTABLE_ROOT + File.separator + ver + File.separator + KSNAME); + return new SSTable.Descriptor(ver, directory, KSNAME, CFNAME, 0, false); + } + + /** + * Generates a test SSTable for use in this classes' tests. Uncomment and run against an older build + * and the output will be copied to a version subdirectory in 'LEGACY_SSTABLE_ROOT' + * + @Test + public void buildTestSSTable() throws IOException + { + // write the output in a version specific directory + SSTable.Descriptor dest = getDescriptor(SSTable.Descriptor.CURRENT_VERSION); + assert dest.directory.mkdirs() : "Could not create " + dest.directory + ". Might it already exist?"; + + SSTableReader ssTable = SSTableUtils.writeRawSSTable(new File(dest.filenameFor(SSTable.COMPONENT_DATA)), + KSNAME, + CFNAME, + TEST_DATA); + assert ssTable.desc.generation == 0 : + "In order to create a generation 0 sstable, please run this test alone."; + System.out.println(">>> Wrote " + dest); + } + */ + + /** + * Between version b and c, on disk bloom filters became incompatible, and needed to be regenerated. + */ + @Test + public void testVerB() throws IOException + { + SSTableReader reader = SSTableReader.open(getDescriptor("b")); + + List keys = new ArrayList(TEST_DATA.keySet()); + Collections.shuffle(keys); + BufferedRandomAccessFile file = new BufferedRandomAccessFile(reader.getFilename(), "r"); + for (String key : keys) + { + // confirm that the bloom filter does not reject any keys + file.seek(reader.getPosition(reader.partitioner.decorateKey(key)).position); + assert key.equals(file.readUTF()); + } + } +} diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java index 4961c1ff7c..8ed14e3a1b 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java @@ -38,13 +38,8 @@ import org.apache.cassandra.io.util.DataOutputBuffer; public class SSTableUtils { // first configured table and cf - public static String TABLENAME; - public static String CFNAME; - static - { - TABLENAME = DatabaseDescriptor.getTables().iterator().next(); - CFNAME = Table.open(TABLENAME).getColumnFamilies().iterator().next(); - } + public static String TABLENAME = "Keyspace1"; + public static String CFNAME = "Standard1"; public static ColumnFamily createCF(long mfda, int ldt, IColumn... cols) { @@ -97,13 +92,26 @@ public class SSTableUtils public static SSTableReader writeRawSSTable(String tablename, String cfname, SortedMap entries) throws IOException { - File f = tempSSTableFile(tablename, cfname); - SSTableWriter writer = new SSTableWriter(f.getAbsolutePath(), entries.size(), StorageService.getPartitioner()); + return writeRawSSTable(null, tablename, cfname, entries); + } + + public static SSTableReader writeRawSSTable(File datafile, String tablename, String cfname, SortedMap entries) throws IOException + { + boolean temporary = false; + if (datafile == null) + { + datafile = tempSSTableFile(tablename, cfname); + temporary = true; + } + SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size(), StorageService.getPartitioner()); for (Map.Entry entry : entries.entrySet()) writer.append(writer.partitioner.decorateKey(entry.getKey()), entry.getValue()); - new File(writer.indexFilename()).deleteOnExit(); - new File(writer.filterFilename()).deleteOnExit(); + if (temporary) + { + new File(writer.indexFilename()).deleteOnExit(); + new File(writer.filterFilename()).deleteOnExit(); + } return writer.closeAndOpenReader(); } }