From df3b6027bc90c658963e7bf72933b1507c6c6beb Mon Sep 17 00:00:00 2001 From: Yuki Morishita Date: Thu, 30 Jul 2015 10:07:25 -0500 Subject: [PATCH] ScrubTest should generate out-of-order SSTable on the fly with some other ScrubTest fixes. patch by yukim; reviewed by Stefania Alborghetti for CASSANDRA-9880 --- build.xml | 4 - .../cassandra/db/compaction/Scrubber.java | 7 +- .../io/sstable/format/SSTableReader.java | 1 - .../cassandra/tools/StandaloneScrubber.java | 2 +- test/data/corrupt-sstables/la-1-big-CRC.db | Bin 8 -> 0 bytes test/data/corrupt-sstables/la-1-big-Data.db | Bin 259 -> 0 bytes .../corrupt-sstables/la-1-big-Digest.adler32 | 1 - test/data/corrupt-sstables/la-1-big-Filter.db | Bin 24 -> 0 bytes test/data/corrupt-sstables/la-1-big-Index.db | Bin 105 -> 0 bytes .../corrupt-sstables/la-1-big-Statistics.db | Bin 4645 -> 0 bytes .../data/corrupt-sstables/la-1-big-Summary.db | Bin 83 -> 0 bytes test/data/corrupt-sstables/la-1-big-TOC.txt | 8 - .../org/apache/cassandra/db/ScrubTest.java | 182 +++++++++--------- 13 files changed, 98 insertions(+), 107 deletions(-) delete mode 100644 test/data/corrupt-sstables/la-1-big-CRC.db delete mode 100644 test/data/corrupt-sstables/la-1-big-Data.db delete mode 100644 test/data/corrupt-sstables/la-1-big-Digest.adler32 delete mode 100644 test/data/corrupt-sstables/la-1-big-Filter.db delete mode 100644 test/data/corrupt-sstables/la-1-big-Index.db delete mode 100644 test/data/corrupt-sstables/la-1-big-Statistics.db delete mode 100644 test/data/corrupt-sstables/la-1-big-Summary.db delete mode 100644 test/data/corrupt-sstables/la-1-big-TOC.txt diff --git a/build.xml b/build.xml index b56e379e02..bc433ae652 100644 --- a/build.xml +++ b/build.xml @@ -1281,7 +1281,6 @@ - @@ -1302,7 +1301,6 @@ - @@ -1323,7 +1321,6 @@ exclude="**/*.java" timeout="${test.timeout}" testtag="compression"> - @@ -1355,7 +1352,6 @@ - diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java index c85315775c..81e307ac08 100644 --- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java +++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java @@ -59,7 +59,6 @@ public class Scrubber implements Closeable private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; private final boolean isOffline; - private final boolean keepOriginals; private SSTableReader newSstable; private SSTableReader newInOrderSstable; @@ -86,7 +85,7 @@ public class Scrubber implements Closeable public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException { - this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData, false); + this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData); } @SuppressWarnings("resource") @@ -95,8 +94,7 @@ public class Scrubber implements Closeable boolean skipCorrupted, OutputHandler outputHandler, boolean isOffline, - boolean checkData, - boolean keepOriginals) throws IOException + boolean checkData) throws IOException { this.cfs = cfs; this.transaction = transaction; @@ -104,7 +102,6 @@ public class Scrubber implements Closeable this.outputHandler = outputHandler; this.skipCorrupted = skipCorrupted; this.isOffline = isOffline; - this.keepOriginals = keepOriginals; this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata, sstable.descriptor.version, sstable.header); diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index e9ac2008d3..6d39d2d047 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -1251,7 +1251,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted 0) { - selfRef().release(); throw new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last)); } } diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java index c00d03624b..9388d98e7b 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java +++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java @@ -121,7 +121,7 @@ public class StandaloneScrubber try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable)) { txn.obsoleteOriginals(); // make sure originals are deleted and avoid NPE if index is missing, CASSANDRA-9591 - try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, true, !options.noValidate, false)) + try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, true, !options.noValidate)) { scrubber.scrub(); } diff --git a/test/data/corrupt-sstables/la-1-big-CRC.db b/test/data/corrupt-sstables/la-1-big-CRC.db deleted file mode 100644 index 1a0c525946d75ded68e93920a0b9c51c4b322e80..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmZQzWMFvB;QSE)1u_C5 diff --git a/test/data/corrupt-sstables/la-1-big-Data.db b/test/data/corrupt-sstables/la-1-big-Data.db deleted file mode 100644 index e6c5eb96f3b8d5e54704b74514e0f229d1aefab9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 259 zcmZQzEUEwh|9=Am1aL426zAur`X%P3LPZ$<|7T=iOe9885;2OZh*6YGjG{_n6r~VW F1OV#*L-7Cr diff --git a/test/data/corrupt-sstables/la-1-big-Digest.adler32 b/test/data/corrupt-sstables/la-1-big-Digest.adler32 deleted file mode 100644 index 93deb45b72..0000000000 --- a/test/data/corrupt-sstables/la-1-big-Digest.adler32 +++ /dev/null @@ -1 +0,0 @@ -3942663153 \ No newline at end of file diff --git a/test/data/corrupt-sstables/la-1-big-Filter.db b/test/data/corrupt-sstables/la-1-big-Filter.db deleted file mode 100644 index 709d2eadc137258de1990a87ae5537860e7e85e6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24 fcmZQzU|?lnU|>>6W@=+-VNqe|O7URmc)$z*AyWik diff --git a/test/data/corrupt-sstables/la-1-big-Index.db b/test/data/corrupt-sstables/la-1-big-Index.db deleted file mode 100644 index 6e5e352f956306737343c5c0767d3a176b4421b0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 105 vcmZQzEMY(fjEPX5Dwt+Wf^xke+$tzHAHq$Ba;HGJl~C?Z2sZ`Fy$7NJ2d)MJ diff --git a/test/data/corrupt-sstables/la-1-big-Statistics.db b/test/data/corrupt-sstables/la-1-big-Statistics.db deleted file mode 100644 index 15220e097484b96640d93b6e75faf3f1fb35d86d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4645 zcmeI$drVVT90%}wZ(CXrI$Q7o3uF_}NtR+nWF{(tHNav9f)BQ7=B2%n8hO|@6Pb&K z;j$2dxQsEQpiV;!5{a)_7||^{*u#xSma#39VN%gxwxaO?YN)rRSAXA={j)!}^du+e zeD3e`_MCgq={>z1$Emrglpq(poHxl?kn=}!R&ku+JeOQ%-(nKWMN6?{vWO0cSZcM4 zCTp?Nl)2L><=U;1U9#qhcBjp0D=U@k>D86jvra|5C!fHV;=Nw)WJENt9dd22@2M4z ze_gA0wH*}p^iA-iAKub*9X#s$zD9rJsXhd%{=(NMntW|(lKTir8IN*Mqf`QA2xBc{ z9pf;@;f(c+4UD50&t@FUI6h#dUr9`SA>*Ztvl*{vyoK>D#tn?mG9ECdOGjS*5 zi)djM<0`a@Ilt-<#?|Z4niO=%GPIT%ukJC%!+Ox+9`p-)(Z(a_nQx(E2GB1hqvsw* zn+nlsP3X)nblwc~=4iBI4Z21|w=na!$Bc1L9GH*0gDzZ)Zo>J=dwzkqP=M=M_#Uk) z{0Z8t)j0pv{Z}Eb`L+%mGS&*#6?nm+lM;B^WF%N`#rbG3--38l$4v0-p%`%NU_Lm$ z7w>n{INtA+`b*G$+3*eU>lxU;H)Gro-};vkyeGFEd|=`=udFpJ?8}Yhsz77g?Sg?_f_~*|1*uNVk%&Ms5&-<(zv=JbUC$V;~c_)pCQ(i)8|3- ziS$kriN}36Vx;#g;ciw=4T-DiJ;TJ!e_ZQoA-?$ev3A;D_vNI4wC8c&_`%gDnZ!9;??0}3Zq5;?E5E_Y zo keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d"); - SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))), - SSTableWriter writer = SSTableWriter.create(cfs.metadata, - Descriptor.fromFilename(filename), - keys.size(), - 0L, - 0, - DatabaseDescriptor.getPartitioner(), - SerializationHeader.make(cfs.metadata, Collections.emptyList())); - - for (String k : keys) - { - RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0L, Util.dk(k)); - PartitionUpdate update = builder.clustering("someName") - .add("val", "someValue") - .buildUpdate(); - - writer.append(update.unfilteredIterator()); - } - writer.finish(false); - */ - - String root = System.getProperty("corrupt-sstable-root"); - assert root != null; - - File rootDir = new File(root); - assert rootDir.isDirectory(); - Descriptor desc = new Descriptor("la", rootDir, KEYSPACE, columnFamily, 1, SSTableFormat.Type.BIG); - CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname); + // This test assumes ByteOrderPartitioner to create out-of-order SSTable + IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner(); + DatabaseDescriptor.setPartitioner(new ByteOrderedPartitioner()); + // Create out-of-order SSTable + File tempDir = File.createTempFile("ScrubTest.testScrubOutOfOrder", "").getParentFile(); + // create ks/cf directory + File tempDataDir = new File(tempDir, String.join(File.separator, KEYSPACE, CF3)); + tempDataDir.mkdirs(); try { - SSTableReader.open(desc, metadata); - fail("SSTR validation should have caught the out-of-order rows"); + CompactionManager.instance.disableAutoCompaction(); + Keyspace keyspace = Keyspace.open(KEYSPACE); + String columnFamily = CF3; + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily); + cfs.clearUnsafe(); + + List keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d"); + String filename = cfs.getSSTablePath(tempDataDir); + Descriptor desc = Descriptor.fromFilename(filename); + + try (SSTableTxnWriter writer = SSTableTxnWriter.create(desc, + keys.size(), + 0L, + 0, + SerializationHeader.make(cfs.metadata, + Collections.emptyList()))) + { + + for (String k : keys) + { + PartitionUpdate update = UpdateBuilder.create(cfs.metadata, Util.dk(k)) + .newRow("someName").add("val", "someValue") + .build(); + + writer.append(update.unfilteredIterator()); + } + writer.finish(false); + } + + try + { + SSTableReader.open(desc, cfs.metadata); + fail("SSTR validation should have caught the out-of-order rows"); + } + catch (IllegalStateException ise) + { /* this is expected */ } + + // open without validation for scrubbing + Set components = new HashSet<>(); + if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists()) + components.add(Component.COMPRESSION_INFO); + components.add(Component.DATA); + components.add(Component.PRIMARY_INDEX); + components.add(Component.FILTER); + components.add(Component.STATS); + components.add(Component.SUMMARY); + components.add(Component.TOC); + + SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs); + if (sstable.last.compareTo(sstable.first) < 0) + sstable.last = sstable.first; + + try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable); + Scrubber scrubber = new Scrubber(cfs, txn, false, true, true)) + { + scrubber.scrub(); + } + cfs.loadNewSSTables(); + assertOrderedAll(cfs, 7); + sstable.selfRef().release(); } - catch (IllegalStateException ise) { /* this is expected */ } - - // open without validation for scrubbing - Set components = new HashSet<>(); - //components.add(Component.COMPRESSION_INFO); - components.add(Component.DATA); - components.add(Component.PRIMARY_INDEX); - components.add(Component.FILTER); - components.add(Component.STATS); - components.add(Component.SUMMARY); - components.add(Component.TOC); - - SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs); - if (sstable.last.compareTo(sstable.first) < 0) - sstable.last = sstable.first; - - try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable); - Scrubber scrubber = new Scrubber(cfs, txn, false, new OutputHandler.LogOutput(), true, true, true)) + finally { - scrubber.scrub(); + FileUtils.deleteRecursive(tempDataDir); + // reset partitioner + DatabaseDescriptor.setPartitioner(oldPartitioner); } - cfs.loadNewSSTables(); - assertOrderedAll(cfs, 7); - sstable.selfRef().release(); } private void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException