From 079890579ba2997b270d8a42bdf7a708f4907aa2 Mon Sep 17 00:00:00 2001 From: Yuki Morishita Date: Wed, 21 Oct 2015 16:05:51 -0500 Subject: [PATCH] Make ScrubTest pass patch by yukim; reviewed by slebresne for CASSANDRA-10558 --- .../io/sstable/SimpleSSTableMultiWriter.java | 2 +- .../io/sstable/format/big/BigTableWriter.java | 4 +- .../org/apache/cassandra/db/ScrubTest.java | 52 +++++++++++++++---- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java index e034d90abf..fd1b9a739b 100644 --- a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java @@ -35,7 +35,7 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter { private final SSTableWriter writer; - private SimpleSSTableMultiWriter(SSTableWriter writer) + protected SimpleSSTableMultiWriter(SSTableWriter writer) { this.writer = writer; } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java index fdc1c64a30..e02c91977f 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java @@ -52,7 +52,7 @@ public class BigTableWriter extends SSTableWriter private final IndexWriter iwriter; private final SegmentedFile.Builder dbuilder; - private final SequentialWriter dataFile; + protected final SequentialWriter dataFile; private DecoratedKey lastWrittenKey; private FileMark dataMark; @@ -98,7 +98,7 @@ public class BigTableWriter extends SSTableWriter /** * Perform sanity checks on @param decoratedKey and @return the position in the data file before any data is written */ - private long beforeAppend(DecoratedKey decoratedKey) + protected long beforeAppend(DecoratedKey decoratedKey) { assert decoratedKey != null : "Keys must not be null"; // empty keys ARE allowed b/c of indexed column values if (lastWrittenKey != null && lastWrittenKey.compareTo(decoratedKey) >= 0) diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java index ab9975085d..a553fe80fd 100644 --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@ -29,6 +29,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.apache.cassandra.*; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.QueryProcessor; @@ -49,7 +50,11 @@ import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.io.compress.CompressionMetadata; import org.apache.cassandra.io.sstable.*; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.sstable.format.big.BigTableWriter; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.ByteBufferUtil; @@ -321,14 +326,8 @@ public class ScrubTest String filename = cfs.getSSTablePath(tempDataDir); Descriptor desc = Descriptor.fromFilename(filename); - try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, - keys.size(), - 0L, - 0, - new SerializationHeader(true, - cfs.metadata, - cfs.metadata.partitionColumns(), - EncodingStats.NO_STATS))) + LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE, desc.directory); + try (SSTableTxnWriter writer = new SSTableTxnWriter(txn, createTestWriter(desc, (long) keys.size(), cfs.metadata, txn))) { for (String k : keys) @@ -365,8 +364,8 @@ public class ScrubTest if (sstable.last.compareTo(sstable.first) < 0) sstable.last = sstable.first; - try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable); - Scrubber scrubber = new Scrubber(cfs, txn, false, true, true)) + try (LifecycleTransaction scrubTxn = LifecycleTransaction.offline(OperationType.SCRUB, sstable); + Scrubber scrubber = new Scrubber(cfs, scrubTxn, false, true, true)) { scrubber.scrub(); } @@ -629,4 +628,37 @@ public class ScrubTest // check index is still working assertOrdered(Util.cmd(cfs).filterOn(colName, Operator.EQ, 1L).build(), numRows / 2); } + + private static SSTableMultiWriter createTestWriter(Descriptor descriptor, long keyCount, CFMetaData metadata, LifecycleTransaction txn) + { + SerializationHeader header = new SerializationHeader(true, metadata, metadata.partitionColumns(), EncodingStats.NO_STATS); + MetadataCollector collector = new MetadataCollector(metadata.comparator).sstableLevel(0); + return new TestMultiWriter(new TestWriter(descriptor, keyCount, 0, metadata, collector, header, txn)); + } + + private static class TestMultiWriter extends SimpleSSTableMultiWriter + { + TestMultiWriter(SSTableWriter writer) + { + super(writer); + } + } + + /** + * Test writer that allows to write out of order SSTable. + */ + private static class TestWriter extends BigTableWriter + { + TestWriter(Descriptor descriptor, long keyCount, long repairedAt, CFMetaData metadata, + MetadataCollector collector, SerializationHeader header, LifecycleTransaction txn) + { + super(descriptor, keyCount, repairedAt, metadata, collector, header, txn); + } + + @Override + protected long beforeAppend(DecoratedKey decoratedKey) + { + return dataFile.position(); + } + } }