From 6ffd5cc5d28658b4d058de79a9bea9c10a82c8d4 Mon Sep 17 00:00:00 2001 From: Marcus Eriksson Date: Thu, 14 Jul 2016 15:31:30 +0200 Subject: [PATCH] run LongLeveledCompactionStrategyTest.testLeveledScanner in a separate table patch by marcuse; reviewed by yukim for CASSANDRA-12202 --- CHANGES.txt | 1 + .../LongLeveledCompactionStrategyTest.java | 86 +++++++++++++------ .../org/apache/cassandra/SchemaLoader.java | 3 + .../LeveledCompactionStrategyTest.java | 2 +- 4 files changed, 63 insertions(+), 29 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1a228146f7..033b366aae 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.2.10 + * Fix flaky LongLeveledCompactionStrategyTest (CASSANDRA-12202) * Fix failing COPY TO STDOUT (CASSANDRA-12497) * Fix ColumnCounter::countAll behaviour for reverse queries (CASSANDRA-13222) * Exceptions encountered calling getSeeds() breaks OTC thread (CASSANDRA-13018) diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java index 5439a72775..8e63006a6e 100644 --- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java +++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java @@ -25,6 +25,7 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.apache.cassandra.SchemaLoader; @@ -41,6 +42,7 @@ public class LongLeveledCompactionStrategyTest { public static final String KEYSPACE1 = "LongLeveledCompactionStrategyTest"; public static final String CF_STANDARDLVL = "StandardLeveled"; + public static final String CF_STANDARDLVL2 = "StandardLeveled2"; @BeforeClass public static void defineSchema() throws ConfigurationException @@ -52,6 +54,9 @@ public class LongLeveledCompactionStrategyTest SimpleStrategy.class, KSMetaData.optsWithRF(1), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLVL) + .compactionStrategyClass(LeveledCompactionStrategy.class) + .compactionStrategyOptions(leveledOptions), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLVL2) .compactionStrategyClass(LeveledCompactionStrategy.class) .compactionStrategyOptions(leveledOptions)); } @@ -144,16 +149,34 @@ public class LongLeveledCompactionStrategyTest @Test public void testLeveledScanner() throws Exception { - testParallelLeveledCompaction(); Keyspace keyspace = Keyspace.open(KEYSPACE1); - ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARDLVL); - store.disableAutoCompaction(); - + final ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARDLVL2); WrappingCompactionStrategy strategy = ((WrappingCompactionStrategy) store.getCompactionStrategy()); - LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategy.getWrappedStrategies().get(1); + final LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategy.getWrappedStrategies().get(1); - ByteBuffer value = ByteBuffer.wrap(new byte[10 * 1024]); // 10 KB value + ByteBuffer value = ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files + // Enough data to have a level 1 and 2 + int rows = 128; + int columns = 10; + + // Adds enough data to trigger multiple sstable per level + for (int r = 0; r < rows; r++) + { + DecoratedKey key = Util.dk(String.valueOf(r)); + Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + for (int c = 0; c < columns; c++) + { + rm.add(CF_STANDARDLVL2, Util.cellname("column" + c), value, 0); + } + rm.apply(); + store.forceBlockingFlush(); + } + + value = ByteBuffer.wrap(new byte[10 * 1024]); // 10 KB value + LeveledCompactionStrategyTest.waitForLeveling(store); + // wait for higher-level compactions to finish + store.disableAutoCompaction(); // Adds 10 partitions for (int r = 0; r < 10; r++) { @@ -161,7 +184,7 @@ public class LongLeveledCompactionStrategyTest Mutation rm = new Mutation(KEYSPACE1, key.getKey()); for (int c = 0; c < 10; c++) { - rm.add(CF_STANDARDLVL, Util.cellname("column" + c), value, 0); + rm.add(CF_STANDARDLVL2, Util.cellname("column" + c), value, 0); } rm.apply(); } @@ -169,33 +192,40 @@ public class LongLeveledCompactionStrategyTest //Flush sstable store.forceBlockingFlush(); - Collection allSSTables = store.getSSTables(); - for (SSTableReader sstable : allSSTables) + store.runWithCompactionsDisabled(new Callable() { - if (sstable.getSSTableLevel() == 0) + public Void call() throws Exception { - System.out.println("Mutating L0-SSTABLE level to L1 to simulate a bug: " + sstable.getFilename()); - sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 1); - sstable.reloadSSTableMetadata(); - } - } - - try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(allSSTables)) - { - //Verify that leveled scanners will always iterate in ascending order (CASSANDRA-9935) - for (ISSTableScanner scanner : scannerList.scanners) - { - DecoratedKey lastKey = null; - while (scanner.hasNext()) + Collection allSSTables = store.getSSTables(); + for (SSTableReader sstable : allSSTables) { - OnDiskAtomIterator row = scanner.next(); - if (lastKey != null) + if (sstable.getSSTableLevel() == 0) { - assertTrue("row " + row.getKey() + " received out of order wrt " + lastKey, row.getKey().compareTo(lastKey) >= 0); + System.out.println("Mutating L0-SSTABLE level to L1 to simulate a bug: " + sstable.getFilename()); + sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 1); + sstable.reloadSSTableMetadata(); } - lastKey = row.getKey(); } + + try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(allSSTables)) + { + //Verify that leveled scanners will always iterate in ascending order (CASSANDRA-9935) + for (ISSTableScanner scanner : scannerList.scanners) + { + DecoratedKey lastKey = null; + while (scanner.hasNext()) + { + OnDiskAtomIterator row = scanner.next(); + if (lastKey != null) + { + assertTrue("row " + row.getKey() + " received out of order wrt " + lastKey, row.getKey().compareTo(lastKey) >= 0); + } + lastKey = row.getKey(); + } + } + } + return null; } - } + }, true); } } diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java index 2048f74fbb..45748a981d 100644 --- a/test/unit/org/apache/cassandra/SchemaLoader.java +++ b/test/unit/org/apache/cassandra/SchemaLoader.java @@ -180,6 +180,9 @@ public class SchemaLoader standardCFMD(ks1, "StandardLeveled") .compactionStrategyClass(LeveledCompactionStrategy.class) .compactionStrategyOptions(leveledOptions), + standardCFMD(ks1, "StandardLeveled2") + .compactionStrategyClass(LeveledCompactionStrategy.class) + .compactionStrategyOptions(leveledOptions), standardCFMD(ks1, "legacyleveled") .compactionStrategyClass(LeveledCompactionStrategy.class) .compactionStrategyOptions(leveledOptions), diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java index 8b9ca0803b..0047678105 100644 --- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java @@ -207,7 +207,7 @@ public class LeveledCompactionStrategyTest /** * wait for leveled compaction to quiesce on the given columnfamily */ - private void waitForLeveling(ColumnFamilyStore cfs) throws InterruptedException + public static void waitForLeveling(ColumnFamilyStore cfs) throws InterruptedException { WrappingCompactionStrategy strategyManager = (WrappingCompactionStrategy)cfs.getCompactionStrategy(); while (true)