diff --git a/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db index 962ed01a99..d5752a5d79 100644 Binary files a/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db and b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db differ diff --git a/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db index 8fb5706d6e..9b57eece86 100644 Binary files a/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db and b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db differ diff --git a/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Data.db b/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Data.db new file mode 100644 index 0000000000..bbcf0b7db0 Binary files /dev/null and b/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Data.db differ diff --git a/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Filter.db b/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Filter.db new file mode 100644 index 0000000000..bd464e517b Binary files /dev/null and b/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Filter.db differ diff --git a/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Index.db b/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Index.db new file mode 100644 index 0000000000..9b3e827f7f Binary files /dev/null and b/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Index.db differ diff --git a/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Statistics.db b/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Statistics.db new file mode 100644 index 0000000000..db1e33d768 Binary files /dev/null and b/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Statistics.db differ diff --git a/test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java b/test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java index 735349b6d3..4a01f778f3 100644 --- a/test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java +++ b/test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java @@ -86,7 +86,7 @@ public class LongCompactionSpeedTest extends CleanupHelper } rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols)); } - SSTableReader sstable = SSTableUtils.writeSSTable(rows); + SSTableReader sstable = SSTableUtils.prepare().write(rows); sstables.add(sstable); store.addSSTable(sstable); } diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index a4c33af3d2..8593bd9802 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -22,15 +22,11 @@ package org.apache.cassandra.io.sstable; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.cassandra.CleanupHelper; -import org.apache.cassandra.io.util.BufferedRandomAccessFile; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.columniterator.SSTableNamesIterator; import org.apache.cassandra.utils.FBUtilities; import org.junit.BeforeClass; import org.junit.Test; @@ -45,7 +41,7 @@ public class LegacySSTableTest extends CleanupHelper public static final String KSNAME = "Keyspace1"; public static final String CFNAME = "Standard1"; - public static Map TEST_DATA; + public static Set TEST_DATA; public static File LEGACY_SSTABLE_ROOT; @BeforeClass @@ -56,11 +52,9 @@ public class LegacySSTableTest extends CleanupHelper LEGACY_SSTABLE_ROOT = new File(scp).getAbsoluteFile(); assert LEGACY_SSTABLE_ROOT.isDirectory(); - TEST_DATA = new HashMap(); + TEST_DATA = new HashSet(); for (int i = 100; i < 1000; ++i) - { - TEST_DATA.put(ByteBuffer.wrap(Integer.toString(i).getBytes()), ByteBuffer.wrap(("Avinash Lakshman is a good man: " + i).getBytes())); - } + TEST_DATA.add(Integer.toString(i)); } /** @@ -83,44 +77,39 @@ public class LegacySSTableTest extends CleanupHelper Descriptor dest = getDescriptor(Descriptor.CURRENT_VERSION); assert dest.directory.mkdirs() : "Could not create " + dest.directory + ". Might it already exist?"; - SSTableReader ssTable = SSTableUtils.writeRawSSTable(new File(dest.filenameFor(SSTable.COMPONENT_DATA)), - KSNAME, - CFNAME, - TEST_DATA); - assert ssTable.desc.generation == 0 : + SSTableReader ssTable = SSTableUtils.prepare().ks(KSNAME).cf(CFNAME).dest(dest).write(TEST_DATA); + assert ssTable.descriptor.generation == 0 : "In order to create a generation 0 sstable, please run this test alone."; System.out.println(">>> Wrote " + dest); } */ @Test - public void testVersions() throws IOException + public void testVersions() throws Throwable { for (File version : LEGACY_SSTABLE_ROOT.listFiles()) if (Descriptor.versionValidate(version.getName())) testVersion(version.getName()); } - public void testVersion(String version) + public void testVersion(String version) throws Throwable { try { SSTableReader reader = SSTableReader.open(getDescriptor(version)); - - List keys = new ArrayList(TEST_DATA.keySet()); - Collections.shuffle(keys); - BufferedRandomAccessFile file = new BufferedRandomAccessFile(reader.getFilename(), "r"); - for (ByteBuffer key : keys) + for (String keystring : TEST_DATA) { - // confirm that the bloom filter does not reject any keys - file.seek(reader.getPosition(reader.partitioner.decorateKey(key), SSTableReader.Operator.EQ)); - assert key.equals( FBUtilities.readShortByteArray(file)); + ByteBuffer key = ByteBuffer.wrap(keystring.getBytes()); + // confirm that the bloom filter does not reject any keys/names + DecoratedKey dk = reader.partitioner.decorateKey(key); + SSTableNamesIterator iter = new SSTableNamesIterator(reader, dk, FBUtilities.singleton(key)); + assert iter.next().name().equals(key); } } catch (Throwable e) { System.err.println("Failed to read " + version); - e.printStackTrace(System.err); + throw e; } } } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java index 6eb3dcded9..1f0cc12828 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java @@ -40,7 +40,7 @@ public class SSTableTest extends CleanupHelper Map map = new HashMap(); map.put(key, bytes); - SSTableReader ssTable = SSTableUtils.writeRawSSTable("Keyspace1", "Standard1", map); + SSTableReader ssTable = SSTableUtils.prepare().cf("Standard1").writeRaw(map); // verify verifySingle(ssTable, bytes, key); @@ -68,7 +68,7 @@ public class SSTableTest extends CleanupHelper } // write - SSTableReader ssTable = SSTableUtils.writeRawSSTable("Keyspace1", "Standard2", map); + SSTableReader ssTable = SSTableUtils.prepare().cf("Standard2").writeRaw(map); // verify verifyMany(ssTable, map); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java index 866dd0d250..cefdb54d76 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java @@ -22,12 +22,7 @@ package org.apache.cassandra.io.sstable; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.*; import org.apache.cassandra.db.Column; import org.apache.cassandra.db.ColumnFamily; @@ -51,6 +46,11 @@ public class SSTableUtils } public static File tempSSTableFile(String tablename, String cfname) throws IOException + { + return tempSSTableFile(tablename, cfname, 0); + } + + public static File tempSSTableFile(String tablename, String cfname, int generation) throws IOException { File tempdir = File.createTempFile(tablename, cfname); if(!tempdir.delete() || !tempdir.mkdir()) @@ -59,49 +59,105 @@ public class SSTableUtils File tabledir = new File(tempdir, tablename); tabledir.mkdir(); tabledir.deleteOnExit(); - File datafile = new File(new Descriptor(tabledir, tablename, cfname, 0, false).filenameFor("Data.db")); + File datafile = new File(new Descriptor(tabledir, tablename, cfname, generation, false).filenameFor("Data.db")); if (!datafile.createNewFile()) throw new IOException("unable to create file " + datafile); datafile.deleteOnExit(); return datafile; } - public static SSTableReader writeSSTable(Set keys) throws IOException + /** + * @return A Context with chainable methods to configure and write a SSTable. + */ + public static Context prepare() { - Map map = new HashMap(); - for (String key : keys) + return new Context(); + } + + public static class Context + { + private String ksname = TABLENAME; + private String cfname = CFNAME; + private Descriptor dest = null; + private boolean cleanup = true; + private int generation = 0; + + Context() {} + + public Context ks(String ksname) { - ColumnFamily cf = ColumnFamily.create(TABLENAME, CFNAME); - cf.addColumn(new Column(ByteBuffer.wrap(key.getBytes()), ByteBuffer.wrap(key.getBytes()), 0)); - map.put(key, cf); + this.ksname = ksname; + return this; } - return writeSSTable(map); - } - public static SSTableReader writeSSTable(Map entries) throws IOException - { - Map map = new HashMap(); - for (Map.Entry entry : entries.entrySet()) + public Context cf(String cfname) { - DataOutputBuffer buffer = new DataOutputBuffer(); - ColumnFamily.serializer().serializeWithIndexes(entry.getValue(), buffer); - map.put(ByteBuffer.wrap(entry.getKey().getBytes()), ByteBuffer.wrap(Arrays.copyOf(buffer.getData(), buffer.getLength()))); + this.cfname = cfname; + return this; } - return writeRawSSTable(TABLENAME, CFNAME, map); - } - public static SSTableReader writeRawSSTable(String tablename, String cfname, Map entries) throws IOException - { - File datafile = tempSSTableFile(tablename, cfname); - SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size()); - SortedMap sortedEntries = new TreeMap(); - for (Map.Entry entry : entries.entrySet()) - sortedEntries.put(writer.partitioner.decorateKey(entry.getKey()), entry.getValue()); - for (Map.Entry entry : sortedEntries.entrySet()) - writer.append(entry.getKey(), entry.getValue()); - new File(writer.descriptor.filenameFor(Component.PRIMARY_INDEX)).deleteOnExit(); - new File(writer.descriptor.filenameFor(Component.FILTER)).deleteOnExit(); - return writer.closeAndOpenReader(); - } + /** + * Set an alternate path for the written SSTable: if unset, the SSTable will + * be cleaned up on JVM exit. + */ + public Context dest(Descriptor dest) + { + this.dest = dest; + this.cleanup = false; + return this; + } + /** + * Sets the generation number for the generated SSTable. Ignored if "dest()" is set. + */ + public Context generation(int generation) + { + this.generation = generation; + return this; + } + + public SSTableReader write(Set keys) throws IOException + { + Map map = new HashMap(); + for (String key : keys) + { + ColumnFamily cf = ColumnFamily.create(ksname, cfname); + cf.addColumn(new Column(ByteBuffer.wrap(key.getBytes()), ByteBuffer.wrap(key.getBytes()), 0)); + map.put(key, cf); + } + return write(map); + } + + public SSTableReader write(Map entries) throws IOException + { + Map map = new HashMap(); + for (Map.Entry entry : entries.entrySet()) + { + DataOutputBuffer buffer = new DataOutputBuffer(); + ColumnFamily.serializer().serializeWithIndexes(entry.getValue(), buffer); + map.put(ByteBuffer.wrap(entry.getKey().getBytes()), + ByteBuffer.wrap(buffer.asByteArray())); + } + return writeRaw(map); + } + + /** + * @Deprecated: Writes the binary content of a row, which should be encapsulated. + */ + public SSTableReader writeRaw(Map entries) throws IOException + { + File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, generation) : new File(dest.filenameFor(Component.DATA)); + SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size()); + SortedMap sortedEntries = new TreeMap(); + for (Map.Entry entry : entries.entrySet()) + sortedEntries.put(writer.partitioner.decorateKey(entry.getKey()), entry.getValue()); + for (Map.Entry entry : sortedEntries.entrySet()) + writer.append(entry.getKey(), entry.getValue()); + SSTableReader reader = writer.closeAndOpenReader(); + if (cleanup) + for (Component comp : reader.components) + new File(reader.descriptor.filenameFor(comp)).deleteOnExit(); + return reader; + } + } } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java index 5878c3629f..75b3aad4b3 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java @@ -75,7 +75,7 @@ public class SSTableWriterTest extends CleanupHelper { ColumnFamily.serializer().serializeWithIndexes(cf, buffer); entries.put(ByteBufferUtil.bytes("k3"), ByteBuffer.wrap(Arrays.copyOf(buffer.getData(), buffer.getLength()))); - SSTableReader orig = SSTableUtils.writeRawSSTable("Keyspace1", "Indexed1", entries); + SSTableReader orig = SSTableUtils.prepare().cf("Indexed1").writeRaw(entries); // whack the index to trigger the recover FileUtils.deleteWithConfirm(orig.descriptor.filenameFor(Component.PRIMARY_INDEX)); FileUtils.deleteWithConfirm(orig.descriptor.filenameFor(Component.FILTER)); diff --git a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java index f352c195b6..3c05da3009 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java @@ -119,7 +119,7 @@ public class StreamingTransferTest extends CleanupHelper content.add("transfer1"); content.add("transfer2"); content.add("transfer3"); - SSTableReader sstable = SSTableUtils.writeSSTable(content); + SSTableReader sstable = SSTableUtils.prepare().write(content); String tablename = sstable.getTableName(); String cfname = sstable.getColumnFamilyName(); @@ -127,7 +127,7 @@ public class StreamingTransferTest extends CleanupHelper content2.add("test"); content2.add("test2"); content2.add("test3"); - SSTableReader sstable2 = SSTableUtils.writeSSTable(content2); + SSTableReader sstable2 = SSTableUtils.prepare().write(content2); // transfer the first and last key IPartitioner p = StorageService.getPartitioner();