diff --git a/build.xml b/build.xml index 1a09910570..d2fc678539 100644 --- a/build.xml +++ b/build.xml @@ -39,6 +39,7 @@ + @@ -369,6 +370,7 @@ + diff --git a/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db new file mode 100644 index 0000000000..962ed01a99 Binary files /dev/null and b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db differ diff --git a/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Filter.db b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Filter.db new file mode 100644 index 0000000000..64d8c5d5d1 Binary files /dev/null and b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Filter.db differ diff --git a/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db new file mode 100644 index 0000000000..8fb5706d6e Binary files /dev/null and b/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db differ diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java new file mode 100644 index 0000000000..d37585c3d1 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -0,0 +1,113 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +package org.apache.cassandra.io.sstable; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +import org.apache.cassandra.CleanupHelper; +import org.apache.cassandra.io.util.BufferedRandomAccessFile; +import org.apache.cassandra.db.DecoratedKey; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; + +/** + * Tests backwards compatibility for SSTables. Requires that older SSTables match up with the existing config file, + * and currently only tests specific cases for specific upgrades. + */ +public class LegacySSTableTest extends CleanupHelper +{ + public static final String LEGACY_SSTABLE_PROP = "legacy-sstable-root"; + public static final String KSNAME = "Keyspace1"; + public static final String CFNAME = "Standard1"; + + public static SortedMap TEST_DATA; + public static File LEGACY_SSTABLE_ROOT; + + @BeforeClass + public static void beforeClass() + { + String scp = System.getProperty(LEGACY_SSTABLE_PROP); + assert scp != null; + LEGACY_SSTABLE_ROOT = new File(scp); + assert LEGACY_SSTABLE_ROOT.isDirectory(); + + TEST_DATA = new TreeMap(); + for ( int i = 100; i < 1000; ++i ) + { + TEST_DATA.put(Integer.toString(i), ("Avinash Lakshman is a good man: " + i).getBytes()); + } + } + + /** + * Get a descriptor for the legacy sstable at the given version. + */ + protected SSTable.Descriptor getDescriptor(String ver) throws IOException + { + File directory = new File(LEGACY_SSTABLE_ROOT + File.separator + ver + File.separator + KSNAME); + return new SSTable.Descriptor(ver, directory, KSNAME, CFNAME, 0, false); + } + + /** + * Generates a test SSTable for use in this classes' tests. Uncomment and run against an older build + * and the output will be copied to a version subdirectory in 'LEGACY_SSTABLE_ROOT' + * + @Test + public void buildTestSSTable() throws IOException + { + // write the output in a version specific directory + SSTable.Descriptor dest = getDescriptor(SSTable.Descriptor.CURRENT_VERSION); + assert dest.directory.mkdirs() : "Could not create " + dest.directory + ". Might it already exist?"; + + SSTableReader ssTable = SSTableUtils.writeRawSSTable(new File(dest.filenameFor(SSTable.COMPONENT_DATA)), + KSNAME, + CFNAME, + TEST_DATA); + assert ssTable.desc.generation == 0 : + "In order to create a generation 0 sstable, please run this test alone."; + System.out.println(">>> Wrote " + dest); + } + */ + + /** + * Between version b and c, on disk bloom filters became incompatible, and needed to be regenerated. + */ + @Test + public void testVerB() throws IOException + { + SSTableReader reader = SSTableReader.open(getDescriptor("b")); + + List keys = new ArrayList(TEST_DATA.keySet()); + Collections.shuffle(keys); + BufferedRandomAccessFile file = new BufferedRandomAccessFile(reader.getFilename(), "r"); + for (String key : keys) + { + // confirm that the bloom filter does not reject any keys + file.seek(reader.getPosition(reader.partitioner.decorateKey(key)).position); + assert key.equals(file.readUTF()); + } + } +} diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java index 4961c1ff7c..8ed14e3a1b 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java @@ -38,13 +38,8 @@ import org.apache.cassandra.io.util.DataOutputBuffer; public class SSTableUtils { // first configured table and cf - public static String TABLENAME; - public static String CFNAME; - static - { - TABLENAME = DatabaseDescriptor.getTables().iterator().next(); - CFNAME = Table.open(TABLENAME).getColumnFamilies().iterator().next(); - } + public static String TABLENAME = "Keyspace1"; + public static String CFNAME = "Standard1"; public static ColumnFamily createCF(long mfda, int ldt, IColumn... cols) { @@ -97,13 +92,26 @@ public class SSTableUtils public static SSTableReader writeRawSSTable(String tablename, String cfname, SortedMap entries) throws IOException { - File f = tempSSTableFile(tablename, cfname); - SSTableWriter writer = new SSTableWriter(f.getAbsolutePath(), entries.size(), StorageService.getPartitioner()); + return writeRawSSTable(null, tablename, cfname, entries); + } + + public static SSTableReader writeRawSSTable(File datafile, String tablename, String cfname, SortedMap entries) throws IOException + { + boolean temporary = false; + if (datafile == null) + { + datafile = tempSSTableFile(tablename, cfname); + temporary = true; + } + SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size(), StorageService.getPartitioner()); for (Map.Entry entry : entries.entrySet()) writer.append(writer.partitioner.decorateKey(entry.getKey()), entry.getValue()); - new File(writer.indexFilename()).deleteOnExit(); - new File(writer.filterFilename()).deleteOnExit(); + if (temporary) + { + new File(writer.indexFilename()).deleteOnExit(); + new File(writer.filterFilename()).deleteOnExit(); + } return writer.closeAndOpenReader(); } }