natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

row level coverage in LegacySSTableTest. patch by stuhood, reviewed by gdusbabek. CASSANDRA-1822 

git-svn-id: https://svn.apache.org/repos/asf/cassandra/branches/cassandra-0.7@1053638 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Gary Dusbabek 2010-12-29 14:16:55 +00:00
parent fd10436005
commit 23067c7bf1
12 changed files with 116 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db
View File

Binary file not shown.

BIN
test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db
View File

Binary file not shown.

BIN
test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Data.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Statistics.db Normal file
View File

Binary file not shown.

2
test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java
View File

@ -86,7 +86,7 @@ public class LongCompactionSpeedTest extends CleanupHelper
}
rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols));
}
SSTableReader sstable = SSTableUtils.writeSSTable(rows);
SSTableReader sstable = SSTableUtils.prepare().write(rows);
sstables.add(sstable);
store.addSSTable(sstable);
}

45
test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
View File

@ -22,15 +22,11 @@ package org.apache.cassandra.io.sstable;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import org.apache.cassandra.CleanupHelper;
import org.apache.cassandra.io.util.BufferedRandomAccessFile;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.columniterator.SSTableNamesIterator;
import org.apache.cassandra.utils.FBUtilities;
import org.junit.BeforeClass;
import org.junit.Test;
@ -45,7 +41,7 @@ public class LegacySSTableTest extends CleanupHelper
public static final String KSNAME = "Keyspace1";
public static final String CFNAME = "Standard1";
public static Map<ByteBuffer, ByteBuffer> TEST_DATA;
public static Set<String> TEST_DATA;
public static File LEGACY_SSTABLE_ROOT;
@BeforeClass
@ -56,11 +52,9 @@ public class LegacySSTableTest extends CleanupHelper
LEGACY_SSTABLE_ROOT = new File(scp).getAbsoluteFile();
assert LEGACY_SSTABLE_ROOT.isDirectory();
TEST_DATA = new HashMap<ByteBuffer,ByteBuffer>();
TEST_DATA = new HashSet<String>();
for (int i = 100; i < 1000; ++i)
{
TEST_DATA.put(ByteBuffer.wrap(Integer.toString(i).getBytes()), ByteBuffer.wrap(("Avinash Lakshman is a good man: " + i).getBytes()));
}
TEST_DATA.add(Integer.toString(i));
}
/**
@ -83,44 +77,39 @@ public class LegacySSTableTest extends CleanupHelper
Descriptor dest = getDescriptor(Descriptor.CURRENT_VERSION);
assert dest.directory.mkdirs() : "Could not create " + dest.directory + ". Might it already exist?";
SSTableReader ssTable = SSTableUtils.writeRawSSTable(new File(dest.filenameFor(SSTable.COMPONENT_DATA)),
KSNAME,
CFNAME,
TEST_DATA);
assert ssTable.desc.generation == 0 :
SSTableReader ssTable = SSTableUtils.prepare().ks(KSNAME).cf(CFNAME).dest(dest).write(TEST_DATA);
assert ssTable.descriptor.generation == 0 :
"In order to create a generation 0 sstable, please run this test alone.";
System.out.println(">>> Wrote " + dest);
}
*/
@Test
public void testVersions() throws IOException
public void testVersions() throws Throwable
{
for (File version : LEGACY_SSTABLE_ROOT.listFiles())
if (Descriptor.versionValidate(version.getName()))
testVersion(version.getName());
}
public void testVersion(String version)
public void testVersion(String version) throws Throwable
{
try
{
SSTableReader reader = SSTableReader.open(getDescriptor(version));
List<ByteBuffer> keys = new ArrayList<ByteBuffer>(TEST_DATA.keySet());
Collections.shuffle(keys);
BufferedRandomAccessFile file = new BufferedRandomAccessFile(reader.getFilename(), "r");
for (ByteBuffer key : keys)
for (String keystring : TEST_DATA)
{
// confirm that the bloom filter does not reject any keys
file.seek(reader.getPosition(reader.partitioner.decorateKey(key), SSTableReader.Operator.EQ));
assert key.equals( FBUtilities.readShortByteArray(file));
ByteBuffer key = ByteBuffer.wrap(keystring.getBytes());
// confirm that the bloom filter does not reject any keys/names
DecoratedKey dk = reader.partitioner.decorateKey(key);
SSTableNamesIterator iter = new SSTableNamesIterator(reader, dk, FBUtilities.singleton(key));
assert iter.next().name().equals(key);
}
}
catch (Throwable e)
{
System.err.println("Failed to read " + version);
e.printStackTrace(System.err);
throw e;
}
}
}

4
test/unit/org/apache/cassandra/io/sstable/SSTableTest.java
View File

@ -40,7 +40,7 @@ public class SSTableTest extends CleanupHelper
Map<ByteBuffer, ByteBuffer> map = new HashMap<ByteBuffer,ByteBuffer>();
map.put(key, bytes);
SSTableReader ssTable = SSTableUtils.writeRawSSTable("Keyspace1", "Standard1", map);
SSTableReader ssTable = SSTableUtils.prepare().cf("Standard1").writeRaw(map);
// verify
verifySingle(ssTable, bytes, key);
@ -68,7 +68,7 @@ public class SSTableTest extends CleanupHelper
}
// write
SSTableReader ssTable = SSTableUtils.writeRawSSTable("Keyspace1", "Standard2", map);
SSTableReader ssTable = SSTableUtils.prepare().cf("Standard2").writeRaw(map);
// verify
verifyMany(ssTable, map);

130
test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
View File

@ -22,12 +22,7 @@ package org.apache.cassandra.io.sstable;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.*;
import org.apache.cassandra.db.Column;
import org.apache.cassandra.db.ColumnFamily;
@ -51,6 +46,11 @@ public class SSTableUtils
}
public static File tempSSTableFile(String tablename, String cfname) throws IOException
{
return tempSSTableFile(tablename, cfname, 0);
}
public static File tempSSTableFile(String tablename, String cfname, int generation) throws IOException
{
File tempdir = File.createTempFile(tablename, cfname);
if(!tempdir.delete() || !tempdir.mkdir())
@ -59,49 +59,105 @@ public class SSTableUtils
File tabledir = new File(tempdir, tablename);
tabledir.mkdir();
tabledir.deleteOnExit();
File datafile = new File(new Descriptor(tabledir, tablename, cfname, 0, false).filenameFor("Data.db"));
File datafile = new File(new Descriptor(tabledir, tablename, cfname, generation, false).filenameFor("Data.db"));
if (!datafile.createNewFile())
throw new IOException("unable to create file " + datafile);
datafile.deleteOnExit();
return datafile;
}
public static SSTableReader writeSSTable(Set<String> keys) throws IOException
/**
* @return A Context with chainable methods to configure and write a SSTable.
*/
public static Context prepare()
{
Map<String, ColumnFamily> map = new HashMap<String, ColumnFamily>();
for (String key : keys)
return new Context();
}
public static class Context
{
private String ksname = TABLENAME;
private String cfname = CFNAME;
private Descriptor dest = null;
private boolean cleanup = true;
private int generation = 0;
Context() {}
public Context ks(String ksname)
{
ColumnFamily cf = ColumnFamily.create(TABLENAME, CFNAME);
cf.addColumn(new Column(ByteBuffer.wrap(key.getBytes()), ByteBuffer.wrap(key.getBytes()), 0));
map.put(key, cf);
this.ksname = ksname;
return this;
}
return writeSSTable(map);
}
public static SSTableReader writeSSTable(Map<String, ColumnFamily> entries) throws IOException
{
Map<ByteBuffer, ByteBuffer> map = new HashMap<ByteBuffer, ByteBuffer>();
for (Map.Entry<String, ColumnFamily> entry : entries.entrySet())
public Context cf(String cfname)
{
DataOutputBuffer buffer = new DataOutputBuffer();
ColumnFamily.serializer().serializeWithIndexes(entry.getValue(), buffer);
map.put(ByteBuffer.wrap(entry.getKey().getBytes()), ByteBuffer.wrap(Arrays.copyOf(buffer.getData(), buffer.getLength())));
this.cfname = cfname;
return this;
}
return writeRawSSTable(TABLENAME, CFNAME, map);
}
public static SSTableReader writeRawSSTable(String tablename, String cfname, Map<ByteBuffer, ByteBuffer> entries) throws IOException
{
File datafile = tempSSTableFile(tablename, cfname);
SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size());
SortedMap<DecoratedKey, ByteBuffer> sortedEntries = new TreeMap<DecoratedKey, ByteBuffer>();
for (Map.Entry<ByteBuffer, ByteBuffer> entry : entries.entrySet())
sortedEntries.put(writer.partitioner.decorateKey(entry.getKey()), entry.getValue());
for (Map.Entry<DecoratedKey, ByteBuffer> entry : sortedEntries.entrySet())
writer.append(entry.getKey(), entry.getValue());
new File(writer.descriptor.filenameFor(Component.PRIMARY_INDEX)).deleteOnExit();
new File(writer.descriptor.filenameFor(Component.FILTER)).deleteOnExit();
return writer.closeAndOpenReader();
}
/**
* Set an alternate path for the written SSTable: if unset, the SSTable will
* be cleaned up on JVM exit.
*/
public Context dest(Descriptor dest)
{
this.dest = dest;
this.cleanup = false;
return this;
}
/**
* Sets the generation number for the generated SSTable. Ignored if "dest()" is set.
*/
public Context generation(int generation)
{
this.generation = generation;
return this;
}
public SSTableReader write(Set<String> keys) throws IOException
{
Map<String, ColumnFamily> map = new HashMap<String, ColumnFamily>();
for (String key : keys)
{
ColumnFamily cf = ColumnFamily.create(ksname, cfname);
cf.addColumn(new Column(ByteBuffer.wrap(key.getBytes()), ByteBuffer.wrap(key.getBytes()), 0));
map.put(key, cf);
}
return write(map);
}
public SSTableReader write(Map<String, ColumnFamily> entries) throws IOException
{
Map<ByteBuffer, ByteBuffer> map = new HashMap<ByteBuffer, ByteBuffer>();
for (Map.Entry<String, ColumnFamily> entry : entries.entrySet())
{
DataOutputBuffer buffer = new DataOutputBuffer();
ColumnFamily.serializer().serializeWithIndexes(entry.getValue(), buffer);
map.put(ByteBuffer.wrap(entry.getKey().getBytes()),
ByteBuffer.wrap(buffer.asByteArray()));
}
return writeRaw(map);
}
/**
* @Deprecated: Writes the binary content of a row, which should be encapsulated.
*/
public SSTableReader writeRaw(Map<ByteBuffer, ByteBuffer> entries) throws IOException
{
File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, generation) : new File(dest.filenameFor(Component.DATA));
SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size());
SortedMap<DecoratedKey, ByteBuffer> sortedEntries = new TreeMap<DecoratedKey, ByteBuffer>();
for (Map.Entry<ByteBuffer, ByteBuffer> entry : entries.entrySet())
sortedEntries.put(writer.partitioner.decorateKey(entry.getKey()), entry.getValue());
for (Map.Entry<DecoratedKey, ByteBuffer> entry : sortedEntries.entrySet())
writer.append(entry.getKey(), entry.getValue());
SSTableReader reader = writer.closeAndOpenReader();
if (cleanup)
for (Component comp : reader.components)
new File(reader.descriptor.filenameFor(comp)).deleteOnExit();
return reader;
}
}
}

2
test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
View File

@ -75,7 +75,7 @@ public class SSTableWriterTest extends CleanupHelper {
ColumnFamily.serializer().serializeWithIndexes(cf, buffer);
entries.put(ByteBufferUtil.bytes("k3"), ByteBuffer.wrap(Arrays.copyOf(buffer.getData(), buffer.getLength())));
SSTableReader orig = SSTableUtils.writeRawSSTable("Keyspace1", "Indexed1", entries);
SSTableReader orig = SSTableUtils.prepare().cf("Indexed1").writeRaw(entries);
// whack the index to trigger the recover
FileUtils.deleteWithConfirm(orig.descriptor.filenameFor(Component.PRIMARY_INDEX));
FileUtils.deleteWithConfirm(orig.descriptor.filenameFor(Component.FILTER));

4
test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
View File

@ -119,7 +119,7 @@ public class StreamingTransferTest extends CleanupHelper
content.add("transfer1");
content.add("transfer2");
content.add("transfer3");
SSTableReader sstable = SSTableUtils.writeSSTable(content);
SSTableReader sstable = SSTableUtils.prepare().write(content);
String tablename = sstable.getTableName();
String cfname = sstable.getColumnFamilyName();
@ -127,7 +127,7 @@ public class StreamingTransferTest extends CleanupHelper
content2.add("test");
content2.add("test2");
content2.add("test3");
SSTableReader sstable2 = SSTableUtils.writeSSTable(content2);
SSTableReader sstable2 = SSTableUtils.prepare().write(content2);
// transfer the first and last key
IPartitioner p = StorageService.getPartitioner();