natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge from 1.0 

git-svn-id: https://svn.apache.org/repos/asf/cassandra/trunk@1211849 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sylvain Lebresne 2011-12-08 12:07:13 +00:00
parent f2744896e9
commit aef5e6aa25
21 changed files with 95 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -42,6 +42,7 @@
(CASSANDRA-3558)
* fix missing response during range slice repair (CASSANDRA-3551)
* 'describe ring' moved from CLI to nodetool and available through JMX (CASSANDRA-3220)
* add back partitioner to sstable metadata (CASSANDRA-3540)
Merged from 0.8:
* use cannonical host for local node in nodetool info (CASSANDRA-3556)
* detect misuses of CounterColumnType (CASSANDRA-3422)

13
src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
View File

@ -29,7 +29,6 @@ import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.EstimatedHistogram;
@ -67,7 +66,7 @@ public class SSTableMetadata
ReplayPosition.NONE,
Long.MIN_VALUE,
Double.MIN_VALUE,
DatabaseDescriptor.getPartitioner().getClass().getCanonicalName());
null);
}
private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts, ReplayPosition replayPosition, long maxTimestamp, double cr, String partitioner)
@ -134,14 +133,14 @@ public class SSTableMetadata
maxTimestamp = Math.max(maxTimestamp, potentialMax);
}
public SSTableMetadata finalizeMetadata()
public SSTableMetadata finalizeMetadata(String partitioner)
{
return new SSTableMetadata(estimatedRowSize,
estimatedColumnCount,
replayPosition,
maxTimestamp,
compressionRatio,
DatabaseDescriptor.getPartitioner().getClass().getCanonicalName());
partitioner);
}
public Collector estimatedRowSize(EstimatedHistogram estimatedRowSize)
@ -169,6 +168,8 @@ public class SSTableMetadata
public void serialize(SSTableMetadata sstableStats, DataOutput dos) throws IOException
{
assert sstableStats.partitioner != null;
EstimatedHistogram.serializer.serialize(sstableStats.estimatedRowSize, dos);
EstimatedHistogram.serializer.serialize(sstableStats.estimatedColumnCount, dos);
ReplayPosition.serializer.serialize(sstableStats.replayPosition, dos);
@ -209,9 +210,7 @@ public class SSTableMetadata
double compressionRatio = desc.hasCompressionRatio
? dis.readDouble()
: Double.MIN_VALUE;
String partitioner = desc.hasPartitioner
? dis.readUTF()
: DatabaseDescriptor.getPartitioner().getClass().getCanonicalName();
String partitioner = desc.hasPartitioner ? dis.readUTF() : null;
return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp, compressionRatio, partitioner);
}
}

6
src/java/org/apache/cassandra/io/sstable/SSTableReader.java
View File

@ -138,9 +138,11 @@ public class SSTableReader extends SSTable
? SSTableMetadata.serializer.deserialize(descriptor)
: SSTableMetadata.createDefaultInstance();
// check if sstable is created using same partitioner as this node
// Check if sstable is created using same partitioner.
// Partitioner can be null, which indicates older version of sstable or no stats available.
// In that case, we skip the check.
String partitionerName = partitioner.getClass().getCanonicalName();
if (!partitionerName.equals(sstableMetadata.partitioner))
if (sstableMetadata.partitioner != null && !partitionerName.equals(sstableMetadata.partitioner))
throw new RuntimeException(String.format("Cannot open %s because partitioner does not match %s",
descriptor, partitionerName));

2
src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
View File

@ -311,7 +311,7 @@ public class SSTableWriter extends SSTable
dataFile.close();
// write sstable statistics
SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata();
SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata(partitioner.getClass().getCanonicalName());
writeMetadata(descriptor, sstableMetadata);
maybeWriteDigest();

BIN
test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Data.db Normal file
View File

Binary file not shown.

1
test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Digest.sha1 Normal file
View File

@ -0,0 +1 @@
095dd05150d499846782cdf1c77544048477cc9b Indexed1-hb-1-Data.db

BIN
test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Data.db Normal file
View File

Binary file not shown.

1
test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Digest.sha1 Normal file
View File

@ -0,0 +1 @@
a6a32d25ca3cdf76ac5a1a6bffa78936463f9648 Indexed1.626972746864617465-hb-1-Data.db

BIN
test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Data.db Normal file
View File

Binary file not shown.

1
test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Digest.sha1 Normal file
View File

@ -0,0 +1 @@
72295cc1b64e6ffdb67a39dec9996c4ff0363cce Standard1-hb-0-Data.db

BIN
test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Statistics.db Normal file
View File

Binary file not shown.

4
test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
View File

@ -28,6 +28,7 @@ import java.io.IOException;
import org.junit.Test;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.utils.EstimatedHistogram;
public class SSTableMetadataSerializerTest
@ -49,7 +50,7 @@ public class SSTableMetadataSerializerTest
.estimatedColumnCount(columnCounts)
.replayPosition(rp);
collector.updateMaxTimestamp(maxTimestamp);
SSTableMetadata originalMetadata = collector.finalizeMetadata();
SSTableMetadata originalMetadata = collector.finalizeMetadata(RandomPartitioner.class.getCanonicalName());
ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(byteOutput);
@ -69,5 +70,6 @@ public class SSTableMetadataSerializerTest
assert stats.replayPosition.equals(rp);
assert stats.maxTimestamp == maxTimestamp;
assert stats.maxTimestamp == originalMetadata.maxTimestamp;
assert RandomPartitioner.class.getCanonicalName().equals(stats.partitioner);
}
}

77
test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
View File

@ -21,6 +21,7 @@ package org.apache.cassandra.io.sstable;
*/
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.ExecutionException;
@ -33,14 +34,21 @@ import org.junit.Test;
import org.apache.cassandra.CleanupHelper;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.MmappedSegmentedFile;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.thrift.IndexClause;
import org.apache.cassandra.thrift.IndexExpression;
import org.apache.cassandra.thrift.IndexOperator;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CLibrary;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.Util;
@ -188,6 +196,75 @@ public class SSTableReaderTest extends CleanupHelper
assert p.right == p7;
}
@Test
public void testPersistentStatisticsWithSecondaryIndex() throws IOException, ExecutionException, InterruptedException
{
// Create secondary index and flush to disk
Table table = Table.open("Keyspace1");
ColumnFamilyStore store = table.getColumnFamilyStore("Indexed1");
ByteBuffer key = ByteBufferUtil.bytes(String.valueOf("k1"));
RowMutation rm = new RowMutation("Keyspace1", key);
rm.add(new QueryPath("Indexed1", null, ByteBufferUtil.bytes("birthdate")), ByteBufferUtil.bytes(1L), System.currentTimeMillis());
rm.apply();
store.forceBlockingFlush();
// check if opening and querying works
assertIndexQueryWorks(store);
}
@Test
public void testPersistentStatisticsFromOlderIndexedSSTable() throws IOException, ExecutionException, InterruptedException
{
// copy legacy indexed sstables
String root = System.getProperty("legacy-sstable-root");
assert root != null;
File rootDir = new File(root + File.separator + "hb" + File.separator + "Keyspace1");
assert rootDir.isDirectory();
String[] destDirs = DatabaseDescriptor.getAllDataFileLocationsForTable("Keyspace1");
assert destDirs != null;
assert destDirs.length > 0;
FileUtils.createDirectory(destDirs[0]);
for (File srcFile : rootDir.listFiles())
{
if (!srcFile.getName().startsWith("Indexed1"))
continue;
File destFile = new File(destDirs[0] + File.separator + srcFile.getName());
CLibrary.createHardLinkWithExec(srcFile, destFile);
destFile = new File(destDirs[0] + File.separator + srcFile.getName());
assert destFile.exists() : destFile.getAbsoluteFile();
}
ColumnFamilyStore store = Table.open("Keyspace1").getColumnFamilyStore("Indexed1");
// check if opening and querying works
assertIndexQueryWorks(store);
}
private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS)
{
assert "Indexed1".equals(indexedCFS.getColumnFamilyName());
// make sure all sstables including 2ary indexes load from disk
indexedCFS.clearUnsafe();
for (ColumnFamilyStore indexCfs : indexedCFS.indexManager.getIndexesBackedByCfs())
{
indexCfs.clearUnsafe();
indexCfs.loadNewSSTables(); // v1.0.4 would fail here (see CASSANDRA-3540)
}
indexedCFS.loadNewSSTables();
// query using index to see if sstable for secondary index opens
IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), IndexOperator.EQ, ByteBufferUtil.bytes(1L));
IndexClause clause = new IndexClause(Arrays.asList(expr), ByteBufferUtil.EMPTY_BYTE_BUFFER, 100);
IPartitioner p = StorageService.getPartitioner();
Range range = new Range(p.getMinimumToken(), p.getMinimumToken());
List<Row> rows = indexedCFS.search(clause, range, new IdentityQueryFilter());
assert rows.size() == 1;
}
private List<Range<Token>> makeRanges(Token left, Token right)
{
return Arrays.<Range<Token>>asList(new Range[]{ new Range<Token>(left, right) });