diff --git a/CHANGES.txt b/CHANGES.txt index ab4323f4d4..13abd7a44a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -42,6 +42,7 @@ (CASSANDRA-3558) * fix missing response during range slice repair (CASSANDRA-3551) * 'describe ring' moved from CLI to nodetool and available through JMX (CASSANDRA-3220) + * add back partitioner to sstable metadata (CASSANDRA-3540) Merged from 0.8: * use cannonical host for local node in nodetool info (CASSANDRA-3556) * detect misuses of CounterColumnType (CASSANDRA-3422) diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java b/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java index 7a102f0a71..4e52993714 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java @@ -29,7 +29,6 @@ import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.commitlog.ReplayPosition; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.EstimatedHistogram; @@ -67,7 +66,7 @@ public class SSTableMetadata ReplayPosition.NONE, Long.MIN_VALUE, Double.MIN_VALUE, - DatabaseDescriptor.getPartitioner().getClass().getCanonicalName()); + null); } private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts, ReplayPosition replayPosition, long maxTimestamp, double cr, String partitioner) @@ -134,14 +133,14 @@ public class SSTableMetadata maxTimestamp = Math.max(maxTimestamp, potentialMax); } - public SSTableMetadata finalizeMetadata() + public SSTableMetadata finalizeMetadata(String partitioner) { return new SSTableMetadata(estimatedRowSize, estimatedColumnCount, replayPosition, maxTimestamp, compressionRatio, - DatabaseDescriptor.getPartitioner().getClass().getCanonicalName()); + partitioner); } public Collector estimatedRowSize(EstimatedHistogram estimatedRowSize) @@ -169,6 +168,8 @@ public class SSTableMetadata public void serialize(SSTableMetadata sstableStats, DataOutput dos) throws IOException { + assert sstableStats.partitioner != null; + EstimatedHistogram.serializer.serialize(sstableStats.estimatedRowSize, dos); EstimatedHistogram.serializer.serialize(sstableStats.estimatedColumnCount, dos); ReplayPosition.serializer.serialize(sstableStats.replayPosition, dos); @@ -209,9 +210,7 @@ public class SSTableMetadata double compressionRatio = desc.hasCompressionRatio ? dis.readDouble() : Double.MIN_VALUE; - String partitioner = desc.hasPartitioner - ? dis.readUTF() - : DatabaseDescriptor.getPartitioner().getClass().getCanonicalName(); + String partitioner = desc.hasPartitioner ? dis.readUTF() : null; return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp, compressionRatio, partitioner); } } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java index d4f7a27d10..e4cf60c280 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java @@ -138,9 +138,11 @@ public class SSTableReader extends SSTable ? SSTableMetadata.serializer.deserialize(descriptor) : SSTableMetadata.createDefaultInstance(); - // check if sstable is created using same partitioner as this node + // Check if sstable is created using same partitioner. + // Partitioner can be null, which indicates older version of sstable or no stats available. + // In that case, we skip the check. String partitionerName = partitioner.getClass().getCanonicalName(); - if (!partitionerName.equals(sstableMetadata.partitioner)) + if (sstableMetadata.partitioner != null && !partitionerName.equals(sstableMetadata.partitioner)) throw new RuntimeException(String.format("Cannot open %s because partitioner does not match %s", descriptor, partitionerName)); diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java index 84a603dead..ef2f1d4dd1 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java @@ -311,7 +311,7 @@ public class SSTableWriter extends SSTable dataFile.close(); // write sstable statistics - SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata(); + SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata(partitioner.getClass().getCanonicalName()); writeMetadata(descriptor, sstableMetadata); maybeWriteDigest(); diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Data.db b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Data.db new file mode 100644 index 0000000000..896897a275 Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Data.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Digest.sha1 b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Digest.sha1 new file mode 100644 index 0000000000..bc0c4e4a78 --- /dev/null +++ b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Digest.sha1 @@ -0,0 +1 @@ +095dd05150d499846782cdf1c77544048477cc9b Indexed1-hb-1-Data.db \ No newline at end of file diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Filter.db b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Filter.db new file mode 100644 index 0000000000..4b61a27af7 Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Filter.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Index.db b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Index.db new file mode 100644 index 0000000000..03b3dc81cf Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Index.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Statistics.db b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Statistics.db new file mode 100644 index 0000000000..78c4ed3333 Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Indexed1-hb-1-Statistics.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Data.db b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Data.db new file mode 100644 index 0000000000..2bdae87404 Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Data.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Digest.sha1 b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Digest.sha1 new file mode 100644 index 0000000000..7af63e1f97 --- /dev/null +++ b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Digest.sha1 @@ -0,0 +1 @@ +a6a32d25ca3cdf76ac5a1a6bffa78936463f9648 Indexed1.626972746864617465-hb-1-Data.db \ No newline at end of file diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Filter.db b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Filter.db new file mode 100644 index 0000000000..1fd53624a6 Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Filter.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Index.db b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Index.db new file mode 100644 index 0000000000..d9a482fa97 Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Index.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Statistics.db b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Statistics.db new file mode 100644 index 0000000000..78c4ed3333 Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Indexed1.626972746864617465-hb-1-Statistics.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Data.db b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Data.db new file mode 100644 index 0000000000..1cd5d15c5d Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Data.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Digest.sha1 b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Digest.sha1 new file mode 100644 index 0000000000..fa25470fcc --- /dev/null +++ b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Digest.sha1 @@ -0,0 +1 @@ +72295cc1b64e6ffdb67a39dec9996c4ff0363cce Standard1-hb-0-Data.db \ No newline at end of file diff --git a/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Filter.db b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Filter.db new file mode 100644 index 0000000000..1fc44602d9 Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Filter.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Index.db b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Index.db new file mode 100644 index 0000000000..bfe71f58bd Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Index.db differ diff --git a/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Statistics.db b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Statistics.db new file mode 100644 index 0000000000..8c256e9168 Binary files /dev/null and b/test/data/legacy-sstables/hb/Keyspace1/Standard1-hb-0-Statistics.db differ diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java index 979b4b13ca..a549352ef0 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java @@ -28,6 +28,7 @@ import java.io.IOException; import org.junit.Test; import org.apache.cassandra.db.commitlog.ReplayPosition; +import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.utils.EstimatedHistogram; public class SSTableMetadataSerializerTest @@ -49,7 +50,7 @@ public class SSTableMetadataSerializerTest .estimatedColumnCount(columnCounts) .replayPosition(rp); collector.updateMaxTimestamp(maxTimestamp); - SSTableMetadata originalMetadata = collector.finalizeMetadata(); + SSTableMetadata originalMetadata = collector.finalizeMetadata(RandomPartitioner.class.getCanonicalName()); ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(byteOutput); @@ -69,5 +70,6 @@ public class SSTableMetadataSerializerTest assert stats.replayPosition.equals(rp); assert stats.maxTimestamp == maxTimestamp; assert stats.maxTimestamp == originalMetadata.maxTimestamp; + assert RandomPartitioner.class.getCanonicalName().equals(stats.partitioner); } } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java index b2e3756092..09eef92bf3 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java @@ -21,6 +21,7 @@ package org.apache.cassandra.io.sstable; */ +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.concurrent.ExecutionException; @@ -33,14 +34,21 @@ import org.junit.Test; import org.apache.cassandra.CleanupHelper; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.columniterator.IdentityQueryFilter; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.filter.QueryPath; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.MmappedSegmentedFile; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.thrift.IndexClause; +import org.apache.cassandra.thrift.IndexExpression; +import org.apache.cassandra.thrift.IndexOperator; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.CLibrary; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.Util; @@ -188,6 +196,75 @@ public class SSTableReaderTest extends CleanupHelper assert p.right == p7; } + @Test + public void testPersistentStatisticsWithSecondaryIndex() throws IOException, ExecutionException, InterruptedException + { + // Create secondary index and flush to disk + Table table = Table.open("Keyspace1"); + ColumnFamilyStore store = table.getColumnFamilyStore("Indexed1"); + ByteBuffer key = ByteBufferUtil.bytes(String.valueOf("k1")); + RowMutation rm = new RowMutation("Keyspace1", key); + rm.add(new QueryPath("Indexed1", null, ByteBufferUtil.bytes("birthdate")), ByteBufferUtil.bytes(1L), System.currentTimeMillis()); + rm.apply(); + store.forceBlockingFlush(); + + // check if opening and querying works + assertIndexQueryWorks(store); + } + + @Test + public void testPersistentStatisticsFromOlderIndexedSSTable() throws IOException, ExecutionException, InterruptedException + { + // copy legacy indexed sstables + String root = System.getProperty("legacy-sstable-root"); + assert root != null; + File rootDir = new File(root + File.separator + "hb" + File.separator + "Keyspace1"); + assert rootDir.isDirectory(); + + String[] destDirs = DatabaseDescriptor.getAllDataFileLocationsForTable("Keyspace1"); + assert destDirs != null; + assert destDirs.length > 0; + + FileUtils.createDirectory(destDirs[0]); + for (File srcFile : rootDir.listFiles()) + { + if (!srcFile.getName().startsWith("Indexed1")) + continue; + File destFile = new File(destDirs[0] + File.separator + srcFile.getName()); + CLibrary.createHardLinkWithExec(srcFile, destFile); + + destFile = new File(destDirs[0] + File.separator + srcFile.getName()); + + assert destFile.exists() : destFile.getAbsoluteFile(); + } + ColumnFamilyStore store = Table.open("Keyspace1").getColumnFamilyStore("Indexed1"); + + // check if opening and querying works + assertIndexQueryWorks(store); + } + + private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS) + { + assert "Indexed1".equals(indexedCFS.getColumnFamilyName()); + + // make sure all sstables including 2ary indexes load from disk + indexedCFS.clearUnsafe(); + for (ColumnFamilyStore indexCfs : indexedCFS.indexManager.getIndexesBackedByCfs()) + { + indexCfs.clearUnsafe(); + indexCfs.loadNewSSTables(); // v1.0.4 would fail here (see CASSANDRA-3540) + } + indexedCFS.loadNewSSTables(); + + // query using index to see if sstable for secondary index opens + IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), IndexOperator.EQ, ByteBufferUtil.bytes(1L)); + IndexClause clause = new IndexClause(Arrays.asList(expr), ByteBufferUtil.EMPTY_BYTE_BUFFER, 100); + IPartitioner p = StorageService.getPartitioner(); + Range range = new Range(p.getMinimumToken(), p.getMinimumToken()); + List rows = indexedCFS.search(clause, range, new IdentityQueryFilter()); + assert rows.size() == 1; + } + private List> makeRanges(Token left, Token right) { return Arrays.>asList(new Range[]{ new Range(left, right) });