diff --git a/CHANGES.txt b/CHANGES.txt index 98a1808261..fe6ab0954a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.14 + * ReverseIndexedReader may drop rows during 2.1 to 3.0 upgrade (CASSANDRA-13525) * Fix repair process violating start/end token limits for small ranges (CASSANDRA-13052) * Add storage port options to sstableloader (CASSANDRA-13518) * Properly handle quoted index names in cqlsh DESCRIBE output (CASSANDRA-12847) diff --git a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java index 92690e1003..ea65633885 100644 --- a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java +++ b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java @@ -256,6 +256,10 @@ public abstract class UnfilteredDeserializer // The position in the input after the last data consumption (readNext/skipNext). private long lastConsumedPosition; + // Tracks the size of the last LegacyAtom read from disk, because this needs to be accounted + // for when marking lastConsumedPosition after readNext/skipNext + private long bytesReadForNextAtom; + private OldFormatDeserializer(CFMetaData metadata, DataInputPlus in, SerializationHelper helper, @@ -272,7 +276,10 @@ public abstract class UnfilteredDeserializer { try { - return LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic); + long pos = currentPosition(); + LegacyLayout.LegacyAtom atom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic); + bytesReadForNextAtom = currentPosition() - pos; + return atom; } catch (IOException e) { @@ -359,7 +366,7 @@ public abstract class UnfilteredDeserializer throw new IllegalStateException(); Unfiltered toReturn = next; next = null; - lastConsumedPosition = currentPosition(); + lastConsumedPosition = currentPosition() - bytesReadForNextAtom(); return toReturn; } @@ -368,7 +375,7 @@ public abstract class UnfilteredDeserializer if (!hasNext()) throw new UnsupportedOperationException(); next = null; - lastConsumedPosition = currentPosition(); + lastConsumedPosition = currentPosition() - bytesReadForNextAtom(); } public long bytesReadForUnconsumedData() @@ -379,6 +386,21 @@ public abstract class UnfilteredDeserializer return currentPosition() - lastConsumedPosition; } + // Reading/skipping an Unfiltered consumes LegacyAtoms from the underlying legacy atom iterator + // e.g. hasNext() -> iterator.hasNext() -> iterator.readRow() -> atoms.next() + // The stop condition of the loop which groups legacy atoms into rows causes that AtomIterator + // to read in the first atom which doesn't belong in the row. So by that point, our position + // is actually past the end of the next Unfiltered. To compensate, we record the size of + // the last LegacyAtom read and subtract it from the current position when we calculate lastConsumedPosition. + // If we don't, then when reading an indexed block, we can over correct and may think that we've + // exhausted the block before we actually have. + private long bytesReadForNextAtom() + { + // If we've read anything at all then we will have recorded this in bytesReadForNextAtom, + // but being extra careful here just incase this method is called before any reads happen. + return iterator.atoms.next == null ? 0 : bytesReadForNextAtom; + } + public void clearState() { next = null; diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db new file mode 100644 index 0000000000..ecd3ddba25 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db new file mode 100644 index 0000000000..d1e4e2faa7 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1 new file mode 100644 index 0000000000..bce117c3d5 --- /dev/null +++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1 @@ -0,0 +1 @@ +76435450 \ No newline at end of file diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db new file mode 100644 index 0000000000..00a88b4c70 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db new file mode 100644 index 0000000000..9ba48940be Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db new file mode 100644 index 0000000000..a57d32bc64 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db new file mode 100644 index 0000000000..d60d8f4ff9 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt new file mode 100644 index 0000000000..25fc8638b7 --- /dev/null +++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Summary.db +TOC.txt +Statistics.db +Digest.sha1 +Filter.db +Index.db +CompressionInfo.db diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db new file mode 100644 index 0000000000..09c4cfa8a5 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db new file mode 100644 index 0000000000..40ee3c691c Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1 new file mode 100644 index 0000000000..55ac08cf2c --- /dev/null +++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1 @@ -0,0 +1 @@ +3851004816 \ No newline at end of file diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db new file mode 100644 index 0000000000..00a88b4c70 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db new file mode 100644 index 0000000000..fb6ceed542 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db new file mode 100644 index 0000000000..b08f500192 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db new file mode 100644 index 0000000000..d60d8f4ff9 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db differ diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt new file mode 100644 index 0000000000..6865ecacf2 --- /dev/null +++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Summary.db +Data.db +Index.db +Statistics.db +TOC.txt +Digest.sha1 +Filter.db diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index cefee526b7..f4f6e851dc 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -146,6 +146,45 @@ public class LegacySSTableTest verifyReads(legacyVersion); } } + @Test + public void testReverseIterationOfLegacyIndexedSSTable() throws Exception + { + // During upgrades from 2.1 to 3.0, reverse queries can drop rows before upgradesstables is completed + QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_indexed (" + + " p int," + + " c int," + + " v1 int," + + " v2 int," + + " PRIMARY KEY(p, c)" + + ")"); + loadLegacyTable("legacy_%s_indexed%s", "ka", ""); + UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * " + + "FROM legacy_tables.legacy_ka_indexed " + + "WHERE p=1 " + + "ORDER BY c DESC"); + Assert.assertEquals(5000, rs.size()); + } + + @Test + public void testReadingLegacyIndexedSSTableWithStaticColumns() throws Exception + { + // During upgrades from 2.1 to 3.0, reading from tables with static columns errors before upgradesstables + // is completed + QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_indexed_static (" + + " p int," + + " c int," + + " v1 int," + + " v2 int," + + " s1 int static," + + " s2 int static," + + " PRIMARY KEY(p, c)" + + ")"); + loadLegacyTable("legacy_%s_indexed_static%s", "ka", ""); + UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * " + + "FROM legacy_tables.legacy_ka_indexed_static " + + "WHERE p=1 "); + Assert.assertEquals(5000, rs.size()); + } private void streamLegacyTables(String legacyVersion) throws Exception {