natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix chunk index overflow due to large sstable with small chunk length 

patch by ZhaoYang; reviewed by brandonwilliams for CASSANDRA-15595
This commit is contained in:
Zhao Yang 2020-04-27 01:09:27 +08:00 committed by Brandon Williams
parent 6cd2d07d9a
commit 4886968c4e
3 changed files with 80 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,5 +1,6 @@
3.0.21
* Fix infinite loop on index query paging in tables with clustering (CASSANDRA-14242)
* Fix chunk index overflow due to large sstable with small chunk length (CASSANDRA-15595)
* cqlsh return non-zero status when STDIN CQL fails (CASSANDRA-15623)
* Don't skip sstables in slice queries based only on local min/max/deletion timestamp (CASSANDRA-15690)
* Memtable memory allocations may deadlock (CASSANDRA-15367)

6
src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
View File

@ -227,11 +227,15 @@ public class CompressionMetadata
public Chunk chunkFor(long position)
{
// position of the chunk
int idx = 8 * (int) (position / parameters.chunkLength());
long idx = 8 * (position / parameters.chunkLength());
if (idx >= chunkOffsetsSize)
throw new CorruptSSTableException(new EOFException(), indexFilePath);
if (idx < 0)
throw new CorruptSSTableException(new IllegalArgumentException(String.format("Invalid negative chunk index %d with position %d", idx, position)),
indexFilePath);
long chunkOffset = chunkOffsets.getLong(idx);
long nextChunkOffset = (idx + 8 == chunkOffsetsSize)
? compressedFileLength

100
test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
View File

@ -18,6 +18,7 @@
*/
package org.apache.cassandra.io.compress;
import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
@ -43,6 +44,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class CompressedRandomAccessReaderTest
{
@ -75,11 +77,11 @@ public class CompressedRandomAccessReaderTest
{
File f = File.createTempFile("compressed6791_", "3");
String filename = f.getAbsolutePath();
try(ChannelProxy channel = new ChannelProxy(f))
try (ChannelProxy channel = new ChannelProxy(f))
{
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
try(CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(32), sstableMetadataCollector))
try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(32), sstableMetadataCollector))
{
for (int i = 0; i < 20; i++)
@ -97,9 +99,9 @@ public class CompressedRandomAccessReaderTest
writer.finish();
}
try(RandomAccessReader reader = new CompressedRandomAccessReader.Builder(channel,
new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32))
.build())
try (RandomAccessReader reader = new CompressedRandomAccessReader.Builder(channel,
new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32))
.build())
{
String res = reader.readLine();
assertEquals(res, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
@ -110,7 +112,47 @@ public class CompressedRandomAccessReaderTest
{
if (f.exists())
assertTrue(f.delete());
File metadata = new File(filename+ ".metadata");
File metadata = new File(filename + ".metadata");
if (metadata.exists())
metadata.delete();
}
}
/**
* JIRA: CASSANDRA-15595 verify large position with small chunk length won't overflow chunk index
*/
@Test
public void testChunkIndexOverflow() throws IOException
{
File file = File.createTempFile("chunk_idx_overflow", "1");
String filename = file.getAbsolutePath();
int chunkLength = 4096; // 4k
try
{
writeSSTable(file, CompressionParams.snappy(chunkLength), 10);
CompressionMetadata metadata = new CompressionMetadata(filename + ".metadata", file.length(), ChecksumType.CRC32);
long chunks = 2761628520L;
long midPosition = (chunks / 2L) * chunkLength;
int idx = 8 * (int) (midPosition / chunkLength); // before patch
assertTrue("Expect integer overflow", idx < 0);
try
{
metadata.chunkFor(midPosition);
fail("Expected to throw EOF exception with chunk idx larger than total number of chunks in the sstable");
}
catch (CorruptSSTableException e)
{
assertTrue("Expect EOF, but got " + e.getCause(), e.getCause() instanceof EOFException);
}
}
finally
{
if (file.exists())
assertTrue(file.delete());
File metadata = new File(filename + ".metadata");
if (metadata.exists())
metadata.delete();
}
@ -121,26 +163,7 @@ public class CompressedRandomAccessReaderTest
final String filename = f.getAbsolutePath();
try(ChannelProxy channel = new ChannelProxy(f))
{
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
try(SequentialWriter writer = compressed
? new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(), sstableMetadataCollector)
: SequentialWriter.open(f))
{
writer.write("The quick ".getBytes());
DataPosition mark = writer.mark();
writer.write("blue fox jumps over the lazy dog".getBytes());
// write enough to be sure to change chunk
for (int i = 0; i < junkSize; ++i)
{
writer.write((byte) 1);
}
writer.resetAndTruncate(mark);
writer.write("brown fox jumps over the lazy dog".getBytes());
writer.finish();
}
assert f.exists();
writeSSTable(f, compressed ? CompressionParams.snappy() : null, junkSize);
CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) : null;
RandomAccessReader.Builder builder = compressed
@ -177,6 +200,31 @@ public class CompressedRandomAccessReaderTest
}
}
private static void writeSSTable(File f, CompressionParams params, int junkSize) throws IOException
{
final String filename = f.getAbsolutePath();
MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
try(SequentialWriter writer = params != null
? new CompressedSequentialWriter(f, filename + ".metadata", params, sstableMetadataCollector)
: SequentialWriter.open(f))
{
writer.write("The quick ".getBytes());
DataPosition mark = writer.mark();
writer.write("blue fox jumps over the lazy dog".getBytes());
// write enough to be sure to change chunk
for (int i = 0; i < junkSize; ++i)
{
writer.write((byte) 1);
}
writer.resetAndTruncate(mark);
writer.write("brown fox jumps over the lazy dog".getBytes());
writer.finish();
}
assert f.exists();
}
/**
* If the data read out doesn't match the checksum, an exception should be thrown
*/