natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce memory footprint of IndexSummaryTest large summary tests 

The runtime overflow guard test (testLargeIndexSummary) required ~2 GiB of
off-heap memory to trigger the Integer.MAX_VALUE boundary in
IndexSummaryBuilder.maybeAddEntry. SafeMemoryWriter's 50% buffer growth
strategy caused ~5 GiB peak memory during reallocation (old + new buffer),
exceeding CI medium node limits (3.5-5 GB).

Extract the hardcoded Integer.MAX_VALUE threshold in IndexSummaryBuilder
into a @VisibleForTesting field (maxEntriesSize), allowing the test to
exercise the same guard logic with a 2 MB threshold instead. Rename both
large summary tests to reflect their distinct code paths: runtime truncation
vs constructor-initiated downsampling.

patch by Sam Lightfoot; reviewed by Brandon Williams,Dmitry Konstantinov,Michael Semb Wever for CASSANDRA-20599
This commit is contained in:
samlightfoot 2026-02-12 21:27:38 +00:00 committed by Dmitry Konstantinov
parent ee124d2658
commit a3a834ad31
2 changed files with 33 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java
View File

@ -22,6 +22,8 @@ import java.nio.ByteOrder;
import java.util.Map;
import java.util.TreeMap;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -41,6 +43,9 @@ public class IndexSummaryBuilder implements AutoCloseable
static long defaultExpectedKeySize = INDEX_SUMMARY_EXPECTED_KEY_SIZE.getLong();
@VisibleForTesting
static long maxEntriesSize = Integer.MAX_VALUE;
// the offset in the keys memory region to look for a given summary boundary
private final SafeMemoryWriter offsets;
private final SafeMemoryWriter entries;
@ -105,13 +110,13 @@ public class IndexSummaryBuilder implements AutoCloseable
long expectedEntrySize = getEntrySize(defaultExpectedKeySize);
long maxExpectedEntries = expectedKeys / minIndexInterval;
long maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize;
if (maxExpectedEntriesSize > Integer.MAX_VALUE)
if (maxExpectedEntriesSize > maxEntriesSize)
{
// that's a _lot_ of keys, and a very low min index interval
int effectiveMinInterval = (int) Math.ceil((double)(expectedKeys * expectedEntrySize) / Integer.MAX_VALUE);
int effectiveMinInterval = (int) Math.ceil((double)(expectedKeys * expectedEntrySize) / maxEntriesSize);
maxExpectedEntries = expectedKeys / effectiveMinInterval;
maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize;
assert maxExpectedEntriesSize <= Integer.MAX_VALUE : maxExpectedEntriesSize;
assert maxExpectedEntriesSize <= maxEntriesSize : maxExpectedEntriesSize;
logger.warn("min_index_interval of {} is too low for {} expected keys of avg size {}; using interval of {} instead",
minIndexInterval, expectedKeys, defaultExpectedKeySize, effectiveMinInterval);
this.minIndexInterval = effectiveMinInterval;
@ -198,7 +203,7 @@ public class IndexSummaryBuilder implements AutoCloseable
{
if (keysWritten == nextSamplePosition)
{
if ((entries.length() + getEntrySize(length)) <= Integer.MAX_VALUE)
if ((entries.length() + getEntrySize(length)) <= maxEntriesSize)
{
offsets.writeInt((int) entries.length());
entries.write(keyBytes, offset, length);
@ -229,7 +234,7 @@ public class IndexSummaryBuilder implements AutoCloseable
{
if (keysWritten == nextSamplePosition)
{
if ((entries.length() + getEntrySize(decoratedKey)) <= Integer.MAX_VALUE)
if ((entries.length() + getEntrySize(decoratedKey)) <= maxEntriesSize)
{
offsets.writeInt((int) entries.length());
entries.write(decoratedKey.getKey());

35
test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java
View File

@ -122,20 +122,26 @@ public class IndexSummaryTest
}
/**
* Test an index summary whose total size is bigger than 2GiB,
* the index summary builder should log an error but it should still
* create an index summary, albeit one that does not cover the entire sstable.
* Test that the index summary builder truncates (stops adding entries) when the entries buffer
* would exceed the max capacity at runtime. The builder doesn't predict the overflow because
* defaultExpectedKeySize is smaller than the actual key size, so the constructor guard doesn't
* trigger. Instead, the runtime guard in maybeAddEntry detects the overflow and stops accepting
* entries, producing a partial summary that covers the beginning of the sstable.
*/
@Test
public void testLargeIndexSummary() throws IOException
public void testLargeIndexSummaryTruncatesWhenOverflowDetectedAtRuntime() throws IOException
{
// On Circle CI we normally don't have enough off-heap memory for this test so ignore it
Assume.assumeTrue(CIRCLECI.getString() == null);
final int numKeys = 1000000;
final int numKeys = 1000;
final int keySize = 3000;
final int minIndexInterval = 1;
// Use a small max so the test doesn't need GiBs of off-heap memory.
// 1000 keys × 3008 bytes/entry = 3,008,000 > 2,000,000, so the runtime guard triggers.
// defaultExpectedKeySize (64) gives estimated total of 72,000 < 2,000,000, so the
// constructor guard does not trigger and minIndexInterval stays at 1.
long oldMaxEntriesSize = IndexSummaryBuilder.maxEntriesSize;
IndexSummaryBuilder.maxEntriesSize = 2_000_000;
try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL))
{
for (int i = 0; i < numKeys; i++)
@ -153,15 +159,20 @@ public class IndexSummaryTest
assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount());
}
}
finally
{
IndexSummaryBuilder.maxEntriesSize = oldMaxEntriesSize;
}
}
/**
* Test an index summary whose total size is bigger than 2GiB,
* having updated IndexSummaryBuilder.defaultExpectedKeySize to match the size,
* the index summary should be downsampled automatically.
* Test that the index summary builder downsamples (increases minIndexInterval) when the
* estimated total entry size exceeds max capacity at construction time. With defaultExpectedKeySize
* matching the actual key size, the constructor detects the overflow upfront and adjusts
* minIndexInterval so the summary covers the entire sstable at a coarser interval.
*/
@Test
public void testLargeIndexSummaryWithExpectedSizeMatching() throws IOException
public void testLargeIndexSummaryDownsamplesWhenOverflowDetectedAtConstruction() throws IOException
{
// On Circle CI we normally don't have enough off-heap memory for this test so ignore it
Assume.assumeTrue(CIRCLECI.getString() == null);