From a3a834ad31907f0401489675390656b2dc570425 Mon Sep 17 00:00:00 2001 From: samlightfoot Date: Thu, 12 Feb 2026 21:27:38 +0000 Subject: [PATCH] Reduce memory footprint of IndexSummaryTest large summary tests The runtime overflow guard test (testLargeIndexSummary) required ~2 GiB of off-heap memory to trigger the Integer.MAX_VALUE boundary in IndexSummaryBuilder.maybeAddEntry. SafeMemoryWriter's 50% buffer growth strategy caused ~5 GiB peak memory during reallocation (old + new buffer), exceeding CI medium node limits (3.5-5 GB). Extract the hardcoded Integer.MAX_VALUE threshold in IndexSummaryBuilder into a @VisibleForTesting field (maxEntriesSize), allowing the test to exercise the same guard logic with a 2 MB threshold instead. Rename both large summary tests to reflect their distinct code paths: runtime truncation vs constructor-initiated downsampling. patch by Sam Lightfoot; reviewed by Brandon Williams,Dmitry Konstantinov,Michael Semb Wever for CASSANDRA-20599 --- .../indexsummary/IndexSummaryBuilder.java | 15 +++++--- .../indexsummary/IndexSummaryTest.java | 35 ++++++++++++------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java index 65e8d1b399..6c9699a013 100644 --- a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java @@ -22,6 +22,8 @@ import java.nio.ByteOrder; import java.util.Map; import java.util.TreeMap; +import com.google.common.annotations.VisibleForTesting; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +43,9 @@ public class IndexSummaryBuilder implements AutoCloseable static long defaultExpectedKeySize = INDEX_SUMMARY_EXPECTED_KEY_SIZE.getLong(); + @VisibleForTesting + static long maxEntriesSize = Integer.MAX_VALUE; + // the offset in the keys memory region to look for a given summary boundary private final SafeMemoryWriter offsets; private final SafeMemoryWriter entries; @@ -105,13 +110,13 @@ public class IndexSummaryBuilder implements AutoCloseable long expectedEntrySize = getEntrySize(defaultExpectedKeySize); long maxExpectedEntries = expectedKeys / minIndexInterval; long maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize; - if (maxExpectedEntriesSize > Integer.MAX_VALUE) + if (maxExpectedEntriesSize > maxEntriesSize) { // that's a _lot_ of keys, and a very low min index interval - int effectiveMinInterval = (int) Math.ceil((double)(expectedKeys * expectedEntrySize) / Integer.MAX_VALUE); + int effectiveMinInterval = (int) Math.ceil((double)(expectedKeys * expectedEntrySize) / maxEntriesSize); maxExpectedEntries = expectedKeys / effectiveMinInterval; maxExpectedEntriesSize = maxExpectedEntries * expectedEntrySize; - assert maxExpectedEntriesSize <= Integer.MAX_VALUE : maxExpectedEntriesSize; + assert maxExpectedEntriesSize <= maxEntriesSize : maxExpectedEntriesSize; logger.warn("min_index_interval of {} is too low for {} expected keys of avg size {}; using interval of {} instead", minIndexInterval, expectedKeys, defaultExpectedKeySize, effectiveMinInterval); this.minIndexInterval = effectiveMinInterval; @@ -198,7 +203,7 @@ public class IndexSummaryBuilder implements AutoCloseable { if (keysWritten == nextSamplePosition) { - if ((entries.length() + getEntrySize(length)) <= Integer.MAX_VALUE) + if ((entries.length() + getEntrySize(length)) <= maxEntriesSize) { offsets.writeInt((int) entries.length()); entries.write(keyBytes, offset, length); @@ -229,7 +234,7 @@ public class IndexSummaryBuilder implements AutoCloseable { if (keysWritten == nextSamplePosition) { - if ((entries.length() + getEntrySize(decoratedKey)) <= Integer.MAX_VALUE) + if ((entries.length() + getEntrySize(decoratedKey)) <= maxEntriesSize) { offsets.writeInt((int) entries.length()); entries.write(decoratedKey.getKey()); diff --git a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java index 451c25aaa7..456fc93463 100644 --- a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java @@ -122,20 +122,26 @@ public class IndexSummaryTest } /** - * Test an index summary whose total size is bigger than 2GiB, - * the index summary builder should log an error but it should still - * create an index summary, albeit one that does not cover the entire sstable. + * Test that the index summary builder truncates (stops adding entries) when the entries buffer + * would exceed the max capacity at runtime. The builder doesn't predict the overflow because + * defaultExpectedKeySize is smaller than the actual key size, so the constructor guard doesn't + * trigger. Instead, the runtime guard in maybeAddEntry detects the overflow and stops accepting + * entries, producing a partial summary that covers the beginning of the sstable. */ @Test - public void testLargeIndexSummary() throws IOException + public void testLargeIndexSummaryTruncatesWhenOverflowDetectedAtRuntime() throws IOException { - // On Circle CI we normally don't have enough off-heap memory for this test so ignore it - Assume.assumeTrue(CIRCLECI.getString() == null); - - final int numKeys = 1000000; + final int numKeys = 1000; final int keySize = 3000; final int minIndexInterval = 1; + // Use a small max so the test doesn't need GiBs of off-heap memory. + // 1000 keys × 3008 bytes/entry = 3,008,000 > 2,000,000, so the runtime guard triggers. + // defaultExpectedKeySize (64) gives estimated total of 72,000 < 2,000,000, so the + // constructor guard does not trigger and minIndexInterval stays at 1. + long oldMaxEntriesSize = IndexSummaryBuilder.maxEntriesSize; + IndexSummaryBuilder.maxEntriesSize = 2_000_000; + try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL)) { for (int i = 0; i < numKeys; i++) @@ -153,15 +159,20 @@ public class IndexSummaryTest assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount()); } } + finally + { + IndexSummaryBuilder.maxEntriesSize = oldMaxEntriesSize; + } } /** - * Test an index summary whose total size is bigger than 2GiB, - * having updated IndexSummaryBuilder.defaultExpectedKeySize to match the size, - * the index summary should be downsampled automatically. + * Test that the index summary builder downsamples (increases minIndexInterval) when the + * estimated total entry size exceeds max capacity at construction time. With defaultExpectedKeySize + * matching the actual key size, the constructor detects the overflow upfront and adjusts + * minIndexInterval so the summary covers the entire sstable at a coarser interval. */ @Test - public void testLargeIndexSummaryWithExpectedSizeMatching() throws IOException + public void testLargeIndexSummaryDownsamplesWhenOverflowDetectedAtConstruction() throws IOException { // On Circle CI we normally don't have enough off-heap memory for this test so ignore it Assume.assumeTrue(CIRCLECI.getString() == null);