Upgrade Jamm version to 0.4.0

This upgrade also fixes issues with PhantomReferences and the test problems from CASSANDRA-17884 anad CASSANDRA-16304

patch by Benjamin Lerer; reviewed by Ekaterina Dimitrova for CASSANDRA-18239
This commit is contained in:
Benjamin Lerer 2023-07-13 19:08:11 +02:00 committed by Ekaterina Dimitrova
parent 6b885a44c3
commit 600f4d9a69
17 changed files with 254 additions and 206 deletions

View File

@ -1,4 +1,5 @@
5.0
* Upgrade Jamm version to 0.4.0 (CASSANDRA-17884, CASSANDRA-16304, CASSANDRA-18329)
* Remove legacy 3.0/3.11 buffer pool metrics (CASSANDRA-18313)
* Add AzureSnitch (CASSANDRA-18646)
* Implementation of the Unified Compaction Strategy as described in CEP-26 (CASSANDRA-18397)

View File

@ -79,7 +79,7 @@ if [ -f "$CASSANDRA_HOME"/lib/jsr223/scala/scala-compiler.jar ] ; then
fi
# set JVM javaagent opts to avoid warnings/errors
JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.2.jar"
JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.4.0.jar"
# Added sigar-bin to the java.library.path CASSANDRA-7838
JAVA_OPTS="$JAVA_OPTS:-Djava.library.path=$CASSANDRA_HOME/lib/sigar-bin"

View File

@ -142,7 +142,7 @@
<property name="jacoco.finalexecfile" value="${jacoco.export.dir}/jacoco.exec" />
<property name="jflex.version" value="1.8.2"/>
<property name="jamm.version" value="0.3.2"/>
<property name="jamm.version" value="0.4.0"/>
<property name="ecj.version" value="4.6.1"/>
<!-- When updating ASM, please, do consider whether you might need to update also FBUtilities#ASM_BYTECODE_VERSION
and the simulator InterceptClasses#BYTECODE_VERSION, in particular if we are looking to provide Cassandra support

View File

@ -191,7 +191,7 @@ fi
JVM_OPTS="$JVM_OPTS -XX:CompileCommandFile=$CASSANDRA_CONF/hotspot_compiler"
# add the jamm javaagent
JVM_OPTS="$JVM_OPTS -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.2.jar"
JVM_OPTS="$JVM_OPTS -javaagent:$CASSANDRA_HOME/lib/jamm-0.4.0.jar"
# set jvm HeapDumpPath with CASSANDRA_HEAPDUMP_DIR
if [ "x$CASSANDRA_HEAPDUMP_DIR" != "x" ]; then

View File

@ -40,7 +40,7 @@ CLASSPATH="$CLASSPATH:$EXTRA_CLASSPATH"
# set JVM javaagent opts to avoid warnings/errors
JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.2.jar"
JAVA_AGENT="$JAVA_AGENT -javaagent:$CASSANDRA_HOME/lib/jamm-0.4.0.jar"
#

View File

@ -98,6 +98,11 @@ public class BinAuditLogger implements IAuditLogger
@VisibleForTesting
public static class Message extends BinLog.ReleaseableWriteMarshallable implements WeightedQueue.Weighable
{
/**
* The shallow size of a {@code Message} object.
*/
private static final long EMPTY_SIZE = ObjectSizes.measure(new Message(""));
private final String message;
public Message(String message)
@ -130,7 +135,7 @@ public class BinAuditLogger implements IAuditLogger
@Override
public int weight()
{
return Ints.checkedCast(ObjectSizes.sizeOf(message));
return Ints.checkedCast(EMPTY_SIZE + ObjectSizes.sizeOf(message));
}
}
}

View File

@ -205,7 +205,7 @@ public class ColumnIdentifier implements IMeasurableMemory, Comparable<ColumnIde
public long unsharedHeapSizeExcludingData()
{
return EMPTY_SIZE
+ ObjectSizes.sizeOnHeapExcludingData(bytes)
+ ObjectSizes.sizeOnHeapExcludingDataOf(bytes)
+ ObjectSizes.sizeOf(text);
}

View File

@ -52,7 +52,7 @@ public class BufferClustering extends AbstractBufferClusteringPrefix implements
{
if (this == Clustering.EMPTY || this == Clustering.STATIC_CLUSTERING)
return 0;
return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(values);
return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingDataOf(values);
}
public static BufferClustering make(ByteBuffer... values)

View File

@ -155,7 +155,7 @@ public class BufferCell extends AbstractCell<ByteBuffer>
@Override
public long unsharedHeapSizeExcludingData()
{
return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(value) + (path == null ? 0 : path.unsharedHeapSizeExcludingData());
return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingDataOf(value) + (path == null ? 0 : path.unsharedHeapSizeExcludingData());
}
@Override

View File

@ -138,7 +138,7 @@ public abstract class CellPath implements IMeasurableMemory
@Override
public long unsharedHeapSizeExcludingData()
{
return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(value);
return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingDataOf(value);
}
}

View File

@ -32,7 +32,8 @@ import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.ObjectSizes;
import org.github.jamm.MemoryLayoutSpecification;
import org.github.jamm.MemoryMeterStrategy;
/**
* In-memory trie built for fast modification and reads executing concurrently with writes from a single mutator thread.
@ -973,7 +974,7 @@ public class InMemoryTrie<T> extends InMemoryReadTrie<T>
/** Returns the on heap size of the memtable trie itself, not counting any space taken by referenced content. */
public long sizeOnHeap()
{
return contentCount * MemoryLayoutSpecification.SPEC.getReferenceSize() +
return contentCount * MemoryMeterStrategy.MEMORY_LAYOUT.getReferenceSize() +
REFERENCE_ARRAY_ON_HEAP_SIZE * getChunkIdx(contentCount, CONTENTS_START_SHIFT, CONTENTS_START_SIZE) +
(bufferType == BufferType.ON_HEAP ? allocatedPos + EMPTY_SIZE_ON_HEAP : EMPTY_SIZE_OFF_HEAP) +
REFERENCE_ARRAY_ON_HEAP_SIZE * getChunkIdx(allocatedPos, BUF_START_SHIFT, BUF_START_SIZE);
@ -1021,7 +1022,7 @@ public class InMemoryTrie<T> extends InMemoryReadTrie<T>
int leadBit = getChunkIdx(index, CONTENTS_START_SHIFT, CONTENTS_START_SIZE);
int ofs = inChunkPointer(index, leadBit, CONTENTS_START_SIZE);
AtomicReferenceArray<T> contentArray = contentArrays[leadBit];
int contentOverhead = ((contentArray != null ? contentArray.length() : 0) - ofs) * MemoryLayoutSpecification.SPEC.getReferenceSize();
int contentOverhead = ((contentArray != null ? contentArray.length() : 0) - ofs) * MemoryMeterStrategy.MEMORY_LAYOUT.getReferenceSize();
return bufferOverhead + contentOverhead;
}

View File

@ -50,7 +50,6 @@ import org.apache.cassandra.utils.binlog.BinLog;
import org.apache.cassandra.utils.binlog.BinLogOptions;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import org.apache.cassandra.utils.concurrent.WeightedQueue;
import org.github.jamm.MemoryLayoutSpecification;
import static com.google.common.base.Preconditions.checkNotNull;
@ -85,9 +84,6 @@ public class FullQueryLogger implements QueryEvents.Listener
private static final int EMPTY_LIST_SIZE = Ints.checkedCast(ObjectSizes.measureDeep(new ArrayList<>(0)));
private static final int EMPTY_BYTEBUF_SIZE;
private static final int OBJECT_HEADER_SIZE = MemoryLayoutSpecification.SPEC.getObjectHeaderSize();
private static final int OBJECT_REFERENCE_SIZE = MemoryLayoutSpecification.SPEC.getReferenceSize();
public static final FullQueryLogger instance = new FullQueryLogger();
volatile BinLog binLog;
@ -332,6 +328,11 @@ public class FullQueryLogger implements QueryEvents.Listener
public static class Query extends AbstractLogEntry
{
/**
* The shallow size of a {@code Query} object.
*/
private static final long EMPTY_SIZE = ObjectSizes.measure(new Query());
private final String query;
public Query(String query, QueryOptions queryOptions, QueryState queryState, long queryStartTime)
@ -340,6 +341,14 @@ public class FullQueryLogger implements QueryEvents.Listener
this.query = query;
}
/**
* Constructor only use to compute this class shallow size.
*/
private Query()
{
this.query = null;
}
@Override
protected String type()
{
@ -356,12 +365,21 @@ public class FullQueryLogger implements QueryEvents.Listener
@Override
public int weight()
{
return Ints.checkedCast(ObjectSizes.sizeOf(query)) + super.weight();
// Object deep size = Object' shallow size + query field deep size + deep size of the parent fields
return Ints.checkedCast(EMPTY_SIZE + ObjectSizes.sizeOf(query) + super.fieldsSize());
}
}
public static class Batch extends AbstractLogEntry
{
/**
* The shallow size of a {@code Batch} object (which includes primitive fields).
*/
private static final long EMPTY_SIZE = ObjectSizes.measure(new Batch());
/**
* The weight is pre-computed in the constructor and represent the object deep size.
*/
private final int weight;
private final BatchStatement.Type batchType;
private final List<String> queries;
@ -380,25 +398,37 @@ public class FullQueryLogger implements QueryEvents.Listener
this.values = values;
this.batchType = batchType;
int weight = super.weight();
// weight, queries, values, batch type
weight += Integer.BYTES + // cached weight
2 * EMPTY_LIST_SIZE + // queries + values lists
3 * OBJECT_REFERENCE_SIZE; // batchType and two lists references
// We assume that all the lists are ArrayLists and that the size of each underlying array is the one of the list
// (which is obviously wrong but not worst than the previous computation that was ignoring part of the arrays size in the computation).
long queriesSize = EMPTY_LIST_SIZE + ObjectSizes.sizeOfReferenceArray(queries.size());
for (String query : queries)
weight += ObjectSizes.sizeOf(checkNotNull(query)) + OBJECT_REFERENCE_SIZE;
queriesSize += ObjectSizes.sizeOf(checkNotNull(query));
long valuesSize = EMPTY_LIST_SIZE + ObjectSizes.sizeOfReferenceArray(values.size());
for (List<ByteBuffer> subValues : values)
{
weight += EMPTY_LIST_SIZE + OBJECT_REFERENCE_SIZE;
for (ByteBuffer value : subValues)
weight += ObjectSizes.sizeOnHeapOf(value) + OBJECT_REFERENCE_SIZE;
valuesSize += EMPTY_LIST_SIZE + ObjectSizes.sizeOfReferenceArray(subValues.size());
for (ByteBuffer subValue : subValues)
valuesSize += ObjectSizes.sizeOnHeapOf(subValue);
}
this.weight = weight;
// No need to add the batch type which is an enum.
this.weight = Ints.checkedCast(EMPTY_SIZE // Shallow size object
+ super.fieldsSize() // deep size of the parent fields (non-primitives as they are included in the shallow size)
+ queriesSize // deep size queries field
+ valuesSize); // deep size values field
}
/**
* Constructor only use to compute this class shallow size.
*/
private Batch()
{
this.weight = 0;
this.batchType = null;
this.queries = null;
this.values = null;
}
@Override
@ -483,6 +513,19 @@ public class FullQueryLogger implements QueryEvents.Listener
}
}
/**
* Constructor only use to compute sub-classes shallow size.
*/
private AbstractLogEntry()
{
this.queryStartTime = 0;
this.protocolVersion = 0;
this.queryOptionsBuffer = null;
this.generatedTimestamp = 0;
this.generatedNowInSeconds = 0;
this.keyspace = null;
}
@Override
protected long version()
{
@ -508,16 +551,14 @@ public class FullQueryLogger implements QueryEvents.Listener
queryOptionsBuffer.release();
}
@Override
public int weight()
/**
* Returns the sum of the non-primitive fields' deep sizes.
* @return the sum of the non-primitive fields' deep sizes.
*/
protected long fieldsSize()
{
return OBJECT_HEADER_SIZE
+ Long.BYTES // queryStartTime
+ Integer.BYTES // protocolVersion
+ OBJECT_REFERENCE_SIZE + EMPTY_BYTEBUF_SIZE + queryOptionsBuffer.capacity() // queryOptionsBuffer
+ Long.BYTES // generatedTimestamp
+ Long.BYTES // generatedNowInSeconds
+ OBJECT_REFERENCE_SIZE + Ints.checkedCast(ObjectSizes.sizeOf(keyspace)); // keyspace
return EMPTY_BYTEBUF_SIZE + queryOptionsBuffer.capacity() // queryOptionsBuffer
+ ObjectSizes.sizeOf(keyspace); // keyspace
}
}

View File

@ -24,23 +24,26 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import org.github.jamm.MemoryLayoutSpecification;
import org.github.jamm.MemoryMeter;
import org.github.jamm.MemoryMeter.ByteBufferMode;
import org.github.jamm.MemoryMeter.Guess;
import static org.github.jamm.MemoryMeterStrategy.MEMORY_LAYOUT;
import static org.github.jamm.utils.ArrayMeasurementUtils.computeArraySize;
/**
* A convenience class for wrapping access to MemoryMeter
* A convenience class for wrapping access to MemoryMeter. Should be used instead of using a {@code MemoryMeter} directly.
* {@code MemoryMeter} can be used directly for testing as it allow a more fine tuned configuration for comparison.
*/
public class ObjectSizes
{
private static final MemoryMeter meter = new MemoryMeter().withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE)
.ignoreKnownSingletons();
private static final MemoryMeter omitSharedMeter = meter.omitSharedBufferOverhead();
private static final MemoryMeter meter = MemoryMeter.builder().withGuessing(Guess.INSTRUMENTATION_AND_SPECIFICATION,
Guess.UNSAFE)
.build();
private static final long EMPTY_HEAP_BUFFER_SIZE = measure(ByteBufferUtil.EMPTY_BYTE_BUFFER);
private static final long EMPTY_BYTE_ARRAY_SIZE = measure(new byte[0]);
private static final long EMPTY_STRING_SIZE = measure("");
private static final long DIRECT_BUFFER_HEAP_SIZE = measure(ByteBuffer.allocateDirect(0));
private static final long HEAP_BUFFER_SHALLOW_SIZE = measure(ByteBufferUtil.EMPTY_BYTE_BUFFER);
private static final long DIRECT_BUFFER_SHALLOW_SIZE = measure(ByteBuffer.allocateDirect(0));
private static final long DIRECT_BUFFER_DEEP_SIZE = measureDeep(ByteBuffer.allocateDirect(0));
public static final long IPV6_SOCKET_ADDRESS_SIZE = ObjectSizes.measureDeep(new InetSocketAddress(getIpvAddress(16), 42));
@ -52,10 +55,7 @@ public class ObjectSizes
*/
public static long sizeOfArray(byte[] bytes)
{
if (bytes == null)
return 0;
return sizeOfArray(bytes.length, 1);
return meter.measureArray(bytes);
}
/**
@ -66,10 +66,7 @@ public class ObjectSizes
*/
public static long sizeOfArray(long[] longs)
{
if (longs == null)
return 0;
return sizeOfArray(longs.length, 8);
return meter.measureArray(longs);
}
/**
@ -80,10 +77,7 @@ public class ObjectSizes
*/
public static long sizeOfArray(int[] ints)
{
if (ints == null)
return 0;
return sizeOfArray(ints.length, 4);
return meter.measureArray(ints);
}
/**
@ -94,7 +88,7 @@ public class ObjectSizes
*/
public static long sizeOfReferenceArray(int length)
{
return sizeOfArray(length, MemoryLayoutSpecification.SPEC.getReferenceSize());
return sizeOfArray(length, MEMORY_LAYOUT.getReferenceSize());
}
/**
@ -105,15 +99,12 @@ public class ObjectSizes
*/
public static long sizeOfArray(Object[] objects)
{
if (objects == null)
return 0;
return sizeOfReferenceArray(objects.length);
return meter.measureArray(objects);
}
private static long sizeOfArray(int length, long elementSize)
private static long sizeOfArray(int length, int elementSize)
{
return MemoryLayoutSpecification.sizeOfArray(length, elementSize);
return computeArraySize(MEMORY_LAYOUT.getArrayHeaderSize(), length, elementSize, MEMORY_LAYOUT.getObjectAlignment());
}
/**
@ -134,65 +125,89 @@ public class ObjectSizes
/**
* Amount of non-data heap memory consumed by the array of byte buffers. It sums memory consumed
* by the array itself and for each included byte buffer using {@link #sizeOnHeapExcludingData(ByteBuffer)}.
* by the array itself and for each included byte buffer using {@link #sizeOnHeapExcludingDataOf(ByteBuffer)}.
*/
public static long sizeOnHeapExcludingData(ByteBuffer[] array)
public static long sizeOnHeapExcludingDataOf(ByteBuffer[] array)
{
if (array == null)
return 0;
long sum = sizeOfArray(array);
for (ByteBuffer b : array)
sum += sizeOnHeapExcludingData(b);
sum += sizeOnHeapExcludingDataOf(b);
return sum;
}
/**
* @return heap memory consumed by the byte buffer. If it is a slice, it counts the data size, but it does not
* include the internal array overhead.
* Measures the heap memory used by the specified byte buffer. If the buffer is a slab only the data size will be
* counted but not the internal overhead. A SLAB is assumed to be created by: {@code buffer.duplicate().position(start).limit(end)} without the use of {@code slice()}.
* <p>This method makes a certain amount of assumptions:
* <ul>
* <li>That slabs are always created using: {@code buffer.duplicate().position(start).limit(end)} and not through slice</li>
* <li>That the input buffers are not read-only buffers</li>
* <li>That the direct buffers that are not slab are not duplicates</li>
* </ul>
* Non-respect of those assumptions can lead to an invalid value being returned.
* @param buffer the buffer to measure
* @return the heap memory used by the specified byte buffer.
*/
public static long sizeOnHeapOf(ByteBuffer buffer)
{
if (buffer == null)
return 0;
assert !buffer.isReadOnly();
// We assume here that slabs are always created using: buffer.duplicate().position(start).limit(end) and not through slice
if (ByteBufferMode.SLAB_ALLOCATION_NO_SLICE.isSlab(buffer))
{
if (buffer.isDirect())
return DIRECT_BUFFER_SHALLOW_SIZE; // We ignore the underlying buffer
return HEAP_BUFFER_SHALLOW_SIZE + buffer.remaining(); // We ignore the array overhead
}
if (buffer.isDirect())
return DIRECT_BUFFER_HEAP_SIZE;
return DIRECT_BUFFER_DEEP_SIZE; // That might not be true if the buffer is a view of another buffer so we could undercount
int arrayLen = buffer.array().length;
int bufLen = buffer.remaining();
// if we're only referencing a sub-portion of the ByteBuffer, don't count the array overhead (assume it is SLAB
// allocated - the overhead amortized over all the allocations is negligible and better to undercount than over)
if (arrayLen > bufLen)
return EMPTY_HEAP_BUFFER_SIZE + bufLen;
return EMPTY_HEAP_BUFFER_SIZE + (arrayLen == 0 ? EMPTY_BYTE_ARRAY_SIZE : sizeOfArray(arrayLen, 1));
return HEAP_BUFFER_SHALLOW_SIZE + meter.measureArray(buffer.array());
}
/**
* @return non-data heap memory consumed by the byte buffer. If it is a slice, it does not include the internal
* array overhead.
* Measures the heap memory used by the specified byte buffer excluding the data. If the buffer shallow size will be counted.
* A SLAB is assumed to be created by: {@code buffer.duplicate().position(start).limit(end)} without the use of {@code slice()}.
* <p>This method makes a certain amount of assumptions:
* <ul>
* <li>That slabs are always created using: {@code buffer.duplicate().position(start).limit(end)} and not through slice</li>
* <li>That the input buffers are not read-only buffers</li>
* <li>That the direct buffers that are not slab are not duplicates</li>
* </ul>
* Non-respect of those assumptions can lead to an invalid value being returned. T
* @param buffer the buffer to measure
* @return the heap memory used by the specified byte buffer excluding the data..
*/
public static long sizeOnHeapExcludingData(ByteBuffer buffer)
public static long sizeOnHeapExcludingDataOf(ByteBuffer buffer)
{
if (buffer == null)
return 0;
assert !buffer.isReadOnly();
// We assume here that slabs are always created using: buffer.duplicate().position(start).limit(end) and not through slice
if (ByteBufferMode.SLAB_ALLOCATION_NO_SLICE.isSlab(buffer))
{
if (buffer.isDirect())
return DIRECT_BUFFER_SHALLOW_SIZE; // We ignore the underlying buffer
return HEAP_BUFFER_SHALLOW_SIZE; // We ignore the array overhead
}
if (buffer.isDirect())
return DIRECT_BUFFER_HEAP_SIZE;
return DIRECT_BUFFER_DEEP_SIZE; // That might not be true if the buffer is a view of another buffer so we could undercount
int arrayLen = buffer.array().length;
int bufLen = buffer.remaining();
// if we're only referencing a sub-portion of the ByteBuffer, don't count the array overhead (assume it is SLAB
// allocated - the overhead amortized over all the allocations is negligible and better to undercount than over)
if (arrayLen > bufLen)
return EMPTY_HEAP_BUFFER_SIZE;
// If buffers are dedicated, account for byte array size and any padding overhead
return EMPTY_HEAP_BUFFER_SIZE + (arrayLen == 0 ? EMPTY_BYTE_ARRAY_SIZE : (sizeOfArray(arrayLen, 1) - arrayLen));
byte[] bytes = buffer.array();
return HEAP_BUFFER_SHALLOW_SIZE + meter.measureArray(bytes) - bytes.length;
}
/**
@ -201,13 +216,9 @@ public class ObjectSizes
* @param str String to calculate memory size of
* @return Total in-memory size of the String
*/
// TODO hard coding this to 2 isn't necessarily correct in Java 11
public static long sizeOf(String str)
{
if (str == null)
return 0;
return EMPTY_STRING_SIZE + sizeOfArray(str.length(), Character.BYTES);
return meter.measureStringDeep(str);
}
/**
@ -230,7 +241,7 @@ public class ObjectSizes
*/
public static long measureDeepOmitShared(Object pojo)
{
return omitSharedMeter.measureDeep(pojo);
return meter.measureDeep(pojo, ByteBufferMode.SLAB_ALLOCATION_NO_SLICE);
}
/**

View File

@ -43,6 +43,7 @@ import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.github.jamm.MemoryMeter;
import org.github.jamm.MemoryMeter.Guess;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
@ -66,10 +67,11 @@ import org.openjdk.jmh.annotations.Warmup;
@State(Scope.Benchmark)
public class ComparisonReadBench
{
// Note: To see a printout of the usage for each object, add .enableDebug() here (most useful with smaller number of
// Note: To see a printout of the usage for each object, add .printVisitedTree() here (most useful with smaller number of
// partitions).
static MemoryMeter meter = new MemoryMeter().ignoreKnownSingletons()
.withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE);
static MemoryMeter meter = MemoryMeter.builder()
.withGuessing(Guess.INSTRUMENTATION_AND_SPECIFICATION, Guess.UNSAFE)
.build();
@Param({"ON_HEAP"})
BufferType bufferType = BufferType.OFF_HEAP;

View File

@ -121,7 +121,7 @@ public class CellSpecTest
private static long valuePtrSize(Object value)
{
if (value instanceof ByteBuffer)
return ObjectSizes.sizeOnHeapExcludingData((ByteBuffer) value);
return ObjectSizes.sizeOnHeapExcludingDataOf((ByteBuffer) value);
else if (value instanceof byte[])
return ObjectSizes.sizeOfArray((byte[]) value) - ((byte[]) value).length;
throw new IllegalArgumentException("Unsupported type: " + value.getClass());

View File

@ -39,16 +39,20 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.utils.FBUtilities;
import org.github.jamm.MemoryMeter;
import org.github.jamm.MemoryMeter.Guess;
// Note: This test can be run in idea with the allocation type configured in the test yaml and memtable using the
// value memtableClass is initialized with.
@RunWith(Parameterized.class)
public class MemtableSizeTest extends CQLTester
{
// Note: To see a printout of the usage for each object, add .enableDebug() here (most useful with smaller number of
// Note: To see a printout of the usage for each object, add .printVisitedTree() here (most useful with smaller number of
// partitions).
static MemoryMeter meter = new MemoryMeter().ignoreKnownSingletons()
.withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE);
static MemoryMeter meter = MemoryMeter.builder()
.withGuessing(Guess.INSTRUMENTATION_AND_SPECIFICATION,
Guess.UNSAFE)
// .printVisitedTreeUpTo(1000)
.build();
static final Logger logger = LoggerFactory.getLogger(MemtableSizeTest.class);
@ -185,18 +189,6 @@ public class MemtableSizeTest extends CQLTester
FBUtilities.prettyPrintMemory(actualHeap),
FBUtilities.prettyPrintMemory(expectedHeap - actualHeap));
logger.info(message);
if (Math.abs(actualHeap - expectedHeap) > max_difference)
{
// Under Java 11, it seems the meter can reach into phantom reference queues and count more space than
// is actually reachable. Unfortunately ignoreNonStrongReferences() does not help (worse, it throws
// exceptions trying to get a phantom referrent). Retrying the measurement appears to clear these up.
Thread.sleep(50);
long secondPass = meter.measureDeep(memtable);
logger.error("Deep size first pass {} second pass {}",
FBUtilities.prettyPrintMemory(deepSizeAfter),
FBUtilities.prettyPrintMemory(secondPass));
expectedHeap = secondPass - deepSizeBefore;
}
Assert.assertTrue(message, Math.abs(actualHeap - expectedHeap) <= max_difference);
}

View File

@ -22,128 +22,123 @@ import java.nio.ByteBuffer;
import org.junit.Test;
import org.github.jamm.MemoryLayoutSpecification;
import org.github.jamm.MemoryMeter;
import org.github.jamm.MemoryMeter.Guess;
import static org.assertj.core.api.Assertions.assertThat;
import static org.github.jamm.MemoryMeter.ByteBufferMode.SLAB_ALLOCATION_NO_SLICE;
import static org.junit.Assert.assertEquals;
public class ObjectSizesTest
{
private static final MemoryMeter meter = new MemoryMeter().withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE).omitSharedBufferOverhead().ignoreKnownSingletons();
private static final long EMPTY_HEAP_BUFFER_RAW_SIZE = meter.measure(ByteBuffer.allocate(0));
private static final long EMPTY_OFFHEAP_BUFFER_RAW_SIZE = meter.measure(ByteBuffer.allocateDirect(0));
private static final ByteBuffer[] EMPTY_BYTE_BUFFER_ARRAY = new ByteBuffer[0];
public static final long REF_ARRAY_0_SIZE = MemoryLayoutSpecification.sizeOfArray(0, MemoryLayoutSpecification.SPEC.getReferenceSize());
public static final long REF_ARRAY_1_SIZE = MemoryLayoutSpecification.sizeOfArray(1, MemoryLayoutSpecification.SPEC.getReferenceSize());
public static final long REF_ARRAY_2_SIZE = MemoryLayoutSpecification.sizeOfArray(2, MemoryLayoutSpecification.SPEC.getReferenceSize());
public static final long BYTE_ARRAY_0_SIZE = MemoryLayoutSpecification.sizeOfArray(0, 1);
public static final long BYTE_ARRAY_10_SIZE = MemoryLayoutSpecification.sizeOfArray(10, 1);
public static final long BYTE_ARRAY_10_EXCEPT_DATA_SIZE = MemoryLayoutSpecification.sizeOfArray(10, 1) - 10;
private ByteBuffer buf10 = ByteBuffer.allocate(10);
private ByteBuffer prefixBuf8 = buf10.duplicate();
private ByteBuffer suffixBuf9 = buf10.duplicate();
private ByteBuffer infixBuf7 = buf10.duplicate();
{
prefixBuf8.limit(8);
suffixBuf9.position(1);
suffixBuf9 = suffixBuf9.slice();
infixBuf7.limit(8);
infixBuf7.position(1);
infixBuf7 = infixBuf7.slice();
}
// We use INSTRUMENTATION as principal strategy as it is our reference strategy
private static final MemoryMeter meter = MemoryMeter.builder()
.withGuessing(Guess.INSTRUMENTATION, Guess.UNSAFE)
.build();
@Test
public void testSizeOnHeapExcludingData()
{
// empty array of byte buffers
ByteBuffer[] buffers = EMPTY_BYTE_BUFFER_ARRAY;
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_0_SIZE);
// single empty heap buffer
buffers = new ByteBuffer[]{ ByteBuffer.allocate(0) };
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_0_SIZE);
checkBufferSizeExcludingData(ByteBuffer.allocate(0), 0);
// single non-empty heap buffer
buffers = new ByteBuffer[]{ buf10 };
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_10_EXCEPT_DATA_SIZE);
checkBufferSizeExcludingData(ByteBuffer.allocate(10), 10);
// single empty direct buffer
buffers = new ByteBuffer[]{ ByteBuffer.allocateDirect(0) };
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE);
checkBufferSizeExcludingData(ByteBuffer.allocateDirect(0), 0);
// single non-empty direct buffer
buffers = new ByteBuffer[]{ ByteBuffer.allocateDirect(10) };
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE);
checkBufferSizeExcludingData(ByteBuffer.allocateDirect(10), 0);
// two different empty byte buffers
buffers = new ByteBuffer[]{ ByteBuffer.allocate(0), ByteBuffer.allocateDirect(0) };
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_2_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_0_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE);
// heap buffer being a prefix slab
ByteBuffer buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().limit(8);
checkBufferSizeExcludingData(buffer, 8);
// two different non-empty byte buffers
buffers = new ByteBuffer[]{ buf10, ByteBuffer.allocateDirect(500) };
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_2_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_10_EXCEPT_DATA_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE);
// heap buffer being a suffix slab
buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1);
checkBufferSizeExcludingData(buffer, 9);
// heap buffer being a prefix slice of other buffer
buffers = new ByteBuffer[]{ prefixBuf8 };
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE);
// heap buffer being an infix slab
buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1).limit(8);
checkBufferSizeExcludingData(buffer, 7);
}
// heap buffer being a suffix slice of other buffer
buffers = new ByteBuffer[]{ suffixBuf9 };
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE);
private void checkBufferSizeExcludingData(ByteBuffer buffer, int dataSize)
{
assertEquals(meter.measureDeep(buffer, SLAB_ALLOCATION_NO_SLICE) - dataSize, ObjectSizes.sizeOnHeapExcludingDataOf(buffer));
}
// heap buffer being an infix slice of other buffer
buffers = new ByteBuffer[]{ infixBuf7 };
assertThat(ObjectSizes.sizeOnHeapExcludingData(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE);
@Test
public void testSizeOnHeapExcludingDataArray()
{
checkBufferSizeExcludingDataArray(0, new ByteBuffer[0]);
// single heap buffer
checkBufferSizeExcludingDataArray(0, ByteBuffer.allocate(0));
// multiple buffers
checkBufferSizeExcludingDataArray(10, ByteBuffer.allocate(0), ByteBuffer.allocate(10), ByteBuffer.allocateDirect(10));
// heap buffer being a prefix slab
ByteBuffer prefix = (ByteBuffer) ByteBuffer.allocate(10).duplicate().limit(8);
// heap buffer being a suffix slab
ByteBuffer suffix = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1);
checkBufferSizeExcludingDataArray(8 + 9, prefix, suffix);
}
private void checkBufferSizeExcludingDataArray(int dataSize, ByteBuffer... buffers)
{
assertEquals(meter.measureDeep(buffers, SLAB_ALLOCATION_NO_SLICE) - dataSize, ObjectSizes.sizeOnHeapExcludingDataOf(buffers));
}
@Test
public void testSizeOnHeapOf()
{
// empty array of byte buffers
ByteBuffer[] buffers = EMPTY_BYTE_BUFFER_ARRAY;
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_0_SIZE);
// single empty heap buffer
buffers = new ByteBuffer[]{ ByteBuffer.allocate(0) };
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_0_SIZE);
checkBufferSize(ByteBuffer.allocate(0));
// single non-empty heap buffer
buffers = new ByteBuffer[]{ buf10 };
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_10_SIZE);
checkBufferSize(ByteBuffer.allocate(10));
// single empty direct buffer
buffers = new ByteBuffer[]{ ByteBuffer.allocateDirect(0) };
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE);
checkBufferSize(ByteBuffer.allocateDirect(0));
// single non-empty direct buffer
buffers = new ByteBuffer[]{ ByteBuffer.allocateDirect(10) };
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE);
checkBufferSize(ByteBuffer.allocateDirect(10));
// two different empty byte buffers
buffers = new ByteBuffer[]{ ByteBuffer.allocate(0), ByteBuffer.allocateDirect(0) };
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_2_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_0_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE);
// heap buffer being a prefix slab
ByteBuffer buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().limit(8);
checkBufferSize(buffer);
// two different non-empty byte buffers
buffers = new ByteBuffer[]{ buf10, ByteBuffer.allocateDirect(500) };
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_2_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + BYTE_ARRAY_10_SIZE + EMPTY_OFFHEAP_BUFFER_RAW_SIZE);
// heap buffer being a suffix slab
buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1);
checkBufferSize(buffer);
// heap buffer being a prefix slice of other buffer
buffers = new ByteBuffer[]{ prefixBuf8 };
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + 8);
// heap buffer being an infix slab
buffer = (ByteBuffer) ByteBuffer.allocate(10).duplicate().position(1).limit(8);
checkBufferSize(buffer);
}
// heap buffer being a suffix slice of other buffer
buffers = new ByteBuffer[]{ suffixBuf9 };
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + 9);
private void checkBufferSize(ByteBuffer buffer)
{
assertEquals(meter.measureDeep(buffer, SLAB_ALLOCATION_NO_SLICE), ObjectSizes.sizeOnHeapOf(buffer));
}
// heap buffer being an infix slice of other buffer
buffers = new ByteBuffer[]{ infixBuf7 };
assertThat(ObjectSizes.sizeOnHeapOf(buffers)).isEqualTo(REF_ARRAY_1_SIZE + EMPTY_HEAP_BUFFER_RAW_SIZE + 7);
@Test
public void testSizeOnHeapOfArray()
{
checkBufferArraySize(new ByteBuffer[0]);
// single heap buffer
checkBufferArraySize(ByteBuffer.allocate(0));
// multiple buffers
checkBufferArraySize(ByteBuffer.allocate(0), ByteBuffer.allocate(10), ByteBuffer.allocateDirect(10));
}
private void checkBufferArraySize(ByteBuffer... buffers)
{
assertEquals(meter.measureDeep(buffers, SLAB_ALLOCATION_NO_SLICE), ObjectSizes.sizeOnHeapOf(buffers));
}
}