From 1e6493083f5d3a065f96994c7bf2d88975d3a7ba Mon Sep 17 00:00:00 2001 From: Sylvain Lebresne Date: Mon, 23 Jun 2014 17:50:08 +0200 Subject: [PATCH 1/2] Fix cqlsh DESC with CompositeType explanation patch by mstepura; reviewed by thobbs for CASSANDRA-7399 --- bin/cqlsh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/cqlsh b/bin/cqlsh index eca641b2e6..c3a4ea6e35 100755 --- a/bin/cqlsh +++ b/bin/cqlsh @@ -532,6 +532,10 @@ class Shell(cmd.Cmd): self.empty_lines = 0 self.statement_error = False self.single_statement = single_statement + #see CASSANDRA-7399 + type_for_composites = lambda cls: "'%s'" % cls.cass_parameterized_type_with(cls.subtypes, True) + cassandra.cqltypes.CompositeType.cql_parameterized_type = classmethod(type_for_composites) + cassandra.cqltypes.DynamicCompositeType.cql_parameterized_type = classmethod(type_for_composites) def set_expanded_cql_version(self, ver): ver, vertuple = full_cql_version(ver) From c33e77ae777d9c639be4ab76f840c69e3617c5a2 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Mon, 23 Jun 2014 16:52:50 +0100 Subject: [PATCH 2/2] Fix heap size calculation for CompoundSparseCellName/.WithCollection patch by Benedict Elliott Smith; reviewed by Dave Brosius CASSANDRA-7421 --- CHANGES.txt | 2 ++ .../cassandra/cql3/ColumnIdentifier.java | 2 +- .../cassandra/db/AtomicBTreeColumns.java | 4 ++-- .../org/apache/cassandra/db/BufferCell.java | 4 ++-- src/java/org/apache/cassandra/db/Cell.java | 2 +- .../org/apache/cassandra/db/NativeCell.java | 2 +- .../cassandra/db/NativeCounterCell.java | 2 +- .../cassandra/db/NativeDeletedCell.java | 2 +- .../cassandra/db/NativeExpiringCell.java | 2 +- .../apache/cassandra/db/RowIndexEntry.java | 2 +- .../cassandra/db/composites/CellName.java | 2 +- .../db/composites/CompoundComposite.java | 8 +++---- .../db/composites/CompoundDenseCellName.java | 2 +- .../db/composites/CompoundSparseCellName.java | 22 +++++++++++++++---- .../db/composites/SimpleDenseCellName.java | 2 +- .../db/composites/SimpleSparseCellName.java | 4 ++-- .../SimpleSparseInternedCellName.java | 2 +- .../cassandra/io/sstable/IndexHelper.java | 2 +- 18 files changed, 42 insertions(+), 26 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4f68cf7b69..04b148c20d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,6 @@ 2.1.0-rc2 + * Fix heap size calculation for CompoundSparseCellName and + CompoundSparseCellName.WithCollection (CASSANDRA-7421) * Allow counter mutations in UNLOGGED batches (CASSANDRA-7351) * Modify reconcile logic to always pick a tombstone over a counter cell (CASSANDRA-7346) diff --git a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java index 1b232cffd0..fa151d29a6 100644 --- a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java +++ b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java @@ -89,7 +89,7 @@ public class ColumnIdentifier implements Selectable, IMeasurableMemory + ObjectSizes.sizeOf(text); } - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(bytes) diff --git a/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java b/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java index 0e38784ef2..03cd0c56d0 100644 --- a/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java +++ b/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java @@ -344,7 +344,7 @@ public class AtomicBTreeColumns extends ColumnFamily indexer.insert(insert); insert = insert.localCopy(metadata, allocator, writeOp); this.dataSize += insert.cellDataSize(); - this.heapSize += insert.excessHeapSizeExcludingData(); + this.heapSize += insert.unsharedHeapSizeExcludingData(); if (inserted == null) inserted = new ArrayList<>(); inserted.add(insert); @@ -359,7 +359,7 @@ public class AtomicBTreeColumns extends ColumnFamily { reconciled = reconciled.localCopy(metadata, allocator, writeOp); dataSize += reconciled.cellDataSize() - existing.cellDataSize(); - heapSize += reconciled.excessHeapSizeExcludingData() - existing.excessHeapSizeExcludingData(); + heapSize += reconciled.unsharedHeapSizeExcludingData() - existing.unsharedHeapSizeExcludingData(); if (inserted == null) inserted = new ArrayList<>(); inserted.add(reconciled); diff --git a/src/java/org/apache/cassandra/db/BufferCell.java b/src/java/org/apache/cassandra/db/BufferCell.java index 93251c8d13..a7d632d908 100644 --- a/src/java/org/apache/cassandra/db/BufferCell.java +++ b/src/java/org/apache/cassandra/db/BufferCell.java @@ -84,9 +84,9 @@ public class BufferCell extends AbstractCell } @Override - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { - return EMPTY_SIZE + name.excessHeapSizeExcludingData() + ObjectSizes.sizeOnHeapExcludingData(value); + return EMPTY_SIZE + name.unsharedHeapSizeExcludingData() + ObjectSizes.sizeOnHeapExcludingData(value); } @Override diff --git a/src/java/org/apache/cassandra/db/Cell.java b/src/java/org/apache/cassandra/db/Cell.java index f91376db7d..7c3926ac9d 100644 --- a/src/java/org/apache/cassandra/db/Cell.java +++ b/src/java/org/apache/cassandra/db/Cell.java @@ -51,7 +51,7 @@ public interface Cell extends OnDiskAtom // returns the size of the Cell and all references on the heap, excluding any costs associated with byte arrays // that would be allocated by a localCopy, as these will be accounted for by the allocator - public long excessHeapSizeExcludingData(); + public long unsharedHeapSizeExcludingData(); public int serializedSize(CellNameType type, TypeSizes typeSizes); diff --git a/src/java/org/apache/cassandra/db/NativeCell.java b/src/java/org/apache/cassandra/db/NativeCell.java index 1c8ebd976a..dac5674690 100644 --- a/src/java/org/apache/cassandra/db/NativeCell.java +++ b/src/java/org/apache/cassandra/db/NativeCell.java @@ -75,7 +75,7 @@ public class NativeCell extends AbstractNativeCell } @Override - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { return SIZE; } diff --git a/src/java/org/apache/cassandra/db/NativeCounterCell.java b/src/java/org/apache/cassandra/db/NativeCounterCell.java index d0df17e366..3fe73ceb04 100644 --- a/src/java/org/apache/cassandra/db/NativeCounterCell.java +++ b/src/java/org/apache/cassandra/db/NativeCounterCell.java @@ -167,7 +167,7 @@ public class NativeCounterCell extends NativeCell implements CounterCell } @Override - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { return SIZE; } diff --git a/src/java/org/apache/cassandra/db/NativeDeletedCell.java b/src/java/org/apache/cassandra/db/NativeDeletedCell.java index 20118a4ff4..e900635969 100644 --- a/src/java/org/apache/cassandra/db/NativeDeletedCell.java +++ b/src/java/org/apache/cassandra/db/NativeDeletedCell.java @@ -112,7 +112,7 @@ public class NativeDeletedCell extends NativeCell implements DeletedCell } @Override - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { return SIZE; } diff --git a/src/java/org/apache/cassandra/db/NativeExpiringCell.java b/src/java/org/apache/cassandra/db/NativeExpiringCell.java index fcadb16403..f2655115a3 100644 --- a/src/java/org/apache/cassandra/db/NativeExpiringCell.java +++ b/src/java/org/apache/cassandra/db/NativeExpiringCell.java @@ -160,7 +160,7 @@ public class NativeExpiringCell extends NativeCell implements ExpiringCell } @Override - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { return SIZE; } diff --git a/src/java/org/apache/cassandra/db/RowIndexEntry.java b/src/java/org/apache/cassandra/db/RowIndexEntry.java index 618cd6176e..05c563c7d2 100644 --- a/src/java/org/apache/cassandra/db/RowIndexEntry.java +++ b/src/java/org/apache/cassandra/db/RowIndexEntry.java @@ -207,7 +207,7 @@ public class RowIndexEntry implements IMeasurableMemory { long entrySize = 0; for (IndexHelper.IndexInfo idx : columnsIndex) - entrySize += idx.excessHeapSize(); + entrySize += idx.unsharedHeapSize(); return BASE_SIZE + entrySize diff --git a/src/java/org/apache/cassandra/db/composites/CellName.java b/src/java/org/apache/cassandra/db/composites/CellName.java index 22b3fbd80a..4d778d3715 100644 --- a/src/java/org/apache/cassandra/db/composites/CellName.java +++ b/src/java/org/apache/cassandra/db/composites/CellName.java @@ -74,5 +74,5 @@ public interface CellName extends Composite @Override public CellName copy(CFMetaData cfm, AbstractAllocator allocator); - public long excessHeapSizeExcludingData(); + public long unsharedHeapSizeExcludingData(); } diff --git a/src/java/org/apache/cassandra/db/composites/CompoundComposite.java b/src/java/org/apache/cassandra/db/composites/CompoundComposite.java index e16b48a79b..7a21b01fe8 100644 --- a/src/java/org/apache/cassandra/db/composites/CompoundComposite.java +++ b/src/java/org/apache/cassandra/db/composites/CompoundComposite.java @@ -28,7 +28,7 @@ import org.apache.cassandra.utils.memory.AbstractAllocator; */ public class CompoundComposite extends AbstractComposite { - private static final long EMPTY_SIZE = ObjectSizes.measure(new CompoundComposite(null, 0, false)); + private static final long HEAP_SIZE = ObjectSizes.measure(new CompoundComposite(null, 0, false)); // We could use a List, but we'll create such object *a lot* and using a array+size is not // all that harder, so we save the List object allocation. @@ -73,12 +73,12 @@ public class CompoundComposite extends AbstractComposite public long unsharedHeapSize() { - return EMPTY_SIZE + ObjectSizes.sizeOnHeapOf(elements); + return HEAP_SIZE + ObjectSizes.sizeOnHeapOf(elements); } - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { - return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(elements); + return HEAP_SIZE + ObjectSizes.sizeOnHeapExcludingData(elements); } public Composite copy(CFMetaData cfm, AbstractAllocator allocator) diff --git a/src/java/org/apache/cassandra/db/composites/CompoundDenseCellName.java b/src/java/org/apache/cassandra/db/composites/CompoundDenseCellName.java index 0a157c2f5d..1f471a8894 100644 --- a/src/java/org/apache/cassandra/db/composites/CompoundDenseCellName.java +++ b/src/java/org/apache/cassandra/db/composites/CompoundDenseCellName.java @@ -73,7 +73,7 @@ public class CompoundDenseCellName extends CompoundComposite implements CellName } @Override - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { return HEAP_SIZE + ObjectSizes.sizeOnHeapExcludingData(elements); } diff --git a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java index ad7bb732a1..03af6d0190 100644 --- a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java +++ b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java @@ -50,6 +50,18 @@ public class CompoundSparseCellName extends CompoundComposite implements CellNam this.columnName = columnName; } + @Override + public long unsharedHeapSize() + { + return HEAP_SIZE + ObjectSizes.sizeOnHeapOf(elements); + } + + @Override + public long unsharedHeapSizeExcludingData() + { + return HEAP_SIZE + ObjectSizes.sizeOnHeapExcludingData(elements); + } + public int size() { return size + 1; @@ -156,13 +168,15 @@ public class CompoundSparseCellName extends CompoundComposite implements CellNam @Override public long unsharedHeapSize() { - return super.unsharedHeapSize() + ObjectSizes.sizeOnHeapOf(collectionElement); + return HEAP_SIZE + ObjectSizes.sizeOnHeapOf(elements) + + ObjectSizes.sizeOnHeapExcludingData(collectionElement); } @Override - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { - return super.excessHeapSizeExcludingData() + ObjectSizes.sizeOnHeapExcludingData(collectionElement); + return HEAP_SIZE + ObjectSizes.sizeOnHeapExcludingData(elements) + + ObjectSizes.sizeOnHeapExcludingData(collectionElement); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/composites/SimpleDenseCellName.java b/src/java/org/apache/cassandra/db/composites/SimpleDenseCellName.java index 5c0f9762fb..2ca7d23353 100644 --- a/src/java/org/apache/cassandra/db/composites/SimpleDenseCellName.java +++ b/src/java/org/apache/cassandra/db/composites/SimpleDenseCellName.java @@ -67,7 +67,7 @@ public class SimpleDenseCellName extends SimpleComposite implements CellName } @Override - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(element); } diff --git a/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java b/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java index a7fafba535..c6351f1dbd 100644 --- a/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java +++ b/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java @@ -87,9 +87,9 @@ public class SimpleSparseCellName extends AbstractComposite implements CellName return true; } - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { - return EMPTY_SIZE + columnName.excessHeapSizeExcludingData(); + return EMPTY_SIZE + columnName.unsharedHeapSizeExcludingData(); } public long unsharedHeapSize() diff --git a/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java b/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java index c5708d8243..c61372046e 100644 --- a/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java +++ b/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java @@ -31,7 +31,7 @@ public class SimpleSparseInternedCellName extends SimpleSparseCellName } @Override - public long excessHeapSizeExcludingData() + public long unsharedHeapSizeExcludingData() { return 0; } diff --git a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java index 72a708bafc..7099a151a2 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java +++ b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java @@ -205,7 +205,7 @@ public class IndexHelper } } - public long excessHeapSize() + public long unsharedHeapSize() { return EMPTY_SIZE + firstName.unsharedHeapSize() + lastName.unsharedHeapSize(); }