diff --git a/CHANGES.txt b/CHANGES.txt index 67985868f0..7a83979055 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,6 @@ +4.0.5 + * Optimise BTree build,update and transform operations (CASSANDRA-15510) + 4.0.4 * Clean up schema migration coordinator and tests (CASSANDRA-17533) * Shut repair task executor down without interruption to avoid compromising shared channel proxies (CASSANDRA-17466) diff --git a/build.xml b/build.xml index eafaeb50c5..4885ec0f03 100644 --- a/build.xml +++ b/build.xml @@ -1236,10 +1236,12 @@ - + + + diff --git a/src/java/org/apache/cassandra/db/Columns.java b/src/java/org/apache/cassandra/db/Columns.java index 929e75dce9..4372f3292e 100644 --- a/src/java/org/apache/cassandra/db/Columns.java +++ b/src/java/org/apache/cassandra/db/Columns.java @@ -40,7 +40,6 @@ import org.apache.cassandra.utils.SearchIterator; import org.apache.cassandra.utils.btree.BTree; import org.apache.cassandra.utils.btree.BTreeSearchIterator; import org.apache.cassandra.utils.btree.BTreeRemoval; -import org.apache.cassandra.utils.btree.UpdateFunction; /** * An immutable and sorted list of (non-PK) columns for a given table. @@ -264,8 +263,7 @@ public class Columns extends AbstractCollection implements Colle if (this == NONE) return other; - Object[] tree = BTree.merge(this.columns, other.columns, Comparator.naturalOrder(), - UpdateFunction.noOp()); + Object[] tree = BTree.update(this.columns, other.columns, Comparator.naturalOrder()); if (tree == this.columns) return this; if (tree == other.columns) diff --git a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java index 801d9e2338..70a9079752 100644 --- a/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java +++ b/src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java @@ -127,7 +127,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition updater.inputDeletionInfoCopy = update.deletionInfo().copy(HeapAllocator.instance); deletionInfo = current.deletionInfo.mutableCopy().add(updater.inputDeletionInfoCopy); - updater.allocated(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize()); + updater.onAllocatedOnHeap(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize()); } else { @@ -135,14 +135,14 @@ public final class AtomicBTreePartition extends AbstractBTreePartition } RegularAndStaticColumns columns = update.columns().mergeTo(current.columns); - updater.allocated(columns.unsharedHeapSize() - current.columns.unsharedHeapSize()); + updater.onAllocatedOnHeap(columns.unsharedHeapSize() - current.columns.unsharedHeapSize()); Row newStatic = update.staticRow(); Row staticRow = newStatic.isEmpty() ? current.staticRow : (current.staticRow.isEmpty() ? updater.apply(newStatic) : updater.apply(current.staticRow, newStatic)); - Object[] tree = BTree.update(current.tree, update.metadata().comparator, update, update.rowCount(), updater); + Object[] tree = BTree.update(current.tree, update.holder().tree, update.metadata().comparator, updater); EncodingStats newStats = current.stats.mergeWith(update.stats()); - updater.allocated(newStats.unsharedHeapSize() - current.stats.unsharedHeapSize()); + updater.onAllocatedOnHeap(newStats.unsharedHeapSize() - current.stats.unsharedHeapSize()); if (tree != null && refUpdater.compareAndSet(this, current, new Holder(columns, tree, deletionInfo, staticRow, newStats))) { @@ -371,7 +371,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition indexer.onInserted(insert); this.dataSize += data.dataSize(); - allocated(data.unsharedHeapSizeExcludingData()); + onAllocatedOnHeap(data.unsharedHeapSizeExcludingData()); if (inserted == null) inserted = new ArrayList<>(); inserted.add(data); @@ -388,7 +388,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition indexer.onUpdated(existing, reconciled); dataSize += reconciled.dataSize() - existing.dataSize(); - allocated(reconciled.unsharedHeapSizeExcludingData() - existing.unsharedHeapSizeExcludingData()); + onAllocatedOnHeap(reconciled.unsharedHeapSizeExcludingData() - existing.unsharedHeapSizeExcludingData()); if (inserted == null) inserted = new ArrayList<>(); inserted.add(reconciled); @@ -408,7 +408,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition return updating.ref != ref; } - public void allocated(long heapSize) + public void onAllocatedOnHeap(long heapSize) { this.heapSize += heapSize; } diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java index 9530e10d3a..a310156b66 100644 --- a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java +++ b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java @@ -858,8 +858,8 @@ public class PartitionUpdate extends AbstractBTreePartition // assert that we are not calling build() several times assert !isBuilt : "A PartitionUpdate.Builder should only get built once"; Object[] add = rowBuilder.build(); - Object[] merged = BTree.merge(tree, add, metadata.comparator, - UpdateFunction.Simple.of(Rows::merge)); + Object[] merged = BTree.update(tree, add, metadata.comparator, + UpdateFunction.Simple.of(Rows::merge)); EncodingStats newStats = EncodingStats.Collector.collect(staticRow, BTree.iterator(merged), deletionInfo); @@ -907,7 +907,7 @@ public class PartitionUpdate extends AbstractBTreePartition public Builder updateAllTimestamp(long newTimestamp) { deletionInfo.updateAllTimestamp(newTimestamp - 1); - tree = BTree.transformAndFilter(tree, (x) -> x.updateAllTimestamp(newTimestamp)); + tree = BTree.transformAndFilter(tree, (x) -> x.updateAllTimestamp(newTimestamp)); staticRow = this.staticRow.updateAllTimestamp(newTimestamp); return this; } diff --git a/src/java/org/apache/cassandra/db/rows/BTreeRow.java b/src/java/org/apache/cassandra/db/rows/BTreeRow.java index bd44b666fd..6ae224a607 100644 --- a/src/java/org/apache/cassandra/db/rows/BTreeRow.java +++ b/src/java/org/apache/cassandra/db/rows/BTreeRow.java @@ -720,7 +720,7 @@ public class BTreeRow extends AbstractRow } } - Object[] btree = BTree.build(buildFrom, UpdateFunction.noOp()); + Object[] btree = BTree.build(buildFrom); return new ComplexColumnData(column, btree, deletion); } } diff --git a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java index 9f35437633..1add2e32b5 100644 --- a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java +++ b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java @@ -20,6 +20,7 @@ package org.apache.cassandra.db.rows; import java.nio.ByteBuffer; import java.util.Iterator; import java.util.Objects; +import java.util.function.BiFunction; import com.google.common.base.Function; @@ -35,6 +36,7 @@ import org.apache.cassandra.schema.DroppedColumn; import org.apache.cassandra.utils.BiLongAccumulator; import org.apache.cassandra.utils.LongAccumulator; import org.apache.cassandra.utils.ObjectSizes; +import org.apache.cassandra.utils.SearchIterator; import org.apache.cassandra.utils.btree.BTree; /** @@ -98,6 +100,11 @@ public class ComplexColumnData extends ColumnData implements Iterable> return BTree.iterator(cells); } + public SearchIterator searchIterator() + { + return BTree.slice(cells, column().asymmetricCellPathComparator(), BTree.Dir.ASC); + } + public Iterator> reverseIterator() { return BTree.iterator(cells, BTree.Dir.DESC); @@ -195,17 +202,25 @@ public class ComplexColumnData extends ColumnData implements Iterable> return transformAndFilter(complexDeletion, (cell) -> filter.fetchedCellIsQueried(column, cell.path()) ? null : cell); } - private ComplexColumnData transformAndFilter(DeletionTime newDeletion, Function, ? extends Cell> function) + private ComplexColumnData update(DeletionTime newDeletion, Object[] newCells) { - Object[] transformed = BTree.transformAndFilter(cells, function); - - if (cells == transformed && newDeletion == complexDeletion) + if (cells == newCells && newDeletion == complexDeletion) return this; - if (newDeletion == DeletionTime.LIVE && BTree.isEmpty(transformed)) + if (newDeletion == DeletionTime.LIVE && BTree.isEmpty(newCells)) return null; - return new ComplexColumnData(column, transformed, newDeletion); + return new ComplexColumnData(column, newCells, newDeletion); + } + + public ComplexColumnData transformAndFilter(DeletionTime newDeletion, Function function) + { + return update(newDeletion, BTree.transformAndFilter(cells, function)); + } + + public ComplexColumnData transformAndFilter(BiFunction function, V param) + { + return update(complexDeletion, BTree.transformAndFilter(cells, function, param)); } public ComplexColumnData updateAllTimestamp(long newTimestamp) diff --git a/src/java/org/apache/cassandra/db/rows/Row.java b/src/java/org/apache/cassandra/db/rows/Row.java index 5c28cd1b42..be9206f208 100644 --- a/src/java/org/apache/cassandra/db/rows/Row.java +++ b/src/java/org/apache/cassandra/db/rows/Row.java @@ -732,7 +732,7 @@ public interface Row extends Unfiltered, Iterable // Because some data might have been shadowed by the 'activeDeletion', we could have an empty row return rowInfo.isEmpty() && rowDeletion.isLive() && dataBuffer.isEmpty() ? null - : BTreeRow.create(clustering, rowInfo, rowDeletion, BTree.build(dataBuffer, UpdateFunction.noOp())); + : BTreeRow.create(clustering, rowInfo, rowDeletion, BTree.build(dataBuffer)); } public Clustering mergedClustering() diff --git a/src/java/org/apache/cassandra/utils/BulkIterator.java b/src/java/org/apache/cassandra/utils/BulkIterator.java new file mode 100644 index 0000000000..5450523b68 --- /dev/null +++ b/src/java/org/apache/cassandra/utils/BulkIterator.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.util.Iterator; + +import org.apache.cassandra.utils.caching.TinyThreadLocalPool; + +public interface BulkIterator extends AutoCloseable +{ + void fetch(Object[] into, int offset, int count); + V next(); + default void close() {}; + + public static class FromArray implements BulkIterator, AutoCloseable + { + private static final TinyThreadLocalPool POOL = new TinyThreadLocalPool<>(); + + private Object[] from; + private int i; + private TinyThreadLocalPool.TinyPool pool; + + private void init(Object[] from, int offset) + { + this.from = from; + this.i = offset; + } + + public void close() + { + pool.offer(this); + from = null; + pool = null; + } + + public void fetch(Object[] into, int offset, int count) + { + System.arraycopy(from, i, into, offset, count); + i += count; + } + + public V next() + { + return (V) from[i++]; + } + } + + public static class Adapter implements BulkIterator + { + final Iterator adapt; + + private Adapter(Iterator adapt) + { + this.adapt = adapt; + } + + public void fetch(Object[] into, int offset, int count) + { + count += offset; + while (offset < count && adapt.hasNext()) + into[offset++] = adapt.next(); + } + + public boolean hasNext() + { + return adapt.hasNext(); + } + + public V next() + { + return adapt.next(); + } + } + + public static FromArray of(Object[] from) + { + return of(from, 0); + } + + public static FromArray of(Object[] from, int offset) + { + TinyThreadLocalPool.TinyPool pool = FromArray.POOL.get(); + FromArray result = pool.poll(); + if (result == null) + result = new FromArray<>(); + result.init(from, offset); + result.pool = pool; + return result; + } + + public static Adapter of(Iterator from) + { + return new Adapter<>(from); + } +} + diff --git a/src/java/org/apache/cassandra/utils/btree/BTree.java b/src/java/org/apache/cassandra/utils/btree/BTree.java index 6c546ac95e..9a66deb2ca 100644 --- a/src/java/org/apache/cassandra/utils/btree/BTree.java +++ b/src/java/org/apache/cassandra/utils/btree/BTree.java @@ -20,283 +20,535 @@ package org.apache.cassandra.utils.btree; import java.util.*; import java.util.function.BiConsumer; +import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; import com.google.common.collect.Iterators; import com.google.common.collect.Ordering; import org.apache.cassandra.utils.BiLongAccumulator; +import org.apache.cassandra.utils.BulkIterator; import org.apache.cassandra.utils.LongAccumulator; import org.apache.cassandra.utils.ObjectSizes; +import org.apache.cassandra.utils.caching.TinyThreadLocalPool; -import static com.google.common.collect.Iterables.concat; -import static com.google.common.collect.Iterables.filter; -import static com.google.common.collect.Iterables.transform; import static java.lang.Math.max; import static java.lang.Math.min; -import static java.util.Comparator.naturalOrder; public class BTree { /** - * Leaf Nodes are a raw array of values: Object[V1, V1, ...,]. - * - * Branch Nodes: Object[V1, V2, ..., child[<V1.key], child[<V2.key], ..., child[< Inf], size], where - * each child is another node, i.e., an Object[]. Thus, the value elements in a branch node are the - * first half of the array (minus one). In our implementation, each value must include its own key; - * we access these via Comparator, rather than directly. - * - * So we can quickly distinguish between leaves and branches, we require that leaf nodes are always an odd number - * of elements (padded with a null, if necessary), and branches are always an even number of elements. - * + * The {@code BRANCH_FACTOR} is defined as the maximum number of children of each branch, with between + * BRANCH_FACTOR/2-1 and BRANCH_FACTOR-1 keys being stored in every node. This yields a minimum tree size of + * {@code (BRANCH_FACTOR/2)^height - 1} and a maximum tree size of {@code BRANCH_FACTOR^height - 1}. + *

+ * Branches differ from leaves only in that they contain a suffix region containing the child nodes that occur + * either side of the keys, and a sizeMap in the last position, permitting seeking by index within the tree. + * Nodes are disambiguated by the length of the array that represents them: an even number is a branch, odd a leaf. + *

+ * Leaf Nodes are represented by an odd-length array of keys, with the final element possibly null, i.e. + * Object[V1, V2, ...,null?] + *

+ * Branch nodes: Object[V1, V2, ..., child[<V1.key], child[<V2.key], ..., child[< Inf], sizeMap] + * Each child is either a branch or leaf, i.e., always an Object[]. + * The key elements in a branch node occupy the first half of the array (minus one) + *

* BTrees are immutable; updating one returns a new tree that reuses unmodified nodes. - * - * There are no references back to a parent node from its children. (This would make it impossible to re-use - * subtrees when modifying the tree, since the modified tree would need new parent references.) + *

+ * There are no references back to a parent node from its children (this would make it impossible to re-use + * subtrees when modifying the tree, since the modified tree would need new parent references). * Instead, we store these references in a Path as needed when navigating the tree. */ + public static final int BRANCH_SHIFT = Integer.getInteger("cassandra.btree.branchshift", 5); - // The maximum fan factor used for B-Trees - static final int FAN_SHIFT; - - // The maximun tree size for certain heigth of tree - static final int[] TREE_SIZE; - - // NB we encode Path indexes as Bytes, so this needs to be less than Byte.MAX_VALUE / 2 - static final int FAN_FACTOR; - - static final int MAX_TREE_SIZE = Integer.MAX_VALUE; - - static - { - int fanfactor = Integer.parseInt(System.getProperty("cassandra.btree.fanfactor", "32")); - assert fanfactor >= 2 : "the minimal btree fanfactor is 2"; - int shift = 1; - while (1 << shift < fanfactor) - shift += 1; - - FAN_SHIFT = shift; - FAN_FACTOR = 1 << FAN_SHIFT; - - // For current FAN_FACTOR, calculate the maximum height of the tree we could build - int maxHeight = 0; - for (long maxSize = 0; maxSize < MAX_TREE_SIZE; maxHeight++) - // each tree node could have (FAN_FACTOR + 1) children, - // plus current node could have FAN_FACTOR number of values - maxSize = maxSize * (FAN_FACTOR + 1) + FAN_FACTOR; - - TREE_SIZE = new int[maxHeight]; - - TREE_SIZE[0] = FAN_FACTOR; - for (int i = 1; i < maxHeight - 1; i++) - TREE_SIZE[i] = TREE_SIZE[i - 1] * (FAN_FACTOR + 1) + FAN_FACTOR; - - TREE_SIZE[maxHeight - 1] = MAX_TREE_SIZE; - } - - - static final int MINIMAL_NODE_SIZE = FAN_FACTOR >> 1; + private static final int BRANCH_FACTOR = 1 << BRANCH_SHIFT; + public static final int MIN_KEYS = BRANCH_FACTOR / 2 - 1; + public static final int MAX_KEYS = BRANCH_FACTOR - 1; // An empty BTree Leaf - which is the same as an empty BTree - static final Object[] EMPTY_LEAF = new Object[1]; + private static final Object[] EMPTY_LEAF = new Object[1]; - // An empty BTree branch - used only for internal purposes in Modifier - static final Object[] EMPTY_BRANCH = new Object[] { null, new int[0] }; + private static final int[][] DENSE_SIZE_MAPS = buildBalancedSizeMaps(BRANCH_SHIFT); + private static final long[] PERFECT_DENSE_SIZE_ON_HEAP = sizeOnHeapOfPerfectTrees(BRANCH_SHIFT); - // direction of iteration - public static enum Dir + /** + * Represents the direction of iteration. + */ + public enum Dir { ASC, DESC; - public Dir invert() { return this == ASC ? DESC : ASC; } - public static Dir asc(boolean asc) { return asc ? ASC : DESC; } - public static Dir desc(boolean desc) { return desc ? DESC : ASC; } + + public Dir invert() + { + return this == ASC ? DESC : ASC; + } + + public static Dir desc(boolean desc) + { + return desc ? DESC : ASC; + } } /** - * Enables methods to consume the contents of iterators, collections, or arrays without duplicating code or - * allocating intermediate objects. Instead of taking an argument that implements an interface, a method takes - * an opaque object as the input, and a singleton helper object it uses as an intermediary to access it's contents. - * The purpose of doing things this way is to avoid memory allocations on hot paths. + * Returns an empty BTree + * + * @return an empty BTree */ - private interface IteratingFunction - { - /** - * Returns the next object at the given index. This method must be called with sequentially increasing index - * values, starting at 0, and must only be called once per index value. The results of calling this method - * without following these rules are undefined. - */ - K nextAt(T input, int idx); - } - - private static final IteratingFunction ITERATOR_FUNCTION = new IteratingFunction() - { - public K nextAt(Iterator input, int idx) - { - return (K) input.next(); - } - }; - - private static final IteratingFunction ARRAY_FUNCTION = new IteratingFunction() - { - public K nextAt(Object[] input, int idx) - { - return (K) input[idx]; - } - }; - public static Object[] empty() { return EMPTY_LEAF; } + /** + * Create a BTree containing only the specified object + * + * @return an new BTree containing only the specified object + */ public static Object[] singleton(Object value) { - return new Object[] { value }; + return new Object[]{ value }; } + @Deprecated + public static Object[] build(Collection source) + { + return build(source, UpdateFunction.noOp()); + } + + @Deprecated public static Object[] build(Collection source, UpdateFunction updateF) { - return buildInternal(source.iterator(), ITERATOR_FUNCTION, source.size(), updateF); + return build(BulkIterator.of(source.iterator()), source.size(), updateF); } - /** - * Creates a BTree containing all of the objects in the provided collection - * - * @param source the items to build the tree with. MUST BE IN STRICTLY ASCENDING ORDER. - * @param size the size of the source iterable - * @return a btree representing the contents of the provided iterable - */ - public static Object[] build(Iterable source, int size, UpdateFunction updateF) - { - if (size < 0) - throw new IllegalArgumentException(Integer.toString(size)); - return buildInternal(source.iterator(), ITERATOR_FUNCTION, size, updateF); - } - - public static Object[] build(Object[] source, int size, UpdateFunction updateF) - { - if (size < 0) - throw new IllegalArgumentException(Integer.toString(size)); - return buildInternal(source, ARRAY_FUNCTION, size, updateF); - } - - private static Object[] buildLeaf(S source, IteratingFunction iterFunc, int size, int startIdx, UpdateFunction updateF) - { - V[] values = (V[]) new Object[size | 1]; - - int idx = startIdx; - for (int i = 0; i < size; i++) - { - K k = iterFunc.nextAt(source, idx); - values[i] = updateF.apply(k); - idx++; - } - if (updateF != UpdateFunction.noOp()) - updateF.allocated(ObjectSizes.sizeOfArray(values)); - return values; - } - - private static Object[] buildInternal(S source, IteratingFunction iterFunc, int size, int level, int startIdx, UpdateFunction updateF) - { - assert size > 0; - assert level >= 0; - if (level == 0) - return buildLeaf(source, iterFunc, size, startIdx, updateF); - - // calcuate child num: (size - (childNum - 1)) / maxChildSize <= childNum - int childNum = size / (TREE_SIZE[level - 1] + 1) + 1; - - V[] values = (V[]) new Object[childNum * 2]; - if (updateF != UpdateFunction.noOp()) - updateF.allocated(ObjectSizes.sizeOfArray(values)); - - int[] indexOffsets = new int[childNum]; - int childPos = childNum - 1; - - int index = 0; - for (int i = 0; i < childNum - 1; i++) - { - // Calculate the next childSize by splitting the remaining values to the remaining child nodes. - // The performance could be improved by pre-compute the childSize (see #9989 comments). - int childSize = (size - index) / (childNum - i); - // Build the tree with inorder traversal - values[childPos + i] = (V) buildInternal(source, iterFunc, childSize, level - 1, startIdx + index, updateF); - index += childSize; - indexOffsets[i] = index; - - K k = iterFunc.nextAt(source, startIdx + index); - values[i] = updateF.apply(k); - index++; - } - - values[childPos + childNum - 1] = (V) buildInternal(source, iterFunc, size - index, level - 1, startIdx + index, updateF); - indexOffsets[childNum - 1] = size; - - values[childPos + childNum] = (V) indexOffsets; - - return values; - } - - private static Object[] buildInternal(S source, IteratingFunction iterFunc, int size, UpdateFunction updateF) + public static Object[] build(BulkIterator source, int size, UpdateFunction updateF) { assert size >= 0; if (size == 0) return EMPTY_LEAF; - // find out the height of the tree - int level = 0; - while (size > TREE_SIZE[level]) - level++; - return buildInternal(source, iterFunc, size, level, 0, updateF); - } + if (size <= MAX_KEYS) + return buildLeaf(source, size, updateF); - public static Object[] update(Object[] btree, - Comparator comparator, - Collection updateWith, - UpdateFunction updateF) - { - return update(btree, comparator, updateWith, updateWith.size(), updateF); + return buildRoot(source, size, updateF); } /** - * Returns a new BTree with the provided collection inserting/replacing as necessary any equal items - * - * @param btree the tree to update - * @param comparator the comparator that defines the ordering over the items in the tree - * @param updateWith the items to either insert / update. MUST BE IN STRICTLY ASCENDING ORDER. - * @param updateWithLength then number of elements in updateWith - * @param updateF the update function to apply to any pairs we are swapping, and maybe abort early - * @param - * @return + * Build a leaf with {@code size} elements taken in bulk from {@code insert}, and apply {@code updateF} to these elements */ - public static Object[] update(Object[] btree, - Comparator comparator, - Iterable updateWith, - int updateWithLength, - UpdateFunction updateF) + private static Object[] buildLeaf(BulkIterator insert, + int size, + UpdateFunction updateF) { - if (isEmpty(btree)) - return build(updateWith, updateWithLength, updateF); - - - TreeBuilder builder = TreeBuilder.newInstance(); - btree = builder.update(btree, comparator, updateWith, updateF); - return btree; + Object[] values = new Object[size | 1]; // ensure that we have an odd-length array + insert.fetch(values, 0, size); + if (!isSimple(updateF)) + { + updateF.onAllocatedOnHeap(ObjectSizes.sizeOfReferenceArray(values.length)); + for (int i = 0; i < size; i++) + values[i] = updateF.apply((I) values[i]); + } + return values; } - public static Object[] merge(Object[] tree1, Object[] tree2, Comparator comparator, UpdateFunction updateF) + /** + * Build a leaf with {@code size} elements taken in bulk from {@code insert}, and apply {@code updateF} to these elements + * Do not invoke {@code updateF.onAllocated}. Used by {@link #buildPerfectDenseWithoutSizeTracking} which + * track the size for the entire tree they build in order to save on work. + */ + private static Object[] buildLeafWithoutSizeTracking(BulkIterator insert, int size, UpdateFunction updateF) { - if (size(tree1) < size(tree2)) + Object[] values = new Object[size | 1]; // ensure that we have an odd-length array + insert.fetch(values, 0, size); + if (!isSimple(updateF)) { - Object[] tmp = tree1; - tree1 = tree2; - tree2 = tmp; + for (int i = 0; i < size; i++) + values[i] = updateF.apply((I) values[i]); + } + return values; + } + + /** + * Build a root node from the first {@code size} elements from {@code source}, applying {@code updateF} to those elements. + * A root node is permitted to have as few as two children, if a branch (i.e. if {@code size > MAX_SIZE}. + */ + private static Object[] buildRoot(BulkIterator source, int size, UpdateFunction updateF) + { + // first calculate the minimum height needed for this size of tree + int height = minHeight(size); + assert height > 1; + assert height * BRANCH_SHIFT < 32; + + int denseChildSize = denseSize(height - 1); + // Divide the size by the child size + 1, adjusting size by +1 to compensate for not having an upper key on the + // last child and rounding up, i.e. (size + 1 + div - 1) / div == size / div + 1 where div = childSize + 1 + int childCount = size / (denseChildSize + 1) + 1; + + return buildMaximallyDense(source, childCount, size, height, updateF); + } + + /** + * Build a tree containing only dense nodes except at most two on any level. This matches the structure that + * a FastBuilder would create, with some optimizations in constructing the dense nodes. + *

+ * We do this by repeatedly constructing fully dense children until we reach a threshold, chosen so that we would + * not be able to create another child with fully dense children and at least MIN_KEYS keys. After the threshold, + * the remainder may fit a single node, or is otherwise split roughly halfway to create one child with at least + * MIN_KEYS+1 fully dense children, and one that has at least MIN_KEYS-1 fully dense and up to two non-dense. + */ + private static Object[] buildMaximallyDense(BulkIterator source, + int childCount, + int size, + int height, + UpdateFunction updateF) + { + assert childCount <= MAX_KEYS + 1; + + int keyCount = childCount - 1; + int[] sizeMap = new int[childCount]; + Object[] branch = new Object[childCount * 2]; + + if (height == 2) + { + // we use the _exact same logic_ as below, only we invoke buildLeaf + int remaining = size; + int threshold = MAX_KEYS + 1 + MIN_KEYS; + int i = 0; + while (remaining >= threshold) + { + branch[keyCount + i] = buildLeaf(source, MAX_KEYS, updateF); + branch[i] = isSimple(updateF) ? source.next() : updateF.apply(source.next()); + remaining -= MAX_KEYS + 1; + sizeMap[i++] = size - remaining - 1; + } + if (remaining > MAX_KEYS) + { + int childSize = remaining / 2; + branch[keyCount + i] = buildLeaf(source, childSize, updateF); + branch[i] = isSimple(updateF) ? source.next() : updateF.apply(source.next()); + remaining -= childSize + 1; + sizeMap[i++] = size - remaining - 1; + } + branch[keyCount + i] = buildLeaf(source, remaining, updateF); + sizeMap[i++] = size; + assert i == childCount; + } + else + { + --height; + int denseChildSize = denseSize(height); + int denseGrandChildSize = denseSize(height - 1); + // The threshold is the point after which we can't add a dense child and still add another child with + // at least MIN_KEYS fully dense children plus at least one more key. + int threshold = denseChildSize + 1 + MIN_KEYS * (denseGrandChildSize + 1); + int remaining = size; + int i = 0; + // Add dense children until we reach the threshold. + while (remaining >= threshold) + { + branch[keyCount + i] = buildPerfectDense(source, height, updateF); + branch[i] = isSimple(updateF) ? source.next() : updateF.apply(source.next()); + remaining -= denseChildSize + 1; + sizeMap[i++] = size - remaining - 1; + } + // At this point the remainder either fits in one child, or too much for one but too little for one + // perfectly dense and a second child with enough grandchildren to be valid. In the latter case, the + // remainder should be split roughly in half, where the first child only has dense grandchildren. + if (remaining > denseChildSize) + { + int grandChildCount = remaining / ((denseGrandChildSize + 1) * 2); + assert grandChildCount >= MIN_KEYS + 1; + int childSize = grandChildCount * (denseGrandChildSize + 1) - 1; + branch[keyCount + i] = buildMaximallyDense(source, grandChildCount, childSize, height, updateF); + branch[i] = isSimple(updateF) ? source.next() : updateF.apply(source.next()); + remaining -= childSize + 1; + sizeMap[i++] = size - remaining - 1; + } + // Put the remainder in the last child, it is now guaranteed to fit and have the required minimum of children. + int grandChildCount = remaining / (denseGrandChildSize + 1) + 1; + assert grandChildCount >= MIN_KEYS + 1; + int childSize = remaining; + branch[keyCount + i] = buildMaximallyDense(source, grandChildCount, childSize, height, updateF); + sizeMap[i++] = size; + assert i == childCount; + } + + branch[2 * keyCount + 1] = sizeMap; + if (!isSimple(updateF)) + updateF.onAllocatedOnHeap(ObjectSizes.sizeOfArray(branch) + ObjectSizes.sizeOfArray(sizeMap)); + + return branch; + } + + private static Object[] buildPerfectDense(BulkIterator source, int height, UpdateFunction updateF) + { + Object[] result = buildPerfectDenseWithoutSizeTracking(source, height, updateF); + updateF.onAllocatedOnHeap(PERFECT_DENSE_SIZE_ON_HEAP[height]); + return result; + } + + /** + * Build a tree of size precisely {@code branchFactor^height - 1} + */ + private static Object[] buildPerfectDenseWithoutSizeTracking(BulkIterator source, int height, UpdateFunction updateF) + { + int keyCount = (1 << BRANCH_SHIFT) - 1; + Object[] node = new Object[(1 << BRANCH_SHIFT) * 2]; + + if (height == 2) + { + int childSize = treeSize2n(1, BRANCH_SHIFT); + for (int i = 0; i < keyCount; i++) + { + node[keyCount + i] = buildLeafWithoutSizeTracking(source, childSize, updateF); + node[i] = isSimple(updateF) ? source.next() : updateF.apply(source.next()); + } + node[2 * keyCount] = buildLeafWithoutSizeTracking(source, childSize, updateF); + } + else + { + for (int i = 0; i < keyCount; i++) + { + Object[] child = buildPerfectDenseWithoutSizeTracking(source, height - 1, updateF); + node[keyCount + i] = child; + node[i] = isSimple(updateF) ? source.next() : updateF.apply(source.next()); + } + node[2 * keyCount] = buildPerfectDenseWithoutSizeTracking(source, height - 1, updateF); + } + node[keyCount * 2 + 1] = DENSE_SIZE_MAPS[height - 2]; + + return node; + } + + public static Object[] update(Object[] toUpdate, Object[] insert, Comparator comparator) + { + return BTree.update(toUpdate, insert, comparator, UpdateFunction.noOp()); + } + + /** + * Inserts {@code insert} into {@code update}, applying {@code updateF} to each new item in {@code insert}, + * as well as any matched items in {@code update}. + *

+ * Note that {@code UpdateFunction.noOp} is assumed to indicate a lack of interest in which value survives. + */ + public static Object[] update(Object[] update, Object[] insert, Comparator comparator, UpdateFunction updateF) + { + // perform some initial obvious optimisations + if (isEmpty(insert)) + return update; // do nothing if update is empty + + if (isEmpty(update)) + { + if (isSimple(updateF)) + return insert; // if update is empty and updateF is trivial, return our new input + + // if update is empty and updateF is non-trivial, perform a simple fast transformation of the input tree + insert = BTree.transform(insert, updateF); + updateF.onAllocatedOnHeap(sizeOnHeapOf(insert)); + return insert; + } + + if (isLeaf(update) && isLeaf(insert)) + { + // if both are leaves, perform a tight-loop leaf variant of update + // possibly flipping the input order if sizes suggest and updateF permits + if (updateF == (UpdateFunction) UpdateFunction.noOp && update.length < insert.length) + { + Object[] tmp = update; + update = insert; + insert = tmp; + } + return updateLeaves(update, insert, comparator, updateF); + } + + if (!isLeaf(insert) && isSimple(updateF)) + { + // consider flipping the order of application, if update is much larger than insert and applying unary no-op + int updateSize = size(update); + int insertSize = size(insert); + int scale = Integer.numberOfLeadingZeros(updateSize) - Integer.numberOfLeadingZeros(insertSize); + if (scale >= 4) + { + // i.e. at roughly 16x the size, or one tier deeper - very arbitrary, should pick more carefully + // experimentally, at least at 64x the size the difference in performance is ~10x + Object[] tmp = insert; + insert = update; + update = tmp; + if (updateF != (UpdateFunction) UpdateFunction.noOp) + updateF = ((UpdateFunction.Simple) updateF).flip(); + } + } + + try (Updater updater = Updater.get()) + { + return updater.update(update, insert, comparator, updateF); + } + } + + /** + * A fast tight-loop variant of updating one btree with another, when both are leaves. + */ + public static Object[] updateLeaves(Object[] unode, Object[] inode, Comparator comparator, UpdateFunction updateF) + { + int upos = -1, usz = sizeOfLeaf(unode); + Existing uk = (Existing) unode[0]; + int ipos = 0, isz = sizeOfLeaf(inode); + Insert ik = (Insert) inode[0]; + + Existing merged = null; + int c = -1; + while (c <= 0) // optimistic: find the first point in the original leaf that is modified (if any) + { + if (c < 0) + { + upos = exponentialSearch(comparator, unode, upos + 1, usz, ik); + c = upos < 0 ? 1 : 0; // positive or zero + if (upos < 0) + upos = -(1 + upos); + if (upos == usz) + break; + uk = (Existing) unode[upos]; + } + else // c == 0 + { + merged = updateF.apply(uk, ik); + if (merged != uk) + break; + if (++ipos == isz) + return unode; + if (++upos == usz) + break; + c = comparator.compare(uk = (Existing) unode[upos], ik = (Insert) inode[ipos]); + } + } + + // exit conditions: c == 0 && merged != uk + // or: c > 0 + // or: upos == usz + + try (FastBuilder builder = fastBuilder()) + { + if (upos > 0) + { + // copy any initial section that is unmodified + System.arraycopy(unode, 0, builder.leaf().buffer, 0, upos); + builder.leaf().count = upos; + } + + // handle prior loop's exit condition + // we always have either an ik, or an ik merged with uk, to handle + if (upos < usz) + { + if (c == 0) + { + builder.add(merged); + if (++upos < usz) + uk = (Existing) unode[upos]; + } + else // c > 0 + { + builder.add(updateF.apply(ik)); + } + if (++ipos < isz) + ik = (Insert) inode[ipos]; + + if (upos < usz && ipos < isz) + { + // note: this code is _identical_ to equivalent section in FastUpdater + c = comparator.compare(uk, ik); + while (true) + { + if (c == 0) + { + builder.leaf().addKey(updateF.apply(uk, ik)); + ++upos; + ++ipos; + if (upos == usz || ipos == isz) + break; + c = comparator.compare(uk = (Existing) unode[upos], ik = (Insert) inode[ipos]); + } + else if (c < 0) + { + int until = exponentialSearch(comparator, unode, upos + 1, usz, ik); + c = until < 0 ? 1 : 0; // must find greater or equal; set >= 0 (equal) to 0; set < 0 (greater) to c=+ve + if (until < 0) + until = -(1 + until); + builder.leaf().copy(unode, upos, until - upos); + if ((upos = until) == usz) + break; + uk = (Existing) unode[upos]; + } + else + { + int until = exponentialSearch(comparator, inode, ipos + 1, isz, uk); + c = until & 0x80000000; // must find less or equal; set >= 0 (equal) to 0, otherwise leave intact + if (until < 0) + until = -(1 + until); + builder.leaf().copy(inode, ipos, until - ipos, updateF); + if ((ipos = until) == isz) + break; + ik = (Insert) inode[ipos]; + } + } + } + if (upos < usz) + { + // ipos == isz + builder.leaf().copy(unode, upos, usz - upos); + } + } + if (ipos < isz) + { + // upos == usz + builder.leaf().copy(inode, ipos, isz - ipos, updateF); + } + return builder.build(); + } + } + + public static void reverseInSitu(Object[] tree) + { + reverseInSitu(tree, height(tree), true); + } + + /** + * The internal implementation of {@link #reverseInSitu(Object[])}. + * Takes two arguments that help minimise garbage generation, by testing sizeMaps against + * known globallyl shared sizeMap for dense nodes that do not need to be modified, and + * for permitting certain users (namely FastBuilder) to declare that non-matching sizeMap + * can be mutated directly without allocating {@code new int[]} + * + * @param tree the tree to reverse in situ + * @param height the height of the tree + * @param copySizeMaps whether or not to copy any non-globally-shared sizeMap before reversing them + */ + private static void reverseInSitu(Object[] tree, int height, boolean copySizeMaps) + { + if (isLeaf(tree)) + { + reverse(tree, 0, sizeOfLeaf(tree)); + } + else + { + int keyCount = shallowSizeOfBranch(tree); + reverse(tree, 0, keyCount); + reverse(tree, keyCount, keyCount * 2 + 1); + for (int i = keyCount; i <= keyCount * 2; ++i) + reverseInSitu((Object[]) tree[i], height - 1, copySizeMaps); + + int[] sizeMap = (int[]) tree[2 * keyCount + 1]; + if (sizeMap != DENSE_SIZE_MAPS[height - 2]) // no need to reverse a dense map; same in both directions + { + if (copySizeMaps) + sizeMap = sizeMap.clone(); + sizeMapToSizes(sizeMap); + reverse(sizeMap, 0, sizeMap.length); + sizesToSizeMap(sizeMap); + } } - return update(tree1, comparator, new BTreeSet<>(tree2, comparator), updateF); } public static Iterator iterator(Object[] btree) @@ -334,8 +586,8 @@ public class BTree /** * Returns an Iterator over the entire tree * - * @param btree the tree to iterate over - * @param dir direction of iteration + * @param btree the tree to iterate over + * @param dir direction of iteration * @param * @return */ @@ -350,8 +602,8 @@ public class BTree * @param comparator the comparator that defines the ordering over the items in the tree * @param start the beginning of the range to return, inclusive (in ascending order) * @param end the end of the range to return, exclusive (in ascending order) - * @param dir if false, the iterator will start at the last item and move backwards - * @return an Iterator over the defined sub-range of the tree + * @param dir if false, the iterator will start at the last item and move backwards + * @return an Iterator over the defined sub-range of the tree */ public static BTreeSearchIterator slice(Object[] btree, Comparator comparator, K start, K end, Dir dir) { @@ -361,10 +613,10 @@ public class BTree /** * @param btree the tree to iterate over * @param comparator the comparator that defines the ordering over the items in the tree - * @param startIndex the start index of the range to return, inclusive - * @param endIndex the end index of the range to return, inclusive - * @param dir if false, the iterator will start at the last item and move backwards - * @return an Iterator over the defined sub-range of the tree + * @param startIndex the start index of the range to return, inclusive + * @param endIndex the end index of the range to return, inclusive + * @param dir if false, the iterator will start at the last item and move backwards + * @return an Iterator over the defined sub-range of the tree */ public static BTreeSearchIterator slice(Object[] btree, Comparator comparator, int startIndex, int endIndex, Dir dir) { @@ -380,7 +632,7 @@ public class BTree * @param end high bound of the range * @param endInclusive inclusivity of higher bound * @param dir direction of iteration - * @return an Iterator over the defined sub-range of the tree + * @return an Iterator over the defined sub-range of the tree */ public static BTreeSearchIterator slice(Object[] btree, Comparator comparator, K start, boolean startInclusive, K end, boolean endInclusive, Dir dir) { @@ -439,7 +691,7 @@ public class BTree return; } - boundary = -1 -boundary; + boundary = -1 - boundary; if (boundary > 0) { assert boundary < sizeMap.length; @@ -479,6 +731,7 @@ public class BTree /** * Honours result semantics of {@link Arrays#binarySearch}, as though it were performed on the tree flattened into an array + * * @return index of item in tree, or (-(insertion point) - 1) if not present */ public static int findIndex(Object[] node, Comparator comparator, V find) @@ -534,7 +787,7 @@ public class BTree return (V) node[boundary]; } - boundary = -1 -boundary; + boundary = -1 - boundary; if (boundary > 0) { assert boundary < sizeMap.length; @@ -555,7 +808,7 @@ public class BTree { int i = findIndex(btree, comparator, find); if (i < 0) - i = -1 -i; + i = -1 - i; return i - 1; } @@ -569,7 +822,7 @@ public class BTree { int i = findIndex(btree, comparator, find); if (i < 0) - i = -2 -i; + i = -2 - i; return i; } @@ -582,8 +835,10 @@ public class BTree public static int higherIndex(Object[] btree, Comparator comparator, V find) { int i = findIndex(btree, comparator, find); - if (i < 0) i = -1 -i; - else i++; + if (i < 0) + i = -1 - i; + else + i++; return i; } @@ -597,7 +852,7 @@ public class BTree { int i = findIndex(btree, comparator, find); if (i < 0) - i = -1 -i; + i = -1 - i; return i; } @@ -689,14 +944,19 @@ public class BTree long size = ObjectSizes.sizeOfArray(tree); if (isLeaf(tree)) return size; - for (int i = getChildStart(tree) ; i < getChildEnd(tree) ; i++) + for (int i = getChildStart(tree); i < getChildEnd(tree); i++) size += sizeOfStructureOnHeap((Object[]) tree[i]); return size; } - // returns true if the provided node is a leaf, false if it is a branch + /** + * Checks is the node is a leaf. + * + * @return {@code true} if the provided node is a leaf, {@code false} if it is a branch. + */ public static boolean isLeaf(Object[] node) { + // Nodes are disambiguated by the length of the array that represents them: an even number is a branch, odd a leaf return (node.length & 1) == 1; } @@ -705,6 +965,44 @@ public class BTree return tree == EMPTY_LEAF; } + // get the upper bound we should search in for keys in the node + static int shallowSize(Object[] node) + { + if (isLeaf(node)) + return sizeOfLeaf(node); + else + return shallowSizeOfBranch(node); + } + + static int sizeOfLeaf(Object[] leaf) + { + int len = leaf.length; + return leaf[len - 1] == null ? len - 1 : len; + } + + // return the boundary position between keys/children for the branch node + // == number of keys, as they are indexed from zero + static int shallowSizeOfBranch(Object[] branch) + { + return (branch.length / 2) - 1; + } + + /** + * @return first index in a branch node containing child nodes + */ + static int childOffset(Object[] branch) + { + return shallowSizeOfBranch(branch); + } + + /** + * @return last index + 1 in a branch node containing child nodes + */ + static int childEndOffset(Object[] branch) + { + return branch.length - 1; + } + public static int depth(Object[] tree) { int depth = 1; @@ -718,8 +1016,9 @@ public class BTree /** * Fill the target array with the contents of the provided subtree, in ascending order, starting at targetOffset - * @param tree source - * @param target array + * + * @param tree source + * @param target array * @param targetOffset offset in target array * @return number of items copied (size of tree) */ @@ -727,6 +1026,7 @@ public class BTree { return toArray(tree, 0, size(tree), target, targetOffset); } + public static int toArray(Object[] tree, int treeStart, int treeEnd, Object[] target, int targetOffset) { if (isLeaf(tree)) @@ -739,7 +1039,7 @@ public class BTree int newTargetOffset = targetOffset; int childCount = getChildCount(tree); int childOffset = getChildStart(tree); - for (int i = 0 ; i < childCount ; i++) + for (int i = 0; i < childCount; i++) { int childStart = treeIndexOffsetOfChild(tree, i); int childEnd = treeIndexOfBranchKey(tree, i); @@ -754,76 +1054,260 @@ public class BTree return newTargetOffset - targetOffset; } - // simple class for avoiding duplicate transformation work - private static class FiltrationTracker implements Function + /** + * An efficient transformAndFilter implementation suitable for a tree consisting of a single leaf root + * NOTE: codewise *identical* to {@link #transformAndFilterLeaf(Object[], BiFunction, Object)} + */ + private static Object[] transformAndFilterLeaf(Object[] leaf, Function apply) { - final Function wrapped; - int index; - boolean failed; - - private FiltrationTracker(Function wrapped) + int i = 0, sz = sizeOfLeaf(leaf); + I in; + O out; + do // optimistic loop, looking for first point transformation modifies the input (if any) { - this.wrapped = wrapped; + in = (I) leaf[i]; + out = apply.apply(in); + } while (in == out && ++i < sz); + + // in == out -> i == sz + // otherwise in == leaf[i] + int identicalUntil = i; + + if (out == null && ++i < sz) + { + // optimistic loop, looking for first key {@code apply} modifies without removing it (if any) + do + { + in = (I) leaf[i]; + out = apply.apply(in); + } while (null == out && ++i < sz); + } + // out == null -> i == sz + // otherwise out == apply.apply(leaf[i]) + + if (i == sz) + { + // if we have reached the end of the input, we're either: + // 1) returning input unmodified; or + // 2) copying some (possibly empty) prefix of it + + if (identicalUntil == sz) + return leaf; + + if (identicalUntil == 0) + return empty(); + + Object[] copy = new Object[identicalUntil | 1]; + System.arraycopy(leaf, 0, copy, 0, identicalUntil); + return copy; } - public V apply(V i) + try (FastBuilder builder = fastBuilder()) { - V o = wrapped.apply(i); - if (o != null) index++; - else failed = true; - return o; + // otherwise copy the initial part that was unmodified, insert the non-null modified key, and continue + if (identicalUntil > 0) + builder.leaf().copyNoOverflow(leaf, 0, identicalUntil); + builder.leaf().addKeyNoOverflow(out); + + while (++i < sz) + { + in = (I) leaf[i]; + out = apply.apply(in); + if (out != null) + builder.leaf().addKeyNoOverflow(out); + } + + return builder.build(); } } /** - * Takes a btree and transforms it using the provided function, filtering out any null results. - * The result of any transformation must sort identically wrt the other results as their originals + * Takes a tree and transforms it using the provided function, filtering out any null results. + * The result of any transformation must sort identically as their originals, wrt other results. + *

+ * If no modifications are made, the original is returned. + * NOTE: codewise *identical* to {@link #transformAndFilter(Object[], Function)} */ - public static Object[] transformAndFilter(Object[] btree, Function function) + public static Object[] transformAndFilter(Object[] tree, BiFunction apply, I2 param) { - if (isEmpty(btree)) - return btree; + if (isEmpty(tree)) + return tree; - // TODO: can be made more efficient - FiltrationTracker wrapped = new FiltrationTracker<>(function); - Object[] result = transformAndFilter(btree, wrapped); - if (!wrapped.failed) - return result; + if (isLeaf(tree)) + return transformAndFilterLeaf(tree, apply, param); - // take the already transformed bits from the head of the partial result - Iterable head = iterable(result, 0, wrapped.index - 1, Dir.ASC); - // and concatenate with remainder of original tree, with transformation applied - Iterable remainder = iterable(btree, wrapped.index + 1, size(btree) - 1, Dir.ASC); - remainder = filter(transform(remainder, function), (x) -> x != null); - Iterable build = concat(head, remainder); - - Builder builder = builder((Comparator) naturalOrder()); - builder.auto(false); - for (V v : build) - builder.add(v); - return builder.build(); + try (BiTransformer transformer = BiTransformer.get(apply, param)) + { + return transformer.apply(tree); + } } - private static Object[] transformAndFilter(Object[] btree, FiltrationTracker function) + /** + * Takes a tree and transforms it using the provided function, filtering out any null results. + * The result of any transformation must sort identically as their originals, wrt other results. + *

+ * If no modifications are made, the original is returned. + *

+ * An efficient transformAndFilter implementation suitable for a tree consisting of a single leaf root + * NOTE: codewise *identical* to {@link #transformAndFilter(Object[], BiFunction, Object)} + */ + public static Object[] transformAndFilter(Object[] tree, Function apply) { - Object[] result = btree; - boolean isLeaf = isLeaf(btree); - int childOffset = isLeaf ? Integer.MAX_VALUE : getChildStart(btree); - int limit = isLeaf ? getLeafKeyEnd(btree) : btree.length - 1; - for (int i = 0 ; i < limit ; i++) + if (isEmpty(tree)) + return tree; + + if (isLeaf(tree)) + return transformAndFilterLeaf(tree, apply); + + try (Transformer transformer = Transformer.get(apply)) { - // we want to visit in iteration order, so we visit our key nodes inbetween our children - int idx = isLeaf ? i : (i / 2) + (i % 2 == 0 ? childOffset : 0); - Object current = btree[idx]; - Object updated = idx < childOffset ? function.apply((V) current) : transformAndFilter((Object[]) current, function); - if (updated != current) + return transformer.apply(tree); + } + } + + /** + * An efficient transformAndFilter implementation suitable for a tree consisting of a single leaf root + * NOTE: codewise *identical* to {@link #transformAndFilterLeaf(Object[], Function)} + */ + private static Object[] transformAndFilterLeaf(Object[] leaf, BiFunction apply, I2 param) + { + int i = 0, sz = sizeOfLeaf(leaf); + I in; + O out; + do // optimistic loop, looking for first point transformation modifies the input (if any) + { + in = (I) leaf[i]; + out = apply.apply(in, param); + } while (in == out && ++i < sz); + + // in == out -> i == sz + // otherwise in == leaf[i] + int identicalUntil = i; + + if (out == null && ++i < sz) + { + // optimistic loop, looking for first key {@code apply} modifies without removing it (if any) + do { - if (result == btree) - result = btree.clone(); - result[idx] = updated; + in = (I) leaf[i]; + out = apply.apply(in, param); + } while (null == out && ++i < sz); + } + // out == null -> i == sz + // otherwise out == apply.apply(leaf[i]) + + if (i == sz) + { + // if we have reached the end of the input, we're either: + // 1) returning input unmodified; or + // 2) copying some (possibly empty) prefix of it + + if (identicalUntil == sz) + return leaf; + + if (identicalUntil == 0) + return empty(); + + Object[] copy = new Object[identicalUntil | 1]; + System.arraycopy(leaf, 0, copy, 0, identicalUntil); + return copy; + } + + try (FastBuilder builder = fastBuilder()) + { + // otherwise copy the initial part that was unmodified, insert the non-null modified key, and continue + if (identicalUntil > 0) + builder.leaf().copyNoOverflow(leaf, 0, identicalUntil); + builder.leaf().addKeyNoOverflow(out); + + while (++i < sz) + { + in = (I) leaf[i]; + out = apply.apply(in, param); + if (out != null) + builder.leaf().addKeyNoOverflow(out); } - if (function.failed) - return result; + + return builder.build(); + } + } + + /** + * Takes a tree and transforms it using the provided function. + * The result of any transformation must sort identically as their originals, wrt other results. + *

+ * If no modifications are made, the original is returned. + */ + public static Object[] transform(Object[] tree, Function function) + { + if (isEmpty(tree)) // isEmpty determined by identity; must return input + return tree; + + if (isLeaf(tree)) // escape hatch for fast leaf transformation + return transformLeaf(tree, function); + + Object[] result = tree; // optimistically assume we'll return our input unmodified + int keyCount = shallowSizeOfBranch(tree); + for (int i = 0; i < keyCount; ++i) + { + // operate on a pair of (child,key) each loop + Object[] curChild = (Object[]) tree[keyCount + i]; + Object[] updChild = transform(curChild, function); + Object curKey = tree[i]; + Object updKey = function.apply((I) curKey); + if (result == tree) + { + if (curChild == updChild && curKey == updKey) + continue; // if output still same as input, loop + + // otherwise initialise output to a copy of input up to this point + result = transformCopyBranchHelper(tree, keyCount, i, i); + } + result[keyCount + i] = updChild; + result[i] = updKey; + } + // final unrolled copy of loop for last child only (unbalanced with keys) + Object[] curChild = (Object[]) tree[2 * keyCount]; + Object[] updChild = transform(curChild, function); + if (result == tree) + { + if (curChild == updChild) + return tree; + result = transformCopyBranchHelper(tree, keyCount, keyCount, keyCount); + } + result[2 * keyCount] = updChild; + result[2 * keyCount + 1] = tree[2 * keyCount + 1]; // take the original sizeMap, as we are exactly the same shape + return result; + } + + // create a copy of a branch, with the exact same size, copying the specified number of keys and children + private static Object[] transformCopyBranchHelper(Object[] branch, int keyCount, int copyKeyCount, int copyChildCount) + { + Object[] result = new Object[branch.length]; + System.arraycopy(branch, 0, result, 0, copyKeyCount); + System.arraycopy(branch, keyCount, result, keyCount, copyChildCount); + return result; + } + + // an efficient transformAndFilter implementation suitable for a tree consisting of a single leaf root + private static Object[] transformLeaf(Object[] leaf, Function apply) + { + Object[] result = leaf; // optimistically assume we'll return our input unmodified + int size = sizeOfLeaf(leaf); + for (int i = 0; i < size; ++i) + { + Object current = leaf[i]; + Object updated = apply.apply((I) current); + if (result == leaf) + { + if (current == updated) + continue; // if output still same as input, loop + + // otherwise initialise output to a copy of input up to this point + result = new Object[leaf.length]; + System.arraycopy(leaf, 0, result, 0, i); + } + result[i] = updated; } return result; } @@ -841,14 +1325,51 @@ public class BTree for (Object v : iterable(btree)) result = 31 * result + Objects.hashCode(v); return result; + } + public static String toString(Object[] btree) + { + return appendBranchOrLeaf(new StringBuilder().append('['), btree).append(']').toString(); + } + + private static StringBuilder appendBranchOrLeaf(StringBuilder builder, Object[] node) + { + return isLeaf(node) ? appendLeaf(builder, node) : appendBranch(builder, node); + } + + private static StringBuilder appendBranch(StringBuilder builder, Object[] branch) + { + int childCount = branch.length / 2; + int keyCount = childCount - 1; + // add keys + for (int i = 0; i < keyCount; i++) + { + if (i != 0) + builder.append(", "); + builder.append(branch[i]); + } + // add children + for (int i = keyCount, m = branch.length - 1; i < m; i++) + { + builder.append(", "); + appendBranchOrLeaf(builder, (Object[]) branch[i]); + } + // add sizeMap + builder.append(", ").append(Arrays.toString((int[]) branch[branch.length - 1])); + return builder; + } + + private static StringBuilder appendLeaf(StringBuilder builder, Object[] leaf) + { + return builder.append(Arrays.toString(leaf)); } /** * tree index => index of key wrt all items in the tree laid out serially - * + *

* This version of the method permits requesting out-of-bounds indexes, -1 and size - * @param root to calculate tree index within + * + * @param root to calculate tree index within * @param keyIndex root-local index of key to calculate tree-index * @return the number of items preceding the key in the whole tree of root */ @@ -875,7 +1396,7 @@ public class BTree } /** - * @param root to calculate tree-index within + * @param root to calculate tree-index within * @param keyIndex root-local index of key to calculate tree-index of * @return the number of items preceding the key in the whole tree of root */ @@ -885,7 +1406,7 @@ public class BTree } /** - * @param root to calculate tree-index within + * @param root to calculate tree-index within * @param childIndex root-local index of *child* to calculate tree-index of * @return the number of items preceding the child in the whole tree of root */ @@ -964,6 +1485,7 @@ public class BTree /** * Creates a copy of this {@code Builder}. + * * @return a copy of this {@code Builder}. */ public Builder copy() @@ -1127,7 +1649,7 @@ public class BTree } else { - a[newCount++] = c < 0 ? a[i++] : a[j++]; + a[newCount++] = c < 0 ? a[i++] : a[j++]; } } @@ -1156,7 +1678,7 @@ public class BTree { assert !auto; int mid = count / 2; - for (int i = 0 ; i < mid ; i++) + for (int i = 0; i < mid; i++) { Object t = values[i]; values[i] = values[count - (1 + i)]; @@ -1179,7 +1701,7 @@ public class BTree sort(); int prevIdx = 0; V prev = (V) values[0]; - for (int i = 1 ; i < count ; i++) + for (int i = 1; i < count; i++) { V next = (V) values[i]; if (comparator.compare(prev, next) != 0) @@ -1198,7 +1720,7 @@ public class BTree { int c = 0; int prev = 0; - for (int i = 1 ; i < count ; i++) + for (int i = 1; i < count; i++) { if (comparator.compare((V) values[i], (V) values[prev]) != 0) { @@ -1216,79 +1738,13 @@ public class BTree { if (auto) autoEnforce(); - return BTree.build(values, count, UpdateFunction.noOp()); + try (BulkIterator iterator = BulkIterator.of(values, 0)) + { + return BTree.build(iterator, count, UpdateFunction.noOp()); + } } } - /** simple static wrapper to calls to cmp.compare() which checks if either a or b are Special (i.e. represent an infinity) */ - static int compare(Comparator cmp, Object a, Object b) - { - if (a == b) - return 0; - if (a == NEGATIVE_INFINITY | b == POSITIVE_INFINITY) - return -1; - if (b == NEGATIVE_INFINITY | a == POSITIVE_INFINITY) - return 1; - return cmp.compare((V) a, (V) b); - } - - static Object POSITIVE_INFINITY = new Object(); - static Object NEGATIVE_INFINITY = new Object(); - - public static boolean isWellFormed(Object[] btree, Comparator cmp) - { - return isWellFormed(cmp, btree, true, NEGATIVE_INFINITY, POSITIVE_INFINITY); - } - - private static boolean isWellFormed(Comparator cmp, Object[] node, boolean isRoot, Object min, Object max) - { - if (cmp != null && !isNodeWellFormed(cmp, node, min, max)) - return false; - - if (isLeaf(node)) - { - if (isRoot) - return node.length <= FAN_FACTOR + 1; - return node.length >= FAN_FACTOR / 2 && node.length <= FAN_FACTOR + 1; - } - - final int keyCount = getBranchKeyEnd(node); - if ((!isRoot && keyCount < FAN_FACTOR / 2) || keyCount > FAN_FACTOR + 1) - return false; - - int type = 0; - int size = -1; - int[] sizeMap = getSizeMap(node); - // compare each child node with the branch element at the head of this node it corresponds with - for (int i = getChildStart(node); i < getChildEnd(node) ; i++) - { - Object[] child = (Object[]) node[i]; - size += size(child) + 1; - if (sizeMap[i - getChildStart(node)] != size) - return false; - Object localmax = i < node.length - 2 ? node[i - getChildStart(node)] : max; - if (!isWellFormed(cmp, child, false, min, localmax)) - return false; - type |= isLeaf(child) ? 1 : 2; - min = localmax; - } - return type < 3; // either all leaves or all branches but not a mix - } - - private static boolean isNodeWellFormed(Comparator cmp, Object[] node, Object min, Object max) - { - Object previous = min; - int end = getKeyEnd(node); - for (int i = 0; i < end; i++) - { - Object current = node[i]; - if (compare(cmp, previous, current) >= 0) - return false; - previous = current; - } - return compare(cmp, previous, max) < 0; - } - private static void applyValue(V value, BiConsumer function, A argument) { function.accept(argument, value); @@ -1298,13 +1754,13 @@ public class BTree { Preconditions.checkArgument(isLeaf(btree)); int limit = getLeafKeyEnd(btree); - for (int i=0; i * Private method * * @param btree @@ -1320,7 +1776,7 @@ public class BTree int childOffset = getChildStart(btree); int limit = btree.length - 1 - childOffset; - for (int i = 0 ; i < limit ; i++) + for (int i = 0; i < limit; i++) { apply((Object[]) btree[childOffset + i], function, argument); @@ -1332,7 +1788,7 @@ public class BTree /** * Simple method to walk the btree forwards and apply a function till a stop condition is reached - * + *

* Private method * * @param btree @@ -1383,7 +1839,7 @@ public class BTree /** * Walk the btree and accumulate a long value using the supplied accumulator function. Iteration will stop if the * accumulator function returns the sentinel values Long.MIN_VALUE or Long.MAX_VALUE - * + *

* If the optional from argument is not null, iteration will start from that value (or the one after it's insertion * point if an exact match isn't found) */ @@ -1413,7 +1869,7 @@ public class BTree } int limit = btree.length - 1 - childOffset; - for (int i=startChild; i full size at height = (1 << (branchShift * height)) - 1 + // => full size at height + 1 = 1 << (branchShift * height) + // => shift(full size at height + 1) = branchShift * height + // => shift(full size at height + 1) / branchShift = height + int lengthInBinary = 64 - Long.numberOfLeadingZeros(size); + return (branchShift - 1 + lengthInBinary) / branchShift; + } + + private static int[][] buildBalancedSizeMaps(int branchShift) + { + int count = (32 / branchShift) - 1; + int childCount = 1 << branchShift; + int[][] sizeMaps = new int[count][childCount]; + for (int height = 0; height < count; ++height) + { + int childSize = treeSize2n(height + 1, branchShift); + int size = 0; + int[] sizeMap = sizeMaps[height]; + for (int i = 0; i < childCount; ++i) + { + sizeMap[i] = size += childSize; + size += 1; + } + } + return sizeMaps; + } + + // simply utility to reverse the contents of array[from..to) + private static void reverse(Object[] array, int from, int to) + { + int mid = (from + to) / 2; + for (int i = from; i < mid; i++) + { + int j = to - (1 + i - from); + Object tmp = array[i]; + array[i] = array[j]; + array[j] = tmp; + } + } + + // simply utility to reverse the contents of array[from..to) + private static void reverse(int[] array, int from, int to) + { + int mid = (from + to) / 2; + for (int i = from; i < mid; i++) + { + int j = to - (1 + i - from); + int tmp = array[i]; + array[i] = array[j]; + array[j] = tmp; + } + } + + /** + * Mutate an array of child sizes into a cumulative sizeMap, returning the total size + */ + private static int sizesToSizeMap(int[] sizeMap) + { + int total = sizeMap[0]; + for (int i = 1; i < sizeMap.length; ++i) + sizeMap[i] = total += 1 + sizeMap[i]; + return total; + } + + private static int sizesToSizeMap(int[] sizes, int count) + { + int total = sizes[0]; + for (int i = 1; i < count; ++i) + sizes[i] = total += 1 + sizes[i]; + return total; + } + + /** + * Mutate an array of child sizes into a cumulative sizeMap, returning the total size + */ + private static void sizeMapToSizes(int[] sizeMap) + { + for (int i = sizeMap.length; i > 1; --i) + sizeMap[i] -= 1 + sizeMap[i - 1]; + } + + /** + * A simple utility method to handle a null upper bound that we treat as infinity + */ + private static int compareWithMaybeInfinity(Comparator comparator, Compare key, Compare ub) + { + if (ub == null) + return -1; + return comparator.compare(key, ub); + } + + /** + * Perform {@link #exponentialSearch} on {@code in[from..to)}, treating a {@code find} of {@code null} as infinity. + */ + static int exponentialSearchForMaybeInfinity(Comparator comparator, Object[] in, int from, int to, Compare find) + { + if (find == null) + return -(1 + to); + return exponentialSearch(comparator, in, from, to, find); + } + + /** + * Equivalent to {@link Arrays#binarySearch}, only more efficient algorithmically for linear merges. + * Binary search has worst case complexity {@code O(n.lg n)} for a linear merge, whereas exponential search + * has a worst case of {@code O(n)}. However compared to a simple linear merge, the best case for exponential + * search is {@code O(lg(n))} instead of {@code O(n)}. + */ + private static int exponentialSearch(Comparator comparator, Object[] in, int from, int to, Compare find) + { + int step = 0; + while (from + step < to) + { + int i = from + step; + int c = comparator.compare(find, (Compare) in[i]); + if (c < 0) + { + to = i; + break; + } + if (c == 0) + return i; + from = i + 1; + step = step * 2 + 1; // jump in perfect binary search increments + } + return Arrays.binarySearch((Compare[]) in, from, to, find, comparator); + } + + /** + * Perform {@link #exponentialSearch} on {@code in[from..to)}; if the value falls outside of the range of these + * elements, test against {@code ub} as though it occurred at position {@code to} + * + * @return same as {@link Arrays#binarySearch} if {@code find} occurs in the range {@code [in[from]..in[to])}; + * otherwise the insertion position {@code -(1+to)} if {@code find} is less than {@code ub}, and {@code -(2+t)} + * if it is greater than or equal to. + *

+ * {@code ub} may be {@code null}, representing infinity. + */ + static int exponentialSearchWithUpperBound(Comparator comparator, Object[] in, int from, int to, Compare ub, Compare find) + { + int step = 0; + while (true) + { + int i = from + step; + if (i >= to) + { + int c = compareWithMaybeInfinity(comparator, find, ub); + if (c >= 0) + return -(2 + to); + break; + } + int c = comparator.compare(find, (Compare) in[i]); + if (c < 0) + { + to = i; + break; + } + if (c == 0) + return i; + from = i + 1; + step = step * 2 + 1; // jump in perfect binary search increments + } + return Arrays.binarySearch((Compare[]) in, from, to, find, comparator); + } + + /** + * Compute the size-in-bytes of full trees of cardinality {@code branchFactor^height - 1} + */ + private static long[] sizeOnHeapOfPerfectTrees(int branchShift) + { + long[] result = new long[heightAtSize2n(Integer.MAX_VALUE, branchShift)]; + int branchFactor = 1 << branchShift; + result[0] = branchFactor - 1; + for (int i = 1; i < result.length; ++i) + result[i] = sizeOnHeapOfPerfectTree(i + 1, branchFactor); + return result; + } + + /** + * Compute the size-in-bytes of a full tree of cardinality {@code branchFactor^height - 1} + * TODO: test + */ + private static long sizeOnHeapOfPerfectTree(int height, int branchShift) + { + int branchFactor = 1 << branchShift; + long branchSize = ObjectSizes.sizeOfReferenceArray(branchFactor * 2); + int branchCount = height == 2 ? 1 : 2 + treeSize2n(height - 2, branchShift); + long leafSize = ObjectSizes.sizeOfReferenceArray((branchFactor - 1) | 1); + int leafCount = 1 + treeSize2n(height - 1, branchShift); + return (branchSize * branchCount) + (leafSize * leafCount); + } + + /** + * @return the actual height of {@code tree} + */ + public static int height(Object[] tree) + { + if (isLeaf(tree)) + return 1; + + int height = 1; + while (!isLeaf(tree)) + { + height++; + tree = (Object[]) tree[shallowSizeOfBranch(tree)]; + } + return height; + } + + /** + * @return the maximum representable size at {@code height}. + */ + private static int denseSize(int height) + { + return treeSize2n(height, BRANCH_SHIFT); + } + + /** + * @return the maximum representable size at {@code height}. + */ + private static int checkedDenseSize(int height) + { + assert height * BRANCH_SHIFT < 32; + return denseSize(height); + } + + /** + * Computes the number of nodes in a full tree of height {@code height} + * and with {@code 2^branchShift} branch factor. + * i.e. computes {@code (2^branchShift)^height - 1} + */ + private static int treeSize2n(int height, int branchShift) + { + return (1 << (branchShift * height)) - 1; + } + + // TODO: test + private static int maxRootHeight(int size) + { + if (size <= BRANCH_FACTOR) + return 1; + return 1 + heightAtSize2n((size - 1) / 2, BRANCH_SHIFT - 1); + } + + private static int sizeOfBranch(Object[] branch) + { + int length = branch.length; + // length - 1 == getChildEnd == getPositionOfSizeMap + // (length / 2) - 1 == getChildCount - 1 == position of full tree size + // hard code this, as will be used often; + return ((int[]) branch[length - 1])[(length / 2) - 1]; + } + + /** + * Checks if the UpdateFunction is an instance of {@code UpdateFunction.Simple}. + */ + private static boolean isSimple(UpdateFunction updateF) + { + return updateF instanceof UpdateFunction.Simple; + } + + /** + * @return the size map for the branch node + */ + static int[] sizeMap(Object[] branch) + { + return (int[]) branch[branch.length - 1]; + } + + public static long sizeOnHeapOf(Object[] tree) + { + long size = ObjectSizes.sizeOfArray(tree); + if (isLeaf(tree)) + return size; + for (int i = childOffset(tree); i < childEndOffset(tree); i++) + size += sizeOnHeapOf((Object[]) tree[i]); + size += ObjectSizes.sizeOfArray(sizeMap(tree)); // may overcount, since we share size maps + return size; + } + + // Arbitrary boundaries + private static Object POSITIVE_INFINITY = new Object(); + private static Object NEGATIVE_INFINITY = new Object(); + + /** + * simple static wrapper to calls to cmp.compare() which checks if either a or b are Special (i.e. represent an infinity) + */ + private static int compareWellFormed(Comparator cmp, Object a, Object b) + { + if (a == b) + return 0; + if (a == NEGATIVE_INFINITY | b == POSITIVE_INFINITY) + return -1; + if (b == NEGATIVE_INFINITY | a == POSITIVE_INFINITY) + return 1; + return cmp.compare((V) a, (V) b); + } + + public static boolean isWellFormed(Object[] btree, Comparator cmp) + { + return isWellFormedReturnHeight(cmp, btree, true, NEGATIVE_INFINITY, POSITIVE_INFINITY) >= 0; + } + + private static int isWellFormedReturnHeight(Comparator cmp, Object[] node, boolean isRoot, Object min, Object max) + { + if (isEmpty(node)) + return 0; + + if (cmp != null && !isNodeWellFormed(cmp, node, min, max)) + return -1; + + int keyCount = shallowSize(node); + if (keyCount < 1) + return -1; + if (!isRoot && keyCount < BRANCH_FACTOR / 2 - 1) + return -1; + if (keyCount >= BRANCH_FACTOR) + return -1; + + if (isLeaf(node)) + return 0; + + int[] sizeMap = sizeMap(node); + int size = 0; + int childHeight = -1; + // compare each child node with the branch element at the head of this node it corresponds with + for (int i = childOffset(node); i < childEndOffset(node); i++) + { + Object[] child = (Object[]) node[i]; + Object localmax = i < node.length - 2 ? node[i - childOffset(node)] : max; + int height = isWellFormedReturnHeight(cmp, child, false, min, localmax); + if (height == -1) + return -1; + if (childHeight == -1) + childHeight = height; + if (childHeight != height) + return -1; + + min = localmax; + size += size(child); + if (sizeMap[i - childOffset(node)] != size) + return -1; + size += 1; + } + + return childHeight + 1; + } + + private static boolean isNodeWellFormed(Comparator cmp, Object[] node, Object min, Object max) + { + Object previous = min; + int end = shallowSize(node); + for (int i = 0; i < end; i++) + { + Object current = node[i]; + if (compareWellFormed(cmp, previous, current) >= 0) + return false; + + previous = current; + } + return compareWellFormed(cmp, previous, max) < 0; + } + + /** + * Build a tree of unknown size, in order. + *

+ * Can be used with {@link #reverseInSitu} to build a tree in reverse. + */ + public static FastBuilder fastBuilder() + { + TinyThreadLocalPool.TinyPool> pool = FastBuilder.POOL.get(); + FastBuilder builder = (FastBuilder) pool.poll(); + if (builder == null) + builder = new FastBuilder<>(); + builder.pool = pool; + return builder; + } + + /** + * Base class for AbstractFastBuilder.BranchBuilder, LeafBuilder and AbstractFastBuilder, + * containing shared behaviour and declaring some useful abstract methods. + */ + private static abstract class LeafOrBranchBuilder + { + final int height; + final LeafOrBranchBuilder child; + BranchBuilder parent; + + /** + * The current buffer contents (if any) of the leaf or branch - always sized to contain a complete + * node of the form being constructed. Always non-null, except briefly during overflow. + */ + Object[] buffer; + /** + * The number of keys in our buffer, whether or not we are building a leaf or branch; if we are building + * a branch, we will ordinarily have the same number of children as well, except temporarily when finishing + * the construction of the node. + */ + int count; + + /** + * either + * 1) an empty leftover buffer from a past usage, which can be used when we exhaust {@code buffer}; or + * 2) a full {@code buffer} that has been parked until we next overflow, so we can steal some back + * if we finish before reaching MIN_KEYS in {@code buffer} + */ + Object[] savedBuffer; + /** + * The key we overflowed on when populating savedBuffer. If null, {@link #savedBuffer} is logically empty. + */ + Object savedNextKey; + + LeafOrBranchBuilder(LeafOrBranchBuilder child) + { + this.height = child == null ? 1 : 1 + child.height; + this.child = child; + } + + /** + * Do we have enough keys in the builder to construct at least one balanced node? + * We could have enough to build two. + */ + final boolean isSufficient() + { + return hasOverflow() || count >= MIN_KEYS; + } + + /** + * Do we have an already constructed node saved, that we can propagate or redistribute? + * This implies we are building two nodes, since {@link #savedNextKey} would overflow {@link #savedBuffer} + */ + final boolean hasOverflow() + { + return savedNextKey != null; + } + + /** + * Do we have an already constructed node saved AND insufficient keys in our buffer, so + * that we need to share the contents of {@link #savedBuffer} with {@link #buffer} to construct + * our results? + */ + final boolean mustRedistribute() + { + return hasOverflow() && count < MIN_KEYS; + } + + /** + * Are we empty, i.e. we have no contents in either {@link #buffer} or {@link #savedBuffer} + */ + final boolean isEmpty() + { + return count == 0 && savedNextKey == null; + } + + /** + * Drain the contents of this builder and build up to two nodes, as necessary. + * If {@code unode != null} and we are building a single node that is identical to it, use {@code unode} instead. + * If {@code propagateTo != null} propagate any nodes we build to it. + * + * @return the last node we construct + */ + abstract Object[] drainAndPropagate(Object[] unode, BranchBuilder propagateTo); + + /** + * Drain the contents of this builder and build at most one node. + * Requires {@code !hasOverflow()} + * + * @return the node we construct + */ + abstract Object[] drain(); + + /** + * Complete the build. Drains the node and any used or newly-required parent and returns the root of the + * resulting tree. + * + * @return the root of the constructed tree. + */ + public Object[] completeBuild() + { + LeafOrBranchBuilder level = this; + while (true) + { + if (!level.hasOverflow()) + return level.drain(); + + BranchBuilder parent = level.ensureParent(); + level.drainAndPropagate(null, parent); + if (level.savedBuffer != null) + Arrays.fill(level.savedBuffer, null); + level = parent; + } + } + + /** + * Takes a node that would logically occur directly preceding the current buffer contents, + * and the key that would separate them in a parent node, and prepends their contents + * to the current buffer's contents. This can be used to redistribute already-propagated + * contents to a parent in cases where this is convenient (i.e. when transforming) + * + * @param predecessor directly preceding node + * @param predecessorNextKey key that would have separated predecessor from buffer contents + */ + abstract void prepend(Object[] predecessor, Object predecessorNextKey); + + /** + * Indicates if this builder produces dense nodes, i.e. those that are populated with MAX_KEYS + * at every level. Only the last two children of any branch may be non-dense, and in some cases only + * the last two nodes in any tier of the tree. + *

+ * This flag switches whether or not we maintain a buffer of sizes, or use the globally shared contents of + * DENSE_SIZE_MAPS. + */ + abstract boolean producesOnlyDense(); + + /** + * Ensure there is a {@code branch.parent}, and return it + */ + final BranchBuilder ensureParent() + { + if (parent == null) + parent = new BranchBuilder(this); + parent.inUse = true; + return parent; + } + + /** + * Mark a branch builder as utilised, so that we must clear it when resetting any {@link AbstractFastBuilder} + * + * @return {@code branch} + */ + static BranchBuilder markUsed(BranchBuilder branch) + { + branch.inUse = true; + return branch; + } + + /** + * A utility method for comparing a range of two arrays + */ + static boolean areIdentical(Object[] a, int aOffset, Object[] b, int bOffset, int count) + { + for (int i = 0; i < count; ++i) + { + if (a[i + aOffset] != b[i + bOffset]) + return false; + } + return true; + } + + /** + * A utility method for comparing a range of two arrays + */ + static boolean areIdentical(int[] a, int aOffset, int[] b, int bOffset, int count) + { + for (int i = 0; i < count; ++i) + { + if (a[i + aOffset] != b[i + bOffset]) + return false; + } + return true; + } + } + + /** + * LeafBuilder for methods pertaining specifically to building a leaf in an {@link AbstractFastBuilder}. + * Note that {@link AbstractFastBuilder} extends this class directly, however it is convenient to maintain + * distinct classes in the hierarchy for clarity of behaviour and intent. + */ + private static abstract class LeafBuilder extends LeafOrBranchBuilder + { + long allocated; + + LeafBuilder() + { + super(null); + buffer = new Object[MAX_KEYS]; + } + + /** + * Add {@code nextKey} to the buffer, overflowing if necessary + */ + public void addKey(Object nextKey) + { + if (count == MAX_KEYS) + overflow(nextKey); + else + buffer[count++] = nextKey; + } + + /** + * Add {@code nextKey} to the buffer; the caller specifying overflow is unnecessary + */ + public void addKeyNoOverflow(Object nextKey) + { + buffer[count++] = nextKey; + } + + /** + * Add {@code nextKey} to the buffer; the caller specifying overflow is unnecessary + */ + public void maybeAddKeyNoOverflow(Object nextKey) + { + buffer[count] = nextKey; + count += nextKey != null ? 1 : 0; + } + + /** + * Add {@code nextKey} to the buffer; the caller specifying overflow is unnecessary + */ + public void maybeAddKey(Object nextKey) + { + if (count == MAX_KEYS) + { + if (nextKey != null) + overflow(nextKey); + } + else + { + buffer[count] = nextKey; + count += nextKey != null ? 1 : 0; + } + } + + /** + * Copy the contents of {@code source[from..to)} to {@code buffer}, overflowing as necessary. + */ + void copy(Object[] source, int offset, int length) + { + if (count + length > MAX_KEYS) + { + int copy = MAX_KEYS - count; + System.arraycopy(source, offset, buffer, count, copy); + offset += copy; +// implicitly: count = MAX_KEYS; + overflow(source[offset++]); + length -= 1 + copy; + } + System.arraycopy(source, offset, buffer, count, length); + count += length; + } + + /** + * Copy the contents of {@code source[from..to)} to {@code buffer}; the caller specifying overflow is unnecessary + */ + void copyNoOverflow(Object[] source, int offset, int length) + { + System.arraycopy(source, offset, buffer, count, length); + count += length; + } + + /** + * Copy the contents of {@code source[from..to)} to {@code buffer}, overflowing as necessary. + * Applies {@code updateF} to the contents before insertion. + */ + void copy(Object[] source, int offset, int length, UpdateFunction updateF) + { + if (isSimple(updateF)) + { + copy(source, offset, length); + return; + } + + if (count + length > MAX_KEYS) + { + int copy = MAX_KEYS - count; + for (int i = 0; i < copy; ++i) + buffer[count + i] = updateF.apply((Insert) source[offset + i]); + offset += copy; +// implicitly: leaf().count = MAX_KEYS; + overflow(updateF.apply((Insert) source[offset++])); + length -= 1 + copy; + } + for (int i = 0; i < length; ++i) + buffer[count + i] = updateF.apply((Insert) source[offset + i]); + count += length; + } + + /** + * {@link #buffer} is full, and we need to make room either by populating {@link #savedBuffer}, + * propagating its current contents, if any, to {@link #parent} + */ + void overflow(Object nextKey) + { + if (hasOverflow()) + propagateOverflow(); + + // precondition: count == MAX_KEYS and savedNextKey == null + + Object[] newBuffer = savedBuffer; + if (newBuffer == null) + newBuffer = new Object[MAX_KEYS]; + + savedBuffer = buffer; + savedNextKey = nextKey; + buffer = newBuffer; + count = 0; + } + + /** + * Redistribute the contents of {@link #savedBuffer} into {@link #buffer}, finalise {@link #savedBuffer} and flush upwards. + * Invoked when we are building from {@link #buffer}, have insufficient values but a complete leaf in {@link #savedBuffer} + * + * @return the size of the leaf we flushed to our parent from {@link #savedBuffer} + */ + Object[] redistributeOverflowAndDrain() + { + Object[] newLeaf = redistributeAndDrain(savedBuffer, MAX_KEYS, savedNextKey); + savedNextKey = null; + return newLeaf; + } + + /** + * Redistribute the contents of {@link #buffer} and an immediate predecessor into a new leaf, + * then construct a new predecessor with the remaining contents and propagate up to our parent + * Invoked when we are building from {@link #buffer}, have insufficient values but either a complete + * leaf in {@link #savedBuffer} or can exfiltrate one from our parent to redistribute. + * + * @return the second of the two new leaves + */ + Object[] redistributeAndDrain(Object[] pred, int predSize, Object predNextKey) + { + // precondition: savedLeafCount == MAX_KEYS && leaf().count < MIN_KEYS + // ensure we have at least MIN_KEYS in leaf().buffer + // first shift leaf().buffer and steal some keys from leaf().savedBuffer and leaf().savedBufferNextKey + int steal = MIN_KEYS - count; + Object[] newLeaf = new Object[MIN_KEYS]; + System.arraycopy(pred, predSize - (steal - 1), newLeaf, 0, steal - 1); + newLeaf[steal - 1] = predNextKey; + System.arraycopy(buffer, 0, newLeaf, steal, count); + + // then create a leaf out of the remainder of savedBuffer + int newPredecessorCount = predSize - steal; + Object[] newPredecessor = new Object[newPredecessorCount | 1]; + System.arraycopy(pred, 0, newPredecessor, 0, newPredecessorCount); + if (allocated >= 0) + allocated += ObjectSizes.sizeOfReferenceArray(newPredecessorCount | 1); + ensureParent().addChildAndNextKey(newPredecessor, newPredecessorCount, pred[newPredecessorCount]); + return newLeaf; + } + + /** + * Invoked to fill our {@link #buffer} to >= MIN_KEYS with data ocurring before {@link #buffer}; + * possibly instead fills {@link #savedBuffer} + * + * @param pred directly preceding node + * @param predNextKey key that would have separated predecessor from buffer contents + */ + void prepend(Object[] pred, Object predNextKey) + { + assert !hasOverflow(); + int predSize = sizeOfLeaf(pred); + int newKeys = 1 + predSize; + if (newKeys + count <= MAX_KEYS) + { + System.arraycopy(buffer, 0, buffer, newKeys, count); + System.arraycopy(pred, 0, buffer, 0, predSize); + buffer[predSize] = predNextKey; + count += newKeys; + } + else + { + if (savedBuffer == null) + savedBuffer = new Object[MAX_KEYS]; + System.arraycopy(pred, 0, savedBuffer, 0, predSize); + if (predSize == MAX_KEYS) + { + savedNextKey = predNextKey; + } + else + { + int removeKeys = MAX_KEYS - predSize; + count -= removeKeys; + savedBuffer[predSize] = predNextKey; + System.arraycopy(buffer, 0, savedBuffer, predSize + 1, MAX_KEYS - newKeys); + savedNextKey = buffer[MAX_KEYS - newKeys]; + System.arraycopy(buffer, removeKeys, buffer, 0, count); + } + } + } + + /** + * Invoked when we want to add a key to the leaf buffer, but it is full + */ + void propagateOverflow() + { + // propagate the leaf we have saved in savedBuffer + // precondition: savedLeafCount == MAX_KEYS + if (allocated >= 0) + allocated += ObjectSizes.sizeOfReferenceArray(MAX_KEYS); + ensureParent().addChildAndNextKey(savedBuffer, MAX_KEYS, savedNextKey); + savedBuffer = null; + savedNextKey = null; + } + + /** + * Construct a new leaf from the contents of {@link #buffer}, unless the contents have not changed + * from {@code unode}, in which case return {@code unode} to avoid allocating unnecessary objects. + * + * This is only called when we have enough data to complete the node, i.e. we have MIN_KEYS or more items added + * or the node is the BTree's root. + */ + Object[] drainAndPropagate(Object[] unode, BranchBuilder propagateTo) + { + Object[] leaf; + int sizeOfLeaf; + if (mustRedistribute()) + { + // we have too few items, so spread the two buffers across two new nodes + leaf = redistributeOverflowAndDrain(); + sizeOfLeaf = MIN_KEYS; + } + else if (!hasOverflow() && unode != null && count == sizeOfLeaf(unode) && areIdentical(buffer, 0, unode, 0, count)) + { + // we have exactly the same contents as the original node, so reuse it + leaf = unode; + sizeOfLeaf = count; + } + else + { + // we have maybe one saved full buffer, and one buffer with sufficient contents to copy + if (hasOverflow()) + propagateOverflow(); + + sizeOfLeaf = count; + leaf = drain(); + if (allocated >= 0 && sizeOfLeaf > 0) + allocated += ObjectSizes.sizeOfReferenceArray(sizeOfLeaf | 1) - (unode == null ? 0 : ObjectSizes.sizeOfArray(unode)); + } + + count = 0; + if (propagateTo != null) + propagateTo.addChild(leaf, sizeOfLeaf); + return leaf; + } + + /** + * Construct a new leaf from the contents of {@code leaf().buffer}, assuming that the node does not overflow. + */ + Object[] drain() + { + // the number of children here may be smaller than MIN_KEYS if this is the root node + assert !hasOverflow(); + if (count == 0) + return empty(); + + Object[] newLeaf = new Object[count | 1]; + System.arraycopy(buffer, 0, newLeaf, 0, count); + count = 0; + return newLeaf; + } + } + + static class BranchBuilder extends LeafOrBranchBuilder + { + final LeafBuilder leaf; + + /** + * sizes of the children in {@link #buffer}. If null, we only produce dense nodes. + */ + int[] sizes; + /** + * sizes of the children in {@link #savedBuffer} + */ + int[] savedSizes; + /** + * marker to limit unnecessary work with unused levels, esp. on reset + */ + boolean inUse; + + BranchBuilder(LeafOrBranchBuilder child) + { + super(child); + buffer = new Object[2 * (MAX_KEYS + 1)]; + if (!child.producesOnlyDense()) + sizes = new int[MAX_KEYS + 1]; + this.leaf = child instanceof LeafBuilder ? (LeafBuilder) child : ((BranchBuilder) child).leaf; + } + + /** + * Ensure there is room to add another key to {@code branchBuffers[branchIndex]}, and add it; + * invoke {@link #overflow} if necessary + */ + void addKey(Object key) + { + if (count == MAX_KEYS) + overflow(key); + else + buffer[count++] = key; + } + + /** + * To be invoked when there's a key already inserted to the buffer that requires a corresponding + * right-hand child, for which the buffers are sized to ensure there is always room. + */ + void addChild(Object[] child, int sizeOfChild) + { + buffer[MAX_KEYS + count] = child; + recordSizeOfChild(sizeOfChild); + } + + void recordSizeOfChild(int sizeOfChild) + { + if (sizes != null) + sizes[count] = sizeOfChild; + } + + /** + * See {@link BranchBuilder#addChild(Object[], int)} + */ + void addChild(Object[] child) + { + addChild(child, sizes == null ? 0 : size(child)); + } + + /** + * Insert a new child into a parent branch, when triggered by {@code overflowLeaf} or {@code overflowBranch} + */ + void addChildAndNextKey(Object[] newChild, int newChildSize, Object nextKey) + { + // we should always have room for a child to the right of any key we have previously inserted + buffer[MAX_KEYS + count] = newChild; + recordSizeOfChild(newChildSize); + // but there may not be room for another key + addKey(nextKey); + } + + /** + * Invoked when we want to add a key to the leaf buffer, but it is full + */ + void propagateOverflow() + { + // propagate the leaf we have saved in leaf().savedBuffer + if (leaf.allocated >= 0) + leaf.allocated += ObjectSizes.sizeOfReferenceArray(2 * (1 + MAX_KEYS)); + int size = setOverflowSizeMap(savedBuffer, MAX_KEYS); + ensureParent().addChildAndNextKey(savedBuffer, size, savedNextKey); + savedBuffer = null; + savedNextKey = null; + } + + /** + * Invoked when a branch already contains {@code MAX_KEYS}, and another child is ready to be added. + * Creates a new neighbouring node containing MIN_KEYS items, shifting back the remaining MIN_KEYS+1 + * items to the start of the buffer(s). + */ + void overflow(Object nextKey) + { + if (hasOverflow()) + propagateOverflow(); + + Object[] restoreBuffer = savedBuffer; + int[] restoreSizes = savedSizes; + + savedBuffer = buffer; + savedSizes = sizes; + savedNextKey = nextKey; + + sizes = restoreSizes == null && savedSizes != null ? new int[MAX_KEYS + 1] : restoreSizes; + buffer = restoreBuffer == null ? new Object[2 * (MAX_KEYS + 1)] : restoreBuffer; + count = 0; + } + + /** + * Redistribute the contents of branch.savedBuffer into branch.buffer, finalise savedBuffer and flush upwards. + * Invoked when we are building from branch, have insufficient values but a complete branch in savedBuffer. + * + * @return the size of the branch we flushed to our parent from savedBuffer + */ + Object[] redistributeOverflowAndDrain() + { + // now ensure we have at least MIN_KEYS in buffer + // both buffer and savedBuffer should be balanced, so that we have count+1 and MAX_KEYS+1 children respectively + // we need to utilise savedNextKey, so we want to take {@code steal-1} keys from savedBuffer, {@code steal) children + // and the dangling key we use in place of savedNextKey for our parent key. + int steal = MIN_KEYS - count; + Object[] newBranch = new Object[2 * (MIN_KEYS + 1)]; + System.arraycopy(savedBuffer, MAX_KEYS - (steal - 1), newBranch, 0, steal - 1); + newBranch[steal - 1] = savedNextKey; + System.arraycopy(buffer, 0, newBranch, steal, count); + System.arraycopy(savedBuffer, 2 * MAX_KEYS + 1 - steal, newBranch, MIN_KEYS, steal); + System.arraycopy(buffer, MAX_KEYS, newBranch, MIN_KEYS + steal, count + 1); + setRedistributedSizeMap(newBranch, steal); + + // then create a branch out of the remainder of savedBuffer + int savedBranchCount = MAX_KEYS - steal; + Object[] savedBranch = new Object[2 * (savedBranchCount + 1)]; + System.arraycopy(savedBuffer, 0, savedBranch, 0, savedBranchCount); + System.arraycopy(savedBuffer, MAX_KEYS, savedBranch, savedBranchCount, savedBranchCount + 1); + int savedBranchSize = setOverflowSizeMap(savedBranch, savedBranchCount); + if (leaf.allocated >= 0) + leaf.allocated += ObjectSizes.sizeOfReferenceArray(2 * (1 + savedBranchCount)); + ensureParent().addChildAndNextKey(savedBranch, savedBranchSize, savedBuffer[savedBranchCount]); + savedNextKey = null; + + return newBranch; + } + + /** + * See {@link LeafOrBranchBuilder#prepend(Object[], Object)} + */ + void prepend(Object[] pred, Object predNextKey) + { + assert !hasOverflow(); + // assumes sizes != null, since only makes sense to use this method in that context + + int predKeys = shallowSizeOfBranch(pred); + int[] sizeMap = (int[]) pred[2 * predKeys + 1]; + int newKeys = 1 + predKeys; + if (newKeys + count <= MAX_KEYS) + { + System.arraycopy(buffer, 0, buffer, newKeys, count); + System.arraycopy(sizes, 0, sizes, newKeys, count + 1); + System.arraycopy(buffer, MAX_KEYS, buffer, MAX_KEYS + newKeys, count + 1); + + System.arraycopy(pred, 0, buffer, 0, predKeys); + buffer[predKeys] = predNextKey; + System.arraycopy(pred, predKeys, buffer, MAX_KEYS, predKeys + 1); + copySizeMapToSizes(sizeMap, 0, sizes, 0, predKeys + 1); + count += newKeys; + } + else + { + if (savedBuffer == null) + { + savedBuffer = new Object[2 * (1 + MAX_KEYS)]; + savedSizes = new int[1 + MAX_KEYS]; + } + + System.arraycopy(pred, 0, savedBuffer, 0, predKeys); + System.arraycopy(pred, predKeys, savedBuffer, MAX_KEYS, predKeys + 1); + copySizeMapToSizes(sizeMap, 0, savedSizes, 0, predKeys + 1); + if (newKeys == MAX_KEYS + 1) + { + savedNextKey = predNextKey; + } + else + { + int removeKeys = (1 + MAX_KEYS - newKeys); + int remainingKeys = count - removeKeys; + + savedBuffer[predKeys] = predNextKey; + System.arraycopy(buffer, 0, savedBuffer, newKeys, MAX_KEYS - newKeys); + savedNextKey = buffer[MAX_KEYS - newKeys]; + System.arraycopy(sizes, 0, savedSizes, newKeys, MAX_KEYS + 1 - newKeys); + System.arraycopy(buffer, MAX_KEYS, savedBuffer, MAX_KEYS + newKeys, MAX_KEYS + 1 - newKeys); + System.arraycopy(buffer, removeKeys, buffer, 0, remainingKeys); + System.arraycopy(buffer, MAX_KEYS + removeKeys, buffer, MAX_KEYS, remainingKeys + 1); + System.arraycopy(sizes, removeKeys, sizes, 0, remainingKeys + 1); + count = remainingKeys; + } + } + } + + boolean producesOnlyDense() + { + return sizes == null; + } + + /** + * Construct a new branch from the contents of {@code branchBuffers[branchIndex]}, unless the contents have + * not changed from {@code unode}, in which case return {@code unode} to avoid allocating unnecessary objects. + * + * This is only called when we have enough data to complete the node, i.e. we have MIN_KEYS or more items added + * or the node is the BTree's root. + */ + Object[] drainAndPropagate(Object[] unode, BranchBuilder propagateTo) + { + int sizeOfBranch; + Object[] branch; + if (mustRedistribute()) + { + branch = redistributeOverflowAndDrain(); + sizeOfBranch = sizeOfBranch(branch); + } + else + { + int usz = unode != null ? shallowSizeOfBranch(unode) : -1; + if (!hasOverflow() && usz == count + && areIdentical(buffer, 0, unode, 0, usz) + && areIdentical(buffer, MAX_KEYS, unode, usz, usz + 1)) + { + branch = unode; + sizeOfBranch = sizeOfBranch(branch); + } + else + { + if (hasOverflow()) + propagateOverflow(); + + // the number of children here may be smaller than MIN_KEYS if this is the root node, but there must + // be at least one key / two children. + assert count > 0; + branch = new Object[2 * (count + 1)]; + System.arraycopy(buffer, 0, branch, 0, count); + System.arraycopy(buffer, MAX_KEYS, branch, count, count + 1); + sizeOfBranch = setDrainSizeMap(unode, usz, branch, count); + } + } + + count = 0; + if (propagateTo != null) + propagateTo.addChild(branch, sizeOfBranch); + + return branch; + } + + /** + * Construct a new branch from the contents of {@code buffer}, assuming that the node does not overflow. + */ + Object[] drain() + { + assert !hasOverflow(); + int keys = count; + count = 0; + + Object[] branch = new Object[2 * (keys + 1)]; + if (keys == MAX_KEYS) + { + Object[] tmp = buffer; + buffer = branch; + branch = tmp; + } + else + { + System.arraycopy(buffer, 0, branch, 0, keys); + System.arraycopy(buffer, MAX_KEYS, branch, keys, keys + 1); + } + setDrainSizeMap(null, -1, branch, keys); + return branch; + } + + /** + * Compute (or fetch from cache) and set the sizeMap in {@code branch}, knowing that it + * was constructed from for the contents of {@code buffer}. + *

+ * For {@link FastBuilder} these are mostly the same, so they are fetched from a global cache and + * resized accordingly, but for {@link AbstractUpdater} we maintain a buffer of sizes. + */ + int setDrainSizeMap(Object[] original, int keysInOriginal, Object[] branch, int keysInBranch) + { + if (producesOnlyDense()) + return setImperfectSizeMap(branch, keysInBranch); + + // first convert our buffer contents of sizes to represent a sizeMap + int size = sizesToSizeMap(this.sizes, keysInBranch + 1); + // then attempt to reuse the sizeMap from the original node, by comparing the buffer's contents with it + int[] sizeMap; + if (keysInOriginal != keysInBranch || !areIdentical(sizeMap = sizeMap(original), 0, this.sizes, 0, keysInBranch + 1)) + { + // if we cannot, then we either take the buffer wholesale and replace its buffer, or copy a prefix + sizeMap = this.sizes; + if (keysInBranch < MAX_KEYS) + sizeMap = Arrays.copyOf(sizeMap, keysInBranch + 1); + else + this.sizes = new int[MAX_KEYS + 1]; + } + branch[2 * keysInBranch + 1] = sizeMap; + return size; + } + + /** + * Compute (or fetch from cache) and set the sizeMap in {@code branch}, knowing that it + * was constructed from for the contents of {@code savedBuffer}. + *

+ * For {@link FastBuilder} these are always the same size, so they are fetched from a global cache, + * but for {@link AbstractUpdater} we maintain a buffer of sizes. + * + * @return the size of {@code branch} + */ + int setOverflowSizeMap(Object[] branch, int keys) + { + if (producesOnlyDense()) + { + int[] sizeMap = DENSE_SIZE_MAPS[height - 2]; + if (keys < MAX_KEYS) + sizeMap = Arrays.copyOf(sizeMap, keys + 1); + branch[2 * keys + 1] = sizeMap; + return keys < MAX_KEYS ? sizeMap[keys] : checkedDenseSize(height + 1); + } + else + { + int[] sizes = savedSizes; + if (keys < MAX_KEYS) + sizes = Arrays.copyOf(sizes, keys + 1); + else + savedSizes = null; + branch[2 * keys + 1] = sizes; + return sizesToSizeMap(sizes); + } + } + + /** + * Compute (or fetch from cache) and set the sizeMap in {@code branch}, knowing that it + * was constructed from the contents of both {@code savedBuffer} and {@code buffer} + *

+ * For {@link FastBuilder} these are mostly the same size, so they are fetched from a global cache + * and only the last items updated, but for {@link AbstractUpdater} we maintain a buffer of sizes. + */ + void setRedistributedSizeMap(Object[] branch, int steal) + { + if (producesOnlyDense()) + { + setImperfectSizeMap(branch, MIN_KEYS); + } + else + { + int[] sizeMap = new int[MIN_KEYS + 1]; + System.arraycopy(sizes, 0, sizeMap, steal, count + 1); + System.arraycopy(savedSizes, MAX_KEYS + 1 - steal, sizeMap, 0, steal); + branch[2 * MIN_KEYS + 1] = sizeMap; + sizesToSizeMap(sizeMap); + } + } + + /** + * Like {@link #setOverflowSizeMap}, but used for building the sizeMap of a node whose + * last two children may have had their contents redistributed; uses the perfect size map + * for all but the final two children, and queries the size of the last children directly + */ + private int setImperfectSizeMap(Object[] branch, int keys) + { + int[] sizeMap = Arrays.copyOf(DENSE_SIZE_MAPS[height - 2], keys + 1); + int size = keys == 1 ? 0 : 1 + sizeMap[keys - 2]; + sizeMap[keys - 1] = size += size((Object[]) branch[2 * keys - 1]); + sizeMap[keys] = size += 1 + size((Object[]) branch[2 * keys]); + branch[2 * keys + 1] = sizeMap; + return size; + } + + /** + * Copy the contents of {@code unode} into {@code branchBuffers[branchIndex]}, + * starting at the child before key with index {@code offset} up to and + * including the key with index {@code offset + length - 1}. + */ + void copyPreceding(Object[] unode, int usz, int offset, int length) + { + int[] uszmap = sizeMap(unode); + if (count + length > MAX_KEYS) + { + // we will overflow, so copy to MAX_KEYS and trigger overflow + int copy = MAX_KEYS - count; + copyPrecedingNoOverflow(unode, usz, uszmap, offset, copy); + offset += copy; + + // copy last child that fits + buffer[MAX_KEYS + MAX_KEYS] = unode[usz + offset]; + sizes[MAX_KEYS] = uszmap[offset] - (offset > 0 ? (1 + uszmap[offset - 1]) : 0); + + overflow(unode[offset]); + + length -= 1 + copy; + ++offset; + } + + copyPrecedingNoOverflow(unode, usz, uszmap, offset, length); + } + + /** + * Copy the contents of {@code unode} into {@code branchBuffers[branchIndex]}, + * between keys {@code from} and {@code to}, with the caller declaring overflow is unnecessary. + * {@code from} may be {@code -1}, representing the first child only; + * all other indices represent the key/child pairs that follow (i.e. a key and its right-hand child). + */ + private void copyPrecedingNoOverflow(Object[] unode, int usz, int[] uszmap, int offset, int length) + { + if (length <= 1) + { + if (length == 0) + return; + + buffer[count] = unode[offset]; + buffer[MAX_KEYS + count] = unode[usz + offset]; + sizes[count] = uszmap[offset] - (offset > 0 ? (1 + uszmap[offset - 1]) : 0); + ++count; + } + else + { + System.arraycopy(unode, offset, buffer, count, length); + System.arraycopy(unode, usz + offset, buffer, MAX_KEYS + count, length); + copySizeMapToSizes(uszmap, offset, sizes, count, length); + count += length; + } + } + + /** + * Copy a region of a cumulative sizeMap into an array of plain sizes + */ + static void copySizeMapToSizes(int[] in, int inOffset, int[] out, int outOffset, int count) + { + assert count > 0; + if (inOffset == 0) + { + // we don't need to subtract anything from the first node, so just copy it so we can keep the rest of the loop simple + out[outOffset++] = in[inOffset++]; + --count; + } + for (int i = 0; i < count; ++i) + out[outOffset + i] = in[inOffset + i] - (1 + in[inOffset + i - 1]); + } + } + + /** + * Shared parent of {@link FastBuilder} and {@link Updater}, both of which + * construct their trees in order without knowing the resultant size upfront. + *

+ * Maintains a simple stack of buffers that we provide utilities to navigate and update. + */ + private static abstract class AbstractFastBuilder extends LeafBuilder + { + final boolean producesOnlyDense() + { + return getClass() == FastBuilder.class; + } + + /** + * An aesthetic convenience for declaring when we are interacting with the leaf, instead of invoking {@code this} directly + */ + final LeafBuilder leaf() + { + return this; + } + + /** + * Clear any references we might still retain, to avoid holding onto memory. + *

+ * While this method is not strictly necessary, it exists to + * ensure the implementing classes are aware they must handle it. + */ + abstract void reset(); + } + + /** + * A pooled builder for constructing a tree in-order, and without needing any reconciliation. + *

+ * Constructs whole nodes in place, so that a flush of a complete node can take its buffer entirely. + * Since we build trees of a predictable shape (i.e. perfectly dense) we do not construct a size map. + */ + public static class FastBuilder extends AbstractFastBuilder implements AutoCloseable + { + private static final TinyThreadLocalPool> POOL = new TinyThreadLocalPool<>(); + private TinyThreadLocalPool.TinyPool> pool; + + FastBuilder() + { + allocated = -1; + } // disable allocation tracking + + public void add(V value) + { + leaf().addKey(value); + } + + public void add(Object[] from, int offset, int count) + { + leaf().copy(from, offset, count); + } + + public Object[] build() + { + return leaf().completeBuild(); + } + + public Object[] buildReverse() + { + Object[] result = build(); + reverseInSitu(result, height(result), false); + return result; + } + + @Override + public void close() + { + reset(); + pool.offer(this); + pool = null; + } + + @Override + void reset() + { + // we clear precisely to leaf().count and branch.count because, in the case of a builder, + // if we ever fill the buffer we will consume it entirely for the tree we are building + // so the last count should match the number of non-null entries + Arrays.fill(leaf().buffer, 0, leaf().count, null); + leaf().count = 0; + BranchBuilder branch = leaf().parent; + while (branch != null && branch.inUse) + { + Arrays.fill(branch.buffer, 0, branch.count, null); + Arrays.fill(branch.buffer, MAX_KEYS, MAX_KEYS + 1 + branch.count, null); + branch.count = 0; + branch.inUse = false; + branch = branch.parent; + } + } + } + + private static abstract class AbstractUpdater extends AbstractFastBuilder implements AutoCloseable + { + void reset() + { + assert leaf().count == 0; + clearLeafBuffer(leaf().buffer); + if (leaf().savedBuffer != null) + Arrays.fill(leaf().savedBuffer, null); + + BranchBuilder branch = leaf().parent; + while (branch != null && branch.inUse) + { + assert branch.count == 0; + clearBranchBuffer(branch.buffer); + if (branch.savedBuffer != null && branch.savedBuffer[0] != null) + Arrays.fill(branch.savedBuffer, null); // by definition full, if non-empty + branch.inUse = false; + branch = branch.parent; + } + } + + /** + * Clear the contents of a branch buffer, aborting once we encounter a null entry + * to save time on small trees + */ + private void clearLeafBuffer(Object[] array) + { + if (array[0] == null) + return; + // find first null entry; loop from beginning, to amortise cost over size of working set + int i = 1; + while (i < array.length && array[i] != null) + ++i; + Arrays.fill(array, 0, i, null); + } + + /** + * Clear the contents of a branch buffer, aborting once we encounter a null entry + * to save time on small trees + */ + private void clearBranchBuffer(Object[] array) + { + if (array[0] == null) + return; + + // find first null entry; loop from beginning, to amortise cost over size of working set + int i = 1; + while (i < MAX_KEYS && array[i] != null) + ++i; + Arrays.fill(array, 0, i, null); + Arrays.fill(array, MAX_KEYS, MAX_KEYS + i + 1, null); + } + } + + /** + * A pooled object for modifying an existing tree with a new (typically smaller) tree. + *

+ * Constructs the new tree around the shape of the existing tree, as though we had performed inserts in + * order, copying as much of the original tree as possible. We achieve this by simply merging leaf nodes + * up to the immediately following key in an ancestor, maintaining up to two complete nodes in a buffer until + * this happens, and flushing any nodes we produce in excess of this immediately into the parent buffer. + *

+ * We construct whole nodes in place, except the size map, so that a flush of a complete node can take its buffer + * entirely. + *

+ * Searches within both trees to accelerate the process of modification, instead of performing a simple + * iteration over the new tree. + */ + private static class Updater extends AbstractUpdater implements AutoCloseable + { + static final TinyThreadLocalPool POOL = new TinyThreadLocalPool<>(); + TinyThreadLocalPool.TinyPool pool; + + // the new tree we navigate linearly, and are always on a key or at the end + final SimpleTreeKeysIterator insert = new SimpleTreeKeysIterator<>(); + + Comparator comparator; + UpdateFunction updateF; + + + static Updater get() + { + TinyThreadLocalPool.TinyPool pool = POOL.get(); + Updater updater = pool.poll(); + if (updater == null) + updater = new Updater<>(); + updater.pool = pool; + return updater; + } + + /** + * Precondition: {@code update} should not be empty. + *

+ * Inserts {@code insert} into {@code update}, after applying {@code updateF} to each item, or matched item pairs. + */ + Object[] update(Object[] update, Object[] insert, Comparator comparator, UpdateFunction updateF) + { + this.insert.init(insert); + this.updateF = updateF; + this.comparator = comparator; + this.allocated = isSimple(updateF) ? -1 : 0; + int leafDepth = BTree.depth(update) - 1; + LeafOrBranchBuilder builder = leaf(); + for (int i = 0; i < leafDepth; ++i) + builder = builder.ensureParent(); + + Insert ik = this.insert.next(); + ik = updateRecursive(ik, update, null, builder); + assert ik == null; + Object[] result = builder.completeBuild(); + + if (allocated > 0) + updateF.onAllocatedOnHeap(allocated); + + return result; + } + + /** + * Merge a BTree recursively with the contents of {@code insert} up to the given upper bound. + * + * @param ik The next key from the inserted data. + * @param unode The source branch to update. + * @param uub The branch's upper bound + * @param builder The builder that will receive the data. It needs to be at the same level of the hierarchy + * as the source unode. + * @return The next key from the inserted data, >= uub. + */ + private Insert updateRecursive(Insert ik, Object[] unode, Existing uub, LeafOrBranchBuilder builder) + { + return builder == leaf() + ? updateRecursive(ik, unode, uub, (LeafBuilder) builder) + : updateRecursive(ik, unode, uub, (BranchBuilder) builder); + } + + private Insert updateRecursive(Insert ik, Object[] unode, Existing uub, BranchBuilder builder) + { + int upos = 0; + int usz = shallowSizeOfBranch(unode); + + while (ik != null) + { + int find = exponentialSearchWithUpperBound(comparator, unode, upos, usz, uub, ik); + int c = searchResultToComparison(find); + if (find < 0) + find = -1 - find; + + if (find > usz) + break; // nothing else needs to be inserted in this branch + if (find > upos) + builder.copyPreceding(unode, usz, upos, find - upos); + + final Existing nextUKey = find < usz ? (Existing) unode[find] : uub; + final Object[] childUNode = (Object[]) unode[find + usz]; + + // process next child + if (c < 0) + { + // ik fall inside it -- recursively merge the child with the update, using next key as an upper bound + LeafOrBranchBuilder childBuilder = builder.child; + ik = updateRecursive(ik, childUNode, nextUKey, childBuilder); + childBuilder.drainAndPropagate(childUNode, builder); + if (find == usz) // this was the right-most child, branch is complete and we can return immediately + return ik; + c = ik != null ? comparator.compare(nextUKey, ik) : -1; + } + else + builder.addChild(childUNode); + + // process next key + if (c == 0) + { + // ik matches next key + builder.addKey(updateF.apply(nextUKey, ik)); + ik = insert.next(); + } + else + builder.addKey(nextUKey); + + upos = find + 1; + } + // copy the rest of the branch and exit + if (upos <= usz) + { + builder.copyPreceding(unode, usz, upos, usz - upos); + builder.addChild((Object[]) unode[usz + usz]); + } + + return ik; + } + + private Insert updateRecursive(Insert ik, Object[] unode, Existing uub, LeafBuilder builder) + { + int upos = 0; + int usz = sizeOfLeaf(unode); + Existing uk = (Existing) unode[upos]; + int c = comparator.compare(uk, ik); + + while (true) + { + if (c == 0) + { + leaf().addKey(updateF.apply(uk, ik)); + if (++upos < usz) + uk = (Existing) unode[upos]; + ik = insert.next(); + if (ik == null) + { + builder.copy(unode, upos, usz - upos); + return null; + } + if (upos == usz) + break; + c = comparator.compare(uk, ik); + } + else if (c < 0) + { + int ulim = exponentialSearch(comparator, unode, upos + 1, usz, ik); + c = -searchResultToComparison(ulim); // 0 if match, 1 otherwise + if (ulim < 0) + ulim = -(1 + ulim); + builder.copy(unode, upos, ulim - upos); + if ((upos = ulim) == usz) + break; + uk = (Existing) unode[upos]; + } + else + { + builder.addKey(isSimple(updateF) ? ik : updateF.apply(ik)); + c = insert.copyKeysSmallerThan(uk, comparator, builder, updateF); // 0 on match, -1 otherwise + ik = insert.next(); + if (ik == null) + { + builder.copy(unode, upos, usz - upos); + return null; + } + } + } + if (uub == null || comparator.compare(ik, uub) < 0) + { + builder.addKey(isSimple(updateF) ? ik : updateF.apply(ik)); + insert.copyKeysSmallerThan(uub, comparator, builder, updateF); // 0 on match, -1 otherwise + ik = insert.next(); + } + return ik; + } + + + public void close() + { + reset(); + pool.offer(this); + pool = null; + } + + void reset() + { + super.reset(); + insert.reset(); + } + } + + static int searchResultToComparison(int searchResult) + { + return searchResult >> 31; + } + + /** + * Attempts to perform a clean transformation of the original tree into a new tree, + * by replicating its original shape as far as possible. + *

+ * We do this by attempting to flush our buffers whenever we finish a source-branch at the given level; + * if there are too few contents, we wait until we finish another node at the same level. + *

+ * This way, we are always resetting at the earliest point we might be able to reuse more parts of the original + * tree, maximising potential reuse. + *

+ * This can permit us to build unbalanced right-most nodes at each level, in which case we simply rebalance + * when done. + *

+ * The approach taken here hopefully balances simplicity, garbage generation and execution time. + */ + private static abstract class AbstractTransformer extends AbstractUpdater implements AutoCloseable + { + /** + * An iterator over the tree we are updating + */ + final SimpleTreeIterator update = new SimpleTreeIterator(); + + /** + * A queue of nodes from update that we are ready to "finish" if we have buffered enough data from them + * The stack pointer is maintained inside of {@link #apply()} + */ + Object[][] queuedToFinish = new Object[1][]; + + AbstractTransformer() + { + allocated = -1; + ensureParent(); + parent.inUse = false; + } + + abstract O apply(I v); + + Object[] apply(Object[] update) + { + int height = this.update.init(update); + if (queuedToFinish.length < height - 1) + queuedToFinish = new Object[height - 1][]; + return apply(); + } + + /** + * We base our operation on the shape of {@code update}, trying to steal as much of the original tree as + * possible for our new tree + */ + private Object[] apply() + { + Object[] unode = update.node(); + int upos = update.position(), usz = sizeOfLeaf(unode); + + while (true) + { + // we always start the loop on a leaf node, for both input and output + boolean propagatedOriginalLeaf = false; + if (leaf().count == 0) + { + if (upos == 0) + { // fast path - buffer is empty and input unconsumed, so may be able to propagate original + I in; + O out; + do + { // optimistic loop - find first point the transformation modified our input + in = (I) unode[upos]; + out = apply(in); + } while (in == out && ++upos < usz); + + if ((propagatedOriginalLeaf = (upos == usz))) + { + // if input is unmodified by transformation, propagate the input node + markUsed(parent).addChild(unode, usz); + } + else + { + // otherwise copy up to the first modified portion, + // and fall-through to our below condition for transforming the remainder + leaf().copyNoOverflow(unode, 0, upos++); + if (out != null) + leaf().addKeyNoOverflow(out); + } + } + + if (!propagatedOriginalLeaf) + transformLeafNoOverflow(unode, upos, usz); + } + else + { + transformLeaf(unode, upos, usz); + } + + // we've finished a leaf, and have to hand it to a parent alongside its right-hand key + // so now we try to do two things: + // 1) find the next unfiltered key from our unfinished parent + // 2) determine how many parents are "finished" and whose buffers we should also attempt to propagate + // we do (1) unconditionally, because: + // a) we need to handle the branch keys somewhere, and it may as well happen in one place + // b) we either need more keys for our incomplete leaf; or + // c) we need a key to go after our last propagated node in any unfinished parent + + int finishToHeight = 0; + O nextKey; + do + { + update.ascendToParent(); // always have a node above leaf level, else we'd invoke transformLeaf + BranchBuilder level = parent; + unode = update.node(); + upos = update.position(); + usz = shallowSizeOfBranch(unode); + + while (upos == usz) + { + queuedToFinish[level.height - 2] = unode; + finishToHeight = max(finishToHeight, level.height); + + if (!update.ascendToParent()) + return finishAndDrain(propagatedOriginalLeaf); + + level = level.ensureParent(); + unode = update.node(); + upos = update.position(); + usz = shallowSizeOfBranch(unode); + } + + nextKey = apply((I) unode[upos]); + if (nextKey == null && leaf().count > MIN_KEYS) // if we don't have a key, try to steal from leaf().buffer + nextKey = (O) leaf().buffer[--leaf().count]; + + update.descendIntoNextLeaf(unode, upos, usz); + unode = update.node(); + upos = update.position(); + usz = sizeOfLeaf(unode); + + // nextKey might have been filtered, so we may need to look in this next leaf for it + while (nextKey == null && upos < usz) + nextKey = apply((I) unode[upos++]); + + // if we still found no key loop and try again on the next parent, leaf, parent... ad infinitum + } while (nextKey == null); + + // we always end with unode a leaf, though it may be that upos == usz and that we will do nothing with it + + // we've found a non-null key, now decide what to do with it: + // 1) if we have insufficient keys in our leaf, simply append to the leaf and continue; + // 2) otherwise, walk our parent branches finishing those *before* {@code finishTo} + // 2a) if any cannot be finished, append our new key to it and stop finishing further parents; they + // will be finished the next time we ascend to their level with a complete chain of finishable branches + // 2b) otherwise, add our new key to {@code finishTo} + + if (!propagatedOriginalLeaf && !finish(leaf(), null)) + { + leaf().addKeyNoOverflow(nextKey); + continue; + } + + BranchBuilder finish = parent; + while (true) + { + if (finish.height <= finishToHeight) + { + Object[] originalNode = queuedToFinish[finish.height - 2]; + if (finish(finish, originalNode)) + { + finish = finish.parent; + continue; + } + } + + // add our key to the last unpropagated parent branch buffer + finish.addKey(nextKey); + break; + } + } + } + + private void transformLeafNoOverflow(Object[] unode, int upos, int usz) + { + while (upos < usz) + { + O v = apply((I) unode[upos++]); + leaf().maybeAddKeyNoOverflow(v); + } + } + + private void transformLeaf(Object[] unode, int upos, int usz) + { + while (upos < usz) + { + O v = apply((I) unode[upos++]); + leaf().maybeAddKey(v); + } + } + + /** + * Invoked when we are finished transforming a branch. If the buffer contains insufficient elements, + * we refuse to construct a leaf and return null. Otherwise we propagate the branch to its parent's buffer + * and return the branch we have constructed. + */ + private boolean finish(LeafOrBranchBuilder level, Object[] unode) + { + if (!level.isSufficient()) + return false; + + level.drainAndPropagate(unode, level.ensureParent()); + return true; + } + + /** + * Invoked once we have consumed all input. + *

+ * Completes all unfinished buffers. If they do not contain enough keys, data is stolen from the preceding + * node to the left on the same level. This is easy if our parent already contains a completed child; if it + * does not, we recursively apply the stealing procedure to obtain a non-empty parent. If this process manages + * to reach the root and still find no preceding branch, this will result in making this branch the new root. + */ + private Object[] finishAndDrain(boolean skipLeaf) + { + LeafOrBranchBuilder level = leaf(); + if (skipLeaf) + { + level = nonEmptyParentMaybeSteal(level); + // handle an edge case, where we have propagated a single complete leaf but have no other contents in any parent + if (level == null) + return (Object[]) leaf().parent.buffer[MAX_KEYS]; + } + while (true) + { + BranchBuilder parent = nonEmptyParentMaybeSteal(level); + if (parent != null && !level.isSufficient()) + { + Object[] result = stealAndMaybeRepropagate(level, parent); + if (result != null) + return result; + } + else + { + + Object[] originalNode = level == leaf() ? null : queuedToFinish[level.height - 2]; + Object[] result = level.drainAndPropagate(originalNode, parent); + if (parent == null) + return result; + } + level = parent; + } + } + + BranchBuilder nonEmptyParentMaybeSteal(LeafOrBranchBuilder level) + { + if (level.hasOverflow()) + return level.ensureParent(); + BranchBuilder parent = level.parent; + if (parent == null || !parent.inUse || (parent.isEmpty() && !tryPrependFromParent(parent))) + return null; + return parent; + } + + /** + * precondition: {@code fill.parentInUse()} must return {@code fill.parent} + *

+ * Steal some data from our ancestors, if possible. + * 1) If no ancestor has any data to steal, simply drain and return the current contents. + * 2) If we exhaust all of our ancestors, and are not now ourselves overflowing, drain and return + * 3) Otherwise propagate the redistributed contents to our parent and return null, indicating we can continue to parent + * + * @return {@code null} if {@code parent} is still logicallly in use after we execute; + * otherwise the return value is the final result + */ + private Object[] stealAndMaybeRepropagate(LeafOrBranchBuilder fill, BranchBuilder parent) + { + // parent already stole, we steal one from it + prependFromParent(fill, parent); + + // if we've emptied our parent, attempt to restore it from our grandparent, + // this is so that we can determine an accurate exhausted status + boolean exhausted = !fill.hasOverflow() && parent.isEmpty() && !tryPrependFromParent(parent); + if (exhausted) + return fill.drain(); + + fill.drainAndPropagate(null, parent); + return null; + } + + private boolean tryPrependFromParent(BranchBuilder parent) + { + BranchBuilder grandparent = nonEmptyParentMaybeSteal(parent); + if (grandparent == null) + return false; + prependFromParent(parent, grandparent); + return true; + } + + // should only be invoked with parent = parentIfStillInUse(fill), if non-null result + private void prependFromParent(LeafOrBranchBuilder fill, BranchBuilder parent) + { + assert !parent.isEmpty(); + + Object[] predecessor; + Object predecessorNextKey; + // parent will have same number of children as shallow key count (and may be empty) + if (parent.count == 0 && parent.hasOverflow()) + { + // use the saved buffer instead of going to our parent + predecessorNextKey = parent.savedNextKey; + predecessor = (Object[]) parent.savedBuffer[2 * MAX_KEYS]; + Object[] tmpBuffer = parent.savedBuffer; + int[] tmpSizes = parent.savedSizes; + parent.savedBuffer = parent.buffer; + parent.savedSizes = parent.sizes; + parent.buffer = tmpBuffer; + parent.sizes = tmpSizes; + parent.savedNextKey = null; + parent.count = MAX_KEYS; + // end with MAX_KEYS keys and children in parent, having stolen MAX_KEYS+1 child and savedNextKey + } + else + { + --parent.count; + predecessor = (Object[]) parent.buffer[MAX_KEYS + parent.count]; + predecessorNextKey = parent.buffer[parent.count]; + } + + fill.prepend(predecessor, predecessorNextKey); + } + + void reset() + { + Arrays.fill(queuedToFinish, 0, update.leafDepth, null); + update.reset(); + super.reset(); + } + } + + + private static class Transformer extends AbstractTransformer + { + static final TinyThreadLocalPool POOL = new TinyThreadLocalPool<>(); + TinyThreadLocalPool.TinyPool pool; + + Function apply; + + O apply(I v) + { + return apply.apply(v); + } + + static Transformer get(Function apply) + { + TinyThreadLocalPool.TinyPool pool = POOL.get(); + Transformer transformer = pool.poll(); + if (transformer == null) + transformer = new Transformer<>(); + transformer.pool = pool; + transformer.apply = apply; + return transformer; + } + + public void close() + { + apply = null; + reset(); + pool.offer(this); + pool = null; + } + } + + private static class BiTransformer extends AbstractTransformer + { + static final TinyThreadLocalPool POOL = new TinyThreadLocalPool<>(); + + BiFunction apply; + I2 i2; + TinyThreadLocalPool.TinyPool pool; + + O apply(I i1) + { + return apply.apply(i1, i2); + } + + static BiTransformer get(BiFunction apply, I2 i2) + { + TinyThreadLocalPool.TinyPool pool = POOL.get(); + BiTransformer transformer = pool.poll(); + if (transformer == null) + transformer = new BiTransformer<>(); + transformer.pool = pool; + transformer.apply = apply; + transformer.i2 = i2; + return transformer; + } + + public void close() + { + apply = null; + i2 = null; + reset(); + pool.offer(this); + pool = null; + } + } + + /** + * A base class for very simple walks of a tree without recursion, supporting reuse + */ + private static abstract class SimpleTreeStack + { + // stack we have descended, with 0 the root node + Object[][] nodes; + /** + * the child node we are in, if at lower height, or the key we are on otherwise + * can be < 0, indicating we have not yet entered the contents of the node, and are deliberating + * whether we descend or consume the contents without descending + */ + int[] positions; + int depth, leafDepth; + + void reset() + { + Arrays.fill(nodes, 0, leafDepth + 1, null); + // positions gets zero'd during descent + } + + Object[] node() + { + return nodes[depth]; + } + + int position() + { + return positions[depth]; + } + } + + // Similar to SimpleTreeNavigator, but visits values eagerly + // (the exception being ascendToParent(), which permits iterating through finished parents). + // Begins by immediately descending to first leaf; if empty terminates immediately. + private static class SimpleTreeIterator extends SimpleTreeStack + { + int init(Object[] tree) + { + int maxHeight = maxRootHeight(size(tree)); + if (positions == null || maxHeight >= positions.length) + { + positions = new int[maxHeight + 1]; + nodes = new Object[maxHeight + 1][]; + } + nodes[0] = tree; + if (isEmpty(tree)) + { + // already done + leafDepth = 0; + depth = -1; + } + else + { + depth = 0; + positions[0] = 0; + while (!isLeaf(tree)) + { + tree = (Object[]) tree[shallowSizeOfBranch(tree)]; + nodes[++depth] = tree; + positions[depth] = 0; + } + leafDepth = depth; + } + return leafDepth + 1; + } + + void descendIntoNextLeaf(Object[] node, int pos, int sz) + { + positions[depth] = ++pos; + ++depth; + nodes[depth] = node = (Object[]) node[sz + pos]; + positions[depth] = 0; + while (depth < leafDepth) + { + ++depth; + nodes[depth] = node = (Object[]) node[shallowSizeOfBranch(node)]; + positions[depth] = 0; + } + } + + boolean ascendToParent() + { + if (depth < 0) + return false; + return --depth >= 0; + } + } + + + private static class SimpleTreeKeysIterator + { + int leafSize; + int leafPos; + Object[] leaf; + Object[][] nodes; + int[] positions; + int depth; + + void init(Object[] tree) + { + int maxHeight = maxRootHeight(size(tree)); + if (positions == null || maxHeight >= positions.length) + { + positions = new int[maxHeight + 1]; + nodes = new Object[maxHeight + 1][]; + } + depth = 0; + descendToLeaf(tree); + } + + void reset() + { + leaf = null; + Arrays.fill(nodes, 0, nodes.length, null); + } + + Insert next() + { + if (leafPos < leafSize) // fast path + return (Insert) leaf[leafPos++]; + + if (depth == 0) + return null; + + Object[] node = nodes[depth - 1]; + final int position = positions[depth - 1]; + Insert result = (Insert) node[position]; + advanceBranch(node, position + 1); + return result; + } + + private void advanceBranch(Object[] node, int position) + { + int count = shallowSizeOfBranch(node); + if (position < count) + positions[depth - 1] = position; + else + --depth; // no more children in this branch, remove from stack + descendToLeaf((Object[]) node[count + position]); + } + + void descendToLeaf(Object[] node) + { + while (!isLeaf(node)) + { + nodes[depth] = node; + positions[depth] = 0; + node = (Object[]) node[shallowSizeOfBranch(node)]; + ++depth; + } + leaf = node; + leafPos = 0; + leafSize = sizeOfLeaf(node); + } + + int copyKeysSmallerThan(Compare bound, Comparator comparator, LeafBuilder builder, UpdateFunction transformer) + { + while (true) + { + int lim = exponentialSearchForMaybeInfinity(comparator, leaf, leafPos, leafSize, bound); + int end = lim >= 0 ? lim : -1 - lim; + if (end > leafPos) + { + builder.copy(leaf, leafPos, end - leafPos, transformer); + leafPos = end; + } + if (end < leafSize) + return searchResultToComparison(lim); // 0 if next is a match for bound, -1 otherwise + + if (depth == 0) + return -1; + + Object[] node = nodes[depth - 1]; + final int position = positions[depth - 1]; + Insert branchKey = (Insert) node[position]; + int cmp = compareWithMaybeInfinity(comparator, branchKey, bound); + if (cmp >= 0) + return -cmp; + builder.addKey(isSimple(transformer) ? branchKey : transformer.apply(branchKey)); + advanceBranch(node, position + 1); + } + } + } } diff --git a/src/java/org/apache/cassandra/utils/btree/BTreeRemoval.java b/src/java/org/apache/cassandra/utils/btree/BTreeRemoval.java index f4d9cd4cab..b887298b0b 100644 --- a/src/java/org/apache/cassandra/utils/btree/BTreeRemoval.java +++ b/src/java/org/apache/cassandra/utils/btree/BTreeRemoval.java @@ -85,9 +85,9 @@ public class BTreeRemoval index -= (1 + BTree.getSizeMap(node)[i - 1]); Object[] nextNode = (Object[]) node[keyEnd + i]; boolean nextNodeNeedsCopy = true; - if (BTree.getKeyEnd(nextNode) > BTree.MINIMAL_NODE_SIZE) + if (BTree.getKeyEnd(nextNode) > BTree.MIN_KEYS) node = copyIfNeeded(node, needsCopy); - else if (i > 0 && BTree.getKeyEnd((Object[]) node[keyEnd + i - 1]) > BTree.MINIMAL_NODE_SIZE) + else if (i > 0 && BTree.getKeyEnd((Object[]) node[keyEnd + i - 1]) > BTree.MIN_KEYS) { node = copyIfNeeded(node, needsCopy); final Object[] leftNeighbour = (Object[]) node[keyEnd + i - 1]; @@ -96,7 +96,7 @@ public class BTreeRemoval index += BTree.size((Object[])leftNeighbour[BTree.getChildEnd(leftNeighbour) - 1]); nextNode = rotateLeft(node, i); } - else if (i < keyEnd && BTree.getKeyEnd((Object[]) node[keyEnd + i + 1]) > BTree.MINIMAL_NODE_SIZE) + else if (i < keyEnd && BTree.getKeyEnd((Object[]) node[keyEnd + i + 1]) > BTree.MIN_KEYS) { node = copyIfNeeded(node, needsCopy); nextNode = rotateRight(node, i); @@ -256,12 +256,12 @@ public class BTreeRemoval private static Object[] merge(final Object[] left, final Object[] right, final V nodeKey) { - assert BTree.getKeyEnd(left) == BTree.MINIMAL_NODE_SIZE; - assert BTree.getKeyEnd(right) == BTree.MINIMAL_NODE_SIZE; + assert BTree.getKeyEnd(left) == BTree.MIN_KEYS; + assert BTree.getKeyEnd(right) == BTree.MIN_KEYS; final boolean leaves = BTree.isLeaf(left); final Object[] result; if (leaves) - result = new Object[BTree.MINIMAL_NODE_SIZE * 2 + 1]; + result = new Object[BTree.MIN_KEYS * 2 + 1]; else result = new Object[left.length + right.length]; int offset = 0; diff --git a/src/java/org/apache/cassandra/utils/btree/BTreeSet.java b/src/java/org/apache/cassandra/utils/btree/BTreeSet.java index 1a28d78c30..b20729d7da 100644 --- a/src/java/org/apache/cassandra/utils/btree/BTreeSet.java +++ b/src/java/org/apache/cassandra/utils/btree/BTreeSet.java @@ -20,7 +20,6 @@ package org.apache.cassandra.utils.btree; import java.util.*; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; import org.apache.cassandra.utils.btree.BTree.Dir; @@ -39,11 +38,6 @@ public class BTreeSet implements NavigableSet, List this.comparator = comparator; } - public BTreeSet update(Collection updateWith) - { - return new BTreeSet<>(BTree.update(tree, comparator, updateWith, UpdateFunction.noOp()), comparator); - } - @Override public Comparator comparator() { @@ -582,31 +576,38 @@ public class BTreeSet implements NavigableSet, List public static class Builder { - final BTree.Builder builder; + final BTree.Builder wrapped; + protected Builder(Comparator comparator) { - builder= BTree.builder(comparator); + wrapped = BTree.builder(comparator); + } + + protected Builder(Comparator comparator, int size) + { + wrapped = BTree.builder(comparator, size); } public Builder add(V v) { - builder.add(v); + wrapped .add(v); return this; } public Builder addAll(Collection iter) { - builder.addAll(iter); + wrapped .addAll(iter); return this; } public boolean isEmpty() { - return builder.isEmpty(); + return wrapped .isEmpty(); } + public BTreeSet build() { - return new BTreeSet<>(builder.build(), builder.comparator); + return new BTreeSet<>(wrapped .build(), wrapped .comparator); } } @@ -615,19 +616,25 @@ public class BTreeSet implements NavigableSet, List return new Builder<>(comparator); } - public static BTreeSet wrap(Object[] btree, Comparator comparator) + /** if you know the precise size of the resultant set use {@code perfectBuilder} instead. */ + public static Builder builder(Comparator comparator, int initialCapacity) + { + return new Builder<>(comparator, initialCapacity); + } + + public static BTreeSet wrap(Object[] btree, Comparator comparator) { return new BTreeSet<>(btree, comparator); } public static > BTreeSet of(Collection sortedValues) { - return new BTreeSet<>(BTree.build(sortedValues, UpdateFunction.noOp()), Ordering.natural()); + return new BTreeSet<>(BTree.build(sortedValues), Ordering.natural()); } public static > BTreeSet of(V value) { - return new BTreeSet<>(BTree.build(ImmutableList.of(value), UpdateFunction.noOp()), Ordering.natural()); + return new BTreeSet<>(BTree.singleton(value), Ordering.natural()); } public static BTreeSet empty(Comparator comparator) @@ -639,4 +646,13 @@ public class BTreeSet implements NavigableSet, List { return new BTreeSet<>(BTree.singleton(value), comparator); } + + public static BTreeSet copy(SortedSet copy, Comparator comparator) + { + try (BTree.FastBuilder builder = BTree.fastBuilder()) + { + copy.forEach(builder::add); + return wrap(builder.build(), comparator); + } + } } diff --git a/src/java/org/apache/cassandra/utils/btree/NodeBuilder.java b/src/java/org/apache/cassandra/utils/btree/NodeBuilder.java deleted file mode 100644 index 93f76fe126..0000000000 --- a/src/java/org/apache/cassandra/utils/btree/NodeBuilder.java +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.utils.btree; - -import org.apache.cassandra.utils.ObjectSizes; - -import java.util.Arrays; -import java.util.Comparator; - -import static org.apache.cassandra.utils.btree.BTree.*; - -/** - * Represents a level / stack item of in progress modifications to a BTree. - */ -final class NodeBuilder -{ - private static final int MAX_KEYS = 1 + (FAN_FACTOR * 2); - - // parent stack - private NodeBuilder parent, child; - - // buffer for building new nodes - private Object[] buildKeys = new Object[MAX_KEYS]; // buffers keys for branches and leaves - private Object[] buildChildren = new Object[1 + MAX_KEYS]; // buffers children for branches only - private int buildKeyPosition; - private int buildChildPosition; - // we null out the contents of buildKeys/buildChildren when clear()ing them for re-use; this is where - // we track how much we actually have to null out - private int maxBuildKeyPosition; - - // current node of the btree we're modifying/copying from - private Object[] copyFrom; - // the index of the first key in copyFrom that has not yet been copied into the build arrays - private int copyFromKeyPosition; - // the index of the first child node in copyFrom that has not yet been copied into the build arrays - private int copyFromChildPosition; - - private UpdateFunction updateFunction; - private Comparator comparator; - - // upper bound of range owned by this level; lets us know if we need to ascend back up the tree - // for the next key we update when bsearch gives an insertion point past the end of the values - // in the current node - private Object upperBound; - - // ensure we aren't referencing any garbage - void clear() - { - NodeBuilder current = this; - while (current != null && current.upperBound != null) - { - current.clearSelf(); - current = current.child; - } - current = parent; - while (current != null && current.upperBound != null) - { - current.clearSelf(); - current = current.parent; - } - } - - void clearSelf() - { - reset(null, null, null, null); - Arrays.fill(buildKeys, 0, maxBuildKeyPosition, null); - Arrays.fill(buildChildren, 0, maxBuildKeyPosition + 1, null); - maxBuildKeyPosition = 0; - } - - // reset counters/setup to copy from provided node - void reset(Object[] copyFrom, Object upperBound, UpdateFunction updateFunction, Comparator comparator) - { - this.copyFrom = copyFrom; - this.upperBound = upperBound; - this.updateFunction = updateFunction; - this.comparator = comparator; - maxBuildKeyPosition = Math.max(maxBuildKeyPosition, buildKeyPosition); - buildKeyPosition = 0; - buildChildPosition = 0; - copyFromKeyPosition = 0; - copyFromChildPosition = 0; - } - - NodeBuilder finish() - { - assert copyFrom != null; - int copyFromKeyEnd = getKeyEnd(copyFrom); - - if (buildKeyPosition + buildChildPosition > 0) - { - // only want to copy if we've already changed something, otherwise we'll return the original - copyKeys(copyFromKeyEnd); - if (!isLeaf(copyFrom)) - copyChildren(copyFromKeyEnd + 1); - } - return isRoot() ? null : ascend(); - } - - /** - * Inserts or replaces the provided key, copying all not-yet-visited keys prior to it into our buffer. - * - * @param key key we are inserting/replacing - * @return the NodeBuilder to retry the update against (a child if we own the range being updated, - * a parent if we do not -- we got here from an earlier key -- and we need to ascend back up), - * or null if we finished the update in this node. - */ - NodeBuilder update(Object key) - { - assert copyFrom != null; - int copyFromKeyEnd = getKeyEnd(copyFrom); - - int i = copyFromKeyPosition; - boolean found; // exact key match? - boolean owns = true; // true if this node (or a child) should contain the key - if (i == copyFromKeyEnd) - { - found = false; - } - else - { - // this optimisation is for the common scenario of updating an existing row with the same columns/keys - // and simply avoids performing a binary search until we've checked the proceeding key; - // possibly we should disable this check if we determine that it fails more than a handful of times - // during any given builder use to get the best of both worlds - int c = -comparator.compare(key, copyFrom[i]); - if (c >= 0) - { - found = c == 0; - } - else - { - i = Arrays.binarySearch(copyFrom, i + 1, copyFromKeyEnd, key, comparator); - found = i >= 0; - if (!found) - i = -i - 1; - } - } - - if (found) - { - Object prev = copyFrom[i]; - Object next = updateFunction.apply(prev, key); - // we aren't actually replacing anything, so leave our state intact and continue - if (prev == next) - return null; - key = next; - } - else if (i == copyFromKeyEnd && compareUpperBound(comparator, key, upperBound) >= 0) - owns = false; - - if (isLeaf(copyFrom)) - { - - if (owns) - { - // copy keys from the original node up to prior to the found index - copyKeys(i); - - if (found) - { - // if found, we've applied updateFunction already - replaceNextKey(key); - } - else - { - // if not found, we need to apply updateFunction still, which is handled in addNewKey - addNewKey(key); - } - - // done, so return null - return null; - } - else - { - // we don't want to copy anything if we're ascending and haven't copied anything previously, - // as in this case we can return the original node. Leaving buildKeyPosition as 0 indicates - // to buildFromRange that it should return the original instead of building a new node - if (buildKeyPosition > 0) - copyKeys(i); - } - - // if we don't own it, all we need to do is ensure we've copied everything in this node - // (which we have done, since not owning means pos >= keyEnd), ascend, and let Modifier.update - // retry against the parent node. The if/ascend after the else branch takes care of that. - } - else - { - // branch - if (found) - { - copyKeys(i); - replaceNextKey(key); - copyChildren(i + 1); - return null; - } - else if (owns) - { - copyKeys(i); - copyChildren(i); - - // belongs to the range owned by this node, but not equal to any key in the node - // so descend into the owning child - Object newUpperBound = i < copyFromKeyEnd ? copyFrom[i] : upperBound; - Object[] descendInto = (Object[]) copyFrom[copyFromKeyEnd + i]; - ensureChild().reset(descendInto, newUpperBound, updateFunction, comparator); - return child; - } - else if (buildKeyPosition > 0 || buildChildPosition > 0) - { - // ensure we've copied all keys and children, but only if we've already copied something. - // otherwise we want to return the original node - copyKeys(copyFromKeyEnd); - copyChildren(copyFromKeyEnd + 1); // since we know that there are exactly 1 more child nodes, than keys - } - } - - return ascend(); - } - - private static int compareUpperBound(Comparator comparator, Object value, Object upperBound) - { - return upperBound == POSITIVE_INFINITY ? -1 : comparator.compare((V)value, (V)upperBound); - } - - // UTILITY METHODS FOR IMPLEMENTATION OF UPDATE/BUILD/DELETE - - boolean isRoot() - { - // if parent == null, or parent.upperBound == null, then we have not initialised a parent builder, - // so we are the top level builder holding modifications; if we have more than FAN_FACTOR items, though, - // we are not a valid root so we would need to spill-up to create a new root - return (parent == null || parent.upperBound == null) && buildKeyPosition <= FAN_FACTOR; - } - - // ascend to the root node, splitting into proper node sizes as we go; useful for building - // where we work only on the newest child node, which may construct many spill-over parents as it goes - NodeBuilder ascendToRoot() - { - NodeBuilder current = this; - while (!current.isRoot()) - current = current.ascend(); - return current; - } - - // builds a new root BTree node - must be called on root of operation - Object[] toNode() - { - // we permit building empty trees as some constructions do not know in advance how many items they will contain - assert buildKeyPosition <= FAN_FACTOR : buildKeyPosition; - return buildFromRange(0, buildKeyPosition, isLeaf(copyFrom), false); - } - - // finish up this level and pass any constructed children up to our parent, ensuring a parent exists - private NodeBuilder ascend() - { - ensureParent(); - boolean isLeaf = isLeaf(copyFrom); - if (buildKeyPosition > FAN_FACTOR) - { - // split current node and move the midpoint into parent, with the two halves as children - int mid = buildKeyPosition / 2; - parent.addExtraChild(buildFromRange(0, mid, isLeaf, true), buildKeys[mid]); - parent.finishChild(buildFromRange(mid + 1, buildKeyPosition - (mid + 1), isLeaf, false)); - } - else - { - parent.finishChild(buildFromRange(0, buildKeyPosition, isLeaf, false)); - } - return parent; - } - - // copy keys from copyf to the builder, up to the provided index in copyf (exclusive) - private void copyKeys(int upToKeyPosition) - { - if (copyFromKeyPosition >= upToKeyPosition) - return; - - int len = upToKeyPosition - copyFromKeyPosition; - assert len <= FAN_FACTOR : upToKeyPosition + "," + copyFromKeyPosition; - - ensureRoom(buildKeyPosition + len); - if (len > 0) - { - System.arraycopy(copyFrom, copyFromKeyPosition, buildKeys, buildKeyPosition, len); - copyFromKeyPosition = upToKeyPosition; - buildKeyPosition += len; - } - } - - // skips the next key in copyf, and puts the provided key in the builder instead - private void replaceNextKey(Object with) - { - // (this first part differs from addNewKey in that we pass the replaced object to replaceF as well) - ensureRoom(buildKeyPosition + 1); - buildKeys[buildKeyPosition++] = with; - - copyFromKeyPosition++; - } - - // applies the updateFunction - // puts the resulting key into the builder - // splits the parent if necessary via ensureRoom - void addNewKey(Object key) - { - ensureRoom(buildKeyPosition + 1); - buildKeys[buildKeyPosition++] = updateFunction.apply(key); - } - - // copies children from copyf to the builder, up to the provided index in copyf (exclusive) - private void copyChildren(int upToChildPosition) - { - // (ensureRoom isn't called here, as we should always be at/behind key additions) - if (copyFromChildPosition >= upToChildPosition) - return; - int len = upToChildPosition - copyFromChildPosition; - if (len > 0) - { - System.arraycopy(copyFrom, getKeyEnd(copyFrom) + copyFromChildPosition, buildChildren, buildChildPosition, len); - copyFromChildPosition = upToChildPosition; - buildChildPosition += len; - } - } - - // adds a new and unexpected child to the builder - called by children that overflow - private void addExtraChild(Object[] child, Object upperBound) - { - ensureRoom(buildKeyPosition + 1); - buildKeys[buildKeyPosition++] = upperBound; - buildChildren[buildChildPosition++] = child; - } - - // adds a replacement expected child to the builder - called by children prior to ascending - private void finishChild(Object[] child) - { - buildChildren[buildChildPosition++] = child; - copyFromChildPosition++; - } - - // checks if we can add the requested keys+children to the builder, and if not we spill-over into our parent - private void ensureRoom(int nextBuildKeyPosition) - { - if (nextBuildKeyPosition < MAX_KEYS) - return; - - // flush even number of items so we don't waste leaf space repeatedly - Object[] flushUp = buildFromRange(0, FAN_FACTOR, isLeaf(copyFrom), true); - ensureParent().addExtraChild(flushUp, buildKeys[FAN_FACTOR]); - int size = FAN_FACTOR + 1; - assert size <= buildKeyPosition : buildKeyPosition + "," + nextBuildKeyPosition; - System.arraycopy(buildKeys, size, buildKeys, 0, buildKeyPosition - size); - buildKeyPosition -= size; - maxBuildKeyPosition = buildKeys.length; - if (buildChildPosition > 0) - { - System.arraycopy(buildChildren, size, buildChildren, 0, buildChildPosition - size); - buildChildPosition -= size; - } - } - - // builds and returns a node from the buffered objects in the given range - private Object[] buildFromRange(int offset, int keyLength, boolean isLeaf, boolean isExtra) - { - // if keyLength is 0, we didn't copy anything from the original, which means we didn't - // modify any of the range owned by it, so can simply return it as is - if (keyLength == 0) - return copyFrom; - - Object[] a; - if (isLeaf) - { - a = new Object[keyLength | 1]; - System.arraycopy(buildKeys, offset, a, 0, keyLength); - } - else - { - a = new Object[2 + (keyLength * 2)]; - System.arraycopy(buildKeys, offset, a, 0, keyLength); - System.arraycopy(buildChildren, offset, a, keyLength, keyLength + 1); - - // calculate the indexOffsets of each key in this node, within the sub-tree rooted at this node - int[] indexOffsets = new int[keyLength + 1]; - int size = BTree.size((Object[]) a[keyLength]); - for (int i = 0 ; i < keyLength ; i++) - { - indexOffsets[i] = size; - size += 1 + BTree.size((Object[]) a[keyLength + 1 + i]); - } - indexOffsets[keyLength] = size; - a[a.length - 1] = indexOffsets; - } - if (isExtra) - updateFunction.allocated(ObjectSizes.sizeOfArray(a)); - else if (a.length != copyFrom.length) - updateFunction.allocated(ObjectSizes.sizeOfArray(a) - - (copyFrom.length == 0 ? 0 : ObjectSizes.sizeOfArray(copyFrom))); - return a; - } - - // checks if there is an initialised parent, and if not creates/initialises one and returns it. - // different to ensureChild, as we initialise here instead of caller, as parents in general should - // already be initialised, and only aren't in the case where we are overflowing the original root node - private NodeBuilder ensureParent() - { - if (parent == null) - { - parent = new NodeBuilder(); - parent.child = this; - } - if (parent.upperBound == null) - parent.reset(EMPTY_BRANCH, upperBound, updateFunction, comparator); - return parent; - } - - // ensures a child level exists and returns it - NodeBuilder ensureChild() - { - if (child == null) - { - child = new NodeBuilder(); - child.parent = this; - } - return child; - } -} diff --git a/src/java/org/apache/cassandra/utils/btree/TreeBuilder.java b/src/java/org/apache/cassandra/utils/btree/TreeBuilder.java deleted file mode 100644 index 128ff6a4a3..0000000000 --- a/src/java/org/apache/cassandra/utils/btree/TreeBuilder.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.utils.btree; - -import java.util.Comparator; - -import io.netty.util.Recycler; - -import static org.apache.cassandra.utils.btree.BTree.POSITIVE_INFINITY; - -/** - * A class for constructing a new BTree, either from an existing one and some set of modifications - * or a new tree from a sorted collection of items. - *

- * This is a fairly heavy-weight object, so a Recycled instance is created for making modifications to a tree - */ -final class TreeBuilder -{ - - private final static Recycler builderRecycler = new Recycler() - { - protected TreeBuilder newObject(Handle handle) - { - return new TreeBuilder(handle); - } - }; - - public static TreeBuilder newInstance() - { - return builderRecycler.get(); - } - - private final Recycler.Handle recycleHandle; - private final NodeBuilder rootBuilder = new NodeBuilder(); - - private TreeBuilder(Recycler.Handle handle) - { - this.recycleHandle = handle; - } - - /** - * At the highest level, we adhere to the classic b-tree insertion algorithm: - * - * 1. Add to the appropriate leaf - * 2. Split the leaf if necessary, add the median to the parent - * 3. Split the parent if necessary, etc. - * - * There is one important difference: we don't actually modify the original tree, but copy each node that we - * modify. Note that every node on the path to the key being inserted or updated will be modified; this - * implies that at a minimum, the root node will be modified for every update, so every root is a "snapshot" - * of a tree that can be iterated or sliced without fear of concurrent modifications. - * - * The NodeBuilder class handles the details of buffering the copied contents of the original tree and - * adding in our changes. Since NodeBuilder maintains parent/child references, it also handles parent-splitting - * (easy enough, since any node affected by the split will already be copied into a NodeBuilder). - * - * One other difference from the simple algorithm is that we perform modifications in bulk; - * we assume @param source has been sorted, e.g. by BTree.update, so the update of each key resumes where - * the previous left off. - */ - public Object[] update(Object[] btree, Comparator comparator, Iterable source, UpdateFunction updateF) - { - assert updateF != null; - - NodeBuilder current = rootBuilder; - current.reset(btree, POSITIVE_INFINITY, updateF, comparator); - - for (K key : source) - { - while (true) - { - if (updateF.abortEarly()) - { - rootBuilder.clear(); - return null; - } - NodeBuilder next = current.update(key); - if (next == null) - break; - // we were in a subtree from a previous key that didn't contain this new key; - // retry against the correct subtree - current = next; - } - } - - // finish copying any remaining keys from the original btree - while (true) - { - NodeBuilder next = current.finish(); - if (next == null) - break; - current = next; - } - - // updating with POSITIVE_INFINITY means that current should be back to the root - assert current.isRoot(); - - Object[] r = current.toNode(); - current.clear(); - - recycleHandle.recycle(this); - - return r; - } -} diff --git a/src/java/org/apache/cassandra/utils/btree/UpdateFunction.java b/src/java/org/apache/cassandra/utils/btree/UpdateFunction.java index 0ab10c2dd5..109f776d29 100644 --- a/src/java/org/apache/cassandra/utils/btree/UpdateFunction.java +++ b/src/java/org/apache/cassandra/utils/btree/UpdateFunction.java @@ -34,15 +34,10 @@ public interface UpdateFunction extends Function */ V apply(V replacing, K update); - /** - * @return true if we should fail the update - */ - boolean abortEarly(); - /** * @param heapSize extra heap space allocated (over previous tree) */ - void allocated(long heapSize); + void onAllocatedOnHeap(long heapSize); public static final class Simple implements UpdateFunction { @@ -52,15 +47,32 @@ public interface UpdateFunction extends Function this.wrapped = wrapped; } - public V apply(V v) { return v; } - public V apply(V replacing, V update) { return wrapped.apply(replacing, update); } - public boolean abortEarly() { return false; } - public void allocated(long heapSize) { } + @Override + public V apply(V v) + { + return v; + } + + @Override + public V apply(V replacing, V update) + { + return wrapped.apply(replacing, update); + } + + @Override + public void onAllocatedOnHeap(long heapSize) + { + } public static Simple of(BiFunction f) { return new Simple<>(f); } + + Simple flip() + { + return of((a, b) -> wrapped.apply(b, a)); + } } static final Simple noOp = Simple.of((a, b) -> a); diff --git a/src/java/org/apache/cassandra/utils/caching/TinyThreadLocalPool.java b/src/java/org/apache/cassandra/utils/caching/TinyThreadLocalPool.java new file mode 100644 index 0000000000..7712c09258 --- /dev/null +++ b/src/java/org/apache/cassandra/utils/caching/TinyThreadLocalPool.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils.caching; + +import io.netty.util.concurrent.FastThreadLocal; + +public class TinyThreadLocalPool extends FastThreadLocal> +{ + protected TinyPool initialValue() + { + return new TinyPool<>(); + } + + // a super-simple pool containing at most two items; useful because we primarily use btrees in a two-tier hierarchy + // so a single thread local item would be insufficient, but an arbitrary length queue too much + public static class TinyPool + { + final Thread thread; + Object val1, val2, val3; + + public TinyPool() + { + this.thread = Thread.currentThread(); + } + + public void offer(V value) + { + if (Thread.currentThread() == thread) + offerSafe(value); + } + private void offerSafe(V value) + { + if (val1 == null) val1 = value; + else if (val2 == null) val2 = value; + else if (val3 == null) val3 = value; + } + public V poll() + { + Object result; + if (val1 != null) + { + result = val1; + val1 = null; + } + else if (val2 != null) + { + result = val2; + val2 = null; + } + else if (val3 != null) + { + result = val3; + val3 = null; + } + else result = null; + return (V) result; + } + } + + public void offer(V value) + { + get().offer(value); + } + + public V poll() + { + return get().poll(); + } +} diff --git a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java b/test/burn/org/apache/cassandra/utils/LongBTreeTest.java index d21f7b34ac..fb6712784a 100644 --- a/test/burn/org/apache/cassandra/utils/LongBTreeTest.java +++ b/test/burn/org/apache/cassandra/utils/LongBTreeTest.java @@ -21,18 +21,19 @@ package org.apache.cassandra.utils; import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.security.SecureRandom; import java.util.*; -import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.IntStream; -import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -52,44 +53,36 @@ import static com.google.common.collect.Iterables.transform; import static java.util.Comparator.naturalOrder; import static java.util.Comparator.reverseOrder; import static org.apache.cassandra.utils.btree.BTree.iterable; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +// TODO: randomise all parameters for all tests, with a target wall time for each iteration +// should dedicate as much wall time to any depth of tree as any other, except depth 1 (which should be less frequent) +// TODO: verify update with no changes returns original +// TODO: verify updateF.allocated() +// TODO: verify reverseInSitu +// TODO: introduce patterns to verification, esp. to transform and update public class LongBTreeTest { - private static final boolean DEBUG = false; - private static int perThreadTrees = 100; + private static int perThreadTrees = 10000; private static int minTreeSize = 4; - private static int maxTreeSize = 10000; - private static float generateTreeByUpdateChance = 0.8f; - private static float generateTreeByCopyChance = 0.1f; - private static float generateTreeByBuilderChance = 0.1f; - private static float generateTreeTotalChance = generateTreeByUpdateChance + generateTreeByCopyChance + generateTreeByBuilderChance; + private static int maxTreeSize = 10000; // TODO randomise this for each test private static int threads = DEBUG ? 1 : Runtime.getRuntime().availableProcessors() * 8; private static final MetricRegistry metrics = new MetricRegistry(); private static final Timer BTREE_TIMER = metrics.timer(MetricRegistry.name(BTree.class, "BTREE")); private static final Timer TREE_TIMER = metrics.timer(MetricRegistry.name(BTree.class, "TREE")); private static final ExecutorService MODIFY = Executors.newFixedThreadPool(threads, new NamedThreadFactory("MODIFY")); private static final ExecutorService COMPARE = DEBUG ? MODIFY : Executors.newFixedThreadPool(threads, new NamedThreadFactory("COMPARE")); - private static final RandomAbort SPORADIC_ABORT = new RandomAbort<>(new Random(), 0.0001f); - - static - { - System.setProperty("cassandra.btree.fanfactor", "4"); - } /************************** TEST ACCESS ********************************************/ @Test public void testSearchIterator() throws InterruptedException { - final int perTreeSelections = 100; - testRandomSelection(perThreadTrees, perTreeSelections, testSearchIteratorFactory()); - } - - private BTreeTestFactory testSearchIteratorFactory() - { - return (test) -> { + final int perTreeSelections = 10; // TODO randomise this for each test + testRandomSelection(randomSeed(), perThreadTrees, perTreeSelections, + (test) -> { IndexedSearchIterator iter1 = test.testAsSet.iterator(); IndexedSearchIterator iter2 = test.testAsList.iterator(); return (key) -> @@ -113,63 +106,56 @@ public class LongBTreeTest // check that by advancing the same key again we get null, but only do it on one of the two iterators // to ensure they both advance differently - if (ThreadLocalRandom.current().nextBoolean()) + if (test.random.nextBoolean()) Assert.assertNull(iter1.next(key)); else Assert.assertNull(iter2.next(key)); }; - }; + }); } @Test public void testInequalityLookups() throws InterruptedException { final int perTreeSelections = 2; - testRandomSelectionOfSet(perThreadTrees, perTreeSelections, testInequalityLookupsFactory()); - } - - private BTreeSetTestFactory testInequalityLookupsFactory() - { - return (test, canonical) -> { - if (!canonical.isEmpty() || !test.isEmpty()) - { - Assert.assertEquals(canonical.isEmpty(), test.isEmpty()); - Assert.assertEquals(canonical.first(), test.first()); - Assert.assertEquals(canonical.last(), test.last()); - } - return (key) -> - { - Assert.assertEquals(test.ceiling(key), canonical.ceiling(key)); - Assert.assertEquals(test.higher(key), canonical.higher(key)); - Assert.assertEquals(test.floor(key), canonical.floor(key)); - Assert.assertEquals(test.lower(key), canonical.lower(key)); - }; - }; + testRandomSelectionOfSet(randomSeed(), perThreadTrees, perTreeSelections, + (test, canonical) -> { + if (!canonical.isEmpty() || !test.isEmpty()) + { + Assert.assertEquals(canonical.isEmpty(), test.isEmpty()); + Assert.assertEquals(canonical.first(), test.first()); + Assert.assertEquals(canonical.last(), test.last()); + } + return (key) -> + { + Assert.assertEquals(test.ceiling(key), canonical.ceiling(key)); + Assert.assertEquals(test.higher(key), canonical.higher(key)); + Assert.assertEquals(test.floor(key), canonical.floor(key)); + Assert.assertEquals(test.lower(key), canonical.lower(key)); + }; + }); } @Test public void testListIndexes() throws InterruptedException { - testRandomSelectionOfList(perThreadTrees, 4, testListIndexesFactory()); - } - - private BTreeListTestFactory testListIndexesFactory() - { - return (test, canonical, cmp) -> - (key) -> - { - int javaIndex = Collections.binarySearch(canonical, key, cmp); - int btreeIndex = test.indexOf(key); - Assert.assertEquals(javaIndex, btreeIndex); - if (javaIndex >= 0) - Assert.assertEquals(canonical.get(javaIndex), test.get(btreeIndex)); - }; + testRandomSelectionOfList(randomSeed(), perThreadTrees, 4, + (test, canonical, cmp) -> + (key) -> + { + int javaIndex = Collections.binarySearch(canonical, key, cmp); + int btreeIndex = test.indexOf(key); + Assert.assertEquals(javaIndex, btreeIndex); + if (javaIndex >= 0) + Assert.assertEquals(canonical.get(javaIndex), test.get(btreeIndex)); + } + ); } @Test public void testToArray() throws InterruptedException { - testRandomSelection(perThreadTrees, 4, + testRandomSelection(randomSeed(), perThreadTrees, 4, (selection) -> { Integer[] array = new Integer[selection.canonicalList.size() + 1]; @@ -196,42 +182,81 @@ public class LongBTreeTest } @Test - public void testTransformAndFilter() throws InterruptedException + public void testTransformAndFilterNone() throws InterruptedException { - testRandomSelection(perThreadTrees, 4, false, false, false, + testRandomSelection(randomSeed(), perThreadTrees, 4, false, false, false, (selection) -> { Map update = new LinkedHashMap<>(); for (Integer i : selection.testKeys) update.put(i, new Integer(i)); - CountingFunction function; + CountingFunction function = new CountingFunction((x) -> x); Object[] original = selection.testAsSet.tree(); - Object[] transformed; + Object[] transformed = BTree.transformAndFilter(original, function); - // test replacing none, leaving all present - function = new CountingFunction((x) -> x); - transformed = BTree.transformAndFilter(original, function); Assert.assertEquals(BTree.size(original), function.count); + assertTrue(BTree.isWellFormed(transformed, naturalOrder())); Assert.assertSame(original, transformed); + }); + } - // test replacing some, leaving all present - function = new CountingFunction((x) -> update.containsKey(x) ? update.get(x) : x); - transformed = BTree.transformAndFilter(original, function); - Assert.assertEquals(BTree.size(original), function.count); - assertSame(transform(selection.canonicalList, function.wrapped), iterable(transformed)); + @Test + public void testTransformAndFilterReplace() throws InterruptedException + { + testRandomSelection(randomSeed(), perThreadTrees, 4, false, false, false, + (selection) -> + { + Map update = new LinkedHashMap<>(); + for (Integer i : selection.testKeys) + update.put(i, new Integer(i)); - // test replacing some, removing some - function = new CountingFunction(update::get); - transformed = BTree.transformAndFilter(original, function); - Assert.assertEquals(BTree.size(original), function.count); - assertSame(filter(transform(selection.canonicalList, function.wrapped), notNull()), iterable(transformed)); + CountingFunction function = new CountingFunction((x) -> update.getOrDefault(x, x)); + Object[] original = selection.testAsSet.tree(); + Object[] transformed = BTree.transformAndFilter(original, function); - // test replacing none, removing some - function = new CountingFunction((x) -> update.containsKey(x) ? null : x); - transformed = BTree.transformAndFilter(selection.testAsList.tree(), function); Assert.assertEquals(BTree.size(original), function.count); - assertSame(filter(transform(selection.canonicalList, function.wrapped), notNull()), iterable(transformed)); + assertTrue(BTree.isWellFormed(transformed, naturalOrder())); + assertSame(transform(selection.canonicalList, function.wrapped::apply), iterable(transformed)); + }); + } + + @Test + public void testTransformAndFilterReplaceAndRemove() throws InterruptedException + { + testRandomSelection(randomSeed(), perThreadTrees, 4, false, false, false, + (selection) -> + { + Map update = new LinkedHashMap<>(); + for (Integer i : selection.testKeys) + update.put(i, new Integer(i)); + + CountingFunction function = new CountingFunction(update::get); + Object[] original = selection.testAsSet.tree(); + Object[] transformed = BTree.transformAndFilter(original, function); + Assert.assertEquals(BTree.size(original), function.count); + assertTrue(BTree.isWellFormed(transformed, naturalOrder())); + assertSame(filter(transform(selection.canonicalList, function.wrapped::apply), notNull()), iterable(transformed)); + }); + } + + @Test + public void testTransformAndFilterRemove() throws InterruptedException + { + testRandomSelection(randomSeed(), perThreadTrees, 4, false, false, false, + (selection) -> + { + Map update = new LinkedHashMap<>(); + for (Integer i : selection.testKeys) + update.put(i, new Integer(i)); + + CountingFunction function = new CountingFunction((x) -> update.containsKey(x) ? null : x); + Object[] original = selection.testAsSet.tree(); + Object[] transformed = BTree.transformAndFilter(selection.testAsList.tree(), function); + Assert.assertEquals(BTree.size(original), function.count); + assertTrue(BTree.isWellFormed(transformed, naturalOrder())); +// Assert.assertEquals(BTree.size(original) - update.size(), BTree.size(transformed)); + assertSame(filter(transform(selection.canonicalList, function.wrapped::apply), notNull()), iterable(transformed)); }); } @@ -247,15 +272,32 @@ public class LongBTreeTest Assert.assertEquals(i1.hasNext(), i2.hasNext()); } - private void testRandomSelectionOfList(int perThreadTrees, int perTreeSelections, BTreeListTestFactory testRun) throws InterruptedException + private static Pair firstDiff(Iterable i1, Iterable i2) { - testRandomSelection(perThreadTrees, perTreeSelections, + return firstDiff(i1.iterator(), i2.iterator()); + } + + private static Pair firstDiff(Iterator i1, Iterator i2) + { + while (i1.hasNext() && i2.hasNext()) + { + Integer v1 = i1.next(); + Integer v2 = i2.next(); + if (v1 != v2) + return Pair.create(v1, v2); + } + return i1.hasNext() ? Pair.create(i1.next(), null) : i2.hasNext() ? Pair.create(null, i2.next()) : null; + } + + private void testRandomSelectionOfList(long testSeed, int perThreadTrees, int perTreeSelections, BTreeListTestFactory testRun) throws InterruptedException + { + testRandomSelection(testSeed, perThreadTrees, perTreeSelections, (BTreeTestFactory) (selection) -> testRun.get(selection.testAsList, selection.canonicalList, selection.comparator)); } - private void testRandomSelectionOfSet(int perThreadTrees, int perTreeSelections, BTreeSetTestFactory testRun) throws InterruptedException + private void testRandomSelectionOfSet(long testSeed, int perThreadTrees, int perTreeSelections, BTreeSetTestFactory testRun) throws InterruptedException { - testRandomSelection(perThreadTrees, perTreeSelections, + testRandomSelection(testSeed, perThreadTrees, perTreeSelections, (BTreeTestFactory) (selection) -> testRun.get(selection.testAsSet, selection.canonicalSet)); } @@ -279,32 +321,23 @@ public class LongBTreeTest void testOne(Integer value); } - private void run(BTreeTestFactory testRun, RandomSelection selection) + private void testRandomSelection(long seed, int perThreadTrees, int perTreeSelections, BTreeTestFactory testRun) throws InterruptedException { - TestEachKey testEachKey = testRun.get(selection); - for (Integer key : selection.testKeys) - testEachKey.testOne(key); + testRandomSelection(seed, perThreadTrees, perTreeSelections, (selection) -> { + TestEachKey testEachKey = testRun.get(selection); + for (Integer key : selection.testKeys) + testEachKey.testOne(key); + }); } - private void run(BTreeSetTestFactory testRun, RandomSelection selection) + private void testRandomSelection(long seed, int perThreadTrees, int perTreeSelections, Consumer testRun) throws InterruptedException { - TestEachKey testEachKey = testRun.get(selection.testAsSet, selection.canonicalSet); - for (Integer key : selection.testKeys) - testEachKey.testOne(key); + testRandomSelection(seed, perThreadTrees, perTreeSelections, true, true, true, testRun); } - private void testRandomSelection(int perThreadTrees, int perTreeSelections, BTreeTestFactory testRun) throws InterruptedException - { - testRandomSelection(perThreadTrees, perTreeSelections, (RandomSelection selection) -> run(testRun, selection)); - } - - private void testRandomSelection(int perThreadTrees, int perTreeSelections, Consumer testRun) throws InterruptedException - { - testRandomSelection(perThreadTrees, perTreeSelections, true, true, true, testRun); - } - - private void testRandomSelection(int perThreadTrees, int perTreeSelections, boolean narrow, boolean mixInNotPresentItems, boolean permitReversal, Consumer testRun) throws InterruptedException + private void testRandomSelection(long seed, int perThreadTrees, int perTreeSelections, boolean narrow, boolean mixInNotPresentItems, boolean permitReversal, Consumer testRun) throws InterruptedException { + final Random outerSeedGenerator = new Random(seed); int threads = Runtime.getRuntime().availableProcessors(); final CountDownLatch latch = new CountDownLatch(threads); final AtomicLong errors = new AtomicLong(); @@ -312,19 +345,18 @@ public class LongBTreeTest final long totalCount = threads * perThreadTrees * perTreeSelections; for (int t = 0 ; t < threads ; t++) { - Runnable runnable = () -> - { + Runnable runnable = () -> { + final Random seedGenerator = new Random(outerSeedGenerator.nextLong()); try { for (int i = 0 ; i < perThreadTrees ; i++) { - // not easy to usefully log seed, as run tests in parallel; need to really pass through to exceptions - long seed = ThreadLocalRandom.current().nextLong(); - Random random = new Random(seed); - RandomTree tree = randomTree(minTreeSize, maxTreeSize, random); + long dataSeed = seedGenerator.nextLong(); + RandomTree tree = randomTree(dataSeed, minTreeSize, maxTreeSize); for (int j = 0 ; j < perTreeSelections ; j++) { - testRun.accept(tree.select(narrow, mixInNotPresentItems, permitReversal)); + long selectionSeed = seedGenerator.nextLong(); + testRun.accept(tree.select(selectionSeed, narrow, mixInNotPresentItems, permitReversal)); count.incrementAndGet(); } } @@ -351,6 +383,9 @@ public class LongBTreeTest private static class RandomSelection { + final long dataSeed; + final long selectionSeed; + final Random random; final List testKeys; final NavigableSet canonicalSet; final List canonicalList; @@ -358,9 +393,13 @@ public class LongBTreeTest final BTreeSet testAsList; final Comparator comparator; - private RandomSelection(List testKeys, NavigableSet canonicalSet, BTreeSet testAsSet, + private RandomSelection(long dataSeed, long selectionSeed, Random random, + List testKeys, NavigableSet canonicalSet, BTreeSet testAsSet, List canonicalList, BTreeSet testAsList, Comparator comparator) { + this.dataSeed = dataSeed; + this.selectionSeed = selectionSeed; + this.random = random; this.testKeys = testKeys; this.canonicalList = canonicalList; this.canonicalSet = canonicalSet; @@ -372,19 +411,22 @@ public class LongBTreeTest private static class RandomTree { - final Random random; + final long dataSeed; final NavigableSet canonical; final BTreeSet test; - private RandomTree(NavigableSet canonical, BTreeSet test, Random random) + private RandomTree(long dataSeed, NavigableSet canonical, BTreeSet test) { + this.dataSeed = dataSeed; this.canonical = canonical; this.test = test; - this.random = random; } - RandomSelection select(boolean narrow, boolean mixInNotPresentItems, boolean permitReversal) + // TODO: revisit logic, document and ensure producing enough distinct patterns + RandomSelection select(long selectionSeed, boolean narrow, boolean mixInNotPresentItems, boolean permitReversal) { + Random random = new Random(selectionSeed); + NavigableSet canonicalSet = this.canonical; BTreeSet testAsSet = this.test; List canonicalList = new ArrayList<>(canonicalSet); @@ -393,8 +435,9 @@ public class LongBTreeTest Assert.assertEquals(canonicalSet.size(), testAsSet.size()); Assert.assertEquals(canonicalList.size(), testAsList.size()); + // TODO: select random patterns of data as well as pure random data (i.e. random sequences, random fixed offsets, random mixes of the above) // sometimes select keys first, so we cover full range - List allKeys = randomKeys(canonical, mixInNotPresentItems, random); + List allKeys = randomKeys(random, canonical, mixInNotPresentItems); List keys = allKeys; int narrowCount = random.nextInt(3); @@ -417,7 +460,7 @@ public class LongBTreeTest if (useLb) { - lbKeyIndex = random.nextInt(indexRange - 1); + lbKeyIndex = nextInt(random, 0, indexRange - 1); Integer candidate = keys.get(lbKeyIndex); if (useLb = (candidate > lbKey && candidate <= ubKey)) { @@ -430,8 +473,7 @@ public class LongBTreeTest } if (useUb) { - int lb = Math.max(lbKeyIndex, keys.size() - indexRange); - ubKeyIndex = random.nextInt(keys.size() - (1 + lb)) + lb; + ubKeyIndex = nextInt(random, Math.max(lbKeyIndex, keys.size() - indexRange), keys.size() - 1); Integer candidate = keys.get(ubKeyIndex); if (useUb = (candidate < ubKey && candidate >= lbKey)) { @@ -491,73 +533,69 @@ public class LongBTreeTest Assert.assertEquals(canonicalSet.last(), testAsList.get(testAsList.size() - 1)); } - return new RandomSelection(keys, canonicalSet, testAsSet, canonicalList, testAsList, comparator); + assertSame(canonicalList, testAsList); + return new RandomSelection(dataSeed, selectionSeed, random, keys, canonicalSet, testAsSet, canonicalList, testAsList, comparator); } } - private static RandomTree randomTree(int minSize, int maxSize, Random random) + private static int nextInt(Random random, int lb, int ub) { + return lb >= ub ? lb : lb + random.nextInt(ub - lb); + } + + private static RandomTree randomTree(long seed, int minSize, int maxSize) + { + Random random = new Random(seed); // perform most of our tree constructions via update, as this is more efficient; since every run uses this // we test builder disproportionately more often than if it had its own test anyway - int maxIntegerValue = random.nextInt(Integer.MAX_VALUE - 1) + 1; - float f = random.nextFloat() / generateTreeTotalChance; - f -= generateTreeByUpdateChance; - if (f < 0) - return randomTreeByUpdate(minSize, maxSize, maxIntegerValue, random); - f -= generateTreeByCopyChance; - if (f < 0) - return randomTreeByCopy(minSize, maxSize, maxIntegerValue, random); - return randomTreeByBuilder(minSize, maxSize, maxIntegerValue, random); + return random.nextFloat() < 0.95 ? randomTreeByUpdate(seed, random, minSize, maxSize) + : randomTreeByBuilder(seed, random, minSize, maxSize); } - private static RandomTree randomTreeByCopy(int minSize, int maxSize, int maxIntegerValue, Random random) + private static RandomTree randomTreeByUpdate(long seed, Random random, int minSize, int maxSize) { assert minSize > 3; TreeSet canonical = new TreeSet<>(); - int targetSize = random.nextInt(maxSize - minSize) + minSize; - int curSize = 0; - while (curSize < targetSize) - { - Integer next = random.nextInt(maxIntegerValue); - if (canonical.add(next)) - ++curSize; - } - return new RandomTree(canonical, BTreeSet.wrap(BTree.build(canonical, UpdateFunction.noOp()), naturalOrder()), random); - } - - private static RandomTree randomTreeByUpdate(int minSize, int maxSize, int maxIntegerValue, Random random) - { - assert minSize > 3; - TreeSet canonical = new TreeSet<>(); - - int targetSize = random.nextInt(maxSize - minSize) + minSize; - int maxModificationSize = random.nextInt(targetSize - 2) + 2; + int targetSize = nextInt(random, minSize, maxSize); + int maxModificationSize = nextInt(random, 2, targetSize); Object[] accmumulate = BTree.empty(); int curSize = 0; while (curSize < targetSize) { - int nextSize = maxModificationSize == 1 ? 1 : random.nextInt(maxModificationSize - 1) + 1; + int nextSize = maxModificationSize == 1 ? 1 : nextInt(random, 1, maxModificationSize); TreeSet build = new TreeSet<>(); + boolean keepOriginal = random.nextBoolean(); + // we don't use no-op, to ensure we know which value will actually result (as no-op doesn't guarantee which makes it through) + UpdateFunction updateF = keepOriginal ? UpdateFunction.Simple.of((a, b) -> a) : InverseNoOp.instance; for (int i = 0 ; i < nextSize ; i++) { - Integer next = random.nextInt(maxIntegerValue); - build.add(next); - canonical.add(next); + Integer next = random.nextInt(); + if (build.add(next)) + { + if (!canonical.add(next) && !keepOriginal) + { + canonical.remove(next); + canonical.add(next); + } + } } - accmumulate = BTree.update(accmumulate, naturalOrder(), build, UpdateFunction.noOp()); + Object[] tmp = BTree.update(accmumulate, BTree.build(build), naturalOrder(), updateF); + assertSame(canonical, BTreeSet.wrap(tmp, naturalOrder())); + accmumulate = tmp; curSize += nextSize; maxModificationSize = Math.min(maxModificationSize, targetSize - curSize); } - return new RandomTree(canonical, BTreeSet.wrap(accmumulate, naturalOrder()), random); + assertSame(canonical, BTreeSet.wrap(accmumulate, naturalOrder())); + return new RandomTree(seed, canonical, BTreeSet.wrap(accmumulate, naturalOrder())); } - private static RandomTree randomTreeByBuilder(int minSize, int maxSize, int maxIntegerValue, Random random) + private static RandomTree randomTreeByBuilder(long seed, Random random, int minSize, int maxSize) { assert minSize > 3; BTree.Builder builder = BTree.builder(naturalOrder()); - int targetSize = random.nextInt(maxSize - minSize) + minSize; + int targetSize = nextInt(random, minSize, maxSize); int maxModificationSize = (int) Math.sqrt(targetSize); TreeSet canonical = new TreeSet<>(); @@ -567,7 +605,7 @@ public class LongBTreeTest List shuffled = new ArrayList<>(); while (curSize < targetSize) { - int nextSize = maxModificationSize <= 1 ? 1 : random.nextInt(maxModificationSize - 1) + 1; + int nextSize = nextInt(random, 1, maxModificationSize); // leave a random selection of previous values (random.nextBoolean() ? ordered.headSet(random.nextInt()) : ordered.tailSet(random.nextInt())).clear(); @@ -575,7 +613,7 @@ public class LongBTreeTest for (int i = 0 ; i < nextSize ; i++) { - Integer next = random.nextInt(maxIntegerValue); + Integer next = random.nextInt(); ordered.add(next); shuffled.add(next); canonical.add(next); @@ -606,12 +644,13 @@ public class LongBTreeTest BTreeSet btree = BTreeSet.wrap(builder.build(), naturalOrder()); Assert.assertEquals(canonical.size(), btree.size()); - return new RandomTree(canonical, btree, random); + assertSame(canonical, btree); + return new RandomTree(seed, canonical, btree); } // select a random subset of the keys, with an optional random population of keys inbetween those that are present // return a value with the search position - private static List randomKeys(Iterable canonical, boolean mixInNotPresentItems, Random random) + private static List randomKeys(Random random, Iterable canonical, boolean mixInNotPresentItems) { boolean useFake = mixInNotPresentItems && random.nextBoolean(); final float fakeRatio = random.nextFloat(); @@ -648,94 +687,144 @@ public class LongBTreeTest return Lists.newArrayList(filter(results, (x) -> random.nextFloat() < useChance)); } - /************************** TEST MUTATION ********************************************/ + /************************** TEST BUILD ********************************************/ @Test - public void testBuildNewTree() + public void testBuild() { - int max = 10000; - final List list = new ArrayList<>(max); - final NavigableSet set = new TreeSet<>(); - BTreeSetTestFactory test = testInequalityLookupsFactory(); - for (int i = 0 ; i < max ; ++i) + Integer[] vs = IntStream.rangeClosed(0, 100000).boxed().toArray(Integer[]::new); + for (UpdateFunction updateF : LongBTreeTest.updateFunctions()) { - list.add(i); - set.add(i); - Object[] tree = BTree.build(list, UpdateFunction.noOp()); - Assert.assertTrue(BTree.isWellFormed(tree, Comparator.naturalOrder())); - BTreeSet btree = new BTreeSet<>(tree, Comparator.naturalOrder()); - RandomSelection selection = new RandomSelection(list, set, btree, list, btree, Comparator.naturalOrder()); - run(test, selection); + try (BulkIterator emptyIter = BulkIterator.of(vs)) + { + Object[] empty = BTree.build(emptyIter, 0, updateF); + assertTrue("" + 0, BTree.isEmpty(empty)); // empty is tested by object identity, so verify we test correctly + } + for (int i = 0 ; i < vs.length ; ++i) + { + try (BulkIterator iter = BulkIterator.of(vs)) + { + Object[] btree = BTree.build(iter, i + 1, updateF); + assertTrue("" + i, BTree.isWellFormed(btree, naturalOrder())); + } + } } } + @Test + public void testFastBuilder() + { + Integer[] vs = IntStream.rangeClosed(0, 100000).boxed().toArray(Integer[]::new); + try (BTree.FastBuilder builder = BTree.fastBuilder()) + { + Object[] empty = builder.build(); + assertTrue("" + 0, BTree.isEmpty(empty)); // empty is tested by object identity, so verify we test correctly + } + for (int i = 0 ; i < vs.length ; ++i) + { + try (BTree.FastBuilder builder = BTree.fastBuilder()) + { + for (int j = 0 ; j <= i ; ++j) + builder.add(vs[j]); + Object[] btree = builder.build(); + assertEquals(i + 1, BTree.size(btree)); + assertTrue(""+i, BTree.isWellFormed(btree, naturalOrder())); + } + } + } + + @Test + public void testBuildByUpdate() + { + Integer[] vs = IntStream.rangeClosed(0, 100000).boxed().toArray(Integer[]::new); + Object[] base = BTree.singleton(vs[0]); + for (int i = 0 ; i < vs.length ; ++i) + { + try (BulkIterator iter = BulkIterator.of(vs)) + { + Object[] insert = BTree.build(iter, i + 1, UpdateFunction.noOp()); + Object[] btree = BTree.update(base, insert, naturalOrder(), InverseNoOp.instance); + assertTrue("" + i, BTree.isWellFormed(btree, naturalOrder())); + } + } + } + + + /************************** TEST MUTATION ********************************************/ + @Test public void testOversizedMiddleInsert() { - TreeSet canon = new TreeSet<>(); - for (int i = 0 ; i < 10000000 ; i++) - canon.add(i); - Object[] btree = BTree.build(Arrays.asList(Integer.MIN_VALUE, Integer.MAX_VALUE), UpdateFunction.noOp()); - btree = BTree.update(btree, naturalOrder(), canon, UpdateFunction.noOp()); - canon.add(Integer.MIN_VALUE); - canon.add(Integer.MAX_VALUE); - assertTrue(BTree.isWellFormed(btree, naturalOrder())); - testEqual("Oversize", BTree.iterator(btree), canon.iterator()); + for (UpdateFunction updateF : LongBTreeTest.updateFunctions()) + { + TreeSet canon = new TreeSet<>(); + for (int i = 0 ; i < 10000000 ; i++) + canon.add(i); + Object[] btree = BTree.build(Arrays.asList(Integer.MIN_VALUE, Integer.MAX_VALUE), updateF); + btree = BTree.update(btree, BTree.build(canon), naturalOrder(), updateF); + canon.add(Integer.MIN_VALUE); + canon.add(Integer.MAX_VALUE); + assertTrue(BTree.isWellFormed(btree, naturalOrder())); + testEqual("Oversize", BTree.iterator(btree), canon.iterator()); + } } @Test public void testIndividualInsertsSmallOverlappingRange() throws ExecutionException, InterruptedException { - testInsertions(50, 1, 1, true); + testInsertions(randomSeed(), 50, 1, 1, true); } @Test public void testBatchesSmallOverlappingRange() throws ExecutionException, InterruptedException { - testInsertions(50, 1, 5, true); + testInsertions(randomSeed(), 50, 1, 5, true); } @Test public void testIndividualInsertsMediumSparseRange() throws ExecutionException, InterruptedException { - testInsertions(perThreadTrees / 10, 500, 10, 1, true); + testInsertions(randomSeed(), perThreadTrees / 10, 500, 10, 1, true); } @Test public void testBatchesMediumSparseRange() throws ExecutionException, InterruptedException { - testInsertions(500, 10, 10, true); + testInsertions(randomSeed(), 500, 10, 10, true); } @Test public void testLargeBatchesLargeRange() throws ExecutionException, InterruptedException { - testInsertions(perThreadTrees / 10, Math.max(maxTreeSize, 5000), 3, 100, true); + testInsertions(randomSeed(), perThreadTrees / 10, Math.max(maxTreeSize, 5000), 3, 100, true); } @Test public void testRandomRangeAndBatches() throws ExecutionException, InterruptedException { - ThreadLocalRandom random = ThreadLocalRandom.current(); - int treeSize = random.nextInt(maxTreeSize / 10, maxTreeSize * 10); + Random seedGenerator = new Random(randomSeed()); for (int i = 0 ; i < perThreadTrees / 10 ; i++) - testInsertions(threads * 10, treeSize, random.nextInt(1, 100) / 10f, treeSize / 100, true); + { + int treeSize = nextInt(seedGenerator, maxTreeSize / 10, maxTreeSize * 10); + testInsertions(seedGenerator.nextLong(), threads * 10, treeSize, nextInt(seedGenerator, 1, 100) / 10f, treeSize / 100, true); + } } @Test public void testSlicingSmallRandomTrees() throws ExecutionException, InterruptedException { - testInsertions(50, 10, 10, false); + testInsertions(randomSeed(), 50, 10, 10, false); } - private static void testInsertions(int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException + private static void testInsertions(long seed, int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException { int tests = perThreadTrees * threads; - testInsertions(tests, perTestCount, testKeyRatio, modificationBatchSize, quickEquality); + testInsertions(seed, tests, perTestCount, testKeyRatio, modificationBatchSize, quickEquality); } - private static void testInsertions(int tests, int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException + private static void testInsertions(long seed, int tests, int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException { + Random random = new Random(seed); int batchesPerTest = perTestCount / modificationBatchSize; int testKeyRange = (int) (perTestCount * testKeyRatio); long totalCount = (long) perTestCount * tests; @@ -749,8 +838,8 @@ public class LongBTreeTest final List>>> outer = new ArrayList<>(); for (int i = 0 ; i < chunkSize ; i++) { - int maxRunLength = modificationBatchSize == 1 ? 1 : ThreadLocalRandom.current().nextInt(1, modificationBatchSize); - outer.add(doOneTestInsertions(testKeyRange, maxRunLength, modificationBatchSize, batchesPerTest, quickEquality)); + int maxRunLength = modificationBatchSize == 1 ? 1 : nextInt(random, 1, modificationBatchSize); + outer.add(doOneTestInsertions(random.nextLong(), testKeyRange, maxRunLength, modificationBatchSize, batchesPerTest, quickEquality)); } final List> inner = new ArrayList<>(); @@ -778,27 +867,32 @@ public class LongBTreeTest log("Done"); } - private static ListenableFutureTask>> doOneTestInsertions(final int upperBound, final int maxRunLength, final int averageModsPerIteration, final int iterations, final boolean quickEquality) + @Test + public void debug() { - ListenableFutureTask>> f = ListenableFutureTask.create(new Callable>>() - { - @Override - public List> call() + randomTree(384037044131282656L, 4, 10000); + } + + private static ListenableFutureTask>> doOneTestInsertions(long seed, final int upperBound, final int maxRunLength, final int averageModsPerIteration, final int iterations, final boolean quickEquality) + { + String id = String.format("<%dL,%d,%d,%d,%d,%b>", seed, upperBound, maxRunLength, averageModsPerIteration, iterations, quickEquality); + Random random = new Random(seed); + ListenableFutureTask>> f = ListenableFutureTask.create(() -> { + try { final List> r = new ArrayList<>(); NavigableMap canon = new TreeMap<>(); Object[] btree = BTree.empty(); final TreeMap buffer = new TreeMap<>(); - ThreadLocalRandom rnd = ThreadLocalRandom.current(); for (int i = 0 ; i < iterations ; i++) { buffer.clear(); - int mods = rnd.nextInt(1, averageModsPerIteration * 2); + int mods = nextInt(random, 1, averageModsPerIteration * 2); while (mods > 0) { - int v = rnd.nextInt(upperBound); + int v = random.nextInt(upperBound); int rc = Math.max(0, Math.min(mods, maxRunLength) - 1); - int c = 1 + (rc <= 0 ? 0 : rnd.nextInt(rc)); + int c = 1 + (rc <= 0 ? 0 : random.nextInt(rc)); for (int j = 0 ; j < c ; j++) { buffer.put(v, v); @@ -811,24 +905,29 @@ public class LongBTreeTest canon.putAll(buffer); ctxt.stop(); ctxt = BTREE_TIMER.time(); - Object[] next = null; - while (next == null) - next = BTree.update(btree, naturalOrder(), buffer.keySet(), SPORADIC_ABORT); - btree = next; + Object[] add = BTree.build(buffer.keySet()); + Object[] newTree = BTree.update(btree, add, naturalOrder(), updateFunction(random)); ctxt.stop(); - if (!BTree.isWellFormed(btree, naturalOrder())) + if (!BTree.isWellFormed(newTree, naturalOrder())) { - log("ERROR: Not well formed"); + log(id + " ERROR: Not well formed"); throw new AssertionError("Not well formed!"); } + btree = newTree; if (quickEquality) - testEqual("", BTree.iterator(btree), canon.keySet().iterator()); + testEqual(id, BTree.iterator(btree), canon.keySet().iterator()); else - r.addAll(testAllSlices("RND", btree, new TreeSet<>(canon.keySet()))); + r.addAll(testAllSlices(id, btree, new TreeSet<>(canon.keySet()))); } return r; } + catch (Throwable t) + { + t.printStackTrace(); + log("Failed %s: %s", id, t.getMessage()); + throw t; + } }); if (DEBUG) f.run(); @@ -840,19 +939,22 @@ public class LongBTreeTest @Test public void testSlicingAllSmallTrees() throws ExecutionException, InterruptedException { - Object[] cur = BTree.empty(); - TreeSet canon = new TreeSet<>(); - // we set FAN_FACTOR to 4, so 128 items is four levels deep, three fully populated - for (int i = 0 ; i < 128 ; i++) + for (UpdateFunction updateF : LongBTreeTest.updateFunctions()) { - String id = String.format("[0..%d)", canon.size()); - log("Testing " + id); - Futures.allAsList(testAllSlices(id, cur, canon)).get(); - Object[] next = null; - while (next == null) - next = BTree.update(cur, naturalOrder(), Arrays.asList(i), SPORADIC_ABORT); - cur = next; - canon.add(i); + Object[] cur = BTree.empty(); + TreeSet canon = new TreeSet<>(); + // we set FAN_FACTOR to 4, so 128 items is four levels deep, three fully populated + for (int i = 0 ; i < 128 ; i++) + { + String id = String.format("[0..%d)", canon.size()); + log("Testing " + id); + Futures.allAsList(testAllSlices(id, cur, canon)).get(); + Object[] next = null; + while (next == null) + next = BTree.update(cur, BTree.singleton(i), naturalOrder(), updateF); + cur = next; + canon.add(i); + } } } @@ -993,36 +1095,37 @@ public class LongBTreeTest }; } - private static final class RandomAbort implements UpdateFunction + private static List> updateFunctions() { - final Random rnd; - final float chance; - private RandomAbort(Random rnd, float chance) - { - this.rnd = rnd; - this.chance = chance; - } + return ImmutableList.of(UpdateFunction.noOp(), InverseNoOp.instance); + } + private static UpdateFunction updateFunction(Random random) + { + return random.nextBoolean() ? InverseNoOp.instance : UpdateFunction.noOp(); + } + + public static final class InverseNoOp implements UpdateFunction + { + public static final InverseNoOp instance = new InverseNoOp(); public V apply(V replacing, V update) { return update; } - - public boolean abortEarly() - { - return rnd.nextFloat() < chance; - } - - public void allocated(long heapSize) + public void onAllocatedOnHeap(long heapSize) { } - public V apply(V v) { return v; } } + private static long randomSeed() + { + return new SecureRandom().nextLong(); + } + public static void main(String[] args) throws ExecutionException, InterruptedException, InvocationTargetException, IllegalAccessException { for (String arg : args) @@ -1076,4 +1179,4 @@ public class LongBTreeTest args[0] = System.currentTimeMillis(); System.out.printf("%tT: " + formatstr + "\n", args); } -} +} \ No newline at end of file diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeBench.java b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeBench.java new file mode 100644 index 0000000000..4490cad6af --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeBench.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench.btree; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.common.collect.ImmutableList; + +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 4, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 8, time = 2, timeUnit = TimeUnit.SECONDS) +@Fork(value = 2) +@Threads(4) +@State(Scope.Benchmark) +public class BTreeBench extends Megamorphism +{ + final AtomicInteger uniqueThreadInitialisation = new AtomicInteger(); + + Integer[] data; + // three iterables to simulate megamorphic callsites for iterable + List dataAsIterable1; + List dataAsIterable2; + List dataAsIterable3; + +// produces sizes between [n/2..n+n/2] + @Param({"1", "4", "16", "64", "256", "1024", "16384"}) + int dataSize; + + @Setup(Level.Trial) + public void setup() + { + setup(dataSize); + } + + void setup(int size) + { + data = new Integer[size + size/2]; + for (int i = 0 ; i < data.length; i++) + data[i] = i; + dataAsIterable1 = Arrays.asList(data); + dataAsIterable2 = ImmutableList.copyOf(data); + dataAsIterable3 = new ArrayList<>(dataSize); + dataAsIterable3.addAll(dataAsIterable1); + } + + @State(Scope.Thread) + public static class BuildSizeState extends IntVisitor + { + @Setup(Level.Trial) + public void setup(BTreeBench bench) + { + super.setup(bench.dataSize); + } + } +} diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeBuildBench.java b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeBuildBench.java new file mode 100644 index 0000000000..cdddd61b53 --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeBuildBench.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench.btree; + +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.cassandra.utils.BulkIterator; +import org.apache.cassandra.utils.btree.BTree; +import org.apache.cassandra.utils.btree.UpdateFunction; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 6, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 8, time = 2, timeUnit = TimeUnit.SECONDS) +@Fork(value = 2) +@Threads(4) +@State(Scope.Benchmark) +public class BTreeBuildBench extends BTreeBench +{ + @Benchmark + public Object[] buildWithIterableStaticBuildMethod(BuildSizeState state) + { + List list; + int size = state.next(); + switch (state.i() % 3) + { + case 0: list = dataAsIterable1; break; + case 1: list = dataAsIterable2; break; + case 2: list = dataAsIterable3; break; + default: throw new IllegalStateException(); + } + return BTree.build(BulkIterator.of(list.iterator()), size, UpdateFunction.noOp()); + } + + @Benchmark + public Object[] buildWithMegamorphicBulkStaticBuildNoop(BuildSizeState state) + { + return buildWithMegamorphicBulkStaticBuild(state, UpdateFunction.noOp()); + } + + public Object[] buildWithMegamorphicBulkStaticBuild(BuildSizeState state, UpdateFunction updateF) + { + BulkIterator iter; + int size = state.next(); + switch (state.i() % 3) + { + case 0: iter = BulkIterator.of(data); break; + case 1: iter = FromArrayCopy.of(data); break; + case 2: iter = FromArrayCopy2.of(data); break; + default: throw new IllegalStateException(); + } + Object[] result = BTree.build(iter, size, updateF); + iter.close(); + return result; + } + + @Benchmark + public Object[] buildWithBulkStaticBuild(BuildSizeState state) + { + int size = state.next(); + try (BulkIterator iter = BulkIterator.of(data)) + { + return BTree.build(iter, size, UpdateFunction.noOp()); + } + } + + @Benchmark + public Object[] buildWithBuilderAuto(BuildSizeState state) + { + int size = state.next(); + BTree.Builder builder = BTree.builder(Comparator.naturalOrder()); + for (int i = 0 ; i < size ; ++i) + builder.add(data[i]); + return builder.build(); + } + + @Benchmark + public Object[] buildWithBuilderManual(BuildSizeState state) + { + int size = state.next(); + BTree.Builder builder = BTree.builder(Comparator.naturalOrder()); + builder.auto(false); + for (int i = 0 ; i < size ; ++i) + builder.add(data[i]); + return builder.build(); + } + + @Benchmark + public Object[] buildWithFastBuilder(BuildSizeState state) + { + int size = state.next(); + try (BTree.FastBuilder builder = BTree.fastBuilder()) + { + for (int i = 0 ; i < size ; ++i) + builder.add(data[i]); + return builder.build(); + } + } +} \ No newline at end of file diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeTransformBench.java b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeTransformBench.java new file mode 100644 index 0000000000..4f2fa78ed3 --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeTransformBench.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench.btree; + +import java.util.BitSet; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import org.apache.cassandra.utils.BulkIterator; +import org.apache.cassandra.utils.btree.BTree; +import org.apache.cassandra.utils.btree.UpdateFunction; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 4, time = 2, timeUnit = TimeUnit.SECONDS) +@Fork(value = 2) +@Threads(4) +@State(Scope.Benchmark) +// TODO: parameterise build method for input to transform +public class BTreeTransformBench extends BTreeBench +{ + public enum Distribution { CONTIGUOUS, RANDOM } + + Integer[] data2; + + @Param({"false"}) + boolean uniquePerTrial; + + @Param({"0", "0.0001", "0.001", "0.01", "0.0625", "0.125", "0.25", "0.5", "1"}) + float ratio; + + @Param({"CONTIGUOUS", "RANDOM"}) + Distribution distribution; + + @Setup(Level.Trial) + public void setup() + { + setup(2 * dataSize); + data2 = data.clone(); + for (int i = 0 ; i < data2.length ; ++i) + data2[i] = new Integer(data2[i]); + } + + @State(Scope.Thread) + public static class ThreadState + { + final Random random = new Random(0); // initialised to a seed below + + final BitSet bitSet = new BitSet(); + + Integer[] data, data2; + boolean uniquePerTrial; + float ratio; + Distribution distribution; + + // unique trials + // instead of doing per-invocation, we do per-iteration, as perfasm measures trial setup costs + Object[][] updates; + BuildSizeState buildSizeState = new BuildSizeState(); + + // current trial + Object[] update; + + @Setup(Level.Trial) + public void doTrialSetup(BTreeTransformBench bench, BuildSizeState invocationBuildSizeState) + { + this.random.setSeed(bench.uniqueThreadInitialisation.incrementAndGet()); + this.data = bench.data; + this.data2 = bench.data2; + this.uniquePerTrial = bench.uniquePerTrial; + this.ratio = bench.ratio; + this.distribution = bench.distribution; + if (!uniquePerTrial) + { + buildSizeState.setup(bench); + buildSizeState.randomise(random); + int numberOfUniqueTrials = (int) Math.min(4096, Runtime.getRuntime().maxMemory() / (4 * 8 * bench.dataSize)); + updates = new Object[numberOfUniqueTrials][]; + for (int i = 0; i < numberOfUniqueTrials; ++i) + updates[i] = createTree(buildSizeState); + } + invocationBuildSizeState.randomise(random); + } + + @Setup(Level.Invocation) + public void doInvocationSetup(BuildSizeState buildSizeState) + { + if (!uniquePerTrial) + { + update = updates[buildSizeState.i() % updates.length]; + buildSizeState.next(); + } + else + { + this.update = createTree(buildSizeState); + } + int size = BTree.size(update); + int setBits = (int) Math.ceil(size * (ratio > 0.5f ? 1 - ratio : ratio)); + switch (distribution) + { + case CONTIGUOUS: setContiguousBits(setBits, size); break; + case RANDOM: setRandomBits(setBits, size); break; + } + } + + private Object[] createTree(BuildSizeState buildSizeState) + { + int buildSize = buildSizeState.next(); + try (BulkIterator.FromArray iter = BulkIterator.of(data)) + { + return BTree.build(iter, buildSize, UpdateFunction.noOp()); + } + } + + private void setRandomBits(int count, int range) + { + ThreadLocalRandom random = ThreadLocalRandom.current(); + bitSet.clear(); + while (count > 0) + { + int next = random.nextInt(range); + if (bitSet.get(next)) + continue; + bitSet.set(next); + --count; + } + } + + private void setContiguousBits(int count, int range) + { + ThreadLocalRandom random = ThreadLocalRandom.current(); + bitSet.clear(); + int start = count >= range ? 0 : random.nextInt(range - count); + bitSet.set(start, start + count); + } + + Function apply(Function replace) + { + return ratio > 0.5f ? i -> bitSet.get(i) ? i : replace.apply(i) + : i -> bitSet.get(i) ? replace.apply(i) : i; + + } + } + + @Benchmark + public Object[] transformReplace(ThreadState state) + { + return BTree.transform(state.update, state.apply(i -> data2[i])); + } + + @Benchmark + public Object[] transformAndFilterReplace(ThreadState state) + { + return BTree.transformAndFilter(state.update, state.apply(i -> data2[i])); + } + + @Benchmark + public Object[] transformAndFilterRemove(ThreadState state) + { + return BTree.transformAndFilter(state.update, state.apply(i -> null)); + } +} \ No newline at end of file diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeUpdateBench.java b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeUpdateBench.java new file mode 100644 index 0000000000..f2be859924 --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/btree/BTreeUpdateBench.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench.btree; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.function.IntFunction; +import java.util.function.Supplier; +import java.util.stream.IntStream; + +import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.btree.BTree; +import org.apache.cassandra.utils.btree.UpdateFunction; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 10, time = 1) +@Measurement(iterations = 10, time = 2) +@Fork(value = 4) +@Threads(4) +@State(Scope.Benchmark) +// TODO: parameterise build method for inputs to update +public class BTreeUpdateBench extends BTreeBench +{ + public enum Distribution { RANDOM, CONTIGUOUS } + + Supplier> comparator = Comparator::naturalOrder; + + Integer[] data2; + + @Param({"1", "4", "16", "64", "256", "1024", "16384"}) + int insertSize; + + @Param({"0", "0.5", "1"}) + float overlap; + + @Param({"RANDOM", "CONTIGUOUS"}) + Distribution distribution; + + @Param({"false", "true"}) + boolean keepOld; + + @Param({"SIMPLE", "SIMPLE_MEGAMORPH", "UNSIMPLE"}) + UpdateF updateF; + + @Param({"false"}) + boolean uniquePerTrial; + + @Setup(Level.Trial) + public void setup() + { + setup(2 * (dataSize + insertSize)); + data2 = data.clone(); + for (int i = 0 ; i < data2.length ; ++i) + data2[i] = new Integer(data2[i]); + } + + @State(Scope.Thread) + public static class InsertSizeState extends IntVisitor + { + @Setup(Level.Trial) + public void setup(BTreeUpdateBench bench) + { + super.setup(bench.insertSize); + } + } + + @State(Scope.Thread) + public static class ThreadState + { + final Random random = new Random(0); // initialised to a seed below + + Comparator comparator; + Integer[] data, data2; + int[] randomOverlaps, randomBuild; + Distribution distribution; + float overlap; + boolean uniquePerTrial; + IntFunction> updateFGetter; + + BTree.Builder buildBuilder = BTree.builder(Comparator.naturalOrder()).auto(false); + BTree.Builder insertBuilder = BTree.builder(Comparator.naturalOrder()).auto(false); + + // unique trials + // instead of doing per-invocation, we do per-iteration, as perfasm measures trial setup costs + Object[][] updates, inserts; + + // current trial + Object[] update, insert; + UpdateFunction updateF; + + @Setup(Level.Trial) + public void doTrialSetup(BTreeUpdateBench bench, BuildSizeState invocationBuildSizeState, InsertSizeState invocationInsertSizeState) + { + random.setSeed(bench.uniqueThreadInitialisation.incrementAndGet()); + this.comparator = bench.comparator.get(); + this.data = bench.data; + this.data2 = bench.data2; + this.randomOverlaps = IntStream.range(0, data.length).toArray(); + this.randomBuild = randomOverlaps.clone(); + this.overlap = bench.overlap; + this.distribution = bench.distribution; + this.updateFGetter = updateFGetter(bench.keepOld, bench.updateF); + this.uniquePerTrial = bench.uniquePerTrial; + if (!uniquePerTrial) + { + BuildSizeState buildSizeState = new BuildSizeState(); + InsertSizeState insertSizeState = new InsertSizeState(); + buildSizeState.setup(bench); + insertSizeState.setup(bench); + buildSizeState.randomise(random); + insertSizeState.randomise(random); + int numberOfUniqueTrials = (int) Math.min(2048, Runtime.getRuntime().maxMemory() / (4 * 8 * (bench.insertSize + bench.dataSize))); + updates = new Object[numberOfUniqueTrials][]; + inserts = new Object[numberOfUniqueTrials][]; + for (int i = 0; i < numberOfUniqueTrials; ++i) + { + Pair updateAndInsert = createUpdateAndInsert(buildSizeState, insertSizeState); + updates[i] = updateAndInsert.left; + inserts[i] = updateAndInsert.right; + } + } + invocationBuildSizeState.randomise(random); + invocationInsertSizeState.randomise(random); + } + + @Setup(Level.Invocation) + public void doInvocationSetup(BuildSizeState buildSizeState, InsertSizeState insertSizeState) + { + if (!uniquePerTrial) + { + update = updates[buildSizeState.i() % updates.length]; + insert = inserts[buildSizeState.i() % inserts.length]; + buildSizeState.next(); + } + else + { + Pair updateAndInsert = createUpdateAndInsert(buildSizeState, insertSizeState); + this.update = updateAndInsert.left; + this.insert = updateAndInsert.right; + } + updateF = updateFGetter.apply(buildSizeState.i()); + } + + /** + * Create an iteration to the benchmark's spec, i.e. two trees with the specified size and overlap + */ + private Pair createUpdateAndInsert(BuildSizeState buildSizeState, InsertSizeState insertSizeState) + { + int buildSize = buildSizeState.next(); + int insertSize = insertSizeState.next(); + int overlapSize = (int) (Math.min(buildSize, insertSize) * overlap); + assert overlapSize <= buildSize && overlapSize <= insertSize; + + BTree.Builder build = buildBuilder; + BTree.Builder insert = insertBuilder; + build.reuse(); + insert.reuse(); + + switch (distribution) + { + case RANDOM: + { + updateOverlap(overlapSize); + buildRandom(build, data, buildSize, overlapSize); + buildRandom(insert, data2, insertSize, overlapSize); + break; + } + case CONTIGUOUS: + { + switch (buildSizeState.i() % 4) + { + case 0: + { + // left-hand insert overlap + int i = 0; + for (int j = 0, mj = insertSize-overlapSize ; j < mj ; ++j) + insert.add(data2[i++]); + for (int j = 0 ; j < overlapSize ; ++j) + { + build.add(data[i]); + insert.add(data2[i++]); + } + for (int j = 0, mj = buildSize-overlapSize ; j < mj ; ++j) + build.add(data[i++]); + break; + } + case 1: + { + // right-hand insert overlap + int i = 0; + for (int j = 0, mj = buildSize-overlapSize ; j < mj ; ++j) + build.add(data[i++]); + for (int j = 0 ; j < overlapSize ; ++j) + { + build.add(data[i]); + insert.add(data2[i++]); + } + for (int j = 0, mj = insertSize-overlapSize ; j < mj ; ++j) + insert.add(data2[i++]); + break; + } + case 2: + { + // straddle insert overlap + int i = 0; + for (int j = 0, mj = (insertSize-overlapSize)/2 ; j < mj ; ++j) + insert.add(data2[i++]); + for (int j = 0, mj = (buildSize-overlapSize)/2 ; j < mj ; ++j) + build.add(data[i++]); + for (int j = 0 ; j < overlapSize ; ++j) + { + build.add(data[i]); + insert.add(data2[i++]); + } + for (int j = 0, mj = buildSize - (overlapSize + (buildSize-overlapSize)/2) ; j < mj ; ++j) + build.add(data[i++]); + for (int j = 0, mj = insertSize - (overlapSize + (insertSize-overlapSize)/2); j < mj ; ++j) + insert.add(data2[i++]); + break; + } + case 3: + { + // straddle update overlap + int i = 0; + for (int j = 0, mj = (buildSize-overlapSize)/2 ; j < mj ; ++j) + build.add(data[i++]); + for (int j = 0, mj = (insertSize-overlapSize)/2 ; j < mj ; ++j) + insert.add(data2[i++]); + for (int j = 0 ; j < overlapSize ; ++j) + { + build.add(data[i]); + insert.add(data2[i++]); + } + for (int j = 0, mj = insertSize - (overlapSize + (insertSize-overlapSize)/2); j < mj ; ++j) + insert.add(data2[i++]); + for (int j = 0, mj = buildSize - (overlapSize + (buildSize-overlapSize)/2) ; j < mj ; ++j) + build.add(data[i++]); + } + } + break; + } + } + + return Pair.create(build.build(), insert.build()); + } + + /** + * Randomise the elements of {@code data} with occur in the first {@code size} elements, then sort them + */ + private void shuffleAndSort(int[] data, int size) + { + for (int i = 0 ; i < size ; ++i) + { + int swap = random.nextInt(data.length); + int tmp = data[swap]; + data[swap] = data[i]; + data[i] = tmp; + } + Arrays.sort(data, 0, size); + } + + /** + * Randomise the elements of {@link #randomOverlaps} with occur in the first {@code size} elements, then sort them + */ + private void updateOverlap(int size) + { + shuffleAndSort(randomOverlaps, size); + } + + private void buildRandom(BTree.Builder build, Integer[] data, int buildSize, int overlapSize) + { + shuffleAndSort(randomBuild, buildSize + overlapSize); + // linear merge + int i = 0, c = 0, j = 0; + for ( ; c < buildSize && j < overlapSize ; ++c) + { + if (randomBuild[i] < randomOverlaps[j]) build.add(data[randomBuild[i++]]); + else if (randomBuild[i] > randomOverlaps[j]) build.add(data[randomOverlaps[j++]]); + else { build.add(data[randomBuild[i++]]); j++; } + } + while (c++ < buildSize) + build.add(data[randomBuild[i++]]); + } + } + + @Benchmark + public Object[] benchUpdate(ThreadState state) + { + return BTree.update(state.update, state.insert, state.comparator, state.updateF); + } + +} diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/IntVisitor.java b/test/microbench/org/apache/cassandra/test/microbench/btree/IntVisitor.java new file mode 100644 index 0000000000..a0a45ab5d3 --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/btree/IntVisitor.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench.btree; + +import java.util.Random; + +/** + * A utility class to visit 2^n integers, around a median value, in an order that rapidly covers a wide range of values + */ +public class IntVisitor +{ + int invocations; + int shift; + int mask; + int base; + + public IntVisitor() + { + } + + public IntVisitor(int median) + { + setup(median); + } + + public void setup(int median) + { + int variance = Integer.highestOneBit(median); + mask = variance - 1; + base = median - variance/2; + shift = 32 - Integer.bitCount(mask); + } + + public int i() + { + return invocations; + } + + public int cur() + { + return size(invocations); + } + + public int next() + { + return size(invocations = invocations == Integer.MAX_VALUE ? 0 : invocations + 1); + } + + public int min() + { + return base; + }; + + public int size() + { + return mask + 1; + }; + + public int size(int invocations) + { + // we reverse the integer to more evenly distribute the visitation of sizes + return base + (Integer.reverse(invocations & mask) >>> shift); + } + + public void randomise(Random random) + { + invocations = random.nextInt(Integer.MAX_VALUE); + } +} \ No newline at end of file diff --git a/test/microbench/org/apache/cassandra/test/microbench/btree/Megamorphism.java b/test/microbench/org/apache/cassandra/test/microbench/btree/Megamorphism.java new file mode 100644 index 0000000000..370acb3d4d --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/btree/Megamorphism.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench.btree; + +import java.util.function.IntFunction; + +import org.apache.cassandra.utils.BulkIterator; +import org.apache.cassandra.utils.btree.UpdateFunction; +import org.apache.cassandra.utils.caching.TinyThreadLocalPool; + +public class Megamorphism +{ + // All functionallity noops, + public enum UpdateF { SIMPLE, SIMPLE_MEGAMORPH, UNSIMPLE } + + private static final UpdateFunction SIMPLE_KEEP_OLD_1 = UpdateFunction.noOp(); + private static final UpdateFunction SIMPLE_KEEP_OLD_2 = UpdateFunction.Simple.of((a, b) -> a); + private static final UpdateFunction SIMPLE_KEEP_OLD_3 = UpdateFunction.Simple.of((a, b) -> a); + + private static final UpdateFunction SIMPLE_KEEP_NEW_1 = UpdateFunction.Simple.of((a, b) -> b); + private static final UpdateFunction SIMPLE_KEEP_NEW_2 = UpdateFunction.Simple.of((a, b) -> b); + private static final UpdateFunction SIMPLE_KEEP_NEW_3 = UpdateFunction.Simple.of((a, b) -> b); + + private static final UpdateFunction UNSIMPLE_KEEP_OLD = new UpdateFunction() + { + public Object apply(Object replacing, Object update) { return replacing; } + public void onAllocatedOnHeap(long heapSize) { } + public Object apply(Object v) { return v; } + }; + + private static final UpdateFunction UNSIMPLE_KEEP_NEW = new UpdateFunction() + { + public Object apply(Object replacing, Object update) { return update; } + public void onAllocatedOnHeap(long heapSize) { } + public Object apply(Object v) { return v; } + }; + + static IntFunction> updateFGetter(boolean keepOld, BTreeBench.UpdateF updateF) + { + switch (updateF) + { + case SIMPLE: return keepOld ? i -> SIMPLE_KEEP_OLD_1 : i -> SIMPLE_KEEP_NEW_1; + case UNSIMPLE: return keepOld ? i -> UNSIMPLE_KEEP_OLD : i -> UNSIMPLE_KEEP_NEW; + case SIMPLE_MEGAMORPH: + if (keepOld) + { + return i -> { + switch (i % 3) + { + case 0: return SIMPLE_KEEP_OLD_1; + case 1: return SIMPLE_KEEP_OLD_2; + case 2: return SIMPLE_KEEP_OLD_3; + default: throw new IllegalStateException(); + } + }; + } + else + { + return i -> { + switch (i % 3) + { + case 0: return SIMPLE_KEEP_NEW_1; + case 1: return SIMPLE_KEEP_NEW_2; + case 2: return SIMPLE_KEEP_NEW_3; + default: throw new IllegalStateException(); + } + }; + } + default: + throw new IllegalStateException(); + } + } + + static class FromArrayCopy implements BulkIterator, AutoCloseable + { + private static final TinyThreadLocalPool cache = new TinyThreadLocalPool<>(); + + private Object[] from; + private int i; + private TinyThreadLocalPool.TinyPool pool; + + public static FromArrayCopy of(Object[] from) + { + TinyThreadLocalPool.TinyPool pool = cache.get(); + FromArrayCopy result = pool.poll(); + if (result == null) + result = new FromArrayCopy<>(); + result.from = from; + result.i = 0; + result.pool = pool; + return result; + } + + public void close() + { + pool.offer(this); + from = null; + pool = null; + } + + public void fetch(Object[] into, int offset, int count) + { + System.arraycopy(from, i, into, offset, count); + i += count; + } + + public V next() + { + + return (V) from[i++]; + } + } + + static class FromArrayCopy2 implements BulkIterator, AutoCloseable + { + private static final TinyThreadLocalPool cache = new TinyThreadLocalPool<>(); + + private Object[] from; + private int i; + private TinyThreadLocalPool.TinyPool pool; + + public static FromArrayCopy2 of(Object[] from) + { + TinyThreadLocalPool.TinyPool pool = cache.get(); + FromArrayCopy2 result = pool.poll(); + if (result == null) + result = new FromArrayCopy2<>(); + result.from = from; + result.i = 0; + result.pool = pool; + return result; + } + + public void close() + { + pool.offer(this); + from = null; + pool = null; + } + + public void fetch(Object[] into, int offset, int count) + { + System.arraycopy(from, i, into, offset, count); + i += count; + } + + public V next() + { + + return (V) from[i++]; + } + } +} diff --git a/test/unit/org/apache/cassandra/utils/btree/BTreeRemovalTest.java b/test/unit/org/apache/cassandra/utils/btree/BTreeRemovalTest.java index a9cf383805..76e7098d86 100644 --- a/test/unit/org/apache/cassandra/utils/btree/BTreeRemovalTest.java +++ b/test/unit/org/apache/cassandra/utils/btree/BTreeRemovalTest.java @@ -37,7 +37,7 @@ public class BTreeRemovalTest { static { - System.setProperty("cassandra.btree.fanfactor", "8"); + System.setProperty("cassandra.btree.branchshift", "3"); } private static final Comparator CMP = new Comparator() @@ -101,7 +101,7 @@ public class BTreeRemovalTest private static Object[] generateLeaf(int from, int size) { - final Object[] result = new Object[(size & 1) == 1 ? size : size + 1]; + final Object[] result = new Object[size | 1]; for (int i = 0; i < size; ++i) result[i] = from + i; return result; @@ -112,16 +112,21 @@ public class BTreeRemovalTest assert keys.length > 0; assert children.length > 1; assert children.length == keys.length + 1; + final Object[] result = new Object[keys.length + children.length + 1]; for (int i = 0; i < keys.length; ++i) result[i] = keys[i]; + for (int i = 0; i < children.length; ++i) result[keys.length + i] = children[i]; + final int[] sizeMap = new int[children.length]; sizeMap[0] = BTree.size(children[0]); for (int i = 1; i < children.length; ++i) sizeMap[i] = sizeMap[i - 1] + BTree.size(children[i]) + 1; + result[result.length - 1] = sizeMap; + return result; } @@ -131,9 +136,11 @@ public class BTreeRemovalTest final Object[][] leaves = new Object[leafSizes.length][]; for (int i = 0; i < leaves.length; ++i) leaves[i] = generateLeaf(10 * i + 1, leafSizes[i]); + final int[] keys = new int[leafSizes.length - 1]; for (int i = 0; i < keys.length; ++i) keys[i] = 10 * (i + 1); + final Object[] btree = generateBranch(keys, leaves); assertTrue(BTree.isWellFormed(btree, CMP)); return btree; @@ -184,7 +191,7 @@ public class BTreeRemovalTest @Test public void testRemoveFromRootWhichIsALeaf() { - for (int size = 1; size < 9; ++size) + for (int size = 1; size <= BTree.MAX_KEYS; ++size) { final Object[] btree = new Object[(size & 1) == 1 ? size : size + 1]; for (int i = 0; i < size; ++i) @@ -218,7 +225,7 @@ public class BTreeRemovalTest @Test public void testRemoveFromNonMinimalLeaf() { - for (int size = 5; size < 9; ++size) + for (int size = 5; size <= BTree.MAX_KEYS; ++size) { final Object[] btree = generateSampleTwoLevelsTree(new int[] {size, 4, 4, 4, 4}); @@ -370,7 +377,7 @@ public class BTreeRemovalTest SortedSet data = new TreeSet<>(); for (int i = 0; i < 1000; ++i) data.add(rand.nextInt()); - Object[] btree = BTree.build(data, UpdateFunction.noOp()); + Object[] btree = BTree.build(data); assertTrue(BTree.isWellFormed(btree, CMP)); assertTrue(Iterables.elementsEqual(data, BTree.iterable(btree))); diff --git a/test/unit/org/apache/cassandra/utils/btree/BTreeSearchIteratorTest.java b/test/unit/org/apache/cassandra/utils/btree/BTreeSearchIteratorTest.java index 69ca93c7df..978bdd4931 100644 --- a/test/unit/org/apache/cassandra/utils/btree/BTreeSearchIteratorTest.java +++ b/test/unit/org/apache/cassandra/utils/btree/BTreeSearchIteratorTest.java @@ -138,7 +138,7 @@ public class BTreeSearchIteratorTest @Test public void testTreeIteratorNormal() { - Object[] btree = BTree.build(seq(30), UpdateFunction.noOp()); + Object[] btree = BTree.build(seq(30)); BTreeSearchIterator fullIter = new FullBTreeSearchIterator<>(btree, CMP, Dir.ASC); BTreeSearchIterator leafIter = new LeafBTreeSearchIterator<>(btree, CMP, Dir.ASC); assertBTreeSearchIteratorEquals(fullIter, leafIter); @@ -181,7 +181,7 @@ public class BTreeSearchIteratorTest @Test public void testTreeIteratorOneElem() { - Object[] btree = BTree.build(seq(1), UpdateFunction.noOp()); + Object[] btree = BTree.build(seq(1)); BTreeSearchIterator fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC); BTreeSearchIterator leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC); assertBTreeSearchIteratorEquals(fullIter, leafIter); @@ -217,7 +217,7 @@ public class BTreeSearchIteratorTest @Test public void testTreeIteratorNotFound() { - Object[] btree = BTree.build(seq(31, 0, 3), UpdateFunction.noOp()); + Object[] btree = BTree.build(seq(31, 0, 3)); BTreeSearchIterator fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC); BTreeSearchIterator leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC); assertBTreeSearchIteratorEquals(fullIter, leafIter, 3 * 5 + 1); diff --git a/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java b/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java index e60fb64560..e72dde2240 100644 --- a/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java +++ b/test/unit/org/apache/cassandra/utils/btree/BTreeTest.java @@ -21,15 +21,10 @@ import java.util.*; import java.util.concurrent.ThreadLocalRandom; import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import org.junit.Test; import org.junit.Assert; -import static org.apache.cassandra.utils.btree.BTree.EMPTY_LEAF; -import static org.apache.cassandra.utils.btree.BTree.FAN_FACTOR; -import static org.apache.cassandra.utils.btree.BTree.FAN_SHIFT; -import static org.apache.cassandra.utils.btree.BTree.POSITIVE_INFINITY; import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; @@ -38,7 +33,7 @@ public class BTreeTest static Integer[] ints = new Integer[20]; static { - System.setProperty("cassandra.btree.fanfactor", "4"); + System.setProperty("cassandra.btree.branchshift", "2"); for (int i = 0 ; i < ints.length ; i++) ints[i] = new Integer(i); } @@ -50,14 +45,8 @@ public class BTreeTest return ints[update]; } - public boolean abortEarly() + public void onAllocatedOnHeap(long heapSize) { - return false; - } - - public void allocated(long heapSize) - { - } public Integer apply(Integer integer) @@ -66,28 +55,6 @@ public class BTreeTest } }; - private static final UpdateFunction noOp = new UpdateFunction() - { - public Integer apply(Integer replacing, Integer update) - { - return update; - } - - public boolean abortEarly() - { - return false; - } - - public void allocated(long heapSize) - { - } - - public Integer apply(Integer k) - { - return k; - } - }; - private static List seq(int count, int interval) { List r = new ArrayList<>(); @@ -102,27 +69,7 @@ public class BTreeTest return seq(count, 1); } - private static List rand(int count) - { - Random rand = ThreadLocalRandom.current(); - List r = seq(count); - for (int i = 0 ; i < count - 1 ; i++) - { - int swap = i + rand.nextInt(count - i); - Integer tmp = r.get(i); - r.set(i, r.get(swap)); - r.set(swap, tmp); - } - return r; - } - - private static final Comparator CMP = new Comparator() - { - public int compare(Integer o1, Integer o2) - { - return Integer.compare(o1, o2); - } - }; + private static final Comparator CMP = Integer::compare; @Test public void testBuilding_UpdateFunctionReplacement() @@ -135,14 +82,14 @@ public class BTreeTest public void testUpdate_UpdateFunctionReplacement() { for (int i = 0; i < 20 ; i++) - checkResult(i, BTree.update(BTree.build(seq(i), noOp), CMP, seq(i), updateF)); + checkResult(i, BTree.update(BTree.build(seq(i)), BTree.build(seq(i)), CMP, updateF)); } @Test public void testApply() { List input = seq(71); - Object[] btree = BTree.build(input, noOp); + Object[] btree = BTree.build(input); final List result = new ArrayList<>(); BTree.apply(btree, i -> result.add(i)); @@ -154,7 +101,7 @@ public class BTreeTest public void inOrderAccumulation() { List input = seq(71); - Object[] btree = BTree.build(input, noOp); + Object[] btree = BTree.build(input); long result = BTree.accumulate(btree, (o, l) -> { Assert.assertEquals((long) o, l + 1); return o; @@ -169,7 +116,7 @@ public class BTreeTest for (int interval=1; interval<=5; interval++) { List input = seq(limit, interval); - Object[] btree = BTree.build(input, noOp); + Object[] btree = BTree.build(input); for (int start=0; start<=limit; start+=interval) { int thisInterval = interval; @@ -190,7 +137,7 @@ public class BTreeTest public void accumulateFromEnd() { List input = seq(100); - Object[] btree = BTree.build(input, noOp); + Object[] btree = BTree.build(input); long result = BTree.accumulate(btree, (o, l) -> 1, Integer::compareTo, 101, 0L); Assert.assertEquals(0, result); } @@ -202,35 +149,35 @@ public class BTreeTest @Test public void testUpdate_UpdateFunctionCallBack() { - Object[] btree = new Object[1]; + Object[] btree = BTree.singleton(1); CallsMonitor monitor = new CallsMonitor(); - btree = BTree.update(btree, CMP, Arrays.asList(1), monitor); + btree = BTree.update(btree, BTree.singleton(1), CMP, monitor); assertArrayEquals(new Object[] {1}, btree); assertEquals(1, monitor.getNumberOfCalls(1)); monitor.clear(); - btree = BTree.update(btree, CMP, Arrays.asList(2), monitor); + btree = BTree.update(btree, BTree.singleton(2), CMP, monitor); assertArrayEquals(new Object[] {1, 2, null}, btree); assertEquals(1, monitor.getNumberOfCalls(2)); // with existing value monitor.clear(); - btree = BTree.update(btree, CMP, Arrays.asList(1), monitor); + btree = BTree.update(btree, BTree.singleton(1), CMP, monitor); assertArrayEquals(new Object[] {1, 2, null}, btree); assertEquals(1, monitor.getNumberOfCalls(1)); // with two non-existing values monitor.clear(); - btree = BTree.update(btree, CMP, Arrays.asList(3, 4), monitor); - assertArrayEquals(new Object[] {1, 2, 3, 4, null}, btree); + btree = BTree.update(btree, BTree.build(Arrays.asList(3, 4)), CMP, monitor); + assertArrayEquals(new Object[] {3, new Object[]{1, 2, null}, new Object[]{4}, new int[]{2, 4}}, btree); assertEquals(1, monitor.getNumberOfCalls(3)); assertEquals(1, monitor.getNumberOfCalls(4)); // with one existing value and one non existing value monitor.clear(); - btree = BTree.update(btree, CMP, Arrays.asList(2, 5), monitor); - assertArrayEquals(new Object[] {3, new Object[]{1, 2, null}, new Object[]{4, 5, null}, new int[]{2, 5}}, btree); + btree = BTree.update(btree, BTree.build(Arrays.asList(2, 5)), CMP, monitor); + assertArrayEquals(new Object[] {3, new Object[]{1, 2, null}, new Object[]{4, 5, null}, new int[]{2, 5}}, btree); assertEquals(1, monitor.getNumberOfCalls(2)); assertEquals(1, monitor.getNumberOfCalls(5)); } @@ -323,6 +270,9 @@ public class BTreeTest builder.reuse(Comparator.reverseOrder()); for (int i = 0; i < 12; i++) builder.add(sorted.get(i)); + System.out.println(BTree.MAX_KEYS); + System.out.println(BTree.MIN_KEYS); + System.out.println(BTree.toString(builder.build())); checkResult(12, builder.build(), BTree.Dir.DESC); builder.reuse(); @@ -458,6 +408,7 @@ public class BTreeTest private static void checkResult(int count, Object[] btree, BTree.Dir dir) { + assertTrue(BTree.isWellFormed(btree, BTree.Dir.DESC == dir ? CMP.reversed() : CMP)); Iterator iter = BTree.slice(btree, CMP, dir); int i = 0; while (iter.hasNext()) @@ -465,44 +416,6 @@ public class BTreeTest assertEquals(count, i); } - @Test - public void testClearOnAbort() - { - Object[] btree = BTree.build(seq(2), noOp); - Object[] copy = Arrays.copyOf(btree, btree.length); - BTree.update(btree, CMP, seq(94), new AbortAfterX(90)); - - assertArrayEquals(copy, btree); - - btree = BTree.update(btree, CMP, seq(94), noOp); - assertTrue(BTree.isWellFormed(btree, CMP)); - } - - private static final class AbortAfterX implements UpdateFunction - { - int counter; - final int abortAfter; - private AbortAfterX(int abortAfter) - { - this.abortAfter = abortAfter; - } - public Integer apply(Integer replacing, Integer update) - { - return update; - } - public boolean abortEarly() - { - return counter++ > abortAfter; - } - public void allocated(long heapSize) - { - } - public Integer apply(Integer v) - { - return v; - } - } - /** * UpdateFunction that count the number of call made to apply for each value. */ @@ -510,22 +423,20 @@ public class BTreeTest { private int[] numberOfCalls = new int[20]; + @Override public Integer apply(Integer replacing, Integer update) { numberOfCalls[update] = numberOfCalls[update] + 1; return update; } - public boolean abortEarly() - { - return false; - } - - public void allocated(long heapSize) + @Override + public void onAllocatedOnHeap(long heapSize) { } + @Override public Integer apply(Integer integer) { numberOfCalls[integer] = numberOfCalls[integer] + 1; @@ -542,106 +453,4 @@ public class BTreeTest Arrays.fill(numberOfCalls, 0); } } - - @Test - public void testTransformAndFilter() - { - List r = seq(100); - - Object[] b1 = BTree.build(r, UpdateFunction.noOp()); - - // replace all values - Object[] b2 = BTree.transformAndFilter(b1, (x) -> (Integer) x * 2); - assertEquals(BTree.size(b1), BTree.size(b2)); - - // remove odd numbers - Object[] b3 = BTree.transformAndFilter(b1, (x) -> (Integer) x % 2 == 1 ? x : null); - assertEquals(BTree.size(b1) / 2, BTree.size(b3)); - - // remove all values - Object[] b4 = BTree.transformAndFilter(b1, (x) -> null); - assertEquals(0, BTree.size(b4)); - } - - private Object[] buildBTreeLegacy(Iterable source, UpdateFunction updateF, int size) - { - assert updateF != null; - NodeBuilder current = new NodeBuilder(); - - while ((size >>= FAN_SHIFT) > 0) - current = current.ensureChild(); - - current.reset(EMPTY_LEAF, POSITIVE_INFINITY, updateF, null); - for (K key : source) - current.addNewKey(key); - - current = current.ascendToRoot(); - - Object[] r = current.toNode(); - current.clear(); - return r; - } - - // Basic BTree validation to check the values and sizeOffsets. Return tree size. - private int validateBTree(Object[] tree, int[] startingPos, boolean isRoot) - { - if (BTree.isLeaf(tree)) - { - int size = BTree.size(tree); - if (!isRoot) - { - assertTrue(size >= FAN_FACTOR / 2); - assertTrue(size <= FAN_FACTOR); - } - for (int i = 0; i < size; i++) - { - assertEquals((int)tree[i], startingPos[0]); - startingPos[0]++; - } - return size; - } - - int childNum = BTree.getChildCount(tree); - assertTrue(childNum >= FAN_FACTOR / 2); - assertTrue(childNum <= FAN_FACTOR + 1); - - int childStart = BTree.getChildStart(tree); - int[] sizeOffsets = BTree.getSizeMap(tree); - int pos = 0; - for (int i = 0; i < childNum; i++) - { - int childSize = validateBTree((Object[])tree[i + childStart], startingPos, false); - - pos += childSize; - assertEquals(sizeOffsets[i], pos); - if (i != childNum - 1) - { - assertEquals((int)tree[i], startingPos[0]); - pos++; - startingPos[0]++; - } - - } - return BTree.size(tree); - } - - @Test - public void testBuildTree() - { - int maxCount = 1000; - - for (int count = 0; count < maxCount; count++) - { - List r = seq(count); - Object[] b1 = BTree.build(r, UpdateFunction.noOp()); - Object[] b2 = buildBTreeLegacy(r, UpdateFunction.noOp(), count); - assertTrue(BTree.equals(b1, b2)); - - int[] startingPos = new int[1]; - startingPos[0] = 0; - assertEquals(count, validateBTree(b1, startingPos, true)); - startingPos[0] = 0; - assertEquals(count, validateBTree(b2, startingPos, true)); - } - } }