natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimise BTree build, update and transform operations 

Patch Benedict Elliott Smith; reviewed by Branimir Lambov and Benjamin Lerer for CASSANDRA-15510
This commit is contained in:
Benedict Elliott Smith 2021-12-10 18:39:55 +01:00 committed by Benjamin Lerer
parent 2873c91269
commit 018c8e0d5e
26 changed files with 4763 additions and 1456 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -1,3 +1,6 @@
4.0.5
* Optimise BTree build,update and transform operations (CASSANDRA-15510)
4.0.4
* Clean up schema migration coordinator and tests (CASSANDRA-17533)
* Shut repair task executor down without interruption to avoid compromising shared channel proxies (CASSANDRA-17466)

4
build.xml
View File

@ -1236,10 +1236,12 @@
<target name="build-jmh" depends="build-test, jar" description="Create JMH uber jar">
<jar jarfile="${build.test.dir}/deps.jar">
<zipgroupfileset dir="${build.dir.lib}/jars">
<zipgroupfileset dir="${test.lib}/jars">
<include name="*jmh*.jar"/>
<include name="jopt*.jar"/>
<include name="commons*.jar"/>
<include name="junit*.jar"/>
<include name="hamcrest*.jar"/>
</zipgroupfileset>
<zipgroupfileset dir="${build.lib}" includes="*.jar"/>
</jar>

4
src/java/org/apache/cassandra/db/Columns.java
View File

@ -40,7 +40,6 @@ import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.btree.BTree;
import org.apache.cassandra.utils.btree.BTreeSearchIterator;
import org.apache.cassandra.utils.btree.BTreeRemoval;
import org.apache.cassandra.utils.btree.UpdateFunction;
/**
* An immutable and sorted list of (non-PK) columns for a given table.
@ -264,8 +263,7 @@ public class Columns extends AbstractCollection<ColumnMetadata> implements Colle
if (this == NONE)
return other;
Object[] tree = BTree.<ColumnMetadata>merge(this.columns, other.columns, Comparator.naturalOrder(),
UpdateFunction.noOp());
Object[] tree = BTree.update(this.columns, other.columns, Comparator.naturalOrder());
if (tree == this.columns)
return this;
if (tree == other.columns)

14
src/java/org/apache/cassandra/db/partitions/AtomicBTreePartition.java
View File

@ -127,7 +127,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
updater.inputDeletionInfoCopy = update.deletionInfo().copy(HeapAllocator.instance);
deletionInfo = current.deletionInfo.mutableCopy().add(updater.inputDeletionInfoCopy);
updater.allocated(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize());
updater.onAllocatedOnHeap(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize());
}
else
{
@ -135,14 +135,14 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
}
RegularAndStaticColumns columns = update.columns().mergeTo(current.columns);
updater.allocated(columns.unsharedHeapSize() - current.columns.unsharedHeapSize());
updater.onAllocatedOnHeap(columns.unsharedHeapSize() - current.columns.unsharedHeapSize());
Row newStatic = update.staticRow();
Row staticRow = newStatic.isEmpty()
? current.staticRow
: (current.staticRow.isEmpty() ? updater.apply(newStatic) : updater.apply(current.staticRow, newStatic));
Object[] tree = BTree.update(current.tree, update.metadata().comparator, update, update.rowCount(), updater);
Object[] tree = BTree.update(current.tree, update.holder().tree, update.metadata().comparator, updater);
EncodingStats newStats = current.stats.mergeWith(update.stats());
updater.allocated(newStats.unsharedHeapSize() - current.stats.unsharedHeapSize());
updater.onAllocatedOnHeap(newStats.unsharedHeapSize() - current.stats.unsharedHeapSize());
if (tree != null && refUpdater.compareAndSet(this, current, new Holder(columns, tree, deletionInfo, staticRow, newStats)))
{
@ -371,7 +371,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
indexer.onInserted(insert);
this.dataSize += data.dataSize();
allocated(data.unsharedHeapSizeExcludingData());
onAllocatedOnHeap(data.unsharedHeapSizeExcludingData());
if (inserted == null)
inserted = new ArrayList<>();
inserted.add(data);
@ -388,7 +388,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
indexer.onUpdated(existing, reconciled);
dataSize += reconciled.dataSize() - existing.dataSize();
allocated(reconciled.unsharedHeapSizeExcludingData() - existing.unsharedHeapSizeExcludingData());
onAllocatedOnHeap(reconciled.unsharedHeapSizeExcludingData() - existing.unsharedHeapSizeExcludingData());
if (inserted == null)
inserted = new ArrayList<>();
inserted.add(reconciled);
@ -408,7 +408,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
return updating.ref != ref;
}
public void allocated(long heapSize)
public void onAllocatedOnHeap(long heapSize)
{
this.heapSize += heapSize;
}

6
src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
View File

@ -858,8 +858,8 @@ public class PartitionUpdate extends AbstractBTreePartition
// assert that we are not calling build() several times
assert !isBuilt : "A PartitionUpdate.Builder should only get built once";
Object[] add = rowBuilder.build();
Object[] merged = BTree.<Row>merge(tree, add, metadata.comparator,
UpdateFunction.Simple.of(Rows::merge));
Object[] merged = BTree.<Row, Row, Row>update(tree, add, metadata.comparator,
UpdateFunction.Simple.of(Rows::merge));
EncodingStats newStats = EncodingStats.Collector.collect(staticRow, BTree.iterator(merged), deletionInfo);
@ -907,7 +907,7 @@ public class PartitionUpdate extends AbstractBTreePartition
public Builder updateAllTimestamp(long newTimestamp)
{
deletionInfo.updateAllTimestamp(newTimestamp - 1);
tree = BTree.<Row>transformAndFilter(tree, (x) -> x.updateAllTimestamp(newTimestamp));
tree = BTree.<Row, Row>transformAndFilter(tree, (x) -> x.updateAllTimestamp(newTimestamp));
staticRow = this.staticRow.updateAllTimestamp(newTimestamp);
return this;
}

2
src/java/org/apache/cassandra/db/rows/BTreeRow.java
View File

@ -720,7 +720,7 @@ public class BTreeRow extends AbstractRow
}
}
Object[] btree = BTree.build(buildFrom, UpdateFunction.noOp());
Object[] btree = BTree.build(buildFrom);
return new ComplexColumnData(column, btree, deletion);
}
}

27
src/java/org/apache/cassandra/db/rows/ComplexColumnData.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.db.rows;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.Objects;
import java.util.function.BiFunction;
import com.google.common.base.Function;
@ -35,6 +36,7 @@ import org.apache.cassandra.schema.DroppedColumn;
import org.apache.cassandra.utils.BiLongAccumulator;
import org.apache.cassandra.utils.LongAccumulator;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.SearchIterator;
import org.apache.cassandra.utils.btree.BTree;
/**
@ -98,6 +100,11 @@ public class ComplexColumnData extends ColumnData implements Iterable<Cell<?>>
return BTree.iterator(cells);
}
public SearchIterator<CellPath, Cell> searchIterator()
{
return BTree.slice(cells, column().asymmetricCellPathComparator(), BTree.Dir.ASC);
}
public Iterator<Cell<?>> reverseIterator()
{
return BTree.iterator(cells, BTree.Dir.DESC);
@ -195,17 +202,25 @@ public class ComplexColumnData extends ColumnData implements Iterable<Cell<?>>
return transformAndFilter(complexDeletion, (cell) -> filter.fetchedCellIsQueried(column, cell.path()) ? null : cell);
}
private ComplexColumnData transformAndFilter(DeletionTime newDeletion, Function<? super Cell<?>, ? extends Cell<?>> function)
private ComplexColumnData update(DeletionTime newDeletion, Object[] newCells)
{
Object[] transformed = BTree.transformAndFilter(cells, function);
if (cells == transformed && newDeletion == complexDeletion)
if (cells == newCells && newDeletion == complexDeletion)
return this;
if (newDeletion == DeletionTime.LIVE && BTree.isEmpty(transformed))
if (newDeletion == DeletionTime.LIVE && BTree.isEmpty(newCells))
return null;
return new ComplexColumnData(column, transformed, newDeletion);
return new ComplexColumnData(column, newCells, newDeletion);
}
public ComplexColumnData transformAndFilter(DeletionTime newDeletion, Function<? super Cell, ? extends Cell> function)
{
return update(newDeletion, BTree.transformAndFilter(cells, function));
}
public <V> ComplexColumnData transformAndFilter(BiFunction<? super Cell, ? super V, ? extends Cell> function, V param)
{
return update(complexDeletion, BTree.transformAndFilter(cells, function, param));
}
public ComplexColumnData updateAllTimestamp(long newTimestamp)

2
src/java/org/apache/cassandra/db/rows/Row.java
View File

@ -732,7 +732,7 @@ public interface Row extends Unfiltered, Iterable<ColumnData>
// Because some data might have been shadowed by the 'activeDeletion', we could have an empty row
return rowInfo.isEmpty() && rowDeletion.isLive() && dataBuffer.isEmpty()
? null
: BTreeRow.create(clustering, rowInfo, rowDeletion, BTree.build(dataBuffer, UpdateFunction.noOp()));
: BTreeRow.create(clustering, rowInfo, rowDeletion, BTree.build(dataBuffer));
}
public Clustering<?> mergedClustering()

112
src/java/org/apache/cassandra/utils/BulkIterator.java Normal file
View File

@ -0,0 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils;
import java.util.Iterator;
import org.apache.cassandra.utils.caching.TinyThreadLocalPool;
public interface BulkIterator<V> extends AutoCloseable
{
void fetch(Object[] into, int offset, int count);
V next();
default void close() {};
public static class FromArray<V> implements BulkIterator<V>, AutoCloseable
{
private static final TinyThreadLocalPool<FromArray> POOL = new TinyThreadLocalPool<>();
private Object[] from;
private int i;
private TinyThreadLocalPool.TinyPool<FromArray> pool;
private void init(Object[] from, int offset)
{
this.from = from;
this.i = offset;
}
public void close()
{
pool.offer(this);
from = null;
pool = null;
}
public void fetch(Object[] into, int offset, int count)
{
System.arraycopy(from, i, into, offset, count);
i += count;
}
public V next()
{
return (V) from[i++];
}
}
public static class Adapter<V> implements BulkIterator<V>
{
final Iterator<V> adapt;
private Adapter(Iterator<V> adapt)
{
this.adapt = adapt;
}
public void fetch(Object[] into, int offset, int count)
{
count += offset;
while (offset < count && adapt.hasNext())
into[offset++] = adapt.next();
}
public boolean hasNext()
{
return adapt.hasNext();
}
public V next()
{
return adapt.next();
}
}
public static <V> FromArray<V> of(Object[] from)
{
return of(from, 0);
}
public static <V> FromArray<V> of(Object[] from, int offset)
{
TinyThreadLocalPool.TinyPool<FromArray> pool = FromArray.POOL.get();
FromArray<V> result = pool.poll();
if (result == null)
result = new FromArray<>();
result.init(from, offset);
result.pool = pool;
return result;
}
public static <V> Adapter<V> of(Iterator<V> from)
{
return new Adapter<>(from);
}
}

3465
src/java/org/apache/cassandra/utils/btree/BTree.java
View File

File diff suppressed because it is too large Load Diff

12
src/java/org/apache/cassandra/utils/btree/BTreeRemoval.java
View File

@ -85,9 +85,9 @@ public class BTreeRemoval
index -= (1 + BTree.getSizeMap(node)[i - 1]);
Object[] nextNode = (Object[]) node[keyEnd + i];
boolean nextNodeNeedsCopy = true;
if (BTree.getKeyEnd(nextNode) > BTree.MINIMAL_NODE_SIZE)
if (BTree.getKeyEnd(nextNode) > BTree.MIN_KEYS)
node = copyIfNeeded(node, needsCopy);
else if (i > 0 && BTree.getKeyEnd((Object[]) node[keyEnd + i - 1]) > BTree.MINIMAL_NODE_SIZE)
else if (i > 0 && BTree.getKeyEnd((Object[]) node[keyEnd + i - 1]) > BTree.MIN_KEYS)
{
node = copyIfNeeded(node, needsCopy);
final Object[] leftNeighbour = (Object[]) node[keyEnd + i - 1];
@ -96,7 +96,7 @@ public class BTreeRemoval
index += BTree.size((Object[])leftNeighbour[BTree.getChildEnd(leftNeighbour) - 1]);
nextNode = rotateLeft(node, i);
}
else if (i < keyEnd && BTree.getKeyEnd((Object[]) node[keyEnd + i + 1]) > BTree.MINIMAL_NODE_SIZE)
else if (i < keyEnd && BTree.getKeyEnd((Object[]) node[keyEnd + i + 1]) > BTree.MIN_KEYS)
{
node = copyIfNeeded(node, needsCopy);
nextNode = rotateRight(node, i);
@ -256,12 +256,12 @@ public class BTreeRemoval
private static <V> Object[] merge(final Object[] left, final Object[] right, final V nodeKey)
{
assert BTree.getKeyEnd(left) == BTree.MINIMAL_NODE_SIZE;
assert BTree.getKeyEnd(right) == BTree.MINIMAL_NODE_SIZE;
assert BTree.getKeyEnd(left) == BTree.MIN_KEYS;
assert BTree.getKeyEnd(right) == BTree.MIN_KEYS;
final boolean leaves = BTree.isLeaf(left);
final Object[] result;
if (leaves)
result = new Object[BTree.MINIMAL_NODE_SIZE * 2 + 1];
result = new Object[BTree.MIN_KEYS * 2 + 1];
else
result = new Object[left.length + right.length];
int offset = 0;

46
src/java/org/apache/cassandra/utils/btree/BTreeSet.java
View File

@ -20,7 +20,6 @@ package org.apache.cassandra.utils.btree;
import java.util.*;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Ordering;
import org.apache.cassandra.utils.btree.BTree.Dir;
@ -39,11 +38,6 @@ public class BTreeSet<V> implements NavigableSet<V>, List<V>
this.comparator = comparator;
}
public BTreeSet<V> update(Collection<V> updateWith)
{
return new BTreeSet<>(BTree.update(tree, comparator, updateWith, UpdateFunction.<V>noOp()), comparator);
}
@Override
public Comparator<? super V> comparator()
{
@ -582,31 +576,38 @@ public class BTreeSet<V> implements NavigableSet<V>, List<V>
public static class Builder<V>
{
final BTree.Builder<V> builder;
final BTree.Builder<V> wrapped;
protected Builder(Comparator<? super V> comparator)
{
builder= BTree.builder(comparator);
wrapped = BTree.builder(comparator);
}
protected Builder(Comparator<? super V> comparator, int size)
{
wrapped = BTree.builder(comparator, size);
}
public Builder<V> add(V v)
{
builder.add(v);
wrapped .add(v);
return this;
}
public Builder<V> addAll(Collection<V> iter)
{
builder.addAll(iter);
wrapped .addAll(iter);
return this;
}
public boolean isEmpty()
{
return builder.isEmpty();
return wrapped .isEmpty();
}
public BTreeSet<V> build()
{
return new BTreeSet<>(builder.build(), builder.comparator);
return new BTreeSet<>(wrapped .build(), wrapped .comparator);
}
}
@ -615,19 +616,25 @@ public class BTreeSet<V> implements NavigableSet<V>, List<V>
return new Builder<>(comparator);
}
public static <V> BTreeSet<V> wrap(Object[] btree, Comparator<V> comparator)
/** if you know the precise size of the resultant set use {@code perfectBuilder} instead. */
public static <V> Builder<V> builder(Comparator<? super V> comparator, int initialCapacity)
{
return new Builder<>(comparator, initialCapacity);
}
public static <V> BTreeSet<V> wrap(Object[] btree, Comparator<? super V> comparator)
{
return new BTreeSet<>(btree, comparator);
}
public static <V extends Comparable<V>> BTreeSet<V> of(Collection<V> sortedValues)
{
return new BTreeSet<>(BTree.build(sortedValues, UpdateFunction.<V>noOp()), Ordering.<V>natural());
return new BTreeSet<>(BTree.build(sortedValues), Ordering.<V>natural());
}
public static <V extends Comparable<V>> BTreeSet<V> of(V value)
{
return new BTreeSet<>(BTree.build(ImmutableList.of(value), UpdateFunction.<V>noOp()), Ordering.<V>natural());
return new BTreeSet<>(BTree.singleton(value), Ordering.<V>natural());
}
public static <V> BTreeSet<V> empty(Comparator<? super V> comparator)
@ -639,4 +646,13 @@ public class BTreeSet<V> implements NavigableSet<V>, List<V>
{
return new BTreeSet<>(BTree.singleton(value), comparator);
}
public static <V> BTreeSet<V> copy(SortedSet<? extends V> copy, Comparator<? super V> comparator)
{
try (BTree.FastBuilder<V> builder = BTree.fastBuilder())
{
copy.forEach(builder::add);
return wrap(builder.build(), comparator);
}
}
}

441
src/java/org/apache/cassandra/utils/btree/NodeBuilder.java
View File

@ -1,441 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cassandra.utils.btree;
import org.apache.cassandra.utils.ObjectSizes;
import java.util.Arrays;
import java.util.Comparator;
import static org.apache.cassandra.utils.btree.BTree.*;
/**
* Represents a level / stack item of in progress modifications to a BTree.
*/
final class NodeBuilder
{
private static final int MAX_KEYS = 1 + (FAN_FACTOR * 2);
// parent stack
private NodeBuilder parent, child;
// buffer for building new nodes
private Object[] buildKeys = new Object[MAX_KEYS]; // buffers keys for branches and leaves
private Object[] buildChildren = new Object[1 + MAX_KEYS]; // buffers children for branches only
private int buildKeyPosition;
private int buildChildPosition;
// we null out the contents of buildKeys/buildChildren when clear()ing them for re-use; this is where
// we track how much we actually have to null out
private int maxBuildKeyPosition;
// current node of the btree we're modifying/copying from
private Object[] copyFrom;
// the index of the first key in copyFrom that has not yet been copied into the build arrays
private int copyFromKeyPosition;
// the index of the first child node in copyFrom that has not yet been copied into the build arrays
private int copyFromChildPosition;
private UpdateFunction updateFunction;
private Comparator comparator;
// upper bound of range owned by this level; lets us know if we need to ascend back up the tree
// for the next key we update when bsearch gives an insertion point past the end of the values
// in the current node
private Object upperBound;
// ensure we aren't referencing any garbage
void clear()
{
NodeBuilder current = this;
while (current != null && current.upperBound != null)
{
current.clearSelf();
current = current.child;
}
current = parent;
while (current != null && current.upperBound != null)
{
current.clearSelf();
current = current.parent;
}
}
void clearSelf()
{
reset(null, null, null, null);
Arrays.fill(buildKeys, 0, maxBuildKeyPosition, null);
Arrays.fill(buildChildren, 0, maxBuildKeyPosition + 1, null);
maxBuildKeyPosition = 0;
}
// reset counters/setup to copy from provided node
void reset(Object[] copyFrom, Object upperBound, UpdateFunction updateFunction, Comparator comparator)
{
this.copyFrom = copyFrom;
this.upperBound = upperBound;
this.updateFunction = updateFunction;
this.comparator = comparator;
maxBuildKeyPosition = Math.max(maxBuildKeyPosition, buildKeyPosition);
buildKeyPosition = 0;
buildChildPosition = 0;
copyFromKeyPosition = 0;
copyFromChildPosition = 0;
}
NodeBuilder finish()
{
assert copyFrom != null;
int copyFromKeyEnd = getKeyEnd(copyFrom);
if (buildKeyPosition + buildChildPosition > 0)
{
// only want to copy if we've already changed something, otherwise we'll return the original
copyKeys(copyFromKeyEnd);
if (!isLeaf(copyFrom))
copyChildren(copyFromKeyEnd + 1);
}
return isRoot() ? null : ascend();
}
/**
* Inserts or replaces the provided key, copying all not-yet-visited keys prior to it into our buffer.
*
* @param key key we are inserting/replacing
* @return the NodeBuilder to retry the update against (a child if we own the range being updated,
* a parent if we do not -- we got here from an earlier key -- and we need to ascend back up),
* or null if we finished the update in this node.
*/
NodeBuilder update(Object key)
{
assert copyFrom != null;
int copyFromKeyEnd = getKeyEnd(copyFrom);
int i = copyFromKeyPosition;
boolean found; // exact key match?
boolean owns = true; // true if this node (or a child) should contain the key
if (i == copyFromKeyEnd)
{
found = false;
}
else
{
// this optimisation is for the common scenario of updating an existing row with the same columns/keys
// and simply avoids performing a binary search until we've checked the proceeding key;
// possibly we should disable this check if we determine that it fails more than a handful of times
// during any given builder use to get the best of both worlds
int c = -comparator.compare(key, copyFrom[i]);
if (c >= 0)
{
found = c == 0;
}
else
{
i = Arrays.binarySearch(copyFrom, i + 1, copyFromKeyEnd, key, comparator);
found = i >= 0;
if (!found)
i = -i - 1;
}
}
if (found)
{
Object prev = copyFrom[i];
Object next = updateFunction.apply(prev, key);
// we aren't actually replacing anything, so leave our state intact and continue
if (prev == next)
return null;
key = next;
}
else if (i == copyFromKeyEnd && compareUpperBound(comparator, key, upperBound) >= 0)
owns = false;
if (isLeaf(copyFrom))
{
if (owns)
{
// copy keys from the original node up to prior to the found index
copyKeys(i);
if (found)
{
// if found, we've applied updateFunction already
replaceNextKey(key);
}
else
{
// if not found, we need to apply updateFunction still, which is handled in addNewKey
addNewKey(key);
}
// done, so return null
return null;
}
else
{
// we don't want to copy anything if we're ascending and haven't copied anything previously,
// as in this case we can return the original node. Leaving buildKeyPosition as 0 indicates
// to buildFromRange that it should return the original instead of building a new node
if (buildKeyPosition > 0)
copyKeys(i);
}
// if we don't own it, all we need to do is ensure we've copied everything in this node
// (which we have done, since not owning means pos >= keyEnd), ascend, and let Modifier.update
// retry against the parent node. The if/ascend after the else branch takes care of that.
}
else
{
// branch
if (found)
{
copyKeys(i);
replaceNextKey(key);
copyChildren(i + 1);
return null;
}
else if (owns)
{
copyKeys(i);
copyChildren(i);
// belongs to the range owned by this node, but not equal to any key in the node
// so descend into the owning child
Object newUpperBound = i < copyFromKeyEnd ? copyFrom[i] : upperBound;
Object[] descendInto = (Object[]) copyFrom[copyFromKeyEnd + i];
ensureChild().reset(descendInto, newUpperBound, updateFunction, comparator);
return child;
}
else if (buildKeyPosition > 0 || buildChildPosition > 0)
{
// ensure we've copied all keys and children, but only if we've already copied something.
// otherwise we want to return the original node
copyKeys(copyFromKeyEnd);
copyChildren(copyFromKeyEnd + 1); // since we know that there are exactly 1 more child nodes, than keys
}
}
return ascend();
}
private static <V> int compareUpperBound(Comparator<V> comparator, Object value, Object upperBound)
{
return upperBound == POSITIVE_INFINITY ? -1 : comparator.compare((V)value, (V)upperBound);
}
// UTILITY METHODS FOR IMPLEMENTATION OF UPDATE/BUILD/DELETE
boolean isRoot()
{
// if parent == null, or parent.upperBound == null, then we have not initialised a parent builder,
// so we are the top level builder holding modifications; if we have more than FAN_FACTOR items, though,
// we are not a valid root so we would need to spill-up to create a new root
return (parent == null || parent.upperBound == null) && buildKeyPosition <= FAN_FACTOR;
}
// ascend to the root node, splitting into proper node sizes as we go; useful for building
// where we work only on the newest child node, which may construct many spill-over parents as it goes
NodeBuilder ascendToRoot()
{
NodeBuilder current = this;
while (!current.isRoot())
current = current.ascend();
return current;
}
// builds a new root BTree node - must be called on root of operation
Object[] toNode()
{
// we permit building empty trees as some constructions do not know in advance how many items they will contain
assert buildKeyPosition <= FAN_FACTOR : buildKeyPosition;
return buildFromRange(0, buildKeyPosition, isLeaf(copyFrom), false);
}
// finish up this level and pass any constructed children up to our parent, ensuring a parent exists
private NodeBuilder ascend()
{
ensureParent();
boolean isLeaf = isLeaf(copyFrom);
if (buildKeyPosition > FAN_FACTOR)
{
// split current node and move the midpoint into parent, with the two halves as children
int mid = buildKeyPosition / 2;
parent.addExtraChild(buildFromRange(0, mid, isLeaf, true), buildKeys[mid]);
parent.finishChild(buildFromRange(mid + 1, buildKeyPosition - (mid + 1), isLeaf, false));
}
else
{
parent.finishChild(buildFromRange(0, buildKeyPosition, isLeaf, false));
}
return parent;
}
// copy keys from copyf to the builder, up to the provided index in copyf (exclusive)
private void copyKeys(int upToKeyPosition)
{
if (copyFromKeyPosition >= upToKeyPosition)
return;
int len = upToKeyPosition - copyFromKeyPosition;
assert len <= FAN_FACTOR : upToKeyPosition + "," + copyFromKeyPosition;
ensureRoom(buildKeyPosition + len);
if (len > 0)
{
System.arraycopy(copyFrom, copyFromKeyPosition, buildKeys, buildKeyPosition, len);
copyFromKeyPosition = upToKeyPosition;
buildKeyPosition += len;
}
}
// skips the next key in copyf, and puts the provided key in the builder instead
private void replaceNextKey(Object with)
{
// (this first part differs from addNewKey in that we pass the replaced object to replaceF as well)
ensureRoom(buildKeyPosition + 1);
buildKeys[buildKeyPosition++] = with;
copyFromKeyPosition++;
}
// applies the updateFunction
// puts the resulting key into the builder
// splits the parent if necessary via ensureRoom
void addNewKey(Object key)
{
ensureRoom(buildKeyPosition + 1);
buildKeys[buildKeyPosition++] = updateFunction.apply(key);
}
// copies children from copyf to the builder, up to the provided index in copyf (exclusive)
private void copyChildren(int upToChildPosition)
{
// (ensureRoom isn't called here, as we should always be at/behind key additions)
if (copyFromChildPosition >= upToChildPosition)
return;
int len = upToChildPosition - copyFromChildPosition;
if (len > 0)
{
System.arraycopy(copyFrom, getKeyEnd(copyFrom) + copyFromChildPosition, buildChildren, buildChildPosition, len);
copyFromChildPosition = upToChildPosition;
buildChildPosition += len;
}
}
// adds a new and unexpected child to the builder - called by children that overflow
private void addExtraChild(Object[] child, Object upperBound)
{
ensureRoom(buildKeyPosition + 1);
buildKeys[buildKeyPosition++] = upperBound;
buildChildren[buildChildPosition++] = child;
}
// adds a replacement expected child to the builder - called by children prior to ascending
private void finishChild(Object[] child)
{
buildChildren[buildChildPosition++] = child;
copyFromChildPosition++;
}
// checks if we can add the requested keys+children to the builder, and if not we spill-over into our parent
private void ensureRoom(int nextBuildKeyPosition)
{
if (nextBuildKeyPosition < MAX_KEYS)
return;
// flush even number of items so we don't waste leaf space repeatedly
Object[] flushUp = buildFromRange(0, FAN_FACTOR, isLeaf(copyFrom), true);
ensureParent().addExtraChild(flushUp, buildKeys[FAN_FACTOR]);
int size = FAN_FACTOR + 1;
assert size <= buildKeyPosition : buildKeyPosition + "," + nextBuildKeyPosition;
System.arraycopy(buildKeys, size, buildKeys, 0, buildKeyPosition - size);
buildKeyPosition -= size;
maxBuildKeyPosition = buildKeys.length;
if (buildChildPosition > 0)
{
System.arraycopy(buildChildren, size, buildChildren, 0, buildChildPosition - size);
buildChildPosition -= size;
}
}
// builds and returns a node from the buffered objects in the given range
private Object[] buildFromRange(int offset, int keyLength, boolean isLeaf, boolean isExtra)
{
// if keyLength is 0, we didn't copy anything from the original, which means we didn't
// modify any of the range owned by it, so can simply return it as is
if (keyLength == 0)
return copyFrom;
Object[] a;
if (isLeaf)
{
a = new Object[keyLength | 1];
System.arraycopy(buildKeys, offset, a, 0, keyLength);
}
else
{
a = new Object[2 + (keyLength * 2)];
System.arraycopy(buildKeys, offset, a, 0, keyLength);
System.arraycopy(buildChildren, offset, a, keyLength, keyLength + 1);
// calculate the indexOffsets of each key in this node, within the sub-tree rooted at this node
int[] indexOffsets = new int[keyLength + 1];
int size = BTree.size((Object[]) a[keyLength]);
for (int i = 0 ; i < keyLength ; i++)
{
indexOffsets[i] = size;
size += 1 + BTree.size((Object[]) a[keyLength + 1 + i]);
}
indexOffsets[keyLength] = size;
a[a.length - 1] = indexOffsets;
}
if (isExtra)
updateFunction.allocated(ObjectSizes.sizeOfArray(a));
else if (a.length != copyFrom.length)
updateFunction.allocated(ObjectSizes.sizeOfArray(a) -
(copyFrom.length == 0 ? 0 : ObjectSizes.sizeOfArray(copyFrom)));
return a;
}
// checks if there is an initialised parent, and if not creates/initialises one and returns it.
// different to ensureChild, as we initialise here instead of caller, as parents in general should
// already be initialised, and only aren't in the case where we are overflowing the original root node
private NodeBuilder ensureParent()
{
if (parent == null)
{
parent = new NodeBuilder();
parent.child = this;
}
if (parent.upperBound == null)
parent.reset(EMPTY_BRANCH, upperBound, updateFunction, comparator);
return parent;
}
// ensures a child level exists and returns it
NodeBuilder ensureChild()
{
if (child == null)
{
child = new NodeBuilder();
child.parent = this;
}
return child;
}
}

121
src/java/org/apache/cassandra/utils/btree/TreeBuilder.java
View File

@ -1,121 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cassandra.utils.btree;
import java.util.Comparator;
import io.netty.util.Recycler;
import static org.apache.cassandra.utils.btree.BTree.POSITIVE_INFINITY;
/**
* A class for constructing a new BTree, either from an existing one and some set of modifications
* or a new tree from a sorted collection of items.
* <p/>
* This is a fairly heavy-weight object, so a Recycled instance is created for making modifications to a tree
*/
final class TreeBuilder
{
private final static Recycler<TreeBuilder> builderRecycler = new Recycler<TreeBuilder>()
{
protected TreeBuilder newObject(Handle handle)
{
return new TreeBuilder(handle);
}
};
public static TreeBuilder newInstance()
{
return builderRecycler.get();
}
private final Recycler.Handle recycleHandle;
private final NodeBuilder rootBuilder = new NodeBuilder();
private TreeBuilder(Recycler.Handle handle)
{
this.recycleHandle = handle;
}
/**
* At the highest level, we adhere to the classic b-tree insertion algorithm:
*
* 1. Add to the appropriate leaf
* 2. Split the leaf if necessary, add the median to the parent
* 3. Split the parent if necessary, etc.
*
* There is one important difference: we don't actually modify the original tree, but copy each node that we
* modify. Note that every node on the path to the key being inserted or updated will be modified; this
* implies that at a minimum, the root node will be modified for every update, so every root is a "snapshot"
* of a tree that can be iterated or sliced without fear of concurrent modifications.
*
* The NodeBuilder class handles the details of buffering the copied contents of the original tree and
* adding in our changes. Since NodeBuilder maintains parent/child references, it also handles parent-splitting
* (easy enough, since any node affected by the split will already be copied into a NodeBuilder).
*
* One other difference from the simple algorithm is that we perform modifications in bulk;
* we assume @param source has been sorted, e.g. by BTree.update, so the update of each key resumes where
* the previous left off.
*/
public <C, K extends C, V extends C> Object[] update(Object[] btree, Comparator<C> comparator, Iterable<K> source, UpdateFunction<K, V> updateF)
{
assert updateF != null;
NodeBuilder current = rootBuilder;
current.reset(btree, POSITIVE_INFINITY, updateF, comparator);
for (K key : source)
{
while (true)
{
if (updateF.abortEarly())
{
rootBuilder.clear();
return null;
}
NodeBuilder next = current.update(key);
if (next == null)
break;
// we were in a subtree from a previous key that didn't contain this new key;
// retry against the correct subtree
current = next;
}
}
// finish copying any remaining keys from the original btree
while (true)
{
NodeBuilder next = current.finish();
if (next == null)
break;
current = next;
}
// updating with POSITIVE_INFINITY means that current should be back to the root
assert current.isRoot();
Object[] r = current.toNode();
current.clear();
recycleHandle.recycle(this);
return r;
}
}

32
src/java/org/apache/cassandra/utils/btree/UpdateFunction.java
View File

@ -34,15 +34,10 @@ public interface UpdateFunction<K, V> extends Function<K, V>
*/
V apply(V replacing, K update);
/**
* @return true if we should fail the update
*/
boolean abortEarly();
/**
* @param heapSize extra heap space allocated (over previous tree)
*/
void allocated(long heapSize);
void onAllocatedOnHeap(long heapSize);
public static final class Simple<V> implements UpdateFunction<V, V>
{
@ -52,15 +47,32 @@ public interface UpdateFunction<K, V> extends Function<K, V>
this.wrapped = wrapped;
}
public V apply(V v) { return v; }
public V apply(V replacing, V update) { return wrapped.apply(replacing, update); }
public boolean abortEarly() { return false; }
public void allocated(long heapSize) { }
@Override
public V apply(V v)
{
return v;
}
@Override
public V apply(V replacing, V update)
{
return wrapped.apply(replacing, update);
}
@Override
public void onAllocatedOnHeap(long heapSize)
{
}
public static <V> Simple<V> of(BiFunction<V, V, V> f)
{
return new Simple<>(f);
}
Simple<V> flip()
{
return of((a, b) -> wrapped.apply(b, a));
}
}
static final Simple<Object> noOp = Simple.of((a, b) -> a);

85
src/java/org/apache/cassandra/utils/caching/TinyThreadLocalPool.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils.caching;
import io.netty.util.concurrent.FastThreadLocal;
public class TinyThreadLocalPool<V> extends FastThreadLocal<TinyThreadLocalPool.TinyPool<V>>
{
protected TinyPool<V> initialValue()
{
return new TinyPool<>();
}
// a super-simple pool containing at most two items; useful because we primarily use btrees in a two-tier hierarchy
// so a single thread local item would be insufficient, but an arbitrary length queue too much
public static class TinyPool<V>
{
final Thread thread;
Object val1, val2, val3;
public TinyPool()
{
this.thread = Thread.currentThread();
}
public void offer(V value)
{
if (Thread.currentThread() == thread)
offerSafe(value);
}
private void offerSafe(V value)
{
if (val1 == null) val1 = value;
else if (val2 == null) val2 = value;
else if (val3 == null) val3 = value;
}
public V poll()
{
Object result;
if (val1 != null)
{
result = val1;
val1 = null;
}
else if (val2 != null)
{
result = val2;
val2 = null;
}
else if (val3 != null)
{
result = val3;
val3 = null;
}
else result = null;
return (V) result;
}
}
public void offer(V value)
{
get().offer(value);
}
public V poll()
{
return get().poll();
}
}

593
test/burn/org/apache/cassandra/utils/LongBTreeTest.java
View File

@ -21,18 +21,19 @@ package org.apache.cassandra.utils;
import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.security.SecureRandom;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.IntStream;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
@ -52,44 +53,36 @@ import static com.google.common.collect.Iterables.transform;
import static java.util.Comparator.naturalOrder;
import static java.util.Comparator.reverseOrder;
import static org.apache.cassandra.utils.btree.BTree.iterable;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
// TODO: randomise all parameters for all tests, with a target wall time for each iteration
// should dedicate as much wall time to any depth of tree as any other, except depth 1 (which should be less frequent)
// TODO: verify update with no changes returns original
// TODO: verify updateF.allocated()
// TODO: verify reverseInSitu
// TODO: introduce patterns to verification, esp. to transform and update
public class LongBTreeTest
{
private static final boolean DEBUG = false;
private static int perThreadTrees = 100;
private static int perThreadTrees = 10000;
private static int minTreeSize = 4;
private static int maxTreeSize = 10000;
private static float generateTreeByUpdateChance = 0.8f;
private static float generateTreeByCopyChance = 0.1f;
private static float generateTreeByBuilderChance = 0.1f;
private static float generateTreeTotalChance = generateTreeByUpdateChance + generateTreeByCopyChance + generateTreeByBuilderChance;
private static int maxTreeSize = 10000; // TODO randomise this for each test
private static int threads = DEBUG ? 1 : Runtime.getRuntime().availableProcessors() * 8;
private static final MetricRegistry metrics = new MetricRegistry();
private static final Timer BTREE_TIMER = metrics.timer(MetricRegistry.name(BTree.class, "BTREE"));
private static final Timer TREE_TIMER = metrics.timer(MetricRegistry.name(BTree.class, "TREE"));
private static final ExecutorService MODIFY = Executors.newFixedThreadPool(threads, new NamedThreadFactory("MODIFY"));
private static final ExecutorService COMPARE = DEBUG ? MODIFY : Executors.newFixedThreadPool(threads, new NamedThreadFactory("COMPARE"));
private static final RandomAbort<Integer> SPORADIC_ABORT = new RandomAbort<>(new Random(), 0.0001f);
static
{
System.setProperty("cassandra.btree.fanfactor", "4");
}
/************************** TEST ACCESS ********************************************/
@Test
public void testSearchIterator() throws InterruptedException
{
final int perTreeSelections = 100;
testRandomSelection(perThreadTrees, perTreeSelections, testSearchIteratorFactory());
}
private BTreeTestFactory testSearchIteratorFactory()
{
return (test) -> {
final int perTreeSelections = 10; // TODO randomise this for each test
testRandomSelection(randomSeed(), perThreadTrees, perTreeSelections,
(test) -> {
IndexedSearchIterator<Integer, Integer> iter1 = test.testAsSet.iterator();
IndexedSearchIterator<Integer, Integer> iter2 = test.testAsList.iterator();
return (key) ->
@ -113,63 +106,56 @@ public class LongBTreeTest
// check that by advancing the same key again we get null, but only do it on one of the two iterators
// to ensure they both advance differently
if (ThreadLocalRandom.current().nextBoolean())
if (test.random.nextBoolean())
Assert.assertNull(iter1.next(key));
else
Assert.assertNull(iter2.next(key));
};
};
});
}
@Test
public void testInequalityLookups() throws InterruptedException
{
final int perTreeSelections = 2;
testRandomSelectionOfSet(perThreadTrees, perTreeSelections, testInequalityLookupsFactory());
}
private BTreeSetTestFactory testInequalityLookupsFactory()
{
return (test, canonical) -> {
if (!canonical.isEmpty() || !test.isEmpty())
{
Assert.assertEquals(canonical.isEmpty(), test.isEmpty());
Assert.assertEquals(canonical.first(), test.first());
Assert.assertEquals(canonical.last(), test.last());
}
return (key) ->
{
Assert.assertEquals(test.ceiling(key), canonical.ceiling(key));
Assert.assertEquals(test.higher(key), canonical.higher(key));
Assert.assertEquals(test.floor(key), canonical.floor(key));
Assert.assertEquals(test.lower(key), canonical.lower(key));
};
};
testRandomSelectionOfSet(randomSeed(), perThreadTrees, perTreeSelections,
(test, canonical) -> {
if (!canonical.isEmpty() || !test.isEmpty())
{
Assert.assertEquals(canonical.isEmpty(), test.isEmpty());
Assert.assertEquals(canonical.first(), test.first());
Assert.assertEquals(canonical.last(), test.last());
}
return (key) ->
{
Assert.assertEquals(test.ceiling(key), canonical.ceiling(key));
Assert.assertEquals(test.higher(key), canonical.higher(key));
Assert.assertEquals(test.floor(key), canonical.floor(key));
Assert.assertEquals(test.lower(key), canonical.lower(key));
};
});
}
@Test
public void testListIndexes() throws InterruptedException
{
testRandomSelectionOfList(perThreadTrees, 4, testListIndexesFactory());
}
private BTreeListTestFactory testListIndexesFactory()
{
return (test, canonical, cmp) ->
(key) ->
{
int javaIndex = Collections.binarySearch(canonical, key, cmp);
int btreeIndex = test.indexOf(key);
Assert.assertEquals(javaIndex, btreeIndex);
if (javaIndex >= 0)
Assert.assertEquals(canonical.get(javaIndex), test.get(btreeIndex));
};
testRandomSelectionOfList(randomSeed(), perThreadTrees, 4,
(test, canonical, cmp) ->
(key) ->
{
int javaIndex = Collections.binarySearch(canonical, key, cmp);
int btreeIndex = test.indexOf(key);
Assert.assertEquals(javaIndex, btreeIndex);
if (javaIndex >= 0)
Assert.assertEquals(canonical.get(javaIndex), test.get(btreeIndex));
}
);
}
@Test
public void testToArray() throws InterruptedException
{
testRandomSelection(perThreadTrees, 4,
testRandomSelection(randomSeed(), perThreadTrees, 4,
(selection) ->
{
Integer[] array = new Integer[selection.canonicalList.size() + 1];
@ -196,42 +182,81 @@ public class LongBTreeTest
}
@Test
public void testTransformAndFilter() throws InterruptedException
public void testTransformAndFilterNone() throws InterruptedException
{
testRandomSelection(perThreadTrees, 4, false, false, false,
testRandomSelection(randomSeed(), perThreadTrees, 4, false, false, false,
(selection) ->
{
Map<Integer, Integer> update = new LinkedHashMap<>();
for (Integer i : selection.testKeys)
update.put(i, new Integer(i));
CountingFunction function;
CountingFunction function = new CountingFunction((x) -> x);
Object[] original = selection.testAsSet.tree();
Object[] transformed;
Object[] transformed = BTree.transformAndFilter(original, function);
// test replacing none, leaving all present
function = new CountingFunction((x) -> x);
transformed = BTree.transformAndFilter(original, function);
Assert.assertEquals(BTree.size(original), function.count);
assertTrue(BTree.<Integer>isWellFormed(transformed, naturalOrder()));
Assert.assertSame(original, transformed);
});
}
// test replacing some, leaving all present
function = new CountingFunction((x) -> update.containsKey(x) ? update.get(x) : x);
transformed = BTree.transformAndFilter(original, function);
Assert.assertEquals(BTree.size(original), function.count);
assertSame(transform(selection.canonicalList, function.wrapped), iterable(transformed));
@Test
public void testTransformAndFilterReplace() throws InterruptedException
{
testRandomSelection(randomSeed(), perThreadTrees, 4, false, false, false,
(selection) ->
{
Map<Integer, Integer> update = new LinkedHashMap<>();
for (Integer i : selection.testKeys)
update.put(i, new Integer(i));
// test replacing some, removing some
function = new CountingFunction(update::get);
transformed = BTree.transformAndFilter(original, function);
Assert.assertEquals(BTree.size(original), function.count);
assertSame(filter(transform(selection.canonicalList, function.wrapped), notNull()), iterable(transformed));
CountingFunction function = new CountingFunction((x) -> update.getOrDefault(x, x));
Object[] original = selection.testAsSet.tree();
Object[] transformed = BTree.transformAndFilter(original, function);
// test replacing none, removing some
function = new CountingFunction((x) -> update.containsKey(x) ? null : x);
transformed = BTree.transformAndFilter(selection.testAsList.tree(), function);
Assert.assertEquals(BTree.size(original), function.count);
assertSame(filter(transform(selection.canonicalList, function.wrapped), notNull()), iterable(transformed));
assertTrue(BTree.<Integer>isWellFormed(transformed, naturalOrder()));
assertSame(transform(selection.canonicalList, function.wrapped::apply), iterable(transformed));
});
}
@Test
public void testTransformAndFilterReplaceAndRemove() throws InterruptedException
{
testRandomSelection(randomSeed(), perThreadTrees, 4, false, false, false,
(selection) ->
{
Map<Integer, Integer> update = new LinkedHashMap<>();
for (Integer i : selection.testKeys)
update.put(i, new Integer(i));
CountingFunction function = new CountingFunction(update::get);
Object[] original = selection.testAsSet.tree();
Object[] transformed = BTree.transformAndFilter(original, function);
Assert.assertEquals(BTree.size(original), function.count);
assertTrue(BTree.<Integer>isWellFormed(transformed, naturalOrder()));
assertSame(filter(transform(selection.canonicalList, function.wrapped::apply), notNull()), iterable(transformed));
});
}
@Test
public void testTransformAndFilterRemove() throws InterruptedException
{
testRandomSelection(randomSeed(), perThreadTrees, 4, false, false, false,
(selection) ->
{
Map<Integer, Integer> update = new LinkedHashMap<>();
for (Integer i : selection.testKeys)
update.put(i, new Integer(i));
CountingFunction function = new CountingFunction((x) -> update.containsKey(x) ? null : x);
Object[] original = selection.testAsSet.tree();
Object[] transformed = BTree.transformAndFilter(selection.testAsList.tree(), function);
Assert.assertEquals(BTree.size(original), function.count);
assertTrue(BTree.<Integer>isWellFormed(transformed, naturalOrder()));
// Assert.assertEquals(BTree.size(original) - update.size(), BTree.size(transformed));
assertSame(filter(transform(selection.canonicalList, function.wrapped::apply), notNull()), iterable(transformed));
});
}
@ -247,15 +272,32 @@ public class LongBTreeTest
Assert.assertEquals(i1.hasNext(), i2.hasNext());
}
private void testRandomSelectionOfList(int perThreadTrees, int perTreeSelections, BTreeListTestFactory testRun) throws InterruptedException
private static Pair<Integer, Integer> firstDiff(Iterable<Integer> i1, Iterable<Integer> i2)
{
testRandomSelection(perThreadTrees, perTreeSelections,
return firstDiff(i1.iterator(), i2.iterator());
}
private static Pair<Integer, Integer> firstDiff(Iterator<Integer> i1, Iterator<Integer> i2)
{
while (i1.hasNext() && i2.hasNext())
{
Integer v1 = i1.next();
Integer v2 = i2.next();
if (v1 != v2)
return Pair.create(v1, v2);
}
return i1.hasNext() ? Pair.create(i1.next(), null) : i2.hasNext() ? Pair.create(null, i2.next()) : null;
}
private void testRandomSelectionOfList(long testSeed, int perThreadTrees, int perTreeSelections, BTreeListTestFactory testRun) throws InterruptedException
{
testRandomSelection(testSeed, perThreadTrees, perTreeSelections,
(BTreeTestFactory) (selection) -> testRun.get(selection.testAsList, selection.canonicalList, selection.comparator));
}
private void testRandomSelectionOfSet(int perThreadTrees, int perTreeSelections, BTreeSetTestFactory testRun) throws InterruptedException
private void testRandomSelectionOfSet(long testSeed, int perThreadTrees, int perTreeSelections, BTreeSetTestFactory testRun) throws InterruptedException
{
testRandomSelection(perThreadTrees, perTreeSelections,
testRandomSelection(testSeed, perThreadTrees, perTreeSelections,
(BTreeTestFactory) (selection) -> testRun.get(selection.testAsSet, selection.canonicalSet));
}
@ -279,32 +321,23 @@ public class LongBTreeTest
void testOne(Integer value);
}
private void run(BTreeTestFactory testRun, RandomSelection selection)
private void testRandomSelection(long seed, int perThreadTrees, int perTreeSelections, BTreeTestFactory testRun) throws InterruptedException
{
TestEachKey testEachKey = testRun.get(selection);
for (Integer key : selection.testKeys)
testEachKey.testOne(key);
testRandomSelection(seed, perThreadTrees, perTreeSelections, (selection) -> {
TestEachKey testEachKey = testRun.get(selection);
for (Integer key : selection.testKeys)
testEachKey.testOne(key);
});
}
private void run(BTreeSetTestFactory testRun, RandomSelection selection)
private void testRandomSelection(long seed, int perThreadTrees, int perTreeSelections, Consumer<RandomSelection> testRun) throws InterruptedException
{
TestEachKey testEachKey = testRun.get(selection.testAsSet, selection.canonicalSet);
for (Integer key : selection.testKeys)
testEachKey.testOne(key);
testRandomSelection(seed, perThreadTrees, perTreeSelections, true, true, true, testRun);
}
private void testRandomSelection(int perThreadTrees, int perTreeSelections, BTreeTestFactory testRun) throws InterruptedException
{
testRandomSelection(perThreadTrees, perTreeSelections, (RandomSelection selection) -> run(testRun, selection));
}
private void testRandomSelection(int perThreadTrees, int perTreeSelections, Consumer<RandomSelection> testRun) throws InterruptedException
{
testRandomSelection(perThreadTrees, perTreeSelections, true, true, true, testRun);
}
private void testRandomSelection(int perThreadTrees, int perTreeSelections, boolean narrow, boolean mixInNotPresentItems, boolean permitReversal, Consumer<RandomSelection> testRun) throws InterruptedException
private void testRandomSelection(long seed, int perThreadTrees, int perTreeSelections, boolean narrow, boolean mixInNotPresentItems, boolean permitReversal, Consumer<RandomSelection> testRun) throws InterruptedException
{
final Random outerSeedGenerator = new Random(seed);
int threads = Runtime.getRuntime().availableProcessors();
final CountDownLatch latch = new CountDownLatch(threads);
final AtomicLong errors = new AtomicLong();
@ -312,19 +345,18 @@ public class LongBTreeTest
final long totalCount = threads * perThreadTrees * perTreeSelections;
for (int t = 0 ; t < threads ; t++)
{
Runnable runnable = () ->
{
Runnable runnable = () -> {
final Random seedGenerator = new Random(outerSeedGenerator.nextLong());
try
{
for (int i = 0 ; i < perThreadTrees ; i++)
{
// not easy to usefully log seed, as run tests in parallel; need to really pass through to exceptions
long seed = ThreadLocalRandom.current().nextLong();
Random random = new Random(seed);
RandomTree tree = randomTree(minTreeSize, maxTreeSize, random);
long dataSeed = seedGenerator.nextLong();
RandomTree tree = randomTree(dataSeed, minTreeSize, maxTreeSize);
for (int j = 0 ; j < perTreeSelections ; j++)
{
testRun.accept(tree.select(narrow, mixInNotPresentItems, permitReversal));
long selectionSeed = seedGenerator.nextLong();
testRun.accept(tree.select(selectionSeed, narrow, mixInNotPresentItems, permitReversal));
count.incrementAndGet();
}
}
@ -351,6 +383,9 @@ public class LongBTreeTest
private static class RandomSelection
{
final long dataSeed;
final long selectionSeed;
final Random random;
final List<Integer> testKeys;
final NavigableSet<Integer> canonicalSet;
final List<Integer> canonicalList;
@ -358,9 +393,13 @@ public class LongBTreeTest
final BTreeSet<Integer> testAsList;
final Comparator<Integer> comparator;
private RandomSelection(List<Integer> testKeys, NavigableSet<Integer> canonicalSet, BTreeSet<Integer> testAsSet,
private RandomSelection(long dataSeed, long selectionSeed, Random random,
List<Integer> testKeys, NavigableSet<Integer> canonicalSet, BTreeSet<Integer> testAsSet,
List<Integer> canonicalList, BTreeSet<Integer> testAsList, Comparator<Integer> comparator)
{
this.dataSeed = dataSeed;
this.selectionSeed = selectionSeed;
this.random = random;
this.testKeys = testKeys;
this.canonicalList = canonicalList;
this.canonicalSet = canonicalSet;
@ -372,19 +411,22 @@ public class LongBTreeTest
private static class RandomTree
{
final Random random;
final long dataSeed;
final NavigableSet<Integer> canonical;
final BTreeSet<Integer> test;
private RandomTree(NavigableSet<Integer> canonical, BTreeSet<Integer> test, Random random)
private RandomTree(long dataSeed, NavigableSet<Integer> canonical, BTreeSet<Integer> test)
{
this.dataSeed = dataSeed;
this.canonical = canonical;
this.test = test;
this.random = random;
}
RandomSelection select(boolean narrow, boolean mixInNotPresentItems, boolean permitReversal)
// TODO: revisit logic, document and ensure producing enough distinct patterns
RandomSelection select(long selectionSeed, boolean narrow, boolean mixInNotPresentItems, boolean permitReversal)
{
Random random = new Random(selectionSeed);
NavigableSet<Integer> canonicalSet = this.canonical;
BTreeSet<Integer> testAsSet = this.test;
List<Integer> canonicalList = new ArrayList<>(canonicalSet);
@ -393,8 +435,9 @@ public class LongBTreeTest
Assert.assertEquals(canonicalSet.size(), testAsSet.size());
Assert.assertEquals(canonicalList.size(), testAsList.size());
// TODO: select random patterns of data as well as pure random data (i.e. random sequences, random fixed offsets, random mixes of the above)
// sometimes select keys first, so we cover full range
List<Integer> allKeys = randomKeys(canonical, mixInNotPresentItems, random);
List<Integer> allKeys = randomKeys(random, canonical, mixInNotPresentItems);
List<Integer> keys = allKeys;
int narrowCount = random.nextInt(3);
@ -417,7 +460,7 @@ public class LongBTreeTest
if (useLb)
{
lbKeyIndex = random.nextInt(indexRange - 1);
lbKeyIndex = nextInt(random, 0, indexRange - 1);
Integer candidate = keys.get(lbKeyIndex);
if (useLb = (candidate > lbKey && candidate <= ubKey))
{
@ -430,8 +473,7 @@ public class LongBTreeTest
}
if (useUb)
{
int lb = Math.max(lbKeyIndex, keys.size() - indexRange);
ubKeyIndex = random.nextInt(keys.size() - (1 + lb)) + lb;
ubKeyIndex = nextInt(random, Math.max(lbKeyIndex, keys.size() - indexRange), keys.size() - 1);
Integer candidate = keys.get(ubKeyIndex);
if (useUb = (candidate < ubKey && candidate >= lbKey))
{
@ -491,73 +533,69 @@ public class LongBTreeTest
Assert.assertEquals(canonicalSet.last(), testAsList.get(testAsList.size() - 1));
}
return new RandomSelection(keys, canonicalSet, testAsSet, canonicalList, testAsList, comparator);
assertSame(canonicalList, testAsList);
return new RandomSelection(dataSeed, selectionSeed, random, keys, canonicalSet, testAsSet, canonicalList, testAsList, comparator);
}
}
private static RandomTree randomTree(int minSize, int maxSize, Random random)
private static int nextInt(Random random, int lb, int ub)
{
return lb >= ub ? lb : lb + random.nextInt(ub - lb);
}
private static RandomTree randomTree(long seed, int minSize, int maxSize)
{
Random random = new Random(seed);
// perform most of our tree constructions via update, as this is more efficient; since every run uses this
// we test builder disproportionately more often than if it had its own test anyway
int maxIntegerValue = random.nextInt(Integer.MAX_VALUE - 1) + 1;
float f = random.nextFloat() / generateTreeTotalChance;
f -= generateTreeByUpdateChance;
if (f < 0)
return randomTreeByUpdate(minSize, maxSize, maxIntegerValue, random);
f -= generateTreeByCopyChance;
if (f < 0)
return randomTreeByCopy(minSize, maxSize, maxIntegerValue, random);
return randomTreeByBuilder(minSize, maxSize, maxIntegerValue, random);
return random.nextFloat() < 0.95 ? randomTreeByUpdate(seed, random, minSize, maxSize)
: randomTreeByBuilder(seed, random, minSize, maxSize);
}
private static RandomTree randomTreeByCopy(int minSize, int maxSize, int maxIntegerValue, Random random)
private static RandomTree randomTreeByUpdate(long seed, Random random, int minSize, int maxSize)
{
assert minSize > 3;
TreeSet<Integer> canonical = new TreeSet<>();
int targetSize = random.nextInt(maxSize - minSize) + minSize;
int curSize = 0;
while (curSize < targetSize)
{
Integer next = random.nextInt(maxIntegerValue);
if (canonical.add(next))
++curSize;
}
return new RandomTree(canonical, BTreeSet.<Integer>wrap(BTree.build(canonical, UpdateFunction.noOp()), naturalOrder()), random);
}
private static RandomTree randomTreeByUpdate(int minSize, int maxSize, int maxIntegerValue, Random random)
{
assert minSize > 3;
TreeSet<Integer> canonical = new TreeSet<>();
int targetSize = random.nextInt(maxSize - minSize) + minSize;
int maxModificationSize = random.nextInt(targetSize - 2) + 2;
int targetSize = nextInt(random, minSize, maxSize);
int maxModificationSize = nextInt(random, 2, targetSize);
Object[] accmumulate = BTree.empty();
int curSize = 0;
while (curSize < targetSize)
{
int nextSize = maxModificationSize == 1 ? 1 : random.nextInt(maxModificationSize - 1) + 1;
int nextSize = maxModificationSize == 1 ? 1 : nextInt(random, 1, maxModificationSize);
TreeSet<Integer> build = new TreeSet<>();
boolean keepOriginal = random.nextBoolean();
// we don't use no-op, to ensure we know which value will actually result (as no-op doesn't guarantee which makes it through)
UpdateFunction<Integer, Integer> updateF = keepOriginal ? UpdateFunction.Simple.of((a, b) -> a) : InverseNoOp.instance;
for (int i = 0 ; i < nextSize ; i++)
{
Integer next = random.nextInt(maxIntegerValue);
build.add(next);
canonical.add(next);
Integer next = random.nextInt();
if (build.add(next))
{
if (!canonical.add(next) && !keepOriginal)
{
canonical.remove(next);
canonical.add(next);
}
}
}
accmumulate = BTree.update(accmumulate, naturalOrder(), build, UpdateFunction.<Integer>noOp());
Object[] tmp = BTree.update(accmumulate, BTree.build(build), naturalOrder(), updateF);
assertSame(canonical, BTreeSet.<Integer>wrap(tmp, naturalOrder()));
accmumulate = tmp;
curSize += nextSize;
maxModificationSize = Math.min(maxModificationSize, targetSize - curSize);
}
return new RandomTree(canonical, BTreeSet.<Integer>wrap(accmumulate, naturalOrder()), random);
assertSame(canonical, BTreeSet.<Integer>wrap(accmumulate, naturalOrder()));
return new RandomTree(seed, canonical, BTreeSet.<Integer>wrap(accmumulate, naturalOrder()));
}
private static RandomTree randomTreeByBuilder(int minSize, int maxSize, int maxIntegerValue, Random random)
private static RandomTree randomTreeByBuilder(long seed, Random random, int minSize, int maxSize)
{
assert minSize > 3;
BTree.Builder<Integer> builder = BTree.builder(naturalOrder());
int targetSize = random.nextInt(maxSize - minSize) + minSize;
int targetSize = nextInt(random, minSize, maxSize);
int maxModificationSize = (int) Math.sqrt(targetSize);
TreeSet<Integer> canonical = new TreeSet<>();
@ -567,7 +605,7 @@ public class LongBTreeTest
List<Integer> shuffled = new ArrayList<>();
while (curSize < targetSize)
{
int nextSize = maxModificationSize <= 1 ? 1 : random.nextInt(maxModificationSize - 1) + 1;
int nextSize = nextInt(random, 1, maxModificationSize);
// leave a random selection of previous values
(random.nextBoolean() ? ordered.headSet(random.nextInt()) : ordered.tailSet(random.nextInt())).clear();
@ -575,7 +613,7 @@ public class LongBTreeTest
for (int i = 0 ; i < nextSize ; i++)
{
Integer next = random.nextInt(maxIntegerValue);
Integer next = random.nextInt();
ordered.add(next);
shuffled.add(next);
canonical.add(next);
@ -606,12 +644,13 @@ public class LongBTreeTest
BTreeSet<Integer> btree = BTreeSet.<Integer>wrap(builder.build(), naturalOrder());
Assert.assertEquals(canonical.size(), btree.size());
return new RandomTree(canonical, btree, random);
assertSame(canonical, btree);
return new RandomTree(seed, canonical, btree);
}
// select a random subset of the keys, with an optional random population of keys inbetween those that are present
// return a value with the search position
private static List<Integer> randomKeys(Iterable<Integer> canonical, boolean mixInNotPresentItems, Random random)
private static List<Integer> randomKeys(Random random, Iterable<Integer> canonical, boolean mixInNotPresentItems)
{
boolean useFake = mixInNotPresentItems && random.nextBoolean();
final float fakeRatio = random.nextFloat();
@ -648,94 +687,144 @@ public class LongBTreeTest
return Lists.newArrayList(filter(results, (x) -> random.nextFloat() < useChance));
}
/************************** TEST MUTATION ********************************************/
/************************** TEST BUILD ********************************************/
@Test
public void testBuildNewTree()
public void testBuild()
{
int max = 10000;
final List<Integer> list = new ArrayList<>(max);
final NavigableSet<Integer> set = new TreeSet<>();
BTreeSetTestFactory test = testInequalityLookupsFactory();
for (int i = 0 ; i < max ; ++i)
Integer[] vs = IntStream.rangeClosed(0, 100000).boxed().toArray(Integer[]::new);
for (UpdateFunction<Integer, Integer> updateF : LongBTreeTest.updateFunctions())
{
list.add(i);
set.add(i);
Object[] tree = BTree.build(list, UpdateFunction.noOp());
Assert.assertTrue(BTree.isWellFormed(tree, Comparator.naturalOrder()));
BTreeSet<Integer> btree = new BTreeSet<>(tree, Comparator.naturalOrder());
RandomSelection selection = new RandomSelection(list, set, btree, list, btree, Comparator.naturalOrder());
run(test, selection);
try (BulkIterator<Integer> emptyIter = BulkIterator.of(vs))
{
Object[] empty = BTree.build(emptyIter, 0, updateF);
assertTrue("" + 0, BTree.isEmpty(empty)); // empty is tested by object identity, so verify we test correctly
}
for (int i = 0 ; i < vs.length ; ++i)
{
try (BulkIterator<Integer> iter = BulkIterator.of(vs))
{
Object[] btree = BTree.build(iter, i + 1, updateF);
assertTrue("" + i, BTree.<Integer>isWellFormed(btree, naturalOrder()));
}
}
}
}
@Test
public void testFastBuilder()
{
Integer[] vs = IntStream.rangeClosed(0, 100000).boxed().toArray(Integer[]::new);
try (BTree.FastBuilder<Integer> builder = BTree.fastBuilder())
{
Object[] empty = builder.build();
assertTrue("" + 0, BTree.isEmpty(empty)); // empty is tested by object identity, so verify we test correctly
}
for (int i = 0 ; i < vs.length ; ++i)
{
try (BTree.FastBuilder<Integer> builder = BTree.fastBuilder())
{
for (int j = 0 ; j <= i ; ++j)
builder.add(vs[j]);
Object[] btree = builder.build();
assertEquals(i + 1, BTree.size(btree));
assertTrue(""+i, BTree.<Integer>isWellFormed(btree, naturalOrder()));
}
}
}
@Test
public void testBuildByUpdate()
{
Integer[] vs = IntStream.rangeClosed(0, 100000).boxed().toArray(Integer[]::new);
Object[] base = BTree.singleton(vs[0]);
for (int i = 0 ; i < vs.length ; ++i)
{
try (BulkIterator<Integer> iter = BulkIterator.of(vs))
{
Object[] insert = BTree.build(iter, i + 1, UpdateFunction.noOp());
Object[] btree = BTree.<Integer, Integer, Integer>update(base, insert, naturalOrder(), InverseNoOp.instance);
assertTrue("" + i, BTree.<Integer>isWellFormed(btree, naturalOrder()));
}
}
}
/************************** TEST MUTATION ********************************************/
@Test
public void testOversizedMiddleInsert()
{
TreeSet<Integer> canon = new TreeSet<>();
for (int i = 0 ; i < 10000000 ; i++)
canon.add(i);
Object[] btree = BTree.build(Arrays.asList(Integer.MIN_VALUE, Integer.MAX_VALUE), UpdateFunction.noOp());
btree = BTree.update(btree, naturalOrder(), canon, UpdateFunction.<Integer>noOp());
canon.add(Integer.MIN_VALUE);
canon.add(Integer.MAX_VALUE);
assertTrue(BTree.isWellFormed(btree, naturalOrder()));
testEqual("Oversize", BTree.iterator(btree), canon.iterator());
for (UpdateFunction<Integer, Integer> updateF : LongBTreeTest.updateFunctions())
{
TreeSet<Integer> canon = new TreeSet<>();
for (int i = 0 ; i < 10000000 ; i++)
canon.add(i);
Object[] btree = BTree.build(Arrays.asList(Integer.MIN_VALUE, Integer.MAX_VALUE), updateF);
btree = BTree.update(btree, BTree.build(canon), naturalOrder(), updateF);
canon.add(Integer.MIN_VALUE);
canon.add(Integer.MAX_VALUE);
assertTrue(BTree.<Integer>isWellFormed(btree, naturalOrder()));
testEqual("Oversize", BTree.iterator(btree), canon.iterator());
}
}
@Test
public void testIndividualInsertsSmallOverlappingRange() throws ExecutionException, InterruptedException
{
testInsertions(50, 1, 1, true);
testInsertions(randomSeed(), 50, 1, 1, true);
}
@Test
public void testBatchesSmallOverlappingRange() throws ExecutionException, InterruptedException
{
testInsertions(50, 1, 5, true);
testInsertions(randomSeed(), 50, 1, 5, true);
}
@Test
public void testIndividualInsertsMediumSparseRange() throws ExecutionException, InterruptedException
{
testInsertions(perThreadTrees / 10, 500, 10, 1, true);
testInsertions(randomSeed(), perThreadTrees / 10, 500, 10, 1, true);
}
@Test
public void testBatchesMediumSparseRange() throws ExecutionException, InterruptedException
{
testInsertions(500, 10, 10, true);
testInsertions(randomSeed(), 500, 10, 10, true);
}
@Test
public void testLargeBatchesLargeRange() throws ExecutionException, InterruptedException
{
testInsertions(perThreadTrees / 10, Math.max(maxTreeSize, 5000), 3, 100, true);
testInsertions(randomSeed(), perThreadTrees / 10, Math.max(maxTreeSize, 5000), 3, 100, true);
}
@Test
public void testRandomRangeAndBatches() throws ExecutionException, InterruptedException
{
ThreadLocalRandom random = ThreadLocalRandom.current();
int treeSize = random.nextInt(maxTreeSize / 10, maxTreeSize * 10);
Random seedGenerator = new Random(randomSeed());
for (int i = 0 ; i < perThreadTrees / 10 ; i++)
testInsertions(threads * 10, treeSize, random.nextInt(1, 100) / 10f, treeSize / 100, true);
{
int treeSize = nextInt(seedGenerator, maxTreeSize / 10, maxTreeSize * 10);
testInsertions(seedGenerator.nextLong(), threads * 10, treeSize, nextInt(seedGenerator, 1, 100) / 10f, treeSize / 100, true);
}
}
@Test
public void testSlicingSmallRandomTrees() throws ExecutionException, InterruptedException
{
testInsertions(50, 10, 10, false);
testInsertions(randomSeed(), 50, 10, 10, false);
}
private static void testInsertions(int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException
private static void testInsertions(long seed, int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException
{
int tests = perThreadTrees * threads;
testInsertions(tests, perTestCount, testKeyRatio, modificationBatchSize, quickEquality);
testInsertions(seed, tests, perTestCount, testKeyRatio, modificationBatchSize, quickEquality);
}
private static void testInsertions(int tests, int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException
private static void testInsertions(long seed, int tests, int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException
{
Random random = new Random(seed);
int batchesPerTest = perTestCount / modificationBatchSize;
int testKeyRange = (int) (perTestCount * testKeyRatio);
long totalCount = (long) perTestCount * tests;
@ -749,8 +838,8 @@ public class LongBTreeTest
final List<ListenableFutureTask<List<ListenableFuture<?>>>> outer = new ArrayList<>();
for (int i = 0 ; i < chunkSize ; i++)
{
int maxRunLength = modificationBatchSize == 1 ? 1 : ThreadLocalRandom.current().nextInt(1, modificationBatchSize);
outer.add(doOneTestInsertions(testKeyRange, maxRunLength, modificationBatchSize, batchesPerTest, quickEquality));
int maxRunLength = modificationBatchSize == 1 ? 1 : nextInt(random, 1, modificationBatchSize);
outer.add(doOneTestInsertions(random.nextLong(), testKeyRange, maxRunLength, modificationBatchSize, batchesPerTest, quickEquality));
}
final List<ListenableFuture<?>> inner = new ArrayList<>();
@ -778,27 +867,32 @@ public class LongBTreeTest
log("Done");
}
private static ListenableFutureTask<List<ListenableFuture<?>>> doOneTestInsertions(final int upperBound, final int maxRunLength, final int averageModsPerIteration, final int iterations, final boolean quickEquality)
@Test
public void debug()
{
ListenableFutureTask<List<ListenableFuture<?>>> f = ListenableFutureTask.create(new Callable<List<ListenableFuture<?>>>()
{
@Override
public List<ListenableFuture<?>> call()
randomTree(384037044131282656L, 4, 10000);
}
private static ListenableFutureTask<List<ListenableFuture<?>>> doOneTestInsertions(long seed, final int upperBound, final int maxRunLength, final int averageModsPerIteration, final int iterations, final boolean quickEquality)
{
String id = String.format("<%dL,%d,%d,%d,%d,%b>", seed, upperBound, maxRunLength, averageModsPerIteration, iterations, quickEquality);
Random random = new Random(seed);
ListenableFutureTask<List<ListenableFuture<?>>> f = ListenableFutureTask.create(() -> {
try
{
final List<ListenableFuture<?>> r = new ArrayList<>();
NavigableMap<Integer, Integer> canon = new TreeMap<>();
Object[] btree = BTree.empty();
final TreeMap<Integer, Integer> buffer = new TreeMap<>();
ThreadLocalRandom rnd = ThreadLocalRandom.current();
for (int i = 0 ; i < iterations ; i++)
{
buffer.clear();
int mods = rnd.nextInt(1, averageModsPerIteration * 2);
int mods = nextInt(random, 1, averageModsPerIteration * 2);
while (mods > 0)
{
int v = rnd.nextInt(upperBound);
int v = random.nextInt(upperBound);
int rc = Math.max(0, Math.min(mods, maxRunLength) - 1);
int c = 1 + (rc <= 0 ? 0 : rnd.nextInt(rc));
int c = 1 + (rc <= 0 ? 0 : random.nextInt(rc));
for (int j = 0 ; j < c ; j++)
{
buffer.put(v, v);
@ -811,24 +905,29 @@ public class LongBTreeTest
canon.putAll(buffer);
ctxt.stop();
ctxt = BTREE_TIMER.time();
Object[] next = null;
while (next == null)
next = BTree.update(btree, naturalOrder(), buffer.keySet(), SPORADIC_ABORT);
btree = next;
Object[] add = BTree.build(buffer.keySet());
Object[] newTree = BTree.update(btree, add, naturalOrder(), updateFunction(random));
ctxt.stop();
if (!BTree.isWellFormed(btree, naturalOrder()))
if (!BTree.<Integer>isWellFormed(newTree, naturalOrder()))
{
log("ERROR: Not well formed");
log(id + " ERROR: Not well formed");
throw new AssertionError("Not well formed!");
}
btree = newTree;
if (quickEquality)
testEqual("", BTree.iterator(btree), canon.keySet().iterator());
testEqual(id, BTree.iterator(btree), canon.keySet().iterator());
else
r.addAll(testAllSlices("RND", btree, new TreeSet<>(canon.keySet())));
r.addAll(testAllSlices(id, btree, new TreeSet<>(canon.keySet())));
}
return r;
}
catch (Throwable t)
{
t.printStackTrace();
log("Failed %s: %s", id, t.getMessage());
throw t;
}
});
if (DEBUG)
f.run();
@ -840,19 +939,22 @@ public class LongBTreeTest
@Test
public void testSlicingAllSmallTrees() throws ExecutionException, InterruptedException
{
Object[] cur = BTree.empty();
TreeSet<Integer> canon = new TreeSet<>();
// we set FAN_FACTOR to 4, so 128 items is four levels deep, three fully populated
for (int i = 0 ; i < 128 ; i++)
for (UpdateFunction<Integer, Integer> updateF : LongBTreeTest.<Integer>updateFunctions())
{
String id = String.format("[0..%d)", canon.size());
log("Testing " + id);
Futures.allAsList(testAllSlices(id, cur, canon)).get();
Object[] next = null;
while (next == null)
next = BTree.update(cur, naturalOrder(), Arrays.asList(i), SPORADIC_ABORT);
cur = next;
canon.add(i);
Object[] cur = BTree.empty();
TreeSet<Integer> canon = new TreeSet<>();
// we set FAN_FACTOR to 4, so 128 items is four levels deep, three fully populated
for (int i = 0 ; i < 128 ; i++)
{
String id = String.format("[0..%d)", canon.size());
log("Testing " + id);
Futures.allAsList(testAllSlices(id, cur, canon)).get();
Object[] next = null;
while (next == null)
next = BTree.update(cur, BTree.singleton(i), naturalOrder(), updateF);
cur = next;
canon.add(i);
}
}
}
@ -993,36 +1095,37 @@ public class LongBTreeTest
};
}
private static final class RandomAbort<V> implements UpdateFunction<V, V>
private static List<UpdateFunction<Integer, Integer>> updateFunctions()
{
final Random rnd;
final float chance;
private RandomAbort(Random rnd, float chance)
{
this.rnd = rnd;
this.chance = chance;
}
return ImmutableList.of(UpdateFunction.noOp(), InverseNoOp.instance);
}
private static UpdateFunction<Integer, Integer> updateFunction(Random random)
{
return random.nextBoolean() ? InverseNoOp.instance : UpdateFunction.noOp();
}
public static final class InverseNoOp<V> implements UpdateFunction<V, V>
{
public static final InverseNoOp instance = new InverseNoOp();
public V apply(V replacing, V update)
{
return update;
}
public boolean abortEarly()
{
return rnd.nextFloat() < chance;
}
public void allocated(long heapSize)
public void onAllocatedOnHeap(long heapSize)
{
}
public V apply(V v)
{
return v;
}
}
private static long randomSeed()
{
return new SecureRandom().nextLong();
}
public static void main(String[] args) throws ExecutionException, InterruptedException, InvocationTargetException, IllegalAccessException
{
for (String arg : args)
@ -1076,4 +1179,4 @@ public class LongBTreeTest
args[0] = System.currentTimeMillis();
System.out.printf("%tT: " + formatstr + "\n", args);
}
}
}

89
test/microbench/org/apache/cassandra/test/microbench/btree/BTreeBench.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.test.microbench.btree;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.ImmutableList;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Warmup(iterations = 4, time = 1, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 8, time = 2, timeUnit = TimeUnit.SECONDS)
@Fork(value = 2)
@Threads(4)
@State(Scope.Benchmark)
public class BTreeBench extends Megamorphism
{
final AtomicInteger uniqueThreadInitialisation = new AtomicInteger();
Integer[] data;
// three iterables to simulate megamorphic callsites for iterable
List<Integer> dataAsIterable1;
List<Integer> dataAsIterable2;
List<Integer> dataAsIterable3;
// produces sizes between [n/2..n+n/2]
@Param({"1", "4", "16", "64", "256", "1024", "16384"})
int dataSize;
@Setup(Level.Trial)
public void setup()
{
setup(dataSize);
}
void setup(int size)
{
data = new Integer[size + size/2];
for (int i = 0 ; i < data.length; i++)
data[i] = i;
dataAsIterable1 = Arrays.asList(data);
dataAsIterable2 = ImmutableList.copyOf(data);
dataAsIterable3 = new ArrayList<>(dataSize);
dataAsIterable3.addAll(dataAsIterable1);
}
@State(Scope.Thread)
public static class BuildSizeState extends IntVisitor
{
@Setup(Level.Trial)
public void setup(BTreeBench bench)
{
super.setup(bench.dataSize);
}
}
}

127
test/microbench/org/apache/cassandra/test/microbench/btree/BTreeBuildBench.java Normal file
View File

@ -0,0 +1,127 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.test.microbench.btree;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.cassandra.utils.BulkIterator;
import org.apache.cassandra.utils.btree.BTree;
import org.apache.cassandra.utils.btree.UpdateFunction;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Warmup(iterations = 6, time = 1, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 8, time = 2, timeUnit = TimeUnit.SECONDS)
@Fork(value = 2)
@Threads(4)
@State(Scope.Benchmark)
public class BTreeBuildBench extends BTreeBench
{
@Benchmark
public Object[] buildWithIterableStaticBuildMethod(BuildSizeState state)
{
List<Integer> list;
int size = state.next();
switch (state.i() % 3)
{
case 0: list = dataAsIterable1; break;
case 1: list = dataAsIterable2; break;
case 2: list = dataAsIterable3; break;
default: throw new IllegalStateException();
}
return BTree.build(BulkIterator.of(list.iterator()), size, UpdateFunction.noOp());
}
@Benchmark
public Object[] buildWithMegamorphicBulkStaticBuildNoop(BuildSizeState state)
{
return buildWithMegamorphicBulkStaticBuild(state, UpdateFunction.noOp());
}
public Object[] buildWithMegamorphicBulkStaticBuild(BuildSizeState state, UpdateFunction<Integer, Integer> updateF)
{
BulkIterator<Integer> iter;
int size = state.next();
switch (state.i() % 3)
{
case 0: iter = BulkIterator.of(data); break;
case 1: iter = FromArrayCopy.of(data); break;
case 2: iter = FromArrayCopy2.of(data); break;
default: throw new IllegalStateException();
}
Object[] result = BTree.build(iter, size, updateF);
iter.close();
return result;
}
@Benchmark
public Object[] buildWithBulkStaticBuild(BuildSizeState state)
{
int size = state.next();
try (BulkIterator<Integer> iter = BulkIterator.of(data))
{
return BTree.build(iter, size, UpdateFunction.noOp());
}
}
@Benchmark
public Object[] buildWithBuilderAuto(BuildSizeState state)
{
int size = state.next();
BTree.Builder<Integer> builder = BTree.builder(Comparator.naturalOrder());
for (int i = 0 ; i < size ; ++i)
builder.add(data[i]);
return builder.build();
}
@Benchmark
public Object[] buildWithBuilderManual(BuildSizeState state)
{
int size = state.next();
BTree.Builder<Integer> builder = BTree.builder(Comparator.naturalOrder());
builder.auto(false);
for (int i = 0 ; i < size ; ++i)
builder.add(data[i]);
return builder.build();
}
@Benchmark
public Object[] buildWithFastBuilder(BuildSizeState state)
{
int size = state.next();
try (BTree.FastBuilder<Integer> builder = BTree.fastBuilder())
{
for (int i = 0 ; i < size ; ++i)
builder.add(data[i]);
return builder.build();
}
}
}

194
test/microbench/org/apache/cassandra/test/microbench/btree/BTreeTransformBench.java Normal file
View File

@ -0,0 +1,194 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.test.microbench.btree;
import java.util.BitSet;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.apache.cassandra.utils.BulkIterator;
import org.apache.cassandra.utils.btree.BTree;
import org.apache.cassandra.utils.btree.UpdateFunction;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 4, time = 2, timeUnit = TimeUnit.SECONDS)
@Fork(value = 2)
@Threads(4)
@State(Scope.Benchmark)
// TODO: parameterise build method for input to transform
public class BTreeTransformBench extends BTreeBench
{
public enum Distribution { CONTIGUOUS, RANDOM }
Integer[] data2;
@Param({"false"})
boolean uniquePerTrial;
@Param({"0", "0.0001", "0.001", "0.01", "0.0625", "0.125", "0.25", "0.5", "1"})
float ratio;
@Param({"CONTIGUOUS", "RANDOM"})
Distribution distribution;
@Setup(Level.Trial)
public void setup()
{
setup(2 * dataSize);
data2 = data.clone();
for (int i = 0 ; i < data2.length ; ++i)
data2[i] = new Integer(data2[i]);
}
@State(Scope.Thread)
public static class ThreadState
{
final Random random = new Random(0); // initialised to a seed below
final BitSet bitSet = new BitSet();
Integer[] data, data2;
boolean uniquePerTrial;
float ratio;
Distribution distribution;
// unique trials
// instead of doing per-invocation, we do per-iteration, as perfasm measures trial setup costs
Object[][] updates;
BuildSizeState buildSizeState = new BuildSizeState();
// current trial
Object[] update;
@Setup(Level.Trial)
public void doTrialSetup(BTreeTransformBench bench, BuildSizeState invocationBuildSizeState)
{
this.random.setSeed(bench.uniqueThreadInitialisation.incrementAndGet());
this.data = bench.data;
this.data2 = bench.data2;
this.uniquePerTrial = bench.uniquePerTrial;
this.ratio = bench.ratio;
this.distribution = bench.distribution;
if (!uniquePerTrial)
{
buildSizeState.setup(bench);
buildSizeState.randomise(random);
int numberOfUniqueTrials = (int) Math.min(4096, Runtime.getRuntime().maxMemory() / (4 * 8 * bench.dataSize));
updates = new Object[numberOfUniqueTrials][];
for (int i = 0; i < numberOfUniqueTrials; ++i)
updates[i] = createTree(buildSizeState);
}
invocationBuildSizeState.randomise(random);
}
@Setup(Level.Invocation)
public void doInvocationSetup(BuildSizeState buildSizeState)
{
if (!uniquePerTrial)
{
update = updates[buildSizeState.i() % updates.length];
buildSizeState.next();
}
else
{
this.update = createTree(buildSizeState);
}
int size = BTree.size(update);
int setBits = (int) Math.ceil(size * (ratio > 0.5f ? 1 - ratio : ratio));
switch (distribution)
{
case CONTIGUOUS: setContiguousBits(setBits, size); break;
case RANDOM: setRandomBits(setBits, size); break;
}
}
private Object[] createTree(BuildSizeState buildSizeState)
{
int buildSize = buildSizeState.next();
try (BulkIterator.FromArray<Integer> iter = BulkIterator.of(data))
{
return BTree.build(iter, buildSize, UpdateFunction.noOp());
}
}
private void setRandomBits(int count, int range)
{
ThreadLocalRandom random = ThreadLocalRandom.current();
bitSet.clear();
while (count > 0)
{
int next = random.nextInt(range);
if (bitSet.get(next))
continue;
bitSet.set(next);
--count;
}
}
private void setContiguousBits(int count, int range)
{
ThreadLocalRandom random = ThreadLocalRandom.current();
bitSet.clear();
int start = count >= range ? 0 : random.nextInt(range - count);
bitSet.set(start, start + count);
}
Function<Integer, Integer> apply(Function<Integer, Integer> replace)
{
return ratio > 0.5f ? i -> bitSet.get(i) ? i : replace.apply(i)
: i -> bitSet.get(i) ? replace.apply(i) : i;
}
}
@Benchmark
public Object[] transformReplace(ThreadState state)
{
return BTree.transform(state.update, state.apply(i -> data2[i]));
}
@Benchmark
public Object[] transformAndFilterReplace(ThreadState state)
{
return BTree.transformAndFilter(state.update, state.apply(i -> data2[i]));
}
@Benchmark
public Object[] transformAndFilterRemove(ThreadState state)
{
return BTree.transformAndFilter(state.update, state.apply(i -> null));
}
}

324
test/microbench/org/apache/cassandra/test/microbench/btree/BTreeUpdateBench.java Normal file
View File

@ -0,0 +1,324 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.test.microbench.btree;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.function.IntFunction;
import java.util.function.Supplier;
import java.util.stream.IntStream;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.btree.BTree;
import org.apache.cassandra.utils.btree.UpdateFunction;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Warmup(iterations = 10, time = 1)
@Measurement(iterations = 10, time = 2)
@Fork(value = 4)
@Threads(4)
@State(Scope.Benchmark)
// TODO: parameterise build method for inputs to update
public class BTreeUpdateBench extends BTreeBench
{
public enum Distribution { RANDOM, CONTIGUOUS }
Supplier<Comparator<? super Integer>> comparator = Comparator::naturalOrder;
Integer[] data2;
@Param({"1", "4", "16", "64", "256", "1024", "16384"})
int insertSize;
@Param({"0", "0.5", "1"})
float overlap;
@Param({"RANDOM", "CONTIGUOUS"})
Distribution distribution;
@Param({"false", "true"})
boolean keepOld;
@Param({"SIMPLE", "SIMPLE_MEGAMORPH", "UNSIMPLE"})
UpdateF updateF;
@Param({"false"})
boolean uniquePerTrial;
@Setup(Level.Trial)
public void setup()
{
setup(2 * (dataSize + insertSize));
data2 = data.clone();
for (int i = 0 ; i < data2.length ; ++i)
data2[i] = new Integer(data2[i]);
}
@State(Scope.Thread)
public static class InsertSizeState extends IntVisitor
{
@Setup(Level.Trial)
public void setup(BTreeUpdateBench bench)
{
super.setup(bench.insertSize);
}
}
@State(Scope.Thread)
public static class ThreadState
{
final Random random = new Random(0); // initialised to a seed below
Comparator<? super Integer> comparator;
Integer[] data, data2;
int[] randomOverlaps, randomBuild;
Distribution distribution;
float overlap;
boolean uniquePerTrial;
IntFunction<UpdateFunction<Integer, Integer>> updateFGetter;
BTree.Builder<Integer> buildBuilder = BTree.<Integer>builder(Comparator.naturalOrder()).auto(false);
BTree.Builder<Integer> insertBuilder = BTree.<Integer>builder(Comparator.naturalOrder()).auto(false);
// unique trials
// instead of doing per-invocation, we do per-iteration, as perfasm measures trial setup costs
Object[][] updates, inserts;
// current trial
Object[] update, insert;
UpdateFunction<Integer, Integer> updateF;
@Setup(Level.Trial)
public void doTrialSetup(BTreeUpdateBench bench, BuildSizeState invocationBuildSizeState, InsertSizeState invocationInsertSizeState)
{
random.setSeed(bench.uniqueThreadInitialisation.incrementAndGet());
this.comparator = bench.comparator.get();
this.data = bench.data;
this.data2 = bench.data2;
this.randomOverlaps = IntStream.range(0, data.length).toArray();
this.randomBuild = randomOverlaps.clone();
this.overlap = bench.overlap;
this.distribution = bench.distribution;
this.updateFGetter = updateFGetter(bench.keepOld, bench.updateF);
this.uniquePerTrial = bench.uniquePerTrial;
if (!uniquePerTrial)
{
BuildSizeState buildSizeState = new BuildSizeState();
InsertSizeState insertSizeState = new InsertSizeState();
buildSizeState.setup(bench);
insertSizeState.setup(bench);
buildSizeState.randomise(random);
insertSizeState.randomise(random);
int numberOfUniqueTrials = (int) Math.min(2048, Runtime.getRuntime().maxMemory() / (4 * 8 * (bench.insertSize + bench.dataSize)));
updates = new Object[numberOfUniqueTrials][];
inserts = new Object[numberOfUniqueTrials][];
for (int i = 0; i < numberOfUniqueTrials; ++i)
{
Pair<Object[], Object[]> updateAndInsert = createUpdateAndInsert(buildSizeState, insertSizeState);
updates[i] = updateAndInsert.left;
inserts[i] = updateAndInsert.right;
}
}
invocationBuildSizeState.randomise(random);
invocationInsertSizeState.randomise(random);
}
@Setup(Level.Invocation)
public void doInvocationSetup(BuildSizeState buildSizeState, InsertSizeState insertSizeState)
{
if (!uniquePerTrial)
{
update = updates[buildSizeState.i() % updates.length];
insert = inserts[buildSizeState.i() % inserts.length];
buildSizeState.next();
}
else
{
Pair<Object[], Object[]> updateAndInsert = createUpdateAndInsert(buildSizeState, insertSizeState);
this.update = updateAndInsert.left;
this.insert = updateAndInsert.right;
}
updateF = updateFGetter.apply(buildSizeState.i());
}
/**
* Create an iteration to the benchmark's spec, i.e. two trees with the specified size and overlap
*/
private Pair<Object[], Object[]> createUpdateAndInsert(BuildSizeState buildSizeState, InsertSizeState insertSizeState)
{
int buildSize = buildSizeState.next();
int insertSize = insertSizeState.next();
int overlapSize = (int) (Math.min(buildSize, insertSize) * overlap);
assert overlapSize <= buildSize && overlapSize <= insertSize;
BTree.Builder<Integer> build = buildBuilder;
BTree.Builder<Integer> insert = insertBuilder;
build.reuse();
insert.reuse();
switch (distribution)
{
case RANDOM:
{
updateOverlap(overlapSize);
buildRandom(build, data, buildSize, overlapSize);
buildRandom(insert, data2, insertSize, overlapSize);
break;
}
case CONTIGUOUS:
{
switch (buildSizeState.i() % 4)
{
case 0:
{
// left-hand insert overlap
int i = 0;
for (int j = 0, mj = insertSize-overlapSize ; j < mj ; ++j)
insert.add(data2[i++]);
for (int j = 0 ; j < overlapSize ; ++j)
{
build.add(data[i]);
insert.add(data2[i++]);
}
for (int j = 0, mj = buildSize-overlapSize ; j < mj ; ++j)
build.add(data[i++]);
break;
}
case 1:
{
// right-hand insert overlap
int i = 0;
for (int j = 0, mj = buildSize-overlapSize ; j < mj ; ++j)
build.add(data[i++]);
for (int j = 0 ; j < overlapSize ; ++j)
{
build.add(data[i]);
insert.add(data2[i++]);
}
for (int j = 0, mj = insertSize-overlapSize ; j < mj ; ++j)
insert.add(data2[i++]);
break;
}
case 2:
{
// straddle insert overlap
int i = 0;
for (int j = 0, mj = (insertSize-overlapSize)/2 ; j < mj ; ++j)
insert.add(data2[i++]);
for (int j = 0, mj = (buildSize-overlapSize)/2 ; j < mj ; ++j)
build.add(data[i++]);
for (int j = 0 ; j < overlapSize ; ++j)
{
build.add(data[i]);
insert.add(data2[i++]);
}
for (int j = 0, mj = buildSize - (overlapSize + (buildSize-overlapSize)/2) ; j < mj ; ++j)
build.add(data[i++]);
for (int j = 0, mj = insertSize - (overlapSize + (insertSize-overlapSize)/2); j < mj ; ++j)
insert.add(data2[i++]);
break;
}
case 3:
{
// straddle update overlap
int i = 0;
for (int j = 0, mj = (buildSize-overlapSize)/2 ; j < mj ; ++j)
build.add(data[i++]);
for (int j = 0, mj = (insertSize-overlapSize)/2 ; j < mj ; ++j)
insert.add(data2[i++]);
for (int j = 0 ; j < overlapSize ; ++j)
{
build.add(data[i]);
insert.add(data2[i++]);
}
for (int j = 0, mj = insertSize - (overlapSize + (insertSize-overlapSize)/2); j < mj ; ++j)
insert.add(data2[i++]);
for (int j = 0, mj = buildSize - (overlapSize + (buildSize-overlapSize)/2) ; j < mj ; ++j)
build.add(data[i++]);
}
}
break;
}
}
return Pair.create(build.build(), insert.build());
}
/**
* Randomise the elements of {@code data} with occur in the first {@code size} elements, then sort them
*/
private void shuffleAndSort(int[] data, int size)
{
for (int i = 0 ; i < size ; ++i)
{
int swap = random.nextInt(data.length);
int tmp = data[swap];
data[swap] = data[i];
data[i] = tmp;
}
Arrays.sort(data, 0, size);
}
/**
* Randomise the elements of {@link #randomOverlaps} with occur in the first {@code size} elements, then sort them
*/
private void updateOverlap(int size)
{
shuffleAndSort(randomOverlaps, size);
}
private void buildRandom(BTree.Builder<Integer> build, Integer[] data, int buildSize, int overlapSize)
{
shuffleAndSort(randomBuild, buildSize + overlapSize);
// linear merge
int i = 0, c = 0, j = 0;
for ( ; c < buildSize && j < overlapSize ; ++c)
{
if (randomBuild[i] < randomOverlaps[j]) build.add(data[randomBuild[i++]]);
else if (randomBuild[i] > randomOverlaps[j]) build.add(data[randomOverlaps[j++]]);
else { build.add(data[randomBuild[i++]]); j++; }
}
while (c++ < buildSize)
build.add(data[randomBuild[i++]]);
}
}
@Benchmark
public Object[] benchUpdate(ThreadState state)
{
return BTree.update(state.update, state.insert, state.comparator, state.updateF);
}
}

85
test/microbench/org/apache/cassandra/test/microbench/btree/IntVisitor.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.test.microbench.btree;
import java.util.Random;
/**
* A utility class to visit 2^n integers, around a median value, in an order that rapidly covers a wide range of values
*/
public class IntVisitor
{
int invocations;
int shift;
int mask;
int base;
public IntVisitor()
{
}
public IntVisitor(int median)
{
setup(median);
}
public void setup(int median)
{
int variance = Integer.highestOneBit(median);
mask = variance - 1;
base = median - variance/2;
shift = 32 - Integer.bitCount(mask);
}
public int i()
{
return invocations;
}
public int cur()
{
return size(invocations);
}
public int next()
{
return size(invocations = invocations == Integer.MAX_VALUE ? 0 : invocations + 1);
}
public int min()
{
return base;
};
public int size()
{
return mask + 1;
};
public int size(int invocations)
{
// we reverse the integer to more evenly distribute the visitation of sizes
return base + (Integer.reverse(invocations & mask) >>> shift);
}
public void randomise(Random random)
{
invocations = random.nextInt(Integer.MAX_VALUE);
}
}

169
test/microbench/org/apache/cassandra/test/microbench/btree/Megamorphism.java Normal file
View File

@ -0,0 +1,169 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.test.microbench.btree;
import java.util.function.IntFunction;
import org.apache.cassandra.utils.BulkIterator;
import org.apache.cassandra.utils.btree.UpdateFunction;
import org.apache.cassandra.utils.caching.TinyThreadLocalPool;
public class Megamorphism
{
// All functionallity noops,
public enum UpdateF { SIMPLE, SIMPLE_MEGAMORPH, UNSIMPLE }
private static final UpdateFunction SIMPLE_KEEP_OLD_1 = UpdateFunction.noOp();
private static final UpdateFunction SIMPLE_KEEP_OLD_2 = UpdateFunction.Simple.of((a, b) -> a);
private static final UpdateFunction SIMPLE_KEEP_OLD_3 = UpdateFunction.Simple.of((a, b) -> a);
private static final UpdateFunction SIMPLE_KEEP_NEW_1 = UpdateFunction.Simple.of((a, b) -> b);
private static final UpdateFunction SIMPLE_KEEP_NEW_2 = UpdateFunction.Simple.of((a, b) -> b);
private static final UpdateFunction SIMPLE_KEEP_NEW_3 = UpdateFunction.Simple.of((a, b) -> b);
private static final UpdateFunction UNSIMPLE_KEEP_OLD = new UpdateFunction()
{
public Object apply(Object replacing, Object update) { return replacing; }
public void onAllocatedOnHeap(long heapSize) { }
public Object apply(Object v) { return v; }
};
private static final UpdateFunction UNSIMPLE_KEEP_NEW = new UpdateFunction()
{
public Object apply(Object replacing, Object update) { return update; }
public void onAllocatedOnHeap(long heapSize) { }
public Object apply(Object v) { return v; }
};
static <V> IntFunction<UpdateFunction<V, V>> updateFGetter(boolean keepOld, BTreeBench.UpdateF updateF)
{
switch (updateF)
{
case SIMPLE: return keepOld ? i -> SIMPLE_KEEP_OLD_1 : i -> SIMPLE_KEEP_NEW_1;
case UNSIMPLE: return keepOld ? i -> UNSIMPLE_KEEP_OLD : i -> UNSIMPLE_KEEP_NEW;
case SIMPLE_MEGAMORPH:
if (keepOld)
{
return i -> {
switch (i % 3)
{
case 0: return SIMPLE_KEEP_OLD_1;
case 1: return SIMPLE_KEEP_OLD_2;
case 2: return SIMPLE_KEEP_OLD_3;
default: throw new IllegalStateException();
}
};
}
else
{
return i -> {
switch (i % 3)
{
case 0: return SIMPLE_KEEP_NEW_1;
case 1: return SIMPLE_KEEP_NEW_2;
case 2: return SIMPLE_KEEP_NEW_3;
default: throw new IllegalStateException();
}
};
}
default:
throw new IllegalStateException();
}
}
static class FromArrayCopy<V> implements BulkIterator<V>, AutoCloseable
{
private static final TinyThreadLocalPool<FromArrayCopy> cache = new TinyThreadLocalPool<>();
private Object[] from;
private int i;
private TinyThreadLocalPool.TinyPool<FromArrayCopy> pool;
public static <V> FromArrayCopy<V> of(Object[] from)
{
TinyThreadLocalPool.TinyPool<FromArrayCopy> pool = cache.get();
FromArrayCopy<V> result = pool.poll();
if (result == null)
result = new FromArrayCopy<>();
result.from = from;
result.i = 0;
result.pool = pool;
return result;
}
public void close()
{
pool.offer(this);
from = null;
pool = null;
}
public void fetch(Object[] into, int offset, int count)
{
System.arraycopy(from, i, into, offset, count);
i += count;
}
public V next()
{
return (V) from[i++];
}
}
static class FromArrayCopy2<V> implements BulkIterator<V>, AutoCloseable
{
private static final TinyThreadLocalPool<FromArrayCopy2> cache = new TinyThreadLocalPool<>();
private Object[] from;
private int i;
private TinyThreadLocalPool.TinyPool<FromArrayCopy2> pool;
public static <V> FromArrayCopy2<V> of(Object[] from)
{
TinyThreadLocalPool.TinyPool<FromArrayCopy2> pool = cache.get();
FromArrayCopy2<V> result = pool.poll();
if (result == null)
result = new FromArrayCopy2<>();
result.from = from;
result.i = 0;
result.pool = pool;
return result;
}
public void close()
{
pool.offer(this);
from = null;
pool = null;
}
public void fetch(Object[] into, int offset, int count)
{
System.arraycopy(from, i, into, offset, count);
i += count;
}
public V next()
{
return (V) from[i++];
}
}
}

17
test/unit/org/apache/cassandra/utils/btree/BTreeRemovalTest.java
View File

@ -37,7 +37,7 @@ public class BTreeRemovalTest
{
static
{
System.setProperty("cassandra.btree.fanfactor", "8");
System.setProperty("cassandra.btree.branchshift", "3");
}
private static final Comparator<Integer> CMP = new Comparator<Integer>()
@ -101,7 +101,7 @@ public class BTreeRemovalTest
private static Object[] generateLeaf(int from, int size)
{
final Object[] result = new Object[(size & 1) == 1 ? size : size + 1];
final Object[] result = new Object[size | 1];
for (int i = 0; i < size; ++i)
result[i] = from + i;
return result;
@ -112,16 +112,21 @@ public class BTreeRemovalTest
assert keys.length > 0;
assert children.length > 1;
assert children.length == keys.length + 1;
final Object[] result = new Object[keys.length + children.length + 1];
for (int i = 0; i < keys.length; ++i)
result[i] = keys[i];
for (int i = 0; i < children.length; ++i)
result[keys.length + i] = children[i];
final int[] sizeMap = new int[children.length];
sizeMap[0] = BTree.size(children[0]);
for (int i = 1; i < children.length; ++i)
sizeMap[i] = sizeMap[i - 1] + BTree.size(children[i]) + 1;
result[result.length - 1] = sizeMap;
return result;
}
@ -131,9 +136,11 @@ public class BTreeRemovalTest
final Object[][] leaves = new Object[leafSizes.length][];
for (int i = 0; i < leaves.length; ++i)
leaves[i] = generateLeaf(10 * i + 1, leafSizes[i]);
final int[] keys = new int[leafSizes.length - 1];
for (int i = 0; i < keys.length; ++i)
keys[i] = 10 * (i + 1);
final Object[] btree = generateBranch(keys, leaves);
assertTrue(BTree.isWellFormed(btree, CMP));
return btree;
@ -184,7 +191,7 @@ public class BTreeRemovalTest
@Test
public void testRemoveFromRootWhichIsALeaf()
{
for (int size = 1; size < 9; ++size)
for (int size = 1; size <= BTree.MAX_KEYS; ++size)
{
final Object[] btree = new Object[(size & 1) == 1 ? size : size + 1];
for (int i = 0; i < size; ++i)
@ -218,7 +225,7 @@ public class BTreeRemovalTest
@Test
public void testRemoveFromNonMinimalLeaf()
{
for (int size = 5; size < 9; ++size)
for (int size = 5; size <= BTree.MAX_KEYS; ++size)
{
final Object[] btree = generateSampleTwoLevelsTree(new int[] {size, 4, 4, 4, 4});
@ -370,7 +377,7 @@ public class BTreeRemovalTest
SortedSet<Integer> data = new TreeSet<>();
for (int i = 0; i < 1000; ++i)
data.add(rand.nextInt());
Object[] btree = BTree.build(data, UpdateFunction.<Integer>noOp());
Object[] btree = BTree.build(data);
assertTrue(BTree.isWellFormed(btree, CMP));
assertTrue(Iterables.elementsEqual(data, BTree.iterable(btree)));

6
test/unit/org/apache/cassandra/utils/btree/BTreeSearchIteratorTest.java
View File

@ -138,7 +138,7 @@ public class BTreeSearchIteratorTest
@Test
public void testTreeIteratorNormal()
{
Object[] btree = BTree.build(seq(30), UpdateFunction.noOp());
Object[] btree = BTree.build(seq(30));
BTreeSearchIterator fullIter = new FullBTreeSearchIterator<>(btree, CMP, Dir.ASC);
BTreeSearchIterator leafIter = new LeafBTreeSearchIterator<>(btree, CMP, Dir.ASC);
assertBTreeSearchIteratorEquals(fullIter, leafIter);
@ -181,7 +181,7 @@ public class BTreeSearchIteratorTest
@Test
public void testTreeIteratorOneElem()
{
Object[] btree = BTree.build(seq(1), UpdateFunction.noOp());
Object[] btree = BTree.build(seq(1));
BTreeSearchIterator fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
BTreeSearchIterator leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
assertBTreeSearchIteratorEquals(fullIter, leafIter);
@ -217,7 +217,7 @@ public class BTreeSearchIteratorTest
@Test
public void testTreeIteratorNotFound()
{
Object[] btree = BTree.build(seq(31, 0, 3), UpdateFunction.noOp());
Object[] btree = BTree.build(seq(31, 0, 3));
BTreeSearchIterator fullIter = new FullBTreeSearchIterator(btree, CMP, Dir.ASC);
BTreeSearchIterator leafIter = new LeafBTreeSearchIterator(btree, CMP, Dir.ASC);
assertBTreeSearchIteratorEquals(fullIter, leafIter, 3 * 5 + 1);

239
test/unit/org/apache/cassandra/utils/btree/BTreeTest.java
View File

@ -21,15 +21,10 @@ import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.junit.Test;
import org.junit.Assert;
import static org.apache.cassandra.utils.btree.BTree.EMPTY_LEAF;
import static org.apache.cassandra.utils.btree.BTree.FAN_FACTOR;
import static org.apache.cassandra.utils.btree.BTree.FAN_SHIFT;
import static org.apache.cassandra.utils.btree.BTree.POSITIVE_INFINITY;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
@ -38,7 +33,7 @@ public class BTreeTest
static Integer[] ints = new Integer[20];
static
{
System.setProperty("cassandra.btree.fanfactor", "4");
System.setProperty("cassandra.btree.branchshift", "2");
for (int i = 0 ; i < ints.length ; i++)
ints[i] = new Integer(i);
}
@ -50,14 +45,8 @@ public class BTreeTest
return ints[update];
}
public boolean abortEarly()
public void onAllocatedOnHeap(long heapSize)
{
return false;
}
public void allocated(long heapSize)
{
}
public Integer apply(Integer integer)
@ -66,28 +55,6 @@ public class BTreeTest
}
};
private static final UpdateFunction<Integer, Integer> noOp = new UpdateFunction<Integer, Integer>()
{
public Integer apply(Integer replacing, Integer update)
{
return update;
}
public boolean abortEarly()
{
return false;
}
public void allocated(long heapSize)
{
}
public Integer apply(Integer k)
{
return k;
}
};
private static List<Integer> seq(int count, int interval)
{
List<Integer> r = new ArrayList<>();
@ -102,27 +69,7 @@ public class BTreeTest
return seq(count, 1);
}
private static List<Integer> rand(int count)
{
Random rand = ThreadLocalRandom.current();
List<Integer> r = seq(count);
for (int i = 0 ; i < count - 1 ; i++)
{
int swap = i + rand.nextInt(count - i);
Integer tmp = r.get(i);
r.set(i, r.get(swap));
r.set(swap, tmp);
}
return r;
}
private static final Comparator<Integer> CMP = new Comparator<Integer>()
{
public int compare(Integer o1, Integer o2)
{
return Integer.compare(o1, o2);
}
};
private static final Comparator<Integer> CMP = Integer::compare;
@Test
public void testBuilding_UpdateFunctionReplacement()
@ -135,14 +82,14 @@ public class BTreeTest
public void testUpdate_UpdateFunctionReplacement()
{
for (int i = 0; i < 20 ; i++)
checkResult(i, BTree.update(BTree.build(seq(i), noOp), CMP, seq(i), updateF));
checkResult(i, BTree.update(BTree.build(seq(i)), BTree.build(seq(i)), CMP, updateF));
}
@Test
public void testApply()
{
List<Integer> input = seq(71);
Object[] btree = BTree.build(input, noOp);
Object[] btree = BTree.build(input);
final List<Integer> result = new ArrayList<>();
BTree.<Integer>apply(btree, i -> result.add(i));
@ -154,7 +101,7 @@ public class BTreeTest
public void inOrderAccumulation()
{
List<Integer> input = seq(71);
Object[] btree = BTree.build(input, noOp);
Object[] btree = BTree.build(input);
long result = BTree.<Integer>accumulate(btree, (o, l) -> {
Assert.assertEquals((long) o, l + 1);
return o;
@ -169,7 +116,7 @@ public class BTreeTest
for (int interval=1; interval<=5; interval++)
{
List<Integer> input = seq(limit, interval);
Object[] btree = BTree.build(input, noOp);
Object[] btree = BTree.build(input);
for (int start=0; start<=limit; start+=interval)
{
int thisInterval = interval;
@ -190,7 +137,7 @@ public class BTreeTest
public void accumulateFromEnd()
{
List<Integer> input = seq(100);
Object[] btree = BTree.build(input, noOp);
Object[] btree = BTree.build(input);
long result = BTree.accumulate(btree, (o, l) -> 1, Integer::compareTo, 101, 0L);
Assert.assertEquals(0, result);
}
@ -202,35 +149,35 @@ public class BTreeTest
@Test
public void testUpdate_UpdateFunctionCallBack()
{
Object[] btree = new Object[1];
Object[] btree = BTree.singleton(1);
CallsMonitor monitor = new CallsMonitor();
btree = BTree.update(btree, CMP, Arrays.asList(1), monitor);
btree = BTree.update(btree, BTree.singleton(1), CMP, monitor);
assertArrayEquals(new Object[] {1}, btree);
assertEquals(1, monitor.getNumberOfCalls(1));
monitor.clear();
btree = BTree.update(btree, CMP, Arrays.asList(2), monitor);
btree = BTree.update(btree, BTree.singleton(2), CMP, monitor);
assertArrayEquals(new Object[] {1, 2, null}, btree);
assertEquals(1, monitor.getNumberOfCalls(2));
// with existing value
monitor.clear();
btree = BTree.update(btree, CMP, Arrays.asList(1), monitor);
btree = BTree.update(btree, BTree.singleton(1), CMP, monitor);
assertArrayEquals(new Object[] {1, 2, null}, btree);
assertEquals(1, monitor.getNumberOfCalls(1));
// with two non-existing values
monitor.clear();
btree = BTree.update(btree, CMP, Arrays.asList(3, 4), monitor);
assertArrayEquals(new Object[] {1, 2, 3, 4, null}, btree);
btree = BTree.update(btree, BTree.build(Arrays.asList(3, 4)), CMP, monitor);
assertArrayEquals(new Object[] {3, new Object[]{1, 2, null}, new Object[]{4}, new int[]{2, 4}}, btree);
assertEquals(1, monitor.getNumberOfCalls(3));
assertEquals(1, monitor.getNumberOfCalls(4));
// with one existing value and one non existing value
monitor.clear();
btree = BTree.update(btree, CMP, Arrays.asList(2, 5), monitor);
assertArrayEquals(new Object[] {3, new Object[]{1, 2, null}, new Object[]{4, 5, null}, new int[]{2, 5}}, btree);
btree = BTree.update(btree, BTree.build(Arrays.asList(2, 5)), CMP, monitor);
assertArrayEquals(new Object[] {3, new Object[]{1, 2, null}, new Object[]{4, 5, null}, new int[]{2, 5}}, btree);
assertEquals(1, monitor.getNumberOfCalls(2));
assertEquals(1, monitor.getNumberOfCalls(5));
}
@ -323,6 +270,9 @@ public class BTreeTest
builder.reuse(Comparator.reverseOrder());
for (int i = 0; i < 12; i++)
builder.add(sorted.get(i));
System.out.println(BTree.MAX_KEYS);
System.out.println(BTree.MIN_KEYS);
System.out.println(BTree.toString(builder.build()));
checkResult(12, builder.build(), BTree.Dir.DESC);
builder.reuse();
@ -458,6 +408,7 @@ public class BTreeTest
private static void checkResult(int count, Object[] btree, BTree.Dir dir)
{
assertTrue(BTree.isWellFormed(btree, BTree.Dir.DESC == dir ? CMP.reversed() : CMP));
Iterator<Integer> iter = BTree.slice(btree, CMP, dir);
int i = 0;
while (iter.hasNext())
@ -465,44 +416,6 @@ public class BTreeTest
assertEquals(count, i);
}
@Test
public void testClearOnAbort()
{
Object[] btree = BTree.build(seq(2), noOp);
Object[] copy = Arrays.copyOf(btree, btree.length);
BTree.update(btree, CMP, seq(94), new AbortAfterX(90));
assertArrayEquals(copy, btree);
btree = BTree.update(btree, CMP, seq(94), noOp);
assertTrue(BTree.isWellFormed(btree, CMP));
}
private static final class AbortAfterX implements UpdateFunction<Integer, Integer>
{
int counter;
final int abortAfter;
private AbortAfterX(int abortAfter)
{
this.abortAfter = abortAfter;
}
public Integer apply(Integer replacing, Integer update)
{
return update;
}
public boolean abortEarly()
{
return counter++ > abortAfter;
}
public void allocated(long heapSize)
{
}
public Integer apply(Integer v)
{
return v;
}
}
/**
* <code>UpdateFunction</code> that count the number of call made to apply for each value.
*/
@ -510,22 +423,20 @@ public class BTreeTest
{
private int[] numberOfCalls = new int[20];
@Override
public Integer apply(Integer replacing, Integer update)
{
numberOfCalls[update] = numberOfCalls[update] + 1;
return update;
}
public boolean abortEarly()
{
return false;
}
public void allocated(long heapSize)
@Override
public void onAllocatedOnHeap(long heapSize)
{
}
@Override
public Integer apply(Integer integer)
{
numberOfCalls[integer] = numberOfCalls[integer] + 1;
@ -542,106 +453,4 @@ public class BTreeTest
Arrays.fill(numberOfCalls, 0);
}
}
@Test
public void testTransformAndFilter()
{
List<Integer> r = seq(100);
Object[] b1 = BTree.build(r, UpdateFunction.noOp());
// replace all values
Object[] b2 = BTree.transformAndFilter(b1, (x) -> (Integer) x * 2);
assertEquals(BTree.size(b1), BTree.size(b2));
// remove odd numbers
Object[] b3 = BTree.transformAndFilter(b1, (x) -> (Integer) x % 2 == 1 ? x : null);
assertEquals(BTree.size(b1) / 2, BTree.size(b3));
// remove all values
Object[] b4 = BTree.transformAndFilter(b1, (x) -> null);
assertEquals(0, BTree.size(b4));
}
private <C, K extends C, V extends C> Object[] buildBTreeLegacy(Iterable<K> source, UpdateFunction<K, V> updateF, int size)
{
assert updateF != null;
NodeBuilder current = new NodeBuilder();
while ((size >>= FAN_SHIFT) > 0)
current = current.ensureChild();
current.reset(EMPTY_LEAF, POSITIVE_INFINITY, updateF, null);
for (K key : source)
current.addNewKey(key);
current = current.ascendToRoot();
Object[] r = current.toNode();
current.clear();
return r;
}
// Basic BTree validation to check the values and sizeOffsets. Return tree size.
private int validateBTree(Object[] tree, int[] startingPos, boolean isRoot)
{
if (BTree.isLeaf(tree))
{
int size = BTree.size(tree);
if (!isRoot)
{
assertTrue(size >= FAN_FACTOR / 2);
assertTrue(size <= FAN_FACTOR);
}
for (int i = 0; i < size; i++)
{
assertEquals((int)tree[i], startingPos[0]);
startingPos[0]++;
}
return size;
}
int childNum = BTree.getChildCount(tree);
assertTrue(childNum >= FAN_FACTOR / 2);
assertTrue(childNum <= FAN_FACTOR + 1);
int childStart = BTree.getChildStart(tree);
int[] sizeOffsets = BTree.getSizeMap(tree);
int pos = 0;
for (int i = 0; i < childNum; i++)
{
int childSize = validateBTree((Object[])tree[i + childStart], startingPos, false);
pos += childSize;
assertEquals(sizeOffsets[i], pos);
if (i != childNum - 1)
{
assertEquals((int)tree[i], startingPos[0]);
pos++;
startingPos[0]++;
}
}
return BTree.size(tree);
}
@Test
public void testBuildTree()
{
int maxCount = 1000;
for (int count = 0; count < maxCount; count++)
{
List<Integer> r = seq(count);
Object[] b1 = BTree.build(r, UpdateFunction.noOp());
Object[] b2 = buildBTreeLegacy(r, UpdateFunction.noOp(), count);
assertTrue(BTree.equals(b1, b2));
int[] startingPos = new int[1];
startingPos[0] = 0;
assertEquals(count, validateBTree(b1, startingPos, true));
startingPos[0] = 0;
assertEquals(count, validateBTree(b2, startingPos, true));
}
}
}