mirror of https://github.com/apache/cassandra
Improve Interval B-Tree test coverage; add pprinter for B-Trees
Patch by Alex Petrov; reviewed by Benedict Elliott Smith for CASSANDRA-20766
This commit is contained in:
Alex Petrov
parent
e8bd2319f4
commit
54a9c407bd
|
|
@ -40,9 +40,6 @@ import accord.local.DurableBefore;
|
|||
import accord.local.RedundantBefore;
|
||||
import accord.utils.Invariants;
|
||||
import accord.utils.UnhandledEnum;
|
||||
import accord.utils.btree.BTree;
|
||||
import accord.utils.btree.BulkIterator;
|
||||
import accord.utils.btree.UpdateFunction;
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.cql3.ColumnIdentifier;
|
||||
import org.apache.cassandra.db.AbstractCompactionController;
|
||||
|
|
@ -105,9 +102,12 @@ import org.apache.cassandra.service.accord.journal.AccordTopologyUpdate;
|
|||
import org.apache.cassandra.service.accord.serializers.Version;
|
||||
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
|
||||
import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
|
||||
import org.apache.cassandra.utils.BulkIterator;
|
||||
import org.apache.cassandra.utils.NoSpamLogger;
|
||||
import org.apache.cassandra.utils.NoSpamLogger.NoSpamLogStatement;
|
||||
import org.apache.cassandra.utils.TimeUUID;
|
||||
import org.apache.cassandra.utils.btree.BTree;
|
||||
import org.apache.cassandra.utils.btree.UpdateFunction;
|
||||
|
||||
import static accord.local.Cleanup.ERASE;
|
||||
import static accord.local.Cleanup.Input.PARTIAL;
|
||||
|
|
|
|||
|
|
@ -26,9 +26,6 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import accord.utils.Invariants;
|
||||
import accord.utils.btree.BTree;
|
||||
import accord.utils.btree.BulkIterator;
|
||||
import accord.utils.btree.UpdateFunction;
|
||||
import org.apache.cassandra.db.BufferClustering;
|
||||
import org.apache.cassandra.db.ColumnFamilyStore;
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
|
|
@ -50,10 +47,13 @@ import org.apache.cassandra.journal.StaticSegment.KeyOrderReader;
|
|||
import org.apache.cassandra.service.ClientState;
|
||||
import org.apache.cassandra.service.accord.AccordJournalValueSerializers.FlyweightImage;
|
||||
import org.apache.cassandra.service.accord.AccordJournalValueSerializers.FlyweightSerializer;
|
||||
import org.apache.cassandra.utils.BulkIterator;
|
||||
import org.apache.cassandra.utils.NoSpamLogger;
|
||||
|
||||
import static java.util.concurrent.TimeUnit.MINUTES;
|
||||
import org.apache.cassandra.service.accord.serializers.Version;
|
||||
import org.apache.cassandra.utils.btree.BTree;
|
||||
import org.apache.cassandra.utils.btree.UpdateFunction;
|
||||
|
||||
/**
|
||||
* Segment compactor: takes static segments and compacts them into a single SSTable.
|
||||
|
|
|
|||
|
|
@ -71,6 +71,10 @@ public class BTree
|
|||
*/
|
||||
public static final int BRANCH_SHIFT = BTREE_BRANCH_SHIFT.getInt();
|
||||
|
||||
/**
|
||||
* _DO NOT_ attempt to modify this field directly. Instead, use BRANCH_SHIFT above instead, as branch factor
|
||||
* should _always_ be a power of 2.
|
||||
*/
|
||||
static final int BRANCH_FACTOR = 1 << BRANCH_SHIFT;
|
||||
public static final int MIN_KEYS = BRANCH_FACTOR / 2 - 1;
|
||||
public static final int MAX_KEYS = BRANCH_FACTOR - 1;
|
||||
|
|
@ -4045,7 +4049,7 @@ public class BTree
|
|||
* 2) If we exhaust all of our ancestors, and are not now ourselves overflowing, drain and return
|
||||
* 3) Otherwise propagate the redistributed contents to our parent and return null, indicating we can continue to parent
|
||||
*
|
||||
* @return {@code null} if {@code parent} is still logicallly in use after we execute;
|
||||
* @return {@code null} if {@code parent} is still logically in use after we execute;
|
||||
* otherwise the return value is the final result
|
||||
*/
|
||||
private Object[] stealAndMaybeRepropagate(LeafOrBranchBuilder fill, BranchBuilder parent)
|
||||
|
|
@ -4148,7 +4152,8 @@ public class BTree
|
|||
if (!remove.hasNext())
|
||||
return -1 - (1 + usz);
|
||||
|
||||
int i = exponentialSearch(comparator, unode, upos, usz, remove.peek());
|
||||
K next = remove.peek();
|
||||
int i = exponentialSearch(comparator, unode, upos, usz, next);
|
||||
if (i == -1 - usz)
|
||||
{
|
||||
// if we sort after the last key in the branch, we may need to descend into the right-most child
|
||||
|
|
@ -4159,7 +4164,7 @@ public class BTree
|
|||
{
|
||||
Object[] pnode = update.nodes[pdepth];
|
||||
int ppos = update.positions[pdepth];
|
||||
if (ppos < shallowSizeOfBranch(pnode) && comparator.compare(remove.peek(), (K)pnode[ppos]) >= 0)
|
||||
if (ppos < shallowSizeOfBranch(pnode) && comparator.compare(next, (K)pnode[ppos]) >= 0)
|
||||
{
|
||||
// increase our result index to point to *after* the last child;
|
||||
// (it's an inequality binary search semantic answer, so will be negated)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,133 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.utils.btree;
|
||||
|
||||
import static org.apache.cassandra.utils.btree.BTree.height;
|
||||
import static org.apache.cassandra.utils.btree.BTree.isEmpty;
|
||||
import static org.apache.cassandra.utils.btree.BTree.isLeaf;
|
||||
import static org.apache.cassandra.utils.btree.BTree.shallowSizeOfBranch;
|
||||
import static org.apache.cassandra.utils.btree.BTree.size;
|
||||
import static org.apache.cassandra.utils.btree.BTree.sizeMap;
|
||||
import static org.apache.cassandra.utils.btree.BTree.sizeOfLeaf;
|
||||
|
||||
// A small utility for debugging / printing B-Tree / IntervalBTrees
|
||||
//
|
||||
// Prints B-Tree in the following format:
|
||||
//
|
||||
// * (branch): [ 9,10:9 | 15,16:15 ]
|
||||
// ├─ [-∞]
|
||||
// │ * (branch): [ 3,4:3 | 6,7:6 ]
|
||||
// │ ├─ [-∞]
|
||||
// │ │ (leaf): [ 0,1:0, 1,2:1, 2,3:2 ]
|
||||
// │ ├─ [3,4:3]
|
||||
// │ │ (leaf): [ 4,5:4, 5,6:5 ]
|
||||
// │ └─ [6,7:6]
|
||||
// │ (leaf): [ 7,8:7, 8,9:8 ]
|
||||
// ├─ [9,10:9]
|
||||
// │ * (branch): [ 12,13:12 ]
|
||||
// │ ├─ [-∞]
|
||||
// │ │ (leaf): [ 10,11:10, 11,12:11 ]
|
||||
// │ └─ [12,13:12]
|
||||
// │ (leaf): [ 13,14:13, 14,15:14 ]
|
||||
// └─ [15,16:15]
|
||||
// * (branch): [ 18,19:18 ]
|
||||
// ├─ [-∞]
|
||||
// │ (leaf): [ 16,17:16, 17,18:17 ]
|
||||
// └─ [18,19:18]
|
||||
// (leaf): [ 19,20:19 ]
|
||||
@SuppressWarnings("unused")
|
||||
public class BTreePrinter
|
||||
{
|
||||
private static boolean PRINT_SIZE_MAP = false;
|
||||
|
||||
public static String print(Object[] btree)
|
||||
{
|
||||
if (isEmpty(btree))
|
||||
return "empty";
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("(size=").append(size(btree))
|
||||
.append(", height=").append(height(btree))
|
||||
.append("):\n");
|
||||
printNode(sb, btree, 0, "");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static void printNode(StringBuilder sb, Object[] node, int level, String prefix)
|
||||
{
|
||||
String indent = " ".repeat(level);
|
||||
|
||||
if (isLeaf(node))
|
||||
{
|
||||
int leafSize = sizeOfLeaf(node);
|
||||
sb.append(prefix).append(indent)
|
||||
.append("(leaf): [ ");
|
||||
for (int i = 0; i < leafSize; i++)
|
||||
{
|
||||
if (i > 0) sb.append(", ");
|
||||
sb.append(node[i]);
|
||||
}
|
||||
sb.append(" ]\n");
|
||||
}
|
||||
else
|
||||
{
|
||||
int keyCount = shallowSizeOfBranch(node);
|
||||
int childCount = keyCount + 1;
|
||||
int[] sizeMap = sizeMap(node);
|
||||
|
||||
sb.append(prefix)
|
||||
.append(indent)
|
||||
.append("* (branch): [ ");
|
||||
|
||||
for (int i = 0; i < keyCount; i++)
|
||||
{
|
||||
if (i > 0) sb.append(" | ");
|
||||
sb.append(node[i]);
|
||||
}
|
||||
sb.append(" ]\n");
|
||||
|
||||
if (PRINT_SIZE_MAP)
|
||||
{
|
||||
sb.append(prefix).append(indent).append("├─ sizeMap: ");
|
||||
for (int i = 0; i < sizeMap.length; i++)
|
||||
{
|
||||
if (i > 0) sb.append(", ");
|
||||
sb.append(sizeMap[i]);
|
||||
}
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
for (int i = 0; i < childCount; i++)
|
||||
{
|
||||
Object[] child = (Object[]) node[keyCount + i];
|
||||
String childPrefix = prefix + indent;
|
||||
String verticalLine = (i == childCount - 1) ? "└─ " : "├─ ";
|
||||
String nextPrefix = (i == childCount - 1) ? " " : "│ ";
|
||||
|
||||
sb.append(childPrefix).append(verticalLine)
|
||||
.append("[")
|
||||
.append(i == 0 ? "-∞" : node[i - 1])
|
||||
.append("]\n");
|
||||
|
||||
// Recursing here should be fine, as we do not expect depth to be too large
|
||||
printNode(sb, child, level + 1, prefix + indent + nextPrefix);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -19,16 +19,21 @@
|
|||
package org.apache.cassandra.utils.btree;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import accord.utils.DefaultRandom;
|
||||
import accord.utils.Invariants;
|
||||
import accord.utils.QuadFunction;
|
||||
import accord.utils.RandomSource;
|
||||
import accord.utils.SymmetricComparator;
|
||||
import org.apache.cassandra.config.CassandraRelevantProperties;
|
||||
|
|
@ -46,7 +51,7 @@ public class IntervalBTreeTest
|
|||
static
|
||||
{
|
||||
// build deeper trees to explore more behaviours
|
||||
CassandraRelevantProperties.BTREE_BRANCH_SHIFT.setInt(3);
|
||||
CassandraRelevantProperties.BTREE_BRANCH_SHIFT.setInt(2);
|
||||
}
|
||||
|
||||
static class TestInterval implements Comparable<TestInterval>
|
||||
|
|
@ -87,11 +92,11 @@ public class IntervalBTreeTest
|
|||
@Override public SymmetricComparator<TestInterval> startWithEndSeeker() { return (a, b) -> startWithEnd(Integer.compare(a.start, b.end)); }
|
||||
@Override public SymmetricComparator<TestInterval> endWithStartSeeker() { return (a, b) -> endWithStart(Integer.compare(a.end, b.start)); }
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testN()
|
||||
{
|
||||
testOne(7817231705170378212L);
|
||||
testOne(-415085593308080411L);
|
||||
Random seeds = new Random();
|
||||
for (int i = 0 ; i < 200 ; ++i)
|
||||
testOne(seeds.nextLong());
|
||||
|
|
@ -236,6 +241,192 @@ public class IntervalBTreeTest
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void printTest()
|
||||
{
|
||||
Random rng = new Random(0);
|
||||
List<TestInterval> intervals = new ArrayList<>();
|
||||
for (int i = 0; i < 20; i++)
|
||||
{
|
||||
intervals.add(interval(i, i + 1, i));
|
||||
}
|
||||
Collections.shuffle(intervals, rng);
|
||||
Object[] tree = IntervalBTree.empty();
|
||||
for (TestInterval v : intervals)
|
||||
tree = IntervalBTree.update(tree, IntervalBTree.singleton(v), TestComparators.INSTANCE);
|
||||
|
||||
String printed = BTreePrinter.print(tree);
|
||||
for (TestInterval interval : intervals)
|
||||
Invariants.require(printed.contains(interval.toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void simpleTest()
|
||||
{
|
||||
Random rng = new Random(0);
|
||||
List<TestInterval> intervals = Arrays.asList(interval(10, 40, 100), interval(20, 50, 200), interval(30, 60, 300));
|
||||
Collections.shuffle(intervals, rng);
|
||||
Object[] tree = IntervalBTree.empty();
|
||||
for (TestInterval v : intervals)
|
||||
{
|
||||
tree = IntervalBTree.update(tree, IntervalBTree.singleton(v), TestComparators.INSTANCE);
|
||||
}
|
||||
|
||||
{
|
||||
Set<TestInterval> actual = new TreeSet<>();
|
||||
IntervalBTree.accumulate(tree, TestComparators.INSTANCE, interval(20, 55, 400), new QuadFunction<Object, Object, TestInterval, Object, Object>()
|
||||
{
|
||||
@Override
|
||||
public Object apply(Object o, Object o2, TestInterval match, Object o3)
|
||||
{
|
||||
actual.add(match);
|
||||
return null;
|
||||
}
|
||||
}, null, null, null);
|
||||
Assert.assertEquals(actual, new TreeSet<>(intervals));
|
||||
}
|
||||
|
||||
{
|
||||
Set<TestInterval> actual = new TreeSet<>();
|
||||
IntervalBTree.accumulate(tree, TestComparators.INSTANCE, interval(10, 15, 400), new QuadFunction<Object, Object, TestInterval, Object, Object>()
|
||||
{
|
||||
@Override
|
||||
public Object apply(Object o, Object o2, TestInterval match, Object o3)
|
||||
{
|
||||
actual.add(match);
|
||||
return null;
|
||||
}
|
||||
}, null, null, null);
|
||||
Assert.assertEquals(actual, new TreeSet<>(Arrays.asList(interval(10, 40, 100))));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void midSizeTest()
|
||||
{
|
||||
for (int i = 0; i < 10_000; i++)
|
||||
{
|
||||
midSizeTestIteration(i);
|
||||
}
|
||||
}
|
||||
|
||||
public void midSizeTestIteration(int iteration)
|
||||
{
|
||||
RandomSource rng = new DefaultRandom(iteration);
|
||||
List<TestInterval> intervals = new ArrayList<>();
|
||||
TreeSet<TestInterval> unique = new TreeSet<>();
|
||||
for (int i = 0; i < rng.nextInt(10_000) + 1; i++)
|
||||
{
|
||||
TestInterval interval = newInterval(rng, 2, 10000);
|
||||
if (unique.add(interval))
|
||||
intervals.add(interval);
|
||||
}
|
||||
|
||||
Collections.shuffle(intervals, rng.asJdkRandom());
|
||||
|
||||
Object[] tree = IntervalBTree.empty();
|
||||
for (TestInterval v : intervals)
|
||||
tree = IntervalBTree.update(tree, IntervalBTree.singleton(v), TestComparators.INSTANCE);
|
||||
|
||||
for (int i = 0; i < 100; i++)
|
||||
{
|
||||
TestInterval searched = newInterval(rng, 2, 10000);
|
||||
Set<TestInterval> actual = new TreeSet<>();
|
||||
IntervalBTree.accumulate(tree, TestComparators.INSTANCE, searched, (o, o2, match, o3) -> {
|
||||
actual.add(match);
|
||||
return null;
|
||||
}, null, null, null);
|
||||
Assert.assertEquals(String.format("Searching for %s", searched),
|
||||
actual, new TreeSet<>(intervals.stream().filter(match -> {
|
||||
if ((match.start <= searched.start && searched.start < match.end) ||
|
||||
(searched.start <= match.start && match.start < searched.end))
|
||||
return true;
|
||||
return false;
|
||||
}).collect(Collectors.toList())));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void subtractTest()
|
||||
{
|
||||
for (int j = 0; j < 10_000; j++)
|
||||
subtractTestIteration(j);
|
||||
}
|
||||
|
||||
public void subtractTestIteration(int iteration)
|
||||
{
|
||||
RandomSource rng = new DefaultRandom(iteration);
|
||||
List<TestInterval> intervals = new ArrayList<>();
|
||||
TreeSet<TestInterval> unique = new TreeSet<>();
|
||||
for (int i = 0; i < rng.nextInt(10_000) + 1; i++)
|
||||
{
|
||||
TestInterval interval = newInterval(rng, 2, 10000);
|
||||
if (unique.add(interval))
|
||||
intervals.add(interval);
|
||||
}
|
||||
|
||||
Collections.shuffle(intervals, rng.asJdkRandom());
|
||||
|
||||
Object[] tree = IntervalBTree.empty();
|
||||
for (TestInterval v : intervals)
|
||||
tree = IntervalBTree.update(tree, IntervalBTree.singleton(v), TestComparators.INSTANCE);
|
||||
|
||||
for (int i = 0; i < 100 && BTree.size(tree) > 0; i++)
|
||||
{
|
||||
int subtractCount = Math.max(1, rng.nextInt(intervals.size() / 10 + 1));
|
||||
List<TestInterval> toSubtract = new ArrayList<>();
|
||||
TreeSet<TestInterval> subtractSet = new TreeSet<>();
|
||||
|
||||
for (int j = 0; j < subtractCount; j++)
|
||||
{
|
||||
TestInterval interval;
|
||||
// pick an existing or a new interval
|
||||
if (rng.nextBoolean() && !intervals.isEmpty())
|
||||
interval = intervals.get(rng.nextInt(intervals.size()));
|
||||
else
|
||||
interval = newInterval(rng, 2, 10000);
|
||||
|
||||
if (subtractSet.add(interval))
|
||||
toSubtract.add(interval);
|
||||
}
|
||||
|
||||
Object[] resultTree;
|
||||
{
|
||||
Object[] subtractTree = IntervalBTree.empty();
|
||||
for (TestInterval v : toSubtract)
|
||||
subtractTree = IntervalBTree.update(subtractTree, IntervalBTree.singleton(v), TestComparators.INSTANCE);
|
||||
|
||||
resultTree = IntervalBTree.subtract(tree, subtractTree, TestComparators.INSTANCE);
|
||||
}
|
||||
// Collect all intervals remaining in the result tree
|
||||
Set<TestInterval> remaining = new TreeSet<>();
|
||||
IntervalBTree.accumulate(resultTree, (o1, o2, interval, o3) -> {
|
||||
remaining.add((TestInterval)interval);
|
||||
return null;
|
||||
}, null, null, null);
|
||||
|
||||
Set<TestInterval> expectedRemaining = new TreeSet<>();
|
||||
for (TestInterval original : intervals)
|
||||
{
|
||||
boolean shouldBeSubtracted = subtractSet.contains(original);
|
||||
if (!shouldBeSubtracted)
|
||||
expectedRemaining.add(original);
|
||||
}
|
||||
|
||||
Assert.assertEquals(String.format("Subtraction iteration %d: expected %d remaining, got %d. Subtracted: %s",
|
||||
i, expectedRemaining.size(), remaining.size(), subtractSet),
|
||||
expectedRemaining, remaining);
|
||||
|
||||
intervals.retainAll(expectedRemaining);
|
||||
tree = resultTree;
|
||||
}
|
||||
}
|
||||
|
||||
public static TestInterval interval(int start, int end, int value)
|
||||
{
|
||||
return new TestInterval(start, end, value);
|
||||
}
|
||||
|
||||
private static int log2uniform(RandomSource random, int max)
|
||||
{
|
||||
int logn = 31 - Integer.numberOfLeadingZeros(max);
|
||||
|
|
@ -268,7 +459,7 @@ public class IntervalBTreeTest
|
|||
|
||||
private static TestInterval newInterval(RandomSource random, int keyDomain, int valueDomain)
|
||||
{
|
||||
int end = 1 + random.nextInt(keyDomain - 1);
|
||||
int end = keyDomain == 1 ? 1 : 1 + random.nextInt(Math.max(1, keyDomain - 1));
|
||||
int start = random.nextInt(end);
|
||||
return new TestInterval(start, end, random.nextInt(valueDomain));
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue