From f1db115e73f7050e82b84a4f642a065c7030414e Mon Sep 17 00:00:00 2001 From: David Capwell Date: Fri, 30 Aug 2024 09:06:39 -0700 Subject: [PATCH] Create a fuzz test that randomizes topology changes, cluster actions, and CQL operations patch by David Capwell; reviewed by Alex Petrov for CASSANDRA-19847 --- modules/accord | 2 +- .../index/accord/RouteIndexTest.java | 129 +++++++----------- .../cassandra/io/util/ChecksumedDataTest.java | 5 +- .../service/accord/EpochSyncTest.java | 107 ++++++--------- .../utils/StatefulRangeTreeTest.java | 98 ++++--------- 5 files changed, 127 insertions(+), 214 deletions(-) diff --git a/modules/accord b/modules/accord index e2ccee4f51..a171322f41 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit e2ccee4f51fe4c7c7f3ea8911897135ed7e37114 +Subproject commit a171322f417c117733ca5b514d03a5202b1ac202 diff --git a/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java b/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java index 17de14e415..edc7f2c517 100644 --- a/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java +++ b/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -49,7 +48,6 @@ import accord.primitives.TxnId; import accord.utils.Gen; import accord.utils.Gens; import accord.utils.Property.Command; -import accord.utils.Property.Commands; import accord.utils.Property.UnitCommand; import accord.utils.RandomSource; import org.agrona.collections.Int2ObjectHashMap; @@ -73,6 +71,7 @@ import org.apache.cassandra.utils.RTree; import org.apache.cassandra.utils.RangeTree; import org.assertj.core.api.Assertions; +import static accord.utils.Property.commands; import static accord.utils.Property.stateful; public class RouteIndexTest extends CQLTester.InMemory @@ -109,85 +108,61 @@ public class RouteIndexTest extends CQLTester.InMemory cfs().disableAutoCompaction(); // let the test control compaction //TODO (coverage): include with the ability to mark ranges as durable for compaction cleanup AccordService.unsafeSetNoop(); // disable accord service since compaction touches it. It would be nice to include this for cleanup support.... - stateful().withExamples(50).check(new Commands() + stateful().withExamples(50).check(commands(() -> State::new, i -> cfs()) + .destroySut(sut -> sut.truncateBlocking()) + .add(FLUSH) + .add(COMPACT) + .add((rs, state) -> { + int storeId = rs.nextInt(0, state.numStores); + Domain domain = state.domainGen.next(rs); + TxnId txnId = state.nextTxnId(domain); + Route route = createRoute(state, rs, domain, rs.nextInt(1, 20)); + return new InsertTxn(storeId, txnId, SaveStatus.PreAccepted, Durability.NotDurable, route); + }) + .add((rs, state) -> new RangeSearch(rs.nextInt(0, state.numStores), state.rangeGen.next(rs))) + .addIf(state -> !state.storeToTableToRangesToTxns.isEmpty(), RouteIndexTest::rangeSearch) + .build()); + } + + private static RangeSearch rangeSearch(RandomSource rs, State state) + { + int storeId = rs.pickUnorderedSet(state.storeToTableToRangesToTxns.keySet()); + var tables = state.storeToTableToRangesToTxns.get(storeId); + TableId tableId = rs.pickUnorderedSet(tables.keySet()); + var ranges = tables.get(tableId); + TreeSet distinctRanges = ranges.stream().map(Map.Entry::getKey).collect(Collectors.toCollection(() -> new TreeSet<>(TokenRange::compareTo))); + TokenRange range; + if (distinctRanges.size() == 1) { - @Override - public Gen genInitialState() + range = Iterables.getFirst(distinctRanges, null); + } + else + { + switch (rs.nextInt(0, 2)) { - return rs -> new State(rs); - } - - @Override - public ColumnFamilyStore createSut(State state) - { - return cfs(); - } - - @Override - public Gen> commands(State state) - { - Map>, Integer> possible = new LinkedHashMap<>(); - possible.put(ignore -> FLUSH, 1); - possible.put(ignore -> COMPACT, 1); - possible.put(rs -> { - int storeId = rs.nextInt(0, state.numStores); - Domain domain = state.domainGen.next(rs); - TxnId txnId = state.nextTxnId(domain); - Route route = createRoute(state, rs, domain, rs.nextInt(1, 20)); - return new InsertTxn(storeId, txnId, SaveStatus.PreAccepted, Durability.NotDurable, route); - }, 10); - possible.put(rs -> new RangeSearch(rs.nextInt(0, state.numStores), state.rangeGen.next(rs)), 1); - if (!state.storeToTableToRangesToTxns.isEmpty()) + case 0: // perfect match + range = rs.pickOrderedSet(distinctRanges); + break; + case 1: // mutli-match { - possible.put(rs -> { - int storeId = rs.pickUnorderedSet(state.storeToTableToRangesToTxns.keySet()); - var tables = state.storeToTableToRangesToTxns.get(storeId); - TableId tableId = rs.pickUnorderedSet(tables.keySet()); - var ranges = tables.get(tableId); - TreeSet distinctRanges = ranges.stream().map(Map.Entry::getKey).collect(Collectors.toCollection(() -> new TreeSet<>(TokenRange::compareTo))); - TokenRange range; - if (distinctRanges.size() == 1) - { - range = Iterables.getFirst(distinctRanges, null); - } - else - { - switch (rs.nextInt(0, 2)) - { - case 0: // perfect match - range = rs.pickOrderedSet(distinctRanges); - break; - case 1: // mutli-match - { - TokenRange a = rs.pickOrderedSet(distinctRanges); - TokenRange b = rs.pickOrderedSet(distinctRanges); - while (a.equals(b)) - b = rs.pickOrderedSet(distinctRanges); - if (b.compareTo(a) < 0) - { - TokenRange tmp = a; - a = b; - b = tmp; - } - range = new TokenRange((AccordRoutingKey) a.start(), (AccordRoutingKey) b.end()); - } - break; - default: - throw new AssertionError(); - } - } - return new RangeSearch(storeId, range); - }, 5); + TokenRange a = rs.pickOrderedSet(distinctRanges); + TokenRange b = rs.pickOrderedSet(distinctRanges); + while (a.equals(b)) + b = rs.pickOrderedSet(distinctRanges); + if (b.compareTo(a) < 0) + { + TokenRange tmp = a; + a = b; + b = tmp; + } + range = new TokenRange((AccordRoutingKey) a.start(), (AccordRoutingKey) b.end()); } - return Gens.oneOf(possible); + break; + default: + throw new AssertionError(); } - - @Override - public void destroySut(ColumnFamilyStore sut) - { - cfs().truncateBlocking(); - } - }); + } + return new RangeSearch(storeId, range); } private static ColumnFamilyStore cfs() @@ -343,7 +318,7 @@ public class RouteIndexTest extends CQLTester.InMemory } } - private class RangeSearch implements Command> + private static class RangeSearch implements Command> { private final int storeId; private final TokenRange range; diff --git a/test/unit/org/apache/cassandra/io/util/ChecksumedDataTest.java b/test/unit/org/apache/cassandra/io/util/ChecksumedDataTest.java index d7b6754a2d..5389c79bb7 100644 --- a/test/unit/org/apache/cassandra/io/util/ChecksumedDataTest.java +++ b/test/unit/org/apache/cassandra/io/util/ChecksumedDataTest.java @@ -24,6 +24,8 @@ import java.util.function.Supplier; import java.util.zip.CRC32C; import java.util.zip.Checksum; +import javax.annotation.Nullable; + import org.junit.Test; import accord.utils.Gen; @@ -149,8 +151,9 @@ public class ChecksumedDataTest } @Override - public void destroySut(List> sut) throws Throwable + public void destroySut(List> sut, @Nullable Throwable t) throws Throwable { + if (t != null) return; ChecksumedDataInputPlus in = new ChecksumedDataInputPlus(new DataInputBuffer(out.unsafeGetBufferAndFlip(), false), CHECKSUM_SUPPLIER); for (StatefulChecksumCommand cmd : sut) { diff --git a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java index 6bc85b6f1e..101ca38a73 100644 --- a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java +++ b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java @@ -28,7 +28,6 @@ import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.NavigableSet; @@ -60,10 +59,7 @@ import accord.primitives.Ranges; import accord.topology.Topology; import accord.topology.TopologyManager; import accord.utils.Gen; -import accord.utils.Gens; import accord.utils.Invariants; -import accord.utils.Property.Command; -import accord.utils.Property.Commands; import accord.utils.Property.UnitCommand; import accord.utils.RandomSource; import accord.utils.async.AsyncChain; @@ -109,6 +105,7 @@ import org.apache.cassandra.utils.ByteArrayUtil; import org.apache.cassandra.utils.Pair; import org.assertj.core.api.Assertions; +import static accord.utils.Property.commands; import static accord.utils.Property.stateful; public class EpochSyncTest @@ -126,72 +123,48 @@ public class EpochSyncTest @Test public void test() { - stateful().withExamples(50).check(new Commands() - { - @Override - public Gen genInitialState() - { - return Cluster::new; - } + stateful().withExamples(50).check(commands(() -> Cluster::new) + .destroyState(cluster -> { + cluster.processAll(); + cluster.validate(true); + }) + .addIf(cluster -> cluster.alive().size() <= cluster.maxNodes, EpochSyncTest::addNode) + .addIf(cluster -> cluster.alive().size() > cluster.minNodes, EpochSyncTest::removeNode) + .addIf(cluster -> cluster.hasWork(), EpochSyncTest::processSome) + .add(rs -> new SimpleCommand("Validate", c -> c.validate(false))) + .add((rs, cluster) -> new SimpleCommand("Bump Epoch " + (cluster.current.epoch.getEpoch() + 1), Cluster::bumpEpoch)) + .build()); + } - @Override - public Void createSut(Cluster Cluster) - { - return null; - } + private static SimpleCommand addNode(RandomSource rs, Cluster cluster) + { + Node.Id id = new Node.Id(++cluster.nodeCounter); + long token = cluster.tokenGen.nextLong(rs); + while (cluster.tokens.contains(token)) + token = cluster.tokenGen.nextLong(rs); + long epoch = cluster.current.epoch.getEpoch() + 1; + long finalToken = token; + return new SimpleCommand("Add Node " + id + "; token=" + token + ", epoch=" + epoch, + c -> c.addNode(id, finalToken)); + } - @Override - public Gen> commands(Cluster cluster) - { - List alive = cluster.alive(); - Map>, Integer> possible = new LinkedHashMap<>(); - if (alive.size() < cluster.maxNodes) - { - // add node - possible.put(rs -> { - Node.Id id = new Node.Id(++cluster.nodeCounter); - long token = cluster.tokenGen.nextLong(rs); - while (cluster.tokens.contains(token)) - token = cluster.tokenGen.nextLong(rs); - long epoch = cluster.current.epoch.getEpoch() + 1; - long finalToken = token; - return new SimpleCommand("Add Node " + id + "; token=" + token + ", epoch=" + epoch, - c -> c.addNode(id, finalToken)); - }, 5); - } - if (alive.size() > cluster.minNodes) - { - possible.put(rs -> { - Node.Id pick = rs.pick(alive); - long token = cluster.instances.get(pick).token; - long epoch = cluster.current.epoch.getEpoch() + 1; - return new SimpleCommand("Remove Node " + pick + "; token=" + token + "; epoch=" + epoch, c -> c.removeNode(pick)); - }, 3); - } - if (cluster.hasWork()) - { - possible.put(rs -> new SimpleCommand("Process Some", - c -> {//noinspection StatementWithEmptyBody - for (int i = 0, attempts = rs.nextInt(1, 100); i < attempts && c.processOne(); i++) - { - } - }), 10); - } + private static SimpleCommand removeNode(RandomSource rs, Cluster cluster) + { + List alive = cluster.alive(); + Node.Id pick = rs.pick(alive); + long token = cluster.instances.get(pick).token; + long epoch = cluster.current.epoch.getEpoch() + 1; + return new SimpleCommand("Remove Node " + pick + "; token=" + token + "; epoch=" + epoch, c -> c.removeNode(pick)); + } - possible.put(rs -> new SimpleCommand("Validate", - c -> c.validate(false)), 1); - possible.put(rs -> new SimpleCommand("Bump Epoch " + (cluster.current.epoch.getEpoch() + 1), - Cluster::bumpEpoch), 10); - return Gens.oneOf(possible); - } - - @Override - public void destroyState(Cluster cluster) - { - cluster.processAll(); - cluster.validate(true); - } - }); + private static SimpleCommand processSome(RandomSource rs) + { + return new SimpleCommand("Process Some", + c -> {//noinspection StatementWithEmptyBody + for (int i = 0, attempts = rs.nextInt(1, 100); i < attempts && c.processOne(); i++) + { + } + }); } private static class SimpleCommand implements UnitCommand diff --git a/test/unit/org/apache/cassandra/utils/StatefulRangeTreeTest.java b/test/unit/org/apache/cassandra/utils/StatefulRangeTreeTest.java index e3e471b550..139f172dfb 100644 --- a/test/unit/org/apache/cassandra/utils/StatefulRangeTreeTest.java +++ b/test/unit/org/apache/cassandra/utils/StatefulRangeTreeTest.java @@ -21,7 +21,6 @@ package org.apache.cassandra.utils; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.TreeSet; @@ -35,12 +34,12 @@ import accord.primitives.Range; import accord.utils.Gen; import accord.utils.Gens; import accord.utils.Property.Command; -import accord.utils.Property.Commands; import accord.utils.Property.UnitCommand; import accord.utils.RandomSource; import org.apache.cassandra.service.accord.RangeTreeRangeAccessor; import org.assertj.core.api.Assertions; +import static accord.utils.Property.commands; import static accord.utils.Property.stateful; public class StatefulRangeTreeTest @@ -61,7 +60,7 @@ public class StatefulRangeTreeTest /** * Stateful test for RTree. - * + *

* This test is very similar to {@link RangeTreeTest#test} but is fully mutable, so can not * use the immutable search trees (else rebuidling becomes a large cost). Both tests should exist as they use different * models, which helps build confidence that the RTree does the correct thing; that test also covers start and end @@ -70,57 +69,25 @@ public class StatefulRangeTreeTest @Test public void test() { - stateful().check(new Commands() - { - @Override - public Gen genInitialState() - { - return rs -> { - Gen rangeGen = rangeGen(rs); - int numChildren = NUM_CHILDREN_GEN.nextInt(rs); - int sizeTarget = SIZE_TARGET_DISTRIBUTION.next(rs).filter(s -> s > numChildren).nextInt(rs); - int createWeight = rs.nextInt(1, 100); - int updateWeight = rs.nextInt(1, 20); - int deleteWeight = rs.nextInt(1, 20); - int clearWeight = rs.nextInt(0, 2); // either disabled or enabled with weight=1 - int readWeight = rs.nextInt(1, 20); - return new State(sizeTarget, numChildren, - TOKEN_DISTRIBUTION.next(rs), rangeGen, - createWeight, updateWeight, deleteWeight, clearWeight, readWeight); - }; - } - - @Override - public Sut createSut(State state) - { - return new Sut(state.sizeTarget, state.numChildren); - } - - @Override - public Gen> commands(State state) - { - Map>, Integer> possible = new LinkedHashMap<>(); - possible.put(rs -> new Create(state.newRange(rs), SMALL_INT_GEN.nextInt(rs)), state.createWeight); - possible.put(rs -> new Read(state.newRange(rs)), state.readWeight); - possible.put(rs -> new KeyRead(IntKey.routing(state.tokenGen.nextInt(rs))), state.readWeight); - possible.put(rs -> new RangeRead(state.rangeGen.next(rs)), state.readWeight); - possible.put(ignore -> Iterate.instance, state.readWeight); - possible.put(ignore -> Clear.instance, state.clearWeight); - if (!state.uniqRanges.isEmpty()) - { - possible.put(rs -> new Read(rs.pickOrderedSet(state.uniqRanges)), state.readWeight); - possible.put(rs -> { - Range range = rs.pickOrderedSet(state.uniqRanges); - int token = rs.nextInt(((IntKey.Routing) range.start()).key, ((IntKey.Routing) range.end()).key) + 1; - return new KeyRead(IntKey.routing(token)); - }, state.readWeight); - possible.put(rs -> new RangeRead(rs.pickOrderedSet(state.uniqRanges)), state.readWeight); - possible.put(rs -> new Update(rs.pickOrderedSet(state.uniqRanges), SMALL_INT_GEN.nextInt(rs)), state.updateWeight); - possible.put(rs -> new Delete(rs.pickOrderedSet(state.uniqRanges)), state.deleteWeight); - } - return Gens.oneOf(possible); - } - }); + stateful().check(commands(() -> State::new, state -> new Sut(state.sizeTarget, state.numChildren)) + .add((rs, state) -> new Create(state.newRange(rs), SMALL_INT_GEN.nextInt(rs))) + .add((rs, state) -> new Read(state.newRange(rs))) + .add((rs, state) -> new KeyRead(IntKey.routing(state.tokenGen.nextInt(rs)))) + .add((rs, state) -> new RangeRead(state.rangeGen.next(rs))) + .add(Iterate.instance) + .add(Clear.instance) + .addAllIf(state -> !state.uniqRanges.isEmpty(), + b -> b.add((rs, state) -> new Read(rs.pickOrderedSet(state.uniqRanges))) + .add((rs, state) -> { + Range range = rs.pickOrderedSet(state.uniqRanges); + int token = rs.nextInt(((IntKey.Routing) range.start()).key, ((IntKey.Routing) range.end()).key) + 1; + return new KeyRead(IntKey.routing(token)); + }) + .add((rs, state) -> new RangeRead(rs.pickOrderedSet(state.uniqRanges))) + .add((rs, state) -> new Update(rs.pickOrderedSet(state.uniqRanges), SMALL_INT_GEN.nextInt(rs))) + .add((rs, state) -> new Delete(rs.pickOrderedSet(state.uniqRanges))) + ) + .build()); } private static Gen rangeGen(RandomSource rand) @@ -401,6 +368,7 @@ public class StatefulRangeTreeTest static class Iterate extends AbstractRead> { static final Iterate instance = new Iterate(); + public Iterate() { super(COMPARATOR); @@ -432,27 +400,21 @@ public class StatefulRangeTreeTest private final int sizeTarget, numChildren; private final Gen.IntGen tokenGen; private final Gen rangeGen; - private final int createWeight, updateWeight, deleteWeight, clearWeight, readWeight; - private State(int sizeTarget, int numChildren, - Gen.IntGen tokenGen, Gen rangeGen, - int createWeight, int updateWeight, int deleteWeight, int clearWeight, int readWeight) + private State(RandomSource rs) { - this.sizeTarget = sizeTarget; - this.numChildren = numChildren; - this.tokenGen = tokenGen; - this.rangeGen = rangeGen; - this.createWeight = createWeight; - this.updateWeight = updateWeight; - this.deleteWeight = deleteWeight; - this.clearWeight = clearWeight; - this.readWeight = readWeight; + this.numChildren = NUM_CHILDREN_GEN.nextInt(rs); + this.sizeTarget = SIZE_TARGET_DISTRIBUTION.next(rs).filter(s -> s > numChildren).nextInt(rs); + this.tokenGen = TOKEN_DISTRIBUTION.next(rs); + this.rangeGen = rangeGen(rs); } public Range newRange(RandomSource rs) { Range range; - while ((uniqRanges.contains(range = rangeGen.next(rs)))) {} + while ((uniqRanges.contains(range = rangeGen.next(rs)))) + { + } return range; }