natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Forward port randomized testing improvements from cep-15-accord to trunk 

patch by David Capwell; reviewed by Caleb Rackliffe for CASSANDRA-20253
This commit is contained in:
David Capwell 2025-01-28 10:46:51 -08:00
parent 8e5bdc6d4b
commit 10c0481f46
16 changed files with 2320 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
View File

@ -554,7 +554,7 @@ public enum CassandraRelevantProperties
TCM_USE_NO_OP_REPLICATOR("cassandra.test.use_no_op_replicator", "false"),
TEST_BBFAILHELPER_ENABLED("test.bbfailhelper.enabled"),
TEST_BLOB_SHARED_SEED("cassandra.test.blob.shared.seed"),
TEST_BLOB_SHARED_SEED("cassandra.test.blob.shared.seed", "42"),
TEST_BYTEMAN_TRANSFORMATIONS_DEBUG("cassandra.test.byteman.transformations.debug"),
TEST_CASSANDRA_KEEPBRIEFBRIEF("cassandra.keepBriefBrief"),
TEST_CASSANDRA_RELEVANT_PROPERTIES("org.apache.cassandra.conf.CassandraRelevantPropertiesTest"),

4
src/java/org/apache/cassandra/dht/ReversedLongLocalPartitioner.java
View File

@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.primitives.Longs;
import org.apache.cassandra.db.CachedHashDecoratedKey;
@ -155,7 +156,8 @@ public class ReversedLongLocalPartitioner implements IPartitioner
return LongType.instance;
}
private static class ReversedLongLocalToken extends Token
@VisibleForTesting
public static class ReversedLongLocalToken extends Token
{
private final long token;

7
src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
View File

@ -25,6 +25,7 @@ import java.util.stream.Stream;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.collect.Iterables;
@ -84,6 +85,12 @@ public final class KeyspaceMetadata implements SchemaElement
this.replicationStrategy = AbstractReplicationStrategy.createReplicationStrategy(keyspaceName, params.replication);
}
@VisibleForTesting
public static KeyspaceMetadata createUnsafe(String keyspaceName, Kind kind, KeyspaceParams params, Tables tables, Views views, Types types, UserFunctions functions)
{
return new KeyspaceMetadata(keyspaceName, kind, params, tables, views, types, functions);
}
public static KeyspaceMetadata create(String name, KeyspaceParams params)
{
return new KeyspaceMetadata(name, Kind.REGULAR, params, Tables.none(), Views.none(), Types.none(), UserFunctions.none());

6
src/java/org/apache/cassandra/schema/ReplicationParams.java
View File

@ -64,6 +64,12 @@ public final class ReplicationParams
this.options = ImmutableMap.copyOf(options);
}
@VisibleForTesting
public static ReplicationParams fromStrategy(AbstractReplicationStrategy strategy)
{
return new ReplicationParams(strategy.getClass(), strategy.configOptions);
}
public static ReplicationParams local()
{
return new ReplicationParams(LocalStrategy.class, ImmutableMap.of());

18
test/distributed/org/apache/cassandra/fuzz/snapshots/SnapshotsTest.java
View File

@ -132,21 +132,21 @@ public class SnapshotsTest
case 1:
if (!state.truncatedSnapshots.isEmpty())
{
keyspace = state.rs.pick(state.truncatedSnapshots.keySet()).split("\\.")[0];
keyspace = state.rs.pickUnorderedSet(state.truncatedSnapshots.keySet()).split("\\.")[0];
picked = true;
}
break;
case 2:
if (!state.droppedSnapshots.isEmpty())
{
keyspace = state.rs.pick(state.droppedSnapshots).split("\\.")[0];
keyspace = state.rs.pickUnorderedSet(state.droppedSnapshots).split("\\.")[0];
picked = true;
}
break;
case 3:
if (!state.snapshots.isEmpty())
{
keyspace = state.rs.pick(state.snapshots).getKeyspaceName();
keyspace = state.rs.pickUnorderedSet(state.snapshots).getKeyspaceName();
picked = true;
}
break;
@ -630,7 +630,7 @@ public class SnapshotsTest
case 1:
if (!state.truncatedSnapshots.isEmpty())
{
String randomKsTb = state.rs.pick(state.truncatedSnapshots.keySet());
String randomKsTb = state.rs.pickUnorderedSet(state.truncatedSnapshots.keySet());
Integer numberOfTruncatedSnapshots = state.truncatedSnapshots.get(randomKsTb);
if (numberOfTruncatedSnapshots == 1)
state.truncatedSnapshots.remove(randomKsTb);
@ -645,14 +645,14 @@ public class SnapshotsTest
case 2:
if (!state.droppedSnapshots.isEmpty())
{
state.droppedSnapshots.remove(state.rs.pick(state.droppedSnapshots));
state.droppedSnapshots.remove(state.rs.pickUnorderedSet(state.droppedSnapshots));
picked = true;
}
break;
case 3:
if (!state.snapshots.isEmpty())
{
TestSnapshot pickedSnapshot = state.rs.pick(state.snapshots);
TestSnapshot pickedSnapshot = state.rs.pickUnorderedSet(state.snapshots);
state.snapshots.remove(pickedSnapshot);
picked = true;
}
@ -817,7 +817,7 @@ public class SnapshotsTest
if (withoutInBuilt.isEmpty())
return Optional.empty();
else
return Optional.of(rs.pick(withoutInBuilt));
return Optional.of(rs.pickUnorderedSet(withoutInBuilt));
}
public Optional<String> pickRandomKeyspace()
@ -825,7 +825,7 @@ public class SnapshotsTest
if (schema.keySet().isEmpty())
return Optional.empty();
else
return Optional.of(rs.pick(schema.keySet()));
return Optional.of(rs.pickUnorderedSet(schema.keySet()));
}
public Optional<Pair<String, String>> pickRandomTable(boolean inBuiltIncluded)
@ -864,7 +864,7 @@ public class SnapshotsTest
if (keyspacesWithTables.isEmpty())
return Optional.empty();
String randomKeyspaceWithTables = rs.pick(keyspacesWithTables.keySet());
String randomKeyspaceWithTables = rs.pickUnorderedSet(keyspacesWithTables.keySet());
List<String> tables = keyspacesWithTables.get(randomKeyspaceWithTables);
String randomTable = rs.pick(tables);

244
test/unit/accord/utils/Gens.java
View File

@ -29,6 +29,7 @@ import java.util.EnumMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
@ -43,6 +44,8 @@ import java.util.stream.Stream;
import com.google.common.collect.Iterables;
import accord.utils.random.Picker;
public class Gens {
private Gens() {
}
@ -88,6 +91,63 @@ public class Gens {
return rs -> gen.next(rs).next(rs);
}
public static <T> OneOfBuilder<T> oneOf()
{
return new OneOfBuilder<>();
}
public static class OneOfBuilder<T>
{
private final Map<Gen<T>, Integer> weighted = new LinkedHashMap<>();
private final Set<Gen<T>> unweighted = new LinkedHashSet<>();
private Gen.IntGen unknownWeightGen = Gens.ints().between(1, 10);
public OneOfBuilder<T> add(Gen<T> gen)
{
unweighted.add(gen);
return this;
}
public OneOfBuilder<T> add(int weight, Gen<T> gen)
{
weighted.put(gen, weight);
return this;
}
public OneOfBuilder<T> unknownWeights(Gen.IntGen gen)
{
this.unknownWeightGen = gen;
return this;
}
public Gen<Gen<T>> buildWithDynamicWeights()
{
if (unweighted.isEmpty())
{
Gen<T> gen = build();
return i -> gen;
}
return rs -> {
Map<Gen<T>, Integer> commands = new LinkedHashMap<>();
commands.putAll(weighted);
for (var gen : unweighted)
commands.put(gen, unknownWeightGen.nextInt(rs));
var top = pick(commands);
return rs2 -> top.next(rs2).next(rs2);
};
}
public Gen<T> build()
{
Map<Gen<T>, Integer> commands = new LinkedHashMap<>();
commands.putAll(weighted);
for (var gen : unweighted)
commands.put(gen, 1);
var top = pick(commands);
return rs -> top.next(rs).next(rs);
}
}
public static Gen.IntGen pickInt(int... ts)
{
return rs -> ts[rs.nextInt(0, ts.length)];
@ -225,6 +285,95 @@ public class Gens {
};
}
public static Gen<Gen.IntGen> randomWeights(int[] array)
{
return rs -> {
float[] weights = Picker.randomWeights(rs, array.length);
return r -> array[index(r, weights)];
};
}
public static Gen<Gen.LongGen> randomWeights(long[] array)
{
return rs -> {
float[] weights = Picker.randomWeights(rs, array.length);
return r -> array[index(r, weights)];
};
}
public static <T> Gen<Gen<T>> randomWeights(T[] array)
{
return rs -> {
float[] weights = Picker.randomWeights(rs, array.length);
return r -> array[index(r, weights)];
};
}
public static <T> Gen<Gen<T>> randomWeights(List<T> array)
{
return rs -> {
float[] weights = Picker.randomWeights(rs, array.size());
return r -> array.get(index(r, weights));
};
}
private static int index(RandomSource rs, float[] weights)
{
int i = Arrays.binarySearch(weights, rs.nextFloat());
if (i < 0) i = -1 - i;
return i;
}
public static Gen<Gen.IntGen> mixedDistribution(int minInclusive, int maxExclusive, int numBuckets)
{
int domainSize = (maxExclusive - minInclusive);
if (domainSize < 0)
throw new IllegalArgumentException("Range is too large; min=" + minInclusive + ", max=" + maxExclusive);
if (numBuckets <= 0 || numBuckets > domainSize)
throw new IllegalArgumentException("Num buckets must be between 1 and " + domainSize + "; given " + numBuckets);
int[] bucket, indexes;
bucket = new int[numBuckets];
int delta = domainSize / numBuckets;
for (int i = 0; i < numBuckets; i++)
bucket[i] = minInclusive + i * delta;
indexes = IntStream.range(0, bucket.length).toArray();
Gen<Gen.IntGen> indexDistro = mixedDistribution(indexes);
return rs -> {
Gen.IntGen indexGen = indexDistro.next(rs);
switch (rs.nextInt(0, 2))
{
case 0: // uniform
{
return r -> {
int idx = indexGen.next(rs);
int start = bucket[idx];
int end = idx == bucket.length - 1 ? maxExclusive : bucket[idx + 1];
return r.nextInt(start, end);
};
}
case 1: // median biased
{
int medians[] = new int[bucket.length];
for (int i = 0; i < medians.length; i++)
{
int start = bucket[i];
int end = i == bucket.length - 1 ? maxExclusive : bucket[i + 1];
medians[i] = rs.nextInt(start, end);
}
return r -> {
int idx = indexGen.next(rs);
int start = bucket[idx];
int end = idx == bucket.length - 1 ? maxExclusive : bucket[idx + 1];
int median = medians[idx];
return r.nextBiasedInt(start, median, end);
};
}
default:
throw new AssertionError();
}
};
}
public static Gen<Gen.IntGen> mixedDistribution(int minInclusive, int maxExclusive)
{
int domainSize = (maxExclusive - minInclusive + 1);
@ -246,11 +395,14 @@ public class Gens {
indexes = null;
}
return rs -> {
switch (rs.nextInt(0, 2))
switch (rs.nextInt(0, 4))
{
case 0: // uniform
return r -> r.nextInt(minInclusive, maxExclusive);
case 1: // zipf
case 1: // median biased
int median = rs.nextInt(minInclusive, maxExclusive);
return r -> r.nextBiasedInt(minInclusive, median, maxExclusive);
case 2: // zipf
if (indexes == null)
return Gens.pickZipf(rs.nextBoolean() ? reverseAndCopy(array) : array);
return Gens.pickZipf(rs.nextBoolean() ? reverseAndCopy(indexes) : indexes).mapAsInt((r, index) -> {
@ -258,6 +410,14 @@ public class Gens {
int end = index == array.length - 1 ? maxExclusive : array[index + 1];
return r.nextInt(start, end);
});
case 3: // random weight
if (indexes == null)
return randomWeights(array).next(rs);
return randomWeights(indexes).next(rs).mapAsInt((r, index) -> {
int start = array[index];
int end = index == array.length - 1 ? maxExclusive : array[index + 1];
return r.nextInt(start, end);
});
default:
throw new AssertionError();
}
@ -298,11 +458,14 @@ public class Gens {
indexes = null;
}
return rs -> {
switch (rs.nextInt(0, 2))
switch (rs.nextInt(0, 4))
{
case 0: // uniform
return r -> r.nextLong(minInclusive, maxExclusive);
case 1: // zipf
case 1: // median biased
long median = rs.nextLong(minInclusive, maxExclusive);
return r -> r.nextBiasedLong(minInclusive, median, maxExclusive);
case 2: // zipf
if (indexes == null)
return Gens.pickZipf(rs.nextBoolean() ? reverseAndCopy(array) : array);
return Gens.pickZipf(rs.nextBoolean() ? reverseAndCopy(indexes) : indexes).mapAsLong((r, index) -> {
@ -310,6 +473,14 @@ public class Gens {
long end = index == array.length - 1 ? maxExclusive : array[index + 1];
return r.nextLong(start, end);
});
case 3: // random weight
if (indexes == null)
return randomWeights(array).next(rs);
return randomWeights(indexes).next(rs).mapAsLong((r, index) -> {
long start = array[index];
long end = index == array.length - 1 ? maxExclusive : array[index + 1];
return r.nextLong(start, end);
});
default:
throw new AssertionError();
}
@ -336,11 +507,14 @@ public class Gens {
public static <T> Gen<Gen<T>> mixedDistribution(List<T> list)
{
return rs -> {
switch (rs.nextInt(0, 2))
switch (rs.nextInt(0, 4))
{
case 0: // uniform
return r -> list.get(rs.nextInt(0, list.size()));
case 1: // zipf
case 1: // median biased
int median = rs.nextInt(0, list.size());
return r -> list.get(r.nextBiasedInt(0, median, list.size()));
case 2: // zipf
List<T> array = list;
if (rs.nextBoolean())
{
@ -348,12 +522,55 @@ public class Gens {
Collections.reverse(array);
}
return pickZipf(array);
case 3: // random weight
return randomWeights(list).next(rs);
default:
throw new AssertionError();
}
};
}
public static <T> Gen<Gen.IntGen> mixedDistribution(int[] list)
{
return rs -> {
switch (rs.nextInt(0, 4))
{
case 0: // uniform
return r -> list[rs.nextInt(0, list.length)];
case 1: // median biased
int median = rs.nextInt(0, list.length);
return r -> list[r.nextBiasedInt(0, median, list.length)];
case 2: // zipf
int[] array = list;
if (rs.nextBoolean())
{
array = Arrays.copyOf(array, array.length);
reverse(array);
}
return pickZipf(array);
case 3: // random weight
return randomWeights(list).next(rs);
default:
throw new AssertionError();
}
};
}
/**
* This is a change from accord as that uses {@link accord.utils.Utils#reverse}, which doesn't exist in this forward port.
*
* To avoid adding another class and merge conflicts to cep-15-accord, this method was inlined
*/
private static void reverse(int[] array)
{
for (int i = 0; i < array.length / 2; i++)
{
int tmp = array[i];
array[i] = array[array.length- 1 - i];
array[array.length - 1 - i] = tmp;
}
}
public static Gen<char[]> charArray(Gen.IntGen sizes, char[] domain)
{
return charArray(sizes, domain, (a, b) -> true);
@ -535,6 +752,11 @@ public class Gens {
{
return Gens.mixedDistribution(minInclusive, maxExclusive);
}
public Gen<Gen.IntGen> mixedDistribution(int minInclusive, int maxExclusive, int numBuckets)
{
return Gens.mixedDistribution(minInclusive, maxExclusive, numBuckets);
}
}
public static class LongDSL {
@ -843,8 +1065,14 @@ public class Gens {
if (!bestEffort)
{
T value;
while (!seen.add((value = fn.next(random)))) {}
return value;
// 10k attempts
for (int i = 0; i < 10_000; i++)
{
if (seen.add((value = fn.next(random))))
return value;
}
throw new IllegalArgumentException("Could not generate a unique value after 10k attempts");
}
else
{

7
test/unit/accord/utils/Property.java
View File

@ -555,7 +555,7 @@ public class Property
default void process(State state, SystemUnderTest sut) throws Throwable
{
checkPostconditions(state, apply(state),
sut, run(sut));
sut, run(sut));
}
}
@ -896,6 +896,11 @@ public class Property
return addIf(predicate, (rs, state) -> cmd.next(rs));
}
public CommandsBuilder<State, SystemUnderTest> addIf(Predicate<State> predicate, Command<State, SystemUnderTest, ?> cmd)
{
return addIf(predicate, (rs, state) -> cmd);
}
public CommandsBuilder<State, SystemUnderTest> addIf(Predicate<State> predicate, Setup<State, SystemUnderTest> cmd)
{
if (conditionalCommands == null)

217
test/unit/accord/utils/RandomSource.java
View File

@ -20,14 +20,22 @@ package accord.utils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.NavigableSet;
import java.util.Random;
import java.util.Set;
import java.util.stream.DoubleStream;
import java.util.stream.IntStream;
import java.util.stream.LongStream;
import java.util.SortedSet;
import java.util.function.BooleanSupplier;
import java.util.function.IntSupplier;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
import com.google.common.collect.Iterables;
import accord.utils.random.Picker;
// TODO (expected): merge with C* RandomSource
public interface RandomSource
{
static RandomSource wrap(Random random)
@ -38,14 +46,40 @@ public interface RandomSource
void nextBytes(byte[] bytes);
boolean nextBoolean();
int nextInt();
default int nextInt(int maxExclusive)
default BooleanSupplier uniformBools() { return this::nextBoolean; }
default BooleanSupplier biasedUniformBools(float chance) { return () -> decide(chance); }
default Supplier<BooleanSupplier> biasedUniformBoolsSupplier(float minChance)
{
return nextInt(0, maxExclusive);
return () -> {
float chance = minChance + (1 - minChance)*nextFloat();
return () -> decide(chance);
};
}
/**
* Returns true with a probability of {@code chance}. This is logically the same as
* <pre>{@code nextFloat() < chance}</pre>
*
* @param chance cumulative probability in range [0..1]
*/
default boolean decide(float chance)
{
return nextFloat() < chance;
}
/**
* Returns true with a probability of {@code chance}. This is logically the same as
* <pre>{@code nextDouble() < chance}</pre>
*
* @param chance cumulative probability in range [0..1]
*/
default boolean decide(double chance)
{
return nextDouble() < chance;
}
int nextInt();
default int nextInt(int maxExclusive) { return nextInt(0, maxExclusive); }
default int nextInt(int minInclusive, int maxExclusive)
{
// this is diff behavior than ThreadLocalRandom, which returns nextInt
@ -74,29 +108,47 @@ public interface RandomSource
}
return result;
}
default IntStream ints()
default int nextBiasedInt(int minInclusive, int median, int maxExclusive)
{
return IntStream.generate(this::nextInt);
checkBiasedUniform(minInclusive, median, maxExclusive);
int range = Math.max(maxExclusive - median, median - minInclusive) * 2;
int next = nextInt(range) - range/2;
next += median;
return next >= median ? next < maxExclusive ? next : nextInt(median, maxExclusive)
: next >= minInclusive ? next : minInclusive == median ? median : nextInt(minInclusive, median);
}
default IntStream ints(int maxExclusive)
default IntSupplier uniformInts(int minInclusive, int maxExclusive) { return () -> nextInt(minInclusive, maxExclusive); }
default IntSupplier biasedUniformInts(int minInclusive, int median, int maxExclusive)
{
return IntStream.generate(() -> nextInt(maxExclusive));
checkBiasedUniform(minInclusive, median, maxExclusive);
return () -> nextBiasedInt(minInclusive, median, maxExclusive);
}
default IntStream ints(int minInclusive, int maxExclusive)
default Supplier<IntSupplier> biasedUniformIntsSupplier(int absoluteMinInclusive, int absoluteMaxExclusive, int minMedian, int maxMedian, int minRange, int maxRange)
{
return IntStream.generate(() -> nextInt(minInclusive, maxExclusive));
return biasedUniformIntsSupplier(absoluteMinInclusive, absoluteMaxExclusive, minMedian, (minMedian+maxMedian)/2, maxMedian, minRange, (minRange+maxRange)/2, maxRange);
}
default Supplier<IntSupplier> biasedUniformIntsSupplier(int absoluteMinInclusive, int absoluteMaxExclusive, int minMedian, int medianMedian, int maxMedian, int minRange, int medianRange, int maxRange)
{
checkBiasedUniform(minMedian, medianMedian, maxMedian);
checkBiasedUniform(minRange, medianRange, maxRange);
if (minMedian < absoluteMinInclusive)
throw new IllegalArgumentException(String.format("absoluteMin (%s) should be less than or equal to minMedian (%s)", absoluteMinInclusive, minMedian));
if (maxMedian > absoluteMaxExclusive)
throw new IllegalArgumentException(String.format("absoluteMax (%s) should be greater than or equal to maxMedian (%s)", absoluteMaxExclusive, maxMedian));
if (minRange < 1)
throw new IllegalArgumentException(String.format("minRange (%s) should be greater than or equal to 1", minRange));
return () -> {
int median = nextBiasedInt(minMedian, medianMedian, maxMedian);
int minInclusive = Math.max(absoluteMinInclusive, median - nextBiasedInt(minRange, medianRange, maxRange)/2);
int maxExclusive = Math.min(absoluteMaxExclusive, median + (nextBiasedInt(minRange, medianRange, maxRange)+1)/2);
return biasedUniformInts(minInclusive, median, maxExclusive);
};
}
long nextLong();
default long nextLong(long maxExclusive)
{
return nextLong(0, maxExclusive);
}
default long nextLong(long maxExclusive) { return nextLong(0, maxExclusive); }
default long nextLong(long minInclusive, long maxExclusive)
{
// this is diff behavior than ThreadLocalRandom, which returns nextLong
@ -125,31 +177,57 @@ public interface RandomSource
}
return result;
}
default LongStream longs()
default long nextBiasedLong(long minInclusive, long median, long maxExclusive)
{
return LongStream.generate(this::nextLong);
checkBiasedUniform(minInclusive, median, maxExclusive);
long range = Math.max(maxExclusive - median, median - minInclusive) * 2;
long next = nextLong(range) - range/2;
next += median;
return next >= median ? next < maxExclusive ? next : nextLong(median, maxExclusive)
: next >= minInclusive ? next : minInclusive == median ? median : nextLong(minInclusive, median);
}
default LongStream longs(long maxExclusive)
default LongSupplier uniformLongs(long minInclusive, long maxExclusive) { return () -> nextLong(minInclusive, maxExclusive); }
default LongSupplier biasedUniformLongs(long minInclusive, long median, long maxExclusive)
{
return LongStream.generate(() -> nextLong(maxExclusive));
checkBiasedUniform(minInclusive, median, maxExclusive);
return () -> nextBiasedLong(minInclusive, median, maxExclusive);
}
default Supplier<LongSupplier> biasedUniformLongsSupplier(long absoluteMinInclusive, long absoluteMaxExclusive, long minMedian, long maxMedian, long minRange, long maxRange)
{
return biasedUniformLongsSupplier(absoluteMinInclusive, absoluteMaxExclusive, minMedian, (minMedian+maxMedian)/2, maxRange, minRange, (minRange+maxRange)/2, maxRange);
}
default Supplier<LongSupplier> biasedUniformLongsSupplier(long absoluteMinInclusive, long absoluteMaxExclusive, long minMedian, long medianMedian, long maxMedian, long minRange, long medianRange, long maxRange)
{
checkBiasedUniform(minMedian, medianMedian, maxMedian);
checkBiasedUniform(minRange, medianRange, maxRange);
if (minMedian < absoluteMinInclusive)
throw new IllegalArgumentException(String.format("absoluteMin (%s) should be less than or equal to minMedian (%s)", absoluteMinInclusive, minMedian));
if (maxMedian > absoluteMaxExclusive)
throw new IllegalArgumentException(String.format("absoluteMax (%s) should be greater than or equal to maxMedian (%s)", absoluteMaxExclusive, maxMedian));
if (minRange < 1)
throw new IllegalArgumentException(String.format("minRange (%s) should be greater than or equal to 1", minRange));
return () -> {
long median = nextBiasedLong(minMedian, medianMedian, maxMedian);
long minInclusive = Math.max(absoluteMinInclusive, median - nextBiasedLong(minRange, medianRange, maxRange)/2);
long maxExclusive = Math.min(absoluteMaxExclusive, median + (1+nextBiasedLong(minRange, medianRange, maxRange))/2);
return biasedUniformLongs(minInclusive, median, maxExclusive);
};
}
default LongStream longs(long minInclusive, long maxExclusive)
static void checkBiasedUniform(long minInclusive, long median, long maxExclusive)
{
return LongStream.generate(() -> nextLong(minInclusive, maxExclusive));
if (minInclusive > median)
throw new IllegalArgumentException(String.format("Min (%s) should be equal to or less than median (%d).", minInclusive, median));
if (median >= maxExclusive)
throw new IllegalArgumentException(String.format("Median (%s) should be less than max (%d).", median, maxExclusive));
}
float nextFloat();
double nextDouble();
default double nextDouble(double maxExclusive)
{
return nextDouble(0, maxExclusive);
}
default double nextDouble(double maxExclusive) { return nextDouble(0, maxExclusive); }
default double nextDouble(double minInclusive, double maxExclusive)
{
if (minInclusive >= maxExclusive)
@ -162,21 +240,6 @@ public interface RandomSource
return result;
}
default DoubleStream doubles()
{
return DoubleStream.generate(this::nextDouble);
}
default DoubleStream doubles(double maxExclusive)
{
return DoubleStream.generate(() -> nextDouble(maxExclusive));
}
default DoubleStream doubles(double minInclusive, double maxExclusive)
{
return DoubleStream.generate(() -> nextDouble(minInclusive, maxExclusive));
}
double nextGaussian();
default int pickInt(int first, int second, int... rest)
@ -227,13 +290,32 @@ public interface RandomSource
return array[nextInt(offset, offset + length)];
}
default <T extends Comparable<T>> T pick(Set<T> set)
default <T> T pickOrderedSet(SortedSet<T> set)
{
int offset = nextInt(0, set.size());
return Iterables.get(set, offset);
}
default <T> T pickOrderedSet(LinkedHashSet<T> set)
{
int offset = nextInt(0, set.size());
return Iterables.get(set, offset);
}
default <T extends Enum<T>> T pickOrderedSet(EnumSet<T> set)
{
int offset = nextInt(0, set.size());
return Iterables.get(set, offset);
}
default <T extends Comparable<? super T>> T pickUnorderedSet(Set<T> set)
{
if (set instanceof SortedSet)
return pickOrderedSet((SortedSet<T>) set);
List<T> values = new ArrayList<>(set);
// Non-ordered sets may have different iteration order on different environments, which would make a seed produce different histories!
// To avoid such a problem, make sure to apply a deterministic function (sort).
if (!(set instanceof NavigableSet))
values.sort(Comparator.naturalOrder());
values.sort(Comparator.naturalOrder());
return pick(values);
}
@ -266,32 +348,13 @@ public interface RandomSource
return values.get(nextInt(offset, offset + length));
}
default <T> Supplier<T> randomWeightedPicker(T[] objects) { return Picker.WeightedObjectPicker.randomWeighted(this, objects); }
default <T> Supplier<T> randomWeightedPicker(T[] objects, float[] bias) { return Picker.WeightedObjectPicker.randomWeighted(this, objects, bias); }
default <T> Supplier<T> weightedPicker(T[] objects, float[] proportionalWeights) { return Picker.WeightedObjectPicker.weighted(this, objects, proportionalWeights); }
void setSeed(long seed);
RandomSource fork();
/**
* Returns true with a probability of {@code chance}. This logic is logically the same as
* <pre>{@code nextFloat() < chance}</pre>
*
* @param chance cumulative probability in range [0..1]
*/
default boolean decide(float chance)
{
return nextFloat() < chance;
}
/**
* Returns true with a probability of {@code chance}. This logic is logically the same as
* <pre>{@code nextDouble() < chance}</pre>
*
* @param chance cumulative probability in range [0..1]
*/
default boolean decide(double chance)
{
return nextDouble() < chance;
}
default long reset()
{
long seed = nextLong();

118
test/unit/accord/utils/random/Picker.java Normal file
View File

@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package accord.utils.random;
import java.util.Arrays;
import java.util.function.Supplier;
import accord.utils.Invariants;
import accord.utils.RandomSource;
public class Picker
{
public static float[] randomWeights(RandomSource random, int length)
{
float[] weights = new float[length - 1];
float sum = 0;
for (int i = 0 ; i < weights.length ; ++i)
weights[i] = sum += random.nextFloat();
sum += random.nextFloat();
for (int i = 0 ; i < weights.length ; ++i)
weights[i] /= sum;
return weights;
}
static abstract class Weighted
{
final RandomSource random;
final float[] weights;
public Weighted(RandomSource random, float[] weights)
{
this.random = random;
this.weights = weights;
}
static float[] randomWeights(RandomSource random, float[] bias)
{
float[] weights = new float[bias.length - 1];
float sum = 0;
for (int i = 0 ; i < weights.length ; ++i)
weights[i] = sum += random.nextFloat() * bias[i];
sum += random.nextFloat() * bias[weights.length];
for (int i = 0 ; i < weights.length ; ++i)
weights[i] /= sum;
return weights;
}
static float[] normaliseWeights(float[] input)
{
float[] output = new float[input.length - 1];
float sum = 0;
for (int i = 0 ; i < output.length ; ++i)
output[i] = sum += input[i];
sum += input[output.length];
for (int i = 0 ; i < output.length ; ++i)
output[i] /= sum;
return output;
}
int pickIndex()
{
int i = Arrays.binarySearch(weights, random.nextFloat());
if (i < 0) i = -1 - i;
return i;
}
}
public static class WeightedObjectPicker<T> extends Weighted implements Supplier<T>
{
final T[] values;
private WeightedObjectPicker(RandomSource random, T[] values, float[] weights)
{
super(random, weights);
this.values = values;
}
@Override
public T get()
{
return values[pickIndex()];
}
public static <T> WeightedObjectPicker<T> randomWeighted(RandomSource random, T[] values)
{
return new WeightedObjectPicker<>(random, values, Picker.randomWeights(random, values.length));
}
public static <T> WeightedObjectPicker<T> randomWeighted(RandomSource random, T[] values, float[] bias)
{
Invariants.checkArgument(values.length == bias.length);
return new WeightedObjectPicker<>(random, values, randomWeights(random, bias));
}
public static <T> WeightedObjectPicker<T> weighted(RandomSource random, T[] values, float[] proportionalWeights)
{
Invariants.checkArgument(values.length == proportionalWeights.length);
return new WeightedObjectPicker<>(random, values, normaliseWeights(proportionalWeights));
}
}
}

6
test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java
View File

@ -105,13 +105,11 @@ public class RandomSchemaTest extends CQLTester.InMemory
.withMaxDepth(2)
.withDefaultSetKey(withoutUnsafeEquality)
.withoutTypeKinds(AbstractTypeGenerators.TypeKind.COUNTER)
.withUDTNames(udtName)
.build())
.withUDTNames(udtName))
.withPartitionColumnsCount(1)
.withPrimaryColumnTypeGen(new TypeGenBuilder(withoutUnsafeEquality)
// map of vector of map crossed the size cut-off for one of the tests, so changed max depth from 2 to 1, so we can't have the second map
.withMaxDepth(1)
.build())
.withMaxDepth(1))
.withClusteringColumnsBetween(1, 2)
.withRegularColumnsBetween(1, 5)
.withStaticColumnsBetween(0, 2)

2
test/unit/org/apache/cassandra/repair/FailedAckTest.java
View File

@ -66,7 +66,7 @@ public class FailedAckTest extends FuzzTestBase
RepairCoordinator repair = coordinator.repair(KEYSPACE, irOption(rs, coordinator, KEYSPACE, ignore -> TABLES), false);
repair.run();
// make sure the failing node is not the coordinator, else messaging isn't used
InetAddressAndPort failingAddress = rs.pick(repair.state.getNeighborsAndRanges().participants);
InetAddressAndPort failingAddress = rs.pickUnorderedSet(repair.state.getNeighborsAndRanges().participants);
Cluster.Node failingNode = cluster.nodes.get(failingAddress);
RepairStage stage = stageGen.next(rs);
switch (stage)

2
test/unit/org/apache/cassandra/repair/FuzzTestBase.java
View File

@ -325,7 +325,7 @@ public abstract class FuzzTestBase extends CQLTester.InMemory
TableId id = ids.get(i);
TableMetadata tableMetadata = new CassandraGenerators.TableMetadataBuilder().withKeyspaceName(ks).withTableName(name).withTableId(id).withTableKinds(TableMetadata.Kind.REGULAR)
// shouldn't matter, just wanted to avoid UDT as that needs more setup
.withDefaultTypeGen(AbstractTypeGenerators.builder().withTypeKinds(AbstractTypeGenerators.TypeKind.PRIMITIVE).withoutPrimitive(EmptyType.instance).build()).build().generate(qt);
.withDefaultTypeGen(AbstractTypeGenerators.builder().withTypeKinds(AbstractTypeGenerators.TypeKind.PRIMITIVE).withoutPrimitive(EmptyType.instance)).build().generate(qt);
tableBuilder.add(tableMetadata);
}
KeyspaceParams params = KeyspaceParams.simple(3);

929
test/unit/org/apache/cassandra/utils/ASTGenerators.java Normal file
View File

@ -0,0 +1,929 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.OptionalLong;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import javax.annotation.Nullable;
import com.google.common.collect.Iterables;
import org.apache.cassandra.cql3.ast.AssignmentOperator;
import org.apache.cassandra.cql3.ast.Bind;
import org.apache.cassandra.cql3.ast.CasCondition;
import org.apache.cassandra.cql3.ast.Conditional;
import org.apache.cassandra.cql3.ast.Expression;
import org.apache.cassandra.cql3.ast.Literal;
import org.apache.cassandra.cql3.ast.Mutation;
import org.apache.cassandra.cql3.ast.Operator;
import org.apache.cassandra.cql3.ast.Reference;
import org.apache.cassandra.cql3.ast.Select;
import org.apache.cassandra.cql3.ast.Symbol;
import org.apache.cassandra.cql3.ast.TableReference;
import org.apache.cassandra.cql3.ast.Txn;
import org.apache.cassandra.cql3.ast.TypeHint;
import org.apache.cassandra.cql3.ast.Value;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.MapType;
import org.apache.cassandra.db.marshal.SetType;
import org.apache.cassandra.db.marshal.ShortType;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableMetadata;
import org.quicktheories.core.Gen;
import org.quicktheories.core.RandomnessSource;
import org.quicktheories.generators.SourceDSL;
import org.quicktheories.impl.Constraint;
import static org.apache.cassandra.utils.Generators.SYMBOL_GEN;
public class ASTGenerators
{
static Gen<Value> valueGen(Object value, AbstractType<?> type)
{
Gen<Boolean> bool = SourceDSL.booleans().all();
return rnd -> bool.generate(rnd) ? new Bind(value, type) : new Literal(value, type);
}
static Gen<Value> valueGen(AbstractType<?> type)
{
Gen<?> v = AbstractTypeGenerators.getTypeSupport(type).valueGen;
return rnd -> valueGen(v.generate(rnd), type).generate(rnd);
}
private static <K, V> Map<K, V> assertDeterministic(Map<K, V> map)
{
if (map instanceof LinkedHashMap || map instanceof TreeMap || map instanceof EnumMap)
return map;
if (map.size() == 1)
return map;
throw new AssertionError("Unsupported map type: " + map.getClass());
}
/**
* Returns a list of all columns in a deterministic order. This method is similar to {@link TableMetadata#columns()},
* but that method uses a hash order, so the values could be different from host to host or jvm to jvm...
*/
public static List<ColumnMetadata> allColumnsInFixedOrder(TableMetadata metadata)
{
List<ColumnMetadata> columns = new ArrayList<>(metadata.columns().size());
metadata.allColumnsInSelectOrder().forEachRemaining(columns::add);
return columns;
}
public static Gen<AssignmentOperator> assignmentOperatorGen(EnumSet<AssignmentOperator.Kind> allowed, Expression right)
{
if (allowed.isEmpty())
throw new IllegalArgumentException("Unable to create a operator gen for empty set of allowed operators");
if (allowed.size() == 1)
return SourceDSL.arbitrary().constant(new AssignmentOperator(Iterables.getFirst(allowed, null), right));
Gen<AssignmentOperator.Kind> kind = SourceDSL.arbitrary().pick(new ArrayList<>(allowed));
return kind.map(k -> new AssignmentOperator(k, right));
}
public static Gen<Operator> operatorGen(Set<Operator.Kind> allowed, Expression e, Gen<Value> paramValueGen)
{
if (allowed.isEmpty())
throw new IllegalArgumentException("Unable to create a operator gen for empty set of allowed operators");
Gen<Operator.Kind> kindGen = allowed.size() == 1 ?
SourceDSL.arbitrary().constant(Iterables.getFirst(allowed, null))
: SourceDSL.arbitrary().pick(new ArrayList<>(allowed));
Gen<Boolean> bool = SourceDSL.booleans().all();
return rnd -> {
Gen<Value> valueGen = paramValueGen;
Operator.Kind kind = kindGen.generate(rnd);
if (kind == Operator.Kind.SUBTRACT && e.type() instanceof MapType)
{
// `map - set` not `map - map`
valueGen = valueGen.map(v -> {
// since we know E is of type map we know the value is a map
Map<?, ?> map = (Map<?, ?>) v.value();
Set<?> newValue = map.keySet();
SetType<Object> newType = SetType.getInstance(((MapType) e.type()).nameComparator(), false);
return v.with(newValue, newType);
});
}
Expression other = valueGen.generate(rnd);
Expression left, right;
if (bool.generate(rnd))
{
left = e;
right = other;
}
else
{
left = other;
right = e;
}
//TODO (correctness): "(smallint) ? - 16250" failed, but is this general or is it a small int thing?
//NOTE: (smallint) of -11843 and 3749 failed as well...
//NOTE: (long) was found and didn't fail...
//NOTE: see https://the-asf.slack.com/archives/CK23JSY2K/p1724819303058669 - varint didn't fail but serialized using int32 which causes equality mismatches for pk/ck lookups
if (e.type().unwrap() == ShortType.instance
|| e.type().unwrap() == IntegerType.instance) // seed=7525457176675272023L
{
left = new TypeHint(left);
right = new TypeHint(right);
}
return new Operator(kind, TypeHint.maybeApplyTypeHint(left), TypeHint.maybeApplyTypeHint(right));
};
}
public static class ExpressionBuilder<T>
{
private final AbstractType<T> type;
private final EnumSet<Operator.Kind> allowedOperators;
private Gen<T> valueGen;
private Gen<Boolean> useOperator = SourceDSL.booleans().all();
private BiFunction<Object, AbstractType<?>, Gen<Value>> literalOrBindGen = ASTGenerators::valueGen;
public ExpressionBuilder(AbstractType<T> type)
{
this.type = type.unwrap();
this.valueGen = AbstractTypeGenerators.getTypeSupport(this.type).valueGen;
this.allowedOperators = Operator.supportsOperators(this.type);
}
public ExpressionBuilder withOperators()
{
useOperator = i -> true;
return this;
}
public ExpressionBuilder withoutOperators()
{
useOperator = i -> false;
return this;
}
public ExpressionBuilder allowOperators()
{
useOperator = SourceDSL.booleans().all();
return this;
}
public ExpressionBuilder withLiteralOrBindGen(BiFunction<Object, AbstractType<?>, Gen<Value>> literalOrBindGen)
{
this.literalOrBindGen = literalOrBindGen;
return this;
}
public Gen<Expression> build()
{
//TODO (coverage): rather than single level operators, allow nested (a + b + c + d)
Gen<Value> leaf = rs -> literalOrBindGen.apply(valueGen.generate(rs), type).generate(rs);
return rs -> {
Expression e = leaf.generate(rs);
if (!allowedOperators.isEmpty() && useOperator.generate(rs))
e = operatorGen(allowedOperators, e, leaf).generate(rs);
return e;
};
}
}
public static class SelectGenBuilder
{
private final TableMetadata metadata;
private Gen<List<Expression>> selectGen;
private Gen<Map<Symbol, Expression>> keyGen;
private Gen<Optional<Value>> limitGen;
private BiFunction<Object, AbstractType<?>, Gen<Value>> literalOrBindGen = ASTGenerators::valueGen;
public SelectGenBuilder(TableMetadata metadata)
{
this.metadata = Objects.requireNonNull(metadata);
this.selectGen = selectColumns(metadata);
this.keyGen = partitionKeyGen(metadata);
withDefaultLimit();
}
public SelectGenBuilder withLiteralOrBindGen(BiFunction<Object, AbstractType<?>, Gen<Value>> literalOrBindGen)
{
this.literalOrBindGen = literalOrBindGen;
return this;
}
public SelectGenBuilder withSelectStar()
{
selectGen = ignore -> Collections.emptyList();
return this;
}
public SelectGenBuilder withDefaultLimit()
{
Gen<Optional<Value>> non = ignore -> Optional.empty();
Constraint limitLength = Constraint.between(1, 10_000);
Gen<Optional<Value>> positive = rnd -> Optional.of(valueGen(Math.toIntExact(rnd.next(limitLength)), Int32Type.instance).generate(rnd));
limitGen = non.mix(positive);
return this;
}
public SelectGenBuilder withLimit1()
{
this.limitGen = rnd -> Optional.of(valueGen(1, Int32Type.instance).generate(rnd));
return this;
}
public SelectGenBuilder withoutLimit()
{
this.limitGen = ignore -> Optional.empty();
return this;
}
public SelectGenBuilder withKeys(Gen<Map<Symbol, Object>> partitionKeys, Gen<Map<Symbol, Object>> clusteringKeys)
{
keyGen = rs -> {
Map<Symbol, Expression> keys = new LinkedHashMap<>();
for (Map.Entry<Symbol, Object> e : assertDeterministic(partitionKeys.generate(rs)).entrySet())
keys.put(e.getKey(), literalOrBindGen.apply(e.getValue(), e.getKey().type()).generate(rs));
if (!metadata.clusteringColumns().isEmpty())
{
for (Map.Entry<Symbol, Object> e : assertDeterministic(clusteringKeys.generate(rs)).entrySet())
keys.put(e.getKey(), literalOrBindGen.apply(e.getValue(), e.getKey().type()).generate(rs));
}
return keys;
};
return this;
}
public Gen<Select> build()
{
Optional<TableReference> ref = Optional.of(TableReference.from(metadata));
return rnd -> {
List<Expression> select = selectGen.generate(rnd);
Conditional keyClause = and(keyGen.generate(rnd));
Optional<Value> limit = limitGen.generate(rnd);
return new Select(select, ref, Optional.of(keyClause), Optional.empty(), limit);
};
}
private static Conditional and(Map<Symbol, Expression> data)
{
Conditional.Builder builder = new Conditional.Builder();
for (Map.Entry<Symbol, Expression> e : assertDeterministic(data).entrySet())
builder.where(e.getKey(), Conditional.Where.Inequality.EQUAL, e.getValue());
return builder.build();
}
private static Gen<List<Expression>> selectColumns(TableMetadata metadata)
{
List<ColumnMetadata> columns = allColumnsInFixedOrder(metadata);
Constraint between = Constraint.between(0, columns.size() - 1);
Gen<int[]> indexGen = rnd -> {
int size = Math.toIntExact(rnd.next(between)) + 1;
Set<Integer> dedup = new LinkedHashSet<>();
while (dedup.size() < size)
dedup.add(Math.toIntExact(rnd.next(between)));
return dedup.stream().mapToInt(Integer::intValue).toArray();
};
return rnd -> {
int[] indexes = indexGen.generate(rnd);
List<Expression> es = new ArrayList<>(indexes.length);
IntStream.of(indexes).mapToObj(columns::get).forEach(c -> es.add(new Symbol(c)));
return es;
};
}
private static Gen<Map<Symbol, Expression>> partitionKeyGen(TableMetadata metadata)
{
Map<ColumnMetadata, Gen<?>> gens = new LinkedHashMap<>();
for (ColumnMetadata col : allColumnsInFixedOrder(metadata))
gens.put(col, AbstractTypeGenerators.getTypeSupport(col.type).valueGen);
return rnd -> {
Map<Symbol, Expression> output = new LinkedHashMap<>();
for (ColumnMetadata col : metadata.partitionKeyColumns())
output.put(new Symbol(col), gens.get(col)
.map(o -> valueGen(o, col.type).generate(rnd))
.generate(rnd));
return output;
};
}
}
public static class MutationGenBuilder
{
public enum DeleteKind { Partition, Row, Column }
private final TableMetadata metadata;
private final LinkedHashSet<Symbol> allColumns;
private final LinkedHashSet<Symbol> partitionColumns, clusteringColumns;
private final LinkedHashSet<Symbol> primaryColumns;
private final LinkedHashSet<Symbol> regularColumns, staticColumns, regularAndStaticColumns;
private Gen<Mutation.Kind> kindGen = SourceDSL.arbitrary().enumValues(Mutation.Kind.class);
private Gen<OptionalInt> ttlGen = SourceDSL.integers().between(1, Math.toIntExact(TimeUnit.DAYS.toSeconds(10))).map(i -> i % 2 == 0 ? OptionalInt.empty() : OptionalInt.of(i));
private Gen<OptionalLong> timestampGen = SourceDSL.longs().between(1, Long.MAX_VALUE).map(i -> i % 2 == 0 ? OptionalLong.empty() : OptionalLong.of(i));
private Collection<Reference> references = Collections.emptyList();
private Gen<Boolean> withCasGen = SourceDSL.booleans().all();
private Gen<Boolean> useCasIf = SourceDSL.booleans().all();
private BiFunction<RandomnessSource, List<Symbol>, List<Symbol>> ifConditionFilter = (rnd, symbols) -> symbols;
private Gen<DeleteKind> deleteKindGen = SourceDSL.arbitrary().enumValues(DeleteKind.class);
private Map<Symbol, ExpressionBuilder<?>> columnExpressions = new LinkedHashMap<>();
public MutationGenBuilder(TableMetadata metadata)
{
this.metadata = Objects.requireNonNull(metadata);
this.allColumns = Mutation.toSet(metadata::allColumnsInSelectOrder);
this.partitionColumns = Mutation.toSet(metadata.partitionKeyColumns());
this.clusteringColumns = Mutation.toSet(metadata.clusteringColumns());
this.primaryColumns = Mutation.toSet(metadata.primaryKeyColumns());
this.regularColumns = Mutation.toSet(metadata.regularColumns());
this.staticColumns = Mutation.toSet(metadata.staticColumns());
this.regularAndStaticColumns = new LinkedHashSet<>();
regularAndStaticColumns.addAll(staticColumns);
regularAndStaticColumns.addAll(regularColumns);
for (Symbol symbol : allColumns)
columnExpressions.put(symbol, new ExpressionBuilder<>(symbol.type()));
}
public MutationGenBuilder withDeletionKind(Gen<DeleteKind> deleteKindGen)
{
this.deleteKindGen = deleteKindGen;
return this;
}
public MutationGenBuilder withDeletionKind(DeleteKind... values)
{
return withDeletionKind(SourceDSL.arbitrary().pick(values));
}
public MutationGenBuilder withLiteralOrBindGen(BiFunction<Object, AbstractType<?>, Gen<Value>> literalOrBindGen)
{
columnExpressions.values().forEach(e -> e.withLiteralOrBindGen(literalOrBindGen));
return this;
}
public MutationGenBuilder withoutTransaction()
{
withoutCas();
return this;
}
public MutationGenBuilder withCas()
{
withCasGen = SourceDSL.arbitrary().constant(true);
return this;
}
public MutationGenBuilder withoutCas()
{
withCasGen = SourceDSL.arbitrary().constant(false);
return this;
}
public MutationGenBuilder withCasGen(Gen<Boolean> withCasGen)
{
withCasGen = Objects.requireNonNull(withCasGen);
return this;
}
public MutationGenBuilder withCasIf()
{
useCasIf = SourceDSL.arbitrary().constant(true);
return this;
}
public MutationGenBuilder withoutCasIf()
{
useCasIf = SourceDSL.arbitrary().constant(false);
return this;
}
public MutationGenBuilder withCasIfGen(Gen<Boolean> gen)
{
useCasIf = Objects.requireNonNull(gen);
return this;
}
public MutationGenBuilder withIfColumnFilter(BiFunction<RandomnessSource, List<Symbol>, List<Symbol>> ifConditionFilter)
{
this.ifConditionFilter = Objects.requireNonNull(ifConditionFilter);
return this;
}
public MutationGenBuilder withoutTimestamp()
{
timestampGen = ignore -> OptionalLong.empty();
return this;
}
public MutationGenBuilder withoutTtl()
{
ttlGen = ignore -> OptionalInt.empty();
return this;
}
public MutationGenBuilder withOperators()
{
columnExpressions.values().forEach(e -> e.withOperators());
return this;
}
public MutationGenBuilder withoutOperators()
{
columnExpressions.values().forEach(e -> e.withoutOperators());
return this;
}
public MutationGenBuilder withReferences(Collection<Reference> references)
{
this.references = references;
return this;
}
private Gen<? extends Map<Symbol, Object>> partitionValueGen = null;
private Gen<? extends Map<Symbol, Object>> clusteringValueGen = null;
public MutationGenBuilder withPartitions(Gen<? extends Map<Symbol, Object>> values)
{
this.partitionValueGen = values;
return this;
}
public MutationGenBuilder withClusterings(Gen<? extends Map<Symbol, Object>> values)
{
this.clusteringValueGen = values;
return this;
}
private static void values(RandomnessSource rnd,
Map<Symbol, ExpressionBuilder<?>> columnExpressions,
Conditional.EqBuilder<?> builder,
LinkedHashSet<Symbol> columns,
@Nullable Gen<? extends Map<Symbol, Object>> gen)
{
if (gen != null)
{
Map<Symbol, Object> map = gen.generate(rnd);
for (Map.Entry<Symbol, ?> e : assertDeterministic(map).entrySet())
builder.value(e.getKey(), valueGen(e.getValue(), e.getKey().type()).generate(rnd));
}
else
{
//TODO (coverage): support IN rather than just EQ
for (Symbol s : columns)
builder.value(s, columnExpressions.get(s).build().generate(rnd));
}
}
public Gen<Mutation> build()
{
Gen<Boolean> bool = SourceDSL.booleans().all();
Map<? extends AbstractType<?>, List<Reference>> typeToReference = references.stream().collect(Collectors.groupingBy(Reference::type));
return rnd -> {
Mutation.Kind kind = kindGen.generate(rnd);
// when there are not non-primary-columns then can't support UPDATE
if (kind == Mutation.Kind.UPDATE && regularColumns.isEmpty())
{
int i;
int maxRetries = 42;
for (i = 0; i < maxRetries && kind == Mutation.Kind.UPDATE; i++)
kind = kindGen.generate(rnd);
if (i == maxRetries)
throw new IllegalArgumentException("Kind gen kept returning UPDATE, but not supported when there are no non-primary columns");
}
boolean isCas = withCasGen.generate(rnd);
boolean isTransaction = isCas; //TODO (coverage): add accord support
switch (kind)
{
case INSERT:
{
Mutation.InsertBuilder builder = Mutation.insert(metadata);
if (isCas)
builder.ifNotExists();
var ttl = ttlGen.generate(rnd);
if (ttl.isPresent())
builder.ttl(valueGen(ttl.getAsInt(), Int32Type.instance).generate(rnd));
var timestamp = timestampGen.generate(rnd);
if (timestamp.isPresent())
builder.timestamp(valueGen(timestamp.getAsLong(), LongType.instance).generate(rnd));
values(rnd, columnExpressions, builder, partitionColumns, partitionValueGen);
values(rnd, columnExpressions, builder, clusteringColumns, clusteringValueGen);
LinkedHashSet<Symbol> columnsToGenerate;
if (regularAndStaticColumns.isEmpty())
{
columnsToGenerate = new LinkedHashSet<>(0);
}
else if (regularAndStaticColumns.size() == 1 || bool.generate(rnd))
{
// all columns
columnsToGenerate = new LinkedHashSet<>(regularAndStaticColumns);
}
else
{
// subset
columnsToGenerate = new LinkedHashSet<>(subsetRegularAndStaticColumns(rnd));
}
generateRemaining(rnd, bool, Mutation.Kind.INSERT, isTransaction, typeToReference, builder, columnsToGenerate);
return builder.build();
}
case UPDATE:
{
Mutation.UpdateBuilder builder = Mutation.update(metadata);
var ttl = ttlGen.generate(rnd);
if (ttl.isPresent())
builder.ttl(valueGen(ttl.getAsInt(), Int32Type.instance).generate(rnd));
var timestamp = timestampGen.generate(rnd);
if (timestamp.isPresent())
builder.timestamp(valueGen(timestamp.getAsLong(), LongType.instance).generate(rnd));
if (isCas)
{
if (useCasIf.generate(rnd))
{
ifGen(new ArrayList<>(regularAndStaticColumns)).generate(rnd).ifPresent(c -> builder.ifCondition(c));
}
else
{
builder.ifExists();
}
}
values(rnd, columnExpressions, builder, partitionColumns, partitionValueGen);
values(rnd, columnExpressions, builder, clusteringColumns, clusteringValueGen);
LinkedHashSet<Symbol> columnsToGenerate;
if (regularAndStaticColumns.size() == 1 || bool.generate(rnd))
{
// all columns
columnsToGenerate = new LinkedHashSet<>(regularAndStaticColumns);
}
else
{
// subset must include a regular column
columnsToGenerate = new LinkedHashSet<>(subset(rnd, regularColumns));
if (!staticColumns.isEmpty() && bool.generate(rnd))
columnsToGenerate.addAll(subset(rnd, staticColumns));
}
Conditional.EqBuilder<Mutation.UpdateBuilder> setBuilder = builder::set;
generateRemaining(rnd, bool, Mutation.Kind.UPDATE, isTransaction, typeToReference, setBuilder, columnsToGenerate);
return builder.build();
}
case DELETE:
{
Mutation.DeleteBuilder builder = Mutation.delete(metadata);
// 3 types of delete: partition, row, columns
DeleteKind deleteKind = deleteKindGen.generate(rnd);
// if there are no columns to delete, fallback to row
if (deleteKind == DeleteKind.Column && regularAndStaticColumns.isEmpty())
deleteKind = DeleteKind.Row;
if (deleteKind == DeleteKind.Row && clusteringColumns.isEmpty())
deleteKind = DeleteKind.Partition;
values(rnd, columnExpressions, builder, partitionColumns, partitionValueGen);
switch (deleteKind)
{
case Partition:
// nothing to do here, already handled
break;
case Row:
values(rnd, columnExpressions, builder, clusteringColumns, clusteringValueGen);
break;
case Column:
if (clusteringColumns.isEmpty())
{
subsetRegularAndStaticColumns(rnd).forEach(builder::column);
}
else if (staticColumns.isEmpty())
{
subset(rnd, regularColumns).forEach(builder::column);
values(rnd, columnExpressions, builder, clusteringColumns, clusteringValueGen);
}
else if (regularColumns.isEmpty())
{
subset(rnd, staticColumns).forEach(builder::column);
}
else
{
// 2 possible states:
// 1) select a row then delete the columns
// 2) select a partition then select static columns only
if (bool.generate(rnd))
{
// select static
subset(rnd, staticColumns).forEach(builder::column);
}
else
{
// select a row, at least 1 regular, and 0 or more statics
values(rnd, columnExpressions, builder, clusteringColumns, clusteringValueGen);
subset(rnd, regularColumns).forEach(builder::column);
if (bool.generate(rnd))
subset(rnd, staticColumns).forEach(builder::column);
}
}
if (!clusteringColumns.isEmpty() && !staticColumns.isEmpty())
{
if (bool.generate(rnd))
{
// static only
subset(rnd, staticColumns).forEach(builder::column);
}
else
{
// mixed (piss
}
}
break;
default:
throw new UnsupportedOperationException();
}
var timestamp = timestampGen.generate(rnd);
if (timestamp.isPresent())
builder.timestamp(valueGen(timestamp.getAsLong(), LongType.instance).generate(rnd));
if (isCas)
{
boolean existAllowed = true;
List<Symbol> columns;
switch (deleteKind)
{
case Partition:
{
// As of this moment delete if partition exists does a full partition read, so its blocked
// due to being too costly... this query is logically correct so we should support as only
// liveness information is needed, but its not supported right now so need to work around
// see ML "[DISCUSS] CASSANDRA-20163 DELETE partition IF static column condition is currently blocked"
// I tried to enable delete partition if static column condition in CASSANDRA-20156, but was
// asked to abandon the patch for consistency reasons.
// Delete partition when there are clustering columns is unsupported, so avoid generating
if (clusteringColumns.isEmpty())
{
// this is the same as delete row
columns = new ArrayList<>(regularAndStaticColumns);
existAllowed = true;
}
else
{
columns = Collections.emptyList();
existAllowed = false;
}
}
break;
case Row:
{
columns = new ArrayList<>(regularAndStaticColumns);
}
break;
case Column:
{
// some column deletes support without clustering, others dont... to avoid
// relearning this, only allow conditions on the followin columns:
// 1) the columns in the query; only valid columns are present
// 2) static columns; these are always safe to include
LinkedHashSet<Symbol> uniq = new LinkedHashSet<>(builder.columns());
uniq.addAll(staticColumns);
columns = new ArrayList<>(uniq);
}
break;
default:
throw new UnsupportedOperationException(deleteKind.name());
}
if (!columns.isEmpty() && useCasIf.generate(rnd))
{
ifGen(columns).generate(rnd).ifPresent(builder::ifCondition);
}
else if (existAllowed)
{
builder.ifExists();
}
else
{
// can't do a CAS query
}
}
return builder.build();
}
default:
throw new UnsupportedOperationException(kind.name());
}
};
}
private void generateRemaining(RandomnessSource rnd,
Gen<Boolean> bool,
Mutation.Kind kind,
boolean isTransaction,
Map<? extends AbstractType<?>, List<Reference>> typeToReference,
Conditional.EqBuilder<?> builder,
LinkedHashSet<Symbol> columnsToGenerate)
{
//TODO (flexability): since expression offers visit to replace things, could also keep the expression in tact but just replace Value with the Reference?
if (!typeToReference.isEmpty())
{
List<Symbol> allowed = new ArrayList<>(columnsToGenerate);
for (Symbol s : allowed)
{
List<Reference> matches = typeToReference.get(s.type());
if (matches == null)
continue;
if (bool.generate(rnd))
{
columnsToGenerate.remove(s);
builder.value(s, SourceDSL.arbitrary().pick(matches).generate(rnd));
}
}
}
if (kind == Mutation.Kind.UPDATE && isTransaction)
{
for (Symbol c : new ArrayList<>(columnsToGenerate))
{
var useOperator = columnExpressions.get(c).useOperator;
EnumSet<AssignmentOperator.Kind> additionOperatorAllowed = AssignmentOperator.supportsOperators(c.type());
if (!additionOperatorAllowed.isEmpty() && useOperator.generate(rnd))
{
Expression expression = columnExpressions.get(c).build().generate(rnd);
builder.value(c, assignmentOperatorGen(additionOperatorAllowed, expression).generate(rnd));
columnsToGenerate.remove(c);
}
}
}
columnsToGenerate.forEach(s -> builder.value(s, columnExpressions.get(s).build().generate(rnd)));
}
private List<Symbol> subsetRegularAndStaticColumns(RandomnessSource rnd)
{
return subset(rnd, regularAndStaticColumns);
}
private static List<Symbol> subset(RandomnessSource rnd, LinkedHashSet<Symbol> columns)
{
if (columns.size() == 1)
return new ArrayList<>(columns);
int numColumns = Math.toIntExact(rnd.next(Constraint.between(1, columns.size())));
List<Symbol> subset = Generators.uniqueList(SourceDSL.arbitrary().pick(new ArrayList<>(columns)), i -> numColumns).generate(rnd);
return subset;
}
private Gen<Optional<CasCondition.IfCondition>> ifGen(List<Symbol> possibleColumns)
{
return rnd -> {
List<Symbol> symbols = ifConditionFilter.apply(rnd, possibleColumns);
if (symbols == null || symbols.isEmpty())
return Optional.empty();
Conditional.Builder builder = new Conditional.Builder();
for (Symbol symbol : symbols)
builder.where(symbol, Conditional.Where.Inequality.EQUAL, columnExpressions.get(symbol).build().generate(rnd));
return Optional.of(new CasCondition.IfCondition(builder.build()));
};
}
}
public static class TxnGenBuilder
{
public enum TxReturn { NONE, TABLE, REF}
private final TableMetadata metadata;
private Constraint letRange = Constraint.between(0, 3);
private Constraint ifUpdateRange = Constraint.between(1, 3);
private Constraint updateRange = Constraint.between(0, 3);
private Gen<Select> selectGen;
private Gen<TxReturn> txReturnGen = SourceDSL.arbitrary().enumValues(TxReturn.class);
private boolean allowReferences = true;
public TxnGenBuilder(TableMetadata metadata)
{
this.metadata = metadata;
this.selectGen = new SelectGenBuilder(metadata)
.withLimit1()
.build();
}
public TxnGenBuilder withoutReferences()
{
this.allowReferences = false;
return this;
}
public Gen<Txn> build()
{
Gen<Boolean> bool = SourceDSL.booleans().all();
return rnd -> {
Txn.Builder builder = new Txn.Builder();
do
{
int numLets = Math.toIntExact(rnd.next(letRange));
for (int i = 0; i < numLets; i++)
{
// LET doesn't use normal symbol logic and acts closer to a common lanaguage; name does not lower
// case... it is possible that a reserved word gets used, so make sure to use a generator that
// filters those out.
String name;
while (builder.lets().containsKey(name = SYMBOL_GEN.generate(rnd))) {}
builder.addLet(name, selectGen.generate(rnd));
}
Gen<Reference> refGen = SourceDSL.arbitrary().pick(new ArrayList<>(builder.allowedReferences()));
if (allowReferences)
{
switch (txReturnGen.generate(rnd))
{
case REF:
{
if (!builder.allowedReferences().isEmpty())
{
Gen<List<Reference>> refsGen = SourceDSL.lists().of(refGen).ofSizeBetween(1, Math.max(10, builder.allowedReferences().size()));
builder.addReturn(new Select((List<Expression>) (List<?>) refsGen.generate(rnd)));
}
}
break;
case TABLE:
builder.addReturn(selectGen.generate(rnd));
break;
}
}
else
{
builder.addReturn(selectGen.generate(rnd));
}
MutationGenBuilder mutationBuilder = new MutationGenBuilder(metadata)
.withoutCas()
.withoutTimestamp()
.withoutTtl()
.withReferences(new ArrayList<>(builder.allowedReferences()));
if (!allowReferences)
mutationBuilder.withReferences(Collections.emptyList());
Gen<Mutation> updateGen = mutationBuilder.build();
if (allowReferences && !builder.lets().isEmpty() && bool.generate(rnd))
{
Gen<Conditional> conditionalGen = conditionalGen(refGen);
int numUpdates = Math.toIntExact(rnd.next(ifUpdateRange));
List<Mutation> mutations = new ArrayList<>(numUpdates);
for (int i = 0; i < numUpdates; i++)
mutations.add(updateGen.generate(rnd));
builder.addIf(new Txn.If(conditionalGen.generate(rnd), mutations));
}
else
{
// Current limitation is that mutations are tied to the condition if present; can't have
// a condition and mutations that don't belong to it in v1... once multiple conditions are
// supported then can always attempt to add updates
int numUpdates = Math.toIntExact(rnd.next(updateRange));
for (int i = 0; i < numUpdates; i++)
builder.addUpdate(updateGen.generate(rnd));
}
} while (builder.isEmpty());
return builder.build();
};
}
private static Gen<Conditional> conditionalGen(Gen<Reference> refGen)
{
Constraint numConditionsConstraint = Constraint.between(1, 10);
return rnd -> {
//TODO support OR
Gen<Conditional.Where> whereGen = whereGen(refGen.generate(rnd));
int size = Math.toIntExact(rnd.next(numConditionsConstraint));
Conditional accum = whereGen.generate(rnd);
for (int i = 1; i < size; i++)
accum = new Conditional.And(accum, whereGen.generate(rnd));
return accum;
};
}
private static Gen<Conditional.Where> whereGen(Reference ref)
{
Gen<Conditional.Where.Inequality> kindGen = SourceDSL.arbitrary().enumValues(Conditional.Where.Inequality.class);
Gen<?> dataGen = AbstractTypeGenerators.getTypeSupport(ref.type()).valueGen;
return rnd -> {
Conditional.Where.Inequality kind = kindGen.generate(rnd);
return Conditional.Where.create(kind, ref, valueGen(dataGen.generate(rnd), ref.type()).generate(rnd));
};
}
}
}

136
test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java
View File

@ -41,6 +41,7 @@ import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
@ -243,17 +244,22 @@ public final class AbstractTypeGenerators
return () -> PRIMITIVE_TYPE_DATA_GENS.put(type, original);
}
public static TypeGenBuilder withoutUnsafeEquality()
public static TypeGenBuilder withoutUnsafeEquality(TypeGenBuilder builder)
{
// make sure to keep UNSAFE_EQUALITY in-sync
return AbstractTypeGenerators.builder()
.withoutEmpty()
.withoutPrimitive(DurationType.instance)
// decimal "normalizes" the data to compare, so primary columns "may" mutate the data, causing missmatches
// see CASSANDRA-18530
.withoutPrimitive(DecimalType.instance)
// counters are only for top level
.withoutTypeKinds(TypeKind.COUNTER);
return builder
.withoutEmpty()
.withoutPrimitive(DurationType.instance)
// decimal "normalizes" the data to compare, so primary columns "may" mutate the data, causing missmatches
// see CASSANDRA-18530
.withoutPrimitive(DecimalType.instance)
// counters are only for top level
.withoutTypeKinds(TypeKind.COUNTER);
}
public static TypeGenBuilder withoutUnsafeEquality()
{
return withoutUnsafeEquality(AbstractTypeGenerators.builder());
}
public interface Releaser extends AutoCloseable
@ -274,6 +280,7 @@ public final class AbstractTypeGenerators
private Function<Integer, Gen<AbstractType<?>>> defaultSetKeyFunc;
private Predicate<AbstractType<?>> typeFilter = null;
private Gen<String> udtName = null;
private Gen<Boolean> multiCellGen = BOOLEAN_GEN;
public TypeGenBuilder()
{
@ -297,6 +304,17 @@ public final class AbstractTypeGenerators
typeFilter = other.typeFilter;
}
public TypeGenBuilder withMultiCell(Gen<Boolean> multiCellGen)
{
this.multiCellGen = multiCellGen;
return this;
}
public TypeGenBuilder withMultiCell(boolean multiCell)
{
return withMultiCell(i -> multiCell);
}
public TypeGenBuilder withTypeFilter(Predicate<AbstractType<?>> fn)
{
typeFilter = fn;
@ -456,7 +474,7 @@ public final class AbstractTypeGenerators
}
else
kindGen = SourceDSL.arbitrary().enumValues(TypeKind.class);
return buildRecursive(maxDepth, maxDepth, kindGen, BOOLEAN_GEN);
return buildRecursive(maxDepth, maxDepth, kindGen, multiCellGen);
}
private Gen<AbstractType<?>> buildRecursive(int maxDepth, int level, Gen<TypeKind> typeKindGen, Gen<Boolean> multiCellGen)
@ -732,6 +750,18 @@ public final class AbstractTypeGenerators
return userTypeGen(elementGen, sizeGen, ksGen, nameGen, BOOLEAN_GEN);
}
private static ThreadLocal<String> OVERRIDE_KEYSPACE = new ThreadLocal<>();
public static void overrideUDTKeyspace(String ks)
{
OVERRIDE_KEYSPACE.set(ks);
}
public static void clearUDTKeyspace()
{
OVERRIDE_KEYSPACE.remove();
}
public static Gen<UserType> userTypeGen(Gen<AbstractType<?>> elementGen, Gen<Integer> sizeGen, Gen<String> ksGen, Gen<String> nameGen, Gen<Boolean> multiCellGen)
{
Gen<FieldIdentifier> fieldNameGen = IDENTIFIER_GEN.map(FieldIdentifier::forQuoted);
@ -740,7 +770,9 @@ public final class AbstractTypeGenerators
int numElements = sizeGen.generate(rnd);
List<AbstractType<?>> fieldTypes = new ArrayList<>(numElements);
LinkedHashSet<FieldIdentifier> fieldNames = new LinkedHashSet<>(numElements);
String ks = ksGen.generate(rnd);
String ks = OVERRIDE_KEYSPACE.get();
if (ks == null)
ks = ksGen.generate(rnd);
String name = nameGen.generate(rnd);
ByteBuffer nameBB = AsciiType.instance.decompose(name);
@ -1089,6 +1121,49 @@ public final class AbstractTypeGenerators
return Math.min(size, uniq);
}
public static boolean contains(AbstractType<?> type, AbstractType<?> searchFor)
{
return contains(type, searchFor::equals);
}
public static boolean contains(AbstractType<?> type, Predicate<AbstractType<?>> searchFor)
{
class Found
{
boolean result = false;
}
Found found = new Found();
visit(type, t -> {
if (searchFor.test(t))
{
found.result = true;
return VisitAction.STOP;
}
return VisitAction.CONTINUE;
});
return found.result;
}
public enum VisitAction { CONTINUE, STOP}
public static VisitAction visit(AbstractType<?> type, Function<AbstractType<?>, VisitAction> fn)
{
VisitAction action = fn.apply(type);
if (action == VisitAction.STOP) return action;
if (type.isReversed())
{
type = type.unwrap();
action = fn.apply(type);
if (action == VisitAction.STOP) return action;
}
for (AbstractType<?> t : type.subTypes())
{
action = visit(t, fn);
if (action == VisitAction.STOP) return action;
}
return VisitAction.CONTINUE;
}
public static Set<UserType> extractUDTs(AbstractType<?> type)
{
Set<UserType> matches = new HashSet<>();
@ -1098,12 +1173,11 @@ public final class AbstractTypeGenerators
public static void extractUDTs(AbstractType<?> type, Set<UserType> matches)
{
if (type instanceof ReversedType)
type = ((ReversedType) type).baseType;
if (type instanceof UserType)
matches.add((UserType) type);
for (AbstractType<?> t : type.subTypes())
extractUDTs(t, matches);
visit(type, t -> {
if (t instanceof UserType)
matches.add((UserType) t);
return VisitAction.CONTINUE;
});
}
public static String typeTree(AbstractType<?> type)
@ -1569,4 +1643,32 @@ public final class AbstractTypeGenerators
forEachTypesPair(frozenAndUnfrozen(l), frozenAndUnfrozen(r), typePairConsumer);
}
public static TypeSupport<?> elementAccess(AbstractType<?> type)
{
type = type.unwrap();
Preconditions.checkArgument(type.isCollection() || type.isUDT(), "Unexpected type: %s", type);
if (type.isUDT())
{
// select a field
UserType ut = (UserType) type;
Gen<ByteBuffer> fieldNameGen = SourceDSL.arbitrary().pick(ut.fieldNames().stream().map(f -> f.bytes).collect(Collectors.toList()));
return new TypeSupport<>(BytesType.instance, fieldNameGen, ByteBuffer::compareTo);
}
else
{
CollectionType<?> ct = (CollectionType<?>) type;
switch (ct.kind)
{
// case SET: // set does not support element access; see org.apache.cassandra.db.marshal.MultiElementType.getElement
case LIST:
// by index
return new TypeSupport<>(Int32Type.instance, SourceDSL.integers().between(0, Integer.MAX_VALUE), Integer::compare);
case MAP:
// by key
return getTypeSupport(ct.nameComparator());
default: throw new UnsupportedOperationException(ct.kind.name());
}
}
}
}

731
test/unit/org/apache/cassandra/utils/CassandraGenerators.java
View File

@ -21,23 +21,33 @@ import java.lang.reflect.Modifier;
import java.math.BigInteger;
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.builder.MultilineRecursiveToStringStyle;
import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
@ -45,6 +55,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.cql3.FieldIdentifier;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.ReadCommand;
import org.apache.cassandra.db.SchemaCQLHelper;
import org.apache.cassandra.db.SinglePartitionReadCommand;
@ -52,7 +63,9 @@ import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.marshal.CounterColumnType;
import org.apache.cassandra.db.marshal.EmptyType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.marshal.UserType;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
@ -62,22 +75,40 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.OrderPreservingPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.ReversedLongLocalPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.EndpointState;
import org.apache.cassandra.gms.HeartBeatState;
import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.LocalStrategy;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.net.ConnectionType;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.NoPayload;
import org.apache.cassandra.net.PingRequest;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.MemtableParams;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableParams;
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.schema.Types;
import org.apache.cassandra.schema.UserFunctions;
import org.apache.cassandra.schema.Views;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.utils.AbstractTypeGenerators.TypeGenBuilder;
import org.apache.cassandra.utils.AbstractTypeGenerators.ValueDomain;
import org.quicktheories.core.Gen;
import org.quicktheories.core.RandomnessSource;
@ -87,6 +118,7 @@ import org.quicktheories.impl.Constraint;
import static org.apache.cassandra.utils.AbstractTypeGenerators.allowReversed;
import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport;
import static org.apache.cassandra.utils.AbstractTypeGenerators.withoutUnsafeEquality;
import static org.apache.cassandra.utils.Generators.IDENTIFIER_GEN;
import static org.apache.cassandra.utils.Generators.SMALL_TIME_SPAN_NANOS;
import static org.apache.cassandra.utils.Generators.TIMESTAMP_NANOS;
@ -111,7 +143,6 @@ public final class CassandraGenerators
return InetAddressAndPort.getByAddressOverrideDefaults(address, NETWORK_PORT_GEN.generate(rnd));
};
public static final Gen<TableId> TABLE_ID_GEN = Generators.UUID_RANDOM_GEN.map(TableId::fromUUID);
private static final Gen<TableMetadata.Kind> TABLE_KIND_GEN = SourceDSL.arbitrary().pick(TableMetadata.Kind.REGULAR, TableMetadata.Kind.INDEX, TableMetadata.Kind.VIRTUAL);
public static final Gen<TableMetadata> TABLE_METADATA_GEN = gen(rnd -> createTableMetadata(IDENTIFIER_GEN.generate(rnd), rnd)).describedAs(CassandraGenerators::toStringRecursive);
@ -183,29 +214,304 @@ public final class CassandraGenerators
return SourceDSL.arbitrary().pick("big", "bti");
}
public static Gen<SSTableFormat<?, ?>> sstableFormat()
{
// make sure ordering is determanstic, else repeatability breaks
NavigableMap<String, SSTableFormat<?, ?>> formats = new TreeMap<>(DatabaseDescriptor.getSSTableFormats());
return SourceDSL.arbitrary().pick(new ArrayList<>(formats.values()));
}
public static class AbstractReplicationStrategyBuilder
{
public enum Strategy
{
Simple(true),
NetworkTopology(true),
Local(false),
Meta(false);
public final boolean userAllowed;
Strategy(boolean userAllowed)
{
this.userAllowed = userAllowed;
}
}
private Gen<Strategy> strategyGen = SourceDSL.arbitrary().enumValues(Strategy.class);
private Gen<String> keyspaceNameGen = KEYSPACE_NAME_GEN;
private Gen<Integer> rfGen = SourceDSL.integers().between(1, 3);
private Gen<List<String>> networkTopologyDCGen = rs -> {
Gen<Integer> numDcsGen = SourceDSL.integers().between(1, 3);
Gen<String> nameGen = IDENTIFIER_GEN;
Set<String> dcs = new HashSet<>();
int targetSize = numDcsGen.generate(rs);
while (dcs.size() != targetSize)
dcs.add(nameGen.generate(rs));
List<String> ordered = new ArrayList<>(dcs);
ordered.sort(Comparator.naturalOrder());
return ordered;
};
public AbstractReplicationStrategyBuilder withKeyspace(Gen<String> keyspaceNameGen)
{
this.keyspaceNameGen = keyspaceNameGen;
return this;
}
public AbstractReplicationStrategyBuilder withKeyspace(String keyspace)
{
this.keyspaceNameGen = i -> keyspace;
return this;
}
public AbstractReplicationStrategyBuilder withUserAllowed()
{
List<Strategy> allowed = Stream.of(Strategy.values()).filter(s -> s.userAllowed).collect(Collectors.toList());
strategyGen = SourceDSL.arbitrary().pick(allowed);
return this;
}
public AbstractReplicationStrategyBuilder withRf(Gen<Integer> rfGen)
{
this.rfGen = rfGen;
return this;
}
public AbstractReplicationStrategyBuilder withRf(int rf)
{
this.rfGen = i -> rf;
return this;
}
public AbstractReplicationStrategyBuilder withDatacenters(Gen<List<String>> networkTopologyDCGen)
{
this.networkTopologyDCGen = networkTopologyDCGen;
return this;
}
public AbstractReplicationStrategyBuilder withDatacenters(String first, String... rest)
{
if (rest.length == 0)
{
this.networkTopologyDCGen = i -> Collections.singletonList(first);
}
else
{
List<String> all = new ArrayList<>(rest.length + 1);
all.add(first);
all.addAll(Arrays.asList(rest));
this.networkTopologyDCGen = i -> all;
}
return this;
}
public Gen<AbstractReplicationStrategy> build()
{
return rs -> {
Strategy strategy = strategyGen.generate(rs);
switch (strategy)
{
case Simple:
return new SimpleStrategy(keyspaceNameGen.generate(rs),
ImmutableMap.of(SimpleStrategy.REPLICATION_FACTOR, rfGen.generate(rs).toString()));
case NetworkTopology:
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
List<String> names = networkTopologyDCGen.generate(rs);
for (String name : names)
builder.put(name, rfGen.generate(rs).toString());
ImmutableMap<String, String> map = builder.build();
return new TestableNetworkTopologyStrategy(keyspaceNameGen.generate(rs), map);
case Meta:
return new MetaStrategy(keyspaceNameGen.generate(rs), ImmutableMap.of());
case Local:
return new LocalStrategy(keyspaceNameGen.generate(rs), ImmutableMap.of());
default:
throw new UnsupportedOperationException(strategy.name());
}
};
}
}
public static class TestableNetworkTopologyStrategy extends NetworkTopologyStrategy
{
public TestableNetworkTopologyStrategy(String keyspaceName, Map<String, String> configOptions) throws ConfigurationException
{
super(keyspaceName, configOptions);
}
@Override
public Collection<String> recognizedOptions(ClusterMetadata metadata)
{
return configOptions.keySet();
}
}
public static KeyspaceMetadataBuilder regularKeyspace()
{
return new KeyspaceMetadataBuilder().withKind(KeyspaceMetadata.Kind.REGULAR);
}
public static class KeyspaceMetadataBuilder
{
private Gen<String> nameGen = KEYSPACE_NAME_GEN;
private Gen<KeyspaceMetadata.Kind> kindGen = SourceDSL.arbitrary().enumValues(KeyspaceMetadata.Kind.class);
private Gen<AbstractReplicationStrategyBuilder> replicationGen = i -> new AbstractReplicationStrategyBuilder();
private Gen<Boolean> durableWritesGen = SourceDSL.booleans().all();
public KeyspaceMetadataBuilder withReplication(Gen<AbstractReplicationStrategyBuilder> replicationGen)
{
this.replicationGen = replicationGen;
return this;
}
public KeyspaceMetadataBuilder withReplication(AbstractReplicationStrategyBuilder replication)
{
this.replicationGen = i -> replication;
return this;
}
public KeyspaceMetadataBuilder withName(Gen<String> nameGen)
{
this.nameGen = nameGen;
return this;
}
public KeyspaceMetadataBuilder withName(String name)
{
this.nameGen = i -> name;
return this;
}
public KeyspaceMetadataBuilder withKind(Gen<KeyspaceMetadata.Kind> kindGen)
{
this.kindGen = kindGen;
return this;
}
public KeyspaceMetadataBuilder withKind(KeyspaceMetadata.Kind kind)
{
this.kindGen = i -> kind;
return this;
}
public Gen<KeyspaceMetadata> build()
{
return rs -> {
String name = nameGen.generate(rs);
KeyspaceMetadata.Kind kind = kindGen.generate(rs);
AbstractReplicationStrategy replication = replicationGen.generate(rs).withKeyspace(nameGen).build().generate(rs);
ReplicationParams replicationParams = ReplicationParams.fromStrategy(replication);
boolean durableWrites = durableWritesGen.generate(rs);
KeyspaceParams params = new KeyspaceParams(durableWrites, replicationParams);
Tables tables = Tables.none();
Views views = Views.none();
Types types = Types.none();
UserFunctions userFunctions = UserFunctions.none();
return KeyspaceMetadata.createUnsafe(name, kind, params, tables, views, types, userFunctions);
};
}
}
public static class TableParamsBuilder
{
@Nullable
private Gen<String> memtableKeyGen = null;
public TableParamsBuilder withKnownMemtables()
{
Set<String> known = MemtableParams.knownDefinitions();
// for testing reason, some invalid types are added; filter out
List<String> valid = known.stream().filter(name -> !name.startsWith("test_")).collect(Collectors.toList());
memtableKeyGen = SourceDSL.arbitrary().pick(valid);
return this;
}
public Gen<TableParams> build()
{
return rnd -> {
TableParams.Builder params = TableParams.builder();
if (memtableKeyGen != null)
params.memtable(MemtableParams.get(memtableKeyGen.generate(rnd)));
return params.build();
};
}
}
public static TableMetadataBuilder regularTable()
{
return new TableMetadataBuilder()
.withTableKinds(TableMetadata.Kind.REGULAR)
.withKnownMemtables();
}
public static class TableMetadataBuilder
{
private Gen<String> ksNameGen = CassandraGenerators.KEYSPACE_NAME_GEN;
private Gen<String> tableNameGen = IDENTIFIER_GEN;
private Gen<AbstractType<?>> defaultTypeGen = AbstractTypeGenerators.builder()
.withDefaultSetKey(AbstractTypeGenerators.withoutUnsafeEquality())
.withMaxDepth(1)
.build();
private Gen<AbstractType<?>> partitionColTypeGen, clusteringColTypeGen, staticColTypeGen, regularColTypeGen;
private TypeGenBuilder defaultTypeGen = defaultTypeGen();
private Gen<Boolean> useCounter = ignore -> false;
private TypeGenBuilder partitionColTypeGen, clusteringColTypeGen, staticColTypeGen, regularColTypeGen;
private Gen<TableId> tableIdGen = TABLE_ID_GEN;
private Gen<TableMetadata.Kind> tableKindGen = SourceDSL.arbitrary().constant(TableMetadata.Kind.REGULAR);
private Gen<Integer> numPartitionColumnsGen = SourceDSL.integers().between(1, 2);
private Gen<Integer> numClusteringColumnsGen = SourceDSL.integers().between(1, 2);
private Gen<Integer> numRegularColumnsGen = SourceDSL.integers().between(1, 5);
private Gen<Integer> numStaticColumnsGen = SourceDSL.integers().between(0, 2);
private Gen<String> memtableKeyGen = null;
@Nullable
private ColumnNameGen columnNameGen = null;
private TableParamsBuilder paramsBuilder = new TableParamsBuilder();
private Gen<IPartitioner> partitionerGen = partitioners();
public static TypeGenBuilder defaultTypeGen()
{
return AbstractTypeGenerators.builder()
.withoutEmpty()
.withDefaultSetKey(withoutUnsafeEquality())
.withMaxDepth(1)
.withoutTypeKinds(AbstractTypeGenerators.TypeKind.COUNTER);
}
public TableMetadataBuilder withSimpleColumnNames()
{
columnNameGen = (i, kind, offset) -> {
switch (kind)
{
case PARTITION_KEY: return "pk" + offset;
case CLUSTERING: return "ck" + offset;
case STATIC: return "s" + offset;
case REGULAR: return "v" + offset;
default: throw new UnsupportedOperationException("Unknown kind: " + kind);
}
};
return this;
}
public TableMetadataBuilder withPartitioner(Gen<IPartitioner> partitionerGen)
{
this.partitionerGen = Objects.requireNonNull(partitionerGen);
return this;
}
public TableMetadataBuilder withPartitioner(IPartitioner partitioner)
{
return withPartitioner(i -> partitioner);
}
public TableMetadataBuilder withUseCounter(boolean useCounter)
{
return withUseCounter(ignore -> useCounter);
}
public TableMetadataBuilder withUseCounter(Gen<Boolean> useCounter)
{
this.useCounter = Objects.requireNonNull(useCounter);
return this;
}
public TableMetadataBuilder withKnownMemtables()
{
Set<String> known = MemtableParams.knownDefinitions();
// for testing reason, some invalid types are added; filter out
List<String> valid = known.stream().filter(name -> !name.startsWith("test_")).collect(Collectors.toList());
memtableKeyGen = SourceDSL.arbitrary().pick(valid);
paramsBuilder.withKnownMemtables();
return this;
}
@ -293,38 +599,44 @@ public final class CassandraGenerators
return this;
}
public TableMetadataBuilder withDefaultTypeGen(Gen<AbstractType<?>> typeGen)
public TableMetadataBuilder withDefaultTypeGen(TypeGenBuilder typeGen)
{
this.defaultTypeGen = typeGen;
return this;
}
public TableMetadataBuilder withPrimaryColumnTypeGen(Gen<AbstractType<?>> typeGen)
public TableMetadataBuilder withoutEmpty()
{
defaultTypeGen.withoutEmpty();
return this;
}
public TableMetadataBuilder withPrimaryColumnTypeGen(TypeGenBuilder typeGen)
{
withPartitionColumnTypeGen(typeGen);
withClusteringColumnTypeGen(typeGen);
return this;
}
public TableMetadataBuilder withPartitionColumnTypeGen(Gen<AbstractType<?>> typeGen)
public TableMetadataBuilder withPartitionColumnTypeGen(TypeGenBuilder typeGen)
{
this.partitionColTypeGen = typeGen;
return this;
}
public TableMetadataBuilder withClusteringColumnTypeGen(Gen<AbstractType<?>> typeGen)
public TableMetadataBuilder withClusteringColumnTypeGen(TypeGenBuilder typeGen)
{
this.clusteringColTypeGen = typeGen;
return this;
}
public TableMetadataBuilder withStaticColumnTypeGen(Gen<AbstractType<?>> typeGen)
public TableMetadataBuilder withStaticColumnTypeGen(TypeGenBuilder typeGen)
{
this.staticColTypeGen = typeGen;
return this;
}
public TableMetadataBuilder withRegularColumnTypeGen(Gen<AbstractType<?>> typeGen)
public TableMetadataBuilder withRegularColumnTypeGen(TypeGenBuilder typeGen)
{
this.regularColTypeGen = typeGen;
return this;
@ -343,42 +655,92 @@ public final class CassandraGenerators
public TableMetadata build(RandomnessSource rnd)
{
if (partitionColTypeGen == null && clusteringColTypeGen == null)
withPrimaryColumnTypeGen(Generators.filter(defaultTypeGen, t -> !AbstractTypeGenerators.UNSAFE_EQUALITY.contains(t.getClass())));
Gen<AbstractType<?>> partitionColTypeGen = withoutUnsafeEquality(new TypeGenBuilder(this.partitionColTypeGen != null ? this.partitionColTypeGen : defaultTypeGen)).build();
Gen<AbstractType<?>> clusteringColTypeGen = withoutUnsafeEquality(new TypeGenBuilder(this.clusteringColTypeGen != null ? this.clusteringColTypeGen : defaultTypeGen)).build();
Gen<AbstractType<?>> staticColTypeGen = (this.staticColTypeGen != null ? this.staticColTypeGen : defaultTypeGen).build();
Gen<AbstractType<?>> regularColTypeGen = (this.regularColTypeGen != null ? this.regularColTypeGen : defaultTypeGen).build();
String ks = ksNameGen.generate(rnd);
String tableName = tableNameGen.generate(rnd);
TableParams.Builder params = TableParams.builder();
if (memtableKeyGen != null)
params.memtable(MemtableParams.get(memtableKeyGen.generate(rnd)));
TableMetadata.Builder builder = TableMetadata.builder(ks, tableName, tableIdGen.generate(rnd))
.partitioner(partitioners().generate(rnd))
.kind(tableKindGen.generate(rnd))
.isCounter(BOOLEAN_GEN.generate(rnd))
.params(params.build());
AbstractTypeGenerators.overrideUDTKeyspace(ks);
try
{
String tableName = tableNameGen.generate(rnd);
TableParams params = paramsBuilder.build().generate(rnd);
boolean isCounter = useCounter.generate(rnd);
TableMetadata.Builder builder = TableMetadata.builder(ks, tableName, tableIdGen.generate(rnd))
.partitioner(partitionerGen.generate(rnd))
.kind(tableKindGen.generate(rnd))
.isCounter(isCounter)
.params(params);
int numPartitionColumns = numPartitionColumnsGen.generate(rnd);
int numClusteringColumns = numClusteringColumnsGen.generate(rnd);
int numRegularColumns = numRegularColumnsGen.generate(rnd);
int numStaticColumns = numStaticColumnsGen.generate(rnd);
int numPartitionColumns = numPartitionColumnsGen.generate(rnd);
int numClusteringColumns = numClusteringColumnsGen.generate(rnd);
Set<String> createdColumnNames = new HashSet<>();
for (int i = 0; i < numPartitionColumns; i++)
builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.PARTITION_KEY, createdColumnNames, partitionColTypeGen == null ? defaultTypeGen : partitionColTypeGen, rnd));
for (int i = 0; i < numClusteringColumns; i++)
builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.CLUSTERING, createdColumnNames, clusteringColTypeGen == null ? defaultTypeGen : clusteringColTypeGen, rnd));
for (int i = 0; i < numStaticColumns; i++)
builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.STATIC, createdColumnNames, staticColTypeGen == null ? defaultTypeGen : staticColTypeGen, rnd));
for (int i = 0; i < numRegularColumns; i++)
builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.REGULAR, createdColumnNames, regularColTypeGen == null ? defaultTypeGen : regularColTypeGen, rnd));
ColumnNameGen nameGen;
if (columnNameGen != null)
{
nameGen = columnNameGen;
}
else
{
Set<String> createdColumnNames = new HashSet<>();
// filter for unique names
nameGen = (r, i1, i2) -> {
String str;
while (!createdColumnNames.add(str = IDENTIFIER_GEN.generate(r)))
{
}
return str;
};
}
for (int i = 0; i < numPartitionColumns; i++)
builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.PARTITION_KEY, i, nameGen, partitionColTypeGen, rnd));
for (int i = 0; i < numClusteringColumns; i++)
builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.CLUSTERING, i, nameGen, clusteringColTypeGen, rnd));
return builder.build();
if (isCounter)
{
builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.REGULAR, 0, nameGen, ignore -> CounterColumnType.instance, rnd));
}
else
{
int numRegularColumns = numRegularColumnsGen.generate(rnd);
int numStaticColumns = numStaticColumnsGen.generate(rnd);
for (int i = 0; i < numStaticColumns; i++)
builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.STATIC, i, nameGen, staticColTypeGen, rnd));
for (int i = 0; i < numRegularColumns; i++)
builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.REGULAR, i, nameGen, regularColTypeGen, rnd));
}
return builder.build();
}
finally
{
AbstractTypeGenerators.clearUDTKeyspace();
}
}
}
public static Gen<ColumnMetadata> columnMetadataGen(Gen<ColumnMetadata.Kind> kindGen, Gen<AbstractType<?>> typeGen)
{
Gen<String> ksNameGen = CassandraGenerators.KEYSPACE_NAME_GEN;
Gen<String> tableNameGen = IDENTIFIER_GEN;
return rs -> {
String ks = ksNameGen.generate(rs);
String table = tableNameGen.generate(rs);
ColumnMetadata.Kind kind = kindGen.generate(rs);
return createColumnDefinition(ks, table, kind, 0, (r, i1, i2) -> IDENTIFIER_GEN.generate(r), typeGen, rs);
};
}
public interface ColumnNameGen
{
String next(RandomnessSource rs, ColumnMetadata.Kind kind, int kindOffset);
}
private static ColumnMetadata createColumnDefinition(String ks, String table,
ColumnMetadata.Kind kind,
Set<String> createdColumnNames, /* This is mutated to check for collisions, so has a side effect outside of normal random generation */
int kindOffset,
ColumnNameGen nameGen,
Gen<AbstractType<?>> typeGen,
RandomnessSource rnd)
{
@ -396,14 +758,12 @@ public final class CassandraGenerators
// when working on a clustering column, add in reversed types periodically
typeGen = allowReversed(typeGen);
}
// filter for unique names
String str;
while (!createdColumnNames.add(str = IDENTIFIER_GEN.generate(rnd)))
{
}
String str = nameGen.next(rnd, kind, kindOffset);
ColumnIdentifier name = new ColumnIdentifier(str, true);
int position = !kind.isPrimaryKeyKind() ? -1 : (int) rnd.next(Constraint.between(0, 30));
return new ColumnMetadata(ks, table, name, typeGen.generate(rnd), position, kind, null);
int position = !kind.isPrimaryKeyKind() ? -1 : kindOffset;
AbstractType<?> type = typeGen.generate(rnd);
return new ColumnMetadata(ks, table, name, type, position, kind, null);
}
public static Gen<ByteBuffer> partitionKeyDataGen(TableMetadata metadata)
@ -425,27 +785,7 @@ public final class CassandraGenerators
public static Gen<ByteBuffer[]> data(TableMetadata metadata, @Nullable Gen<ValueDomain> valueDomainGen)
{
AbstractTypeGenerators.TypeSupport<?>[] types = new AbstractTypeGenerators.TypeSupport[metadata.columns().size()];
Iterator<ColumnMetadata> it = metadata.allColumnsInSelectOrder();
int partitionColumns = metadata.partitionKeyColumns().size();
int clusteringColumns = metadata.clusteringColumns().size();
int primaryKeyColumns = partitionColumns + clusteringColumns;
for (int i = 0; it.hasNext(); i++)
{
ColumnMetadata col = it.next();
types[i] = AbstractTypeGenerators.getTypeSupportWithNulls(col.type, i < partitionColumns ? null : valueDomainGen);
if (i < partitionColumns)
types[i] = types[i].withoutEmptyData();
if (i >= partitionColumns && i < primaryKeyColumns)
// clustering doesn't allow null...
types[i] = types[i].mapBytes(b -> b == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : b);
}
return rnd -> {
ByteBuffer[] row = new ByteBuffer[types.length];
for (int i = 0; i < row.length; i++)
row[i] = types[i].bytesGen().generate(rnd);
return row;
};
return new DataGeneratorBuilder(metadata).withValueDomain(valueDomainGen).build();
}
/**
@ -573,7 +913,8 @@ public final class CassandraGenerators
public static Gen<Token> byteOrderToken()
{
Constraint size = Constraint.between(0, 10);
// empty token only happens if partition key is byte[0], which isn't allowed
Constraint size = Constraint.between(1, 10);
Constraint byteRange = Constraint.between(Byte.MIN_VALUE, Byte.MAX_VALUE);
return rs -> {
byte[] token = new byte[Math.toIntExact(rs.next(size))];
@ -585,7 +926,9 @@ public final class CassandraGenerators
public static Gen<Token> randomPartitionerToken()
{
Constraint domain = Constraint.none();
// valid range is -1 -> 2^127
Constraint domain = Constraint.between(-1, Long.MAX_VALUE);
// TODO (coverage): handle the range [2^63-1, 2^127]
return rs -> new RandomPartitioner.BigIntegerToken(BigInteger.valueOf(rs.next(domain)));
}
@ -595,7 +938,7 @@ public final class CassandraGenerators
return rs -> partitioner.getToken(bytes.generate(rs));
}
public static Gen<IPartitioner> localPartitioner()
public static Gen<LocalPartitioner> localPartitioner()
{
return AbstractTypeGenerators.safeTypeGen().map(LocalPartitioner::new);
}
@ -610,9 +953,25 @@ public final class CassandraGenerators
};
}
public static Gen<Token> reversedLongLocalToken()
{
Constraint range = Constraint.between(0, Long.MAX_VALUE);
return rs -> new ReversedLongLocalPartitioner.ReversedLongLocalToken(rs.next(range));
}
public static Gen<ByteBuffer> reversedLongLocalKeys()
{
Constraint range = Constraint.between(0, Long.MAX_VALUE);
return rs -> {
long value = rs.next(range);
return ByteBufferUtil.bytes(value);
};
}
public static Gen<Token> orderPreservingToken()
{
Gen<String> string = Generators.utf8(0, 10);
// empty token only happens if partition key is byte[0], which isn't allowed
Gen<String> string = Generators.utf8(1, 10);
return rs -> new OrderPreservingPartitioner.StringToken(string.generate(rs));
}
@ -625,23 +984,38 @@ public final class CassandraGenerators
private enum SupportedPartitioners
{
Murmur(ignore -> Murmur3Partitioner.instance),
ByteOrdered(ignore -> ByteOrderedPartitioner.instance),
Random(ignore -> RandomPartitioner.instance),
Local(localPartitioner()),
OrderPreserving(ignore -> OrderPreservingPartitioner.instance);
Murmur(Murmur3Partitioner.class, ignore -> Murmur3Partitioner.instance),
ByteOrdered(ByteOrderedPartitioner.class, ignore -> ByteOrderedPartitioner.instance),
Random(RandomPartitioner.class, ignore -> RandomPartitioner.instance),
Local(LocalPartitioner.class, localPartitioner()),
OrderPreserving(OrderPreservingPartitioner.class, ignore -> OrderPreservingPartitioner.instance);
private final Gen<IPartitioner> partitioner;
private final Class<? extends IPartitioner> clazz;
private final Gen<? extends IPartitioner> partitioner;
SupportedPartitioners(Gen<IPartitioner> partitionerGen)
<T extends IPartitioner> SupportedPartitioners(Class<T> clazz, Gen<T> partitionerGen)
{
this.clazz = clazz;
partitioner = partitionerGen;
}
public Gen<IPartitioner> partitioner()
public Gen<? extends IPartitioner> partitioner()
{
return partitioner;
}
public static Set<Class<? extends IPartitioner>> knownPartitioners()
{
ImmutableSet.Builder<Class<? extends IPartitioner>> builder = ImmutableSet.builder();
for (SupportedPartitioners p : values())
builder.add(p.clazz);
return builder.build();
}
}
public static Set<Class<? extends IPartitioner>> knownPartitioners()
{
return SupportedPartitioners.knownPartitioners();
}
public static Gen<IPartitioner> partitioners()
@ -650,6 +1024,7 @@ public final class CassandraGenerators
.flatMap(SupportedPartitioners::partitioner);
}
public static Gen<IPartitioner> nonLocalPartitioners()
{
return SourceDSL.arbitrary().enumValues(SupportedPartitioners.class)
@ -657,6 +1032,24 @@ public final class CassandraGenerators
.flatMap(SupportedPartitioners::partitioner);
}
/**
* For {@link LocalPartitioner} it can have a very complex type which can lead to generating data larger than
* allowed in a primary key. If a test needs to filter out those cases, can just
* {@code .map(CassandraGenerators::simplify)} to resolve.
*/
public static IPartitioner simplify(IPartitioner partitioner)
{
// serializers require tokens to fit within 1 << 16, but that makes the test flakey when LocalPartitioner with a nested type is found...
if (!(partitioner instanceof LocalPartitioner)) return partitioner;
if (!shouldSimplify(partitioner.getTokenValidator())) return partitioner;
return new LocalPartitioner(Int32Type.instance);
}
private static boolean shouldSimplify(AbstractType<?> type)
{
return AbstractTypeGenerators.contains(type, t -> t.isCollection());
}
public static Gen<Token> token()
{
return partitioners().flatMap(CassandraGenerators::token);
@ -810,4 +1203,174 @@ public final class CassandraGenerators
return Duration.newInstance(months, days, nanoseconds);
};
}
public static Gen<DecoratedKey> decoratedKeys()
{
return decoratedKeys(partitioners(), Generators.bytes(0, 100));
}
public static Gen<DecoratedKey> decoratedKeys(Gen<IPartitioner> partitionerGen)
{
return decoratedKeys(partitionerGen, Generators.bytes(0, 100));
}
public static Gen<DecoratedKey> decoratedKeys(Gen<IPartitioner> partitionerGen, Gen<ByteBuffer> keyGen)
{
return rs -> {
IPartitioner partitioner = partitionerGen.generate(rs);
Gen<ByteBuffer> valueGen = keyGen;
if (partitioner instanceof LocalPartitioner)
{
LocalPartitioner lp = (LocalPartitioner) partitioner;
valueGen = AbstractTypeGenerators.getTypeSupport(lp.getTokenValidator()).bytesGen();
}
else if (partitioner instanceof ReversedLongLocalPartitioner)
{
valueGen = reversedLongLocalKeys();
}
return partitioner.decorateKey(valueGen.generate(rs));
};
}
public static void visitUDTs(TableMetadata metadata, Consumer<UserType> fn)
{
Set<UserType> udts = CassandraGenerators.extractUDTs(metadata);
if (!udts.isEmpty())
{
Deque<UserType> pending = new ArrayDeque<>(udts);
Set<ByteBuffer> visited = new HashSet<>();
while (!pending.isEmpty())
{
UserType next = pending.poll();
Set<UserType> subTypes = AbstractTypeGenerators.extractUDTs(next);
subTypes.remove(next); // it includes self
if (subTypes.isEmpty() || subTypes.stream().allMatch(t -> visited.contains(t.name)))
{
fn.accept(next);
visited.add(next.name);
}
else
{
pending.add(next);
}
}
}
}
public static class DataGeneratorBuilder
{
private final TableMetadata metadata;
@Nullable
private Gen<ValueDomain> valueDomainGen = null;
public DataGeneratorBuilder(TableMetadata metadata)
{
this.metadata = metadata;
}
public DataGeneratorBuilder withValueDomain(@Nullable Gen<ValueDomain> valueDomainGen)
{
this.valueDomainGen = valueDomainGen;
return this;
}
public Gen<Gen<ByteBuffer[]>> build(Gen<Integer> numUniqPartitionsGen)
{
AbstractTypeGenerators.TypeSupport<?>[] types = typeSupport();
return rnd -> {
int numPartitions = numUniqPartitionsGen.generate(rnd);
Set<List<ByteBuffer>> partitions = Sets.newHashSetWithExpectedSize(numPartitions);
int partitionColumns = metadata.partitionKeyColumns().size();
for (int i = 0; i < numPartitions; i++)
{
List<ByteBuffer> pk = new ArrayList<>(partitionColumns);
int attempts = 0;
do
{
attempts++;
pk.clear();
for (int c = 0; c < partitionColumns; c++)
pk.add(types[c].bytesGen().generate(rnd));
}
while (!partitions.add(pk) && attempts < 42);
}
List<List<ByteBuffer>> deterministicOrder = new ArrayList<>(partitions);
deterministicOrder.sort((a, b) -> {
int rc = 0;
for (int i = 0; i < a.size(); i++)
{
rc = a.get(i).compareTo(b.get(i));
if (rc != 0) return rc;
}
return rc;
});
Gen<List<ByteBuffer>> pkGen = SourceDSL.arbitrary().pick(deterministicOrder);
return next -> {
// select partition
List<ByteBuffer> pk = pkGen.generate(next);
// generate rest
ByteBuffer[] row = new ByteBuffer[types.length];
for (int i = 0; i < pk.size(); i++)
row[i] = pk.get(i);
for (int i = partitionColumns; i < row.length; i++)
row[i] = types[i].bytesGen().generate(rnd);
return row;
};
};
}
public Gen<ByteBuffer[]> build()
{
AbstractTypeGenerators.TypeSupport<?>[] types = typeSupport();
return rnd -> {
ByteBuffer[] row = new ByteBuffer[types.length];
for (int i = 0; i < row.length; i++)
row[i] = types[i].bytesGen().generate(rnd);
return row;
};
}
private AbstractTypeGenerators.TypeSupport<?>[] typeSupport()
{
AbstractTypeGenerators.TypeSupport<?>[] types = new AbstractTypeGenerators.TypeSupport[metadata.columns().size()];
Iterator<ColumnMetadata> it = metadata.allColumnsInSelectOrder();
int partitionColumns = metadata.partitionKeyColumns().size();
int clusteringColumns = metadata.clusteringColumns().size();
int primaryKeyColumns = partitionColumns + clusteringColumns;
for (int i = 0; it.hasNext(); i++)
{
ColumnMetadata col = it.next();
types[i] = AbstractTypeGenerators.getTypeSupportWithNulls(col.type, i < partitionColumns ? null : valueDomainGen);
if (i < partitionColumns)
types[i] = types[i].withoutEmptyData();
if (i >= partitionColumns && i < primaryKeyColumns)
// clustering doesn't allow null...
types[i] = types[i].mapBytes(b -> b == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : b);
}
return types;
}
}
private enum EpochConstants { FIRST, EMPTY, UPGRADE_STARTUP, UPGRADE_GOSSIP}
public static Gen<Epoch> epochs()
{
return rnd -> {
if (SourceDSL.booleans().all().generate(rnd))
{
switch (SourceDSL.arbitrary().enumValues(EpochConstants.class).generate(rnd))
{
case FIRST: return Epoch.FIRST;
case EMPTY: return Epoch.EMPTY;
case UPGRADE_STARTUP: return Epoch.UPGRADE_STARTUP;
case UPGRADE_GOSSIP: return Epoch.UPGRADE_GOSSIP;
default: throw new UnsupportedOperationException();
}
}
return Epoch.create(SourceDSL.longs().between(2, Long.MAX_VALUE).generate(rnd));
};
}
}

102
test/unit/org/apache/cassandra/utils/Generators.java
View File

@ -26,8 +26,11 @@ import java.nio.charset.StandardCharsets;
import java.sql.Timestamp;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
@ -39,6 +42,7 @@ import org.apache.commons.lang3.ArrayUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.cql3.ReservedKeywords;
import org.quicktheories.core.Gen;
import org.quicktheories.core.RandomnessSource;
import org.quicktheories.generators.SourceDSL;
@ -58,9 +62,12 @@ public final class Generators
private static final Constraint DNS_DOMAIN_PARTS_CONSTRAINT = Constraint.between(1, 127);
private static final char CHAR_UNDERSCORE = 95;
private static final char[] LETTER_DOMAIN = createLetterDomain();
private static final Constraint LETTER_CONSTRAINT = Constraint.between(0, LETTER_DOMAIN.length - 1).withNoShrinkPoint();
private static final char[] LETTER_OR_DIGIT_DOMAIN = createLetterOrDigitDomain();
private static final char[] LETTER_OR_DIGIT_DOMAIN_WITH_UNDERSCORE = createLetterOrDigitDomainWithUnderscore();
private static final Constraint LETTER_OR_DIGIT_CONSTRAINT = Constraint.between(0, LETTER_OR_DIGIT_DOMAIN.length - 1).withNoShrinkPoint();
private static final char[] REGEX_WORD_DOMAIN = createRegexWordDomain();
private static final Constraint REGEX_WORD_CONSTRAINT = Constraint.between(0, REGEX_WORD_DOMAIN.length - 1).withNoShrinkPoint();
@ -68,6 +75,12 @@ public final class Generators
private static final Constraint DNS_DOMAIN_PART_CONSTRAINT = Constraint.between(0, DNS_DOMAIN_PART_DOMAIN.length - 1).withNoShrinkPoint();
public static final Gen<String> IDENTIFIER_GEN = Generators.regexWord(SourceDSL.integers().between(1, 50));
public static final Gen<String> SYMBOL_GEN = Generators.filter(symbolGen(SourceDSL.integers().between(1, 48)), s -> !ReservedKeywords.isReserved(s));
public static Gen<String> symbolGen(Gen<Integer> size)
{
return string(size, LETTER_OR_DIGIT_DOMAIN_WITH_UNDERSCORE, (index, c) -> !(index == 0 && !Character.isLetter(c)));
}
public static Gen<Character> letterOrDigit()
{
@ -259,7 +272,23 @@ public final class Generators
return charArray(sizes, domain).map(c -> new String(c));
}
public static Gen<String> string(Gen<Integer> sizes, char[] domain, IntCharBiPredicate fn)
{
// note, map is overloaded so String::new is ambugious to javac, so need a lambda here
return charArray(sizes, domain, fn).map(c -> new String(c));
}
public interface IntCharBiPredicate
{
boolean test(int a, char b);
}
public static Gen<char[]> charArray(Gen<Integer> sizes, char[] domain)
{
return charArray(sizes, domain, (a, b) -> true);
}
public static Gen<char[]> charArray(Gen<Integer> sizes, char[] domain, IntCharBiPredicate fn)
{
Constraint constraints = Constraint.between(0, domain.length - 1).withNoShrinkPoint();
Gen<char[]> gen = td -> {
@ -267,8 +296,13 @@ public final class Generators
char[] is = new char[size];
for (int i = 0; i != size; i++)
{
int idx = (int) td.next(constraints);
is[i] = domain[idx];
char c;
do
{
int idx = (int) td.next(constraints);
c = domain[idx];
} while (!fn.test(i, c));
is[i] = c;
}
return is;
};
@ -309,6 +343,14 @@ public final class Generators
return domain;
}
private static char[] createLetterOrDigitDomainWithUnderscore()
{
char[] domain = new char[LETTER_OR_DIGIT_DOMAIN.length + 1];
System.arraycopy(LETTER_OR_DIGIT_DOMAIN, 0, domain, 0, LETTER_OR_DIGIT_DOMAIN.length);
domain[domain.length - 1] = CHAR_UNDERSCORE;
return domain;
}
private static char[] createRegexWordDomain()
{
// \w == [a-zA-Z_0-9] the only difference with letterOrDigit is the addition of _
@ -326,6 +368,11 @@ public final class Generators
return bytes(min, max, SourceDSL.arbitrary().constant(BBCases.HEAP));
}
public static Gen<ByteBuffer> directBytes(int min, int max)
{
return bytes(min, max, SourceDSL.arbitrary().pick(BBCases.DIRECT, BBCases.READ_ONLY_DIRECT));
}
public static Gen<ByteBuffer> bytesAnyType(int min, int max)
{
return bytes(min, max, SourceDSL.arbitrary().enumValues(BBCases.class));
@ -371,7 +418,7 @@ public final class Generators
return bb;
}
/**
/**
* Implements a valid utf-8 generator.
*
* Implementation note, currently relies on getBytes to strip out non-valid utf-8 chars, so is slow
@ -379,9 +426,9 @@ public final class Generators
public static Gen<String> utf8(int min, int max)
{
return SourceDSL.strings()
.basicMultilingualPlaneAlphabet()
.ofLengthBetween(min, max)
.map(s -> new String(s.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8));
.basicMultilingualPlaneAlphabet()
.ofLengthBetween(min, max)
.map(s -> new String(s.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8));
}
public static Gen<BigInteger> bigInt()
@ -432,6 +479,28 @@ public final class Generators
return filter(gen, dedup::add);
}
public static <T> Gen<Set<T>> set(Gen<T> gen, Gen<Integer> sizeGen)
{
return rnd -> {
Set<T> set = new HashSet<>();
int size = sizeGen.generate(rnd);
for (int i = 0; i < size; i++)
{
while (!set.add(gen.generate(rnd))) {}
}
return set;
};
}
public static <T extends Comparable<? super T>> Gen<List<T>> uniqueList(Gen<T> gen, Gen<Integer> sizeGen)
{
return set(gen, sizeGen).map(t -> {
List<T> list = new ArrayList<>(t);
list.sort(Comparator.naturalOrder());
return list;
});
}
public static <T> Gen<T> cached(Gen<T> gen)
{
Object cacheMissed = new Object();
@ -466,7 +535,7 @@ public final class Generators
static
{
long blobSeed = TEST_BLOB_SHARED_SEED.getLong(System.currentTimeMillis());
long blobSeed = TEST_BLOB_SHARED_SEED.getLong();
logger.info("Shared blob Gen used seed {}", blobSeed);
Random random = new Random(blobSeed);
@ -542,4 +611,23 @@ public final class Generators
return qt.generate(r);
};
}
public static Gen<TimeUUID> timeUUID()
{
ZonedDateTime now = ZonedDateTime.of(2020, 8, 20,
0, 0, 0, 0, ZoneOffset.UTC);
ZonedDateTime startOfTime = now.minusYears(50);
ZonedDateTime endOfDays = now.plusYears(50);
Constraint micros = Constraint.between(toMicros(startOfTime), toMicros(endOfDays));
return rs -> {
long nowMicro = rs.next(micros);
long lsb = rs.next(Constraint.none());
return new TimeUUID(TimeUUID.unixMicrosToRawTimestamp(nowMicro), lsb);
};
}
private static long toMicros(ZonedDateTime zdt)
{
return zdt.toInstant().toEpochMilli() * 1000 + zdt.getNano() / 1000;
}
}