diff --git a/build.xml b/build.xml index e51299bd46..a2ff92a1a4 100644 --- a/build.xml +++ b/build.xml @@ -710,9 +710,9 @@ - + - + diff --git a/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java b/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java index 7d13844eb0..b13bb1596d 100644 --- a/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java +++ b/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java @@ -234,6 +234,13 @@ public class ByteArrayAccessor implements ValueAccessor return new UUID(getLong(value, 0), getLong(value, 8)); } + @Override + public int putByte(byte[] dst, int offset, byte value) + { + dst[offset] = value; + return TypeSizes.BYTE_SIZE; + } + @Override public int putShort(byte[] dst, int offset, short value) { diff --git a/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java b/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java index a0f9c1d8d0..57a13735d3 100644 --- a/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java +++ b/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java @@ -238,6 +238,13 @@ public class ByteBufferAccessor implements ValueAccessor return UUIDGen.getUUID(value); } + @Override + public int putByte(ByteBuffer dst, int offset, byte value) + { + dst.put(dst.position() + offset, value); + return TypeSizes.BYTE_SIZE; + } + @Override public int putShort(ByteBuffer dst, int offset, short value) { diff --git a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java index 5df3600995..90f0992915 100644 --- a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java +++ b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java @@ -22,6 +22,7 @@ import java.nio.charset.CharacterCodingException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +61,8 @@ public class DynamicCompositeType extends AbstractCompositeType { private static final Logger logger = LoggerFactory.getLogger(DynamicCompositeType.class); - private final Map> aliases; + @VisibleForTesting + public final Map> aliases; // interning instances private static final ConcurrentHashMap>, DynamicCompositeType> instances = new ConcurrentHashMap<>(); @@ -378,7 +380,8 @@ public class DynamicCompositeType extends AbstractCompositeType * A comparator that always sorts it's first argument before the second * one. */ - private static class FixedValueComparator extends AbstractType + @VisibleForTesting + public static class FixedValueComparator extends AbstractType { public static final FixedValueComparator alwaysLesserThan = new FixedValueComparator(-1); public static final FixedValueComparator alwaysGreaterThan = new FixedValueComparator(1); diff --git a/src/java/org/apache/cassandra/db/marshal/TupleType.java b/src/java/org/apache/cassandra/db/marshal/TupleType.java index 83fbb25d54..f76a074d2c 100644 --- a/src/java/org/apache/cassandra/db/marshal/TupleType.java +++ b/src/java/org/apache/cassandra/db/marshal/TupleType.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.List; import java.util.regex.Pattern; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Objects; import com.google.common.collect.Lists; @@ -63,7 +64,8 @@ public class TupleType extends AbstractType this(types, true); } - protected TupleType(List> types, boolean freezeInner) + @VisibleForTesting + public TupleType(List> types, boolean freezeInner) { super(ComparisonType.CUSTOM); diff --git a/src/java/org/apache/cassandra/db/marshal/TypeParser.java b/src/java/org/apache/cassandra/db/marshal/TypeParser.java index 7416d49143..db0d33d251 100644 --- a/src/java/org/apache/cassandra/db/marshal/TypeParser.java +++ b/src/java/org/apache/cassandra/db/marshal/TypeParser.java @@ -225,7 +225,7 @@ public class TypeParser String alias = readNextIdentifier(); if (alias.length() != 1) - throwSyntaxError("An alias should be a single character"); + throwSyntaxError("An alias should be a single character: '" + alias + "', string: " + str); char aliasChar = alias.charAt(0); if (aliasChar < 33 || aliasChar > 127) throwSyntaxError("An alias should be a single character in [0..9a..bA..B-+._&]"); diff --git a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java index 10532ff304..fd09f31023 100644 --- a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java +++ b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java @@ -322,6 +322,12 @@ public interface ValueAccessor /** returns a UUID from offset 0 */ UUID toUUID(V value); + /** + * writes the byte value {@param value} to {@param dst} at offset {@param offset} + * @return the number of bytes written to {@param value} + */ + int putByte(V dst, int offset, byte value); + /** * writes the short value {@param value} to {@param dst} at offset {@param offset} * @return the number of bytes written to {@param value} diff --git a/src/java/org/apache/cassandra/utils/CassandraVersion.java b/src/java/org/apache/cassandra/utils/CassandraVersion.java index 56290fb3c2..a24c14c148 100644 --- a/src/java/org/apache/cassandra/utils/CassandraVersion.java +++ b/src/java/org/apache/cassandra/utils/CassandraVersion.java @@ -279,4 +279,9 @@ public class CassandraVersion implements Comparable sb.append('+').append(StringUtils.join(build, ".")); return sb.toString(); } + + public String toMajorMinorString() + { + return String.format("%d.%d", major, minor); + } } diff --git a/src/java/org/apache/cassandra/utils/FastByteOperations.java b/src/java/org/apache/cassandra/utils/FastByteOperations.java index ffaee1f084..2a86712951 100644 --- a/src/java/org/apache/cassandra/utils/FastByteOperations.java +++ b/src/java/org/apache/cassandra/utils/FastByteOperations.java @@ -70,6 +70,11 @@ public class FastByteOperations return BestHolder.BEST.compare(b1, b2); } + public static int compareUnsigned(byte[] b1, byte[] b2) + { + return compareUnsigned(b1, 0, b1.length, b2, 0, b2.length); + } + public static void copy(byte[] src, int srcPosition, byte[] trg, int trgPosition, int length) { BestHolder.BEST.copy(src, srcPosition, trg, trgPosition, length); diff --git a/test/data/types-compatibility/4.0.json.gz b/test/data/types-compatibility/4.0.json.gz new file mode 100644 index 0000000000..8f220d7fb3 Binary files /dev/null and b/test/data/types-compatibility/4.0.json.gz differ diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java index f9ef4cc253..2698617217 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java @@ -34,6 +34,8 @@ import org.junit.Test; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.SchemaCQLHelper; +import org.apache.cassandra.db.marshal.DecimalType; +import org.apache.cassandra.db.marshal.DurationType; import org.apache.cassandra.db.marshal.TupleType; import org.apache.cassandra.utils.AbstractTypeGenerators.TypeSupport; import org.quicktheories.core.Gen; @@ -334,7 +336,8 @@ public class TupleTypeTest extends CQLTester private static Gen typesAndRowsGen(int numRows) { - Gen typeGen = tupleTypeGen(primitiveTypeGen(), SourceDSL.integers().between(1, 10)); + // duration type is invalid for keys and decimal type is problematic for equality checks + Gen typeGen = tupleTypeGen(primitiveTypeGen(DurationType.instance, DecimalType.instance), SourceDSL.integers().between(1, 10)); Set distinctRows = new HashSet<>(numRows); // reuse the memory Gen gen = rnd -> { TypeAndRows c = new TypeAndRows(); diff --git a/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java new file mode 100644 index 0000000000..ed7020496f --- /dev/null +++ b/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java @@ -0,0 +1,998 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.marshal; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.security.CodeSource; +import java.security.ProtectionDomain; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.BiPredicate; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import com.google.common.base.Charsets; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; +import com.google.common.collect.Streams; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.openhft.chronicle.core.util.ThrowingFunction; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.db.rows.ComplexColumnData; +import org.apache.cassandra.db.rows.DeserializationHelper; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.SerializationHelper; +import org.apache.cassandra.db.rows.UnfilteredSerializer; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.AbstractTypeGenerators; +import org.apache.cassandra.utils.CassandraVersion; +import org.apache.cassandra.utils.Collectors3; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.asserts.SoftAssertionsWithLimit; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.description.Description; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.quicktheories.core.Gen; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ConfigurationBuilder; + +import static org.apache.cassandra.db.marshal.AbstractType.ComparisonType.CUSTOM; +import static org.apache.cassandra.utils.AbstractTypeGenerators.UNSUPPORTED; +import static org.apache.cassandra.utils.AbstractTypeGenerators.forEachPrimitiveTypePair; +import static org.apache.cassandra.utils.AbstractTypeGenerators.forEachTypesPair; +import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport; +import static org.apache.cassandra.utils.AbstractTypeGenerators.unfreeze; +import static org.apache.cassandra.utils.AbstractTypeGenerators.unwrap; +import static org.apache.cassandra.utils.ByteBufferUtil.bytesToHex; +import static org.assertj.core.api.Assertions.assertThat; +import static org.quicktheories.QuickTheory.qt; + +@SuppressWarnings({ "unchecked", "rawtypes" }) +public class AbstractTypeTest +{ + private final static Logger logger = LoggerFactory.getLogger(AbstractTypeTest.class); + + private static final Pattern TYPE_PREFIX_PATTERN = Pattern.compile("org\\.apache\\.cassandra\\.db\\.marshal\\."); + + private static final Reflections reflections = new Reflections(new ConfigurationBuilder() + .forPackage("org.apache.cassandra") + .setScanners(Scanners.SubTypes) + .setExpandSuperTypes(true) + .setParallel(true)); + + private static TypesCompatibility currentTypesCompatibility; + private static LoadedTypesCompatibility cassandra40TypesCompatibility; + + private final static String CASSANDRA_VERSION = new CassandraVersion(FBUtilities.getReleaseVersionString()).toMajorMinorString(); + private final static Path BASE_OUTPUT_PATH = Paths.get("test", "data", "types-compatibility"); + + @BeforeClass + public static void beforeClass() throws IOException + { + DatabaseDescriptor.daemonInitialization(); + cassandra40TypesCompatibility = new LoadedTypesCompatibility(compatibilityFile(CassandraVersion.CASSANDRA_4_0.toMajorMinorString()), ImmutableSet.of()); + currentTypesCompatibility = new CurrentTypesCompatibility(); + } + + @Test + public void allTypesCovered() + { + // this test just makes sure that all types are covered and no new type is left out + Set> subTypes = reflections.getSubTypesOf(AbstractType.class); + Set> coverage = AbstractTypeGenerators.knownTypes(); + StringBuilder sb = new StringBuilder(); + for (Class klass : Sets.difference(subTypes, coverage)) + { + if (Modifier.isAbstract(klass.getModifiers())) + continue; + if (isTestType(klass)) + continue; + String name = klass.getCanonicalName(); + if (name == null) + name = klass.getName(); + sb.append(name).append('\n'); + } + if (sb.length() > 0) + throw new AssertionError("Uncovered types:\n" + sb); + } + + private boolean isTestType(Class klass) + { + String name = klass.getCanonicalName(); + if (name == null) + name = klass.getName(); + if (name == null) + name = klass.toString(); + if (name.contains("Test")) + return true; + ProtectionDomain domain = klass.getProtectionDomain(); + if (domain == null) return false; + CodeSource src = domain.getCodeSource(); + if (src == null) return false; + return "test".equals(new File(src.getLocation().getPath()).getName()); + } + + @Test + public void testAssumedCompatibility() + { + SoftAssertions assertions = new SoftAssertionsWithLimit(100); + forEachPrimitiveTypePair((l, r) -> currentTypesCompatibility.checkExpectedTypeCompatibility(l, r, assertions)); + assertions.assertAll(); + } + + @Test + public void testBackwardCompatibility() + { + testBackwardCompatibility(currentTypesCompatibility, cassandra40TypesCompatibility); // test compatibility with the previous patch release + cassandra40TypesCompatibility.assertLoaded(); + } + + public void testBackwardCompatibility(TypesCompatibility upgradeTo, TypesCompatibility upgradeFrom) + { + SoftAssertions assertions = new SoftAssertionsWithLimit(100); + + assertions.assertThat(upgradeTo.knownTypes()).containsAll(upgradeFrom.knownTypes()); + assertions.assertThat(upgradeTo.primitiveTypes()).containsAll(upgradeFrom.primitiveTypes()); + + // for compatibility, we ensure that this version can read values of all the types the previous version can write + assertions.assertThat(upgradeTo.multiCellSupportingTypesForReading()).containsAll(upgradeFrom.multiCellSupportingTypes()); + + forEachTypesPair(true, (l, r) -> { + if (upgradeFrom.expectCompatibleWith(l, r)) + assertions.assertThat(upgradeTo.expectCompatibleWith(l, r)).describedAs(isCompatibleWithDesc(l, r)).isTrue(); + if (upgradeFrom.expectSerializationCompatibleWith(l, r)) + assertions.assertThat(upgradeTo.expectSerializationCompatibleWith(l, r)).describedAs(isSerializationCompatibleWithDesc(l, r)).isTrue(); + if (upgradeFrom.expectValueCompatibleWith(l, r)) + assertions.assertThat(upgradeTo.expectValueCompatibleWith(l, r)).describedAs(isValueCompatibleWithDesc(l, r)).isTrue(); + }); + + assertions.assertAll(); + } + + @Test + public void testImplementedCompatibility() + { + SoftAssertions assertions = new SoftAssertionsWithLimit(100); + + forEachTypesPair(true, (l, r) -> { + assertions.assertThat(l.equals(r)).describedAs("equals symmetricity for %s and %s", l, r).isEqualTo(r.equals(l)); + verifyTypesCompatibility(l, r, getTypeSupport(r).valueGen, assertions); + }); + + assertions.assertAll(); + } + + private static Path compatibilityFile(String version) + { + return BASE_OUTPUT_PATH.resolve(String.format("%s.json.gz", version)); + } + + @Test + @Ignore + public void testStoreAllCompatibleTypePairs() throws IOException + { + currentTypesCompatibility.store(compatibilityFile(CASSANDRA_VERSION)); + } + + private static void verifyTypesCompatibility(AbstractType left, AbstractType right, Gen rightGen, SoftAssertions assertions) + { + if (left.equals(right)) + return; + + verifyTypeSerializers(left, right, assertions); + + if (!left.isValueCompatibleWith(right)) + return; + + ColumnMetadata rightColumn1 = new ColumnMetadata("k", "t", ColumnIdentifier.getInterned("c", false), right, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.REGULAR); + ColumnMetadata rightColumn2 = new ColumnMetadata("k", "t", ColumnIdentifier.getInterned("d", false), right, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.REGULAR); + ColumnMetadata leftColumn1 = new ColumnMetadata("k", "t", ColumnIdentifier.getInterned("c", false), left, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.REGULAR); + ColumnMetadata leftColumn2 = new ColumnMetadata("k", "t", ColumnIdentifier.getInterned("d", false), left, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.REGULAR); + + TableMetadata leftTable = TableMetadata.builder("k", "t").addPartitionKeyColumn("pk", EmptyType.instance).addColumn(leftColumn1).addColumn(leftColumn2).build(); + TableMetadata rightTable = TableMetadata.builder("k", "t").addPartitionKeyColumn("pk", EmptyType.instance).addColumn(rightColumn1).addColumn(rightColumn2).build(); + + SerializationHeader leftHeader = new SerializationHeader(false, leftTable, leftTable.regularAndStaticColumns(), EncodingStats.NO_STATS); + SerializationHeader rightHeader = new SerializationHeader(false, rightTable, rightTable.regularAndStaticColumns(), EncodingStats.NO_STATS); + + DeserializationHelper leftHelper = new DeserializationHelper(leftTable, MessagingService.current_version, DeserializationHelper.Flag.LOCAL, ColumnFilter.all(leftTable)); + SerializationHelper rightHelper = new SerializationHelper(rightHeader); + + assertions.assertThatCode(() -> { + qt().withExamples(10).forAll(rightGen).checkAssert(v -> { + // value compatibility means that we can use left's type serializer to decompose a value of right's type + ByteBuffer rightDecomposed = right.decompose(v); + Object leftComposed = left.compose(rightDecomposed); + ByteBuffer leftDecomposed = left.decompose(leftComposed); + assertThat(leftDecomposed.hasRemaining()).describedAs(typeRelDesc(".decompose", left, right)).isEqualTo(rightDecomposed.hasRemaining()); + + // serialization compatibility means that we can read a cell written using right's type serializer with left's type serializer; + // this additinoally imposes the requirement for storing the buffer lenght in the serialized form if the value is of variable length + // as well as, either both types serialize into a single or multiple cells + if (left.isSerializationCompatibleWith(right)) + { + if (!left.isMultiCell() && !right.isMultiCell()) + verifySerializationCompatibilityForSimpleCells(left, right, v, rightTable, rightColumn1, rightHelper, leftHeader, leftHelper, leftColumn1); + else if (currentTypesCompatibility.multiCellSupportingTypes().contains(left.getClass()) && currentTypesCompatibility.multiCellSupportingTypes().contains(right.getClass())) + verifySerializationCompatibilityForComplexCells(left, right, v, rightTable, rightColumn1, rightHelper, leftHeader, leftHelper, leftColumn1); + } + }); + }).describedAs(typeRelDesc("isSerializationCompatibleWith", left, right)).doesNotThrowAnyException(); + + // if types are not (comparison) compatible, no reason to verify that + if (!left.isCompatibleWith(right) || right.comparisonType == AbstractType.ComparisonType.NOT_COMPARABLE || left.comparisonType == AbstractType.ComparisonType.NOT_COMPARABLE) + return; + + // types compatibility means that we can compare values of right's type using left's type comparator additionally + // to types being serialization compatible + if (!left.isMultiCell() && !right.isMultiCell()) + { + // make sure that frozen isCompatibleWith frozen ==> left isCompatibleWith right + assertions.assertThat(unfreeze(left).isCompatibleWith(unfreeze(right))).isTrue(); + + assertions.assertThatCode(() -> qt().withExamples(10) + .forAll(rightGen, rightGen) + .checkAssert((rightValue1, rightValue2) -> verifyComparisonCompatibilityForSimpleCells(left, right, rightValue1, rightValue2))) + .describedAs(typeRelDesc("isCompatibleWith", left, right)).doesNotThrowAnyException(); + } + else if (left.isMultiCell() && right.isMultiCell()) + { + if (currentTypesCompatibility.multiCellSupportingTypes().contains(left.getClass()) && currentTypesCompatibility.multiCellSupportingTypes().contains(right.getClass())) + { + assertions.assertThatCode(() -> qt().withExamples(10) + .forAll(rightGen, rightGen) + .checkAssert((rightValue1, rightValue2) -> verifyComparisonCompatibilityForMultiCell(left, right, rightValue1, rightValue2, rightTable, rightColumn1, rightColumn2, rightHelper, leftHeader, leftHelper, leftColumn1, leftColumn2))) + .describedAs(typeRelDesc("isCompatibleWith", left, right)).doesNotThrowAnyException(); + } + } + } + + /** + * Assert that (comparison) incompatible types which use custom comparison are not using the same serializer. + */ + private static void verifyTypeSerializers(AbstractType l, AbstractType r, SoftAssertions assertions) + { + AbstractType lt = unfreeze(unwrap(l)); + AbstractType rt = unfreeze(unwrap(r)); + + if (lt.comparisonType != CUSTOM && rt.comparisonType != CUSTOM) + return; + + if (lt.isCompatibleWith(rt) && rt.isCompatibleWith(lt)) + return; + + assertions.assertThat(l.getSerializer()).describedAs(typeRelDesc("should have different serializer to", l, r)).isNotEqualTo(r.getSerializer()); + } + + private static int sign(int value) + { + return Integer.compare(value, 0); + } + + private static void verifyComparison(Comparator leftComparator, Comparator rightComparator, T lv1, T lv2, T rv1, T rv2, int expectedResult, Function desc) + { + SoftAssertions checks = new SoftAssertions(); + + expectedResult = sign(expectedResult); + + // first just check that the comparison is antisymmetric + checks.assertThat(sign(rightComparator.compare(rv2, rv1))).describedAs(desc.apply("Using R for inverse comparison of R values")).isEqualTo(-expectedResult); + + // then, check if we can compare buffers using left's comparator + checks.assertThat(sign(leftComparator.compare(lv1, lv2))).describedAs(desc.apply("Using L for comparison of L values")).isEqualTo(expectedResult); + checks.assertThat(sign(leftComparator.compare(lv1, rv2))).describedAs(desc.apply("Using L for comparison of L and R values")).isEqualTo(expectedResult); + checks.assertThat(sign(leftComparator.compare(rv1, lv2))).describedAs(desc.apply("Using L for comparison of R and L values")).isEqualTo(expectedResult); + checks.assertThat(sign(leftComparator.compare(rv1, rv2))).describedAs(desc.apply("Using L for comparison of R values")).isEqualTo(expectedResult); + + checks.assertThat(sign(leftComparator.compare(lv2, lv1))).describedAs(desc.apply("Using L for inverse comparison of L values")).isEqualTo(-expectedResult); + checks.assertThat(sign(leftComparator.compare(lv2, rv1))).describedAs(desc.apply("Using L for inverse comparison of L and R values")).isEqualTo(-expectedResult); + checks.assertThat(sign(leftComparator.compare(rv2, lv1))).describedAs(desc.apply("Using L for inverse comparison of R and L values")).isEqualTo(-expectedResult); + checks.assertThat(sign(leftComparator.compare(rv2, rv1))).describedAs(desc.apply("Using L for inverse comparison of R values")).isEqualTo(-expectedResult); + + checks.assertAll(); + } + + private static void verifyComparisonCompatibilityForSimpleCells(AbstractType left, AbstractType right, Object r1, Object r2) + { + Function desc = s -> typeRelDesc(".compare", left, right, String.format("%s: '%s' and '%s'", s, r1, r2)); + + ByteBuffer rBuf1 = right.decompose(r1); + ByteBuffer rBuf2 = right.decompose(r2); + ByteBuffer lBuf1 = left.decompose(left.compose(rBuf1)); + ByteBuffer lBuf2 = left.decompose(left.compose(rBuf2)); + + int c = right.compare(rBuf1, rBuf2); + verifyComparison(left, right, lBuf1, lBuf2, rBuf1, rBuf2, c, desc); + } + + private static void verifyComparisonCompatibilityForMultiCell(AbstractType left, AbstractType right, Object r1, Object r2, + TableMetadata rightTable, ColumnMetadata rightColumn1, ColumnMetadata rightColumn2, SerializationHelper rightHelper, + SerializationHeader leftHeader, DeserializationHelper leftHelper, ColumnMetadata leftColumn1, ColumnMetadata leftColumn2) + { + Function desc = s -> typeRelDesc(".compare", left, right, String.format("%s: %s and %s", s, r1, r2)); + + Row rightRow = Rows.simpleBuilder(rightTable) + .noPrimaryKeyLivenessInfo() + .add(rightColumn1.name.toString(), r1) + .add(rightColumn2.name.toString(), r2) + .build(); + + try (DataOutputBuffer out = new DataOutputBuffer()) + { + UnfilteredSerializer.serializer.serialize(rightRow, rightHelper, out, MessagingService.current_version); + try (DataInputBuffer in = new DataInputBuffer(out.getData())) + { + Row.Builder builder = BTreeRow.sortedBuilder(); + builder.addPrimaryKeyLivenessInfo(rightRow.primaryKeyLivenessInfo()); + Row leftRow = (Row) UnfilteredSerializer.serializer.deserialize(in, leftHeader, leftHelper, builder); + ComplexColumnData leftData1 = leftRow.getComplexColumnData(leftColumn1); + ComplexColumnData leftData2 = leftRow.getComplexColumnData(leftColumn2); + ComplexColumnData rightData1 = rightRow.getComplexColumnData(rightColumn1); + ComplexColumnData rightData2 = rightRow.getComplexColumnData(rightColumn2); + + for (int i = 0; i < Math.min(leftData1.cellsCount(), leftData2.cellsCount()); i++) + { + CellPath lp1 = leftData1.getCellByIndex(i).path(); + CellPath lp2 = leftData2.getCellByIndex(i).path(); + CellPath rp1 = rightData1.getCellByIndex(i).path(); + CellPath rp2 = rightData2.getCellByIndex(i).path(); + + int c = rightColumn1.cellPathComparator().compare(rp1, rp2); + verifyComparison(leftColumn1.cellPathComparator(), rightColumn1.cellPathComparator(), lp1, lp2, rp1, rp2, c, desc); + } + } + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private static void verifySerializationCompatibilityForSimpleCells(AbstractType left, AbstractType right, Object v, + TableMetadata rightTable, ColumnMetadata rightColumn, SerializationHelper rightHelper, + SerializationHeader leftHeader, DeserializationHelper leftHelper, ColumnMetadata leftColumn) + { + Row rightRow = Rows.simpleBuilder(rightTable).noPrimaryKeyLivenessInfo().add(rightColumn.name.toString(), v).build(); + try (DataOutputBuffer out = new DataOutputBuffer()) + { + UnfilteredSerializer.serializer.serialize(rightRow, rightHelper, out, MessagingService.current_version); + try (DataInputBuffer in = new DataInputBuffer(out.getData())) + { + Row.Builder builder = BTreeRow.sortedBuilder(); + builder.addPrimaryKeyLivenessInfo(rightRow.primaryKeyLivenessInfo()); + Row leftRow = (Row) UnfilteredSerializer.serializer.deserialize(in, leftHeader, leftHelper, builder); + Cell leftData = (Cell) leftRow.getColumnData(leftColumn); + Cell rightData = (Cell) rightRow.getColumnData(rightColumn); + assertThat(leftData.buffer()).describedAs(typeRelDesc(".deserialize", left, right)).isEqualTo(rightData.buffer()); + } + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private static void verifySerializationCompatibilityForComplexCells(AbstractType left, AbstractType right, Object v, + TableMetadata rightTable, ColumnMetadata rightColumn, SerializationHelper rightHelper, + SerializationHeader leftHeader, DeserializationHelper leftHelper, ColumnMetadata leftColumn) + { + SoftAssertions checks = new SoftAssertions(); + Row rightRow = Rows.simpleBuilder(rightTable).noPrimaryKeyLivenessInfo().add(rightColumn.name.toString(), v).build(); + try (DataOutputBuffer out = new DataOutputBuffer()) + { + UnfilteredSerializer.serializer.serialize(rightRow, rightHelper, out, MessagingService.current_version); + try (DataInputBuffer in = new DataInputBuffer(out.getData())) + { + Row.Builder builder = BTreeRow.sortedBuilder(); + builder.addPrimaryKeyLivenessInfo(rightRow.primaryKeyLivenessInfo()); + Row leftRow = (Row) UnfilteredSerializer.serializer.deserialize(in, leftHeader, leftHelper, builder); + ComplexColumnData leftData = leftRow.getComplexColumnData(leftColumn); + ComplexColumnData rightData = rightRow.getComplexColumnData(rightColumn); + checks.assertThat(leftData.cellsCount()).describedAs(typeRelDesc(".cellsCountIsEqualTo", left, right)).isEqualTo(rightData.cellsCount()); + for (int i = 0; i < leftData.cellsCount(); i++) + { + Cell leftCell = leftData.getCellByIndex(i); + Cell rightCell = rightData.getCellByIndex(i); + checks.assertThat(leftCell.buffer()).describedAs(bytesToHex(leftCell.buffer())).isEqualTo(rightCell.buffer()).describedAs(bytesToHex(rightCell.buffer())); + checks.assertThat(leftCell.path().size()).describedAs(typeRelDesc(".cellPathSizeIsEqualTo", left, right)).isEqualTo(rightCell.path().size()); + for (int j = 0; j < leftCell.path().size(); j++) + checks.assertThat(leftCell.path().get(j)).describedAs(bytesToHex(leftCell.path().get(j))).isEqualTo(rightCell.path().get(j)).describedAs(bytesToHex(rightCell.path().get(j))); + } + } + } + catch (IOException e) + { + throw new RuntimeException(e); + } + + checks.assertAll(); + } + + @Test + public void testMultiCellSupport() + { + SoftAssertions assertions = new SoftAssertions(); + + Set> multiCellSupportingTypesForReading = new HashSet<>(); + Set> multiCellSupportingTypes = new HashSet<>(); + + forEachTypesPair(true, (l, r) -> { + if (l.equals(r)) + { + if (l.isMultiCell()) + { + // types which can be created as multicell + multiCellSupportingTypes.add(l.getClass()); + + AbstractType frozen = l.freeze(); + assertThat(frozen.isMultiCell()).isFalse(); + assertions.assertThat(l).isNotEqualTo(frozen); + } + else + { + // some complex types cannot be created as multicell, but can be parsed as multicell for backward + // compatibility; here we want to collect such types + AbstractType t = TypeParser.parse(l.toString(true)); + if (t.isMultiCell()) + { + multiCellSupportingTypesForReading.add(l.getClass()); + + assertions.assertThat(t).isNotEqualTo(l); + assertions.assertThat(t.freeze()).isNotEqualTo(t); + assertions.assertThat(t.freeze()).isEqualTo(l); + } + else + { + assertions.assertThat(l.freeze()).isSameAs(l); + assertions.assertThat(unfreeze(l)).isSameAs(l); + } + } + } + }); + + assertions.assertThat(multiCellSupportingTypes).isEqualTo(currentTypesCompatibility.multiCellSupportingTypes()); + assertions.assertThat(multiCellSupportingTypesForReading).isEqualTo(currentTypesCompatibility.multiCellSupportingTypesForReading()); + + // all primitive types should be freezing agnostic + currentTypesCompatibility.primitiveTypes().forEach(type -> { + assertThat(type.freeze()).isSameAs(type); + assertThat(unfreeze(type)).isSameAs(type); + }); + } + + private static Description typeRelDesc(String rel, AbstractType left, AbstractType right) + { + return typeRelDesc(rel, left, right, null); + } + + private static Description typeRelDesc(String rel, AbstractType left, AbstractType right, String extraInfo) + { + return new Description() + { + @Override + public String value() + { + if (extraInfo != null) + { + return TYPE_PREFIX_PATTERN.matcher(String.format("%s %s %s, %s", left, rel, right, extraInfo)).replaceAll(""); + } + else if (!left.equals(right)) + { + String extraInfo = Streams.zip(left.subTypes().stream(), right.subTypes().stream(), (l, r) -> { + if (l.equals(r)) + return ""; + + StringBuilder out = new StringBuilder(); + if (l.isCompatibleWith(r)) + out.append(" cmp"); + if (l.isValueCompatibleWith(r)) + out.append(" val"); + if (l.isSerializationCompatibleWith(r)) + out.append(" ser"); + if (out.length() > 0) + return String.format("%s is%s compatible with %s", l, out, r); + else + return String.format("%s is not compatible with %s", l, r); + }).collect(Collectors.joining("; ", "{", "}")); + return TYPE_PREFIX_PATTERN.matcher(String.format("%s %s %s, %s", left, rel, right, extraInfo)).replaceAll(""); + } + else + { + return TYPE_PREFIX_PATTERN.matcher(String.format("%s %s %s", left, rel, right)).replaceAll(""); + } + } + }; + } + + private static Description isCompatibleWithDesc(AbstractType left, AbstractType right) + { + return typeRelDesc("isCompatibleWith", left, right); + } + + private static Description isValueCompatibleWithDesc(AbstractType left, AbstractType right) + { + return typeRelDesc("isValueCompatibleWith", left, right); + } + + private static Description isSerializationCompatibleWithDesc(AbstractType left, AbstractType right) + { + return typeRelDesc("isSerializationCompatibleWith", left, right); + } + + /** + * The instances of this class provides types compatibility checks valid for a certain version of Cassandra. + * This way we can verify whether the current implementation satisfy assumed compatibility rules, as well as + * upgrade compatibility (that is, whether the new implementation ensures the compatibility rules from the previous + * verion of Cassandra are still satisfied). + */ + public abstract static class TypesCompatibility + { + protected static final String KNOWN_TYPES_KEY = "known_types"; + protected static final String MULTICELL_TYPES_KEY = "multicell_types"; + protected static final String MULTICELL_TYPES_FOR_READING_KEY = "multicell_types_for_reading"; + protected static final String PRIMITIVE_TYPES_KEY = "primitive_types"; + protected static final String COMPATIBLE_TYPES_KEY = "compatible_types"; + protected static final String SERIALIZATION_COMPATIBLE_TYPES_KEY = "serialization_compatible_types"; + protected static final String VALUE_COMPATIBLE_TYPES_KEY = "value_compatible_types"; + protected static final String UNSUPPORTED_TYPES_KEY = "unsupported_types"; + + public final String name; + + public TypesCompatibility(String name) + { + this.name = name; + } + + public void checkExpectedTypeCompatibility(T left, T right, SoftAssertions assertions) + { + assertions.assertThat(left.isCompatibleWith(right)).as(isCompatibleWithDesc(left, right)).isEqualTo(expectCompatibleWith(left, right)); + assertions.assertThat(left.isSerializationCompatibleWith(right)).as(isSerializationCompatibleWithDesc(left, right)).isEqualTo(expectSerializationCompatibleWith(left, right)); + assertions.assertThat(left.isValueCompatibleWith(right)).as(isValueCompatibleWithDesc(left, right)).isEqualTo(expectValueCompatibleWith(left, right)); + } + + public abstract boolean expectCompatibleWith(AbstractType left, AbstractType right); + + public abstract boolean expectValueCompatibleWith(AbstractType left, AbstractType right); + + public abstract boolean expectSerializationCompatibleWith(AbstractType left, AbstractType right); + + public abstract Set> knownTypes(); + + public abstract Set> primitiveTypes(); + + public abstract Set> multiCellSupportingTypes(); + + public abstract Set> multiCellSupportingTypesForReading(); + + public abstract Set> unsupportedTypes(); + + public void store(Path path) throws IOException + { + Set> primitiveTypeClasses = primitiveTypes().stream().map(AbstractType::getClass).collect(Collectors.toSet()); + HashSet> knownTypes = new HashSet<>(knownTypes()); + knownTypes.removeAll(unsupportedTypes()); + Multimap, Class> knownPairs = Multimaps.newMultimap(new HashMap<>(), HashSet::new); + knownTypes.forEach(l -> knownTypes.forEach(r -> { + if (l == r) + knownPairs.put(l, r); + if (primitiveTypeClasses.contains(l) && primitiveTypeClasses.contains(r)) + knownPairs.put(l, r); + })); + + Multimap compatibleWithMap = Multimaps.newListMultimap(new HashMap<>(), ArrayList::new); + Multimap serializationCompatibleWithMap = Multimaps.newListMultimap(new HashMap<>(), ArrayList::new); + Multimap valueCompatibleWithMap = Multimaps.newListMultimap(new HashMap<>(), ArrayList::new); + + Map, String> typeToStringMap = new HashMap<>(); + Map> stringToTypeMap = new HashMap<>(); + + forEachTypesPair(true, (l, r) -> { + knownPairs.remove(l.getClass(), r.getClass()); + + if (l.equals(r)) + return; + + AbstractType l1 = TypeParser.parse(l.toString()); + AbstractType r1 = TypeParser.parse(r.toString()); + assertThat(l1).isEqualTo(l); + assertThat(r1).isEqualTo(r); + + if (l.isCompatibleWith(r)) + { + assertThat(l1.isCompatibleWith(r1)).isTrue(); + compatibleWithMap.put(l.toString(), r.toString()); + } + + if (l.isSerializationCompatibleWith(r)) + { + assertThat(l1.isSerializationCompatibleWith(r1)).isTrue(); + serializationCompatibleWithMap.put(l.toString(), r.toString()); + } + + if (l.isValueCompatibleWith(r)) + { + assertThat(l1.isValueCompatibleWith(r1)).isTrue(); + valueCompatibleWithMap.put(l.toString(), r.toString()); + } + }); + + // make sure that all pairs were covered + assertThat(knownPairs.entries()).isEmpty(); + + assertThat(typeToStringMap).hasSameSizeAs(stringToTypeMap); + + JSONObject json = new JSONObject(); + json.put(KNOWN_TYPES_KEY, knownTypes().stream().map(Class::getName).collect(Collectors.toList())); + json.put(MULTICELL_TYPES_KEY, multiCellSupportingTypes().stream().map(Class::getName).collect(Collectors.toList())); + json.put(MULTICELL_TYPES_FOR_READING_KEY, multiCellSupportingTypesForReading().stream().map(Class::getName).collect(Collectors.toList())); + json.put(COMPATIBLE_TYPES_KEY, compatibleWithMap.asMap()); + json.put(SERIALIZATION_COMPATIBLE_TYPES_KEY, serializationCompatibleWithMap.asMap()); + json.put(VALUE_COMPATIBLE_TYPES_KEY, valueCompatibleWithMap.asMap()); + json.put(UNSUPPORTED_TYPES_KEY, unsupportedTypes().stream().map(Class::getName).collect(Collectors.toList())); + json.put(PRIMITIVE_TYPES_KEY, primitiveTypes().stream().map(AbstractType::toString).collect(Collectors.toList())); + + try (GZIPOutputStream out = new GZIPOutputStream(Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))) + { + out.write(json.toJSONString().getBytes(Charsets.UTF_8)); + } + + logger.info("Stored types compatibility to {}: knownTypes: {}, multiCellSupportingTypes: {}, " + + "multiCellSupportingTypesForReading: {}, unsupportedTypes: {}, primitiveTypes: {}, " + + "compatibleWith: {}, serializationCompatibleWith: {}, valueCompatibleWith: {}", + path.getFileName(), knownTypes().size(), multiCellSupportingTypes().size(), + multiCellSupportingTypesForReading().size(), unsupportedTypes().size(), primitiveTypes().size(), + compatibleWithMap.entries().size(), serializationCompatibleWithMap.entries().size(), valueCompatibleWithMap.entries().size()); + } + + @Override + public String toString() + { + return String.format("TypesCompatibility[%s]", name); + } + } + + private final static class LoadedTypesCompatibility extends TypesCompatibility + { + private final Multimap, AbstractType> valueCompatibleWith; + private final Multimap, AbstractType> serializationCompatibleWith; + private final Multimap, AbstractType> compatibleWith; + private final Set> unsupportedTypes; + private final Set> knownTypes; + private final Set> multiCellSupportingTypes; + private final Set> multiCellSupportingTypesForReading; + private final Set> primitiveTypes; + private final SoftAssertions loadAssertions = new SoftAssertionsWithLimit(100); + private final Set excludedTypes; + + private Function> safeParse(ThrowingFunction consumer) + { + return obj -> { + String typeName = (String) obj; + if (refersExcludedType(typeName)) + return Stream.empty(); + try + { + return Stream.of(consumer.apply(typeName)); + } + catch (InterruptedException ex) + { + Thread.currentThread().interrupt(); + throw new RuntimeException(ex); + } + catch (Exception th) + { + loadAssertions.fail("Failed to parse type: " + typeName, th); + } + return Stream.empty(); + }; + } + + private boolean refersExcludedType(String typeName) + { + for (String unsupportedType : excludedTypes) + if (typeName.contains(unsupportedType)) + return true; + return false; + } + + private Set> getTypesArray(Object json) + { + return (Set>) ((JSONArray) json).stream() + .map(String.class::cast) + .flatMap(safeParse(((ThrowingFunction, Exception>) Class::forName))) + .collect(Collectors3.toImmutableSet()); + } + + private Multimap, AbstractType> getTypesCompatibilityMultimap(Object json) + { + Multimap, AbstractType> map = Multimaps.newListMultimap(new HashMap<>(), ArrayList::new); + ((JSONObject) json).forEach((l, collection) -> safeParse(TypeParser::parse).apply(l).forEach(left -> { + ((JSONArray) collection).forEach(r -> { + safeParse(TypeParser::parse).apply(r).forEach(right -> map.put(left, right)); + }); + })); + return map; + } + + private LoadedTypesCompatibility(Path path, Set excludedTypes) throws IOException + { + super(path.getFileName().toString()); + + this.excludedTypes = ImmutableSet.copyOf(excludedTypes); + logger.info("Loading types compatibility from {} skipping {} as unsupported", path.toAbsolutePath(), excludedTypes); + try (GZIPInputStream in = new GZIPInputStream(Files.newInputStream(path))) + { + JSONObject json = (JSONObject) new JSONParser().parse(new InputStreamReader(in, Charsets.UTF_8)); + knownTypes = getTypesArray(json.get(KNOWN_TYPES_KEY)); + multiCellSupportingTypes = getTypesArray(json.get(MULTICELL_TYPES_KEY)); + multiCellSupportingTypesForReading = getTypesArray(json.get(MULTICELL_TYPES_FOR_READING_KEY)); + unsupportedTypes = getTypesArray(json.get(UNSUPPORTED_TYPES_KEY)); + primitiveTypes = (Set>) ((JSONArray) json.get(PRIMITIVE_TYPES_KEY)).stream().flatMap(safeParse(TypeParser::parse)).collect(Collectors.toSet()); + compatibleWith = getTypesCompatibilityMultimap(json.get(COMPATIBLE_TYPES_KEY)); + serializationCompatibleWith = getTypesCompatibilityMultimap(json.get(SERIALIZATION_COMPATIBLE_TYPES_KEY)); + valueCompatibleWith = getTypesCompatibilityMultimap(json.get(VALUE_COMPATIBLE_TYPES_KEY)); + } + catch (ParseException | NoSuchFileException e) + { + throw new IOException(path.toAbsolutePath().toString(), e); + } + + logger.info("Loaded types compatibility from {}: knownTypes: {}, multiCellSupportingTypes: {}, " + + "multiCellSupportingTypesForReading: {}, unsupportedTypes: {}, primitiveTypes: {}, " + + "compatibleWith: {}, serializationCompatibleWith: {}, valueCompatibleWith: {}", + path.getFileName(), knownTypes.size(), multiCellSupportingTypes.size(), + multiCellSupportingTypesForReading.size(), unsupportedTypes.size(), primitiveTypes.size(), + compatibleWith.size(), serializationCompatibleWith.size(), valueCompatibleWith.size()); + } + + public void assertLoaded() + { + loadAssertions.assertAll(); + } + @Override + public boolean expectCompatibleWith(AbstractType left, AbstractType right) + { + return compatibleWith.containsEntry(left, right); + } + + @Override + public boolean expectValueCompatibleWith(AbstractType left, AbstractType right) + { + return valueCompatibleWith.containsEntry(left, right); + } + + @Override + public boolean expectSerializationCompatibleWith(AbstractType left, AbstractType right) + { + return serializationCompatibleWith.containsEntry(left, right); + } + + @Override + public Set> knownTypes() + { + return knownTypes; + } + + @Override + public Set> primitiveTypes() + { + return primitiveTypes; + } + + @Override + public Set> multiCellSupportingTypes() + { + return multiCellSupportingTypes; + } + + @Override + public Set> multiCellSupportingTypesForReading() + { + return multiCellSupportingTypesForReading; + } + + @Override + public Set> unsupportedTypes() + { + return unsupportedTypes; + } + } + + private static class CurrentTypesCompatibility extends TypesCompatibility + { + protected final Multimap, AbstractType> primitiveValueCompatibleWith = HashMultimap.create(); + protected final Multimap, AbstractType> primitiveSerializationCompatibleWith = HashMultimap.create(); + protected final Multimap, AbstractType> primitiveCompatibleWith = HashMultimap.create(); + + protected final Set> knownTypes = ImmutableSet.copyOf(AbstractTypeGenerators.knownTypes()); + protected final Set> primitiveTypes = ImmutableSet.copyOf(AbstractTypeGenerators.primitiveTypes()); + protected final Set> unsupportedTypes = ImmutableSet.>builder().addAll(UNSUPPORTED.keySet()).add(CounterColumnType.class).build(); + protected final Set> multiCellSupportingTypes; + protected final Set> multiCellSupportingTypesForReading; + + private CurrentTypesCompatibility() + { + super("current"); + primitiveValueCompatibleWith.put(BytesType.instance, AsciiType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, BooleanType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, ByteType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, DecimalType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, DoubleType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, DurationType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, EmptyType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, FloatType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, InetAddressType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, Int32Type.instance); + primitiveValueCompatibleWith.put(BytesType.instance, IntegerType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, LexicalUUIDType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, LongType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, ShortType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, SimpleDateType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, TimeType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, TimeUUIDType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, TimestampType.instance); + primitiveValueCompatibleWith.put(BytesType.instance, UTF8Type.instance); + primitiveValueCompatibleWith.put(BytesType.instance, UUIDType.instance); + primitiveValueCompatibleWith.put(IntegerType.instance, Int32Type.instance); + primitiveValueCompatibleWith.put(IntegerType.instance, LongType.instance); + primitiveValueCompatibleWith.put(IntegerType.instance, TimestampType.instance); + primitiveValueCompatibleWith.put(LongType.instance, TimestampType.instance); + primitiveValueCompatibleWith.put(SimpleDateType.instance, Int32Type.instance); + primitiveValueCompatibleWith.put(TimeType.instance, LongType.instance); + primitiveValueCompatibleWith.put(TimestampType.instance, LongType.instance); + primitiveValueCompatibleWith.put(UTF8Type.instance, AsciiType.instance); + primitiveValueCompatibleWith.put(UUIDType.instance, TimeUUIDType.instance); + + primitiveSerializationCompatibleWith.put(BytesType.instance, AsciiType.instance); + primitiveSerializationCompatibleWith.put(BytesType.instance, ByteType.instance); + primitiveSerializationCompatibleWith.put(BytesType.instance, DecimalType.instance); + primitiveSerializationCompatibleWith.put(BytesType.instance, DurationType.instance); + primitiveSerializationCompatibleWith.put(BytesType.instance, InetAddressType.instance); + primitiveSerializationCompatibleWith.put(BytesType.instance, IntegerType.instance); + primitiveSerializationCompatibleWith.put(BytesType.instance, ShortType.instance); + primitiveSerializationCompatibleWith.put(BytesType.instance, SimpleDateType.instance); + primitiveSerializationCompatibleWith.put(BytesType.instance, TimeType.instance); + primitiveSerializationCompatibleWith.put(BytesType.instance, UTF8Type.instance); + primitiveSerializationCompatibleWith.put(LongType.instance, TimestampType.instance); + primitiveSerializationCompatibleWith.put(TimestampType.instance, LongType.instance); + primitiveSerializationCompatibleWith.put(UTF8Type.instance, AsciiType.instance); + primitiveSerializationCompatibleWith.put(UUIDType.instance, TimeUUIDType.instance); + + primitiveCompatibleWith.put(BytesType.instance, AsciiType.instance); + primitiveCompatibleWith.put(BytesType.instance, UTF8Type.instance); + primitiveCompatibleWith.put(UTF8Type.instance, AsciiType.instance); + + for (AbstractType t : primitiveTypes) + { + primitiveValueCompatibleWith.put(t, t); + primitiveSerializationCompatibleWith.put(t, t); + primitiveCompatibleWith.put(t, t); + } + + multiCellSupportingTypes = ImmutableSet.of(MapType.class, SetType.class, ListType.class); + multiCellSupportingTypesForReading = ImmutableSet.of(MapType.class, SetType.class, ListType.class, UserType.class); + } + + @Override + public Set> knownTypes() + { + return knownTypes; + } + + @Override + public Set> primitiveTypes() + { + return primitiveTypes; + } + + @Override + public Set> multiCellSupportingTypes() + { + return multiCellSupportingTypes; + } + + @Override + public Set> multiCellSupportingTypesForReading() + { + return multiCellSupportingTypesForReading; + } + + @Override + public Set> unsupportedTypes() + { + return unsupportedTypes; + } + + private boolean expectedCompatibility(AbstractType left, AbstractType right, BiPredicate primitiveTypesPredicate, BiPredicate complexTypesPredicate) + { + if (left.equals(right)) + return true; + + boolean leftIsPrimitve = primitiveTypes().contains(left); + boolean rightIsPrimitve = primitiveTypes().contains(right); + + if (leftIsPrimitve && rightIsPrimitve) + return primitiveTypesPredicate.test(left, right); + + if (leftIsPrimitve || rightIsPrimitve) + return false; + + return complexTypesPredicate.test(left, right); + } + + @Override + public boolean expectCompatibleWith(AbstractType left, AbstractType right) + { + return expectedCompatibility(left, right, primitiveCompatibleWith::containsEntry, AbstractType::isCompatibleWith); + } + + @Override + public boolean expectValueCompatibleWith(AbstractType left, AbstractType right) + { + return expectedCompatibility(left, right, primitiveValueCompatibleWith::containsEntry, AbstractType::isValueCompatibleWith); + } + + @Override + public boolean expectSerializationCompatibleWith(AbstractType left, AbstractType right) + { + return expectedCompatibility(left, right, primitiveSerializationCompatibleWith::containsEntry, AbstractType::isSerializationCompatibleWith); + } + } +} diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java index 8d3b6a7099..81d7acf150 100644 --- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java @@ -71,7 +71,6 @@ import org.apache.cassandra.utils.UUIDGen; import org.apache.cassandra.utils.asserts.SyncTaskListAssert; import static org.apache.cassandra.utils.asserts.SyncTaskAssert.assertThat; -import static org.apache.cassandra.utils.asserts.SyncTaskListAssert.assertThat; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertTrue; @@ -661,7 +660,7 @@ public class RepairJobTest false, PreviewKind.ALL)); - assertThat(tasks.values()).areAllInstanceOf(AsymmetricRemoteSyncTask.class); + SyncTaskListAssert.assertThat(tasks.values()).areAllInstanceOf(AsymmetricRemoteSyncTask.class); // addr1 streams range1 from addr3: assertThat(tasks.get(pair(addr1, addr3)).rangesToSync).contains(RANGE_1); diff --git a/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java b/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java index 6b685dcd1e..103fa59457 100644 --- a/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java +++ b/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java @@ -19,52 +19,102 @@ package org.apache.cassandra.utils; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.BiConsumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.Nullable; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.FieldIdentifier; +import org.apache.cassandra.db.marshal.AbstractCompositeType; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.db.marshal.BooleanType; +import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.db.marshal.ByteType; import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.marshal.CounterColumnType; +import org.apache.cassandra.db.marshal.DateType; +import org.apache.cassandra.db.marshal.DecimalType; import org.apache.cassandra.db.marshal.DoubleType; +import org.apache.cassandra.db.marshal.DurationType; +import org.apache.cassandra.db.marshal.DynamicCompositeType; import org.apache.cassandra.db.marshal.EmptyType; import org.apache.cassandra.db.marshal.FloatType; +import org.apache.cassandra.db.marshal.FrozenType; import org.apache.cassandra.db.marshal.InetAddressType; import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.marshal.LexicalUUIDType; import org.apache.cassandra.db.marshal.ListType; import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.marshal.PartitionerDefinedOrder; import org.apache.cassandra.db.marshal.ReversedType; import org.apache.cassandra.db.marshal.SetType; import org.apache.cassandra.db.marshal.ShortType; +import org.apache.cassandra.db.marshal.SimpleDateType; +import org.apache.cassandra.db.marshal.TimeType; +import org.apache.cassandra.db.marshal.TimeUUIDType; import org.apache.cassandra.db.marshal.TimestampType; import org.apache.cassandra.db.marshal.TupleType; +import org.apache.cassandra.db.marshal.TypeParser; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.db.marshal.UserType; +import org.apache.cassandra.db.marshal.ValueAccessor; +import org.apache.cassandra.serializers.MarshalException; import org.quicktheories.core.Gen; import org.quicktheories.core.RandomnessSource; import org.quicktheories.generators.SourceDSL; +import org.quicktheories.impl.JavaRandom; import static org.apache.cassandra.utils.Generators.IDENTIFIER_GEN; +@SuppressWarnings({ "unchecked", "rawtypes" }) public final class AbstractTypeGenerators { + private final static Logger logger = LoggerFactory.getLogger(AbstractTypeGenerators.class); + private static final Gen VERY_SMALL_POSITIVE_SIZE_GEN = SourceDSL.integers().between(1, 3); private static final Gen BOOLEAN_GEN = SourceDSL.booleans().all(); - private static final Map, TypeSupport> PRIMITIVE_TYPE_DATA_GENS = + + @SuppressWarnings("RedundantCast") + public static final Map>, String> UNSUPPORTED = ImmutableMap.>, String>builder() + .put(DateType.class, "Says its CQL type is timestamp, but that maps to TimestampType; is this actually dead code at this point?") + .put(PartitionerDefinedOrder.class, "This is a fake type used for ordering partitions using a Partitioner") + // ignore intellij saying that "(Class)" isn't needed; jdk8 fails to compile without it! + .put((Class>) (Class) ReversedType.class, "Implementation detail for cluster ordering... its expected the caller will unwrap the clustering type to always get access to the real type") + .put(DynamicCompositeType.FixedValueComparator.class, "Hack type used for special ordering case, not a real/valid type") + .put(FrozenType.class, "Fake class only used during parsing... the parsing creates this and the real type under it, then this gets swapped for the real type") + .build(); + + public static final Map, TypeSupport> PRIMITIVE_TYPE_DATA_GENS = Stream.of(TypeSupport.of(BooleanType.instance, BOOLEAN_GEN), TypeSupport.of(ByteType.instance, SourceDSL.integers().between(0, Byte.MAX_VALUE * 2 + 1).map(Integer::byteValue)), TypeSupport.of(ShortType.instance, SourceDSL.integers().between(0, Short.MAX_VALUE * 2 + 1).map(Integer::shortValue)), @@ -74,40 +124,87 @@ public final class AbstractTypeGenerators TypeSupport.of(DoubleType.instance, SourceDSL.doubles().any()), TypeSupport.of(BytesType.instance, Generators.bytes(1, 1024)), TypeSupport.of(UUIDType.instance, Generators.UUID_RANDOM_GEN), + TypeSupport.of(TimeUUIDType.instance, Generators.UUID_TIME_GEN), + TypeSupport.of(LexicalUUIDType.instance, Generators.UUID_RANDOM_GEN.mix(Generators.UUID_TIME_GEN)), TypeSupport.of(InetAddressType.instance, Generators.INET_ADDRESS_UNRESOLVED_GEN), // serialization strips the hostname, only keeps the address TypeSupport.of(AsciiType.instance, SourceDSL.strings().ascii().ofLengthBetween(1, 1024)), TypeSupport.of(UTF8Type.instance, Generators.utf8(1, 1024)), TypeSupport.of(TimestampType.instance, Generators.DATE_GEN), + TypeSupport.of(SimpleDateType.instance, SourceDSL.integers().between(0, Integer.MAX_VALUE)), // can't use time gen as this is an int, and in Milliseconds... so overflows... + TypeSupport.of(TimeType.instance, SourceDSL.longs().between(0, 24L * 60L * 60L * 1_000_000_000L - 1L)), // null is desired here as #decompose will call org.apache.cassandra.serializers.EmptySerializer.serialize which ignores the input and returns empty bytes - TypeSupport.of(EmptyType.instance, rnd -> null) - //TODO add the following - // IntegerType.instance, - // DecimalType.instance, - // TimeUUIDType.instance, - // LexicalUUIDType.instance, - // SimpleDateType.instance, - // TimeType.instance, - // DurationType.instance, + TypeSupport.of(EmptyType.instance, rnd -> null), + TypeSupport.of(DurationType.instance, CassandraGenerators.duration()), + TypeSupport.of(IntegerType.instance, Generators.bigInt()), + TypeSupport.of(DecimalType.instance, Generators.bigDecimal()) ).collect(Collectors.toMap(t -> t.type, t -> t)); // NOTE not supporting reversed as CQL doesn't allow nested reversed types // when generating part of the clustering key, it would be good to allow reversed types as the top level private static final Gen> PRIMITIVE_TYPE_GEN = SourceDSL.arbitrary().pick(new ArrayList<>(PRIMITIVE_TYPE_DATA_GENS.keySet())); + private static final Set> NON_PRIMITIVE_TYPES = ImmutableSet.>builder() + .add(SetType.class) + .add(ListType.class) + .add(MapType.class) + .add(TupleType.class) + .add(UserType.class) + .add(CompositeType.class) + .add(DynamicCompositeType.class) + .add(CounterColumnType.class) + .build(); + private AbstractTypeGenerators() { } public enum TypeKind - {PRIMITIVE, SET, LIST, MAP, TUPLE, UDT} + { + PRIMITIVE, + SET, LIST, MAP, + TUPLE, UDT, + COMPOSITE, DYNAMIC_COMPOSITE, + COUNTER + } - private static final Gen TYPE_KIND_GEN = SourceDSL.arbitrary().enumValuesWithNoOrder(TypeKind.class); + private static final Gen TYPE_KIND_GEN = SourceDSL.arbitrary().pick(ImmutableList.copyOf(EnumSet.complementOf(EnumSet.of(TypeKind.COUNTER)))); + + public static Set> knownTypes() + { + Set> types = PRIMITIVE_TYPE_DATA_GENS.keySet().stream().map(a -> a.getClass()).collect(Collectors.toSet()); + types.addAll(NON_PRIMITIVE_TYPES); + types.addAll(UNSUPPORTED.keySet()); + return types; + } + + public static Collection> primitiveTypes() + { + return PRIMITIVE_TYPE_DATA_GENS.keySet(); + } + + public static Stream, AbstractType>> primitiveTypePairs() + { + return primitiveTypePairs(a -> true); + } + + public static Stream, AbstractType>> primitiveTypePairs(Predicate> filter) + { + return primitiveTypes().stream().filter(filter).flatMap(a -> primitiveTypes().stream().filter(filter).map(b -> Pair.create(a, b))); + } public static Gen> primitiveTypeGen() { return PRIMITIVE_TYPE_GEN; } + public static Gen> primitiveTypeGen(AbstractType... without) + { + List> types = new ArrayList<>(PRIMITIVE_TYPE_DATA_GENS.keySet()); + for (AbstractType t : without) + types.remove(t); + return SourceDSL.arbitrary().pick(types); + } + public static Gen> typeGen() { return typeGen(3); @@ -139,12 +236,63 @@ public final class AbstractTypeGenerators return tupleTypeGen(atBottom ? PRIMITIVE_TYPE_GEN : typeGen(maxDepth - 1, typeKindGen, sizeGen), sizeGen).generate(rnd); case UDT: return userTypeGen(atBottom ? PRIMITIVE_TYPE_GEN : typeGen(maxDepth - 1, typeKindGen, sizeGen), sizeGen).generate(rnd); + case COMPOSITE: + return compositeTypeGen(atBottom ? PRIMITIVE_TYPE_GEN : typeGen(maxDepth - 1, typeKindGen, sizeGen), sizeGen).generate(rnd); + case DYNAMIC_COMPOSITE: + Gen aliasGen = Generators.letterOrDigit().map(c -> (byte) c.charValue()); + // stores alias names by class and not what is actually valid by cql... so only primitive types match! + return dynamicCompositeGen(PRIMITIVE_TYPE_GEN, aliasGen, sizeGen).generate(rnd); + case COUNTER: + return CounterColumnType.instance; default: throw new IllegalArgumentException("Unknown kind: " + kind); } }; } + @SuppressWarnings("unused") + public static Gen compositeTypeGen() + { + return compositeTypeGen(typeGen(2)); + } + + public static Gen compositeTypeGen(Gen> typeGen) + { + return compositeTypeGen(typeGen, VERY_SMALL_POSITIVE_SIZE_GEN); + } + + public static Gen compositeTypeGen(Gen> typeGen, Gen sizeGen) + { + return rnd -> { + int size = sizeGen.generate(rnd); + List> types = new ArrayList<>(size); + for (int i = 0; i < size; i++) + { + AbstractType type = typeGen.generate(rnd); + while (type == BytesType.instance || type == UUIDType.instance) + type = typeGen.generate(rnd); + types.add(type); + } + return CompositeType.getInstance(types); + }; + } + + public static Gen dynamicCompositeGen(Gen> typeGen, Gen aliasGen, Gen sizeGen) + { + return rnd -> { + int size = sizeGen.generate(rnd); + Map> aliases = Maps.newHashMapWithExpectedSize(size); + for (int i = 0; i < size; i++) + { + byte alias = aliasGen.generate(rnd); + while (aliases.containsKey(alias)) + alias = aliasGen.generate(rnd); + aliases.put(alias, unfreeze(typeGen.generate(rnd))); + } + return DynamicCompositeType.getInstance(aliases); + }; + } + @SuppressWarnings("unused") public static Gen> setTypeGen() { @@ -173,12 +321,12 @@ public final class AbstractTypeGenerators return mapTypeGen(typeGen(2)); // lower the default depth since this is already a nested type } - public static Gen> mapTypeGen(Gen> typeGen) + public static Gen> mapTypeGen(Gen> typeGen) { return mapTypeGen(typeGen, typeGen); } - public static Gen> mapTypeGen(Gen> keyGen, Gen> valueGen) + public static Gen> mapTypeGen(Gen> keyGen, Gen> valueGen) { return rnd -> MapType.getInstance(keyGen.generate(rnd), valueGen.generate(rnd), BOOLEAN_GEN.generate(rnd)); } @@ -249,76 +397,339 @@ public final class AbstractTypeGenerators return getTypeSupport(type, VERY_SMALL_POSITIVE_SIZE_GEN); } + public enum ValueDomain + {NULL, EMPTY_BYTES, NORMAL} + + public static TypeSupport getTypeSupport(AbstractType type, Gen sizeGen) + { + return getTypeSupport(type, sizeGen, null); + } + /** * For a type, create generators for data that matches that type */ - public static TypeSupport getTypeSupport(AbstractType type, Gen sizeGen) + public static TypeSupport getTypeSupport(AbstractType type, Gen sizeGen, @Nullable Gen valueDomainGen) { + Objects.requireNonNull(sizeGen, "sizeGen"); // this doesn't affect the data, only sort order, so drop it - if (type.isReversed()) - type = ((ReversedType) type).baseType; + type = unwrap(type); // cast is safe since type is a constant and was type cast while inserting into the map @SuppressWarnings("unchecked") TypeSupport gen = (TypeSupport) PRIMITIVE_TYPE_DATA_GENS.get(type); + TypeSupport support; if (gen != null) - return gen; + { + support = gen; + } // might be... complex... - if (type instanceof SetType) + else if (type instanceof SetType) { // T = Set so can not use T here SetType setType = (SetType) type; - TypeSupport elementSupport = getTypeSupport(setType.getElementsType(), sizeGen); - @SuppressWarnings("unchecked") - TypeSupport support = (TypeSupport) TypeSupport.of(setType, rnd -> { + TypeSupport elementSupport = getTypeSupport(setType.getElementsType(), sizeGen, valueDomainGen); + support = (TypeSupport) TypeSupport.of(setType, rnd -> { int size = sizeGen.generate(rnd); + size = normalizeSizeFromType(elementSupport, size); HashSet set = Sets.newHashSetWithExpectedSize(size); for (int i = 0; i < size; i++) - set.add(elementSupport.valueGen.generate(rnd)); + { + Object generate = elementSupport.valueGen.generate(rnd); + for (int attempts = 0; set.contains(generate); attempts++) + { + if (attempts == 42) + throw new AssertionError(String.format("Unable to get unique element for type %s with the size %d", typeTree(elementSupport.type), size)); + rnd = JavaRandom.wrap(rnd); + generate = elementSupport.valueGen.generate(rnd); + } + + set.add(generate); + } return set; }); - return support; } else if (type instanceof ListType) { // T = List so can not use T here ListType listType = (ListType) type; - TypeSupport elementSupport = getTypeSupport(listType.getElementsType(), sizeGen); - @SuppressWarnings("unchecked") - TypeSupport support = (TypeSupport) TypeSupport.of(listType, rnd -> { + TypeSupport elementSupport = getTypeSupport(listType.getElementsType(), sizeGen, valueDomainGen); + support = (TypeSupport) TypeSupport.of(listType, rnd -> { int size = sizeGen.generate(rnd); List list = new ArrayList<>(size); for (int i = 0; i < size; i++) list.add(elementSupport.valueGen.generate(rnd)); return list; }); - return support; } else if (type instanceof MapType) { // T = Map so can not use T here MapType mapType = (MapType) type; - TypeSupport keySupport = getTypeSupport(mapType.getKeysType(), sizeGen); - TypeSupport valueSupport = getTypeSupport(mapType.getValuesType(), sizeGen); - @SuppressWarnings("unchecked") - TypeSupport support = (TypeSupport) TypeSupport.of(mapType, rnd -> { + // do not use valueDomainGen as map doesn't allow null/empty + TypeSupport keySupport = getTypeSupport(mapType.getKeysType(), sizeGen, valueDomainGen); + TypeSupport valueSupport = getTypeSupport(mapType.getValuesType(), sizeGen, valueDomainGen); + support = (TypeSupport) TypeSupport.of(mapType, rnd -> { int size = sizeGen.generate(rnd); + size = normalizeSizeFromType(keySupport, size); Map map = Maps.newHashMapWithExpectedSize(size); // if there is conflict thats fine for (int i = 0; i < size; i++) - map.put(keySupport.valueGen.generate(rnd), valueSupport.valueGen.generate(rnd)); + { + Object key = keySupport.valueGen.generate(rnd); + for (int attempts = 0; map.containsKey(key); attempts++) + { + if (attempts == 42) + throw new AssertionError(String.format("Unable to get unique element for type %s with the size %d", typeTree(keySupport.type), size)); + rnd = JavaRandom.wrap(rnd); + key = keySupport.valueGen.generate(rnd); + } + map.put(key, valueSupport.valueGen.generate(rnd)); + } return map; }); - return support; } else if (type instanceof TupleType) // includes UserType { // T is ByteBuffer TupleType tupleType = (TupleType) type; - @SuppressWarnings("unchecked") - TypeSupport support = (TypeSupport) TypeSupport.of(tupleType, new TupleGen(tupleType, sizeGen)); - return support; + support = (TypeSupport) TypeSupport.of(tupleType, new TupleGen(tupleType, sizeGen, valueDomainGen)); } - throw new UnsupportedOperationException("Unsupported type: " + type); + else if (type instanceof CompositeType) + { + CompositeType ct = (CompositeType) type; + List> elementSupport = (List>) (List) ct.types.stream().map(AbstractTypeGenerators::getTypeSupport).collect(Collectors.toList()); + //noinspection Convert2Diamond + Serde> serde = new Serde>() + { + @Override + public ByteBuffer from(List objects) + { + return ct.decompose(objects.toArray()); + } + + @Override + public List to(ByteBuffer buffer) + { + ByteBuffer[] bbs = ct.split(buffer); + List values = new ArrayList<>(bbs.length); + for (int i = 0; i < bbs.length; i++) + values.add(bbs[i] == null ? null : elementSupport.get(i).type.compose(bbs[i])); + return values; + } + }; + support = (TypeSupport) TypeSupport.of(ct, serde, rnd -> { + List values = new ArrayList<>(ct.types.size()); + for (int i = 0, size = ct.types.size(); i < size; i++) + values.add(elementSupport.get(i).valueGen.generate(rnd)); + return values; + }); + } + else if (type instanceof DynamicCompositeType) + { + // data generation limits to what is valid for the type, and sadly the type + DynamicCompositeType dct = (DynamicCompositeType) type; + Map> supports = new TreeMap<>(); + for (Map.Entry> e : dct.aliases.entrySet()) + supports.put(e.getKey(), getTypeSupport(e.getValue())); + List orderedAliases = new ArrayList<>(supports.keySet()); + //noinspection Convert2Diamond + Serde> serde = new Serde>() + { + @Override + public ByteBuffer from(Map byteObjectMap) + { + return new DynamicCompositeTypeValueBuilder(dct).build(byteObjectMap); + } + + @Override + public Map to(ByteBuffer buffer) + { + ByteBuffer[] parts = dct.split(buffer); + Map values = Maps.newHashMapWithExpectedSize(parts.length); + for (int i = 0; i < parts.length; i++) + { + Byte alias = orderedAliases.get(i); + AbstractType type = dct.aliases.get(alias); + ByteBuffer bytes = parts[i]; + type.validate(bytes); + values.put(alias, type.compose(bytes)); + } + return values; + } + }; + support = (TypeSupport) TypeSupport.of(dct, serde, rnd -> { + Map byteObjectMap = new TreeMap<>(); + for (Byte alias : orderedAliases) + byteObjectMap.put(alias, supports.get(alias).valueGen.generate(rnd)); + return byteObjectMap; + }); + } + else if (type instanceof CounterColumnType) + { + support = (TypeSupport) TypeSupport.of(CounterColumnType.instance, SourceDSL.longs().all()); + } + else + { + throw new UnsupportedOperationException("No TypeSupport for: " + type); + } + return support.withValueDomain(valueDomainGen); + } + + private static int uniqueElementsForDomain(AbstractType type) + { + type = unwrap(type); + if (type instanceof BooleanType) + return 2; + if (type instanceof EmptyType) + return 1; + if (type instanceof SetType) + return uniqueElementsForDomain(((SetType) type).getElementsType()); + if (type instanceof MapType) + return uniqueElementsForDomain(((MapType) type).getKeysType()); + if (type instanceof TupleType || type instanceof CompositeType || type instanceof DynamicCompositeType) + { + int product = 1; + for (AbstractType f : type.subTypes()) + { + int uniq = uniqueElementsForDomain(f); + if (uniq == -1) + return -1; + product *= uniq; + } + return product; + } + return -1; + } + + private static int normalizeSizeFromType(TypeSupport keySupport, int size) + { + int uniq = uniqueElementsForDomain(keySupport.type); + if (uniq == -1) + return size; + return Math.min(size, uniq); + } + + public static String typeTree(AbstractType type) + { + StringBuilder sb = new StringBuilder(); + typeTree(sb, type, 0); + return sb.toString().trim(); + } + + private static void typeTree(StringBuilder sb, AbstractType type, int indent) + { + if (type.isUDT()) + { + if (indent != 0) + { + indent += 2; + newline(sb, indent); + } + UserType ut = (UserType) type; + if (!type.isMultiCell()) sb.append("frozen "); + sb.append("udt[").append(ColumnIdentifier.maybeQuote(ut.elementName())).append("]:"); + int elementIndent = indent + 2; + for (int i = 0; i < ut.size(); i++) + { + newline(sb, elementIndent); + FieldIdentifier fieldName = ut.fieldName(i); + AbstractType fieldType = ut.fieldType(i); + sb.append(ColumnIdentifier.maybeQuote(fieldName.toString())).append(": "); + typeTree(sb, fieldType, elementIndent); + } + newline(sb, elementIndent); + } + else if (type.isTuple()) + { + if (indent != 0) + { + indent += 2; + newline(sb, indent); + } + TupleType tt = (TupleType) type; + sb.append("tuple:"); + int elementIndent = indent + 2; + for (int i = 0; i < tt.size(); i++) + { + newline(sb, elementIndent); + AbstractType fieldType = tt.type(i); + sb.append(i).append(": "); + typeTree(sb, fieldType, elementIndent); + } + } + else if (type.isCollection()) + { + CollectionType ct = (CollectionType) type; + if (indent != 0) + { + indent += 2; + newline(sb, indent); + } + if (!type.isMultiCell()) sb.append("frozen "); + switch (ct.kind) + { + case MAP: + { + MapType mt = (MapType) type; + sb.append("map:"); + indent += 2; + newline(sb, indent); + sb.append("key: "); + int subTypeIndent = indent + 2; + typeTree(sb, mt.getKeysType(), subTypeIndent); + newline(sb, indent); + sb.append("value: "); + typeTree(sb, mt.getValuesType(), subTypeIndent); + } + break; + case LIST: + { + ListType lt = (ListType) type; + sb.append("list: "); + indent += 2; + typeTree(sb, lt.getElementsType(), indent); + } + break; + case SET: + { + SetType st = (SetType) type; + sb.append("set: "); + indent += 2; + typeTree(sb, st.getElementsType(), indent); + } + break; + default: + throw new UnsupportedOperationException("Unknown kind: " + ct.kind); + } + } + else if (type instanceof CompositeType) + { + CompositeType ct = (CompositeType) type; + if (indent != 0) + { + indent += 2; + newline(sb, indent); + } + sb.append("CompositeType:"); + indent += 2; + int idx = 0; + for (AbstractType subtype : ct.subTypes()) + { + newline(sb, indent); + sb.append(idx++).append(": "); + typeTree(sb, subtype, indent); + } + } + else + { + sb.append(type.asCQL3Type().toString().replaceAll("org.apache.cassandra.db.marshal.", "")); + } + } + + private static void newline(StringBuilder sb, int indent) + { + sb.append('\n'); + for (int i = 0; i < indent; i++) + sb.append(' '); } private static final class TupleGen implements Gen @@ -326,9 +737,9 @@ public final class AbstractTypeGenerators private final List> elementsSupport; @SuppressWarnings("unchecked") - private TupleGen(TupleType tupleType, Gen sizeGen) + private TupleGen(TupleType tupleType, Gen sizeGen, @Nullable Gen valueDomainGen) { - this.elementsSupport = tupleType.allTypes().stream().map(t -> getTypeSupport((AbstractType) t, sizeGen)).collect(Collectors.toList()); + this.elementsSupport = tupleType.allTypes().stream().map(t -> getTypeSupport((AbstractType) t, sizeGen, valueDomainGen)).collect(Collectors.toList()); } public ByteBuffer generate(RandomnessSource rnd) @@ -344,6 +755,12 @@ public final class AbstractTypeGenerators } } + public interface Serde + { + A from(B b); + B to(A a); + } + /** * Pair of {@link AbstractType} and a Generator of values that are handled by that type. */ @@ -351,11 +768,20 @@ public final class AbstractTypeGenerators { public final AbstractType type; public final Gen valueGen; + private final Gen bytesGen; private TypeSupport(AbstractType type, Gen valueGen) { this.type = Objects.requireNonNull(type); this.valueGen = Objects.requireNonNull(valueGen); + this.bytesGen = rnd -> type.decompose(valueGen.generate(rnd)); + } + + private TypeSupport(AbstractType type, Gen valueGen, Gen bytesGen) + { + this.type = type; + this.valueGen = valueGen; + this.bytesGen = bytesGen; } public static TypeSupport of(AbstractType type, Gen valueGen) @@ -363,12 +789,43 @@ public final class AbstractTypeGenerators return new TypeSupport<>(type, valueGen); } + public static TypeSupport of(AbstractType type, Serde serde, Gen valueGen) + { + return of(type, valueGen.map(serde::from)); + } + /** * Generator which composes the values gen with {@link AbstractType#decompose(Object)} */ public Gen bytesGen() { - return rnd -> type.decompose(valueGen.generate(rnd)); + return bytesGen; + } + + public TypeSupport withValueDomain(@Nullable Gen valueDomainGen) + { + if (valueDomainGen == null || !allowsEmpty(type)) + return this; + Gen gen = rnd -> { + ValueDomain domain = valueDomainGen.generate(rnd); + ByteBuffer value; + switch (domain) + { + case NULL: + value = null; + break; + case EMPTY_BYTES: + value = ByteBufferUtil.EMPTY_BYTE_BUFFER; + break; + case NORMAL: + value = bytesGen.generate(rnd); + break; + default: + throw new IllegalArgumentException("Unknown domain: " + domain); + } + return value; + }; + return new TypeSupport<>(type, valueGen, gen); } public String toString() @@ -378,4 +835,288 @@ public final class AbstractTypeGenerators '}'; } } + + public static boolean allowsEmpty(AbstractType type) + { + try + { + type.validate(ByteBufferUtil.EMPTY_BYTE_BUFFER); + return true; + } + catch (MarshalException e) + { + return false; + } + } + + public static AbstractType unwrap(AbstractType type) + { + if (type instanceof ReversedType) + return ((ReversedType) type).baseType; + return type; + } + + public static AbstractType unfreeze(AbstractType t) + { + if (t.isMultiCell()) + return t; + + AbstractType unfrozen = TypeParser.parse(t.toString(true)); + if (unfrozen.isMultiCell()) + return unfrozen; + + return t; + } + + private static class DynamicCompositeTypeValueBuilder + { + private final Map> aliases; + + public DynamicCompositeTypeValueBuilder(DynamicCompositeType dct) + { + this.aliases = dct.aliases; + } + + public ByteBuffer build(Map valuesMap) + { + Sets.SetView unknownAliases = Sets.difference(valuesMap.keySet(), aliases.keySet()); + if (!unknownAliases.isEmpty()) + throw new IllegalArgumentException(String.format("Aliases %s used; only valid values are %s", unknownAliases, aliases.keySet())); + List> types = new ArrayList<>(valuesMap.size()); + List values = new ArrayList<>(valuesMap.size()); + List aliasesList = new ArrayList<>(valuesMap.size()); + for (Map.Entry e : valuesMap.entrySet()) + { + @SuppressWarnings("rawtype") + AbstractType type = aliases.get(e.getKey()); + types.add(type); + values.add(type.decompose(e.getValue())); + aliasesList.add(e.getKey()); + } + return build(ByteBufferAccessor.instance, types, values, aliasesList, (byte) 0); + } + + @VisibleForTesting + public static V build(ValueAccessor accessor, + List> types, + List values, + List aliasesList, + byte lastEoc) + { + assert types.size() == values.size(); + + int numComponents = types.size(); + // Compute the total number of bytes that we'll need to store the types and their payloads. + int totalLength = 0; + for (int i = 0; i < numComponents; ++i) + { + AbstractType type = types.get(i); + Byte alias = aliasesList.get(i); + int typeNameLength = alias == null ? type.toString().getBytes(StandardCharsets.UTF_8).length : 0; + // The type data will be stored by means of the type's fully qualified name, not by aliasing, so: + // 1. The type data header should be the fully qualified name length in bytes. + // 2. The length should be small enough so that it fits in 15 bits (2 bytes with the first bit zero). + assert typeNameLength <= 0x7FFF; + int valueLength = accessor.size(values.get(i)); + // The value length should also expect its first bit to be 0, as the length should be stored as a signed + // 2-byte value (short). + assert valueLength <= 0x7FFF; + totalLength += 2 + typeNameLength + 2 + valueLength + 1; + } + + V result = accessor.allocate(totalLength); + int offset = 0; + for (int i = 0; i < numComponents; ++i) + { + AbstractType type = types.get(i); + Byte alias = aliasesList.get(i); + if (alias == null) + { + // Write the type data (2-byte length header + the fully qualified type name in UTF-8). + byte[] typeNameBytes = type.toString().getBytes(StandardCharsets.UTF_8); + accessor.putShort(result, + offset, + (short) typeNameBytes.length); // this should work fine also if length >= 32768 + offset += 2; + accessor.copyByteArrayTo(typeNameBytes, 0, result, offset, typeNameBytes.length); + offset += typeNameBytes.length; + } + else + { + accessor.putShort(result, offset, (short) (alias | 0x8000)); + offset += 2; + } + + // Write the type payload data (2-byte length header + the payload). + V value = values.get(i); + int bytesToCopy = accessor.size(value); + if ((short) bytesToCopy != bytesToCopy) + throw new IllegalArgumentException(String.format("Value of type %s is of length %d; does not fit in a short", type.asCQL3Type(), bytesToCopy)); + accessor.putShort(result, offset, (short) bytesToCopy); + offset += 2; + accessor.copyTo(value, 0, result, accessor, offset, bytesToCopy); + offset += bytesToCopy; + + // Write the end-of-component byte. + accessor.putByte(result, offset, i != numComponents - 1 ? (byte) 0 : lastEoc); + offset += 1; + } + return result; + } + } + + public static void forEachTypesPair(boolean withVariants, BiConsumer typesPairConsumer) + { + forEachPrimitiveTypePair(typesPairConsumer); + forEachMapTypesPair(withVariants, typesPairConsumer); + forEachSetTypesPair(withVariants, typesPairConsumer); + forEachListTypesPair(withVariants, typesPairConsumer); + forEachUserTypesPair(withVariants, typesPairConsumer); + forEachCompositeTypesPair(withVariants, typesPairConsumer); + } + + public static void forEachPrimitiveTypePair(BiConsumer typePairConsumer) + { + logger.info("Iterating over primitive types pairs..."); + primitiveTypePairs().forEach(p -> typePairConsumer.accept(p.left, p.right)); + } + + private static > Set frozenAndUnfrozen(T... types) + { + return Stream.of(types).flatMap(t -> Stream.of((T) t.freeze(), (T) unfreeze(t))).collect(Collectors3.toImmutableSet()); + } + + private static UserType withAddedField(UserType type, String fieldName, AbstractType fieldType) + { + ArrayList fieldNames = new ArrayList<>(type.fieldNames()); + fieldNames.add(FieldIdentifier.forUnquoted(fieldName)); + List> fieldTypes = new ArrayList<>(type.fieldTypes()); + fieldTypes.add(fieldType); + return new UserType(type.keyspace, type.name, fieldNames, fieldTypes, true); + } + + private static Set tupleTypeVariants(UserType type) + { + UserType extType = withAddedField(type, "extra", EmptyType.instance); + return frozenAndUnfrozen(type, + new TupleType(type.subTypes(), false), + extType, + new TupleType(extType.subTypes(), false)); + } + + private static void forEachUserTypeVariantPair(UserType leftType, UserType rightType, BiConsumer typePairConsumer) + { + forEachTypesPair(tupleTypeVariants(leftType), tupleTypeVariants(rightType), typePairConsumer); + } + + private static DynamicCompositeType withAddedField(DynamicCompositeType type, AbstractType fieldType) + { + Map> aliases = new HashMap<>(type.aliases); + aliases.put((byte) ('a' + type.aliases.size()), fieldType); + return DynamicCompositeType.getInstance(aliases); + } + + private static Set compositeTypeVariants(DynamicCompositeType type) + { + DynamicCompositeType extType = withAddedField(type, EmptyType.instance); + return frozenAndUnfrozen(type, + CompositeType.getInstance(new TreeMap<>(type.aliases).values()), + extType, + CompositeType.getInstance(new TreeMap<>(extType.aliases).values())); + } + + private static void forEachCompositeTypeVariantsPair(DynamicCompositeType leftType, DynamicCompositeType rightType, BiConsumer typePairConsumer) + { + forEachTypesPair(compositeTypeVariants(leftType), compositeTypeVariants(rightType), typePairConsumer); + } + + public static void forEachMapTypesPair(boolean withVariants, BiConsumer typePairConsumer) + { + logger.info("Iterating over map types pairs..."); + primitiveTypePairs(t -> t.getClass() != EmptyType.class).forEach(keyPair -> { // key cannot be empty + primitiveTypePairs().forEach(valuePair -> { + MapType leftMap = MapType.getInstance(keyPair.left, valuePair.left, true); + MapType rightMap = MapType.getInstance(keyPair.right, valuePair.right, true); + if (withVariants) + forEachCollectionTypeVariantsPair(leftMap, rightMap, typePairConsumer); + else + typePairConsumer.accept(leftMap, rightMap); + }); + }); + } + + public static void forEachSetTypesPair(boolean withVariants, BiConsumer typePairConsumer) + { + logger.info("Iterating over set types pairs..."); + primitiveTypePairs().forEach(keyPair -> { + SetType leftSet = SetType.getInstance(keyPair.left, true); + SetType rightSet = SetType.getInstance(keyPair.right, true); + if (withVariants) + forEachCollectionTypeVariantsPair(leftSet, rightSet, typePairConsumer); + else + typePairConsumer.accept(leftSet, rightSet); + }); + } + + public static void forEachListTypesPair(boolean withVariants, BiConsumer typePairConsumer) + { + logger.info("Iterating over list types pairs..."); + primitiveTypePairs().forEach(valuePair -> { + ListType leftList = ListType.getInstance(valuePair.left, true); + ListType rightList = ListType.getInstance(valuePair.right, true); + if (withVariants) + forEachCollectionTypeVariantsPair(leftList, rightList, typePairConsumer); + else + typePairConsumer.accept(leftList, rightList); + }); + } + + public static void forEachUserTypesPair(boolean withVariants, BiConsumer typePairConsumer) + { + logger.info("Iterating over user types pairs..."); + + String ks = "ks"; + ByteBuffer t = ByteBufferUtil.bytes("t"); + List names = Stream.of("a", "b").map(FieldIdentifier::forUnquoted).collect(Collectors3.toImmutableList()); + + primitiveTypePairs().forEach(elem1Pair -> { + primitiveTypePairs().forEach(elem2Pair -> { + UserType leftType = new UserType(ks, t, names, ImmutableList.of(elem1Pair.left, elem2Pair.left), true); + UserType rightType = new UserType(ks, t, names, ImmutableList.of(elem1Pair.right, elem2Pair.right), true); + if (withVariants) + forEachUserTypeVariantPair(leftType, rightType, typePairConsumer); + else + typePairConsumer.accept(leftType, rightType); + }); + }); + } + + public static void forEachCompositeTypesPair(boolean withVariants, BiConsumer typePairConsumer) + { + logger.info("Iterating over composite types pairs..."); + primitiveTypePairs().forEach(elem1Pair -> { + primitiveTypePairs().forEach(elem2Pair -> { + DynamicCompositeType leftType = DynamicCompositeType.getInstance(ImmutableMap.of((byte) 'a', elem1Pair.left, (byte) 'b', elem2Pair.left)); + DynamicCompositeType rightType = DynamicCompositeType.getInstance(ImmutableMap.of((byte) 'a', elem1Pair.right, (byte) 'b', elem2Pair.right)); + if (withVariants) + forEachCompositeTypeVariantsPair(leftType, rightType, typePairConsumer); + else + typePairConsumer.accept(leftType, rightType); + }); + }); + } + + private static void forEachTypesPair(Collection leftVariants, Collection rightVariants, BiConsumer typePairConsumer) + { + for (T left : leftVariants) + for (T right : rightVariants) + typePairConsumer.accept(left, right); + } + + + private static void forEachCollectionTypeVariantsPair(T l, T r, BiConsumer typePairConsumer) + { + forEachTypesPair(frozenAndUnfrozen(l), frozenAndUnfrozen(r), typePairConsumer); + } + } diff --git a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java index 4d51f5c122..45a1f0f0fe 100644 --- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java +++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java @@ -33,6 +33,7 @@ import org.apache.commons.lang3.builder.MultilineRecursiveToStringStyle; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.Duration; import org.apache.cassandra.cql3.FieldIdentifier; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.SchemaCQLHelper; @@ -316,4 +317,24 @@ public final class CassandraGenerators } }, true); } + + public static Gen duration() + { + Constraint ints = Constraint.between(0, Integer.MAX_VALUE); + Constraint longs = Constraint.between(0, Long.MAX_VALUE); + Gen neg = SourceDSL.booleans().all(); + return rnd -> { + int months = (int) rnd.next(ints); + int days = (int) rnd.next(ints); + long nanoseconds = rnd.next(longs); + if (neg.generate(rnd)) + { + months = -1 * months; + days = -1 * days; + nanoseconds = -1 * nanoseconds; + } + return Duration.newInstance(months, days, nanoseconds); + }; + } + } diff --git a/test/unit/org/apache/cassandra/utils/Generators.java b/test/unit/org/apache/cassandra/utils/Generators.java index 179c0f4155..4389aa68a2 100644 --- a/test/unit/org/apache/cassandra/utils/Generators.java +++ b/test/unit/org/apache/cassandra/utils/Generators.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.utils; +import java.math.BigDecimal; +import java.math.BigInteger; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; @@ -61,6 +63,11 @@ public final class Generators public static final Gen IDENTIFIER_GEN = Generators.regexWord(SourceDSL.integers().between(1, 50)); + public static Gen letterOrDigit() + { + return SourceDSL.integers().between(0, LETTER_OR_DIGIT_DOMAIN.length - 1).map(idx -> LETTER_OR_DIGIT_DOMAIN[idx]); + } + public static final Gen UUID_RANDOM_GEN = rnd -> { long most = rnd.next(Constraint.none()); most &= 0x0f << 8; /* clear version */ @@ -71,6 +78,16 @@ public final class Generators return new UUID(most, least); }; + public static final Gen UUID_TIME_GEN = rnd -> { + long most = rnd.next(Constraint.none()); + most &= 0x0f << 8; /* clear version */ + most += 0x10 << 8; /* set to version 1 */ + long least = rnd.next(Constraint.none()); + least &= 0x3fl << 56; /* clear variant */ + least |= 0x80l << 56; /* set to IETF variant */ + return new UUID(most, least); + }; + public static final Gen DNS_DOMAIN_NAME = rnd -> { // how many parts to generate int numParts = (int) rnd.next(DNS_DOMAIN_PARTS_CONSTRAINT); @@ -331,6 +348,48 @@ public final class Generators .map(s -> new String(s.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8)); } + public static Gen bigInt() + { + return bigInt(SourceDSL.integers().between(1, 32)); + } + + public static Gen bigInt(Gen numBitsGen) + { + Gen signumGen = SourceDSL.arbitrary().pick(-1, 0, 1); + return rnd -> { + int signum = signumGen.generate(rnd); + if (signum == 0) + return BigInteger.ZERO; + int numBits = numBitsGen.generate(rnd); + if (numBits < 0) + throw new IllegalArgumentException("numBits must be non-negative"); + int numBytes = (int)(((long)numBits+7)/8); // avoid overflow + + // Generate random bytes and mask out any excess bits + byte[] randomBits = new byte[0]; + if (numBytes > 0) { + randomBits = bytes(numBytes, numBytes).map(bb -> ByteBufferUtil.getArray(bb)).generate(rnd); + int excessBits = 8*numBytes - numBits; + randomBits[0] &= (1 << (8-excessBits)) - 1; + } + return new BigInteger(signum, randomBits); + }; + } + + public static Gen bigDecimal() + { + return bigDecimal(SourceDSL.integers().between(1, 100), bigInt()); + } + + public static Gen bigDecimal(Gen scaleGen, Gen bigIntegerGen) + { + return rnd -> { + int scale = scaleGen.generate(rnd); + BigInteger bigInt = bigIntegerGen.generate(rnd); + return new BigDecimal(bigInt, scale); + }; + } + private static boolean isDash(char c) { switch (c) diff --git a/test/unit/org/apache/cassandra/utils/asserts/SoftAssertionsWithLimit.java b/test/unit/org/apache/cassandra/utils/asserts/SoftAssertionsWithLimit.java new file mode 100644 index 0000000000..d7a82e98e8 --- /dev/null +++ b/test/unit/org/apache/cassandra/utils/asserts/SoftAssertionsWithLimit.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils.asserts; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.assertj.core.api.SoftAssertions; + +public class SoftAssertionsWithLimit extends SoftAssertions +{ + private final AtomicInteger counter = new AtomicInteger(); + private final int limit; + + public SoftAssertionsWithLimit(int limit) + { + this.limit = limit; + } + + @Override + public void collectAssertionError(AssertionError error) + { + int cnt = counter.incrementAndGet(); + if (cnt < limit) + super.collectAssertionError(error); + else if (cnt == limit) + super.collectAssertionError(new AssertionError("Too many assertion errors, stopping collecting them.")); + } +} diff --git a/test/unit/org/quicktheories/impl/JavaRandom.java b/test/unit/org/quicktheories/impl/JavaRandom.java new file mode 100644 index 0000000000..a1e4a780e8 --- /dev/null +++ b/test/unit/org/quicktheories/impl/JavaRandom.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.quicktheories.impl; + +import java.util.Objects; +import java.util.Random; + +import org.quicktheories.core.DetatchedRandomnessSource; +import org.quicktheories.core.RandomnessSource; + +/** + * The {@link Constraint} class does not expose {@link Constraint#min()} or {@link Constraint#max()} outside the package, so + * the only way to build a {@link RandomnessSource} is to define it in the package "org.quicktheories.impl" + */ +public class JavaRandom implements RandomnessSource, DetatchedRandomnessSource +{ + private final Random random; + + public JavaRandom(long seed) + { + this.random = new Random(seed); + } + + public JavaRandom(Random random) + { + this.random = Objects.requireNonNull(random); + } + + public void setSeed(long seed) + { + this.random.setSeed(seed); + } + + public static JavaRandom wrap(RandomnessSource rnd) + { + if (rnd instanceof JavaRandom) + return (JavaRandom) rnd; + return new JavaRandom(rnd.next(Constraint.none().withNoShrinkPoint())); + } + + @Override + public long next(Constraint constraint) + { + long max = constraint.max(); + return nextLong(constraint.min(), max == Long.MAX_VALUE ? max : max + 1); + } + + private long nextLong(long minInclusive, long maxExclusive) + { + // pulled from accord.utils.RandomSource#nextLong(long, long)... + // long term it will be great to unify the two and drop QuickTheories classes all together + // this is diff behavior than ThreadLocalRandom, which returns nextLong + if (minInclusive >= maxExclusive) + throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive)); + + long result = random.nextLong(); + long delta = maxExclusive - minInclusive; + long mask = delta - 1; + if ((delta & mask) == 0L) // power of two + result = (result & mask) + minInclusive; + else if (delta > 0L) + { + // reject over-represented candidates + for (long u = result >>> 1; // ensure nonnegative + u + mask - (result = u % delta) < 0L; // rejection check + u = random.nextLong() >>> 1) // retry + ; + result += minInclusive; + } + else + { + // range not representable as long + while (result < minInclusive || result >= maxExclusive) + result = random.nextLong(); + } + return result; + } + + @Override + public DetatchedRandomnessSource detach() + { + return this; + } + + @Override + public void registerFailedAssumption() + { + + } + + @Override + public void commit() + { + // no-op + } +}