natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

To improve accord interoperability test coverage, need to extend the harry model domain to handle more possible CQL states 

patch by David Capwell; reviewed by Ariel Weisberg for CASSANDRA-20156
This commit is contained in:
David Capwell 2025-01-31 19:44:22 -08:00
parent 113d83417e
commit d543cc9107
13 changed files with 449 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/accord

@ -1 +1 @@
Subproject commit f543f7dae959e804a39e465654d702bc34db1bd1
Subproject commit e5e108adfeb817f899ca89507e8fa6bc0b191759

111
test/distributed/org/apache/cassandra/distributed/test/cql3/AccordInteropMultiNodeTableWalkBase.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.cql3;
import accord.utils.Property;
import accord.utils.RandomSource;
import org.apache.cassandra.cql3.KnownIssue;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
public abstract class AccordInteropMultiNodeTableWalkBase extends MultiNodeTableWalkBase
{
private final TransactionalMode transactionalMode;
protected AccordInteropMultiNodeTableWalkBase(TransactionalMode transactionalMode)
{
super(ReadRepairStrategy.NONE);
this.transactionalMode = transactionalMode;
}
@Override
protected void preCheck(Cluster cluster, Property.StatefulBuilder builder)
{
addUncaughtExceptionsFilter(cluster);
}
static void addUncaughtExceptionsFilter(Cluster cluster)
{
if (IGNORED_ISSUES.contains(KnownIssue.ACCORD_JOURNAL_SUPPORT_DROP_TABLE))
{
cluster.setUncaughtExceptionsFilter(t -> {
// There is a known issue with drop table and journal snapshots where the journal can't find the given table...
// To make these tests stable need to ignore this error as it's unrelated to the test and is target to be fixed
// directly.
/*
Suppressed: java.lang.AssertionError: Unknown keyspace ks12
at org.apache.cassandra.db.Keyspace.open(Keyspace.java:149)
at org.apache.cassandra.db.Keyspace.openAndGetStoreIfExists(Keyspace.java:172)
at org.apache.cassandra.service.accord.AccordDataStore.lambda$snapshot$2(AccordDataStore.java:104)
*/
if (t instanceof AssertionError
&& t.getMessage() != null
&& t.getMessage().startsWith("Unknown keyspace ks"))
return true;
return false;
});
}
}
@Override
protected void clusterConfig(IInstanceConfig c)
{
super.clusterConfig(c);
c.set("transaction_timeout", "180s");
}
@Override
protected TableMetadata defineTable(RandomSource rs, String ks)
{
TableMetadata metadata = super.defineTable(rs, ks);
return metadata.withSwapped(metadata.params.unbuild().transactionalMode(transactionalMode).build());
}
@Override
protected State createState(RandomSource rs, Cluster cluster)
{
return new AccordInteropMultiNodeState(rs, cluster);
}
public class AccordInteropMultiNodeState extends MultiNodeState
{
public AccordInteropMultiNodeState(RandomSource rs, Cluster cluster)
{
super(rs, cluster);
}
@Override
protected ConsistencyLevel selectCl()
{
return ConsistencyLevel.QUORUM;
}
@Override
protected ConsistencyLevel mutationCl()
{
return ConsistencyLevel.QUORUM;
}
}
}

65
test/distributed/org/apache/cassandra/distributed/test/cql3/AccordInteropMultiNodeTokenConflictBase.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.cql3;
import accord.utils.Property;
import accord.utils.RandomSource;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.service.consensus.TransactionalMode;
public abstract class AccordInteropMultiNodeTokenConflictBase extends MultiNodeTokenConflictTest
{
protected AccordInteropMultiNodeTokenConflictBase(TransactionalMode transactionalMode)
{
super(transactionalMode);
}
@Override
protected void preCheck(Cluster cluster, Property.StatefulBuilder builder)
{
AccordInteropMultiNodeTableWalkBase.addUncaughtExceptionsFilter(cluster);
}
@Override
protected State createState(RandomSource rs, Cluster cluster)
{
return new AccordInteropState(rs, cluster);
}
private class AccordInteropState extends State
{
AccordInteropState(RandomSource rs, Cluster cluster)
{
super(rs, cluster);
}
@Override
protected ConsistencyLevel selectCl()
{
return ConsistencyLevel.QUORUM;
}
@Override
protected ConsistencyLevel mutationCl()
{
return ConsistencyLevel.QUORUM;
}
}
}

41
test/distributed/org/apache/cassandra/distributed/test/cql3/FullAccordInteropMultiNodeTableWalkTest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.cql3;
import accord.utils.Property;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.service.consensus.TransactionalMode;
public class FullAccordInteropMultiNodeTableWalkTest extends AccordInteropMultiNodeTableWalkBase
{
public FullAccordInteropMultiNodeTableWalkTest()
{
super(TransactionalMode.full);
}
@Override
protected void preCheck(Cluster cluster, Property.StatefulBuilder builder)
{
super.preCheck(cluster, builder);
// if a failing seed is detected, populate here
// Example: builder.withSeed(42L);
// CQL operations may have opertors such as +, -, and / (example 4 + 4), to "apply" them to get a constant value
// CQL_DEBUG_APPLY_OPERATOR = true;
}
}

41
test/distributed/org/apache/cassandra/distributed/test/cql3/FullAccordInteropMultiNodeTokenConflictTest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.cql3;
import accord.utils.Property;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.service.consensus.TransactionalMode;
public class FullAccordInteropMultiNodeTokenConflictTest extends AccordInteropMultiNodeTokenConflictBase
{
public FullAccordInteropMultiNodeTokenConflictTest()
{
super(TransactionalMode.full);
}
@Override
protected void preCheck(Cluster cluster, Property.StatefulBuilder builder)
{
super.preCheck(cluster, builder);
// if a failing seed is detected, populate here
// Example: builder.withSeed(42L);
// CQL operations may have opertors such as +, -, and / (example 4 + 4), to "apply" them to get a constant value
// CQL_DEBUG_APPLY_OPERATOR = true;
}
}

41
test/distributed/org/apache/cassandra/distributed/test/cql3/MixedReadsAccordInteropMultiNodeTableWalkTest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.cql3;
import accord.utils.Property;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.service.consensus.TransactionalMode;
public class MixedReadsAccordInteropMultiNodeTableWalkTest extends AccordInteropMultiNodeTableWalkBase
{
public MixedReadsAccordInteropMultiNodeTableWalkTest()
{
super(TransactionalMode.mixed_reads);
}
@Override
protected void preCheck(Cluster cluster, Property.StatefulBuilder builder)
{
super.preCheck(cluster, builder);
// if a failing seed is detected, populate here
// Example: builder.withSeed(42L);
// CQL operations may have opertors such as +, -, and / (example 4 + 4), to "apply" them to get a constant value
// CQL_DEBUG_APPLY_OPERATOR = true;
}
}

41
test/distributed/org/apache/cassandra/distributed/test/cql3/MixedReadsAccordInteropMultiNodeTokenConflictTest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.cql3;
import accord.utils.Property;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.service.consensus.TransactionalMode;
public class MixedReadsAccordInteropMultiNodeTokenConflictTest extends AccordInteropMultiNodeTokenConflictBase
{
public MixedReadsAccordInteropMultiNodeTokenConflictTest()
{
super(TransactionalMode.mixed_reads);
}
@Override
protected void preCheck(Cluster cluster, Property.StatefulBuilder builder)
{
super.preCheck(cluster, builder);
// if a failing seed is detected, populate here
// Example: builder.withSeed(42L);
// CQL operations may have opertors such as +, -, and / (example 4 + 4), to "apply" them to get a constant value
// CQL_DEBUG_APPLY_OPERATOR = true;
}
}

15
test/distributed/org/apache/cassandra/distributed/test/cql3/MultiNodeTokenConflictTest.java
View File

@ -20,18 +20,31 @@ package org.apache.cassandra.distributed.test.cql3;
import java.io.IOException;
import javax.annotation.Nullable;
import accord.utils.Property;
import accord.utils.RandomSource;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
public class MultiNodeTokenConflictTest extends SingleNodeTokenConflictTest
{
protected MultiNodeTokenConflictTest(@Nullable TransactionalMode transactionalMode)
{
super(transactionalMode);
}
public MultiNodeTokenConflictTest()
{
super();
}
@Override
protected void preCheck(Property.StatefulBuilder builder)
protected void preCheck(Cluster cluster, Property.StatefulBuilder builder)
{
// if a failing seed is detected, populate here
// Example: builder.withSeed(42L);

4
test/distributed/org/apache/cassandra/distributed/test/cql3/SingleNodeTableWalkTest.java
View File

@ -94,6 +94,10 @@ public class SingleNodeTableWalkTest extends StatefulASTBase
protected static boolean READ_AFTER_WRITE = false;
public SingleNodeTableWalkTest()
{
}
protected void preCheck(Cluster cluster, Property.StatefulBuilder builder)
{
// if a failing seed is detected, populate here

60
test/distributed/org/apache/cassandra/distributed/test/cql3/SingleNodeTokenConflictTest.java
View File

@ -33,6 +33,8 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -55,6 +57,7 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
import org.apache.cassandra.utils.ASTGenerators;
import org.apache.cassandra.utils.AbstractTypeGenerators;
@ -86,7 +89,20 @@ public class SingleNodeTokenConflictTest extends StatefulASTBase
AbstractTypeGenerators.withoutUnsafeEquality(AbstractTypeGenerators.builder()
.withTypeKinds(TypeKind.PRIMITIVE));
protected void preCheck(Property.StatefulBuilder builder)
@Nullable
private final TransactionalMode transactionalMode;
protected SingleNodeTokenConflictTest(@Nullable TransactionalMode transactionalMode)
{
this.transactionalMode = transactionalMode;
}
public SingleNodeTokenConflictTest()
{
this(null);
}
protected void preCheck(Cluster cluster, Property.StatefulBuilder builder)
{
// if a failing seed is detected, populate here
// Example: builder.withSeed(42L);
@ -249,7 +265,7 @@ public class SingleNodeTokenConflictTest extends StatefulASTBase
try (Cluster cluster = createCluster())
{
Property.StatefulBuilder statefulBuilder = stateful().withExamples(10);
preCheck(statefulBuilder);
preCheck(cluster, statefulBuilder);
statefulBuilder.check(commands(() -> rs -> createState(rs, cluster))
.add(StatefulASTBase::insert)
//TODO (now, coverage): this is flakey and non-deterministic. When this fails (gives bad response) rerunning the seed yields a passing test!
@ -279,23 +295,26 @@ public class SingleNodeTokenConflictTest extends StatefulASTBase
protected TableMetadata defineTable(RandomSource rs, String ks)
{
return toGen(new TableMetadataBuilder()
.withTableKinds(TableMetadata.Kind.REGULAR)
.withKnownMemtables()
.withKeyspaceName(ks).withTableName("tbl")
.withSimpleColumnNames()
.withDefaultTypeGen(SUPPORTED_TYPES)
.withPartitioner(Murmur3Partitioner.instance)
.withPartitionColumnsCount(1)
// this should produce vector<bigint, 2>... should make this easier...
.withPartitionColumnTypeGen(new TypeGenBuilder()
.withMaxDepth(0)
.withTypeKinds(TypeKind.VECTOR)
.withPrimitives(LongType.instance)
.withVectorSizeGen(i -> 2)
.withDefaultSizeGen(1))
.build())
.next(rs);
TableMetadata metadata = toGen(new TableMetadataBuilder()
.withTableKinds(TableMetadata.Kind.REGULAR)
.withKnownMemtables()
.withKeyspaceName(ks).withTableName("tbl")
.withSimpleColumnNames()
.withDefaultTypeGen(SUPPORTED_TYPES)
.withPartitioner(Murmur3Partitioner.instance)
.withPartitionColumnsCount(1)
// this should produce vector<bigint, 2>... should make this easier...
.withPartitionColumnTypeGen(new TypeGenBuilder()
.withMaxDepth(0)
.withTypeKinds(TypeKind.VECTOR)
.withPrimitives(LongType.instance)
.withVectorSizeGen(i -> 2)
.withDefaultSizeGen(1))
.build())
.next(rs);
if (transactionalMode != null)
metadata = metadata.withSwapped(metadata.params.unbuild().transactionalMode(transactionalMode).build());
return metadata;
}
class State extends CommonState
@ -402,8 +421,7 @@ public class SingleNodeTokenConflictTest extends StatefulASTBase
LinkedHashSet<ByteBuffer> pks = new LinkedHashSet<>();
for (int i = 0; i < numPks; i++)
{
// TODO: add back when accord-core Property supports it
ByteBuffer value = null;//rs.pickOrderedSet(available);
ByteBuffer value = rs.pickOrderedSet(available);
pks.add(value);
available.remove(value);
}

2
test/unit/org/apache/cassandra/cql3/KnownIssue.java
View File

@ -47,6 +47,8 @@ public enum KnownIssue
"When you do a CAS to the partition level the read is SELECT statics LIMIT 1, if the CAS doesn't apply the response includes the first row in the partition with its values redacted... this statement is partition level and not row level, would expect just the applied column like the other cases where the static row isn't present"),
STATIC_LIST_APPEND_WITH_CLUSTERING_IN("",
"When an 'UPDATE SET s += [0] WHERE pk = ? AND ck IN (?, ?)' happens the static operation happens twice, so the list append adds 2 elements!"),
ACCORD_JOURNAL_SUPPORT_DROP_TABLE("",
"When DROP TABLE is done, this is currently not plumbed through to Journals snapshot logic, which leads to unknown keyspace/table errors"),
;
KnownIssue(String url, String description)

2
test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java
View File

@ -1382,7 +1382,7 @@ public final class AbstractTypeGenerators
TypeSupport<Object> support = eSupport.get(i);
elements.add(support.type.decompose(support.valueGen.generate(rnd)));
}
return type.pack(elements, ByteBufferAccessor.instance);
return type.pack(elements);
}
}

48
test/unit/org/apache/cassandra/utils/GeneratorsTest.java
View File

@ -18,11 +18,21 @@
package org.apache.cassandra.utils;
import java.util.List;
import java.util.Objects;
import java.util.Random;
import com.google.common.net.InternetDomainName;
import org.junit.Test;
import accord.utils.Property;
import org.apache.cassandra.db.marshal.AsciiType;
import org.assertj.core.api.Assertions;
import org.quicktheories.core.Gen;
import org.quicktheories.generators.SourceDSL;
import org.quicktheories.impl.JavaRandom;
import static org.apache.cassandra.utils.AbstractTypeGenerators.stringComparator;
import static org.quicktheories.QuickTheory.qt;
public class GeneratorsTest
@ -45,4 +55,42 @@ public class GeneratorsTest
{
qt().forAll(Generators.DNS_DOMAIN_NAME).checkAssert(InternetDomainName::from);
}
@Test
public void asciiDeterministic()
{
AbstractTypeGenerators.TypeSupport<String> support = AbstractTypeGenerators.TypeSupport.of(AsciiType.instance, SourceDSL.strings().ascii().ofLengthBetween(1, 10), stringComparator(AsciiType.instance));
int samples = 100;
int attempts = 100;
Property.qt().check(rs -> checkDeterministicGeneration(attempts, samples, rs.nextLong(), support.valueGen));
Property.qt().check(rs -> checkDeterministicGeneration(attempts, samples, rs.nextLong(), support.bytesGen()));
}
@Test
public void asciiThereAndBackAgain()
{
qt().forAll(SourceDSL.strings().ascii().ofLengthBetween(1, 100)).checkAssert(ascii -> {
String accum = ascii;
for (int i = 0; i < 100; i++)
accum = AsciiType.instance.compose(AsciiType.instance.decompose(accum));
Assertions.assertThat(accum).isEqualTo(ascii);
});
}
private static <T> void checkDeterministicGeneration(int attempts, int samples, long seed, Gen<T> gen)
{
List<T> goldSet = null;
for (int i = 0; i < attempts; i++)
{
JavaRandom qt = new JavaRandom(new Random(seed));
List<T> sample = SourceDSL.lists().of(gen).ofSize(samples).generate(qt);
if (goldSet == null)
{
goldSet = sample;
continue;
}
if (!Objects.equals(sample, goldSet))
throw new AssertionError("seed=" + seed);
}
}
}