natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

AST fuzz tests can be flakey in multi node clusters due to ephemeral read errors caused by a race condition issue with SAIUtils test class 

patch by David Capwell; reviewed by Bernardo Botella Corbi, Caleb Rackliffe for CASSANDRA-20550
This commit is contained in:
David Capwell 2025-04-15 10:24:08 -07:00
parent f7462446a4
commit 6b2cdba56b
3 changed files with 41 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
test/distributed/org/apache/cassandra/distributed/test/cql3/MultiNodeTableWalkBase.java
View File

@ -92,27 +92,6 @@ public abstract class MultiNodeTableWalkBase extends SingleNodeTableWalkTest
super(rs, cluster);
}
@Override
public boolean allowNonPartitionQuery()
{
// This is disabled to make CI stable. There are known issues that are being fixed so have to exclude for now
return false;
}
@Override
public boolean allowNonPartitionMultiColumnQuery()
{
// This is disabled to make CI stable. There are known issues that are being fixed so have to exclude for now
return false;
}
@Override
public boolean allowPartitionQuery()
{
// This is disabled to make CI stable. There are known issues that are being fixed so have to exclude for now
return false;
}
@Override
protected boolean isMultiNode()
{

30
test/distributed/org/apache/cassandra/distributed/test/cql3/StatefulASTBase.java
View File

@ -34,6 +34,7 @@ import java.util.stream.Stream;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
import org.slf4j.Logger;
import accord.utils.Gen;
@ -46,6 +47,8 @@ import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.SimpleStatement;
import com.datastax.driver.core.SocketOptions;
import com.datastax.driver.core.exceptions.ReadFailureException;
import com.datastax.driver.core.exceptions.WriteFailureException;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.KnownIssue;
@ -73,6 +76,7 @@ import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.test.JavaDriverUtils;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.harry.model.ASTSingleTableModel;
import org.apache.cassandra.harry.util.StringUtils;
import org.apache.cassandra.schema.TableMetadata;
@ -496,11 +500,35 @@ public class StatefulASTBase extends TestBaseImpl
.findAny()
.get();
ss.setHost(host);
ResultSet result = session.execute(ss);
ResultSet result;
try
{
result = session.execute(ss);
}
catch (ReadFailureException t)
{
throw new AssertionError("failed from=" + Maps.transformValues(t.getFailuresMap(), BaseState::safeErrorCode), t);
}
catch (WriteFailureException t)
{
throw new AssertionError("failed from=" + Maps.transformValues(t.getFailuresMap(), BaseState::safeErrorCode), t);
}
return getRowsAsByteBuffer(result);
}
}
private static String safeErrorCode(Integer code)
{
try
{
return RequestFailureReason.fromCode(code).name();
}
catch (IllegalArgumentException e)
{
return "Unexpected code " + code + ": " + e.getMessage();
}
}
@VisibleForTesting
static ByteBuffer[][] getRowsAsByteBuffer(ResultSet result)
{

22
test/distributed/org/apache/cassandra/distributed/test/sai/SAIUtil.java
View File

@ -29,7 +29,6 @@ import java.util.stream.Collectors;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInstance;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.IndexStatusManager;
@ -86,23 +85,26 @@ public class SAIUtil
*/
private static void assertIndexesQueryable(Cluster cluster, String keyspace, final Iterable<String> indexes)
{
IInvokableInstance localNode = cluster.get(1);
final List<InetAddressAndPort> nodes =
cluster.stream()
.map(node -> nodeAddress(node.broadcastAddress()))
.collect(Collectors.toList());
localNode.runOnInstance(() -> {
for (String index : indexes)
{
for (InetAddressAndPort node : nodes)
for (var localNode : cluster)
{
if (localNode.isShutdown()) continue;
localNode.runOnInstance(() -> {
for (String index : indexes)
{
Index.Status status = IndexStatusManager.instance.getIndexStatus(node, keyspace, index);
assert status == Index.Status.BUILD_SUCCEEDED
for (InetAddressAndPort node : nodes)
{
Index.Status status = IndexStatusManager.instance.getIndexStatus(node, keyspace, index);
assert status == Index.Status.BUILD_SUCCEEDED
: "Index " + index + " not queryable on node " + node + " (status = " + status + ')';
}
}
}
});
});
}
}
private static InetAddressAndPort nodeAddress(InetSocketAddress address)