natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Harry Simulation test halts the JVM when errors are detected which loose all history in CI 

patch by David Capwell; reviewed by Alex Petrov for CASSANDRA-19802
This commit is contained in:
ci worker 2024-07-25 10:28:35 -07:00 committed by dcapwell
parent 93ddd3a976
commit 7f246d5419
5 changed files with 58 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.xml
View File

@ -1797,6 +1797,8 @@
<jvmarg line="-XX:Tier4CompileThreshold=1000"/>
<jvmarg line="-XX:ReservedCodeCacheSize=256M"/>
<jvmarg line="-Xmx8G"/>
<!-- Harry tests kept failing due to direct memory failures and looks like its undersized... so upping to allow more stable runs -->
<jvmarg line="-XX:MaxDirectMemorySize=8G"/>
</testmacro>
</target>

15
test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java
View File

@ -433,13 +433,16 @@ public class SimulationRunner
}
private static Optional<Long> parseHex(Optional<String> value)
public static Optional<Long> parseHex(Optional<String> value)
{
return value.map(s -> {
if (s.startsWith("0x"))
return Hex.parseLong(s, 2, s.length());
throw new IllegalArgumentException("Invalid hex string: " + s);
});
return value.map(SimulationRunner::parseHex);
}
public static long parseHex(String s)
{
if (s.startsWith("0x"))
return Hex.parseLong(s, 2, s.length());
throw new IllegalArgumentException("Invalid hex string: " + s);
}
private static final Pattern CHANCE_PATTERN = Pattern.compile("(uniform|(?<qlog>qlog(\\((?<quantizations>[0-9]+)\\))?):)?(?<min>0(\\.[0-9]+)?)(..(?<max>0\\.[0-9]+))?", Pattern.CASE_INSENSITIVE);

8
test/simulator/main/org/apache/cassandra/simulator/harry/HarryValidatingQuery.java
View File

@ -113,8 +113,9 @@ public class HarryValidatingQuery extends SimulatedAction
if (!throwables.isEmpty())
{
logger.error(String.format("Could not validate %d out of %d replicas %s", throwables.size(), replicas.size(), replicas), throwables.get(0));
System.exit(0);
AssertionError error = new AssertionError(String.format("Could not validate %d out of %d replicas %s", throwables.size(), replicas.size(), replicas));
throwables.forEach(error::addSuppressed);
throw error;
}
}
@ -137,9 +138,8 @@ public class HarryValidatingQuery extends SimulatedAction
catch (Throwable t)
{
logger.error("Caught an exception while validating", t);
t.printStackTrace();
throw t;
}
finally { }
}
};
}

68
test/simulator/test/org/apache/cassandra/simulator/test/HarrySimulatorTest.java
View File

@ -27,6 +27,8 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@ -80,6 +82,7 @@ import org.apache.cassandra.simulator.OrderOn;
import org.apache.cassandra.simulator.RandomSource;
import org.apache.cassandra.simulator.RunnableActionScheduler;
import org.apache.cassandra.simulator.Simulation;
import org.apache.cassandra.simulator.SimulationException;
import org.apache.cassandra.simulator.SimulationRunner;
import org.apache.cassandra.simulator.SimulatorUtils;
import org.apache.cassandra.simulator.cluster.ClusterActionListener.NoOpListener;
@ -187,6 +190,8 @@ public class HarrySimulatorTest
public int rowsPerPhase = 10;
@Option(name = {"--nodes-per-dc"}, description = "How many nodes per dc for replication")
public int nodesPerDc = 3;
@Option(name = {"-s", "--seed"}, title = "0x", description = "What seed to run with; in hex format... example: 0x190e6ff01d6")
public String seed = null;
public static void main(String... args) throws Throwable
{
@ -201,6 +206,8 @@ public class HarrySimulatorTest
public void test() throws Exception
{
rowsPerPhase = 1;
// To rerun a failing test for a given seed, uncomment the below and set the seed
// this.seed = "<your seed here>";
harryTest();
}
@ -436,7 +443,23 @@ public class HarrySimulatorTest
try (CloseableIterator<?> iter = iterator())
{
while (iter.hasNext())
{
checkForErrors();
iter.next();
}
checkForErrors();
}
}
private void checkForErrors()
{
if (simulated.failures.hasFailure())
{
AssertionError error = new AssertionError("Errors detected during simulation");
// don't care about the stack trace... the issue is the errors found and not what part of the scheduler we stopped
error.setStackTrace(new StackTraceElement[0]);
simulated.failures.get().forEach(error::addSuppressed);
throw error;
}
}
@ -504,43 +527,38 @@ public class HarrySimulatorTest
/**
* Simulation entrypoint; syntax sugar for creating a simulation.
*/
static void simulate(Consumer<ClusterSimulation.Builder<HarrySimulation>> configure,
Consumer<IInstanceConfig> instanceConfigUpdater,
Configuration.ConfigurationBuilder harryConfig,
String[] properties,
Function<HarrySimulation, ActionSchedule.Work[]>... phases) throws IOException
void simulate(Consumer<ClusterSimulation.Builder<HarrySimulation>> configure,
Consumer<IInstanceConfig> instanceConfigUpdater,
Configuration.ConfigurationBuilder harryConfig,
String[] properties,
Function<HarrySimulation, ActionSchedule.Work[]>... phases) throws IOException
{
try (WithProperties p = new WithProperties().with(properties))
{
HarrySimulationBuilder factory = new HarrySimulationBuilder(harryConfig, instanceConfigUpdater);
SimulationRunner.beforeAll();
long seed = System.currentTimeMillis();
// Development seed:
//long seed = 1687184561194L;
System.out.println("Simulation seed: " + seed + "L");
long seed = SimulationRunner.parseHex(Optional.ofNullable(this.seed)).orElseGet(() -> new Random().nextLong());
logger.info("Seed 0x{}", Long.toHexString(seed));
configure.accept(factory);
try (ClusterSimulation<HarrySimulation> clusterSimulation = factory.create(seed))
{
try
{
HarrySimulation simulation = clusterSimulation.simulation();
HarrySimulation simulation = clusterSimulation.simulation();
// For better determinism during startup, we allow instances to fully start (including daemon work)
for (int i = 0; i < phases.length; i++)
{
HarrySimulation current = simulation;
if (i == 0)
current = current.withScheduler(new RunnableActionScheduler.Immediate()).withSchedulers((s) -> Collections.emptyMap());
current.withSchedule(phases[i]).run();
}
}
catch (Throwable t)
// For better determinism during startup, we allow instances to fully start (including daemon work)
for (int i = 0; i < phases.length; i++)
{
throw new AssertionError(String.format("Failed on seed %s", Long.toHexString(seed)),
t);
HarrySimulation current = simulation;
if (i == 0)
current = current.withScheduler(new RunnableActionScheduler.Immediate()).withSchedulers((s) -> Collections.emptyMap());
current.withSchedule(phases[i]).run();
}
}
catch (Throwable t)
{
if (t instanceof SimulationException) throw t;
throw new SimulationException(seed, "Failure creating the simulation", t);
}
}
}
@ -976,9 +994,7 @@ public class HarrySimulatorTest
public void onFailure(Throwable t)
{
super.onFailure(t);
t.printStackTrace();
logger.error("Caught an exception, going to halt", t);
//while (true) { }
}
}
}

2
test/simulator/test/org/apache/cassandra/simulator/test/SimulationTestBase.java
View File

@ -206,7 +206,7 @@ public class SimulationTestBase
long seed = System.currentTimeMillis();
// Development seed:
//long seed = 1687184561194L;
System.out.println("Simulation seed: " + seed + "L");
logger.info("Simulation seed: {}L", seed);
configure.accept(factory);
try (ClusterSimulation<?> cluster = factory.create(seed))
{