From a277c2015d1c43ebb1b422167a3ad0cf8409ac45 Mon Sep 17 00:00:00 2001 From: Jon Haddad Date: Wed, 10 Jun 2026 19:02:08 -0700 Subject: [PATCH] Widen the randomized differential soak's workload space MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The soak previously generated inserts, row deletes, and partition deletes over NORMAL-domain values with single-column partition keys — much narrower than the deterministic corpus. It now also generates: UPDATE-built rows (cells without primary-key liveness), primary-key-only INSERTs (liveness without cells), long-TTL inserts, explicit USING TIMESTAMP collisions from a small pool (same-timestamp tie-breaks on overwritten keys), null values on non-key columns (cell tombstones on simple columns; empty buffers on clustering columns), single-sided and prefix range deletes, multi-column and static cell deletes, and composite partition keys. Any schema the generator produces that needs multi-cell-column support is rejected before use by the same unsupportedMetadata filter production uses, so the null-value generation is safe on this branch even though it does not support multi-cell columns. Two new knobs, both forwarded via -Dtest.jvm.args: - cassandra.test.differential.examples=N scales the example count (default 10; the pre-JIRA validation run uses thousands) - cassandra.test.differential.seed=N replays a failing seed as example 0 — no code edit needed to reproduce The widened space found a real bug in its first 25-example run: the reversed-column empty-clustering comparison fix ("Respect clustering reversal when comparing absent components"), caught as an 8-byte covered-clustering divergence in Statistics.db (seed 99303954147053). --- .../RandomDifferentialCompactionTest.java | 244 ++++++++++++++++-- 1 file changed, 227 insertions(+), 17 deletions(-) diff --git a/test/unit/org/apache/cassandra/db/compaction/differential/RandomDifferentialCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/differential/RandomDifferentialCompactionTest.java index 9d8c191284..85b8d92d47 100644 --- a/test/unit/org/apache/cassandra/db/compaction/differential/RandomDifferentialCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/differential/RandomDifferentialCompactionTest.java @@ -21,8 +21,10 @@ package org.apache.cassandra.db.compaction.differential; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.Set; @@ -30,6 +32,7 @@ import com.google.common.collect.ImmutableList; import org.junit.Test; import org.quicktheories.core.Gen; +import org.quicktheories.generators.SourceDSL; import org.quicktheories.impl.JavaRandom; import org.apache.cassandra.config.CassandraRelevantProperties; @@ -39,6 +42,7 @@ import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.AbstractTypeGenerators; import org.apache.cassandra.utils.AbstractTypeGenerators.TypeGenBuilder; +import org.apache.cassandra.utils.AbstractTypeGenerators.ValueDomain; import org.apache.cassandra.utils.CassandraGenerators; import org.apache.cassandra.utils.CassandraGenerators.TableMetadataBuilder; import org.apache.cassandra.utils.Generators; @@ -51,11 +55,23 @@ import static org.apache.cassandra.utils.Generators.IDENTIFIER_GEN; * production uses), random multi-round workloads with overwrites and deletes flushed into * overlapping sstables, then byte+logical differential comparison of both compaction paths. * - * Reproducing a failure: every failure message is wrapped in a seed; plug it into - * {@code withFixedSeed} below. + * Workload space: INSERT / UPDATE (no row liveness) / primary-key-only INSERT (liveness, no + * cells) / INSERT USING TTL (long-lived, far from the expiry boundary) / explicit USING + * TIMESTAMP collisions from a small pool (same-timestamp tie-breaks); null values on + * non-key columns (cell tombstones on simple columns; mapped to empty buffers on clustering + * columns); row deletes, partition deletes, single-sided and prefix range deletes, + * multi-column cell deletes, static cell deletes. Composite partition keys (1-2 components). * - * Known coverage gaps (deliberate): null/empty - * value domains and range deletes are exercised by the deterministic corpus only. + * Example count is property-gated: -Dcassandra.test.differential.examples=N (default + * {@value #DEFAULT_EXAMPLES}; the pre-JIRA validation run uses thousands). + * + * Reproducing a failure: every failure message is wrapped in a seed; rerun with + * -Dcassandra.test.differential.seed=N (the failing seed becomes example 0), or plug it + * into {@code withFixedSeed} below. + * + * Known coverage gaps (deliberate, covered by the deterministic corpus): EMPTY_BYTES value + * domain (invalid CQL for some generated multi-cell shapes), collection-element deletes + * (DELETE m['k'] needs element values of the right type), expired TTLs (timing-dependent). */ public class RandomDifferentialCompactionTest extends DifferentialCompactionTester { @@ -66,7 +82,11 @@ public class RandomDifferentialCompactionTest extends DifferentialCompactionTest } private static final Set ALLOWLIST = Set.of(); - private static final int EXAMPLES = 10; + private static final int DEFAULT_EXAMPLES = 10; + private static final int EXAMPLES = Integer.getInteger("cassandra.test.differential.examples", DEFAULT_EXAMPLES); + + /** Long enough that expiry can never fall between the two differential runs. */ + private static final int SOAK_TTL_SECONDS = 30 * 24 * 60 * 60; @Test public void randomizedDifferential() throws Throwable @@ -94,15 +114,14 @@ public class RandomDifferentialCompactionTest extends DifferentialCompactionTest .withDefaultSetKey(safePrimary) .withoutTypeKinds(AbstractTypeGenerators.TypeKind.COUNTER) .withUDTNames(udtName)) - .withPartitionColumnsCount(1) + .withPartitionColumnsBetween(1, 2) .withPrimaryColumnTypeGen(new TypeGenBuilder(safePrimary).withMaxDepth(1)) .withClusteringColumnsBetween(0, 3) .withRegularColumnsBetween(1, 5) .withStaticColumnsBetween(0, 2) .build(qtRandom); } - // Same filter production uses to route to the cursor pipeline. When increment 2 lands - // (multi-cell columns), this loop disappears and the generator space widens. + // Same filter production uses to route to the cursor pipeline. // Also rejects invalid CQL the generator can produce: static columns require // clustering columns. while (CursorCompactor.unsupportedMetadata(metadata) @@ -117,13 +136,35 @@ public class RandomDifferentialCompactionTest extends DifferentialCompactionTest ColumnFamilyStore cfs = getColumnFamilyStore(KEYSPACE, metadata.name); cfs.disableAutoCompaction(); - Gen dataGen = CassandraGenerators.data(metadata, ignore -> AbstractTypeGenerators.ValueDomain.NORMAL); + // ~12% of non-key values are null: cell tombstones on simple columns. The data + // generator maps null to an empty buffer on clustering columns (null clustering is + // invalid; empty is legal and exercises the empty-vs-valued clustering comparison) + // and never applies the domain to partition keys. + Gen valueDomains = SourceDSL.integers().between(0, 99) + .map(i -> i < 12 ? ValueDomain.NULL : ValueDomain.NORMAL); + Gen dataGen = CassandraGenerators.data(metadata, valueDomains); + int partitionColumnCount = metadata.partitionKeyColumns().size(); - int primaryColumnCount = partitionColumnCount + metadata.clusteringColumns().size(); + int clusteringColumnCount = metadata.clusteringColumns().size(); + int primaryColumnCount = partitionColumnCount + clusteringColumnCount; String insertStmt = insertStmt(metadata); String deleteRowStmt = deleteStmt(metadata, primaryColumnCount); String deletePartitionStmt = deleteStmt(metadata, partitionColumnCount); + // select-order index of every column, for UPDATE/cell-delete binding + Map selectOrderIndex = new HashMap<>(); + { + Iterator it = metadata.allColumnsInSelectOrder(); + for (int i = 0; it.hasNext(); i++) + selectOrderIndex.put(it.next().name.toString(), i); + } + List regularColumns = ImmutableList.copyOf(metadata.regularColumns()); + List staticColumns = ImmutableList.copyOf(metadata.staticColumns()); + + // small explicit-timestamp pool: cross-sstable same-timestamp conflicts on + // overwritten primary keys exercise the reconciliation tie-break rules + long tiePoolBase = 1_000_000; + List rows = new ArrayList<>(); int rounds = 2 + workload.nextInt(3); // 2-4 sstables for (int round = 0; round < rounds; round++) @@ -132,14 +173,43 @@ public class RandomDifferentialCompactionTest extends DifferentialCompactionTest for (int i = 0; i < inserts; i++) { ByteBuffer[] row = dataGen.generate(qtRandom); - if (!rows.isEmpty() && workload.nextInt(100) < 30) + boolean overwrite = !rows.isEmpty() && workload.nextInt(100) < 30; + if (overwrite) { // overwrite: keep a previously used primary key, fresh non-key values — // this is what makes the merge actually reconcile rather than concatenate ByteBuffer[] prev = rows.get(workload.nextInt(rows.size())); System.arraycopy(prev, 0, row, 0, primaryColumnCount); } - execute(insertStmt, (Object[]) row); + + int mode = workload.nextInt(100); + if (overwrite && workload.nextInt(100) < 40) + { + // explicit-timestamp collision candidate: two writes to the same primary + // key with the same timestamp force the same-ts tie-break path + long ts = tiePoolBase + workload.nextInt(3); + execute(insertStmt + " USING TIMESTAMP " + ts, (Object[]) row); + } + else if (mode < 15 && !regularColumns.isEmpty()) + { + // UPDATE: writes cells without primary-key liveness (different row flags) + execute(updateStmt(metadata, regularColumns), + updateParams(row, regularColumns, selectOrderIndex, primaryColumnCount)); + } + else if (mode < 22) + { + // primary-key-only INSERT: row liveness with zero cells + execute(pkOnlyInsertStmt(metadata), (Object[]) Arrays.copyOf(row, primaryColumnCount)); + } + else if (mode < 30) + { + // long TTL: liveness info with ttl + expiration far from the runs + execute(insertStmt + " USING TTL " + SOAK_TTL_SECONDS, (Object[]) row); + } + else + { + execute(insertStmt, (Object[]) row); + } rows.add(row); } @@ -149,6 +219,47 @@ public class RandomDifferentialCompactionTest extends DifferentialCompactionTest ByteBuffer[] victim = rows.get(workload.nextInt(rows.size())); execute(deleteRowStmt, (Object[]) Arrays.copyOf(victim, primaryColumnCount)); } + + // range deletes (clustering tables only): single-sided slices and clustering-prefix + // deletes against known keys; single-sided bounds cannot produce inverted ranges + for (int i = 0; i < 2 && clusteringColumnCount > 0 && !rows.isEmpty(); i++) + { + ByteBuffer[] victim = rows.get(workload.nextInt(rows.size())); + if (clusteringColumnCount >= 2 && workload.nextBoolean()) + { + // prefix delete: equality on a strict prefix of the clustering columns + int depth = 1 + workload.nextInt(clusteringColumnCount - 1); + execute(deleteStmt(metadata, partitionColumnCount + depth), + (Object[]) Arrays.copyOf(victim, partitionColumnCount + depth)); + } + else + { + int eqDepth = workload.nextInt(clusteringColumnCount); + String op = new String[]{ ">=", ">", "<=", "<" }[workload.nextInt(4)]; + execute(rangeDeleteStmt(metadata, eqDepth, op), + (Object[]) Arrays.copyOf(victim, partitionColumnCount + eqDepth + 1)); + } + } + + // cell deletes: random subset of regular columns at a known row; occasionally a + // static cell delete instead + for (int i = 0; i < 2 && !rows.isEmpty(); i++) + { + ByteBuffer[] victim = rows.get(workload.nextInt(rows.size())); + if (!staticColumns.isEmpty() && workload.nextInt(100) < 30) + { + ColumnMetadata col = staticColumns.get(workload.nextInt(staticColumns.size())); + execute(cellDeleteStmt(metadata, List.of(col), partitionColumnCount), + (Object[]) Arrays.copyOf(victim, partitionColumnCount)); + } + else + { + List subset = randomSubset(regularColumns, workload); + execute(cellDeleteStmt(metadata, subset, primaryColumnCount), + (Object[]) Arrays.copyOf(victim, primaryColumnCount)); + } + } + // occasional partition delete if (workload.nextInt(100) < 40 && !rows.isEmpty()) { @@ -179,14 +290,113 @@ public class RandomDifferentialCompactionTest extends DifferentialCompactionTest return sb.append(')').toString(); } + /** INSERT binding only the primary key columns: row liveness without any cells. */ + private static String pkOnlyInsertStmt(TableMetadata metadata) + { + List keys = primaryKeyColumns(metadata); + StringBuilder sb = new StringBuilder("INSERT INTO ").append(metadata).append(" ("); + for (int i = 0; i < keys.size(); i++) + { + if (i > 0) sb.append(", "); + sb.append(keys.get(i).name.toCQLString()); + } + sb.append(") VALUES ("); + for (int i = 0; i < keys.size(); i++) + { + if (i > 0) sb.append(", "); + sb.append('?'); + } + return sb.append(')').toString(); + } + + /** UPDATE setting every regular column: cells without primary-key liveness. */ + private static String updateStmt(TableMetadata metadata, List regularColumns) + { + StringBuilder sb = new StringBuilder("UPDATE ").append(metadata).append(" SET "); + for (int i = 0; i < regularColumns.size(); i++) + { + if (i > 0) sb.append(", "); + sb.append(regularColumns.get(i).name.toCQLString()).append(" = ?"); + } + sb.append(" WHERE "); + List keys = primaryKeyColumns(metadata); + for (int i = 0; i < keys.size(); i++) + { + if (i > 0) sb.append(" AND "); + sb.append(keys.get(i).name.toCQLString()).append(" = ?"); + } + return sb.toString(); + } + + private static Object[] updateParams(ByteBuffer[] row, List regularColumns, + Map selectOrderIndex, int primaryColumnCount) + { + Object[] params = new Object[regularColumns.size() + primaryColumnCount]; + for (int i = 0; i < regularColumns.size(); i++) + params[i] = row[selectOrderIndex.get(regularColumns.get(i).name.toString())]; + for (int i = 0; i < primaryColumnCount; i++) + params[regularColumns.size() + i] = row[i]; + return params; + } + + /** DELETE col1, col2 FROM t WHERE first {@code keyColumnCount} primary key columns bound. */ + private static String cellDeleteStmt(TableMetadata metadata, List columns, int keyColumnCount) + { + StringBuilder sb = new StringBuilder("DELETE "); + for (int i = 0; i < columns.size(); i++) + { + if (i > 0) sb.append(", "); + sb.append(columns.get(i).name.toCQLString()); + } + sb.append(" FROM ").append(metadata).append(" WHERE "); + List keys = primaryKeyColumns(metadata); + for (int i = 0; i < keyColumnCount; i++) + { + if (i > 0) sb.append(" AND "); + sb.append(keys.get(i).name.toCQLString()).append(" = ?"); + } + return sb.toString(); + } + + /** + * DELETE with equality on the partition key plus the first {@code eqDepth} clustering + * columns, and a single-sided {@code op} bound on clustering column {@code eqDepth}. + */ + private static String rangeDeleteStmt(TableMetadata metadata, int eqDepth, String op) + { + StringBuilder sb = new StringBuilder("DELETE FROM ").append(metadata).append(" WHERE "); + List keys = primaryKeyColumns(metadata); + int partitionColumnCount = metadata.partitionKeyColumns().size(); + int bound = partitionColumnCount + eqDepth; + for (int i = 0; i < bound; i++) + { + if (i > 0) sb.append(" AND "); + sb.append(keys.get(i).name.toCQLString()).append(" = ?"); + } + sb.append(" AND ").append(keys.get(bound).name.toCQLString()).append(' ').append(op).append(" ?"); + return sb.toString(); + } + + private static List primaryKeyColumns(TableMetadata metadata) + { + return ImmutableList.builder() + .addAll(metadata.partitionKeyColumns()) + .addAll(metadata.clusteringColumns()) + .build(); + } + + private static List randomSubset(List columns, Random workload) + { + List shuffled = new ArrayList<>(columns); + java.util.Collections.shuffle(shuffled, workload); + return shuffled.subList(0, 1 + workload.nextInt(shuffled.size())); + } + /** DELETE with the first {@code keyColumnCount} primary key columns bound (partition or full row). */ private static String deleteStmt(TableMetadata metadata, int keyColumnCount) { StringBuilder sb = new StringBuilder("DELETE FROM ").append(metadata).append(" WHERE "); - List keys = ImmutableList.builder() - .addAll(metadata.partitionKeyColumns()) - .addAll(metadata.clusteringColumns()) - .build(); + List keys = primaryKeyColumns(metadata); for (int i = 0; i < keyColumnCount; i++) { if (i > 0) sb.append(" AND "); @@ -202,7 +412,7 @@ public class RandomDifferentialCompactionTest extends DifferentialCompactionTest private static final long addend = 0xBL; private static final long mask = (1L << 48) - 1; - private long seed = System.currentTimeMillis(); + private long seed = Long.getLong("cassandra.test.differential.seed", System.currentTimeMillis()); private final int examples; SeedRunner(int examples)