Harry Stress Tool and Offline Levelled SSTable Generator

.build/build-rat.xml

@@ -60,6 +60,7 @@
                  <exclude name="test/conf/sstableloader_with_encryption.yaml"/>
                  <exclude name="test/conf/unit-test-conf/test-native-port.yaml"/>
                  <exclude name="test/resources/data/config/YamlConfigurationLoaderTest/*.yaml"/>
+                 <exclude name="test/resources/harry/stress/default-stress-schema.yaml"/>
                  <exclude name="test/resources/nodetool/help/**"/>
                  <exclude name="tools/cqlstress-*.yaml"/>
                  <!-- test data -->

build.xml

@@ -1939,7 +1939,7 @@
       <fileset dir="${test.conf}"/>
     </copy>
     <unzip dest="${build.dir}/dtest" overwrite="false">
-      <fileset dir="${test.lib}/jars" includes="jimfs-1.1.jar,dtest-api-*.jar,asm-*.jar,javassist-*.jar,reflections-*.jar,semver4j-*.jar"/>
+      <fileset dir="${test.lib}/jars" includes="jimfs-1.1.jar,dtest-api-*.jar,asm-*.jar,javassist-*.jar,reflections-*.jar,semver4j-*.jar,cassandra-accord-*.jar"/>
       <patternset excludes="META-INF/license/**"/>
     </unzip>
     <unzip dest="${build.dir}/dtest" overwrite="false">

src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java

@@ -1097,6 +1097,17 @@ public class PartitionUpdate extends AbstractBTreePartition
             return metadata;
         }
 
+        /**
+         * Returns the number of non-static rows accumulated so far.
+         * Accurate when rows are added in clustering order; may overcount with unordered input.
+         */
+        public int rowCount()
+        {
+            if (rowBuilder != null)
+                return rowBuilder.count();
+            return firstRow != null ? 1 : 0;
+        }
+
         private static final UpdateFunction<Row, Row> ROWS_MERGE_FUNCTION = UpdateFunction.Simple.of(Rows::merge);
 
         public PartitionUpdate build()

src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.stream.Stream;
 
@@ -56,10 +57,12 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
     protected static final AtomicReference<SSTableId> id = new AtomicReference<>(SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get());
     protected boolean makeRangeAware = false;
     protected final Collection<Index.Group> indexGroups;
-    protected Consumer<Collection<SSTableReader>> sstableProducedListener;
+    protected BiConsumer<SSTableTxnWriter, Collection<SSTableReader>> sstableProducedListener;
     protected boolean openSSTableOnProduced = false;
     protected CompressionDictionary compressionDictionary;
     protected SSTable.Owner owner;
+    protected int sstableLevel = 0;
+    protected long reparedAtMillis = ActiveRepairService.UNREPAIRED_SSTABLE;
 
     protected AbstractSSTableSimpleWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns)
     {
@@ -69,11 +72,21 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
         indexGroups = new ArrayList<>();
     }
 
+    protected  void setReparedAtMillis(long reparedAtMillis)
+    {
+        this.reparedAtMillis = reparedAtMillis;
+    }
+
     protected void setSSTableFormatType(SSTableFormat<?, ?> type)
     {
         this.format = type;
     }
 
+    protected void setSSTableLevel(int level)
+    {
+        this.sstableLevel = level;
+    }
+
     protected void setRangeAwareWriting(boolean makeRangeAware)
     {
         this.makeRangeAware = makeRangeAware;
@@ -90,6 +103,12 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
     }
 
     protected void setSSTableProducedListener(Consumer<Collection<SSTableReader>> listener)
+    {
+        Objects.requireNonNull(listener, "sstableProducedListener cannot be null");
+        this.sstableProducedListener = (writer, readers) -> listener.accept(readers);
+    }
+
+    protected void setSSTableProducedListener(BiConsumer<SSTableTxnWriter, Collection<SSTableReader>> listener)
     {
         this.sstableProducedListener = Objects.requireNonNull(listener, "sstableProducedListener cannot be null");
     }
@@ -107,12 +126,12 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
         return openSSTableOnProduced;
     }
 
-    protected void notifySSTableProduced(Collection<SSTableReader> sstables)
+    protected void notifySSTableProduced(SSTableTxnWriter writer, Collection<SSTableReader> sstables)
     {
         if (sstableProducedListener == null)
             return;
 
-        sstableProducedListener.accept(sstables);
+        sstableProducedListener.accept(writer, sstables);
     }
 
     protected SSTableTxnWriter createWriter(SSTable.Owner owner) throws IOException
@@ -120,7 +139,7 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
         SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS);
 
         if (makeRangeAware)
-            return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, format, header);
+            return SSTableTxnWriter.createRangeAware(metadata, 0, reparedAtMillis, ActiveRepairService.NO_PENDING_REPAIR, false, format, header);
 
 
         SSTable.Owner effectiveOwner;
@@ -139,9 +158,10 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
         return SSTableTxnWriter.create(metadata,
                                        createDescriptor(directory, metadata.keyspace, metadata.name, format),
                                        0,
-                                       ActiveRepairService.UNREPAIRED_SSTABLE,
+                                       reparedAtMillis,
                                        ActiveRepairService.NO_PENDING_REPAIR,
                                        false,
+                                       sstableLevel,
                                        header,
                                        indexGroups,
                                        effectiveOwner);

src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java

@@ -54,13 +54,14 @@ import static org.apache.cassandra.utils.concurrent.BlockingQueues.newBlockingQu
  *
  * @see SSTableSimpleWriter
  */
-class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
+public class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
 {
     private static final Buffer SENTINEL = new Buffer();
 
     private Buffer buffer = new Buffer();
     private final long maxSStableSizeInBytes;
     private long currentSize;
+    private boolean syncPending;
 
     // Used to compute the row serialized size
     private final SerializationHeader header;
@@ -91,6 +92,18 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
         PartitionUpdate.Builder previous = buffer.get(key);
         if (previous == null)
         {
+            if (syncPending)
+            {
+                syncPending = false;
+                try
+                {
+                    sync();
+                }
+                catch (IOException e)
+                {
+                    throw new SyncException(e);
+                }
+            }
             // todo: inefficient - we create and serialize a PU just to get its size, then recreate it
             // todo: either allow PartitionUpdateBuilder to have .build() called several times or pre-calculate the size
             currentSize += PartitionUpdate.serializer.serializedSize(createPartitionUpdateBuilder(key).build(), format.getLatestVersion().correspondingMessagingVersion());
@@ -110,19 +123,10 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
         currentSize += UnfilteredSerializer.serializer.serializedSize(row, helper, 0, format.getLatestVersion().correspondingMessagingVersion());
     }
 
-    private void maybeSync() throws SyncException
+    private void maybeSync()
     {
-        try
-        {
-            if (currentSize > maxSStableSizeInBytes)
-                sync();
-        }
-        catch (IOException e)
-        {
-            // addColumn does not throw IOException but we want to report this to the user,
-            // so wrap it in a temporary RuntimeException that we'll catch in rawAddRow above.
-            throw new SyncException(e);
-        }
+        if (currentSize > maxSStableSizeInBytes)
+            syncPending = true;
     }
 
     private PartitionUpdate.Builder createPartitionUpdateBuilder(DecoratedKey key)
@@ -132,8 +136,11 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
             @Override
             public void add(Row row)
             {
+                int before = rowCount();
                 super.add(row);
-                countRow(row);
+
+                if (rowCount() > before)
+                    countRow(row);
                 maybeSync();
             }
         };
@@ -229,7 +236,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
                         for (Map.Entry<DecoratedKey, PartitionUpdate.Builder> entry : b.entrySet())
                             writer.append(entry.getValue().build().unfilteredIterator());
                         Collection<SSTableReader> finished = writer.finish(shouldOpenSSTables());
-                        notifySSTableProduced(finished);
+                        notifySSTableProduced(writer, finished);
                     }
                 }
                 catch (Throwable e)

src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java

@@ -44,7 +44,7 @@ import org.apache.cassandra.schema.TableMetadataRef;
  * The output will be a series of SSTables that do not exceed a specified size.
  * By default, all sorted data are written into a single SSTable.
  */
-class SSTableSimpleWriter extends AbstractSSTableSimpleWriter
+public class SSTableSimpleWriter extends AbstractSSTableSimpleWriter
 {
     private final long maxSSTableSizeInBytes;
 
@@ -161,7 +161,7 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter
                 return;
 
             Collection<SSTableReader> finished = writer.finish(shouldOpenSSTables());
-            notifySSTableProduced(finished);
+            notifySSTableProduced(writer, finished);
         }
         catch (Throwable t)
         {

src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java

@@ -151,10 +151,25 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
                                           SerializationHeader header,
                                           Collection<Index.Group> indexGroups,
                                           SSTable.Owner owner)
+    {
+        return create(metadata, descriptor, keyCount, repairedAt, pendingRepair, isTransient, 0, header, indexGroups, owner);
+    }
+
+    @SuppressWarnings({"resource", "RedundantSuppression"}) // log and writer closed during doPostCleanup
+    public static SSTableTxnWriter create(TableMetadataRef metadata,
+                                          Descriptor descriptor,
+                                          long keyCount,
+                                          long repairedAt,
+                                          TimeUUID pendingRepair,
+                                          boolean isTransient,
+                                          int sstableLevel,
+                                          SerializationHeader header,
+                                          Collection<Index.Group> indexGroups,
+                                          SSTable.Owner owner)
     {
         // if the column family store does not exist, we create a new default SSTableMultiWriter to use:
         LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
-        SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, null, 0, header, indexGroups, txn, owner);
+        SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, null, sstableLevel, header, indexGroups, txn, owner);
         return new SSTableTxnWriter(txn, writer);
     }
 }

src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java

@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
+import com.google.common.base.Preconditions;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -62,6 +63,9 @@ public class DataPlacement
             this.writes = reads; // performance optimization for a typical case, to not search for endpoints twice
         else
             this.writes = writes;
+
+        Preconditions.checkArgument(reads.ranges.equals(writes.ranges),
+                                    "Read ranges (%s) are not the same as write ranges (%s)", reads.ranges, writes.ranges);
     }
 
     /**

src/java/org/apache/cassandra/utils/btree/BTree.java

@@ -1611,6 +1611,11 @@ public class BTree
             return new Builder<>(this);
         }
 
+        public int count()
+        {
+            return count;
+        }
+
         public Builder<V> setQuickResolver(QuickResolver<V> quickResolver)
         {
             this.quickResolver = quickResolver;

test/distributed/org/apache/cassandra/distributed/test/accord/AccordHostReplacementTest.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.distributed.shared.ClusterUtils;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
 import org.apache.cassandra.harry.execution.RingAwareInJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.gen.Generator;
@@ -43,6 +44,7 @@ import org.apache.cassandra.service.consensus.TransactionalMode;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce;
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.*;
 
 public class AccordHostReplacementTest extends TestBaseImpl
 {
@@ -72,9 +74,10 @@ public class AccordHostReplacementTest extends TestBaseImpl
                         SchemaSpec.optionsBuilder().withTransactionalMode(transactionalModeGen.generate(rng))
                                 .withSpeculativeRetry("ALWAYS"));
                 SchemaSpec schema = schemaGen.generate(rng);
-                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), 1000)));
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), 1000);
+                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(Generators.int64(0, Math.min(valueGenerators.pkGen().population(), 1000))));
 
-                HistoryBuilder history = historyBuilder(schema, cluster);
+                HistoryBuilder history = historyBuilder(schema, valueGenerators, cluster);
                 waitForCMSToQuiesce(cluster, cluster.get(1));
 
                 for (int i = 0; i < 1000; i++)
@@ -93,13 +96,13 @@ public class AccordHostReplacementTest extends TestBaseImpl
         }
     }
 
-    private static HistoryBuilder historyBuilder(SchemaSpec schema, Cluster cluster)
+    private static HistoryBuilder historyBuilder(SchemaSpec schema, IndexedValueGenerators valueGenerators, Cluster cluster)
     {
-        HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+        HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                 hb -> RingAwareInJvmDTestVisitExecutor.builder()
                         .replicationFactor(new TokenPlacementModel.SimpleReplicationFactor(3))
                         .consistencyLevel(ConsistencyLevel.ALL)
-                        .build(schema, hb, cluster));
+                        .build(schema, hb.valueGenerators(), cluster));
         history.customThrowing(() -> cluster.schemaChange(schema.compile()), "Setup");
         return history;
     }

test/distributed/org/apache/cassandra/distributed/test/log/AlterTopologyTest.java

@@ -34,6 +34,7 @@ import org.apache.cassandra.distributed.shared.ClusterUtils;
 import org.apache.cassandra.exceptions.ExceptionCode;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
 import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.gen.Generator;
@@ -54,6 +55,7 @@ import static java.time.Duration.ofSeconds;
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce;
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 import static org.junit.Assert.assertEquals;
 
 public class AlterTopologyTest extends FuzzTestBase
@@ -75,13 +77,13 @@ public class AlterTopologyTest extends FuzzTestBase
 
             withRandom(rng -> {
                 SchemaSpec schema = schemaGen.generate(rng);
-                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), 1000)));
-                Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), 1000);
+                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(Generators.int64(0, Math.min(valueGenerators.pkGen().population(), 1000))));
 
-                HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+                HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                                                                      (hb) -> InJvmDTestVisitExecutor.builder()
                                                                                                     .nodeSelector(i -> 1)
-                                                                                                    .build(schema, hb, cluster));
+                                                                                                    .build(schema, valueGenerators, cluster));
                 history.custom(() -> {
                     cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE +
                                          " WITH replication = {'class': 'NetworkTopologyStrategy', 'dc1' : 3 };");
@@ -92,7 +94,10 @@ public class AlterTopologyTest extends FuzzTestBase
 
                 Runnable writeAndValidate = () -> {
                     for (int i = 0; i < 2000; i++)
-                        history.insert(pkGen.generate(rng), ckGen.generate(rng));
+                    {
+                        int pdIdx = pkGen.generate(rng);
+                        history.insert(pdIdx, valueGenerators.forPdIdx(pdIdx).ckIdxGen().generate(rng));
+                    }
 
                     for (int pk : pkGen.generated())
                         history.selectPartition(pk);

test/distributed/org/apache/cassandra/distributed/test/log/BounceGossipTest.java

@@ -40,7 +40,7 @@ import org.apache.cassandra.gms.EndpointState;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
-import org.apache.cassandra.harry.dsl.HistoryBuilderHelper;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.execution.CQLVisitExecutor;
 import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.gen.Generator;
@@ -53,6 +53,7 @@ import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFac
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 import static org.junit.Assert.fail;
 import static org.psjava.util.AssertStatus.assertTrue;
 
@@ -75,8 +76,9 @@ public class BounceGossipTest extends TestBaseImpl
 
             withRandom(rng -> {
                 SchemaSpec schema = schemaGen.generate(rng);
-                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), 1000)));
-                Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), 1000);
+
+                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(valueGenerators.pkIdxGen()));
 
                 cluster.schemaChange("CREATE KEYSPACE " + schema.keyspace +
                                      " WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};");
@@ -84,12 +86,12 @@ public class BounceGossipTest extends TestBaseImpl
 
                 Future<?> f = es.submit(new Runnable()
                 {
-                    final HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
+                    final HistoryBuilder history = new HistoryBuilder(valueGenerators);
                     final Iterator<Visit> iterator = history.iterator();
                     final CQLVisitExecutor executor = InJvmDTestVisitExecutor.builder()
                                                                              .nodeSelector(lts -> 1)
                                                                              .retryPolicy(InJvmDTestVisitExecutor.RetryPolicy.RETRY_ON_TIMEOUT)
-                                                                             .build(schema, history, cluster);
+                                                                             .build(schema, history.valueGenerators(), cluster);
                     @Override
                     public void run()
                     {
@@ -97,7 +99,7 @@ public class BounceGossipTest extends TestBaseImpl
                         {
                             // Rate limit to ~10 per second
                             LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(500));
-                            HistoryBuilderHelper.insertRandomData(schema, pkGen, ckGen, rng, history);
+                            history.insert();
                             history.selectPartition(pkGen.generate(rng));
 
                             while (iterator.hasNext() && !stop.get())

test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java

@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.distributed.test.log;
 
-
 import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.Random;
@@ -38,11 +37,13 @@ import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.ValueGeneratorHelper;
 import org.apache.cassandra.harry.cql.WriteHelper;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.execution.CompiledStatement;
 import org.apache.cassandra.harry.gen.Generator;
 import org.apache.cassandra.harry.gen.SchemaGenerators;
 import org.apache.cassandra.harry.model.TokenPlacementModel;
 import org.apache.cassandra.harry.model.TokenPlacementModel.Replica;
+import org.apache.cassandra.harry.op.Kind;
 import org.apache.cassandra.harry.op.Operations;
 import org.apache.cassandra.harry.util.ByteUtils;
 import org.apache.cassandra.harry.util.TokenUtil;
@@ -77,12 +78,12 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase
 
             withRandom(rng -> {
                 SchemaSpec schema = schemaGen.generate(rng);
+                IndexedValueGenerators valueGenerators = HistoryBuilder.valueGenerators(schema, rng.next(), 1000);
                 cluster.schemaChange("CREATE KEYSPACE " + schema.keyspace +
                                      " WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};");
                 cluster.schemaChange(schema.compile());
 
-                HistoryBuilder.IndexedValueGenerators valueGenerators = (HistoryBuilder.IndexedValueGenerators) schema.valueGenerators;
-                for (int i = 0; i < valueGenerators.pkPopulation(); i++)
+                for (int i = 0; i < valueGenerators.pkGen().population(); i++)
                 {
                     long pd = valueGenerators.pkGen().descriptorAt(i);
 
@@ -102,12 +103,12 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase
 
                     long lts = 1L;
                     Future<?> writeQuery = async(() -> {
-
                         CompiledStatement s = WriteHelper.inflateInsert(new Operations.WriteOp(lts, pd, 0,
-                                                                                               ValueGeneratorHelper.randomDescriptors(rng, valueGenerators::regularColumnGen, valueGenerators.regularColumnCount()),
-                                                                                               ValueGeneratorHelper.randomDescriptors(rng, valueGenerators::staticColumnGen, valueGenerators.staticColumnCount()),
-                                                                                               Operations.Kind.INSERT),
+                                                                                               ValueGeneratorHelper.randomDescriptors(rng, valueGenerators.forPd(pd)::regularColumnGen, valueGenerators.forPd(pd).regularColumnCount()),
+                                                                                               ValueGeneratorHelper.randomDescriptors(rng, valueGenerators.forPd(pd)::staticColumnGen, valueGenerators.forPd(pd).staticColumnCount()),
+                                                                                               Kind.INSERT),
                                                                         schema,
+                                                                        valueGenerators,
                                                                         lts);
                         cluster.coordinator(1).execute(s.cql(), ConsistencyLevel.QUORUM, s.bindings());
                         return null;

test/distributed/org/apache/cassandra/distributed/test/log/FailedLeaveTest.java

@@ -41,6 +41,7 @@ import org.apache.cassandra.distributed.shared.ClusterUtils;
 import org.apache.cassandra.distributed.shared.ClusterUtils.SerializableBiPredicate;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
 import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.gen.Generator;
@@ -60,6 +61,7 @@ import static org.apache.cassandra.distributed.shared.ClusterUtils.decommission;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.getClusterMetadataVersion;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.getSequenceAfterCommit;
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 
 public class FailedLeaveTest extends FuzzTestBase
 {
@@ -98,13 +100,13 @@ public class FailedLeaveTest extends FuzzTestBase
 
             withRandom(rng -> {
                 SchemaSpec schema = schemaGen.generate(rng);
-                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), 1000)));
-                Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), 1000);
+                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(Generators.int64(0, Math.min(valueGenerators.pkGen().population(), 1000))));
 
-                HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+                HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                                                                      (hb) -> InJvmDTestVisitExecutor.builder()
                                                                                                     .nodeSelector(i -> 1)
-                                                                                                    .build(schema, hb, cluster));
+                                                                                                    .build(schema, hb.valueGenerators(), cluster));
                 history.custom(() -> {
                     cluster.schemaChange("CREATE KEYSPACE " + schema.keyspace +
                                          " WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 2};");
@@ -113,7 +115,10 @@ public class FailedLeaveTest extends FuzzTestBase
 
                 Runnable writeAndValidate = () -> {
                     for (int i = 0; i < WRITES; i++)
-                        history.insert(pkGen.generate(rng), ckGen.generate(rng));
+                    {
+                        int pkIdx = pkGen.generate(rng);
+                        history.insert(pkIdx, valueGenerators.forPdIdx(pkIdx).ckIdxGen().generate(rng));
+                    }
 
                     for (int pk : pkGen.generated())
                         history.selectPartition(pk);

test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java

@@ -38,6 +38,7 @@ import org.apache.cassandra.distributed.shared.ClusterUtils;
 import org.apache.cassandra.distributed.shared.NetworkTopology;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.execution.DataTracker;
 import org.apache.cassandra.harry.execution.RingAwareInJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.gen.Generator;
@@ -79,10 +80,10 @@ public class ResumableStartupTest extends FuzzTestBase
 
             withRandom(rng -> {
                 SchemaSpec schema = schemaGen.generate(rng);
-                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), 1000)));
-                Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
+                IndexedValueGenerators valueGenerators = HistoryBuilder.valueGenerators(schema, rng.next(), 1000);
+                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(Generators.int64(0, Math.min(valueGenerators.pkGen().population(), 1000))));
 
-                HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
+                HistoryBuilder history = new HistoryBuilder(valueGenerators);
                 history.customThrowing(() -> {
                     cluster.schemaChange(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 2};", schema.keyspace));
                     cluster.schemaChange(schema.compile());
@@ -91,7 +92,10 @@ public class ResumableStartupTest extends FuzzTestBase
 
                 Runnable writeAndValidate = () -> {
                     for (int i = 0; i < WRITES; i++)
-                        history.insert(pkGen.generate(rng), ckGen.generate(rng));
+                    {
+                        int pdIdx = pkGen.generate(rng);
+                        history.insert(pdIdx, history.valueGenerators().forPdIdx(pdIdx).ckIdxGen().generate(rng));
+                    }
 
                     for (int pk : pkGen.generated())
                         history.selectPartition(pk);
@@ -100,15 +104,15 @@ public class ResumableStartupTest extends FuzzTestBase
 
                 // First write with ONE, as we only have 1 node
                 TokenPlacementModel.ReplicationFactor rf = new TokenPlacementModel.SimpleReplicationFactor(1);
-                DataTracker tracker = new DataTracker.SequentialDataTracker();
-                QuiescentChecker checker = new QuiescentChecker(schema.valueGenerators, tracker, history);
+                DataTracker.SequentialDataTracker tracker = new DataTracker.SequentialDataTracker();
+                QuiescentChecker checker = new QuiescentChecker(valueGenerators, tracker);
 
                 RingAwareInJvmDTestVisitExecutor executor;
                 // RF is ONE here since we have no pending nodes
                 executor = RingAwareInJvmDTestVisitExecutor.builder()
                                                            .replicationFactor(rf)
                                                            .consistencyLevel(ConsistencyLevel.ONE)
-                                                           .build(schema, tracker, checker, cluster);
+                                                           .build(schema, valueGenerators, tracker, checker, cluster);
                 Iterator<Visit> iterator = history.iterator();
                 while (iterator.hasNext())
                     executor.execute(iterator.next());
@@ -130,7 +134,7 @@ public class ResumableStartupTest extends FuzzTestBase
                 executor = RingAwareInJvmDTestVisitExecutor.builder()
                                                            .replicationFactor(rf)
                                                            .consistencyLevel(ConsistencyLevel.ONE)
-                                                           .build(schema, tracker, checker, cluster);
+                                                           .build(schema, valueGenerators, tracker, checker, cluster);
                 while (iterator.hasNext())
                     executor.execute(iterator.next());
 

test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeHarryTest.java

@@ -89,9 +89,7 @@ public class ClusterMetadataUpgradeHarryTest extends UpgradeTestBase
             .upgradesToCurrentFrom(v41)
             .withUpgradeListener(listener)
             .setup((cluster) -> {
-                SchemaSpec schema = new SchemaSpec(rng.next(),
-                                                   10_000,
-                                                   "harry", "test_table",
+                SchemaSpec schema = new SchemaSpec("harry", "test_table",
                                                    asList(pk("pk1", asciiType), pk("pk2", int64Type)),
                                                    asList(ck("ck1", asciiType, false), ck("ck2", int64Type, false)),
                                                    asList(regularColumn("regular1", asciiType), regularColumn("regular2", int64Type)),
@@ -99,7 +97,7 @@ public class ClusterMetadataUpgradeHarryTest extends UpgradeTestBase
                 cluster.schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d};", schema.keyspace, 3));
                 cluster.schemaChange(schema.compile());
 
-                HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+                HistoryBuilder history = new ReplayingHistoryBuilder(HistoryBuilder.valueGenerators(schema, rng.next()),
                                                                      hb -> InJvmDTestVisitExecutor.builder()
                                                                                                   .retryPolicy(retry -> true)
                                                                                                   .nodeSelector(lts -> {
@@ -112,9 +110,9 @@ public class ClusterMetadataUpgradeHarryTest extends UpgradeTestBase
                                                                                                       }
                                                                                                   })
                                                                                                   .consistencyLevel(ConsistencyLevel.QUORUM)
-                                                                                                  .build(schema, hb, cluster));
+                                                                                                  .build(schema, hb.valueGenerators(), cluster));
 
-                Generator<Integer> pkIdxGen = Generators.int32(0, Math.min(10_000, schema.valueGenerators.ckPopulation()));
+                Generator<Integer> pkIdxGen = Generators.adaptLongToInt(Generators.int64(0, Math.min(10_000, history.valueGenerators().pkGen().population())));
 
                 executor.set(executorFactory().infiniteLoop("R/W Worload",
                                                             () -> {

test/distributed/org/apache/cassandra/fuzz/harry/examples/RangeTombstoneBurnTest.java

@@ -33,10 +33,14 @@ import org.apache.cassandra.harry.gen.SchemaGenerators;
 
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
 
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
+
 public class RangeTombstoneBurnTest extends IntegrationTestBase
 {
     private final int ITERATIONS = 10;
     private final int STEPS_PER_ITERATION = 1000;
+    private final int POPULATION = 1000;
 
     @Test
     public void rangeTombstoneBurnTest()
@@ -47,8 +51,9 @@ public class RangeTombstoneBurnTest extends IntegrationTestBase
             cluster.get(1).nodetool("disableautocompaction");
             cluster.schemaChange(schema.compile());
 
-            int perIteration = Math.min(10, schema.valueGenerators.pkPopulation());;
-            int maxPartitions = Math.max(perIteration, schema.valueGenerators.pkPopulation());
+            IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), POPULATION);
+            int perIteration = 10;
+            int maxPartitions = Math.toIntExact(Math.max(perIteration, valueGenerators.pkGen().population()));
 
             for (int iteration = 0; iteration < ITERATIONS; iteration++)
             {
@@ -61,13 +66,13 @@ public class RangeTombstoneBurnTest extends IntegrationTestBase
                 float deleteColumnsChance = rng.nextFloat(0.95f, 1.0f);
                 float deleteRangeChance = rng.nextFloat(0.95f, 1.0f);
                 float flushChance = rng.nextFloat(0.999f, 1.0f);
-                int maxPartitionSize = Math.min(rng.nextInt(1, 1 << rng.nextInt(5, 11)), schema.valueGenerators.ckPopulation());
+                int maxPartitionSize = Math.min(rng.nextInt(1, 1 << rng.nextInt(5, 11)), POPULATION);
 
                 Generator<Integer> partitionPicker = Generators.pick(partitions);
                 Generator<Integer> rowPicker = Generators.int32(0, maxPartitionSize);
                 ModelChecker<SingleOperationBuilder, Void> model = new ModelChecker<>();
-                ReplayingHistoryBuilder historyBuilder = new ReplayingHistoryBuilder(schema.valueGenerators,
-                                                                                     (hb) -> InJvmDTestVisitExecutor.builder().build(schema, hb, cluster));
+                ReplayingHistoryBuilder historyBuilder = new ReplayingHistoryBuilder(valueGenerators,
+                                                                                     (hb) -> InJvmDTestVisitExecutor.builder().build(schema, hb.valueGenerators(), cluster));
 
                 model.init(historyBuilder)
                      .step((history, rng_) -> {

test/distributed/org/apache/cassandra/fuzz/harry/examples/RepairBurnTest.java

@@ -26,7 +26,7 @@ import org.apache.cassandra.distributed.test.IntegrationTestBase;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.checker.ModelChecker;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
-import org.apache.cassandra.harry.dsl.HistoryBuilderHelper;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.gen.Generator;
 import org.apache.cassandra.harry.gen.Generators;
@@ -47,19 +47,19 @@ public class RepairBurnTest extends IntegrationTestBase
     public void repairBurnTest()
     {
         int maxPartitionSize = 10;
-        int partitions = 1000;
         Generator<SchemaSpec> schemaGen = SchemaGenerators.schemaSpecGen(KEYSPACE, "repair_burn", 1000);
 
         withRandom(rng -> {
             SchemaSpec schema = schemaGen.generate(rng);
+            IndexedValueGenerators valueGenerators = HistoryBuilder.valueGenerators(schema, rng.next(), 1000);
 
-            Generators.TrackingGenerator<Integer > pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), partitions)));
-            Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), maxPartitionSize));
+            Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(valueGenerators.pkIdxGen()));
+            Generator<Integer> ckGen = Generators.int32(0, maxPartitionSize);
 
             ModelChecker<HistoryBuilder, Void> modelChecker = new ModelChecker<>();
 
-            modelChecker.init(new HistoryBuilder(schema.valueGenerators))
-                        .step((history, rng_) -> HistoryBuilderHelper.insertRandomData(schema, pkGen, ckGen, rng, history))
+            modelChecker.init(new HistoryBuilder(valueGenerators))
+                        .step((history, rng_) -> history.insert(pkGen.generate(rng)))
                         .step((history, rng_) -> history.deleteRow(pkGen.generate(rng), ckGen.generate(rng)))
                         .exitCondition((history) -> {
                             if (history.size() < 10_000)
@@ -73,7 +73,7 @@ public class RepairBurnTest extends IntegrationTestBase
 
                             cluster.schemaChange(schema.compile());
 
-                            InJvmDTestVisitExecutor.replay(InJvmDTestVisitExecutor.builder().build(schema, history, cluster),
+                            InJvmDTestVisitExecutor.replay(InJvmDTestVisitExecutor.builder().build(schema, history.valueGenerators(), cluster),
                                                            history);
 
                             return true;

test/distributed/org/apache/cassandra/fuzz/harry/sstable/LevelledSSTableGeneratorTest.java

@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fuzz.harry.sstable;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
+import org.apache.cassandra.harry.execution.QueryBuildingVisitExecutor;
+import org.apache.cassandra.harry.gen.Generators;
+import org.apache.cassandra.harry.model.QuiescentChecker;
+import org.apache.cassandra.harry.op.ClusteringOrderBy;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.stress.ActivePartition;
+import org.apache.cassandra.harry.stress.LevelledSStableGenerator;
+import org.apache.cassandra.harry.stress.RotationStrategy;
+import org.apache.cassandra.harry.stress.config.StressSchemaConfig;
+import org.apache.cassandra.harry.stress.TokenIndex;
+import org.apache.cassandra.harry.stress.TokenIndexGenerator;
+import org.apache.cassandra.harry.stress.VisitGenerator;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+import org.apache.cassandra.harry.stress.distribution.Distributions;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+
+public class LevelledSSTableGeneratorTest extends TestBaseImpl
+{
+    private static final Logger logger = LoggerFactory.getLogger(LevelledSSTableGeneratorTest.class);
+
+    private static final String SCHEMA_CONFIG = "test/resources/harry/stress/levelled-sstable-schema.yaml";
+
+    private static final long INITIAL_LTS = 1;
+    private static final long VISITS = 100_000;                // total writes; with visitSize=1, one op per LTS
+    private static final long END_LTS = INITIAL_LTS + VISITS; // first LTS past the written history; reads use it
+    private static final int SSTABLE_SIZE_MIB = 1;
+    private static final int[] LEVEL_WEIGHTS = { 1, 2, 4, 8, 16 }; // index == LCS level; weights spread writes across levels
+
+    // Initialize static state for offline tool usage, since we are generating SSTables on the main class loader rather
+    // than in-jvm dtest nodes
+    public static void initForOfflineTool()
+    {
+        DatabaseDescriptor.toolInitialization(false);
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        ClusterMetadataService.initializeForClients();
+    }
+
+    @Test
+    public void generateAndValidateLevelledSSTables() throws Throwable
+    {
+        initForOfflineTool();
+
+        StressSchemaConfig config = StressSchemaConfig.load(Paths.get(SCHEMA_CONFIG));
+        SchemaSpec schema = config.schema();
+
+        Distribution visitSize = Distributions.fixed(1);
+        VisitGenerator.OpKindGenFactory opKindGen = new VisitGenerator.RandomOpKindGenFactory();
+
+        try (Cluster cluster = init(Cluster.build(1).start(), 1))
+        {
+            cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + schema.keyspace +
+                                 " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+            cluster.schemaChange(schema.compile());
+            cluster.get(1).nodetool("disableautocompaction", schema.keyspace, schema.table);
+
+
+            // Build TokenIndex
+            File tokenDir = new File(Files.createTempDirectory("harry-token-index"));
+            TokenIndexGenerator.generate(tokenDir.toJavaIOFile(), schema, config.rotationStrategy(),
+                                         config.rowPopulation(), Generators.constant(VisitGenerator.VisitType.MUTATE),
+                                         visitSize, INITIAL_LTS, VISITS);
+            TokenIndex tokenIndex = new TokenIndex(new File(tokenDir, "merged_tokens"),
+                                                   new File(tokenDir, "merged_tokens.idx"));
+
+            // Generate levelled SSTables
+            File sstableDir = new File(Files.createDirectories(Files.createTempDirectory("harry-levelled-sstables")
+                                                                    .resolve(schema.keyspace)
+                                                                    .resolve(schema.table + '-' + "0".repeat(32))));
+            LevelledSStableGenerator generator =
+            new LevelledSStableGenerator(schema, config.rowPopulation(), config.columnPopulation(), visitSize, opKindGen,
+                                         false, SSTABLE_SIZE_MIB,
+                                         new LevelledSStableGenerator.SSTableLevelPicker(LEVEL_WEIGHTS),
+                                         tokenIndex, sstableDir);
+            generator.generate(Long.MIN_VALUE, Long.MAX_VALUE);
+
+            // Import SSTables; -l to keeps generated levels
+            cluster.get(1).nodetoolResult("import", "-l", schema.keyspace, schema.table, sstableDir.absolutePath())
+                   .asserts().success();
+
+            TestOracle model = replay(schema, config, tokenIndex, visitSize, opKindGen);
+            tokenIndex.close();
+
+            // Validate
+            InJvmDTestVisitExecutor validator = new InJvmDTestVisitExecutor(schema, model.partitions, new DataTracker.NoOpDataTracker(),
+                                                                            new QuiescentChecker(model.partitions, model.tracker), cluster,
+                                                                            lts -> 1,
+                                                                            v -> InJvmDTestVisitExecutor.PageSizeSelector.NO_PAGING,
+                                                                            InJvmDTestVisitExecutor.RetryPolicy.NO_RETRY,
+                                                                            v -> ConsistencyLevel.NODE_LOCAL,
+                                                                            QueryBuildingVisitExecutor.WrapQueries.EMPTY);
+            for (long pd : model.generatedPds)
+                validator.execute(new Visit(END_LTS, new Operations.Operation[]{ new Operations.SelectPartition(END_LTS, pd, ClusteringOrderBy.ASC) }));
+            logger.info("Validated {} partitions", model.generatedPds.size());
+        }
+    }
+
+    private static TestOracle replay(SchemaSpec schema, StressSchemaConfig config, TokenIndex tokenIndex,
+                                     Distribution visitSize, VisitGenerator.OpKindGenFactory opKindGen)
+    {
+        RotationStrategy rotation = config.rotationStrategy();
+        ActivePartition.Partitions partitions = new ActivePartition.Partitions(schema, config.rowPopulation(), config.columnPopulation(),
+                                                                               rotation, 0, rotation.targetSize(), INITIAL_LTS);
+        partitions.populate();
+        DataTracker.SimpleDataTracker modelTracker = new DataTracker.SimpleDataTracker();
+        Set<Long> generatedPds = new TreeSet<>();
+        TokenIndex.EntryIterator iter = tokenIndex.range(Long.MIN_VALUE, Long.MAX_VALUE);
+        while (iter.hasNext())
+        {
+            long pd = iter.pd();
+            generatedPds.add(pd);
+            for (long lts : iter.readLts())
+            {
+                Visit visit = VisitGenerator.mutatingVisit(lts, visitSize, opKindGen, rng -> partitions.forPd(pd));
+                modelTracker.begin(visit);
+                modelTracker.end(visit);
+            }
+            iter.advance();
+        }
+        return new TestOracle(partitions, modelTracker, generatedPds);
+    }
+
+    private static final class TestOracle
+    {
+        final ActivePartition.Partitions partitions;
+        final DataTracker.SimpleDataTracker tracker;
+        final Set<Long> generatedPds;
+
+        TestOracle(ActivePartition.Partitions partitions, DataTracker.SimpleDataTracker tracker, Set<Long> generatedPds)
+        {
+            this.partitions = partitions;
+            this.tracker = tracker;
+            this.generatedPds = generatedPds;
+        }
+    }
+}

test/distributed/org/apache/cassandra/fuzz/harry/sstable/RangeAwareSSTableLoadAndStressTest.java

@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fuzz.harry.sstable;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.Generators;
+import org.apache.cassandra.harry.stress.HarryStress;
+import org.apache.cassandra.harry.stress.LevelledSStableGenerator;
+import org.apache.cassandra.harry.stress.RotationStrategy;
+import org.apache.cassandra.harry.stress.config.StressSchemaConfig;
+import org.apache.cassandra.harry.stress.TokenIndex;
+import org.apache.cassandra.harry.stress.TokenIndexGenerator;
+import org.apache.cassandra.harry.stress.VisitGenerator;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+import org.apache.cassandra.harry.stress.distribution.Distributions;
+import org.apache.cassandra.io.util.File;
+
+public class RangeAwareSSTableLoadAndStressTest extends TestBaseImpl
+{
+    private static final Logger logger = LoggerFactory.getLogger(RangeAwareSSTableLoadAndStressTest.class);
+
+    private static final String SCHEMA_CONFIG = "test/resources/harry/stress/levelled-sstable-schema.yaml";
+
+    private static final int NUM_NODES = 4;
+    private static final int RF = 3;
+
+    private static final long INITIAL_LTS = 1;
+    private static final long VISITS = 100_000;
+    private static final long END_LTS = INITIAL_LTS + VISITS;
+    private static final long STRESS_VISITS = 16_000;
+    private static final int CONCURRENCY = 2;
+    private static final int RATE_PER_SECOND = 20_000;
+    private static final int SSTABLE_SIZE_MIB = 1;
+    private static final int[] LEVEL_WEIGHTS = { 1, 2, 4, 8, 16 };
+
+    @Test
+    public void rangeAwareLoadAndStress() throws Throwable
+    {
+        LevelledSSTableGeneratorTest.initForOfflineTool();
+
+        StressSchemaConfig config = StressSchemaConfig.load(Paths.get(SCHEMA_CONFIG));
+        SchemaSpec schema = config.schema();
+        RotationStrategy rotation = config.rotationStrategy();
+
+        Distribution visitSize = Distributions.fixed(1);
+        VisitGenerator.OpKindGenFactory opKindGen = new VisitGenerator.RandomOpKindGenFactory();
+        Generator<VisitGenerator.VisitType> visitTypeGen = Generators.constant(VisitGenerator.VisitType.MUTATE);
+
+        try (Cluster cluster = init(Cluster.build(NUM_NODES)
+                                           .withConfig(c -> c.set("num_tokens", 1))
+                                           .start(), RF))
+        {
+            cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + schema.keyspace +
+                                 " WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter0': " + RF + "}");
+            cluster.schemaChange(schema.compile());
+            cluster.forEach(n -> n.nodetool("disableautocompaction", schema.keyspace, schema.table));
+
+            List<RangeReplicas> ranges = getTopology(cluster, schema);
+            for (RangeReplicas r : ranges)
+                System.out.println(String.format("  range (%d, %d] -> nodes %s", r.startToken, r.endToken, java.util.Arrays.toString(r.replicas)));
+            logger.info("data_placements returned {} ranges", ranges.size());
+
+            File tokenDir = new File(Files.createTempDirectory("harry-token-index"));
+            TokenIndexGenerator.generate(tokenDir.toJavaIOFile(), schema, rotation,
+                                         config.rowPopulation(), visitTypeGen,
+                                         visitSize, INITIAL_LTS, VISITS);
+            TokenIndex tokenIndex = new TokenIndex(new File(tokenDir, "merged_tokens"),
+                                                   new File(tokenDir, "merged_tokens.idx"));
+
+            // Ror each range, generate levelled SSTables and import into replicas
+            for (RangeReplicas range : ranges)
+            {
+                File sstableDir = new File(Files.createDirectories(Files.createTempDirectory("range-aware-sstables")
+                                                                        .resolve(String.format("%d-%d", range.startToken, range.endToken))
+                                                                        .resolve(schema.keyspace)
+                                                                        .resolve(schema.table + '-' + "0".repeat(32))));
+                LevelledSStableGenerator generator = new LevelledSStableGenerator(schema, config.rowPopulation(), config.columnPopulation(), visitSize, opKindGen,
+                                                                                  false, SSTABLE_SIZE_MIB,
+                                                                                  new LevelledSStableGenerator.SSTableLevelPicker(LEVEL_WEIGHTS),
+                                                                                  tokenIndex, sstableDir);
+                // Handle wraparound ranges
+                if (range.startToken <= range.endToken)
+                {
+                    generator.generate(range.startToken, range.endToken);
+                }
+                else
+                {
+                    generator.generate(Long.MIN_VALUE, range.endToken);
+                    generator.generate(range.startToken, Long.MAX_VALUE);
+                }
+
+                for (int node : range.replicas)
+                    cluster.get(node).nodetoolResult("import", "-cd", "-t", schema.keyspace, schema.table, sstableDir.absolutePath())
+                                     .asserts().success();
+            }
+            logger.info("Generated and imported SSTables for all ranges");
+            tokenIndex.close();
+
+            // Continue HarryStress live from where SSTableGenerator has finished
+            HarryStress stress = new HarryStress(schema,
+                                                 config.rowPopulation(),
+                                                 config.columnPopulation(),
+                                                 visitTypeGen,
+                                                 visitSize,
+                                                 opKindGen,
+                                                 config.rotationStrategy(),
+                                                 /*metricsOut=*/ null,
+                                                 /*reportIntervalSeconds=*/ 30,
+                                                 () -> (statement, run) -> {
+                                                     Object[][] rs = cluster.coordinator(1).execute(statement.cql(),
+                                                                                                    ConsistencyLevel.QUORUM,
+                                                                                                    statement.bindings());
+                                                     System.out.println("rs = " + Arrays.toString(rs));
+                                                     if (run != null)
+                                                         run.run();
+                                                     return rs;
+                                                 },
+                                                 CONCURRENCY,
+                                                 RATE_PER_SECOND,
+                                                 /*minPartitionIdx=*/ 0,
+                                                 /*maxPartitionIdx=*/ Long.MAX_VALUE,
+                                                 /*initialLts=*/ END_LTS);
+
+            stress.replay(INITIAL_LTS, END_LTS);
+            stress.start(END_LTS + STRESS_VISITS, Long.MAX_VALUE);
+            logger.info("HarryStress completed an additional {} visits past the loaded SSTable history", STRESS_VISITS);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static List<RangeReplicas> getTopology(Cluster cluster, SchemaSpec schema)
+    {
+        Object[][] rows = cluster.coordinator(1).execute(
+            "SELECT range_start, range_end, write_replicas FROM system_views.data_placements " +
+            "WHERE keyspace_name = ? AND table_name = ?",
+            ConsistencyLevel.ONE, schema.keyspace, schema.table);
+        List<RangeReplicas> result = new ArrayList<>();
+
+        for (Object[] row : rows)
+        {
+            long startToken = Long.parseLong((String) row[0]);
+            long endToken = Long.parseLong((String) row[1]);
+            int[] replicas = toIdx((Set<Integer>) row[2]);
+            result.add(new RangeReplicas(startToken, endToken, replicas));
+        }
+        return result;
+    }
+
+    private static int[] toIdx(Set<Integer> nodeIds)
+    {
+        // In dtest clusters, TCM node IDs are assigned 1..N matching the instance indices
+        int[] idxs = new int[nodeIds.size()];
+        int i = 0;
+        for (int nodeId : nodeIds)
+            idxs[i++] = nodeId;
+        return idxs;
+    }
+
+    // A token range and its write-replica set
+    private static final class RangeReplicas
+    {
+        final long startToken;
+        final long endToken;
+        final int[] replicas;
+
+        RangeReplicas(long startToken, long endToken, int[] replicas)
+        {
+            this.startToken = startToken;
+            this.endToken = endToken;
+            this.replicas = replicas;
+        }
+    }
+}

test/distributed/org/apache/cassandra/fuzz/harry/stress/HarryStressValidationSmokeTest.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fuzz.harry.stress;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.gen.Generators;
+import org.apache.cassandra.harry.stress.HarryStress;
+import org.apache.cassandra.harry.stress.RotationStrategy;
+import org.apache.cassandra.harry.stress.VisitGenerator;
+import org.apache.cassandra.harry.stress.distribution.Distributions;
+
+import static java.util.Arrays.asList;
+import static org.apache.cassandra.harry.ColumnSpec.asciiType;
+import static org.apache.cassandra.harry.ColumnSpec.ck;
+import static org.apache.cassandra.harry.ColumnSpec.int64Type;
+import static org.apache.cassandra.harry.ColumnSpec.pk;
+import static org.apache.cassandra.harry.ColumnSpec.regularColumn;
+
+public class HarryStressValidationSmokeTest extends TestBaseImpl
+{
+    private static final Logger logger = LoggerFactory.getLogger(HarryStressValidationSmokeTest.class);
+
+    private static final String TABLE = "stress_smoke";
+
+    private static final long ITERATIONS = 100_000;
+    private static final int NUM_PARTITIONS = 100;
+    private static final int REPLACE_WITH_NEW = 10;
+    private static final int ROW_POPULATION = 100;
+    private static final int COLUMN_POPULATION = 100;
+    private static final int CONCURRENCY = 2;
+    private static final int RATE_PER_SECOND = 20_000;
+
+    @Test
+    public void validatingStressSmokeTest() throws Throwable
+    {
+        try (Cluster cluster = init(Cluster.build(1).start(), 1))
+        {
+            SchemaSpec schema = schema();
+            cluster.schemaChange(schema.compile());
+
+            HarryStress stress =
+                new HarryStress(schema,
+                                Distributions.fixed(ROW_POPULATION),
+                                column -> Distributions.fixed(COLUMN_POPULATION),
+                                Generators.pick(VisitGenerator.VisitType.values()),
+                                Distributions.fixed(1),
+                                new VisitGenerator.RandomOpKindGenFactory(),
+                                new RotationStrategy.FixedRotationStrategy(NUM_PARTITIONS, REPLACE_WITH_NEW, 0),
+                                null,
+                                30,
+                                () -> (statement, onComplete) -> {
+                                    Object[][] rows = cluster.coordinator(1).execute(statement.cql(),
+                                                                                     ConsistencyLevel.QUORUM,
+                                                                                     statement.bindings());
+                                    if (onComplete != null)
+                                        onComplete.run();
+                                    return rows;
+                                },
+                                CONCURRENCY,
+                                RATE_PER_SECOND,
+                                0,
+                                Long.MAX_VALUE,
+                                0);
+
+            stress.start(ITERATIONS, Long.MAX_VALUE);
+            logger.info("Validating stress smoke test ran ~{} visits across {} partitions with no validation failures",
+                        ITERATIONS, NUM_PARTITIONS);
+        }
+    }
+
+    private static SchemaSpec schema()
+    {
+        return new SchemaSpec(KEYSPACE, TABLE,
+                              asList(pk("pk1", asciiType), pk("pk2", int64Type)),
+                              asList(ck("ck1", asciiType, false), ck("ck2", int64Type, false)),
+                              asList(regularColumn("v1", asciiType), regularColumn("v2", int64Type), regularColumn("v3", asciiType)),
+                              asList(),
+                              SchemaSpec.optionsBuilder().ifNotExists(true).addWriteTimestamps(true));
+    }
+}

test/distributed/org/apache/cassandra/fuzz/ring/ConsistentBootstrapTest.java

@@ -39,7 +39,7 @@ import org.apache.cassandra.distributed.shared.NetworkTopology;
 import org.apache.cassandra.distributed.test.log.FuzzTestBase;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
-import org.apache.cassandra.harry.dsl.HistoryBuilderHelper;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
 import org.apache.cassandra.harry.execution.DataTracker;
 import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
@@ -59,6 +59,7 @@ import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseBeforeCo
 import static org.apache.cassandra.distributed.shared.ClusterUtils.unpauseCommits;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce;
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 
 public class ConsistentBootstrapTest extends FuzzTestBase
 {
@@ -83,13 +84,16 @@ public class ConsistentBootstrapTest extends FuzzTestBase
 
             withRandom(rng -> {
                 SchemaSpec schema = schemaGen.generate(rng);
-                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), 1000)));
-                Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), 1000);
+                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(valueGenerators.pkIdxGen()));
 
-                HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
+                HistoryBuilder history = new HistoryBuilder(valueGenerators);
                 Runnable writeAndValidate = () -> {
                     for (int i = 0; i < WRITES; i++)
-                        HistoryBuilderHelper.insertRandomData(schema, pkGen, ckGen, rng, history);
+                    {
+                        int pkIdx = pkGen.generate(rng);
+                        history.insert(pkIdx, valueGenerators.forPdIdx(pkIdx).ckIdxGen().generate(rng));
+                    }
 
                     for (int pk : pkGen.generated())
                         history.selectPartition(pk);
@@ -134,7 +138,7 @@ public class ConsistentBootstrapTest extends FuzzTestBase
                 RingAwareInJvmDTestVisitExecutor.replay(RingAwareInJvmDTestVisitExecutor.builder()
                                                                                         .replicationFactor(new TokenPlacementModel.SimpleReplicationFactor(2))
                                                                                         .consistencyLevel(ConsistencyLevel.ALL)
-                                                                                        .build(schema, history, cluster),
+                                                                                        .build(schema, history.valueGenerators(), cluster),
                                                         history);
             });
         }
@@ -178,13 +182,13 @@ public class ConsistentBootstrapTest extends FuzzTestBase
 
             withRandom(rng -> {
                 SchemaSpec schema = schemaGen.generate(rng);
-                Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), 1000);
 
-                HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+                HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                                                                      hb -> RingAwareInJvmDTestVisitExecutor.builder()
                                                                                                            .replicationFactor(new TokenPlacementModel.SimpleReplicationFactor(3))
                                                                                                            .consistencyLevel(ConsistencyLevel.ALL)
-                                                                                                           .build(schema, hb, cluster));
+                                                                                                           .build(schema, hb.valueGenerators(), cluster));
 
                 cluster.schemaChange(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};", KEYSPACE));
                 cluster.schemaChange(schema.compile());
@@ -198,15 +202,16 @@ public class ConsistentBootstrapTest extends FuzzTestBase
                        .drop()
                        .on();
 
-                DataTracker tracker = new DataTracker.SequentialDataTracker();
+                DataTracker.SequentialDataTracker tracker = new DataTracker.SequentialDataTracker();
                 RingAwareInJvmDTestVisitExecutor executor = RingAwareInJvmDTestVisitExecutor.builder()
                                                                                             .replicationFactor(new TokenPlacementModel.SimpleReplicationFactor(2))
                                                                                             .nodeSelector(i -> 2)
                                                                                             .consistencyLevel(ConsistencyLevel.ALL)
                                                                                             .retryPolicy(InJvmDTestVisitExecutor.RetryPolicy.NO_RETRY)
                                                                                             .build(schema,
+                                                                                                   history.valueGenerators(),
                                                                                                    tracker,
-                                                                                                   new QuiescentChecker(schema.valueGenerators, tracker, history),
+                                                                                                   new QuiescentChecker(valueGenerators, tracker),
                                                                                                    cluster);
 
                 // Prime the CMS node to pause before the finish join event is committed
@@ -231,7 +236,7 @@ public class ConsistentBootstrapTest extends FuzzTestBase
                 boolean triggered = false;
 
                 outer:
-                for (int i = 0; i < history.valueGenerators().pkPopulation(); i++)
+                for (int i = 0; i < history.valueGenerators().pkGen().population(); i++)
                 {
                     long pd = history.valueGenerators().pkGen().descriptorAt(i);
                     for (TokenPlacementModel.Replica replica : executor.getReplicasFor(pd))
@@ -240,7 +245,7 @@ public class ConsistentBootstrapTest extends FuzzTestBase
                         {
                             try
                             {
-                                HistoryBuilderHelper.insertRandomData(schema, i, ckGen.generate(rng), rng, history);
+                                history.insert(i, valueGenerators.forPdIdx(i).ckIdxGen().generate(rng));
                             }
                             catch (Throwable t)
                             {

test/distributed/org/apache/cassandra/fuzz/ring/ConsistentLeaveTest.java

@@ -57,6 +57,8 @@ import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseBeforeCo
 import static org.apache.cassandra.distributed.shared.ClusterUtils.unpauseCommits;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce;
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import static org.junit.Assert.assertFalse;
 
 public class ConsistentLeaveTest extends FuzzTestBase
@@ -82,19 +84,22 @@ public class ConsistentLeaveTest extends FuzzTestBase
 
             withRandom(rng -> {
                 SchemaSpec schema = schemaGen.generate(rng);
-                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), 1000)));
-                Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
+                IndexedValueGenerators valueGenerators = HistoryBuilder.valueGenerators(schema, rng.next(), 1000);
+                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(valueGenerators.pkIdxGen()));
 
-                HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+                HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                                                                      (hb) -> RingAwareInJvmDTestVisitExecutor.builder()
                                                                                                              .replicationFactor(new TokenPlacementModel.SimpleReplicationFactor(2))
                                                                                                              .consistencyLevel(ConsistencyLevel.ALL)
                                                                                                              .retryPolicy(InJvmDTestVisitExecutor.RetryPolicy.RETRY_ON_TIMEOUT)
                                                                                                              .nodeSelector(lts -> 1)
-                                                                                                             .build(schema, hb, cluster));
+                                                                                                             .build(schema, hb.valueGenerators(), cluster));
                 Runnable writeAndValidate = () -> {
                     for (int i = 0; i < WRITES; i++)
-                        history.insert(pkGen.generate(rng), ckGen.generate(rng));
+                    {
+                        int pkIdx = pkGen.generate(rng);
+                        history.insert(pkIdx, valueGenerators.forPdIdx(pkIdx).ckIdxGen().generate(rng));
+                    }
 
                     for (int pk : pkGen.generated())
                         history.selectPartition(pk);

test/distributed/org/apache/cassandra/fuzz/ring/ConsistentMoveTest.java

@@ -40,6 +40,7 @@ import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
 import org.apache.cassandra.harry.execution.RingAwareInJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.gen.Generator;
@@ -83,17 +84,20 @@ public class ConsistentMoveTest extends FuzzTestBase
 
             withRandom(rng -> {
                 SchemaSpec schema = schemaGen.generate(rng);
-                Generators.TrackingGenerator<Integer > pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), 1000)));
-                Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
+                IndexedValueGenerators valueGenerators = HistoryBuilder.valueGenerators(schema, rng.next(), 1000);
+                Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(valueGenerators.pkIdxGen()));
 
-                HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+                HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                                                                      (hb) -> RingAwareInJvmDTestVisitExecutor.builder()
                                                                                                              .replicationFactor(new TokenPlacementModel.SimpleReplicationFactor(2))
                                                                                                              .consistencyLevel(ConsistencyLevel.ALL)
-                                                                                                             .build(schema, hb, cluster));
+                                                                                                             .build(schema, hb.valueGenerators(), cluster));
                 Runnable writeAndValidate = () -> {
                     for (int i = 0; i < WRITES; i++)
-                        history.insert(pkGen.generate(rng), ckGen.generate(rng));
+                    {
+                        int pkIdx = pkGen.generate(rng);
+                        history.insert(pkIdx, valueGenerators.forPdIdx(pkIdx).ckIdxGen().generate(rng));
+                    }
 
                     for (int pk : pkGen.generated())
                         history.selectPartition(pk);

test/distributed/org/apache/cassandra/fuzz/sai/SingleNodeSAITestBase.java

@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.fuzz.sai;
 
-
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -43,6 +42,7 @@ import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
 import org.apache.cassandra.harry.dsl.HistoryBuilderHelper;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
 import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.gen.EntropySource;
@@ -55,6 +55,7 @@ import org.apache.cassandra.service.consensus.TransactionalMode;
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 import static org.apache.cassandra.harry.dsl.SingleOperationBuilder.IdxRelation;
 
 // TODO: "WITH OPTIONS = {'case_sensitive': 'false', 'normalize': 'true', 'ascii': 'true'};",
@@ -135,7 +136,7 @@ public abstract class SingleNodeSAITestBase extends TestBaseImpl
     public void simplifiedSaiTest()
     {
         withRandom(rng -> saiTest(rng,
-                                  SchemaGenerators.trivialSchema(KEYSPACE, "simplified", 1000).generate(rng),
+                                  SchemaGenerators.trivialSchema(KEYSPACE, "simplified").generate(rng),
                                   () -> true,
                                   DEFAULT_REPAIR_SKIP));
     }
@@ -169,9 +170,8 @@ public abstract class SingleNodeSAITestBase extends TestBaseImpl
     private void saiTest(EntropySource rng, SchemaSpec schema, Supplier<Boolean> createIndex, int repairSkip)
     {
         logger.info(schema.compile());
-
-        Generator<Integer> globalPkGen = Generators.int32(0, Math.min(NUM_PARTITIONS, schema.valueGenerators.pkPopulation()));
-        Generator<Integer> ckGen = Generators.int32(0, schema.valueGenerators.ckPopulation());
+        IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), MAX_PARTITION_SIZE);
+        Generator<Integer> globalPkGen = Generators.adaptLongToInt(Generators.int64(0, Math.min(NUM_PARTITIONS, valueGenerators.pkGen().population())));
 
         beforeEach();
         cluster.forEach(i -> i.nodetool("disableautocompaction"));
@@ -201,11 +201,11 @@ public abstract class SingleNodeSAITestBase extends TestBaseImpl
             CassandraRelevantProperties.SAI_INTERSECTION_CLAUSE_LIMIT.setInt(indexCount.get());
             waitForIndexesQueryable(schema);
 
-            HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+            HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                                                                  (hb) -> InJvmDTestVisitExecutor.builder()
                                                                                                 .pageSizeSelector(pageSizeSelector(rng))
                                                                                                 .consistencyLevel(consistencyLevelSelector())
-                                                                                                .doubleWriting(schema, hb, cluster, "debug_table"));
+                                                                                                .doubleWriting(schema, hb.valueGenerators(), cluster, "debug_table"));
             Set<Integer> partitions = new HashSet<>();
             int attempts = 0;
             while (partitions.size() < NUM_VISITED_PARTITIONS && attempts < NUM_VISITED_PARTITIONS * 10)
@@ -237,14 +237,15 @@ public abstract class SingleNodeSAITestBase extends TestBaseImpl
 
             for (int i = 0; i < OPERATIONS_PER_RUN; i++)
             {
-                int partitionIndex = pkGen.generate(rng);
-                HistoryBuilderHelper.insertRandomData(schema, partitionIndex, ckGen.generate(rng), rng, 0.5d, history);
+                int pdIdx = pkGen.generate(rng);
+                IndexedValueGenerators.IndexedPartitionValues partitionValues = valueGenerators.forPdIdx(pdIdx);
+                HistoryBuilderHelper.insertRandomData(schema, pdIdx, partitionValues.ckIdxGen().generate(rng), rng, 0.5d, history);
 
                 if (rng.nextFloat() > 0.99f)
                 {
-                    int row1 = ckGen.generate(rng);
-                    int row2 = ckGen.generate(rng);
-                    history.deleteRowRange(partitionIndex,
+                    int row1 = partitionValues.ckIdxGen().generate(rng);
+                    int row2 = partitionValues.ckIdxGen().generate(rng);
+                    history.deleteRowRange(pdIdx,
                                            Math.min(row1, row2),
                                            Math.max(row1, row2),
                                            rng.nextInt(schema.clusteringKeys.size()),
@@ -253,10 +254,10 @@ public abstract class SingleNodeSAITestBase extends TestBaseImpl
                 }
 
                 if (rng.nextFloat() > 0.995f)
-                    HistoryBuilderHelper.deleteRandomColumns(schema, partitionIndex, ckGen.generate(rng), rng, history);
+                    HistoryBuilderHelper.deleteRandomColumns(schema, pdIdx, partitionValues.ckIdxGen().generate(rng), rng, history);
 
                 if (rng.nextFloat() > 0.9995f)
-                    history.deletePartition(partitionIndex);
+                    history.deletePartition(pdIdx);
 
                 if (i % FLUSH_SKIP == 0)
                     history.custom(() -> flush(schema), "Flush");
@@ -272,13 +273,13 @@ public abstract class SingleNodeSAITestBase extends TestBaseImpl
                         List<IdxRelation> regularRelations =
                         HistoryBuilderHelper.generateValueRelations(rng,
                                                                     schema.regularColumns.size(),
-                                                                    column -> Math.min(schema.valueGenerators.regularPopulation(column), UNIQUE_CELL_VALUES),
+                                                                    column -> Math.min(partitionValues.regularPopulation(column), UNIQUE_CELL_VALUES),
                                                                     eqOnlyRegularColumns::contains);
 
                         List<IdxRelation> staticRelations =
                         HistoryBuilderHelper.generateValueRelations(rng,
                                                                     schema.staticColumns.size(),
-                                                                    column -> Math.min(schema.valueGenerators.staticPopulation(column), UNIQUE_CELL_VALUES),
+                                                                    column -> Math.min(partitionValues.staticPopulation(column), UNIQUE_CELL_VALUES),
                                                                     eqOnlyStaticColumns::contains);
 
                         Integer pk = pkGen.generate(rng);
@@ -286,7 +287,7 @@ public abstract class SingleNodeSAITestBase extends TestBaseImpl
                         IdxRelation[] ckRelations =
                         HistoryBuilderHelper.generateClusteringRelations(rng,
                                                                          schema.clusteringKeys.size(),
-                                                                         ckGen,
+                                                                         partitionValues.ckIdxGen(),
                                                                          eqOnlyClusteringColumns).toArray(new IdxRelation[0]);
 
                         IdxRelation[] regularRelationsArray = regularRelations.toArray(new IdxRelation[regularRelations.size()]);
@@ -353,7 +354,7 @@ public abstract class SingleNodeSAITestBase extends TestBaseImpl
     protected InJvmDTestVisitExecutor.ConsistencyLevelSelector consistencyLevelSelector()
     {
         return visit -> {
-            if (visit.selectOnly)
+            if (visit.validating)
                 return ConsistencyLevel.ALL;
 
             // The goal here is to make replicas as out of date as possible, modulo the efforts of repair

test/distributed/org/apache/cassandra/fuzz/sai/StaticsTortureTest.java

@@ -36,6 +36,8 @@ import org.apache.cassandra.harry.gen.Generators;
 import org.apache.cassandra.harry.gen.rng.JdkRandomEntropySource;
 import org.apache.cassandra.harry.util.BitSet;
 
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 import static org.apache.cassandra.harry.dsl.HistoryBuilderHelper.generateClusteringRelations;
 import static org.apache.cassandra.harry.dsl.HistoryBuilderHelper.generateValueRelations;
 import static org.apache.cassandra.harry.dsl.SingleOperationBuilder.IdxRelation;
@@ -45,7 +47,7 @@ public class StaticsTortureTest extends IntegrationTestBase
     private static final int MAX_PARTITION_SIZE = 10_000;
     private static final int NUM_PARTITIONS = 100;
     private static final int UNIQUE_CELL_VALUES = 5;
-
+    private static final int POPULATION = 1000;
     @Test
     public void staticsTortureTest()
     {
@@ -64,9 +66,7 @@ public class StaticsTortureTest extends IntegrationTestBase
 
     public void staticsTortureTest(List<ColumnSpec<?>> cks, int idx)
     {
-        SchemaSpec schema = new SchemaSpec(idx,
-                                           10_000,
-                                           KEYSPACE,
+        SchemaSpec schema = new SchemaSpec(KEYSPACE,
                                            "tbl" + idx,
                                            Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.int64Type, Generators.int64()),
                                                          ColumnSpec.pk("pk2", ColumnSpec.asciiType, Generators.ascii(4, 1000)),
@@ -124,8 +124,9 @@ public class StaticsTortureTest extends IntegrationTestBase
         Generator<BitSet> staticColumnBitSet = Generators.bitSet(schema.staticColumns.size());
         EntropySource rng = new JdkRandomEntropySource(1l);
 
-        ReplayingHistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators, hb -> {
-            return InJvmDTestVisitExecutor.builder().pageSizeSelector(i -> rng.nextInt(1, 10)).build(schema, hb, cluster);
+        IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), POPULATION);
+        ReplayingHistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators, hb -> {
+            return InJvmDTestVisitExecutor.builder().pageSizeSelector(i -> rng.nextInt(1, 10)).build(schema, hb.valueGenerators(), cluster);
         });
 
 
@@ -175,10 +176,8 @@ public class StaticsTortureTest extends IntegrationTestBase
             for (int i = 0; i < 10; i++)
             {
                 List<IdxRelation> ckRelations = generateClusteringRelations(rng, schema.clusteringKeys.size(), ckIdxGen);
-                List<IdxRelation> regularRelations = generateValueRelations(rng, schema.regularColumns.size(),
-                                                                            column -> Math.min(schema.valueGenerators.regularPopulation(column), MAX_PARTITION_SIZE));
-                List<IdxRelation> staticRelations = generateValueRelations(rng, schema.staticColumns.size(),
-                                                                           column -> Math.min(schema.valueGenerators.staticPopulation(column), MAX_PARTITION_SIZE));
+                List<IdxRelation> regularRelations = generateValueRelations(rng, schema.regularColumns.size(),column -> POPULATION);
+                List<IdxRelation> staticRelations = generateValueRelations(rng, schema.staticColumns.size(),column -> POPULATION);
                 history.select(pdx,
                                ckRelations.toArray(new IdxRelation[0]),
                                regularRelations.toArray(new IdxRelation[0]),

test/distributed/org/apache/cassandra/fuzz/topology/AccordBootstrapTest.java

@@ -36,6 +36,7 @@ import org.apache.cassandra.distributed.test.log.FuzzTestBase;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
 import org.apache.cassandra.harry.dsl.HistoryBuilderHelper;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
 import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.execution.QueryBuildingVisitExecutor;
@@ -48,6 +49,7 @@ import org.apache.cassandra.service.consensus.TransactionalMode;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.unpauseCommits;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce;
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 
 public class AccordBootstrapTest extends FuzzTestBase
 {
@@ -77,16 +79,17 @@ public class AccordBootstrapTest extends FuzzTestBase
 
             HashSet<Integer> downInstances = new HashSet<>();
             withRandom(rng -> {
-                Generator<SchemaSpec> schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bootstrap_fuzz", POPULATION,
+                Generator<SchemaSpec> schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bootstrap_fuzz",
                                                                                  SchemaSpec.optionsBuilder()
                                                                                            .addWriteTimestamps(false)
                                                                                            .withTransactionalMode(TransactionalMode.full)
                 );
 
                 SchemaSpec schema = schemaGen.generate(rng);
-                TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), POPULATION)));
-                Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), POPULATION));
-                HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), 1000);
+                TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(Generators.int64(0, Math.min(valueGenerators.pkGen().population(), POPULATION))));
+                HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                                                                      hb -> InJvmDTestVisitExecutor.builder()
                                                                                                   .consistencyLevel(ConsistencyLevel.QUORUM)
                                                                                                   .wrapQueries(QueryBuildingVisitExecutor.WrapQueries.TRANSACTION)
@@ -100,11 +103,15 @@ public class AccordBootstrapTest extends FuzzTestBase
 
                                                                                                       }
                                                                                                   })
-                                                                                                  .build(schema, hb, cluster));
+                                                                                                  .build(schema, valueGenerators, cluster));
 
                 Runnable writeAndValidate = () -> {
                     for (int i = 0; i < WRITES; i++)
-                        HistoryBuilderHelper.insertRandomData(schema, pkGen, ckGen, rng, history);
+                    {
+                        int pdIdx = pkGen.generate(rng);
+                        IndexedValueGenerators.IndexedPartitionValues partitionValues = valueGenerators.forPdIdx(pdIdx);
+                        HistoryBuilderHelper.insertRandomData(schema, pdIdx, partitionValues.ckIdxGen.generate(rng), rng, 0.5d, history);
+                    }
 
                     for (int pk : pkGen.generated())
                         history.selectPartition(pk);

test/distributed/org/apache/cassandra/fuzz/topology/AccordBounceTest.java

@@ -36,6 +36,7 @@ import org.apache.cassandra.distributed.shared.ClusterUtils;
 import org.apache.cassandra.distributed.test.log.FuzzTestBase;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
 import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.execution.QueryBuildingVisitExecutor;
@@ -45,6 +46,7 @@ import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 
 public class AccordBounceTest extends FuzzTestBase
 {
@@ -67,7 +69,7 @@ public class AccordBounceTest extends FuzzTestBase
                                                                                      {
                                                                                          return "bootstrap_fuzz" + (i++);
                                                                                      }
-                                                                                 }, POPULATION,
+                                                                                 },
                                                                                  SchemaSpec.optionsBuilder()
                                                                                          .addWriteTimestamps(false)
                                                                                          .withTransactionalMode(TransactionalMode.full)
@@ -78,12 +80,13 @@ public class AccordBounceTest extends FuzzTestBase
                 {
                     SchemaSpec schema = schemaGen.generate(rng);
                     cluster.schemaChange(schema.compile());
-                    historyBuilders.add(new ReplayingHistoryBuilder(schema.valueGenerators,
+                    IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), POPULATION);
+                    historyBuilders.add(new ReplayingHistoryBuilder(valueGenerators,
                                                                     hb -> InJvmDTestVisitExecutor.builder()
                                                                             .consistencyLevel(ConsistencyLevel.QUORUM)
                                                                             .wrapQueries(QueryBuildingVisitExecutor.WrapQueries.TRANSACTION)
                                                                             .pageSizeSelector(p -> InJvmDTestVisitExecutor.PageSizeSelector.NO_PAGING)
-                                                                            .build(schema, hb, cluster)));
+                                                                            .build(schema, valueGenerators, cluster)));
                 }
 
                 for (HistoryBuilder hb : historyBuilders)
@@ -125,7 +128,7 @@ public class AccordBounceTest extends FuzzTestBase
                                                                                      {
                                                                                          return  "bootstrap_fuzz" + (i++);
                                                                                      }
-                                                                                 }, POPULATION,
+                                                                                 },
                                                                                  SchemaSpec.optionsBuilder()
                                                                                          .addWriteTimestamps(false)
                                                                                          .withTransactionalMode(TransactionalMode.full)
@@ -136,12 +139,13 @@ public class AccordBounceTest extends FuzzTestBase
                 {
                     SchemaSpec schema = schemaGen.generate(rng);
                     cluster.schemaChange(schema.compile());
-                    historyBuilders.add(new ReplayingHistoryBuilder(schema.valueGenerators,
+                    IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), POPULATION);
+                    historyBuilders.add(new ReplayingHistoryBuilder(valueGenerators,
                                                                     hb -> InJvmDTestVisitExecutor.builder()
                                                                             .consistencyLevel(ConsistencyLevel.QUORUM)
                                                                             .wrapQueries(QueryBuildingVisitExecutor.WrapQueries.TRANSACTION)
                                                                             .pageSizeSelector(p -> InJvmDTestVisitExecutor.PageSizeSelector.NO_PAGING)
-                                                                            .build(schema, hb, cluster)));
+                                                                            .build(schema, valueGenerators, cluster)));
                 }
 
                 Runnable writeAndValidate = () -> {

test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java

@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fuzz.topology;
+
+import java.util.HashSet;
+
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.distributed.shared.ClusterUtils;
+import org.apache.cassandra.distributed.shared.NetworkTopology;
+import org.apache.cassandra.distributed.test.log.FuzzTestBase;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.HistoryBuilderHelper;
+import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
+import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
+import org.apache.cassandra.harry.execution.QueryBuildingVisitExecutor;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.Generators;
+import org.apache.cassandra.harry.gen.Generators.TrackingGenerator;
+import org.apache.cassandra.harry.gen.SchemaGenerators;
+import org.apache.cassandra.service.consensus.TransactionalMode;
+
+import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce;
+import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
+
+public class AccordHardCatchupTest extends FuzzTestBase
+{
+    private static final int WRITES = 10;
+    private static final int POPULATION = 1000;
+
+    @Test
+    public void hardCatchupFuzzTest() throws Throwable
+    {
+        CassandraRelevantProperties.SYSTEM_TRACES_DEFAULT_RF.setInt(3);
+        Cluster.Builder builder = builder();
+        try (Cluster cluster = builder.withNodes(3)
+                                      .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(100))
+                                      .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(100, "dc0", "rack0"))
+                                      .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP))
+                                      .start())
+        {
+            IInvokableInstance cmsInstance = cluster.get(1);
+            waitForCMSToQuiesce(cluster, cmsInstance);
+
+            HashSet<Integer> downInstances = new HashSet<>();
+            withRandom(rng -> {
+                Generator<SchemaSpec> schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bootstrap_fuzz",
+                                                                                 SchemaSpec.optionsBuilder()
+                                                                                           .addWriteTimestamps(false)
+                                                                                           .withTransactionalMode(TransactionalMode.full)
+                );
+
+                SchemaSpec schema = schemaGen.generate(rng);
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), POPULATION);
+                TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(Generators.int64(0, Math.min(valueGenerators.pkGen().population(), POPULATION))));
+                HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
+                                                                     hb -> InJvmDTestVisitExecutor.builder()
+                                                                                                  .consistencyLevel(ConsistencyLevel.QUORUM)
+                                                                                                  .wrapQueries(QueryBuildingVisitExecutor.WrapQueries.TRANSACTION)
+                                                                                                  .pageSizeSelector(p -> InJvmDTestVisitExecutor.PageSizeSelector.NO_PAGING)
+                                                                                                  .nodeSelector(lts -> {
+                                                                                                      while (true)
+                                                                                                      {
+                                                                                                          int pick = rng.nextInt(1, cluster.size() + 1);
+                                                                                                          if (!downInstances.contains(pick))
+                                                                                                              return pick;
+
+                                                                                                      }
+                                                                                                  })
+                                                                                                  .build(schema, valueGenerators, cluster));
+
+                Runnable writeAndValidate = () -> {
+                    for (int i = 0; i < WRITES; i++)
+                    {
+                        int pdIdx = pkGen.generate(rng);
+                        IndexedValueGenerators.IndexedPartitionValues partitionValues = valueGenerators.forPdIdx(pdIdx);
+                        HistoryBuilderHelper.insertRandomData(schema, pdIdx, partitionValues.ckIdxGen.generate(rng), rng, 0.5d, history);
+                    }
+
+                    for (int pk : pkGen.generated())
+                        history.selectPartition(pk);
+                };
+
+                history.customThrowing(() -> {
+                    cluster.schemaChange(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};", KEYSPACE));
+                    cluster.schemaChange(schema.compile());
+                    waitForCMSToQuiesce(cluster, cmsInstance);
+                }, "Setup");
+                Thread.sleep(1000);
+                writeAndValidate.run();
+
+                history.customThrowing(() -> {
+                    downInstances.add(2);
+                    ClusterUtils.stopUnchecked(cluster.get(2));
+                    cluster.get(1).logs().watchFor("/127.0.0.2:.* is now DOWN");
+                }, "Shut down node 2");
+
+                writeAndValidate.run();
+
+                history.customThrowing(() -> {
+                    cluster.get(2).config().set("accord.catchup_on_start", "HARD");
+                    cluster.get(2).startup();
+                    cluster.get(2).logs().watchFor(".*Catchup.*");
+                    cluster.get(1).logs().watchFor("/127.0.0.2:.* is now UP");
+                    downInstances.remove(2);
+                }, "Start down node 2");
+
+                writeAndValidate.run();
+
+                history.customThrowing(() -> {
+                    downInstances.add(1);
+                    ClusterUtils.stopUnchecked(cluster.get(1));
+                    cluster.get(2).logs().watchFor("/127.0.0.1:.* is now DOWN");
+                }, "Shut down node 1");
+
+                writeAndValidate.run();
+            });
+        }
+    }
+}

test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java

@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fuzz.topology;
+
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.IntPredicate;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.distributed.shared.ClusterUtils;
+import org.apache.cassandra.distributed.shared.NetworkTopology;
+import org.apache.cassandra.distributed.test.log.FuzzTestBase;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.HistoryBuilderHelper;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
+import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
+import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
+import org.apache.cassandra.harry.execution.QueryBuildingVisitExecutor;
+import org.apache.cassandra.harry.gen.EntropySource;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.Generators;
+import org.apache.cassandra.harry.gen.Generators.TrackingGenerator;
+import org.apache.cassandra.harry.gen.SchemaGenerators;
+import org.apache.cassandra.harry.util.ThrowingRunnable;
+import org.apache.cassandra.io.util.PathUtils;
+import org.apache.cassandra.service.consensus.TransactionalMode;
+
+import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce;
+import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
+
+public class AccordRebootstrapTest extends FuzzTestBase
+{
+    private static final int WRITES = 10;
+    private static final int POPULATION = 1000;
+
+    @Test
+    public void rebootstrapFuzzTest() throws Throwable
+    {
+        CassandraRelevantProperties.SYSTEM_TRACES_DEFAULT_RF.setInt(3);
+        Cluster.Builder builder = builder();
+        try (Cluster cluster = builder.withNodes(3)
+                                      .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(100))
+                                      .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(100, "dc0", "rack0"))
+                                      .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP))
+                                      .start())
+        {
+            IInvokableInstance cmsInstance = cluster.get(1);
+            waitForCMSToQuiesce(cluster, cmsInstance);
+
+            HashSet<Integer> downInstances = new HashSet<>();
+            AtomicInteger nextId = new AtomicInteger();
+            withRandom(rng -> {
+                Generator<SchemaSpec> schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bootstrap_fuzz" + (nextId.incrementAndGet()),
+                                                                                 SchemaSpec.optionsBuilder()
+                                                                                           .addWriteTimestamps(false)
+                                                                                           .withTransactionalMode(TransactionalMode.full)
+                );
+
+                History history1 = createNewSchemaWithWriteAndValidate(schemaGen, rng, cluster, downInstances::contains);
+                history1.writeAndValidate();
+
+                history1.run(() -> {
+                    downInstances.add(2);
+                    ClusterUtils.stopUnchecked(cluster.get(2));
+                    cluster.get(1).logs().watchFor("/127.0.0.2:.* is now DOWN");
+                }, "Shut down node 2");
+
+                history1.writeAndValidate();
+                History history2 = createNewSchemaWithWriteAndValidate(schemaGen, rng, cluster, downInstances::contains);
+                history2.writeAndValidate();
+                History history3 = createNewSchemaWithWriteAndValidate(schemaGen, rng, cluster, downInstances::contains);
+                history3.writeAndValidate();
+
+                history1.run(() -> {
+                    cluster.get(2).config().set("accord.journal.stop_marker_failure_policy", "REBOOTSTRAP");
+                    Path journalDir = Path.of(cluster.get(2).config().get("accord.journal_directory").toString());
+                    Path stopMarker = journalDir.resolve("stopped");
+                    PathUtils.delete(stopMarker);
+                    cluster.get(2).startup();
+                    cluster.get(2).logs().watchFor(".*Rebootstrapping.*");
+                    cluster.get(1).logs().watchFor("/127.0.0.2:.* is now UP");
+                    downInstances.remove(2);
+                }, "Start down node 2");
+
+                history1.writeAndValidate();
+                history2.writeAndValidate();
+                history3.writeAndValidate();
+            });
+        }
+    }
+
+    interface History
+    {
+        void writeAndValidate();
+        void run(ThrowingRunnable run, String tag);
+    }
+
+    private History createNewSchemaWithWriteAndValidate(Generator<SchemaSpec> schemaGen, EntropySource rng, Cluster cluster, IntPredicate downInstances) throws InterruptedException
+    {
+        IInvokableInstance cmsInstance = cluster.get(1);
+        SchemaSpec schema = schemaGen.generate(rng);
+        IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), POPULATION);
+        TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.adaptLongToInt(Generators.int64(0, Math.min(valueGenerators.pkGen().population(), POPULATION))));
+        HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
+                                                             hb -> InJvmDTestVisitExecutor.builder()
+                                                                                          .consistencyLevel(ConsistencyLevel.QUORUM)
+                                                                                          .wrapQueries(QueryBuildingVisitExecutor.WrapQueries.TRANSACTION)
+                                                                                          .pageSizeSelector(p -> InJvmDTestVisitExecutor.PageSizeSelector.NO_PAGING)
+                                                                                          .nodeSelector(lts -> {
+                                                                                              while (true)
+                                                                                              {
+                                                                                                  int pick = rng.nextInt(1, cluster.size() + 1);
+                                                                                                  if (!downInstances.test(pick))
+                                                                                                      return pick;
+                                                                                              }
+                                                                                          })
+                                                                                          .build(schema, valueGenerators, cluster));
+
+        history.customThrowing(() -> {
+            cluster.schemaChangeIgnoringStoppedInstances(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};", KEYSPACE));
+            cluster.schemaChangeIgnoringStoppedInstances(schema.compile());
+            waitForCMSToQuiesce(cluster, cmsInstance);
+        }, "Setup");
+        Thread.sleep(1000);
+
+        return new History()
+        {
+            @Override
+            public void writeAndValidate()
+            {
+                for (int i = 0; i < WRITES; i++)
+                {
+                    int pdIdx = pkGen.generate(rng);
+                    IndexedValueGenerators.IndexedPartitionValues partitionValues = valueGenerators.forPdIdx(pdIdx);
+                    HistoryBuilderHelper.insertRandomData(schema, pdIdx, partitionValues.ckIdxGen.generate(rng), rng, 0.5d, history);
+                }
+
+                for (int pk : pkGen.generated())
+                    history.selectPartition(pk);
+            }
+
+            @Override
+            public void run(ThrowingRunnable run, String tag)
+            {
+                history.customThrowing(run, tag);
+            }
+        };
+    }
+}

test/distributed/org/apache/cassandra/fuzz/topology/HarryTopologyMixupTest.java

@@ -55,7 +55,7 @@ import org.apache.cassandra.harry.gen.Generator;
 import org.apache.cassandra.harry.gen.Generators;
 import org.apache.cassandra.harry.gen.SchemaGenerators;
 import org.apache.cassandra.harry.gen.rng.JdkRandomEntropySource;
-import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.ClusteringOrderBy;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.utils.AssertionUtils;
 
@@ -135,8 +135,7 @@ public class HarryTopologyMixupTest extends TopologyMixupTestBase<HarryTopologyM
                 schemaGen = SchemaGenerators.schemaSpecGen("harry", "table", 1000);
 
             schema = schemaGen.generate(rng);
-
-            HistoryBuilder harry = new ReplayingHistoryBuilder(schema.valueGenerators,
+            HistoryBuilder harry = new ReplayingHistoryBuilder(HistoryBuilder.valueGenerators(schema, rng.next(), 1000),
                     hb -> {
                         InJvmDTestVisitExecutor.Builder builder = InJvmDTestVisitExecutor.builder();
                         if (mode.kind == AccordMode.Kind.Direct)
@@ -182,7 +181,7 @@ public class HarryTopologyMixupTest extends TopologyMixupTestBase<HarryTopologyM
                                     }
                                     return false;
                                 })
-                                .build(schema, hb, cluster);
+                                .build(schema, hb.valueGenerators(), cluster);
                     });
             cluster.schemaChange(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};", schema.keyspace));
             cluster.schemaChange(schema.compile());
@@ -239,7 +238,7 @@ public class HarryTopologyMixupTest extends TopologyMixupTestBase<HarryTopologyM
 
             TransactionalMode transationalMode = spec.schema.options.transactionalMode();
             if (TransactionalMode.full == transationalMode)
-                reads.add(new HarryCommand(s -> String.format("Harry Reverse Validate pd=%d%s", pd, state.commandNamePostfix()), s -> spec.harry.selectPartition(pkIdx, Operations.ClusteringOrderBy.DESC)));
+                reads.add(new HarryCommand(s -> String.format("Harry Reverse Validate pd=%d%s", pd, state.commandNamePostfix()), s -> spec.harry.selectPartition(pkIdx, ClusteringOrderBy.DESC)));
         }
         reads.add(new HarryCommand(s -> "Reset Harry Write State" + state.commandNamePostfix(), s -> ((HarryState) s).numInserts = 0));
         return Property.multistep(reads);
@@ -255,7 +254,7 @@ public class HarryTopologyMixupTest extends TopologyMixupTestBase<HarryTopologyM
         {
             this.harry = harry;
             this.schema = schema;
-            this.pkGen = Generators.tracking(Generators.int32(0, schema.valueGenerators.pkPopulation()));
+            this.pkGen = Generators.tracking(Generators.adaptLongToInt(harry.valueGenerators().pkIdxGen()));
         }
 
         @Override

test/distributed/org/apache/cassandra/fuzz/topology/IdenticalTopologyTest.java

@@ -27,6 +27,7 @@ import org.apache.cassandra.distributed.shared.ClusterUtils;
 import org.apache.cassandra.distributed.test.log.FuzzTestBase;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
 import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
 import org.apache.cassandra.harry.execution.QueryBuildingVisitExecutor;
@@ -38,6 +39,7 @@ import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 
 public class IdenticalTopologyTest extends FuzzTestBase
 {
@@ -64,12 +66,13 @@ public class IdenticalTopologyTest extends FuzzTestBase
 
                 SchemaSpec schema = schemaGen.generate(rng);
                 cluster.schemaChange(schema.compile());
-                HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), POPULATION);
+                HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                                                                      hb -> InJvmDTestVisitExecutor.builder()
                                                                                                   .consistencyLevel(ConsistencyLevel.QUORUM)
                                                                                                   .wrapQueries(QueryBuildingVisitExecutor.WrapQueries.TRANSACTION)
                                                                                                   .pageSizeSelector(p -> InJvmDTestVisitExecutor.PageSizeSelector.NO_PAGING)
-                                                                                                  .build(schema, hb, cluster));
+                                                                                                  .build(schema, valueGenerators, cluster));
 
                 for (int i = 0; i <= 100; i++)
                 {

test/distributed/org/apache/cassandra/fuzz/topology/JournalGCTest.java

@@ -22,6 +22,7 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -47,11 +48,10 @@ import org.apache.cassandra.service.consensus.TransactionalMode;
 
 import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
 import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.valueGenerators;
 
 public class JournalGCTest extends FuzzTestBase
 {
-    private static final int POPULATION = 1000;
-
     @Test
     public void journalGCTest() throws Throwable
     {
@@ -70,19 +70,20 @@ public class JournalGCTest extends FuzzTestBase
                     Keyspace.open(SchemaConstants.ACCORD_KEYSPACE_NAME).getColumnFamilyStore(AccordKeyspace.JOURNAL).disableAutoCompaction();
                 });
 
-                Generator<SchemaSpec> schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bootstrap_fuzz", POPULATION,
+                Generator<SchemaSpec> schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bootstrap_fuzz",
                                                                                  SchemaSpec.optionsBuilder()
                                                                                          .addWriteTimestamps(false)
                                                                                          .withTransactionalMode(TransactionalMode.full));
 
                 SchemaSpec schema = schemaGen.generate(rng);
                 cluster.schemaChange(schema.compile());
-                HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
+                IndexedValueGenerators valueGenerators = valueGenerators(schema, rng.next(), 1000);
+                HistoryBuilder history = new ReplayingHistoryBuilder(valueGenerators,
                                                                      hb -> InJvmDTestVisitExecutor.builder()
                                                                              .consistencyLevel(ConsistencyLevel.QUORUM)
                                                                              .wrapQueries(QueryBuildingVisitExecutor.WrapQueries.TRANSACTION)
                                                                              .pageSizeSelector(p -> InJvmDTestVisitExecutor.PageSizeSelector.NO_PAGING)
-                                                                             .build(schema, hb, cluster));
+                                                                             .build(schema, valueGenerators, cluster));
 
                 for (int pk = 0; pk <= 500; pk++) {
                     for (int i = 0; i < 100; i++)

test/harry/main/org/apache/cassandra/harry/MagicConstants.java

@@ -55,6 +55,11 @@ public class MagicConstants
     public static final long UNSET_DESCR = Long.MIN_VALUE + 3;
     public static final long NIL_DESCR = Long.MIN_VALUE;
     public static final Set<Long> MAGIC_DESCRIPTOR_VALS = Set.of(UNKNOWN_DESCR, EMPTY_VALUE_DESCR, UNSET_DESCR, NIL_DESCR);
+
+    public static boolean isMagicDescriptor(long val)
+    {
+        return val <= UNSET_DESCR;
+    }
     /**
      * For LTS
      */

test/harry/main/org/apache/cassandra/harry/SchemaSpec.java

@@ -25,10 +25,8 @@ import java.util.Objects;
 import java.util.function.Consumer;
 
 import org.apache.cassandra.cql3.ast.Symbol;
-import org.apache.cassandra.harry.dsl.HistoryBuilder;
 import org.apache.cassandra.harry.gen.Generator;
 import org.apache.cassandra.harry.gen.Generators;
-import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.util.IteratorsUtil;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.utils.ByteArrayUtil;
@@ -46,25 +44,20 @@ public class SchemaSpec
     public final List<ColumnSpec<?>> staticColumns;
 
     public final List<ColumnSpec<?>> allColumnInSelectOrder;
-    public final ValueGenerators<Object[], Object[]> valueGenerators;
     public final Options options;
 
-    public SchemaSpec(long seed,
-                      int populationPerColumn,
-                      String keyspace,
+    public SchemaSpec(String keyspace,
                       String table,
                       List<ColumnSpec<?>> partitionKeys,
                       List<ColumnSpec<?>> clusteringKeys,
                       List<ColumnSpec<?>> regularColumns,
                       List<ColumnSpec<?>> staticColumns)
     {
-        this(seed, populationPerColumn, keyspace, table, partitionKeys, clusteringKeys, regularColumns, staticColumns, optionsBuilder());
+        this(keyspace, table, partitionKeys, clusteringKeys, regularColumns, staticColumns, optionsBuilder());
     }
 
     @SuppressWarnings({ "unchecked" })
-    public SchemaSpec(long seed,
-                      int populationPerColumn,
-                      String keyspace,
+    public SchemaSpec(String keyspace,
                       String table,
                       List<ColumnSpec<?>> partitionKeys,
                       List<ColumnSpec<?>> clusteringKeys,
@@ -93,11 +86,11 @@ public class SchemaSpec
                                                          regularSelectOrder))
             selectOrder.add(column);
         this.allColumnInSelectOrder = Collections.unmodifiableList(selectOrder);
-
-        // TODO: empty gen
-        this.valueGenerators = HistoryBuilder.valueGenerators(this, seed, populationPerColumn);
     }
 
+    /**
+     * A number of unique values that can be generated for a given combination of columns.
+     */
     public static /* unsigned */ long cumulativeEntropy(List<ColumnSpec<?>> columns)
     {
         if (columns.isEmpty())
@@ -383,7 +376,6 @@ public class SchemaSpec
         private boolean trackLts = false;
         private boolean compactStorage = false;
         private String speculativeRetry = null;
-
         private OptionsBuilder()
         {
         }

test/harry/main/org/apache/cassandra/harry/cql/DeleteHelper.java

@@ -18,16 +18,19 @@
 
 package org.apache.cassandra.harry.cql;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
 
 import org.apache.cassandra.cql3.ast.Symbol;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.harry.ColumnSpec;
 import org.apache.cassandra.harry.Relations;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.op.Operations;
 import org.apache.cassandra.harry.util.BitSet;
 
@@ -35,6 +38,7 @@ public class DeleteHelper
 {
     public static CompiledStatement inflateDelete(Operations.DeletePartition delete,
                                                   SchemaSpec schema,
+                                                  ValueGenerators<Object[], Object[]> valueGenerators,
                                                   long timestamp)
     {
         StringBuilder b = new StringBuilder();
@@ -50,7 +54,7 @@ public class DeleteHelper
 
         List<Object> bindings = new ArrayList<>();
 
-        Object[] pk = schema.valueGenerators.pkGen().inflate(delete.pd());
+        Object[] pk = valueGenerators.pkGen().inflate(delete.pd());
 
         RelationWriter writer = new RelationWriter(b, bindings::add) ;
 
@@ -61,11 +65,19 @@ public class DeleteHelper
 
         Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
 
-        return new CompiledStatement(b.toString(), bindingsArr);
+        CompiledStatement compiled = new CompiledStatement(false, b.toString(), bindingsArr);
+        {
+            ByteBuffer[] pkBuffers = new ByteBuffer[pk.length];
+            for (int i = 0; i < pk.length; i++)
+                pkBuffers[i] = ((AbstractType)schema.partitionKeys.get(i).type.asServerType()).decompose(pk[i]);
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
     public static CompiledStatement inflateDelete(Operations.DeleteRow delete,
                                                   SchemaSpec schema,
+                                                  ValueGenerators<Object[], Object[]> generators,
                                                   long timestamp)
     {
         StringBuilder b = new StringBuilder();
@@ -81,8 +93,9 @@ public class DeleteHelper
 
         List<Object> bindings = new ArrayList<>();
 
-        Object[] pk = schema.valueGenerators.pkGen().inflate(delete.pd());
-        Object[] ck = schema.valueGenerators.ckGen().inflate(delete.cd());
+        Object[] pk = generators.pkGen().inflate(delete.pd());
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = generators.forPd(delete.pd);
+        Object[] ck = valueGenerators.ckGen().inflate(delete.cd());
 
         RelationWriter writer = new RelationWriter(b, bindings::add);
 
@@ -95,11 +108,19 @@ public class DeleteHelper
 
         Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
 
-        return new CompiledStatement(b.toString(), bindingsArr);
+        CompiledStatement compiled = new CompiledStatement(false, b.toString(), bindingsArr);
+        {
+            ByteBuffer[] pkBuffers = new ByteBuffer[pk.length];
+            for (int i = 0; i < pk.length; i++)
+                pkBuffers[i] = ((AbstractType)schema.partitionKeys.get(i).type.asServerType()).decompose(pk[i]);
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
     public static CompiledStatement inflateDelete(Operations.DeleteColumns delete,
                                                   SchemaSpec schema,
+                                                  ValueGenerators<Object[], Object[]> generators,
                                                   long timestamp)
     {
         StringBuilder b = new StringBuilder();
@@ -139,8 +160,9 @@ public class DeleteHelper
 
         List<Object> bindings = new ArrayList<>();
 
-        Object[] pk = schema.valueGenerators.pkGen().inflate(delete.pd());
-        Object[] ck = schema.valueGenerators.ckGen().inflate(delete.cd());
+        Object[] pk = generators.pkGen().inflate(delete.pd());
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = generators.forPd(delete.pd);
+        Object[] ck = valueGenerators.ckGen().inflate(delete.cd());
 
         RelationWriter writer = new RelationWriter(b, bindings::add);
 
@@ -153,11 +175,19 @@ public class DeleteHelper
 
         Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
 
-        return new CompiledStatement(b.toString(), bindingsArr);
+        CompiledStatement compiled = new CompiledStatement(false, b.toString(), bindingsArr);
+        {
+            ByteBuffer[] pkBuffers = new ByteBuffer[pk.length];
+            for (int i = 0; i < pk.length; i++)
+                pkBuffers[i] = ((AbstractType)schema.partitionKeys.get(i).type.asServerType()).decompose(pk[i]);
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
     public static CompiledStatement inflateDelete(Operations.DeleteRange delete,
                                                   SchemaSpec schema,
+                                                  ValueGenerators<Object[], Object[]> generators,
                                                   long timestamp)
     {
         StringBuilder b = new StringBuilder();
@@ -173,9 +203,10 @@ public class DeleteHelper
 
         List<Object> bindings = new ArrayList<>();
 
-        Object[] pk = schema.valueGenerators.pkGen().inflate(delete.pd());
-        Object[] lowBound = schema.valueGenerators.ckGen().inflate(delete.lowerBound());
-        Object[] highBound = schema.valueGenerators.ckGen().inflate(delete.upperBound());
+        Object[] pk = generators.pkGen().inflate(delete.pd());
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = generators.forPd(delete.pd);
+        Object[] lowBound = valueGenerators.ckGen().inflate(delete.lowerBound());
+        Object[] highBound = valueGenerators.ckGen().inflate(delete.upperBound());
 
         RelationWriter writer = new RelationWriter(b, bindings::add);
 
@@ -196,7 +227,14 @@ public class DeleteHelper
 
         Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
 
-        return new CompiledStatement(b.toString(), bindingsArr);
+        CompiledStatement compiled = new CompiledStatement(false, b.toString(), bindingsArr);
+        {
+            ByteBuffer[] pkBuffers = new ByteBuffer[pk.length];
+            for (int i = 0; i < pk.length; i++)
+                pkBuffers[i] = ((AbstractType)schema.partitionKeys.get(i).type.asServerType()).decompose(pk[i]);
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
     private static final class RelationWriter

test/harry/main/org/apache/cassandra/harry/cql/SelectHelper.java

@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.harry.cql;
 
+import java.nio.ByteBuffer;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -27,19 +28,26 @@ import org.apache.cassandra.cql3.ast.Conditional.Where;
 import org.apache.cassandra.cql3.ast.FunctionCall;
 import org.apache.cassandra.cql3.ast.Select;
 import org.apache.cassandra.cql3.ast.Symbol;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.MagicConstants;
 import org.apache.cassandra.harry.Relations;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.gen.ValueGenerators;
+import org.apache.cassandra.harry.op.ClusteringOrderBy;
 import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Selection;
 
 public class SelectHelper
 {
-    public static CompiledStatement select(Operations.SelectPartition select, SchemaSpec schema)
+    public static CompiledStatement select(Operations.SelectPartition select,
+                                           SchemaSpec schema,
+                                           ValueGenerators<Object[], Object[]> generators)
     {
-        Select.Builder builder = commmonPart(select, schema);
+        Select.Builder builder = commmonPart(select, schema, generators);
 
-        if (select.orderBy() == Operations.ClusteringOrderBy.DESC)
+        if (select.orderBy() == ClusteringOrderBy.DESC)
         {
             for (int i = 0; i < schema.clusteringKeys.size(); i++)
             {
@@ -48,14 +56,28 @@ public class SelectHelper
             }
         }
 
-        return toCompiled(builder.build());
+        CompiledStatement compiled = toCompiled(builder.build());
+        {
+            Object[] pk = generators.pkGen().inflate(select.pd());
+            ByteBuffer[] pkBuffers = new ByteBuffer[pk.length];
+            for (int i = 0; i < schema.partitionKeys.size(); i++)
+            {
+                ColumnSpec<?> column = schema.partitionKeys.get(i);
+                Object value = pk[i];
+                pkBuffers[i] = ((AbstractType) column.type.asServerType()).decompose(value);
+            }
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
-    public static CompiledStatement select(Operations.SelectRow select, SchemaSpec schema)
+    public static CompiledStatement select(Operations.SelectRow select,
+                                           SchemaSpec schema,
+                                           ValueGenerators<Object[], Object[]> generators)
     {
-        Select.Builder builder = commmonPart(select, schema);
-
-        Object[] ck = schema.valueGenerators.ckGen().inflate(select.cd());
+        Select.Builder builder = commmonPart(select, schema, generators);
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = generators.forPd(select.pd);
+        Object[] ck = valueGenerators.ckGen().inflate(select.cd());
 
         for (int i = 0; i < schema.clusteringKeys.size(); i++)
         {
@@ -65,27 +87,42 @@ public class SelectHelper
                           new Bind(ck[i], column.type.asServerType()));
         }
 
-        return toCompiled(builder.build());
+        CompiledStatement compiled = toCompiled(builder.build());
+        {
+            Object[] pk = generators.pkGen().inflate(select.pd());
+            ByteBuffer[] pkBuffers = new ByteBuffer[pk.length];
+            for (int i = 0; i < schema.partitionKeys.size(); i++)
+            {
+                ColumnSpec<?> column = schema.partitionKeys.get(i);
+                Object value = pk[i];
+                pkBuffers[i] = ((AbstractType) column.type.asServerType()).decompose(value);
+            }
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
-    public static CompiledStatement select(Operations.SelectRange select, SchemaSpec schema)
+    public static CompiledStatement select(Operations.SelectRange select,
+                                           SchemaSpec schema,
+                                           ValueGenerators<Object[], Object[]> generators)
     {
-        Select.Builder builder = commmonPart(select, schema);
+        Select.Builder builder = commmonPart(select, schema, generators);
 
-        Object[] lowBound = schema.valueGenerators.ckGen().inflate(select.lowerBound());
-        Object[] highBound = schema.valueGenerators.ckGen().inflate(select.upperBound());
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = generators.forPd(select.pd);
+        Object[] lowBound = select.lowerBound() == MagicConstants.UNSET_DESCR ? null : valueGenerators.ckGen().inflate(select.lowerBound());
+        Object[] highBound = select.upperBound() == MagicConstants.UNSET_DESCR ? null : valueGenerators.ckGen().inflate(select.upperBound());
 
         for (int i = 0; i < schema.clusteringKeys.size(); i++)
         {
             ColumnSpec<?> column = schema.clusteringKeys.get(i);
-            if (select.lowerBoundRelation()[i] != null)
+            if (lowBound != null && select.lowerBoundRelation()[i] != null)
             {
                 builder.where(new Symbol(column.name, column.type.asServerType()),
                               toInequality(select.lowerBoundRelation()[i]),
                               new Bind(lowBound[i], column.type.asServerType()));
             }
 
-            if (select.upperBoundRelation()[i] != null)
+            if (highBound != null && select.upperBoundRelation()[i] != null)
             {
                 builder.where(new Symbol(column.name, column.type.asServerType()),
                               toInequality(select.upperBoundRelation()[i]),
@@ -93,7 +130,7 @@ public class SelectHelper
             }
         }
 
-        if (select.orderBy() == Operations.ClusteringOrderBy.DESC)
+        if (select.orderBy() == ClusteringOrderBy.DESC)
         {
             for (int i = 0; i < schema.clusteringKeys.size(); i++)
             {
@@ -102,17 +139,32 @@ public class SelectHelper
             }
         }
 
-        return toCompiled(builder.build());
+        CompiledStatement compiled = toCompiled(builder.build());
+        {
+            Object[] pk = generators.pkGen().inflate(select.pd());
+            ByteBuffer[] pkBuffers = new ByteBuffer[pk.length];
+            for (int i = 0; i < schema.partitionKeys.size(); i++)
+            {
+                ColumnSpec<?> column = schema.partitionKeys.get(i);
+                Object value = pk[i];
+                pkBuffers[i] = ((AbstractType) column.type.asServerType()).decompose(value);
+            }
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
-    public static CompiledStatement select(Operations.SelectCustom select, SchemaSpec schema)
+    public static CompiledStatement select(Operations.SelectCustom select,
+                                           SchemaSpec schema,
+                                           ValueGenerators<Object[], Object[]> generators)
     {
-        Select.Builder builder = commmonPart(select, schema);
+        Select.Builder builder = commmonPart(select, schema, generators);
 
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = generators.forPd(select.pd);
         Map<Long, Object[]> cache = new HashMap<>();
         for (Relations.Relation relation : select.ckRelations())
         {
-            Object[] query = cache.computeIfAbsent(relation.descriptor, schema.valueGenerators.ckGen()::inflate);
+            Object[] query = cache.computeIfAbsent(relation.descriptor, valueGenerators.ckGen()::inflate);
             ColumnSpec<?> column = schema.clusteringKeys.get(relation.column);
             builder.where(new Symbol(column.name, column.type.asServerType()),
                           toInequality(relation.kind),
@@ -122,7 +174,7 @@ public class SelectHelper
         for (Relations.Relation relation : select.regularRelations())
         {
             ColumnSpec<?> column = schema.regularColumns.get(relation.column);
-            Object query = schema.valueGenerators.regularColumnGen(relation.column).inflate(relation.descriptor);
+            Object query = valueGenerators.regularColumnGen(relation.column).inflate(relation.descriptor);
             builder.where(new Symbol(column.name, column.type.asServerType()),
                           toInequality(relation.kind),
                           new Bind(query, column.type.asServerType()));
@@ -130,14 +182,14 @@ public class SelectHelper
 
         for (Relations.Relation relation : select.staticRelations())
         {
-            Object query = schema.valueGenerators.staticColumnGen(relation.column).inflate(relation.descriptor);
+            Object query = valueGenerators.staticColumnGen(relation.column).inflate(relation.descriptor);
             ColumnSpec<?> column = schema.staticColumns.get(relation.column);
             builder.where(new Symbol(column.name, column.type.asServerType()),
                           toInequality(relation.kind),
                           new Bind(query, column.type.asServerType()));
         }
 
-        if (select.orderBy() == Operations.ClusteringOrderBy.DESC)
+        if (select.orderBy() == ClusteringOrderBy.DESC)
         {
             for (int i = 0; i < schema.clusteringKeys.size(); i++)
             {
@@ -148,14 +200,28 @@ public class SelectHelper
 
         builder.allowFiltering();
 
-        return toCompiled(builder.build());
+        CompiledStatement compiled = toCompiled(builder.build());
+        {
+            Object[] pk = generators.pkGen().inflate(select.pd());
+            ByteBuffer[] pkBuffers = new ByteBuffer[pk.length];
+            for (int i = 0; i < schema.partitionKeys.size(); i++)
+            {
+                ColumnSpec<?> column = schema.partitionKeys.get(i);
+                Object value = pk[i];
+                pkBuffers[i] = ((AbstractType) column.type.asServerType()).decompose(value);
+            }
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
-    public static Select.Builder commmonPart(Operations.SelectStatement select, SchemaSpec schema)
+    public static Select.Builder commmonPart(Operations.SelectStatement select,
+                                             SchemaSpec schema,
+                                             ValueGenerators<Object[], Object[]> valueGenerators)
     {
         Select.Builder builder = new Select.Builder();
 
-        Operations.Selection selection = Operations.Selection.fromBitSet(select.selection(), schema);
+        Selection selection = Selection.fromBitSet(select.selection(), schema);
         if (selection.isWildcard())
         {
             builder.wildcard();
@@ -191,7 +257,7 @@ public class SelectHelper
 
         builder.table(schema.keyspace, schema.table);
 
-        Object[] pk = schema.valueGenerators.pkGen().inflate(select.pd());
+        Object[] pk = valueGenerators.pkGen().inflate(select.pd());
         for (int i = 0; i < schema.partitionKeys.size(); i++)
         {
             ColumnSpec<?> column = schema.partitionKeys.get(i);
@@ -235,7 +301,7 @@ public class SelectHelper
         // Select does not add ';' by default, but CompiledStatement expects this
         String cql = select.toCQL(CQLFormatter.None.instance) + ';';
         Object[] bindingsArr = select.binds();
-        return new CompiledStatement(cql, bindingsArr);
+        return new CompiledStatement(true, cql, bindingsArr);
     }
 
 }

test/harry/main/org/apache/cassandra/harry/cql/WriteHelper.java

@@ -18,30 +18,34 @@
 
 package org.apache.cassandra.harry.cql;
 
+import java.nio.ByteBuffer;
 import java.util.List;
 
 import accord.utils.Invariants;
-
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.harry.ColumnSpec;
 import org.apache.cassandra.harry.MagicConstants;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.op.Operations;
 
 public class WriteHelper
 {
     public static CompiledStatement inflateInsert(Operations.WriteOp op,
                                                   SchemaSpec schema,
+                                                  ValueGenerators<Object[], Object[]> gens,
                                                   long timestamp)
     {
         assert op.vds().length == schema.regularColumns.size();
         assert op.sds().length == schema.staticColumns.size();
-        assert op.vds().length == schema.valueGenerators.regularColumnCount();
-        assert op.sds().length == schema.valueGenerators.staticColumnCount();
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = gens.forPd(op.pd);
+        assert op.vds().length == valueGenerators.regularColumnCount();
+        assert op.sds().length == valueGenerators.staticColumnCount();
 
-        Object[] partitionKey = schema.valueGenerators.pkGen().inflate(op.pd());
+        Object[] partitionKey = gens.pkGen().inflate(op.pd());
         assert partitionKey.length == schema.partitionKeys.size();
-        Object[] clusteringKey = schema.valueGenerators.ckGen().inflate(op.cd());
+        Object[] clusteringKey = valueGenerators.ckGen().inflate(op.cd());
         assert clusteringKey.length == schema.clusteringKeys.size();
         Object[] regularColumns = new Object[op.vds().length];
         Object[] staticColumns = new Object[op.sds().length];
@@ -52,7 +56,7 @@ public class WriteHelper
             if (descriptor == MagicConstants.UNSET_DESCR)
                 regularColumns[i] = MagicConstants.UNSET_VALUE;
             else
-                regularColumns[i] = schema.valueGenerators.regularColumnGen(i).inflate(descriptor);
+                regularColumns[i] = valueGenerators.regularColumnGen(i).inflate(descriptor);
         }
 
         for (int i = 0; i < op.sds().length; i++)
@@ -61,7 +65,7 @@ public class WriteHelper
             if (descriptor == MagicConstants.UNSET_DESCR)
                 staticColumns[i] = MagicConstants.UNSET_VALUE;
             else
-                staticColumns[i] = schema.valueGenerators.staticColumnGen(i).inflate(descriptor);
+                staticColumns[i] = valueGenerators.staticColumnGen(i).inflate(descriptor);
         }
 
         Object[] bindings = new Object[schema.allColumnInSelectOrder.size()];
@@ -96,7 +100,15 @@ public class WriteHelper
         }
 
         b.append(";");
-        return new CompiledStatement(b.toString(), adjustArraySize(bindings, bindingsCount));
+
+        CompiledStatement compiled = new CompiledStatement(false, b.toString(), adjustArraySize(bindings, bindingsCount));
+        {
+            ByteBuffer[] pkBuffers = new ByteBuffer[partitionKey.length];
+            for (int i = 0; i < partitionKey.length; i++)
+                pkBuffers[i] = ((AbstractType)schema.partitionKeys.get(i).type.asServerType()).decompose(partitionKey[i]);
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
     public static Object[] adjustArraySize(Object[] bindings, int bindingsCount)
@@ -112,25 +124,28 @@ public class WriteHelper
 
     public static CompiledStatement inflateUpdate(Operations.WriteOp op,
                                                   SchemaSpec schema,
+                                                  ValueGenerators<Object[], Object[]> gens,
                                                   long timestamp)
     {
         assert op.vds().length == schema.regularColumns.size();
         assert op.sds().length == schema.staticColumns.size();
-        assert op.vds().length == schema.valueGenerators.regularColumnCount();
-        assert op.sds().length == schema.valueGenerators.staticColumnCount();
 
-        Object[] partitionKey = schema.valueGenerators.pkGen().inflate(op.pd);
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = gens.forPd(op.pd);
+        assert op.vds().length == valueGenerators.regularColumnCount();
+        assert op.sds().length == valueGenerators.staticColumnCount();
+
+        Object[] partitionKey = gens.pkGen().inflate(op.pd);
         assert partitionKey.length == schema.partitionKeys.size();
-        Object[] clusteringKey = schema.valueGenerators.ckGen().inflate(op.cd());
+        Object[] clusteringKey = valueGenerators.ckGen().inflate(op.cd());
         assert clusteringKey.length == schema.clusteringKeys.size();
         Object[] regularColumns = new Object[op.vds().length];
         Object[] staticColumns = new Object[op.sds().length];
 
         for (int i = 0; i < op.vds().length; i++)
-            regularColumns[i] = schema.valueGenerators.regularColumnGen(i).inflate(op.vds()[i]);
+            regularColumns[i] = valueGenerators.regularColumnGen(i).inflate(op.vds()[i]);
 
         for (int i = 0; i < op.sds().length; i++)
-            staticColumns[i] = schema.valueGenerators.staticColumnGen(i).inflate(op.sds()[i]);
+            staticColumns[i] = valueGenerators.staticColumnGen(i).inflate(op.sds()[i]);
 
         Object[] bindings = new Object[schema.allColumnInSelectOrder.size()];
 
@@ -143,10 +158,11 @@ public class WriteHelper
         if (timestamp != -1 && schema.options.addWriteTimestamps())
         {
             b.append(" USING TIMESTAMP ")
-             .append(timestamp)
-             .append(" SET ");
+             .append(timestamp);
         }
 
+        b.append(" SET ");
+
         int bindingsCount = 0;
         bindingsCount += addSetStatements(b, bindings, schema.regularColumns, regularColumns, bindingsCount);
         if (staticColumns.length != 0)
@@ -158,7 +174,15 @@ public class WriteHelper
         bindingsCount += addWhereStatements(b, bindings, schema.partitionKeys, partitionKey, bindingsCount, true);
         bindingsCount += addWhereStatements(b, bindings, schema.clusteringKeys, clusteringKey, bindingsCount, false);
         b.append(";");
-        return new CompiledStatement(b.toString(), adjustArraySize(bindings, bindingsCount));
+
+        CompiledStatement compiled = new CompiledStatement(false, b.toString(), adjustArraySize(bindings, bindingsCount));
+        {
+            ByteBuffer[] pkBuffers = new ByteBuffer[partitionKey.length];
+            for (int i = 0; i < partitionKey.length; i++)
+                pkBuffers[i] = ((AbstractType)schema.partitionKeys.get(i).type.asServerType()).decompose(partitionKey[i]);
+            compiled.setPk(pkBuffers);
+        }
+        return compiled;
     }
 
     private static int addSetStatements(StringBuilder b,

test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilder.java

@@ -27,16 +27,16 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import accord.utils.Invariants;
 import org.apache.cassandra.harry.ColumnSpec;
 import org.apache.cassandra.harry.MagicConstants;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.gen.Bijections;
 import org.apache.cassandra.harry.gen.EntropySource;
-import org.apache.cassandra.harry.gen.IndexGenerators;
 import org.apache.cassandra.harry.gen.InvertibleGenerator;
-import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.gen.rng.JdkRandomEntropySource;
 import org.apache.cassandra.harry.model.Model;
+import org.apache.cassandra.harry.op.ClusteringOrderBy;
 import org.apache.cassandra.harry.op.Operations;
 import org.apache.cassandra.harry.op.Visit;
 import org.apache.cassandra.harry.util.BitSet;
@@ -47,33 +47,25 @@ import static org.apache.cassandra.harry.SchemaSpec.forKeys;
 import static org.apache.cassandra.harry.gen.InvertibleGenerator.fromType;
 
 // TODO: either create or replay timestamps out of order
-public class HistoryBuilder implements SingleOperationBuilder, Model.Replay
+public class HistoryBuilder implements SingleOperationBuilder, Model.FullReplay
 {
     protected final IndexedValueGenerators valueGenerators;
-    protected final IndexGenerators indexGenerators;
 
     protected int nextOpIdx = 0;
 
-    // TODO: would be great to have a very simple B-Tree here
-    protected final Map<Long, Visit> log;
+    // TODO: would be great to have a very simple B-Tree here, potentially append-only
+    protected final ArrayList<Visit> log;
 
     public static HistoryBuilder fromSchema(SchemaSpec schemaSpec, long seed, int population)
     {
         IndexedValueGenerators generators = valueGenerators(schemaSpec, seed, population);
-        return new HistoryBuilder(generators, IndexGenerators.withDefaults(generators));
+        return new HistoryBuilder(generators);
     }
 
-    public HistoryBuilder(ValueGenerators generators)
+    public HistoryBuilder(IndexedValueGenerators valueGenerators)
     {
-        this((IndexedValueGenerators) generators, IndexGenerators.withDefaults(generators));
-    }
-
-    public HistoryBuilder(IndexedValueGenerators valueGenerators,
-                          IndexGenerators indexGenerators)
-    {
-        this.log = new HashMap<>();
+        this.log = new ArrayList<>(1024);
         this.valueGenerators = valueGenerators;
-        this.indexGenerators = indexGenerators;
     }
 
     public IndexedValueGenerators valueGenerators()
@@ -86,12 +78,11 @@ public class HistoryBuilder implements SingleOperationBuilder, Model.Replay
         return log.size();
     }
 
-    @Override
     public Iterator<Visit> iterator()
     {
         return new Iterator<>()
         {
-            long replayed = 0;
+            int replayed = 0;
 
             public boolean hasNext()
             {
@@ -105,25 +96,13 @@ public class HistoryBuilder implements SingleOperationBuilder, Model.Replay
         };
     }
 
-    @Override
-    public Visit replay(long lts)
-    {
-        return log.get(lts);
-    }
-
-    @Override
-    public Operations.Operation replay(long lts, int opId)
-    {
-        return replay(lts).operations[opId];
-    }
-
     SingleOperationVisitBuilder singleOpVisitBuilder()
     {
         long visitLts = nextOpIdx++;
+        Invariants.require(visitLts == log.size());
         return new SingleOperationVisitBuilder(visitLts,
                                                valueGenerators,
-                                               indexGenerators,
-                                               (visit) -> log.put(visit.lts, visit));
+                                               log::add);
     }
 
     ;
@@ -133,8 +112,7 @@ public class HistoryBuilder implements SingleOperationBuilder, Model.Replay
         long visitLts = nextOpIdx++;
         return new MultiOperationVisitBuilder(visitLts,
                                               valueGenerators,
-                                              indexGenerators,
-                                              visit -> log.put(visit.lts, visit));
+                                              log::add);
     }
 
     @Override
@@ -240,7 +218,7 @@ public class HistoryBuilder implements SingleOperationBuilder, Model.Replay
     }
 
     @Override
-    public SingleOperationBuilder selectPartition(int pdIdx, Operations.ClusteringOrderBy orderBy)
+    public SingleOperationBuilder selectPartition(int pdIdx, ClusteringOrderBy orderBy)
     {
         singleOpVisitBuilder().selectPartition(pdIdx, orderBy);
         return this;
@@ -313,21 +291,23 @@ public class HistoryBuilder implements SingleOperationBuilder, Model.Replay
      */
     public interface IndexedBijection<T> extends Bijections.Bijection<T>
     {
-        int idxFor(long descriptor);
+        long idxFor(long descriptor);
 
-        long descriptorAt(int idx);
+        long descriptorAt(long idx);
 
         @Override
         default String toString(long pd)
         {
             if (pd == MagicConstants.UNSET_DESCR)
-                return Integer.toString(MagicConstants.UNSET_IDX);
+                return Long.toString(MagicConstants.UNSET_IDX);
 
             if (pd == MagicConstants.NIL_DESCR)
-                return Integer.toString(MagicConstants.NIL_IDX);
+                return Long.toString(MagicConstants.NIL_IDX);
 
-            return Integer.toString(idxFor(pd));
+            return Long.toString(idxFor(pd));
         }
+
+        default void discard(){}
     }
 
     public static IndexedValueGenerators valueGenerators(SchemaSpec schema, long seed)
@@ -358,64 +338,20 @@ public class HistoryBuilder implements SingleOperationBuilder, Model.Replay
             map.computeIfAbsent(column, (a) -> (InvertibleGenerator<Object>) fromType(rng, populationPerColumn, column));
 
         // TODO: empty gen
-        return new IndexedValueGenerators(new InvertibleGenerator<>(rng, cumulativeEntropy(schema.partitionKeys), populationPerColumn, forKeys(schema.partitionKeys), keyComparator(schema.partitionKeys)),
-                                          new InvertibleGenerator<>(rng, cumulativeEntropy(schema.clusteringKeys), populationPerColumn, forKeys(schema.clusteringKeys), keyComparator(schema.clusteringKeys)),
-                                          schema.regularColumns.stream()
-                                                               .map(map::get)
-                                                               .collect(Collectors.toList()),
-                                          schema.staticColumns.stream()
-                                                              .map(map::get)
-                                                              .collect(Collectors.toList()),
-                                          pkComparators,
-                                          ckComparators,
-                                          regularComparators,
-                                          staticComparators);
+        return new IndexedValueGenerators.Shared(new InvertibleGenerator<>(rng, cumulativeEntropy(schema.partitionKeys), populationPerColumn, forKeys(schema.partitionKeys), keyComparator(schema.partitionKeys)),
+                                                 new InvertibleGenerator<>(rng, cumulativeEntropy(schema.clusteringKeys), populationPerColumn, forKeys(schema.clusteringKeys), keyComparator(schema.clusteringKeys)),
+                                                 schema.regularColumns.stream()
+                                                                      .map(map::get)
+                                                                      .collect(Collectors.toList()),
+                                                 schema.staticColumns.stream()
+                                                                     .map(map::get)
+                                                                     .collect(Collectors.toList()),
+                                                 ckComparators,
+                                                 regularComparators,
+                                                 staticComparators);
     }
 
-    public static class IndexedValueGenerators extends ValueGenerators<Object[], Object[]>
-    {
-        public IndexedValueGenerators(IndexedBijection<Object[]> pkGen,
-                                      IndexedBijection<Object[]> ckGen,
-                                      List<IndexedBijection<Object>> regularColumnGens,
-                                      List<IndexedBijection<Object>> staticColumnGens,
-                                      List<Comparator<Object>> pkComparators,
-                                      List<Comparator<Object>> ckComparators,
-                                      List<Comparator<Object>> regularComparators,
-                                      List<Comparator<Object>> staticComparators)
-        {
-            super(pkGen, ckGen, ArrayAccessor.instance,
-                  (List<Bijections.Bijection<Object>>) (List<?>) regularColumnGens,
-                  (List<Bijections.Bijection<Object>>) (List<?>) staticColumnGens,
-                  pkComparators, ckComparators, regularComparators, staticComparators);
-        }
-
-        @Override
-        public IndexedBijection<Object[]> pkGen()
-        {
-            return (IndexedBijection<Object[]>) super.pkGen();
-        }
-
-        @Override
-        public IndexedBijection<Object[]> ckGen()
-        {
-            return (IndexedBijection<Object[]>) super.ckGen();
-        }
-
-        @Override
-        public IndexedBijection<Object> regularColumnGen(int idx)
-        {
-            return (IndexedBijection<Object>) super.regularColumnGen(idx);
-        }
-
-        @Override
-        public IndexedBijection<Object> staticColumnGen(int idx)
-        {
-            return (IndexedBijection<Object>) super.staticColumnGen(idx);
-        }
-    }
-
-
-    private static Comparator<Object[]> keyComparator(List<ColumnSpec<?>> columns)
+    public static Comparator<Object[]> keyComparator(List<ColumnSpec<?>> columns)
     {
         return (o1, o2) -> compareKeys(columns, o1, o2);
     }

test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilderHelper.java

@@ -49,41 +49,32 @@ public class HistoryBuilderHelper
     /**
      * Perform a random insert to any row
      */
-    public static void insertRandomData(SchemaSpec schema, Generator<Integer> pkGen, Generator<Integer> ckGen, EntropySource rng, HistoryBuilder history)
-    {
-        insertRandomData(schema, pkGen.generate(rng), ckGen.generate(rng), rng, history);
-    }
+    // TODO: bring back
+//    public static void insertRandomData(SchemaSpec schema, Generator<Integer> pkGen, Generator<Integer> ckGen, EntropySource rng, HistoryBuilder history)
+//    {
+//        insertRandomData(schema, pkGen.generate(rng), ckGen.generate(rng), rng, history);
+//    }
 
-    public static void insertRandomData(SchemaSpec schema, int partitionIdx, int rowIdx, EntropySource rng, HistoryBuilder history)
+//    public static void insertRandomData(SchemaSpec schema, int partitionIdx, int rowIdx, EntropySource rng, HistoryBuilder history)
+//    {
+//        int[] vIdxs = new int[schema.regularColumns.size()];
+//        for (int i = 0; i < schema.regularColumns.size(); i++)
+//            vIdxs[i] = rng.nextInt(history.valueGenerators().regularPopulation(i));
+//        int[] sIdxs = new int[schema.staticColumns.size()];
+//        for (int i = 0; i < schema.staticColumns.size(); i++)
+//            sIdxs[i] = rng.nextInt(history.valueGenerators().staticPopulation(i));
+//        history.insert(partitionIdx, rowIdx, vIdxs, sIdxs);
+//    }
+
+    public static void insertRandomData(SchemaSpec schema, int pdIdx, int rowIdx, EntropySource rng, double chanceOfUnset, HistoryBuilder history)
     {
         int[] vIdxs = new int[schema.regularColumns.size()];
         for (int i = 0; i < schema.regularColumns.size(); i++)
-            vIdxs[i] = rng.nextInt(history.valueGenerators().regularPopulation(i));
+            vIdxs[i] = rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(history.valueGenerators().forPdIdx(pdIdx).regularPopulation(i));
         int[] sIdxs = new int[schema.staticColumns.size()];
         for (int i = 0; i < schema.staticColumns.size(); i++)
-            sIdxs[i] = rng.nextInt(history.valueGenerators().staticPopulation(i));
-        history.insert(partitionIdx, rowIdx, vIdxs, sIdxs);
-    }
-
-    public static void insertRandomData(SchemaSpec schema, int pkIdx, EntropySource rng, HistoryBuilder history)
-    {
-        insertRandomData(schema,
-                         pkIdx,
-                         rng.nextInt(0, history.valueGenerators().ckPopulation()),
-                         rng,
-                         0,
-                         history);
-    }
-
-    public static void insertRandomData(SchemaSpec schema, int partitionIdx, int rowIdx, EntropySource rng, double chanceOfUnset, HistoryBuilder history)
-    {
-        int[] vIdxs = new int[schema.regularColumns.size()];
-        for (int i = 0; i < schema.regularColumns.size(); i++)
-            vIdxs[i] = rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(history.valueGenerators().regularPopulation(i));
-        int[] sIdxs = new int[schema.staticColumns.size()];
-        for (int i = 0; i < schema.staticColumns.size(); i++)
-            sIdxs[i] = rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(history.valueGenerators().staticPopulation(i));
-        history.insert(partitionIdx, rowIdx, vIdxs, sIdxs);
+            sIdxs[i] = rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(history.valueGenerators().forPdIdx(pdIdx).staticPopulation(i));
+        history.insert(pdIdx, rowIdx, vIdxs, sIdxs);
     }
 
     public static void deleteRandomColumns(SchemaSpec schema, int partitionIdx, int rowIdx, EntropySource rng, SingleOperationBuilder history)

test/harry/main/org/apache/cassandra/harry/dsl/IndexedValueGenerators.java

@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.dsl;
+
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.cassandra.harry.gen.IndexGenerators;
+import org.apache.cassandra.harry.gen.ValueGenerators;
+import org.apache.cassandra.harry.gen.Bijections;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.Generators;
+
+public abstract class IndexedValueGenerators extends ValueGenerators<Object[], Object[]>
+{
+    private final Generator<Long> pkIdxGen;
+
+    public IndexedValueGenerators(HistoryBuilder.IndexedBijection<Object[]> pkGen)
+    {
+        super(pkGen);
+        this.pkIdxGen = Generators.int64(0, pkGen().population());
+    }
+
+    public IndexedPartitionValues forPdIdx(long pdIdx)
+    {
+        return forPd(pkGen().descriptorAt(pdIdx));
+    }
+
+    @Override
+    public HistoryBuilder.IndexedBijection<Object[]> pkGen()
+    {
+        return (HistoryBuilder.IndexedBijection<Object[]>) super.pkGen();
+    }
+
+    @Override
+    public abstract IndexedPartitionValues forPd(long pd);
+
+    public Generator<Long> pkIdxGen()
+    {
+        return pkIdxGen;
+    }
+
+    /**
+     * Indexed value generators, for which values that can be generated inside each partition are shared
+     */
+    public static class Shared extends IndexedValueGenerators
+    {
+        private final IndexedPartitionValues partitionValues;
+
+        public Shared(HistoryBuilder.IndexedBijection<Object[]> pkGen,
+                      HistoryBuilder.IndexedBijection<Object[]> ckGen,
+                      List<HistoryBuilder.IndexedBijection<Object>> regularColumnGens,
+                      List<HistoryBuilder.IndexedBijection<Object>> staticColumnGens,
+                      List<Comparator<Object>> ckComparators,
+                      List<Comparator<Object>> regularComparators,
+                      List<Comparator<Object>> staticComparators)
+        {
+            super(pkGen);
+            this.partitionValues = IndexedPartitionValues.uniform(ckGen,
+                                                                  regularColumnGens,
+                                                                  staticColumnGens,
+                                                                  ckComparators,
+                                                                  regularComparators,
+                                                                  staticComparators);
+        }
+
+        @Override
+        public IndexedPartitionValues forPd(long pd)
+        {
+            return partitionValues;
+        }
+    }
+
+    public static class IndexedPartitionValues extends PartitionValues<Object[]>
+    {
+        public final Generator<Integer> ckIdxGen;
+        public final Generator<Integer>[] regularIdxGens;
+        public final Generator<Integer>[] staticIdxGens;
+
+        /**
+         * Index partition value wrapper with random value pickers
+         */
+        public static IndexedPartitionValues uniform(Bijections.Bijection<Object[]> ckGen,
+                                                     List<? extends Bijections.Bijection<?>> regularColumnGens,
+                                                     List<? extends Bijections.Bijection<?>> staticColumnGens,
+                                                     List<Comparator<Object>> ckComparators,
+                                                     List<Comparator<Object>> regularComparators,
+                                                     List<Comparator<Object>> staticComparators)
+        {
+            return new IndexedPartitionValues(ckGen,
+
+                                              regularColumnGens,
+                                              staticColumnGens,
+                                              ckComparators,
+                                              regularComparators,
+                                              staticComparators,
+
+                                              IndexGenerators.uniform(ckGen),
+                                              IndexGenerators.uniform(regularColumnGens),
+                                              IndexGenerators.uniform(regularColumnGens));
+        }
+
+        public IndexedPartitionValues(Bijections.Bijection<Object[]> ckGen,
+
+                                      List<? extends Bijections.Bijection<?>> regularColumnGens,
+                                      List<? extends Bijections.Bijection<?>> staticColumnGens,
+                                      List<Comparator<Object>> ckComparators,
+                                      List<Comparator<Object>> regularComparators,
+                                      List<Comparator<Object>> staticComparators,
+
+                                      Generator<Integer> ckIdxGen,
+                                      Generator<Integer>[] regularIdxGens,
+                                      Generator<Integer>[] staticIdxGens)
+        {
+            super(ckGen, ArrayAccessor.instance, regularColumnGens, staticColumnGens, ckComparators, regularComparators, staticComparators);
+
+            this.ckIdxGen = ckIdxGen;
+            this.regularIdxGens = regularIdxGens;
+            this.staticIdxGens = staticIdxGens;
+        }
+
+        @Override
+        public HistoryBuilder.IndexedBijection<Object[]> ckGen()
+        {
+            return (HistoryBuilder.IndexedBijection<Object[]>) super.ckGen();
+        }
+
+        @Override
+        public HistoryBuilder.IndexedBijection<Object> regularColumnGen(int idx)
+        {
+            return (HistoryBuilder.IndexedBijection<Object>) super.regularColumnGen(idx);
+        }
+
+        @Override
+        public HistoryBuilder.IndexedBijection<Object> staticColumnGen(int idx)
+        {
+            return (HistoryBuilder.IndexedBijection<Object>) super.staticColumnGen(idx);
+        }
+
+        public Generator<Integer> ckIdxGen()
+        {
+            return ckIdxGen;
+        }
+        public Generator<Integer>[] regularIdxGens()
+        {
+            return regularIdxGens;
+        }
+
+        public Generator<Integer>[] staticIdxGens()
+        {
+            return staticIdxGens;
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/dsl/MultiOperationVisitBuilder.java

@@ -21,17 +21,14 @@ package org.apache.cassandra.harry.dsl;
 import java.io.Closeable;
 import java.util.function.Consumer;
 
-import org.apache.cassandra.harry.gen.IndexGenerators;
 import org.apache.cassandra.harry.op.Visit;
 import org.apache.cassandra.harry.util.BitSet;
 
-import static org.apache.cassandra.harry.dsl.HistoryBuilder.IndexedValueGenerators;
-
 public class MultiOperationVisitBuilder extends SingleOperationVisitBuilder implements Closeable
 {
-    MultiOperationVisitBuilder(long lts, IndexedValueGenerators valueGenerators, IndexGenerators indexGenerators, Consumer<Visit> appendToLog)
+    MultiOperationVisitBuilder(long lts, IndexedValueGenerators valueGenerators, Consumer<Visit> appendToLog)
     {
-        super(lts, valueGenerators, indexGenerators, appendToLog);
+        super(lts, valueGenerators, appendToLog);
     }
 
     @Override

test/harry/main/org/apache/cassandra/harry/dsl/ReplayingHistoryBuilder.java

@@ -20,28 +20,28 @@ package org.apache.cassandra.harry.dsl;
 
 import java.util.function.Function;
 
+import accord.utils.Invariants;
 import org.apache.cassandra.harry.execution.CQLVisitExecutor;
-import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.op.Visit;
 
 public class ReplayingHistoryBuilder extends HistoryBuilder
 {
     private final CQLVisitExecutor executor;
 
-    public ReplayingHistoryBuilder(ValueGenerators generators, Function<HistoryBuilder, CQLVisitExecutor> executorFactory)
+    public ReplayingHistoryBuilder(IndexedValueGenerators generators, Function<HistoryBuilder, CQLVisitExecutor> executorFactory)
     {
-        super((IndexedValueGenerators) generators);
+        super(generators);
         this.executor = executorFactory.apply(this);
-     }
+    }
 
     SingleOperationVisitBuilder singleOpVisitBuilder()
     {
         long visitLts = nextOpIdx++;
         HistoryBuilder this_ = this;
+        Invariants.require(visitLts == log.size());
         return new SingleOperationVisitBuilder(visitLts,
                                                valueGenerators,
-                                               indexGenerators,
-                                               (visit) -> log.put(visit.lts, visit)) {
+                                               log::add) {
             @Override
             Visit build()
             {
@@ -59,8 +59,7 @@ public class ReplayingHistoryBuilder extends HistoryBuilder
         HistoryBuilder this_ = this;
         return new MultiOperationVisitBuilder(visitLts,
                                               valueGenerators,
-                                              indexGenerators,
-                                              visit -> log.put(visit.lts, visit)) {
+                                              log::add) {
             @Override
             Visit buildInternal()
             {

test/harry/main/org/apache/cassandra/harry/dsl/SingleOperationBuilder.java

@@ -19,6 +19,7 @@
 package org.apache.cassandra.harry.dsl;
 
 import org.apache.cassandra.harry.Relations;
+import org.apache.cassandra.harry.op.ClusteringOrderBy;
 import org.apache.cassandra.harry.op.Operations;
 import org.apache.cassandra.harry.util.BitSet;
 import org.apache.cassandra.harry.util.ThrowingRunnable;
@@ -46,7 +47,7 @@ public interface SingleOperationBuilder
     SingleOperationBuilder selectRowRange(int pdIdx, int lowerBoundRowIdx, int upperBoundRowIdx,
                                           int nonEqFrom, boolean includeLowBound, boolean includeHighBound);
     SingleOperationBuilder selectPartition(int pdIdx);
-    SingleOperationBuilder selectPartition(int pdIdx, Operations.ClusteringOrderBy orderBy);
+    SingleOperationBuilder selectPartition(int pdIdx, ClusteringOrderBy orderBy);
     SingleOperationBuilder selectRow(int pdIdx, int cdIdx);
     SingleOperationBuilder selectRowSliceByLowerBound(int pdIdx, int lowerBoundRowIdx, int nonEqFrom, boolean isEq);
     SingleOperationBuilder selectRowSliceByUpperBound(int pdIdx, int upperBoundRowIdx, int nonEqFrom, boolean isEq);

test/harry/main/org/apache/cassandra/harry/dsl/SingleOperationVisitBuilder.java

@@ -27,23 +27,23 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import accord.utils.Invariants;
-
 import org.apache.cassandra.harry.MagicConstants;
 import org.apache.cassandra.harry.Relations;
-import org.apache.cassandra.harry.gen.IndexGenerators;
 import org.apache.cassandra.harry.gen.rng.PCGFastPure;
 import org.apache.cassandra.harry.gen.rng.PureRng;
 import org.apache.cassandra.harry.gen.rng.SeedableEntropySource;
+import org.apache.cassandra.harry.op.ClusteringOrderBy;
+import org.apache.cassandra.harry.op.Kind;
 import org.apache.cassandra.harry.op.Operations;
 import org.apache.cassandra.harry.op.Visit;
 import org.apache.cassandra.harry.util.BitSet;
 
-import static org.apache.cassandra.harry.dsl.HistoryBuilder.IndexedValueGenerators;
-import static org.apache.cassandra.harry.op.Operations.Kind;
+import static org.apache.cassandra.harry.dsl.IndexedValueGenerators.IndexedPartitionValues;
 import static org.apache.cassandra.harry.op.Operations.Operation;
 import static org.apache.cassandra.harry.op.Operations.WriteOp;
 
-class SingleOperationVisitBuilder implements SingleOperationBuilder
+// TODO: make package-private again
+public class SingleOperationVisitBuilder implements SingleOperationBuilder
 {
     private static final Logger logger = LoggerFactory.getLogger(SingleOperationVisitBuilder.class);
 
@@ -59,11 +59,9 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     protected int opIdCounter;
 
     protected final IndexedValueGenerators valueGenerators;
-    protected final IndexGenerators indexGenerators;
 
     SingleOperationVisitBuilder(long lts,
                                 IndexedValueGenerators valueGenerators,
-                                IndexGenerators indexGenerators,
                                 Consumer<Visit> appendToLog)
     {
         this.operations = new ArrayList<>();
@@ -73,7 +71,6 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
         this.opIdCounter = 0;
 
         this.valueGenerators = valueGenerators;
-        this.indexGenerators = indexGenerators;
 
         this.seedSelector = new PureRng.PCGFast(lts);
     }
@@ -84,7 +81,8 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
         int opId = opIdCounter++;
         long seed = PCGFastPure.shuffle(PCGFastPure.advanceState(lts, opId, lts));
         return rngSupplier.computeWithSeed(seed, rng -> {
-            int pdIdx = indexGenerators.pkIdxGen.generate(rng);
+            long pdIdx = valueGenerators.pkIdxGen().generate(rng);
+            IndexedPartitionValues indexGenerators = valueGenerators.forPdIdx(pdIdx);
             int cdIdx = indexGenerators.ckIdxGen.generate(rng);
 
             int[] valueIdxs = new int[indexGenerators.regularIdxGens.length];
@@ -93,7 +91,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
             int[] sValueIdxs = new int[indexGenerators.staticIdxGens.length];
             for (int i = 0; i < sValueIdxs.length; i++)
                 sValueIdxs[i] = indexGenerators.staticIdxGens[i].generate(rng);
-            return insert(pdIdx, cdIdx, valueIdxs, sValueIdxs);
+            return write(pdIdx, cdIdx, valueIdxs, sValueIdxs, Kind.INSERT);
         });
     }
 
@@ -102,6 +100,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     {
         int opId = opIdCounter++;
         long seed = PCGFastPure.shuffle(PCGFastPure.advanceState(lts, opId, lts));
+        IndexedPartitionValues indexGenerators = valueGenerators.forPdIdx(pdIdx);
         return rngSupplier.computeWithSeed(seed, rng -> {
             int cdIdx = indexGenerators.ckIdxGen.generate(rng);
 
@@ -120,6 +119,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     {
         int opId = opIdCounter++;
         long seed = PCGFastPure.shuffle(PCGFastPure.advanceState(lts, opId, lts));
+        IndexedPartitionValues indexGenerators = valueGenerators.forPdIdx(pdIdx);
         return rngSupplier.computeWithSeed(seed, rng -> {
             int[] valueIdxs = new int[indexGenerators.regularIdxGens.length];
             for (int i = 0; i < valueIdxs.length; i++)
@@ -166,7 +166,8 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
         int opId = opIdCounter++;
         long seed = PCGFastPure.shuffle(PCGFastPure.advanceState(lts, opId, lts));
         return rngSupplier.computeWithSeed(seed, rng -> {
-            int pdIdx = indexGenerators.pkIdxGen.generate(rng);
+            long pdIdx = valueGenerators.pkIdxGen().generate(rng);
+            IndexedPartitionValues indexGenerators = valueGenerators.forPdIdx(pdIdx);
             int cdIdx = indexGenerators.ckIdxGen.generate(rng);
 
             int[] valueIdxs = new int[indexGenerators.regularIdxGens.length];
@@ -175,7 +176,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
             int[] sValueIdxs = new int[indexGenerators.staticIdxGens.length];
             for (int i = 0; i < sValueIdxs.length; i++)
                 sValueIdxs[i] = indexGenerators.staticIdxGens[i].generate(rng);
-            return update(pdIdx, cdIdx, valueIdxs, sValueIdxs);
+            return write(pdIdx, cdIdx, valueIdxs, sValueIdxs, Kind.UPDATE);
         });
     }
 
@@ -184,6 +185,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     {
         int opId = opIdCounter++;
         long seed = PCGFastPure.shuffle(PCGFastPure.advanceState(lts, opId, lts));
+        IndexedPartitionValues indexGenerators = valueGenerators.forPdIdx(pdIdx);
         return rngSupplier.computeWithSeed(seed, rng -> {
             int cdIdx = indexGenerators.ckIdxGen.generate(rng);
 
@@ -202,6 +204,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     {
         int opId = opIdCounter++;
         long seed = PCGFastPure.shuffle(PCGFastPure.advanceState(lts, opId, lts));
+        IndexedPartitionValues indexGenerators = valueGenerators.forPdIdx(pdIdx);
         return rngSupplier.computeWithSeed(seed, rng -> {
             int[] valueIdxs = new int[indexGenerators.regularIdxGens.length];
             for (int i = 0; i < valueIdxs.length; i++)
@@ -219,15 +222,25 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
         return write(pdIdx, cdIdx, valueIdxs, sValueIdxs, Kind.UPDATE);
     }
 
-    private SingleOperationBuilder write(int pdIdx, int cdIdx, int[] valueIdxs, int[] sValueIdxs, Kind kind)
+    public SingleOperationBuilder write(long pdIdx, int cdIdx, int[] valueIdxs, int[] sValueIdxs, Kind kind)
+    {
+        WriteOp op = write(valueGenerators.pkGen(), valueGenerators.forPdIdx(pdIdx), lts, pdIdx, cdIdx, valueIdxs, sValueIdxs, kind);
+        opIdCounter++;
+        operations.add(op);
+        build();
+        return this;
+    }
+
+    // TODO: extract
+    public static WriteOp write(HistoryBuilder.IndexedBijection<Object[]> pkGen, IndexedPartitionValues valueGenerators,
+                                long lts, long pdIdx, int cdIdx, int[] valueIdxs, int[] sValueIdxs, Kind kind)
     {
         assert valueIdxs.length == valueGenerators.regularColumnCount();
         assert sValueIdxs.length == valueGenerators.staticColumnCount();
 
-        long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
+        long pd = pkGen.descriptorAt(pdIdx);
         long cd = valueGenerators.ckGen().descriptorAt(cdIdx);
 
-        opIdCounter++;
         long[] vds = new long[valueIdxs.length];
         for (int i = 0; i < valueGenerators.regularColumnCount(); i++)
         {
@@ -248,24 +261,23 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
                 sds[i] = valueGenerators.staticColumnGen(i).descriptorAt(valueIdx);
         }
 
-        operations.add(new WriteOp(lts, pd, cd, vds, sds, kind) {
+        return new WriteOp(lts, pd, cd, vds, sds, kind) {
             @Override
             public String toString()
             {
                 return String.format("%s (%d, %d, %s, %s)",
                                      kind, pdIdx, cdIdx, Arrays.toString(valueIdxs), Arrays.toString(sValueIdxs));
             }
-        });
-        build();
-        return this;
+        };
     }
 
     @Override
     public SingleOperationVisitBuilder deleteRowRange(int pdIdx, int lowerBoundRowIdx, int upperBoundRowIdx,
                                                       int nonEqFrom, boolean includeLowerBound, boolean includeUpperBound)
     {
-        long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
+        long pd = this.valueGenerators.pkGen().descriptorAt(pdIdx);
 
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         long lowerBoundCd = valueGenerators.ckGen().descriptorAt(lowerBoundRowIdx);
         long upperBoundCd = valueGenerators.ckGen().descriptorAt(upperBoundRowIdx);
 
@@ -325,6 +337,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     {
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
         opIdCounter++;
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         long cd = valueGenerators.ckGen().descriptorAt(rowIdx);
         operations.add(new Operations.DeleteRow(lts, pd, cd) {
             @Override
@@ -342,6 +355,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     {
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
         opIdCounter++;
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         long cd = valueGenerators.ckGen().descriptorAt(rowIdx);
         operations.add(new Operations.DeleteColumns(lts, pd, cd, regularSelection, staticSelection)  {
             @Override
@@ -358,6 +372,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     public SingleOperationBuilder deleteRowSliceByLowerBound(int pdIdx, int lowerBoundRowIdx, int nonEqFrom, boolean includeBound)
     {
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         long lowerBoundCd = valueGenerators.ckGen().descriptorAt(lowerBoundRowIdx);
 
         Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()];
@@ -391,6 +406,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     public SingleOperationBuilder deleteRowSliceByUpperBound(int pdIdx, int upperBoundRowIdx, int nonEqFrom, boolean includeBound)
     {
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         long upperBoundCd = valueGenerators.ckGen().descriptorAt(upperBoundRowIdx);
 
         Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()];
@@ -426,6 +442,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
                                                       int nonEqFrom, boolean includeLowerBound, boolean includeUpperBound)
     {
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         long lowerBoundCd = valueGenerators.ckGen().descriptorAt(lowerBoundRowIdx);
         long upperBoundCd = valueGenerators.ckGen().descriptorAt(upperBoundRowIdx);
 
@@ -459,6 +476,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
         opIdCounter++;
 
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         Relations.Relation[] ckRelations = new Relations.Relation[ckIdxRelations.length];
         for (int i = 0; i < ckRelations.length; i++)
         {
@@ -503,7 +521,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     }
 
     @Override
-    public SingleOperationBuilder selectPartition(int pdIdx, Operations.ClusteringOrderBy orderBy)
+    public SingleOperationBuilder selectPartition(int pdIdx, ClusteringOrderBy orderBy)
     {
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
         opIdCounter++;
@@ -518,6 +536,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     {
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
         opIdCounter++;
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         long cd = valueGenerators.ckGen().descriptorAt(rowIdx);
         operations.add(new Operations.SelectRow(lts, pd, cd));
         build();
@@ -528,6 +547,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     public SingleOperationBuilder selectRowSliceByLowerBound(int pdIdx, int lowerBoundRowIdx, int nonEqFrom, boolean includeBound)
     {
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         long lowerBoundCd = valueGenerators.ckGen().descriptorAt(lowerBoundRowIdx);
 
         Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()];
@@ -554,6 +574,7 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder
     public SingleOperationBuilder selectRowSliceByUpperBound(int pdIdx, int upperBoundRowIdx, int nonEqFrom, boolean includeBound)
     {
         long pd = valueGenerators.pkGen().descriptorAt(pdIdx);
+        IndexedPartitionValues valueGenerators = this.valueGenerators.forPd(pd);
         long upperBoundCd = valueGenerators.ckGen().descriptorAt(upperBoundRowIdx);
 
         Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()];

test/harry/main/org/apache/cassandra/harry/execution/CQLTesterVisitExecutor.java

@@ -30,8 +30,10 @@ import accord.utils.Invariants;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.harry.ColumnSpec;
 import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.model.Model;
 import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Selection;
 import org.apache.cassandra.harry.op.Visit;
 
 import static org.apache.cassandra.harry.MagicConstants.LTS_UNKNOWN;
@@ -43,15 +45,20 @@ import static org.apache.cassandra.harry.MagicConstants.UNSET_DESCR;
 public class CQLTesterVisitExecutor extends CQLVisitExecutor
 {
     private static final Logger logger = LoggerFactory.getLogger(CQLTesterVisitExecutor.class);
+
     private final Function<CompiledStatement, UntypedResultSet> execute;
+    private final ValueGenerators<Object[], Object[]> valueGenerators;
 
     public CQLTesterVisitExecutor(SchemaSpec schema,
+                                  ValueGenerators<Object[], Object[]> valueGenerators,
                                   DataTracker dataTracker,
                                   Model model,
                                   Function<CompiledStatement, UntypedResultSet> execute)
     {
-        super(schema, dataTracker, model, new QueryBuildingVisitExecutor(schema, QueryBuildingVisitExecutor.WrapQueries.UNLOGGED_BATCH));
+        super(schema, dataTracker, model,
+              new QueryBuildingVisitExecutor(schema, QueryBuildingVisitExecutor.WrapQueries.UNLOGGED_BATCH, valueGenerators));
         this.execute = execute;
+        this.valueGenerators = valueGenerators;
     }
 
     @Override
@@ -60,8 +67,9 @@ public class CQLTesterVisitExecutor extends CQLVisitExecutor
         List<ResultSetRow> actual = new ArrayList<>();
         // TODO: Have never tested with multiple
         Invariants.require(visit.operations.length == 1);
+        // TODO: for now, cql tester only supports "global" value gens
         for (UntypedResultSet.Row row : execute.apply(statement))
-            actual.add(resultSetToRow(schema, (Operations.SelectStatement) visit.operations[0], row));
+            actual.add(resultSetToRow(schema, valueGenerators, (Operations.SelectStatement) visit.operations[0], row));
         return actual;
     }
 
@@ -70,10 +78,12 @@ public class CQLTesterVisitExecutor extends CQLVisitExecutor
         execute.apply(statement);
     }
 
-    public static ResultSetRow resultSetToRow(SchemaSpec schema, Operations.SelectStatement select, UntypedResultSet.Row row)
+    public static ResultSetRow resultSetToRow(SchemaSpec schema,
+                                              ValueGenerators<Object[], Object[]> generators,
+                                              Operations.SelectStatement select, UntypedResultSet.Row row)
     {
         // TODO: do we want to use selection?
-        Operations.Selection selection = Operations.Selection.fromBitSet(select.selection(), schema);
+        Selection selection = Selection.fromBitSet(select.selection(), schema);
 
         long pd = UNKNOWN_DESCR;
         if (selection.selectsAllOf(schema.partitionKeys))
@@ -85,10 +95,10 @@ public class CQLTesterVisitExecutor extends CQLVisitExecutor
                 partitionKey[i] = column.type.asServerType().compose(row.getBytes(column.name));
             }
 
-            pd = schema.valueGenerators.pkGen().deflate(partitionKey);
+            pd = generators.pkGen().deflate(partitionKey);
         }
 
-
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = Invariants.nonNull(generators.forPd(pd));
         long cd = UNKNOWN_DESCR;
         if (selection.selectsAllOf(schema.clusteringKeys))
         {
@@ -116,7 +126,7 @@ public class CQLTesterVisitExecutor extends CQLVisitExecutor
             if (clusteringKey == NIL_KEY)
                 cd = UNSET_DESCR;
             else
-                cd = schema.valueGenerators.ckGen().deflate(clusteringKey);
+                cd = valueGenerators.ckGen().deflate(clusteringKey);
         }
 
         long[] regularColumns = new long[schema.regularColumns.size()];
@@ -128,7 +138,7 @@ public class CQLTesterVisitExecutor extends CQLVisitExecutor
                 if (row.has(column.name))
                 {
                     Object value = column.type.asServerType().compose(row.getBytes(column.name));
-                    regularColumns[i] = schema.valueGenerators.regularColumnGen(i).deflate(value);
+                    regularColumns[i] = valueGenerators.regularColumnGen(i).deflate(value);
                 }
                 else
                 {
@@ -150,7 +160,7 @@ public class CQLTesterVisitExecutor extends CQLVisitExecutor
                 if (row.has(column.name))
                 {
                     Object value = column.type.asServerType().compose(row.getBytes(column.name));
-                    staticColumns[i] = schema.valueGenerators.staticColumnGen(i).deflate(value);
+                    staticColumns[i] = valueGenerators.staticColumnGen(i).deflate(value);
                 }
                 else
                 {

test/harry/main/org/apache/cassandra/harry/execution/CQLVisitExecutor.java

@@ -19,6 +19,7 @@
 package org.apache.cassandra.harry.execution;
 
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -30,13 +31,10 @@ import accord.utils.Invariants;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.model.Model;
+import org.apache.cassandra.harry.op.Kind;
 import org.apache.cassandra.harry.op.Operations;
 import org.apache.cassandra.harry.op.Visit;
 
-/**
- *
- * TODO: Transactional results ; LET
- */
 public abstract class CQLVisitExecutor
 {
     private static final Logger logger = LoggerFactory.getLogger(QueryBuildingVisitExecutor.class);
@@ -54,7 +52,7 @@ public abstract class CQLVisitExecutor
         this.queryBuilder = queryBuilder;
     }
 
-    public static void replay(CQLVisitExecutor executor, Model.Replay replay)
+    public static void replay(CQLVisitExecutor executor, Model.FullReplay replay)
     {
         for (Visit visit : replay)
         {
@@ -64,7 +62,7 @@ public abstract class CQLVisitExecutor
         }
     }
 
-    public static void executeVisit(Visit visit, CQLVisitExecutor executor, Model.Replay replay)
+    public static void executeVisit(Visit visit, CQLVisitExecutor executor, Model.FullReplay replay)
     {
         try
         {
@@ -88,7 +86,7 @@ public abstract class CQLVisitExecutor
         LAST_50
     }
 
-    public static void replayAfterFailure(Visit visit, CQLVisitExecutor executor, Model.Replay replay)
+    public static void replayAfterFailure(Visit visit, CQLVisitExecutor executor, Model.FullReplay replay)
     {
         QueryBuildingVisitExecutor queryBuilder = executor.queryBuilder;
         if (!visit.hasCustom)
@@ -123,9 +121,12 @@ public abstract class CQLVisitExecutor
         }
     }
 
-    private static boolean intersects(Set<?> set1, Set<?> set2)
+    private static boolean intersects(long[] v1, long[] v2)
     {
-        for (Object o : set1)
+        Set<Long> set2 = new HashSet<>();
+        for (Long l : v2)
+            set2.add(l);
+        for (long o : v1)
         {
             if (set2.contains(o))
                 return true;
@@ -136,25 +137,31 @@ public abstract class CQLVisitExecutor
     public void execute(Visit visit)
     {
         dataTracker.begin(visit);
-        QueryBuildingVisitExecutor.BuiltQuery compiledStatement = queryBuilder.compile(visit);
-        // All operations are not touching any data
-        if (compiledStatement == null)
+        try
         {
-            Invariants.requireArgument(Arrays.stream(visit.operations).allMatch(op -> op.kind() == Operations.Kind.CUSTOM));
-            return;
-        }
+            QueryBuildingVisitExecutor.BuiltQuery compiledStatement = queryBuilder.compile(visit);
+            // All operations are not touching any data
+            if (compiledStatement == null)
+            {
+                Invariants.requireArgument(Arrays.stream(visit.operations).allMatch(op -> op.kind() == Kind.CUSTOM));
+                return;
+            }
 
-        List<Operations.SelectStatement> selects = compiledStatement.selects;
-        if (selects.isEmpty())
-        {
-            executeMutatingVisit(visit, compiledStatement);
+            List<Operations.SelectStatement> selects = compiledStatement.selects;
+            if (selects.isEmpty())
+            {
+                executeMutatingVisit(visit, compiledStatement);
+            }
+            else
+            {
+                Invariants.require(selects.size() == 1);
+                executeValidatingVisit(visit, selects, compiledStatement);
+            }
         }
-        else
+        finally
         {
-            Invariants.require(selects.size() == 1);
-            executeValidatingVisit(visit, selects, compiledStatement);
+            dataTracker.end(visit);
         }
-        dataTracker.end(visit);
     }
 
     // Lives in a separate method so that it is easier to override it

test/harry/main/org/apache/cassandra/harry/execution/CompiledStatement.java

@@ -18,18 +18,22 @@
 
 package org.apache.cassandra.harry.execution;
 
-import java.net.InetAddress;
-import java.util.UUID;
-
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
 public class CompiledStatement
 {
+    public final boolean validating;
     private final String cql;
     private final Object[] bindings;
+    private ByteBuffer[] pk;
 
-    public CompiledStatement(String cql, Object... bindings)
+    public CompiledStatement(boolean validating, String cql, Object... bindings)
     {
+        this.validating = validating;
         this.cql = cql;
         this.bindings = bindings;
     }
@@ -41,28 +45,35 @@ public class CompiledStatement
 
     public CompiledStatement withSchema(String oldKs, String oldTable, String newKs, String newTable)
     {
-        return new CompiledStatement(cql.replace(oldKs + "." + oldTable,
-                                                 newKs + "." + newTable),
-                                     bindings);
+        CompiledStatement statement = new CompiledStatement(validating, cql.replace(oldKs + "." + oldTable,
+                newKs + "." + newTable),
+                bindings);
+        statement.setPk(pk);
+        return statement;
     }
 
     public CompiledStatement withFiltering()
     {
-        return new CompiledStatement(cql.replace(";",
+        return new CompiledStatement(validating, cql.replace(";",
                                                  " ALLOW FILTERING;"),
                                      bindings);
     }
 
+    public void setPk(ByteBuffer[] pk)
+    {
+        this.pk = pk;
+    }
+
+    public ByteBuffer[] pk()
+    {
+        return pk;
+    }
+
     public Object[] bindings()
     {
         return bindings;
     }
 
-    public static CompiledStatement create(String cql, Object... bindings)
-    {
-        return new CompiledStatement(cql, bindings);
-    }
-
     public String toString()
     {
         return "CompiledStatement{" +

test/harry/main/org/apache/cassandra/harry/execution/DataTracker.java

@@ -20,24 +20,26 @@ package org.apache.cassandra.harry.execution;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
 import accord.utils.Invariants;
-
 import org.apache.cassandra.harry.model.Model;
-import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Kind;
 import org.apache.cassandra.harry.op.Visit;
 
-import static org.apache.cassandra.harry.op.Operations.Kind.CUSTOM;
-import static org.apache.cassandra.harry.op.Operations.Kind.SELECT_CUSTOM;
-import static org.apache.cassandra.harry.op.Operations.Kind.SELECT_PARTITION;
-import static org.apache.cassandra.harry.op.Operations.Kind.SELECT_RANGE;
-import static org.apache.cassandra.harry.op.Operations.Kind.SELECT_ROW;
+import static org.apache.cassandra.harry.op.Kind.CUSTOM;
+import static org.apache.cassandra.harry.op.Kind.SELECT_CUSTOM;
+import static org.apache.cassandra.harry.op.Kind.SELECT_PARTITION;
+import static org.apache.cassandra.harry.op.Kind.SELECT_RANGE;
+import static org.apache.cassandra.harry.op.Kind.SELECT_ROW;
+import static org.apache.cassandra.harry.op.Operations.Operation;
+import static org.apache.cassandra.harry.op.Operations.PartitionOperation;
 
 /**
  * Data tracker tracks every operation that was started and finished.
@@ -50,22 +52,21 @@ public interface DataTracker
 {
     void begin(Visit visit);
     void end(Visit visit);
+    default void gc(long pd) {}
 
-    Iterable<Model.LtsOperationPair> potentialVisits(long pd);
-    boolean isFinished(long lts);
-    boolean allFinished();
-
-    Set<Operations.Kind> OPS_WITHOUT_EFFECT = Set.of(SELECT_CUSTOM, SELECT_PARTITION, SELECT_ROW, SELECT_RANGE, CUSTOM);
+    Set<Kind> OPS_WITHOUT_EFFECT = Set.of(SELECT_CUSTOM, SELECT_PARTITION, SELECT_ROW, SELECT_RANGE, CUSTOM);
 
     /**
-     * Data tracker that only allows partition visits to be done _in sequence_
+     * Data tracker that only allows partition visits to be done _in sequence_.
+     *
+     * This data tracker is _not_ thread safe.
      */
-    class SequentialDataTracker implements DataTracker
+    class SequentialDataTracker implements DataTracker, Model.PartialReplay
     {
         private final AtomicLong started = new AtomicLong();
         private final AtomicLong finished = new AtomicLong();
 
-        private Map<Long, List<Model.LtsOperationPair>> partitionVisits = new HashMap<>();
+        private Map<Long, List<Operation>> partitionVisits = new HashMap<>();
 
         public void begin(Visit visit)
         {
@@ -74,15 +75,15 @@ public interface DataTracker
             started.set(visit.lts);
             for (int i = 0; i < visit.operations.length; i++)
             {
-                Operations.Operation operation = visit.operations[i];
+                Operation operation = visit.operations[i];
 
                 // SELECT statements have no effect on the model
                 if (OPS_WITHOUT_EFFECT.contains(operation.kind()))
                     continue;
 
-                Operations.PartitionOperation partitionOp = (Operations.PartitionOperation) operation;
+                PartitionOperation partitionOp = (PartitionOperation) operation;
                 partitionVisits.computeIfAbsent(partitionOp.pd, pd_ -> new ArrayList<>())
-                               .add(new Model.LtsOperationPair(visit.lts, i));
+                               .add(operation);
             }
         }
 
@@ -93,79 +94,91 @@ public interface DataTracker
             finished.set(visit.lts);
         }
 
-        public Iterable<Model.LtsOperationPair> potentialVisits(long pd)
+        @Override
+        public Iterable<Operation> potentialVisits(long pd)
         {
-            Iterable<Model.LtsOperationPair> res = partitionVisits.get(pd);
+            Iterable<Operation> res = partitionVisits.get(pd);
             if (res != null)
                 return res;
 
             return Collections.emptyList();
         }
+    }
 
-        public boolean isFinished(long lts)
+    public static interface ReplayingDataTracker extends DataTracker, Model.PartialReplay {}
+
+    public static class NoOpDataTracker implements ReplayingDataTracker
+    {
+
+        @Override
+        public void begin(Visit visit)
         {
-            return finished.get() >= lts;
         }
 
         @Override
-        public boolean allFinished()
+        public void end(Visit visit)
         {
-            return started.get() == finished.get();
+        }
+
+        @Override
+        public Iterable<Operation> potentialVisits(long pd)
+        {
+            return Collections.emptyList();
         }
     }
-
-    // TODO: optimize for sequential accesses
-
     /**
-     * Data tracker able to track LTS out of order
+     * Data tracker able to track LTS out of order.
+     *
+     * Intended to be used either in a single-threaded environment, or in conjuction with a locking/concurrent tracker
      */
-    class SimpleDataTracker implements DataTracker
+    // TODO: optimize for sequential accesses
+    class SimpleDataTracker implements ReplayingDataTracker
     {
-        private final Set<Long> started = new HashSet<>();
-        private final Set<Long> finished = new HashSet<>();
-
-        private Map<Long, List<Model.LtsOperationPair>> partitionVisits = new HashMap<>();
+        // WARNING: you can access partitions concurrently, but make sure to use locking tracker to guard the op list
+        private Map<Long, List<Operation>> partitionVisits = new ConcurrentHashMap<>();
 
         public void begin(Visit visit)
         {
-            started.add(visit.lts);
             for (int i = 0; i < visit.operations.length; i++)
             {
-                Operations.Operation operation = visit.operations[i];
+                Operation operation = visit.operations[i];
 
                 // SELECT statements have no effect on the model
                 if (OPS_WITHOUT_EFFECT.contains(operation.kind()))
                     continue;
 
-                Operations.PartitionOperation partitionOp = (Operations.PartitionOperation) operation;
+                PartitionOperation partitionOp = (PartitionOperation) operation;
                 partitionVisits.computeIfAbsent(partitionOp.pd, pd_ -> new ArrayList<>())
-                               .add(new Model.LtsOperationPair(visit.lts, i));
+                               .add(operation);
             }
         }
 
+        @Override
+        public void gc(long pd)
+        {
+            partitionVisits.remove(pd);
+        }
+
         public void end(Visit visit)
         {
-            finished.add(visit.lts);
-        }
-
-        public Iterable<Model.LtsOperationPair> potentialVisits(long pd)
-        {
-            Iterable<Model.LtsOperationPair> res = partitionVisits.get(pd);
-            if (res != null)
-                return res;
-
-            return Collections.emptyList();
-        }
-
-        public boolean isFinished(long lts)
-        {
-            return finished.contains(lts);
         }
 
         @Override
-        public boolean allFinished()
+        public Iterable<Operation> potentialVisits(long pd)
         {
-            return started.size() == finished.size();
+            List<Operation> res = partitionVisits.get(pd);
+            if (res == null)
+                return Collections.emptyList();
+
+            // TODO: this won't hold for Accord or Paxos, so we will need to also have a separate wall clock
+            //       tracker for operations.
+            // Operations are appended in begin() order, which under concurrent execution is not logical-timestamp
+            // order. The quiescent checker requires them grouped and applied in increasing lts (matching the DB's
+            // last-write-wins by USING TIMESTAMP lts), so return an lts-sorted copy. The caller holds the partition
+            // read lock, so no writer is appending to the underlying list while we copy it.
+            List<Operation> sorted = new ArrayList<>(res);
+            sorted.sort(Comparator.comparingLong(Operation::lts));
+            return sorted;
         }
     }
 }

test/harry/main/org/apache/cassandra/harry/execution/InJvmDTestVisitExecutor.java

@@ -37,9 +37,12 @@ import org.apache.cassandra.exceptions.RequestTimeoutException;
 import org.apache.cassandra.harry.ColumnSpec;
 import org.apache.cassandra.harry.MagicConstants;
 import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
+import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.model.Model;
 import org.apache.cassandra.harry.model.QuiescentChecker;
 import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Selection;
 import org.apache.cassandra.harry.op.Visit;
 import org.apache.cassandra.utils.AssertionUtils;
 
@@ -61,18 +64,24 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
     protected final PageSizeSelector pageSizeSelector;
     protected final RetryPolicy retryPolicy;
 
-    protected InJvmDTestVisitExecutor(SchemaSpec schema,
-                                      DataTracker dataTracker,
-                                      Model model,
-                                      ICluster<?> cluster,
+    protected final ValueGenerators<Object[], Object[]> valueGenerators;
 
-                                      NodeSelector nodeSelector,
-                                      PageSizeSelector pageSizeSelector,
-                                      RetryPolicy retryPolicy,
-                                      ConsistencyLevelSelector consistencyLevel,
-                                      WrapQueries wrapQueries)
+    public InJvmDTestVisitExecutor(SchemaSpec schema,
+
+                                   ValueGenerators<Object[], Object[]> valueGenerators,
+                                   DataTracker dataTracker,
+                                   Model model,
+                                   ICluster<?> cluster,
+
+                                   NodeSelector nodeSelector,
+                                   PageSizeSelector pageSizeSelector,
+                                   RetryPolicy retryPolicy,
+                                   ConsistencyLevelSelector consistencyLevel,
+                                   WrapQueries wrapQueries)
     {
-        super(schema, dataTracker, model, new QueryBuildingVisitExecutor(schema, wrapQueries));
+        super(schema, dataTracker, model, new QueryBuildingVisitExecutor(schema, wrapQueries, valueGenerators));
+
+        this.valueGenerators = valueGenerators;
         this.cluster = cluster;
         this.consistencyLevel = consistencyLevel;
 
@@ -121,7 +130,7 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
         if (logger.isTraceEnabled())
             logger.trace("{} returned {} results", statement, rows.length);
 
-        return rowsToResultSet(schema, (Operations.SelectStatement) visit.operations[0], rows);
+        return rowsToResultSet(schema, valueGenerators, (Operations.SelectStatement) visit.operations[0], rows);
     }
 
     protected static Object[][] iterToArr(Iterator<Object[]> iter)
@@ -160,15 +169,20 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
             cluster.coordinator(node).execute(statement.cql(), consistencyLevel, statement.bindings());
     }
 
-    public static List<ResultSetRow> rowsToResultSet(SchemaSpec schema, Operations.SelectStatement select, Object[][] result)
+    public static List<ResultSetRow> rowsToResultSet(SchemaSpec schema,
+                                                     ValueGenerators<Object[], Object[]> generators,
+                                                     Operations.SelectStatement select, Object[][] result)
     {
         List<ResultSetRow> rs = new ArrayList<>();
         for (Object[] res : result)
-            rs.add(rowToResultSet(schema, select, res));
+            rs.add(rowToResultSet(schema, generators, select, res));
         return rs;
     }
 
-    public static ResultSetRow rowToResultSet(SchemaSpec schema, Operations.SelectStatement select, Object[] result)
+    public static ResultSetRow rowToResultSet(SchemaSpec schema,
+                                              ValueGenerators<Object[], Object[]> generators,
+                                              Operations.SelectStatement select,
+                                              Object[] result)
     {
         long[] staticColumns = new long[schema.staticColumns.size()];
         long[] regularColumns = new long[schema.regularColumns.size()];
@@ -179,16 +193,18 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
         long[] regularLts = LTS_UNKNOWN;
 
         long pd = UNKNOWN_DESCR;
-        Operations.Selection selection = Operations.Selection.fromBitSet(select.selection(), schema);
+        Selection selection = Selection.fromBitSet(select.selection(), schema);
+        Invariants.require(selection.selectsAllOf(schema.partitionKeys));
         if (selection.selectsAllOf(schema.partitionKeys))
         {
             Object[] partitionKey = new Object[schema.partitionKeys.size()];
             for (int i = 0; i < schema.partitionKeys.size(); i++)
                 partitionKey[i] = result[selection.indexOf(schema.partitionKeys.get(i))];
 
-            pd = schema.valueGenerators.pkGen().deflate(partitionKey);
+            pd = generators.pkGen().deflate(partitionKey);
         }
 
+        ValueGenerators.PartitionValues<Object[]> valueGenerators = generators.forPd(pd);
         // Deflate logic for clustering key is a bit more involved, since CK can be nil in case of a single static row.
         long cd = UNKNOWN_DESCR;
         if (selection.selectsAllOf(schema.clusteringKeys))
@@ -214,7 +230,7 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
             if (clusteringKey == NIL_KEY)
                 cd = UNSET_DESCR;
             else
-                cd = schema.valueGenerators.ckGen().deflate(clusteringKey);
+                cd = valueGenerators.ckGen().deflate(clusteringKey);
         }
 
         for (int i = 0; i < schema.regularColumns.size(); i++)
@@ -226,7 +242,7 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
                 if (v == null)
                     regularColumns[i] = NIL_DESCR;
                 else
-                    regularColumns[i] = schema.valueGenerators.regularColumnGen(i).deflate(v);
+                    regularColumns[i] = valueGenerators.regularColumnGen(i).deflate(v);
             }
             else
             {
@@ -243,7 +259,7 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
                 if (v == null)
                     staticColumns[i] = NIL_DESCR;
                 else
-                    staticColumns[i] = schema.valueGenerators.staticColumnGen(i).deflate(v);
+                    staticColumns[i] = valueGenerators.staticColumnGen(i).deflate(v);
             }
             else
             {
@@ -371,12 +387,12 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
             }
         }
 
-        public InJvmDTestVisitExecutor build(SchemaSpec schema, Model.Replay replay, ICluster<?> cluster)
+        public InJvmDTestVisitExecutor build(SchemaSpec schema, IndexedValueGenerators valueGenerators, ICluster<?> cluster)
         {
             setDefaults(schema, cluster);
-            DataTracker tracker = new DataTracker.SequentialDataTracker();
-            Model model = new QuiescentChecker(schema.valueGenerators, tracker, replay);
-            return new InJvmDTestVisitExecutor(schema, tracker, model, cluster,
+            DataTracker.SequentialDataTracker tracker = new DataTracker.SequentialDataTracker();
+            Model model = new QuiescentChecker(valueGenerators, tracker);
+            return new InJvmDTestVisitExecutor(schema, valueGenerators, tracker, model, cluster,
                                                nodeSelector, pageSizeSelector, retryPolicy, consistencyLevel, wrapQueries);
         }
 
@@ -389,12 +405,12 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
         /**
          * WARNING: highly experimental
          */
-        public InJvmDTestVisitExecutor doubleWriting(SchemaSpec schema, Model.Replay replay, ICluster<?> cluster, String secondTable)
+        public InJvmDTestVisitExecutor doubleWriting(SchemaSpec schema, IndexedValueGenerators valueGens, ICluster<?> cluster, String secondTable)
         {
             setDefaults(schema, cluster);
-            DataTracker tracker = new DataTracker.SequentialDataTracker();
-            Model model = new QuiescentChecker(schema.valueGenerators, tracker, replay);
-            return new InJvmDTestVisitExecutor(schema, tracker, model, cluster,
+            DataTracker.SequentialDataTracker tracker = new DataTracker.SequentialDataTracker();
+            Model model = new QuiescentChecker(valueGens, tracker);
+            return new InJvmDTestVisitExecutor(schema, valueGens, tracker, model, cluster,
                                                nodeSelector, pageSizeSelector, retryPolicy, consistencyLevel, wrapQueries)
             {
                 @Override
@@ -408,7 +424,7 @@ public class InJvmDTestVisitExecutor extends CQLVisitExecutor
                     {
                         logger.debug("Second opinion: ");
                         for (ResultSetRow resultSetRow : secondOpinion)
-                            logger.debug(resultSetRow.toString(schema.valueGenerators));
+                            logger.debug(resultSetRow.toString(valueGens));
                     }
                     return rows;
                 }

test/harry/main/org/apache/cassandra/harry/execution/LockingDataTracker.java

@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.execution;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.utils.concurrent.WaitQueue;
+
+/**
+ * Locking data tracker, that can be used with a quiescent model checker while providing
+ * a high degree of concurrency. It works by isolating readers from writers. In other words,
+ * readers can intersect with other readers, and writers can coincide with other writers.
+ *
+ * We achieve quiescence on a partition level, not on LTS level, and we know for sure
+ * which operations have finished for a partition, even if their LTS are non-contiguous.
+ *
+ * We use a simple wait queue for queuing up waiters, and a compact long counter for
+ * tracking the number of concurrent readers and writers. Lower 32 bits hold a number of
+ * readers, and higher 32 bits - a number of writers.
+ */
+public class LockingDataTracker implements DataTracker
+{
+    private final Map<Long, ReadersWritersLock> locked = new ConcurrentHashMap<>();
+
+    private final WaitQueue readersQueue = WaitQueue.newWaitQueue();
+    private final WaitQueue writersQueue = WaitQueue.newWaitQueue();
+
+    // TODO: primitive concurrent lock!
+    private Set<Long> readingFrom = new ConcurrentSkipListSet<>();
+    private Set<Long> writingTo = new ConcurrentSkipListSet<>();
+
+    private final DataTracker delegate;
+
+    private final int readConcurrency;
+    private final int writeConcurrency;
+
+    public LockingDataTracker(DataTracker delegate)
+    {
+        // By default, until we have a better/stronger model, we allow however many readers, but only 1 writer at a time
+        this(delegate, Integer.MAX_VALUE, 1);
+    }
+
+    public LockingDataTracker(DataTracker delegate, int readConcurrency, int writeConcurrency)
+    {
+        this.delegate = delegate;
+        this.readConcurrency = readConcurrency;
+        this.writeConcurrency = writeConcurrency;
+    }
+
+    @Override
+    public void begin(Visit visit)
+    {
+        while (true)
+        {
+            int lockedUpTo = -1;
+            // Grab all locks, or release all locks, to avoid deadlocking with other visitors
+            for (int i = 0; i < visit.visitedPartitions.length; i++)
+            {
+                Long pd = visit.visitedPartitions[i];
+                ReadersWritersLock partitionLock = getLock(pd);
+                if (visit.validating())
+                {
+                    if (!partitionLock.tryLockForRead())
+                    {
+//                        System.out.println("Could not lock for read");
+                        break;
+                    }
+                    assert !writingTo.contains(pd) : String.format("Writing to should not have contained %d", pd);
+                    readingFrom.add(pd);
+                }
+                else
+                {
+                    if (!partitionLock.tryLockForWrite())
+                    {
+//                        System.out.println("Could not lock for write");
+                        break;
+                    }
+//                    assert !readingFrom.contains(partitionLock.descriptor) : String.format("Reading from should not have contained %d", partitionLock.descriptor);
+                    writingTo.add(partitionLock.descriptor);
+                }
+
+                lockedUpTo = i;
+            }
+
+            if (lockedUpTo != visit.visitedPartitions.length - 1)
+            {
+                for (int i = 0; i < lockedUpTo; i++)
+                {
+                    Long pd = visit.visitedPartitions[i];
+                    ReadersWritersLock partitionLock = getLock(pd);
+                    if (visit.validating())
+                        partitionLock.unlockAfterRead();
+                    else
+                        partitionLock.unlockAfterWrite();
+                }
+                continue;
+            }
+
+            break;
+        }
+        delegate.begin(visit);
+    }
+
+    @Override
+    public void end(Visit visit)
+    {
+        for (Long pd : visit.visitedPartitions)
+        {
+            ReadersWritersLock partitionLock = getLock(pd);
+            if (visit.validating())
+            {
+                readingFrom.remove(pd);
+                partitionLock.unlockAfterRead();
+            }
+            else
+            {
+                writingTo.remove(pd);
+                partitionLock.unlockAfterWrite();
+            }
+        }
+        delegate.end(visit);
+    }
+
+    private ReadersWritersLock getLock(long pd)
+    {
+        return locked.computeIfAbsent(pd, (pd_) -> new ReadersWritersLock(readersQueue, writersQueue, pd, readConcurrency, writeConcurrency));
+    }
+
+    /**
+     * Readers/writers lock. It was decided not to use signals here, and instead go for a
+     * busyspin instead, since we expect locks to be released briefly and contention to be minimal.
+     */
+    public static class ReadersWritersLock
+    {
+        private static final AtomicLongFieldUpdater<ReadersWritersLock> fieldUpdater = AtomicLongFieldUpdater.newUpdater(ReadersWritersLock.class, "lock");
+        private volatile long lock;
+
+        final long descriptor;
+        // TODO: we do not need to use queues here, just using a signal will suffice
+        final WaitQueue readersQueue;
+        final WaitQueue writersQueue;
+        private final int readConcurrency;
+        private final int writeConcurrency;
+
+        public ReadersWritersLock(WaitQueue readersQueue, WaitQueue writersQueue, long descriptor, int readConcurrency, int writeConcurrency)
+        {
+            this.readersQueue = readersQueue;
+            this.writersQueue = writersQueue;
+            this.lock = 0L;
+            this.descriptor = descriptor;
+            Invariants.require(readConcurrency > 0);
+            this.readConcurrency = readConcurrency;
+            Invariants.require(writeConcurrency > 0);
+            this.writeConcurrency = writeConcurrency;
+        }
+
+        @Override
+        public String toString()
+        {
+            long lock = this.lock;
+            return "PartitionLock{" +
+                   "pd = " + descriptor +
+                   ", readers = " + getReaders(lock) +
+                   ", writers = " + getWriters(lock) +
+                   '}';
+        }
+
+        public void lockForWrite()
+        {
+            while (true)
+            {
+                WaitQueue.Signal signal = writersQueue.register();
+                long v = lock;
+                if (getReaders(v) == 0 && getWriters(v) < writeConcurrency)
+                {
+                    if (fieldUpdater.compareAndSet(this, v, incWriters(v)))
+                    {
+                        signal.cancel();
+                        return;
+                    }
+                }
+                signal.awaitUninterruptibly();
+            }
+        }
+
+        public boolean tryLockForWrite()
+        {
+            long v = lock;
+            if (getReaders(v) == 0 && getWriters(v) < writeConcurrency && fieldUpdater.compareAndSet(this, v, incWriters(v)))
+                return true;
+
+            return false;
+        }
+
+        public void unlockAfterWrite()
+        {
+            while (true)
+            {
+                long v = lock;
+                if (fieldUpdater.compareAndSet(this, v, decWriters(v)))
+                {
+                    readersQueue.signalAll();
+                    writersQueue.signalAll();
+                    return;
+                }
+            }
+        }
+
+        public void lockForRead()
+        {
+            while (true)
+            {
+                WaitQueue.Signal signal = readersQueue.register();
+                long v = lock;
+                if (getWriters(v) == 0 && getReaders(v) < readConcurrency)
+                {
+                    if (fieldUpdater.compareAndSet(this, v, incReaders(v)))
+                    {
+                        signal.cancel();
+                        return;
+                    }
+                }
+                signal.awaitUninterruptibly();
+            }
+        }
+
+        public boolean tryLockForRead()
+        {
+            long v = lock;
+            if (getWriters(v) == 0 && getReaders(v) < readConcurrency && fieldUpdater.compareAndSet(this, v, incReaders(v)))
+                return true;
+
+            return false;
+        }
+
+        public void unlockAfterRead()
+        {
+            while (true)
+            {
+                long v = lock;
+                if (fieldUpdater.compareAndSet(this, v, decReaders(v)))
+                {
+                    writersQueue.signalAll();
+                    readersQueue.signalAll();
+                    return;
+                }
+            }
+        }
+
+        private long incReaders(long v)
+        {
+            long readers = getReaders(v);
+            assert getWriters(v) == 0;
+            v &= ~0x00000000ffffffffL; // erase all readers
+            return v | (readers + 1L);
+        }
+
+        private long decReaders(long v)
+        {
+            long readers = getReaders(v);
+            assert getWriters(v) == 0;
+            assert readers >= 1;
+            v &= ~0x00000000ffffffffL; // erase all readers
+            return v | (readers - 1L);
+        }
+
+        private long incWriters(long v)
+        {
+            long writers = getWriters(v);
+            assert getReaders(v) == 0;
+            v &= ~0xffffffff00000000L; // erase all writers
+            return v | ((writers + 1L) << 32);
+        }
+
+        private long decWriters(long v)
+        {
+            long writers = getWriters(v);
+            assert getReaders(v) == 0;
+            assert writers >= 1 : "Writers left " + writers;
+            v &= ~0xffffffff00000000L; // erase all writers
+            return v | ((writers - 1L) << 32);
+        }
+
+        public int getReaders(long v)
+        {
+            v &= 0xffffffffL;
+            return (int) v;
+        }
+
+        public int getWriters(long v)
+        {
+            v >>= 32;
+            v &= 0xffffffffL;
+            return (int) v;
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Locking" + super.toString();
+    }
+}

test/harry/main/org/apache/cassandra/harry/execution/QueryBuildingVisitExecutor.java

@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.harry.execution;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -33,6 +34,7 @@ import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.cql.DeleteHelper;
 import org.apache.cassandra.harry.cql.SelectHelper;
 import org.apache.cassandra.harry.cql.WriteHelper;
+import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.op.Operations;
 import org.apache.cassandra.harry.op.Visit;
 
@@ -42,11 +44,13 @@ public class QueryBuildingVisitExecutor extends VisitExecutor
     private static final Logger logger = LoggerFactory.getLogger(QueryBuildingVisitExecutor.class);
     protected final SchemaSpec schema;
     protected final WrapQueries wrapQueries;
+    protected final ValueGenerators<Object[], Object[]> valueGenerators;
 
-    public QueryBuildingVisitExecutor(SchemaSpec schema, WrapQueries wrapQueries)
+    public QueryBuildingVisitExecutor(SchemaSpec schema, WrapQueries wrapQueries, ValueGenerators<Object[], Object[]> valueGenerators)
     {
         this.schema = schema;
         this.wrapQueries = wrapQueries;
+        this.valueGenerators = valueGenerators;
     }
 
     public BuiltQuery compile(Visit visit)
@@ -66,9 +70,11 @@ public class QueryBuildingVisitExecutor extends VisitExecutor
         {
             Object[] bindingsArray = new Object[bindings.size()];
             bindings.toArray(bindingsArray);
-            BuiltQuery query = new BuiltQuery(selects,
+            BuiltQuery query = new BuiltQuery(selects, visit.validating,
                                    wrapQueries.wrap(visit, String.join("\n ", statements)),
                                    bindingsArray);
+            if (pks.size() == 1)
+                query.setPk(pks.get(0));
             clear();
             return query;
         }
@@ -80,9 +86,9 @@ public class QueryBuildingVisitExecutor extends VisitExecutor
     public static class BuiltQuery extends CompiledStatement
     {
         protected final List<Operations.SelectStatement> selects;
-        public BuiltQuery(List<Operations.SelectStatement> selects, String cql, Object... bindings)
+        public BuiltQuery(List<Operations.SelectStatement> selects, boolean mutates, String cql, Object... bindings)
         {
-            super(cql, bindings);
+            super(mutates, cql, bindings);
             this.selects = selects;
         }
     }
@@ -93,14 +99,13 @@ public class QueryBuildingVisitExecutor extends VisitExecutor
     private final List<String> statements = new ArrayList<>();
     private final List<Object> bindings = new ArrayList<>();
     private final Set<Long> visitedPds = new HashSet<>();
+    private final List<ByteBuffer[]> pks = new ArrayList<>();
 
     private List<Operations.SelectStatement> selects = null;
 
     protected void beginLts(long lts)
     {
-        statements.clear();
-        bindings.clear();
-        visitedPds.clear();
+        clear();
         selects = new ArrayList<>();
     }
 
@@ -120,9 +125,11 @@ public class QueryBuildingVisitExecutor extends VisitExecutor
         statements.clear();
         bindings.clear();
         visitedPds.clear();
+        pks.clear();
         selects = null;
     }
 
+    @SuppressWarnings("unchecked")
     protected void operation(Operations.Operation operation)
     {
         if (operation instanceof Operations.PartitionOperation)
@@ -131,37 +138,47 @@ public class QueryBuildingVisitExecutor extends VisitExecutor
         switch (operation.kind())
         {
             case UPDATE:
-                statement = WriteHelper.inflateUpdate((Operations.WriteOp) operation, schema, operation.lts());
+            {
+                Operations.WriteOp write = (Operations.WriteOp) operation;
+                statement = WriteHelper.inflateUpdate(write, schema, valueGenerators, operation.lts());
                 break;
+            }
             case INSERT:
-                statement = WriteHelper.inflateInsert((Operations.WriteOp) operation, schema, operation.lts());
+            {
+                Operations.WriteOp write = (Operations.WriteOp) operation;
+                statement = WriteHelper.inflateInsert(write, schema, valueGenerators, operation.lts());
                 break;
+            }
             case DELETE_RANGE:
-                statement = DeleteHelper.inflateDelete((Operations.DeleteRange) operation, schema, operation.lts());
+                // TODO: unroll
+                statement = DeleteHelper.inflateDelete((Operations.DeleteRange) operation, schema, valueGenerators, operation.lts());
                 break;
             case DELETE_PARTITION:
-                statement = DeleteHelper.inflateDelete((Operations.DeletePartition) operation, schema, operation.lts());
+                statement = DeleteHelper.inflateDelete((Operations.DeletePartition) operation, schema, valueGenerators, operation.lts());
                 break;
             case DELETE_ROW:
-                statement = DeleteHelper.inflateDelete((Operations.DeleteRow) operation, schema, operation.lts());
+                statement = DeleteHelper.inflateDelete((Operations.DeleteRow) operation, schema, valueGenerators, operation.lts());
                 break;
             case DELETE_COLUMNS:
-                statement = DeleteHelper.inflateDelete((Operations.DeleteColumns) operation, schema, operation.lts());
+                statement = DeleteHelper.inflateDelete((Operations.DeleteColumns) operation, schema, valueGenerators, operation.lts());
                 break;
             case SELECT_PARTITION:
-                statement = SelectHelper.select((Operations.SelectPartition) operation, schema);
+            {
+                Operations.SelectPartition select = (Operations.SelectPartition) operation;
+                statement = SelectHelper.select(select, schema, valueGenerators);
                 selects.add((Operations.SelectStatement) operation);
                 break;
+            }
             case SELECT_ROW:
-                statement = SelectHelper.select((Operations.SelectRow) operation, schema);
+                statement = SelectHelper.select((Operations.SelectRow) operation, schema, valueGenerators);
                 selects.add((Operations.SelectStatement) operation);
                 break;
             case SELECT_RANGE:
-                statement = SelectHelper.select((Operations.SelectRange) operation, schema);
+                statement = SelectHelper.select((Operations.SelectRange) operation, schema, valueGenerators);
                 selects.add((Operations.SelectStatement) operation);
                 break;
             case SELECT_CUSTOM:
-                statement = SelectHelper.select((Operations.SelectCustom) operation, schema);
+                statement = SelectHelper.select((Operations.SelectCustom) operation, schema, valueGenerators);
                 selects.add((Operations.SelectStatement) operation);
                 break;
 
@@ -172,6 +189,7 @@ public class QueryBuildingVisitExecutor extends VisitExecutor
                 throw new IllegalArgumentException();
         }
         statements.add(statement.cql());
+        pks.add(statement.pk());
         Collections.addAll(bindings, statement.bindings());
     }
 
@@ -193,6 +211,8 @@ public class QueryBuildingVisitExecutor extends VisitExecutor
 
         WrapQueries TRANSACTION = (visit, compiled) -> String.format(wrapInTxnFormat, compiled);
 
+        WrapQueries EMPTY = (visit, compiled) -> compiled;
+
 
         String wrap(Visit visit, String compiled);
     }

test/harry/main/org/apache/cassandra/harry/execution/ResultSetRow.java

@@ -118,10 +118,10 @@ public class ResultSetRow
     {
         return "resultSetRow("
                + valueGenerators.pkGen().toString(pd)
-               + ", " + descrToIdx(valueGenerators.ckGen(), cd) +
-               (sds == null ? "" : ", statics(" + toString(sds, valueGenerators::staticColumnGen) + ")") +
+               + ", " + descrToIdx(valueGenerators.forPd(pd).ckGen(), cd) +
+               (sds == null ? "" : ", statics(" + toString(sds, valueGenerators.forPd(pd)::staticColumnGen) + ")") +
                (slts == null ? "" : ", slts(" + StringUtils.toString(slts) + ")") +
-               ", values(" + toString(vds, valueGenerators::regularColumnGen) + ")" +
+               ", values(" + toString(vds, valueGenerators.forPd(pd)::regularColumnGen) + ")" +
                ", lts(" + StringUtils.toString(lts) + ")" +
                ")";
     }

test/harry/main/org/apache/cassandra/harry/execution/RingAwareInJvmDTestVisitExecutor.java

@@ -33,6 +33,7 @@ import org.apache.cassandra.distributed.api.ICluster;
 import org.apache.cassandra.distributed.api.ICoordinator;
 import org.apache.cassandra.distributed.api.IInstance;
 import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.model.Model;
 import org.apache.cassandra.harry.model.QuiescentChecker;
 import org.apache.cassandra.harry.model.TokenPlacementModel;
@@ -52,6 +53,7 @@ public class RingAwareInJvmDTestVisitExecutor extends InJvmDTestVisitExecutor
     private final TokenPlacementModel.ReplicationFactor rf;
 
     private RingAwareInJvmDTestVisitExecutor(SchemaSpec schema,
+                                             ValueGenerators<Object[], Object[]> valueGenerators,
                                              DataTracker dataTracker,
                                              Model model,
                                              ICluster<?> cluster,
@@ -62,7 +64,7 @@ public class RingAwareInJvmDTestVisitExecutor extends InJvmDTestVisitExecutor
                                              TokenPlacementModel.ReplicationFactor rf,
                                              QueryBuildingVisitExecutor.WrapQueries wrapQueries)
     {
-        super(schema, dataTracker, model, cluster, nodeSelector, pageSizeSelector, retryPolicy, consistencyLevel, QueryBuildingVisitExecutor.WrapQueries.UNLOGGED_BATCH);
+        super(schema, valueGenerators, dataTracker, model, cluster, nodeSelector, pageSizeSelector, retryPolicy, consistencyLevel, wrapQueries);
         this.rf = rf;
     }
 
@@ -86,7 +88,7 @@ public class RingAwareInJvmDTestVisitExecutor extends InJvmDTestVisitExecutor
 
     protected long token(long pd)
     {
-        return TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(schema.valueGenerators.pkGen().inflate(pd))));
+        return TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(valueGenerators.pkGen().inflate(pd))));
     }
 
     @Override
@@ -120,9 +122,10 @@ public class RingAwareInJvmDTestVisitExecutor extends InJvmDTestVisitExecutor
     {
         try
         {
-            Invariants.require(visit.visitedPartitions.size() == 1,
+            Invariants.require(visit.visitedPartitions.length == 1,
                                   "Ring aware executor can only read and write one partition at a time");
-            for (TokenPlacementModel.Replica replica : getReplicasFor(visit.visitedPartitions.iterator().next().longValue()))
+
+            for (TokenPlacementModel.Replica replica : getReplicasFor(visit.visitedPartitions[0]))
             {
                 IInstance instance = cluster
                                      .stream()
@@ -198,17 +201,17 @@ public class RingAwareInJvmDTestVisitExecutor extends InJvmDTestVisitExecutor
             }
         }
 
-        public RingAwareInJvmDTestVisitExecutor build(SchemaSpec schema, Model.Replay replay, ICluster<?> cluster)
+        public RingAwareInJvmDTestVisitExecutor build(SchemaSpec schema, ValueGenerators<Object[], Object[]> valueGenerators, ICluster<?> cluster)
         {
-            DataTracker tracker = new DataTracker.SequentialDataTracker();
-            Model model = new QuiescentChecker(schema.valueGenerators, tracker, replay);
-            return build(schema, tracker, model, cluster);
+            DataTracker.SequentialDataTracker tracker = new DataTracker.SequentialDataTracker();
+            Model model = new QuiescentChecker(valueGenerators, tracker);
+            return build(schema, valueGenerators, tracker, model, cluster);
         }
 
-        public RingAwareInJvmDTestVisitExecutor build(SchemaSpec schema, DataTracker tracker, Model model, ICluster<?> cluster)
+        public RingAwareInJvmDTestVisitExecutor build(SchemaSpec schema, ValueGenerators<Object[], Object[]> valueGenerators, DataTracker tracker, Model model, ICluster<?> cluster)
         {
             setDefaults(schema, cluster);
-            return new RingAwareInJvmDTestVisitExecutor(schema, tracker, model, cluster,
+            return new RingAwareInJvmDTestVisitExecutor(schema, valueGenerators, tracker, model, cluster,
                                                         nodeSelector, pageSizeSelector, retryPolicy, consistencyLevel, rf, wrapQueries);
         }
     }

test/harry/main/org/apache/cassandra/harry/gen/BijectionCache.java

@@ -76,7 +76,7 @@ public class BijectionCache<T> implements Bijections.Bijection<T>
     }
 
     @Override
-    public int population()
+    public long population()
     {
         throw new UnsupportedOperationException();
     }

test/harry/main/org/apache/cassandra/harry/gen/Bijections.java

@@ -53,7 +53,7 @@ public class Bijections
 
         // TODO: byteSize is great, but you know what's better? Bit size! For example, for `boolean`, we only need a single bit.
         int byteSize();
-        default int population()
+        default long population()
         {
             return byteSize() * Byte.SIZE;
         }

test/harry/main/org/apache/cassandra/harry/gen/BytesBijection.java

@@ -0,0 +1,260 @@
+/*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package org.apache.cassandra.harry.gen;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.TreeMap;
+
+import org.apache.cassandra.harry.gen.rng.RngUtils;
+
+/**
+ * A nibble-based bijection for ByteBuffers, similar to {@link StringBijection}.
+ * <p>
+ * Each byte of the descriptor is used as an index into a nibble table of 256 pre-generated
+ * byte[] fragments. The 8 nibbles form a unique prefix that guarantees bijectivity;
+ * additional random bytes are appended for variable-length output.
+ */
+public class BytesBijection implements Bijections.Bijection<ByteBuffer>
+{
+    public static final int NIBBLES_SIZE = 256;
+    private final byte[][] nibbles;
+    private final int nibbleSize;
+    private final int maxRandomBytes;
+
+    public BytesBijection()
+    {
+        this(8, 10);
+    }
+
+    public BytesBijection(int nibbleSize, int maxRandomBytes)
+    {
+        this(generateNibbles(nibbleSize), nibbleSize, maxRandomBytes);
+    }
+
+    public BytesBijection(byte[][] nibbles, int nibbleSize, int maxRandomBytes)
+    {
+        assert nibbles.length == NIBBLES_SIZE;
+        this.nibbles = nibbles;
+        this.nibbleSize = nibbleSize;
+        this.maxRandomBytes = maxRandomBytes;
+
+        for (int i = 0; i < nibbles.length; i++)
+            assert nibbles[i].length == nibbleSize;
+    }
+
+    public ByteBuffer inflate(long descriptor)
+    {
+        // Prefix: 8 nibbles selected by each byte of the descriptor
+        int prefixLen = Long.BYTES * nibbleSize;
+
+        // Determine suffix length
+        int suffixLen = suffixLength(descriptor);
+
+        // Determine extra length (subclasses may override)
+        int extraLen = extraLength(descriptor);
+
+        byte[] result = new byte[prefixLen + suffixLen + extraLen];
+
+        // Write prefix nibbles
+        for (int i = 0; i < Long.BYTES; i++)
+        {
+            int idx = getByte(descriptor, i);
+            System.arraycopy(nibbles[idx], 0, result, i * nibbleSize, nibbleSize);
+        }
+
+        // Append suffix bytes
+        appendSuffix(result, prefixLen, suffixLen, descriptor);
+
+        // Append extra bytes (subclasses may override)
+        appendExtra(result, prefixLen + suffixLen, extraLen, descriptor);
+
+        return ByteBuffer.wrap(result);
+    }
+
+    protected int suffixLength(long descriptor)
+    {
+        long rnd = RngUtils.next(descriptor);
+        return RngUtils.asInt(rnd, 0, maxRandomBytes);
+    }
+
+    protected int extraLength(long descriptor)
+    {
+        return 0;
+    }
+
+    protected void appendSuffix(byte[] result, int offset, int length, long descriptor)
+    {
+        long rnd = RngUtils.next(descriptor);
+        // skip past the rnd used for length
+        rnd = RngUtils.next(rnd);
+
+        int pos = offset;
+        int remaining = length;
+        while (remaining > 0)
+        {
+            rnd = RngUtils.next(rnd);
+            for (int i = 0; i < remaining && i < Long.BYTES; i++)
+            {
+                result[pos++] = (byte) ((rnd >> (i * 8)) & 0xff);
+                remaining--;
+            }
+        }
+    }
+
+    protected void appendExtra(byte[] result, int offset, int length, long descriptor)
+    {
+    }
+
+    public long deflate(ByteBuffer value)
+    {
+        long res = 0;
+        for (int i = 0; i < Long.BYTES; i++)
+        {
+            int idx = findNibble(value, value.position() + i * nibbleSize);
+            long v = idx;
+            if (i == 0)
+                v ^= 0x80;
+            res |= v << (Long.BYTES - i - 1) * Byte.SIZE;
+        }
+        return res;
+    }
+
+    private int findNibble(ByteBuffer value, int offset)
+    {
+        for (int n = 0; n < NIBBLES_SIZE; n++)
+        {
+            boolean match = true;
+            for (int j = 0; j < nibbleSize; j++)
+            {
+                if (value.get(offset + j) != nibbles[n][j])
+                {
+                    match = false;
+                    break;
+                }
+            }
+            if (match)
+                return n;
+        }
+        throw new IllegalArgumentException("No matching nibble found at offset " + offset);
+    }
+
+    public static int getByte(long l, int idx)
+    {
+        int b = (int) ((l >> (Long.BYTES - idx - 1) * Byte.SIZE) & 0xff);
+        if (idx == 0)
+            b ^= 0x80;
+        return b;
+    }
+
+    public int compare(long l, long r)
+    {
+        for (int i = 0; i < Long.BYTES; i++)
+        {
+            int cmp = Integer.compare(getByte(l, i), getByte(r, i));
+            if (cmp != 0)
+                return cmp;
+        }
+        return 0;
+    }
+
+    public int byteSize()
+    {
+        return Long.BYTES;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "bytes(" +
+               "nibbleSize=" + nibbleSize +
+               ", maxRandomBytes=" + maxRandomBytes +
+               ')';
+    }
+
+    /**
+     * Generate 256 unique byte[] nibbles of the given size, sorted lexicographically.
+     * Uses a deterministic RNG so nibbles are stable across runs.
+     */
+    public static byte[][] generateNibbles(int nibbleSize)
+    {
+        TreeMap<ByteArrayWrapper, byte[]> sorted = new TreeMap<>();
+        long seed = 0xdeadbeefcafeL;
+
+        while (sorted.size() < NIBBLES_SIZE)
+        {
+            seed = RngUtils.next(seed);
+            byte[] nibble = new byte[nibbleSize];
+            long s = seed;
+            for (int j = 0; j < nibbleSize; j++)
+            {
+                nibble[j] = (byte) (s & 0xff);
+                s = RngUtils.next(s);
+            }
+            sorted.put(new ByteArrayWrapper(nibble), nibble);
+        }
+
+        byte[][] nibbles = new byte[NIBBLES_SIZE][];
+        int i = 0;
+        for (byte[] nibble : sorted.values())
+            nibbles[i++] = nibble;
+
+        return nibbles;
+    }
+
+    /**
+     * Wrapper for byte[] that provides equals/hashCode/compareTo for use in sorted collections.
+     */
+    private static class ByteArrayWrapper implements Comparable<ByteArrayWrapper>
+    {
+        private final byte[] data;
+
+        ByteArrayWrapper(byte[] data)
+        {
+            this.data = data;
+        }
+
+        @Override
+        public int compareTo(ByteArrayWrapper other)
+        {
+            int len = Math.min(data.length, other.data.length);
+            for (int i = 0; i < len; i++)
+            {
+                int cmp = Integer.compare(data[i] & 0xff, other.data[i] & 0xff);
+                if (cmp != 0)
+                    return cmp;
+            }
+            return Integer.compare(data.length, other.data.length);
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (!(o instanceof ByteArrayWrapper)) return false;
+            return Arrays.equals(data, ((ByteArrayWrapper) o).data);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Arrays.hashCode(data);
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/gen/Generators.java

@@ -25,14 +25,18 @@ import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.UUID;
 import java.util.function.Supplier;
 
 import accord.utils.Invariants;
-
+import org.apache.cassandra.harry.stress.distribution.Distribution;
 import org.apache.cassandra.harry.util.BitSet;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.utils.TimeUUID;
@@ -229,6 +233,19 @@ public class Generators
         return new TrackingGenerator<>(delegate);
     }
 
+    public static Generator<Integer> adaptLongToInt(Generator<Long> orig)
+    {
+        return new Generator<Integer>()
+        {
+            final Generator<Long> longGen = orig;
+            @Override
+            public Integer generate(EntropySource rng)
+            {
+                return Math.toIntExact(longGen.generate(rng));
+            }
+        };
+    }
+
     public static class TrackingGenerator<T> implements Generator<T>
     {
         private final Set<T> generated;
@@ -325,6 +342,60 @@ public class Generators
         }
     }
 
+    public static final class VariableSizeStringGenerator implements Generator<String>
+    {
+        final char[] chars;
+        final Distribution sizeDistribution;
+
+        public VariableSizeStringGenerator(char[] chars, Distribution sizeDistribution)
+        {
+            this.chars = chars;
+            this.sizeDistribution = sizeDistribution;
+        }
+
+        public VariableSizeStringGenerator(int minChar, int maxChar, Distribution sizeDistribution)
+        {
+            int range = maxChar - minChar + 1;
+            this.chars = new char[range];
+            for (int i = 0; i < range; i++)
+                chars[i] = (char) (minChar + i);
+            this.sizeDistribution = sizeDistribution;
+        }
+
+        @Override
+        public String generate(EntropySource rng)
+        {
+            int length = Math.toIntExact(sizeDistribution.next(rng.next()));
+            char[] buf = new char[length];
+            for (int i = 0; i < length; i++)
+                buf[i] = chars[rng.nextInt(chars.length)];
+            return new String(buf);
+        }
+    }
+
+    public static final class VariableSizeByteBufferGenerator implements Generator<ByteBuffer>
+    {
+        final Distribution sizeDistribution;
+
+        public VariableSizeByteBufferGenerator(Distribution sizeDistribution)
+        {
+            this.sizeDistribution = sizeDistribution;
+        }
+
+        @Override
+        public ByteBuffer generate(EntropySource rng)
+        {
+            int size = Math.toIntExact(sizeDistribution.next(rng.next()));
+            byte[] bytes = new byte[size];
+            for (int i = 0; i < size; )
+                for (long v = rng.next(),
+                     n = Math.min(size - i, Long.SIZE / Byte.SIZE);
+                     n-- > 0; v >>= Byte.SIZE)
+                    bytes[i++] = (byte) v;
+            return ByteBuffer.wrap(bytes);
+        }
+    }
+
     public static final class LongGenerator implements Generator<Long>
     {
         @Override
@@ -391,7 +462,7 @@ public class Generators
     {
         if (ts.isEmpty())
             throw new IllegalStateException("Can't pick from an empty list");
-        return (rng) -> ts.get(rng.nextInt(0, ts.size()));
+        return (rng) -> ts.get(rng.nextInt(ts.size()));
     }
 
     public static <T> Generator<T> pick(T... ts)
@@ -466,4 +537,38 @@ public class Generators
     {
         return (random) -> constant.get();
     }
+
+
+    public static <T> Map<T, Integer> normalize(Map<T, Integer> weights)
+    {
+        Map<T, Integer> normalized = new HashMap<>();
+        int sum = 0;
+        for (Integer value : weights.values())
+            sum += value;
+
+        for (T kind : weights.keySet())
+        {
+            double dbl = (sum * ((double) weights.get(kind)) / sum);
+            normalized.put(kind, (int) Math.round(dbl));
+        }
+
+        return normalized;
+    }
+    public static <T> Generator<T> weighted(Map<T, Integer> weights)
+    {
+        NavigableMap<Integer, T> weightMap = weights instanceof NavigableMap ? (TreeMap<Integer, T>) weights : new TreeMap<Integer, T>();
+        int sum = 0;
+        for (Map.Entry<T, Integer> entry : weights.entrySet())
+        {
+            sum += entry.getValue();
+            weightMap.put(sum, entry.getKey());
+        }
+
+        int max = sum;
+        return (rng) -> {
+            int weight = rng.nextInt(max);
+            return weightMap.ceilingEntry(weight).getValue();
+        };
+    }
+
 }

test/harry/main/org/apache/cassandra/harry/gen/IndexGenerators.java

@@ -1,4 +1,4 @@
-/*
+package org.apache.cassandra.harry.gen;/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -16,115 +16,23 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.harry.gen;
-
-import org.apache.cassandra.harry.MagicConstants;
+import java.util.List;
 
 public class IndexGenerators
 {
-    private final ValueGenerators valueGenerators;
-    public final Generator<Integer> pkIdxGen;
-    public final Generator<Integer> ckIdxGen;
-    public final Generator<Integer>[] regularIdxGens;
-    public final Generator<Integer>[] staticIdxGens;
-
-    public static IndexGenerators withDefaults(ValueGenerators valueGenerators)
+    public static Generator<Integer> uniform(Bijections.Bijection<?> gen)
     {
-        Generator<Integer>[] regularIdxGens = new Generator[valueGenerators.regularColumnGens.size()];
-        Generator<Integer>[] staticIdxGens = new Generator[valueGenerators.staticColumnGens.size()];
-
-        for (int i = 0; i < regularIdxGens.length; i++)
-        {
-            int column = i;
-            regularIdxGens[i] = (rng) -> rng.nextInt(valueGenerators.regularPopulation(column));
-        }
-
-        for (int i = 0; i < staticIdxGens.length; i++)
-        {
-            int column = i;
-            staticIdxGens[i] = (rng) -> rng.nextInt(valueGenerators.staticPopulation(column));
-        }
-
-        return new IndexGenerators(valueGenerators,
-                                   // TODO: distribution for visits
-                                   Generators.int32(0, valueGenerators.pkPopulation()),
-                                   Generators.int32(0, valueGenerators.ckPopulation()),
-                                   regularIdxGens,
-                                   staticIdxGens);
+        return (rng) -> rng.nextInt((int) gen.population());
     }
 
-    public IndexGenerators(ValueGenerators valueGenerators,
-                           Generator<Integer> pkIdxGen,
-                           Generator<Integer> ckIdxGen,
-                           Generator<Integer>[] regularIdxGens,
-                           Generator<Integer>[] staticIdxGens)
+    public static Generator<Integer>[] uniform(List<? extends Bijections.Bijection<?>> gens)
     {
-        this.valueGenerators = valueGenerators;
-        this.pkIdxGen = pkIdxGen;
-        this.ckIdxGen = ckIdxGen;
-        this.regularIdxGens = regularIdxGens;
-        this.staticIdxGens = staticIdxGens;
-
-    }
-
-    public IndexGenerators roundRobinPk()
-    {
-        Generator<Integer> pkIdxGen = new Generator<Integer>()
-        {
-            int offset = 0;
-            @Override
-            public Integer generate(EntropySource rng)
-            {
-                int next = offset++;
-
-                if (offset < 0)
-                    offset = 0;
-
-                return next % valueGenerators.pkPopulation();
-            }
-        };
-
-        return new IndexGenerators(valueGenerators,
-                                   pkIdxGen,
-                                   ckIdxGen,
-                                   regularIdxGens,
-                                   staticIdxGens);
-    }
-
-    public IndexGenerators trackPk()
-    {
-        if (pkIdxGen instanceof Generators.TrackingGenerator<?>)
-            return this;
-
-        return new IndexGenerators(valueGenerators,
-                                   Generators.tracking(pkIdxGen),
-                                   ckIdxGen,
-                                   regularIdxGens,
-                                   staticIdxGens);
-    }
-
-    public IndexGenerators withChanceOfUnset(double chanceOfUnset)
-    {
-        Generator<Integer>[] regularIdxGens = new Generator[valueGenerators.regularColumnGens.size()];
-        Generator<Integer>[] staticIdxGens = new Generator[valueGenerators.staticColumnGens.size()];
-
-        for (int i = 0; i < regularIdxGens.length; i++)
+        Generator<Integer>[] res = new Generator[gens.size()];
+        for (int i = 0; i < res.length; i++)
         {
             int column = i;
-            regularIdxGens[i] = (rng) -> rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(valueGenerators.regularPopulation(column));
+            res[i] = (rng) -> rng.nextInt((int) gens.get(column).population());
         }
-
-        for (int i = 0; i < staticIdxGens.length; i++)
-        {
-            int column = i;
-            staticIdxGens[i] = (rng) -> rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(valueGenerators.staticPopulation(column));
-        }
-
-        return new IndexGenerators(valueGenerators,
-                                   // TODO: distribution for visits
-                                   Generators.int32(0, valueGenerators.pkPopulation()),
-                                   Generators.int32(0, valueGenerators.ckPopulation()),
-                                   regularIdxGens,
-                                   staticIdxGens);
+        return res;
     }
 }

test/harry/main/org/apache/cassandra/harry/gen/InvertibleGenerator.java

@@ -20,14 +20,15 @@ package org.apache.cassandra.harry.gen;
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.function.LongFunction;
 import java.util.stream.Collectors;
 
-import org.agrona.collections.IntHashSet;
+import org.agrona.collections.Long2ObjectHashMap;
 
 import accord.utils.Invariants;
 
@@ -51,25 +52,41 @@ import org.apache.cassandra.utils.ArrayUtils;
  *
  * TODO (expected): custom invertible generator for bool, u8, u16, u32, etc, for efficiency.
  * TODO (expected): implement support for tuple/vector/udt, and other multi-cell types.
+ * TODO (expected): use smaller-entropy values for descriptors, as we only need to generate _distinct_ values. In principe, we can just use counters.
  */
 public class InvertibleGenerator<T> implements HistoryBuilder.IndexedBijection<T>
 {
     public static long MAX_ENTROPY = 1L << 63;
 
-    private static final boolean PARANOIA = true;
+    private static final boolean PARANOIA = false;
+    private static final int INFLATE_CACHE_SIZE = 10_000;
+    // Number of top levels of the deflate binary-search tree to pin permanently (up to ~2^depth entries), so
+    // the hottest comparisons are never evicted by the FIFO backstop. Tunable.
+    private static final int BINARY_SEARCH_CACHE_DEPTH = 10;
 
-    // TODO (required): switch to use a primitive array; will need to implement a sort comparator for primitive types
-    private final List<Long> allocatedDescriptors;
+    private static final ConcurrentSkipListMap<Integer, long[]> REUSE_DESCRIPTOR_ARRAYS = new ConcurrentSkipListMap<>();
+
+    private long[] descriptors;
+    private final long descriptorCount;
 
-    private final Generator<T> gen;
     private final Comparator<T> comparator;
+    private final Cache<T> inflateCache;
 
-    // To avoid <?> erased types
-    public static <T> InvertibleGenerator<T> fromType(EntropySource rng, int population, ColumnSpec<T> spec)
+    public static <T> HistoryBuilder.IndexedBijection<T> fromType(EntropySource rng, int population, ColumnSpec<T> spec)
     {
+        if (spec.gen instanceof HistoryBuilder.IndexedBijection)
+            return (HistoryBuilder.IndexedBijection<T>) spec.gen;
         return new InvertibleGenerator<>(rng, spec.type.typeEntropy(), population, spec.gen, spec.type.comparator());
     }
 
+    @Override
+    public void discard()
+    {
+        if (descriptors.length > 100)
+            REUSE_DESCRIPTOR_ARRAYS.putIfAbsent(descriptors.length, descriptors);
+        descriptors = null;
+    }
+
     public InvertibleGenerator(EntropySource rng,
                                /* unsigned */ long typeEntropy,
                                int population,
@@ -87,62 +104,80 @@ public class InvertibleGenerator<T> implements HistoryBuilder.IndexedBijection<T
 
         population = (int) Math.min(typeEntropy, population);
 
-        this.gen = gen;
         this.comparator = comparator;
-        this.allocatedDescriptors = new ArrayList<>();
+        LongFunction<T> compute = descriptor -> SeedableEntropySource.computeWithSeed(descriptor, gen::generate);
 
-        // Generate a population of _unique_ values. We do not want to store all values, only their hashes.
-        IntHashSet hashes = new IntHashSet(population);
-        while (allocatedDescriptors.size() < population)
+        Map.Entry<Integer, long[]> e = REUSE_DESCRIPTOR_ARRAYS.ceilingEntry(population);
+        if (population > 100 && e != null && e.getValue().length < population * 2 && REUSE_DESCRIPTOR_ARRAYS.remove(e.getKey(), e.getValue()))
+            this.descriptors = e.getValue();
+        else
+            this.descriptors = new long[population];
+
+        // Generate a population of values into a throwaway boxed list, sort it by value, and copy the distinct
+        // values (now adjacent) into the primitive descriptor array. The list and the value cache exist only
+        // for the duration of construction; the long-lived state is descriptors[]/descriptorCount.
+        List<Long> candidates = new ArrayList<>(population);
+        for (int i = 0 ; i < population ; ++i)
         {
             long candidate = rng.next();
 
             // Should never allocate these, however improbable that is
-            if (MagicConstants.MAGIC_DESCRIPTOR_VALS.contains(candidate))
+            if (MagicConstants.isMagicDescriptor(candidate))
                 continue;
 
-            Object inflated = inflate(candidate);
-            int hash = ArrayUtils.hashCode(inflated);
-            Invariants.require(hash  != System.identityHashCode(inflated), "hashCode was not overridden for type %s", inflated.getClass());
-
-            if (hashes.add(hash))
-                allocatedDescriptors.add(candidate);
+            candidates.add(candidate);
         }
-        hashes.clear();
 
-        allocatedDescriptors.sort(this::compare);
+        Cache<T> tmpCache = new UnboundedCache<>(population, compute);
+        candidates.sort((d1, d2) -> comparator.compare(tmpCache.get(d1), tmpCache.get(d2)));
+        int count = 0;
+        for (int i = 0; i < candidates.size(); i++)
+        {
+            long candidate = candidates.get(i);
+            if (count > 0 && comparator.compare(tmpCache.get(descriptors[count - 1]), tmpCache.get(candidate)) == 0)
+                continue;
+            descriptors[count++] = candidate;
+        }
+        descriptorCount = count;
+
+        // Inflate cache: pin the hottest binary-search entries (the top few levels, visited by every deflate)
+        // so they are never evicted, and fall back to a bounded FIFO for everything else.
+        this.inflateCache = new HierarchicalCache<>(new BinarySearchPathCache<>(descriptors, count, BINARY_SEARCH_CACHE_DEPTH, compute),
+                                                    new FifoCache<>(INFLATE_CACHE_SIZE, compute));
 
         // Check there are no duplicates, and items are properly sorted.
         if (PARANOIA)
         {
-            T prev = inflate(allocatedDescriptors.get(0));
-            for (int i = 1; i < allocatedDescriptors.size(); i++)
+            T prev = inflate(descriptors[0]);
+            for (int i = 1; i < descriptorCount; i++)
             {
-                T current = inflate(allocatedDescriptors.get(i));
+                T current = inflate(descriptors[i]);
                 Invariants.require( comparator.compare(current, prev) > 0,
-                                       () -> String.format("%s should be strictly after %s", prev, current));
+                                       "%s should be strictly after %s", prev, current);
+                prev = current;
             }
         }
     }
 
     @Override
-    public int idxFor(long descriptor)
+    public long idxFor(long descriptor)
     {
-        return Collections.binarySearch(allocatedDescriptors, descriptor, this.descriptorsComparator());
+        // descriptors[] is value-sorted, so the index of a descriptor is where its value sorts.
+        return binarySearch(inflate(descriptor));
     }
 
     @Override
-    public long descriptorAt(int idx)
+    public long descriptorAt(long idx)
     {
-        return allocatedDescriptors.get(idx);
+        return descriptors[(int) idx];
     }
 
     @Override
     public T inflate(long descriptor)
     {
-        Invariants.require(!MagicConstants.MAGIC_DESCRIPTOR_VALS.contains(descriptor),
-                              String.format("Should not be able to inflate %d, as it's magic value", descriptor));
-        return SeedableEntropySource.computeWithSeed(descriptor, gen::generate);
+        Invariants.require(!MagicConstants.isMagicDescriptor(descriptor),
+                           "Should not be able to inflate %d, as it's magic value", descriptor);
+        return inflateCache.get(descriptor);
     }
 
     @Override
@@ -153,28 +188,26 @@ public class InvertibleGenerator<T> implements HistoryBuilder.IndexedBijection<T
         {
             if (idx < 0)
             {
-                for (long descriptor : allocatedDescriptors)
+                for (int i = 0; i < descriptorCount; i++)
                 {
-                    Object expected = inflate(descriptor);
+                    Object expected = inflate(descriptors[i]);
                     if (value.getClass().isArray())
                     {
                         Object[] valueArr = (Object[]) value;
                         Object[] expectedArr = (Object[]) expected;
                         Invariants.require(comparator.compare((T) expected, value) != 0,
-                                              "%s was found: %s", Arrays.toString(expectedArr), Arrays.toString(valueArr));
-
+                                           "%s was found: %s", Arrays.toString(expectedArr), Arrays.toString(valueArr));
                     }
                     else
                     {
                         Invariants.require(comparator.compare((T) expected, value) != 0,
-                                              "%s was found: %s", expected, value);
+                                           "%s was found: %s", expected, value);
                     }
-
                 }
             }
             else
             {
-                long res = allocatedDescriptors.get(idx);
+                long res = descriptors[idx];
                 Object expected = inflate(res);
                 if (value.getClass().isArray())
                 {
@@ -182,13 +215,12 @@ public class InvertibleGenerator<T> implements HistoryBuilder.IndexedBijection<T
                     Object[] expectedArr = (Object[]) expected;
 
                     Invariants.require(comparator.compare((T) expected, value) == 0,
-                                          "%s != %s", Arrays.toString(expectedArr), Arrays.toString(valueArr));
-
+                                       "%s != %s", Arrays.toString(expectedArr), Arrays.toString(valueArr));
                 }
                 else
                 {
                     Invariants.require(comparator.compare((T) expected, value) == 0,
-                                          "%s != %s", expected, value);
+                                       "%s != %s", expected, value);
                 }
 
                 return res;
@@ -200,12 +232,12 @@ public class InvertibleGenerator<T> implements HistoryBuilder.IndexedBijection<T
             int start = Math.max(0, idx - 2);
             List<Object> nearby = new ArrayList<>();
             for (int i = start; i < start + 2; i++)
-                nearby.add(inflate(allocatedDescriptors.get(i)));
+                nearby.add(inflate(descriptors[i]));
             throw new IllegalStateException(String.format("Could not find: %s\nNearby objects: %s",
                                                           ArrayUtils.toString(value), nearby.stream().map(ArrayUtils::toString).collect(Collectors.toList())));
         }
 
-        return allocatedDescriptors.get(idx);
+        return descriptors[idx];
     }
 
 
@@ -217,11 +249,11 @@ public class InvertibleGenerator<T> implements HistoryBuilder.IndexedBijection<T
 
     private int binarySearch(T key)
     {
-        int low = 0, mid = allocatedDescriptors.size(), high = mid - 1, result = -1;
+        int low = 0, mid = (int) descriptorCount, high = mid - 1, result = -1;
         while (low <= high)
         {
             mid = (low + high) >>> 1;
-            result = comparator.compare(key, inflate(allocatedDescriptors.get(mid)));
+            result = comparator.compare(key, inflate(descriptors[mid]));
             if (result > 0)
                 low = mid + 1;
             else if (result == 0)
@@ -232,7 +264,6 @@ public class InvertibleGenerator<T> implements HistoryBuilder.IndexedBijection<T
         return -mid - (result < 0 ? 1 : 2);
     }
 
-
     @Override
     public int compare(long d1, long d2)
     {
@@ -247,17 +278,179 @@ public class InvertibleGenerator<T> implements HistoryBuilder.IndexedBijection<T
      * Returns a number of allocated descriptors
      */
     @Override
-    public int population()
+    public long population()
     {
-        return allocatedDescriptors.size();
+        return descriptorCount;
     }
 
     public Comparator<Long> descriptorsComparator()
     {
-        // TODO: this can be cached
         Map<Long, Integer> descriptorToIdx = new HashMap<>();
-        for (int i = 0; i < allocatedDescriptors.size(); i++)
-            descriptorToIdx.put(allocatedDescriptors.get(i), i);
+        for (int i = 0; i < descriptorCount; i++)
+            descriptorToIdx.put(descriptors[i], i);
         return Comparator.comparingInt(descriptorToIdx::get);
     }
+
+    /**
+     * A value cache keyed by descriptor.
+     */
+    public interface Cache<T>
+    {
+        /** Returns the cached value for {@code descriptor}, or {@code null} if it is not cached; never computes. */
+        T lookup(long descriptor);
+
+        /** Returns the value for {@code descriptor}, computing it (and possibly caching it) on a miss. */
+        T get(long descriptor);
+    }
+
+    /**
+     * Fixed-size FIFO cache: a ring buffer tracks insertion order for eviction while a map provides O(1)
+     * lookup by descriptor. Misses are filled with the supplied compute function.
+     */
+    public static final class FifoCache<T> implements Cache<T>
+    {
+        private final int capacity;
+        private final LongFunction<T> compute;
+        private final long[] ring;
+        private final Long2ObjectHashMap<T> map;
+        private int pos = 0;
+        private int count = 0;
+
+        public FifoCache(int capacity, LongFunction<T> compute)
+        {
+            this.capacity = capacity;
+            this.compute = compute;
+            this.ring = new long[capacity];
+            this.map = new Long2ObjectHashMap<>(capacity, 0.75f);
+        }
+
+        @Override
+        public T lookup(long descriptor)
+        {
+            return map.get(descriptor);
+        }
+
+        @Override
+        public T get(long descriptor)
+        {
+            T cached = map.get(descriptor);
+            if (cached != null)
+                return cached;
+
+            T value = compute.apply(descriptor);
+            if (count == capacity)
+                map.remove(ring[pos]);   // evict the oldest entry to make room
+            else
+                count++;
+
+            ring[pos] = descriptor;
+            map.put(descriptor, value);
+            pos = (pos + 1) % capacity;
+            return value;
+        }
+    }
+
+    /**
+     * Unbounded cache that retains every entry it computes; intended for short-lived, build-time use where the
+     * whole population is touched and recomputation must be avoided. Not suitable as a long-lived cache.
+     */
+    public static final class UnboundedCache<T> implements Cache<T>
+    {
+        private final LongFunction<T> compute;
+        private final Long2ObjectHashMap<T> map;
+
+        public UnboundedCache(int initialCapacity, LongFunction<T> compute)
+        {
+            this.compute = compute;
+            this.map = new Long2ObjectHashMap<>(Math.max(1, initialCapacity), 0.75f);
+        }
+
+        @Override
+        public T lookup(long descriptor)
+        {
+            return map.get(descriptor);
+        }
+
+        @Override
+        public T get(long descriptor)
+        {
+            T cached = map.get(descriptor);
+            if (cached != null)
+                return cached;
+
+            T value = compute.apply(descriptor);
+            map.put(descriptor, value);
+            return value;
+        }
+    }
+
+    /**
+     * Caches exactly the entries that a binary search over a sorted descriptor array would touch in its first {@code depth} steps.
+     */
+    public static final class BinarySearchPathCache<T> implements Cache<T>
+    {
+        private final LongFunction<T> compute;
+        private final Long2ObjectHashMap<T> map;
+
+        public BinarySearchPathCache(long[] sortedDescriptors, int size, int depth, LongFunction<T> compute)
+        {
+            this.compute = compute;
+            this.map = new Long2ObjectHashMap<>(Math.max(1, 1 << Math.min(depth, 16)), 0.75f);
+            cachePath(sortedDescriptors, 0, size - 1, depth);
+        }
+
+        private void cachePath(long[] sortedDescriptors, int lo, int hi, int depth)
+        {
+            if (depth <= 0 || lo > hi)
+                return;
+            int mid = (lo + hi) >>> 1;
+            long descriptor = sortedDescriptors[mid];
+            map.put(descriptor, compute.apply(descriptor));
+            cachePath(sortedDescriptors, lo, mid - 1, depth - 1);
+            cachePath(sortedDescriptors, mid + 1, hi, depth - 1);
+        }
+
+        @Override
+        public T lookup(long descriptor)
+        {
+            return map.get(descriptor);
+        }
+
+        @Override
+        public T get(long descriptor)
+        {
+            T cached = map.get(descriptor);
+            return cached != null ? cached : compute.apply(descriptor);
+        }
+    }
+
+    /**
+     * Composes two caches into a hierarchy: lookups try {@code primary} first and fall back to {@code secondry}
+     * on a miss.
+     */
+    public static final class HierarchicalCache<T> implements Cache<T>
+    {
+        private final Cache<T> primary;
+        private final Cache<T> secondary;
+
+        public HierarchicalCache(Cache<T> primary, Cache<T> secondary)
+        {
+            this.primary = primary;
+            this.secondary = secondary;
+        }
+
+        @Override
+        public T lookup(long descriptor)
+        {
+            T value = primary.lookup(descriptor);
+            return value != null ? value : secondary.lookup(descriptor);
+        }
+
+        @Override
+        public T get(long descriptor)
+        {
+            T value = primary.lookup(descriptor);
+            return value != null ? value : secondary.get(descriptor);
+        }
+    }
 }

test/harry/main/org/apache/cassandra/harry/gen/OperationsGenerators.java

@@ -19,7 +19,8 @@
 package org.apache.cassandra.harry.gen;
 
 import org.apache.cassandra.harry.SchemaSpec;
-import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
+import org.apache.cassandra.harry.op.Kind;
 import org.apache.cassandra.harry.op.Operations;
 
 public class OperationsGenerators
@@ -38,16 +39,14 @@ public class OperationsGenerators
         };
     }
 
-    // TODO: distributions
-    public static Generator<Long> sequentialPd(SchemaSpec schema)
+    // TODO: can we use this, it is not functionally pure
+    public static Generator<Long> sequentialPd(IndexedValueGenerators valueGenerators)
     {
-        // TODO: switch away from Indexed generators here
-        HistoryBuilder.IndexedValueGenerators valueGenerators = (HistoryBuilder.IndexedValueGenerators) schema.valueGenerators;
-        int population = valueGenerators.pkPopulation();
+        long population = valueGenerators.pkGen.population();
 
         return new Generator<>()
         {
-            int counter = 0;
+            long counter = 0;
 
             @Override
             public Long generate(EntropySource rng)
@@ -57,53 +56,43 @@ public class OperationsGenerators
         };
     }
 
-    public static Generator<Long> sequentialCd(SchemaSpec schema)
-    {
-        // TODO: switch away from Indexed generators here
-        HistoryBuilder.IndexedValueGenerators valueGenerators = (HistoryBuilder.IndexedValueGenerators) schema.valueGenerators;
-        int population = valueGenerators.ckPopulation();
-
-        return new Generator<>()
-        {
-            int counter = 0;
-
-            @Override
-            public Long generate(EntropySource rng)
-            {
-                return valueGenerators.ckGen().descriptorAt(counter++ % population);
-            }
-        };
-    }
-
-    public static Generator<ToOp> writeOp(SchemaSpec schema)
-    {
-        return writeOp(schema, sequentialPd(schema), sequentialCd(schema));
-    }
+    // TODO: useful?
+//    public static Generator<Long> sequentialCd(IndexedValueGenerators valueGenerators)
+//    {
+//        int population = valueGenerators.ckPopulation();
+//
+//        return new Generator<>()
+//        {
+//            int counter = 0;
+//
+//            @Override
+//            public Long generate(EntropySource rng)
+//            {
+//                return valueGenerators.ckGen().descriptorAt(counter++ % population);
+//            }
+//        };
+//    }
 
     // TODO: chance of unset
     public static Generator<ToOp> writeOp(SchemaSpec schema,
-                                          Generator<Long> pdGen,
-                                          Generator<Long> cdGen)
+                                          IndexedValueGenerators valueGenerators)
     {
-        // TODO: switch away from Indexed generators here
-        HistoryBuilder.IndexedValueGenerators valueGenerators = (HistoryBuilder.IndexedValueGenerators) schema.valueGenerators;
-
         return (rng) -> {
-            long pd = pdGen.generate(rng);
-            long cd = cdGen.generate(rng);
+            long pd = valueGenerators.pkIdxGen().generate(rng);
+            long cd = valueGenerators.forPd(pd).ckIdxGen().generate(rng);
             long[] vds = new long[schema.regularColumns.size()];
             for (int i = 0; i < schema.regularColumns.size(); i++)
             {
-                int idx = rng.nextInt(valueGenerators.regularPopulation(i));
-                vds[i] = valueGenerators.regularColumnGen(i).descriptorAt(idx);
+                long idx = rng.nextInt(valueGenerators.forPd(pd).regularPopulation(i));
+                vds[i] = valueGenerators.forPd(pd).regularColumnGen(i).descriptorAt(idx);
             }
             long[] sds = new long[schema.staticColumns.size()];
             for (int i = 0; i < schema.staticColumns.size(); i++)
             {
-                int idx = rng.nextInt(valueGenerators.staticPopulation(i));
-                sds[i] = valueGenerators.staticColumnGen(i).descriptorAt(idx);
+                long idx = rng.nextInt(valueGenerators.forPd(pd).staticPopulation(i));
+                sds[i] = valueGenerators.forPd(pd).staticColumnGen(i).descriptorAt(idx);
             }
-            return lts -> new Operations.WriteOp(lts, pd, cd, vds, sds, Operations.Kind.INSERT);
+            return lts -> new Operations.WriteOp(lts, pd, cd, vds, sds, Kind.INSERT);
         };
     }
 

test/harry/main/org/apache/cassandra/harry/gen/SchemaGenerators.java

@@ -46,9 +46,7 @@ public class SchemaGenerators
             public SchemaSpec generate(EntropySource rng)
             {
                 int idx = counter++;
-                return new SchemaSpec(rng.next(),
-                                      expectedValues,
-                                      ks,
+                return new SchemaSpec(ks,
                                       prefix + idx,
                                       pkGen.generate(rng),
                                       ckGen.generate(rng),
@@ -97,17 +95,25 @@ public class SchemaGenerators
         };
     }
 
-    public static Generator<SchemaSpec> trivialSchema(String ks, String table, int population)
+    public static Generator<SchemaSpec> trivialSchema(String ks, String table)
     {
-        return trivialSchema(ks, () -> table, population, SchemaSpec.optionsBuilder().build());
+        return trivialSchema(ks, table, SchemaSpec.optionsBuilder().build());
+    }
+
+    public static Generator<SchemaSpec> trivialSchema(String ks, String table, SchemaSpec.Options options)
+    {
+        return trivialSchema(ks, () -> table, options);
+    }
+
+    public static Generator<SchemaSpec> trivialSchema(String ks, Supplier<String> table, SchemaSpec.Options options)
+    {
+        return trivialSchema(ks, table, 1000, options);
     }
 
     public static Generator<SchemaSpec> trivialSchema(String ks, Supplier<String> table, int population, SchemaSpec.Options options)
     {
         return (rng) -> {
-            return new SchemaSpec(rng.next(),
-                                  population,
-                                  ks, table.get(),
+            return new SchemaSpec(ks, table.get(),
                                   Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.int64Type, Generators.int64())),
                                   Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.int64Type, Generators.int64(), false)),
                                   Arrays.asList(ColumnSpec.regularColumn("v1", ColumnSpec.int64Type)),

test/harry/main/org/apache/cassandra/harry/gen/SharedValueGenerators.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.gen;
+
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.cassandra.harry.gen.Bijections.Bijection;
+
+public class SharedValueGenerators<PartitionKey, ClusteringKey> extends ValueGenerators<PartitionKey, ClusteringKey>
+{
+    protected final PartitionValues<ClusteringKey> partitionValues;
+
+    public SharedValueGenerators(Bijection<PartitionKey> pkGen,
+
+                                 Bijection<ClusteringKey> ckGen,
+                                 Accessor<ClusteringKey> ckAccessor,
+
+                                 List<? extends Bijection<? extends Object>> regularColumnGens,
+                                 List<? extends Bijection<? extends Object>> staticColumnGens,
+
+                                 List<Comparator<Object>> ckComparators,
+                                 List<Comparator<Object>> regularComparators,
+                                 List<Comparator<Object>> staticComparators)
+    {
+        super(pkGen);
+        this.partitionValues = new PartitionValues<>(ckGen, ckAccessor, regularColumnGens, staticColumnGens, ckComparators, regularComparators, staticComparators);
+    }
+
+    public Bijection<PartitionKey> pkGen()
+    {
+        return pkGen;
+    }
+
+    @Override
+    public PartitionValues<ClusteringKey> forPd(long pd)
+    {
+        return partitionValues;
+    }
+}

test/harry/main/org/apache/cassandra/harry/gen/StringBijection.java

@@ -70,10 +70,12 @@ public class StringBijection implements Bijections.Bijection<String>
             builder.append(nibbles[idx]);
         }
 
-        appendRandomBytes(builder, descriptor);
+        appendSuffix(builder, descriptor);
+
+        // everything after the nibble prefix is just random padding, since strings are guaranteed
+        // to have unique prefixes. Subclasses may append additional content after this.
+        appendExtra(builder, descriptor);
 
-        // everything after this point can be just random, since strings are guaranteed
-        // to have unique prefixes
         return builder.toString();
     }
 
@@ -85,7 +87,7 @@ public class StringBijection implements Bijections.Bijection<String>
         return b;
     }
 
-    private void appendRandomBytes(StringBuilder builder, long descriptor)
+    protected void appendSuffix(StringBuilder builder, long descriptor)
     {
         long rnd = RngUtils.next(descriptor);
         int remaining = RngUtils.asInt(rnd, 0, maxRandomBytes);
@@ -101,6 +103,14 @@ public class StringBijection implements Bijections.Bijection<String>
         }
     }
 
+    /**
+     * Hook for subclasses to append additional content after the nibble prefix and suffix.
+     * By default, does nothing.
+     */
+    protected void appendExtra(StringBuilder builder, long descriptor)
+    {
+    }
+
     public long deflate(String descriptor)
     {
         long res = 0;

test/harry/main/org/apache/cassandra/harry/gen/TypeAdapters.java

@@ -44,6 +44,7 @@ import org.apache.cassandra.db.marshal.TimeUUIDType;
 import org.apache.cassandra.db.marshal.TimestampType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
 import org.apache.cassandra.schema.ColumnMetadata;
 
 /**
@@ -111,6 +112,41 @@ public class TypeAdapters
         return forValues(type, defaults);
     }
 
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    public static Generator<Object> forValues(AbstractType<?> type, Distribution distribution)
+    {
+        return forValues(type, distribution, 0xdeadbeefcafeL);
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    public static Generator<Object> forValues(AbstractType<?> type, Distribution distribution, long seed)
+    {
+        if (distribution == null)
+            return forValues(type, defaults);
+
+        AbstractType<?> baseType = type instanceof ReversedType ? ((ReversedType<?>) type).baseType : type;
+
+        if (baseType == AsciiType.instance)
+        {
+            char[] asciiChars = new char[128];
+            for (int i = 0; i < 128; i++)
+                asciiChars[i] = (char) i;
+            return (Generator<Object>) (Generator<?>) new UnorderedBijections.UnorderedStringBijection(0xaabbccddeeffL, seed, distribution, asciiChars);
+        }
+        else if (baseType == UTF8Type.instance)
+        {
+            int range = 0xD7FF + 1;
+            char[] utf8Chars = new char[range];
+            for (int i = 0; i < range; i++)
+                utf8Chars[i] = (char) i;
+            return (Generator<Object>) (Generator<?>) new UnorderedBijections.UnorderedStringBijection(0xaabbccddeeffL, seed, distribution, utf8Chars);
+        }
+        else if (baseType == BytesType.instance)
+            return (Generator<Object>) (Generator<?>) new UnorderedBijections.UnorderedBytesBijection(0xaabbccddeeffL, seed, distribution);
+        else
+            throw new IllegalArgumentException(String.format("Distribution-based generator is not supported for type %s", type));
+    }
+
     private static Generator<Object> forValues(AbstractType type, Map<AbstractType<?>, Generator<?>> typeToGen)
     {
         Generator<Object> gen = (Generator<Object>) typeToGen.get(type);

test/harry/main/org/apache/cassandra/harry/gen/UnorderedBijections.java

@@ -0,0 +1,187 @@
+/*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package org.apache.cassandra.harry.gen;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.gen.rng.PureRng;
+import org.apache.cassandra.harry.gen.rng.RngUtils;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+
+/**
+ * Bijections where values are NOT sorted by index. Indices are assigned via a pseudo-random
+ * permutation (PCG), so the mapping between index and descriptor is invertible but unordered:
+ * iterating indices 0, 1, 2, ... produces values in an effectively random order.
+ */
+public class UnorderedBijections
+{
+    public static class UnorderedStringBijection extends StringBijection implements HistoryBuilder.IndexedBijection<String>, Generator<String>
+    {
+        private final long stream;
+        private final PureRng rng;
+        private final Distribution extraSizeDistribution;
+        private final char[] allowedChars;
+
+        public UnorderedStringBijection(long seed)
+        {
+            this(0xaabbccddeeffL, seed, null, null);
+        }
+
+        public UnorderedStringBijection(long stream, long seed)
+        {
+            this(stream, seed, null, null);
+        }
+
+        public UnorderedStringBijection(long stream, long seed, Distribution extraSizeDistribution, char[] allowedChars)
+        {
+            this.stream = stream;
+            this.rng = new PureRng.PCGFast(seed);
+            this.extraSizeDistribution = extraSizeDistribution;
+            this.allowedChars = allowedChars;
+        }
+
+        public UnorderedStringBijection(int nibbleSize, int maxRandomBytes, long stream, long seed,
+                                        Distribution extraSizeDistribution, char[] allowedChars)
+        {
+            super(nibbleSize, maxRandomBytes);
+            this.stream = stream;
+            this.rng = new PureRng.PCGFast(seed);
+            this.extraSizeDistribution = extraSizeDistribution;
+            this.allowedChars = allowedChars;
+        }
+
+        public UnorderedStringBijection(String[] nibbles, int nibbleSize, int maxRandomBytes, long stream, long seed,
+                                        Distribution extraSizeDistribution, char[] allowedChars)
+        {
+            super(nibbles, nibbleSize, maxRandomBytes);
+            this.stream = stream;
+            this.rng = new PureRng.PCGFast(seed);
+            this.extraSizeDistribution = extraSizeDistribution;
+            this.allowedChars = allowedChars;
+        }
+
+        @Override
+        public long idxFor(long descriptor)
+        {
+            return rng.sequenceNumber(descriptor, stream);
+        }
+
+        @Override
+        public long descriptorAt(long idx)
+        {
+            return rng.randomNumber(idx, stream);
+        }
+
+        @Override
+        public String generate(EntropySource rng)
+        {
+            return inflate(rng.next());
+        }
+
+        @Override
+        protected void appendExtra(StringBuilder builder, long descriptor)
+        {
+            if (extraSizeDistribution == null || allowedChars == null)
+                return;
+
+            // Use a different seed derivation to avoid correlation with the suffix
+            long rnd = RngUtils.next(RngUtils.next(RngUtils.next(descriptor)));
+            int remaining = Math.toIntExact(extraSizeDistribution.next(rnd));
+
+            while (remaining > 0)
+            {
+                rnd = RngUtils.next(rnd);
+                builder.append(allowedChars[RngUtils.asInt(rnd, 0, allowedChars.length - 1)]);
+                remaining--;
+            }
+        }
+    }
+
+    public static class UnorderedBytesBijection extends BytesBijection implements HistoryBuilder.IndexedBijection<ByteBuffer>, Generator<ByteBuffer>
+    {
+        private final long stream;
+        private final PureRng rng;
+        private final Distribution extraSizeDistribution;
+
+        public UnorderedBytesBijection(long seed)
+        {
+            this(0xaabbccddeeffL, seed, null);
+        }
+
+        public UnorderedBytesBijection(long stream, long seed)
+        {
+            this(stream, seed, null);
+        }
+
+        public UnorderedBytesBijection(long stream, long seed, Distribution extraSizeDistribution)
+        {
+            this.stream = stream;
+            this.rng = new PureRng.PCGFast(seed);
+            this.extraSizeDistribution = extraSizeDistribution;
+        }
+
+        public UnorderedBytesBijection(int nibbleSize, int maxRandomBytes, long stream, long seed,
+                                       Distribution extraSizeDistribution)
+        {
+            super(nibbleSize, maxRandomBytes);
+            this.stream = stream;
+            this.rng = new PureRng.PCGFast(seed);
+            this.extraSizeDistribution = extraSizeDistribution;
+        }
+
+        public UnorderedBytesBijection(byte[][] nibbles, int nibbleSize, int maxRandomBytes, long stream, long seed,
+                                       Distribution extraSizeDistribution)
+        {
+            super(nibbles, nibbleSize, maxRandomBytes);
+            this.stream = stream;
+            this.rng = new PureRng.PCGFast(seed);
+            this.extraSizeDistribution = extraSizeDistribution;
+        }
+
+        @Override
+        public long idxFor(long descriptor)
+        {
+            return rng.sequenceNumber(descriptor, stream);
+        }
+
+        @Override
+        public long descriptorAt(long idx)
+        {
+            return rng.randomNumber(idx, stream);
+        }
+
+        @Override
+        public ByteBuffer generate(EntropySource rng)
+        {
+            return inflate(rng.next());
+        }
+
+        @Override
+        protected int extraLength(long descriptor)
+        {
+            if (extraSizeDistribution == null)
+                return 0;
+
+            // Use a different seed derivation to avoid correlation with the suffix
+            long rnd = RngUtils.next(RngUtils.next(RngUtils.next(descriptor)));
+            return Math.toIntExact(extraSizeDistribution.next(rnd));
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/gen/ValueGenerators.java

@@ -23,42 +23,13 @@ import java.util.List;
 
 import org.apache.cassandra.harry.gen.Bijections.Bijection;
 
-public class ValueGenerators<PartitionKey, ClusteringKey>
+public abstract class ValueGenerators<PartitionKey, ClusteringKey>
 {
     protected final Bijection<PartitionKey> pkGen;
-    protected final Bijection<ClusteringKey> ckGen;
 
-    protected final Accessor<ClusteringKey> ckAccessor;
-
-    protected final List<? extends Bijection<? extends Object>> regularColumnGens;
-    protected final List<? extends Bijection<? extends Object>> staticColumnGens;
-
-    protected final List<Comparator<Object>> pkComparators;
-    protected final List<Comparator<Object>> ckComparators;
-    protected final List<Comparator<Object>> regularComparators;
-    protected final List<Comparator<Object>> staticComparators;
-
-    public ValueGenerators(Bijection<PartitionKey> pkGen,
-                           Bijection<ClusteringKey> ckGen,
-                           Accessor<ClusteringKey> ckAccessor,
-
-                           List<? extends Bijection<? extends Object>> regularColumnGens,
-                           List<? extends Bijection<? extends Object>> staticColumnGens,
-
-                           List<Comparator<Object>> pkComparators,
-                           List<Comparator<Object>> ckComparators,
-                           List<Comparator<Object>> regularComparators,
-                           List<Comparator<Object>> staticComparators)
+    public ValueGenerators(Bijection<PartitionKey> pkGen)
     {
         this.pkGen = pkGen;
-        this.ckGen = ckGen;
-        this.ckAccessor = ckAccessor;
-        this.regularColumnGens = regularColumnGens;
-        this.staticColumnGens = staticColumnGens;
-        this.pkComparators = pkComparators;
-        this.ckComparators = ckComparators;
-        this.regularComparators = regularComparators;
-        this.staticComparators = staticComparators;
     }
 
     public Bijection<PartitionKey> pkGen()
@@ -66,79 +37,105 @@ public class ValueGenerators<PartitionKey, ClusteringKey>
         return pkGen;
     }
 
-    public Bijection<ClusteringKey> ckGen()
-    {
-        return ckGen;
-    }
+    public abstract PartitionValues<ClusteringKey> forPd(long pd);
 
-    public Bijection regularColumnGen(int idx)
+    public static class PartitionValues<CK>
     {
-        return regularColumnGens.get(idx);
-    }
+        protected final Bijection<CK> ckGen;
 
-    public Bijection staticColumnGen(int idx)
-    {
-        return staticColumnGens.get(idx);
-    }
+        protected final Accessor<CK> ckAccessor;
 
-    public int ckColumnCount()
-    {
-        return ckComparators.size();
-    }
+        protected final List<? extends Bijection<? extends Object>> regularColumnGens;
+        protected final List<? extends Bijection<? extends Object>> staticColumnGens;
 
-    public int regularColumnCount()
-    {
-        return regularColumnGens.size();
-    }
+        protected final List<Comparator<Object>> ckComparators;
+        protected final List<Comparator<Object>> regularComparators;
+        protected final List<Comparator<Object>> staticComparators;
 
-    public int staticColumnCount()
-    {
-        return staticColumnGens.size();
-    }
+        public PartitionValues(Bijection<CK> ckGen,
+                               Accessor<CK> ckAccessor,
 
-    public Comparator<Object> pkComparator(int idx)
-    {
-        return pkComparators.get(idx);
-    }
+                               List<? extends Bijection<? extends Object>> regularColumnGens,
+                               List<? extends Bijection<? extends Object>> staticColumnGens,
 
-    public Comparator<Object> ckComparator(int idx)
-    {
-        return ckComparators.get(idx);
-    }
+                               List<Comparator<Object>> ckComparators,
+                               List<Comparator<Object>> regularComparators,
+                               List<Comparator<Object>> staticComparators)
+        {
 
-    public Comparator<Object> regularComparator(int idx)
-    {
-        return regularComparators.get(idx);
-    }
+            this.ckGen = ckGen;
+            this.ckAccessor = ckAccessor;
+            this.regularColumnGens = regularColumnGens;
+            this.staticColumnGens = staticColumnGens;
+            this.ckComparators = ckComparators;
+            this.regularComparators = regularComparators;
+            this.staticComparators = staticComparators;
+        }
 
-    public Comparator<Object> staticComparator(int idx)
-    {
-        return staticComparators.get(idx);
-    }
+        public Bijection<CK> ckGen()
+        {
+            return ckGen;
+        }
 
-    public Accessor<ClusteringKey> ckAccessor()
-    {
-        return ckAccessor;
-    }
+        public Bijection regularColumnGen(int idx)
+        {
+            return regularColumnGens.get(idx);
+        }
 
-    public int pkPopulation()
-    {
-        return pkGen.population();
-    }
+        public Bijection staticColumnGen(int idx)
+        {
+            return staticColumnGens.get(idx);
+        }
 
-    public int ckPopulation()
-    {
-        return ckGen.population();
-    }
+        public int ckColumnCount()
+        {
+            return ckComparators.size();
+        }
 
-    public int regularPopulation(int i)
-    {
-        return regularColumnGens.get(i).population();
-    }
+        public int regularColumnCount()
+        {
+            return regularColumnGens.size();
+        }
 
-    public int staticPopulation(int i)
-    {
-        return staticColumnGens.get(i).population();
+        public int staticColumnCount()
+        {
+            return staticColumnGens.size();
+        }
+
+        public Comparator<Object> ckComparator(int idx)
+        {
+            return ckComparators.get(idx);
+        }
+
+        public Comparator<Object> regularComparator(int idx)
+        {
+            return regularComparators.get(idx);
+        }
+
+        public Comparator<Object> staticComparator(int idx)
+        {
+            return staticComparators.get(idx);
+        }
+
+        public Accessor<CK> ckAccessor()
+        {
+            return ckAccessor;
+        }
+
+        public int ckPopulation()
+        {
+            return Math.toIntExact(ckGen.population());
+        }
+
+        public int regularPopulation(int i)
+        {
+            return Math.toIntExact(regularColumnGens.get(i).population());
+        }
+
+        public int staticPopulation(int i)
+        {
+            return Math.toIntExact(staticColumnGens.get(i).population());
+        }
     }
 
     public interface Accessor<T>

test/harry/main/org/apache/cassandra/harry/gen/rng/SeedableEntropySource.java

@@ -27,19 +27,30 @@ import io.netty.util.concurrent.FastThreadLocal;
 
 public class SeedableEntropySource
 {
+    // TODO (required): forbid reentrancy
     private static final FastThreadLocal<EntropySource> THREAD_LOCAL = new FastThreadLocal<>();
 
+    public static <T> T computeWithSeed(long seed, long stream, Function<EntropySource, T> fn)
+    {
+        return fn.apply(entropySource(seed, stream));
+    }
+
     public static <T> T computeWithSeed(long seed, Function<EntropySource, T> fn)
     {
-        return fn.apply(entropySource(seed));
+        return fn.apply(entropySource(seed, seed));
+    }
+
+    public static long computeLongWithSeed(long seed, ToLong<EntropySource> fn)
+    {
+        return fn.apply(entropySource(seed, seed));
     }
 
     public static void doWithSeed(long seed, Consumer<EntropySource> fn)
     {
-        fn.accept(entropySource(seed));
+        fn.accept(entropySource(seed, seed));
     }
 
-    private static EntropySource entropySource(long seed)
+    public static EntropySource entropySource(long seed, long stream)
     {
         EntropySource entropySource = THREAD_LOCAL.get();
         if (entropySource == null)
@@ -47,7 +58,15 @@ public class SeedableEntropySource
             entropySource = new JdkRandomEntropySource(0);
             THREAD_LOCAL.set(entropySource);
         }
+        // scramble seed even more, since it seems to be not enough entropy for small cardinality values, such as rng.nextInt(2)
+        // in default JDK scrambler
+        seed = PCGFastPure.shuffle(PCGFastPure.advanceState(seed, stream, 100));
         entropySource.seed(seed);
         return entropySource;
     }
+
+    public static interface ToLong<T>
+    {
+        long apply(T value);
+    }
 }

test/harry/main/org/apache/cassandra/harry/model/BytesPartitionState.java

@@ -43,6 +43,7 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.harry.MagicConstants;
 import org.apache.cassandra.harry.gen.BijectionCache;
 import org.apache.cassandra.harry.gen.Bijections;
+import org.apache.cassandra.harry.gen.SharedValueGenerators;
 import org.apache.cassandra.harry.gen.ValueGenerators;
 import org.apache.cassandra.harry.util.BitSet;
 import org.apache.cassandra.schema.ColumnMetadata;
@@ -598,10 +599,9 @@ public class BytesPartitionState
             clusteringCache = new BijectionCache<>(clusteringComparator);
 
             ValueGenerators.Accessor<Clustering<ByteBuffer>> clusteringAccessor = (offset, clustering) -> clustering.bufferAt(offset);
-            valueGenerators = new ValueGenerators<>(partitionCache, clusteringCache, clusteringAccessor,
-                                                    regularColumnGens, staticColumnGens,
-                                                    pkComparators, ckComparators,
-                                                    regularComparators, staticComparators);
+            valueGenerators = new SharedValueGenerators<>(partitionCache, clusteringCache, clusteringAccessor,
+                                                          regularColumnGens, staticColumnGens, ckComparators,
+                                                          regularComparators, staticComparators);
         }
 
         private Comparator<Object> compareValue(AbstractType<?> type)

test/harry/main/org/apache/cassandra/harry/model/Model.java

@@ -28,22 +28,21 @@ public interface Model
 {
     void validate(Operations.SelectStatement select, List<ResultSetRow> actual);
 
-    class LtsOperationPair
+    interface FullReplay extends Iterable<Visit>
     {
-        final long lts;
-        final int opId;
+    }
 
-        public LtsOperationPair(long lts, int opId)
+    interface PartialReplay
+    {
+        Iterable<Operations.Operation> potentialVisits(long pd);
+    }
+
+    public Model NO_OP = new Model()
+    {
+        @Override
+        public void validate(Operations.SelectStatement select, List<ResultSetRow> actual)
         {
-            this.lts = lts;
-            this.opId = opId;
+
         }
-    }
-
-    interface Replay extends Iterable<Visit>
-    {
-        Visit replay(long lts);
-        Operations.Operation replay(long lts, int opId);
-    }
-
+    };
 }

test/harry/main/org/apache/cassandra/harry/model/PartitionState.java

@@ -56,16 +56,22 @@ public class PartitionState implements Iterable<PartitionState.RowState>
     NavigableMap<Long, RowState> rows;
 
     final ValueGenerators valueGenerators;
-
+    final ValueGenerators.PartitionValues partitionValues;
     public PartitionState(long pd, ValueGenerators valueGenerators)
     {
         this.pd = pd;
-        this.rows = new TreeMap<>(valueGenerators.ckGen().descriptorsComparator());
         this.valueGenerators = valueGenerators;
+        this.partitionValues = valueGenerators.forPd(pd);
+        // TODO: ideally, we want to switch to ck _indexes_ here, since they are still in the value order.
+        //       this is just an unfortunate consequence of going from index to value t descriptor.
+        // Order rows by clustering value rather than by the raw clustering descriptor: for indexed bijections
+        // the descriptor is only a seed and is not monotonic with value, so a natural-ordered map would not
+        // match the clustering order the database returns.
+        this.rows = new TreeMap<Long, RowState>(partitionValues.ckGen()::compare);
         this.staticRow = new RowState(this,
                                       STATIC_CLUSTERING,
-                                      arr(valueGenerators.staticColumnCount(), MagicConstants.NIL_DESCR),
-                                      arr(valueGenerators.staticColumnCount(), MagicConstants.NO_TIMESTAMP));
+                                      arr(partitionValues.staticColumnCount(), MagicConstants.NIL_DESCR),
+                                      arr(partitionValues.staticColumnCount(), MagicConstants.NO_TIMESTAMP));
     }
 
     /**
@@ -78,20 +84,21 @@ public class PartitionState implements Iterable<PartitionState.RowState>
 
     public void writeStatic(long[] sds, long lts)
     {
-        staticRow = updateRowState(staticRow, valueGenerators::staticColumnGen, STATIC_CLUSTERING, sds, lts, false);
+        staticRow = updateRowState(staticRow, partitionValues::staticColumnGen, STATIC_CLUSTERING, sds, lts, false);
     }
 
     public void writeRegular(long cd, long[] vds, long lts, boolean writePrimaryKeyLiveness)
     {
-        rows.compute(cd, (cd_, current) -> updateRowState(current, valueGenerators::regularColumnGen, cd, vds, lts, writePrimaryKeyLiveness));
+        rows.compute(cd, (cd_, current) -> updateRowState(current, partitionValues::regularColumnGen, cd, vds, lts, writePrimaryKeyLiveness));
     }
 
+    // TODO: Make sure to use LTS, as writes can be propagated out of order!
     public void delete(Operations.DeleteRange delete, long lts)
     {
         // TODO: inefficient; need to search for lower/higher bounds
-        rows.entrySet().removeIf(e -> Relations.matchRange(valueGenerators.ckGen(),
-                                                           valueGenerators::ckComparator,
-                                                           valueGenerators.ckColumnCount(),
+        rows.entrySet().removeIf(e -> Relations.matchRange(partitionValues.ckGen(),
+                                                           partitionValues::ckComparator,
+                                                           partitionValues.ckColumnCount(),
                                                            delete.lowerBound(),
                                                            delete.upperBound(),
                                                            delete.lowerBoundRelation(),
@@ -134,9 +141,9 @@ public class PartitionState implements Iterable<PartitionState.RowState>
     private void filterInternal(Operations.SelectRange select)
     {
         // TODO: inefficient; need to search for lower/higher bounds
-        rows.entrySet().removeIf(e -> !Relations.matchRange(valueGenerators.ckGen(),
-                                                            valueGenerators::ckComparator,
-                                                            valueGenerators.ckColumnCount(),
+        rows.entrySet().removeIf(e -> !Relations.matchRange(partitionValues.ckGen(),
+                                                            partitionValues::ckComparator,
+                                                            partitionValues.ckColumnCount(),
                                                             select.lowerBound(),
                                                             select.upperBound(),
                                                             select.lowerBoundRelation(),
@@ -151,10 +158,10 @@ public class PartitionState implements Iterable<PartitionState.RowState>
             Map<Long, Object> cache = new HashMap<>();
             for (Relations.Relation relation : select.ckRelations())
             {
-                Object query = cache.computeIfAbsent(relation.descriptor, valueGenerators.ckGen()::inflate);
-                Object match = cache.computeIfAbsent(e.getValue().cd, valueGenerators.ckGen()::inflate);
-                var accessor = valueGenerators.ckAccessor();
-                if (!relation.kind.match(valueGenerators.ckComparator(relation.column),
+                Object query = cache.computeIfAbsent(relation.descriptor, partitionValues.ckGen()::inflate);
+                Object match = cache.computeIfAbsent(e.getValue().cd, partitionValues.ckGen()::inflate);
+                var accessor = partitionValues.ckAccessor();
+                if (!relation.kind.match(partitionValues.ckComparator(relation.column),
                                          accessor.access(relation.column, match),
                                          accessor.access(relation.column, query)))
                     return true; // true means "no match", so remove from resultset
@@ -162,23 +169,23 @@ public class PartitionState implements Iterable<PartitionState.RowState>
 
             for (Relations.Relation relation : select.regularRelations())
             {
-                Object query = valueGenerators.regularColumnGen(relation.column).inflate(relation.descriptor);
+                Object query = partitionValues.regularColumnGen(relation.column).inflate(relation.descriptor);
                 long descriptor = e.getValue().vds[relation.column];
                 if (MagicConstants.MAGIC_DESCRIPTOR_VALS.contains(descriptor)) // TODO: do we allow UNSET queries?
                     return true;
-                Object match = valueGenerators.regularColumnGen(relation.column).inflate(e.getValue().vds[relation.column]);
-                if (!relation.kind.match(valueGenerators.regularComparator(relation.column), match, query))
+                Object match = partitionValues.regularColumnGen(relation.column).inflate(e.getValue().vds[relation.column]);
+                if (!relation.kind.match(partitionValues.regularComparator(relation.column), match, query))
                     return true;
             }
 
             for (Relations.Relation relation : select.staticRelations())
             {
-                Object query = valueGenerators.staticColumnGen(relation.column).inflate(relation.descriptor);
+                Object query = partitionValues.staticColumnGen(relation.column).inflate(relation.descriptor);
                 long descriptor = e.getValue().partitionState.staticRow.vds[relation.column];
                 if (MagicConstants.MAGIC_DESCRIPTOR_VALS.contains(descriptor)) // TODO: do we allow UNSET queries?
                     return true;
-                Object match = valueGenerators.staticColumnGen(relation.column).inflate(e.getValue().partitionState.staticRow.vds[relation.column]);
-                if (!relation.kind.match(valueGenerators.staticComparator(relation.column), match, query))
+                Object match = partitionValues.staticColumnGen(relation.column).inflate(e.getValue().partitionState.staticRow.vds[relation.column]);
+                if (!relation.kind.match(partitionValues.staticComparator(relation.column), match, query))
                     return true;
             }
 
@@ -239,7 +246,8 @@ public class PartitionState implements Iterable<PartitionState.RowState>
                 if (vds[i] == MagicConstants.UNSET_DESCR)
                     continue;
 
-                assert lts >= currentState.lts[i] : String.format("Out-of-order LTS: %d. Max seen: %s", lts, currentState.lts[i]); // sanity check; we're iterating in lts order
+                // TODO: lts could be out of order with accord, but these asserts could still be useful
+                //assert lts >= currentState.lts[i] : String.format("Out-of-order LTS: %d. Max seen: %s", lts, currentState.lts[i]); // sanity check; we're iterating in lts order
 
                 if (lts != MagicConstants.NO_TIMESTAMP && currentState.lts[i] == lts)
                 {
@@ -254,7 +262,7 @@ public class PartitionState implements Iterable<PartitionState.RowState>
                 }
                 else
                 {
-                    assert lts == MagicConstants.NO_TIMESTAMP || lts > currentState.lts[i];
+                    // assert lts == MagicConstants.NO_TIMESTAMP || lts > currentState.lts[i];
                     currentState.vds[i] = vds[i];
                     currentState.lts[i] = lts;
                 }
@@ -345,12 +353,12 @@ public class PartitionState implements Iterable<PartitionState.RowState>
 
         if (staticRow != null)
         {
-            sb.append("Static row:\n" + staticRow.toString(valueGenerators)).append("\n");
+            sb.append("Static row:\n" + staticRow).append("\n");
             sb.append("\n");
         }
 
         for (RowState row : rows.values())
-            sb.append(row.toString(valueGenerators)).append("\n");
+            sb.append(row.toString()).append("\n");
 
         return sb.toString();
     }
@@ -400,31 +408,26 @@ public class PartitionState implements Iterable<PartitionState.RowState>
             return gen.toString(descr);
         }
 
-        public String toString(ValueGenerators valueGenerators)
+        @Override
+        public String toString()
         {
             if (cd == STATIC_CLUSTERING)
             {
                 return " rowStateRow("
-                       +  valueGenerators.pkGen().toString(partitionState.pd) +
+                       +  partitionState.valueGenerators.pkGen().toString(partitionState.pd) +
                        ", STATIC" +
-                       ", statics(" + toString(partitionState.staticRow.vds, valueGenerators::staticColumnGen) + ")" +
+                       ", statics(" + toString(partitionState.staticRow.vds, partitionState.partitionValues::staticColumnGen) + ")" +
                        ", lts(" + StringUtils.toString(partitionState.staticRow.lts) + ")";
             }
             else
             {
                 return " rowStateRow("
-                       + valueGenerators.pkGen().toString(partitionState.pd) +
-                       ", " + descrToIdxForToString(valueGenerators.ckGen(), cd) +
-                       ", vds(" + toString(vds, valueGenerators::regularColumnGen) + ")" +
+                       + partitionState.valueGenerators.pkGen().toString(partitionState.pd) +
+                       ", " + descrToIdxForToString(partitionState.partitionValues.ckGen(), cd) +
+                       ", vds(" + toString(vds, partitionState.partitionValues::regularColumnGen) + ")" +
                        ", lts(" + StringUtils.toString(lts) + ")";
             }
         }
-
-        @Override
-        public String toString()
-        {
-            return toString(partitionState.valueGenerators);
-        }
     }
 
     public static long[] arr(int length, long fill)

test/harry/main/org/apache/cassandra/harry/model/PartitionStateBuilder.java

@@ -32,7 +32,7 @@ import org.apache.cassandra.harry.gen.ValueGenerators;
 import static org.apache.cassandra.harry.op.Operations.DeleteColumnsOp;
 import static org.apache.cassandra.harry.op.Operations.DeleteRange;
 import static org.apache.cassandra.harry.op.Operations.DeleteRow;
-import static org.apache.cassandra.harry.op.Operations.Kind.INSERT;
+import static org.apache.cassandra.harry.op.Kind.INSERT;
 import static org.apache.cassandra.harry.op.Operations.Operation;
 import static org.apache.cassandra.harry.op.Operations.WriteOp;
 
@@ -46,9 +46,9 @@ class PartitionStateBuilder extends VisitExecutor
     private final List<Operation> rangeDeletes;
     private final List<Operation> writes;
     private final List<Operation> columnDeletes;
-    private final ValueGenerators valueGenerators;
+    private final ValueGenerators<Object[], Object[]> valueGenerators;
 
-    PartitionStateBuilder(ValueGenerators valueGenerators,
+    PartitionStateBuilder(ValueGenerators<Object[], Object[]> valueGenerators,
                           PartitionState partitionState)
     {
         this.valueGenerators = valueGenerators;
@@ -122,7 +122,7 @@ class PartitionStateBuilder extends VisitExecutor
 
             if (hadTrackingRowWrite)
             {
-                long[] statics = new long[valueGenerators.staticColumnCount()];
+                long[] statics = new long[valueGenerators.forPd(writeOp.pd).staticColumnCount()];
                 Arrays.fill(statics, MagicConstants.UNSET_DESCR);
                 partitionState.writeStatic(statics, lts);
             }

test/harry/main/org/apache/cassandra/harry/model/QuiescentChecker.java

@@ -25,11 +25,10 @@ import java.util.List;
 import java.util.NavigableMap;
 
 import accord.utils.Invariants;
-
-import org.apache.cassandra.harry.execution.DataTracker;
-import org.apache.cassandra.harry.execution.ResultSetRow;
 import org.apache.cassandra.harry.gen.ValueGenerators;
+import org.apache.cassandra.harry.op.ClusteringOrderBy;
 import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.execution.ResultSetRow;
 
 import static org.apache.cassandra.harry.MagicConstants.LTS_UNKNOWN;
 import static org.apache.cassandra.harry.MagicConstants.NIL_DESCR;
@@ -41,19 +40,15 @@ import static org.apache.cassandra.harry.MagicConstants.UNSET_DESCR;
  * Unfortunately model is not reducible to a simple interface of apply/validate, at least not if
  * we intend to support concurrent validation and in-flight/timed out queries. Validation needs to
  * know which queries might have been applied.
- *
- *
  */
 public class QuiescentChecker implements Model
 {
-    private final DataTracker tracker;
-    private final Replay replay;
-    private final ValueGenerators valueGenerators;
+    private final PartialReplay replay;
+    private final ValueGenerators<Object[], Object[]> valueGenerators;
 
-    public QuiescentChecker(ValueGenerators valueGenerators, DataTracker tracker, Replay replay)
+    public QuiescentChecker(ValueGenerators<Object[], Object[]> valueGenerators, PartialReplay replay)
     {
         this.valueGenerators = valueGenerators;
-        this.tracker = tracker;
         this.replay = replay;
     }
 
@@ -64,20 +59,19 @@ public class QuiescentChecker implements Model
         PartitionStateBuilder stateBuilder = new PartitionStateBuilder(valueGenerators, partitionState);
 
         long prevLts = -1;
-        for (LtsOperationPair potentialVisit : tracker.potentialVisits(select.pd()))
+
+        // In case of quiescent checkers, all potential visits have to be finished
+        for (Operations.Operation potentialVisit : replay.potentialVisits(select.pd()))
         {
-            if (tracker.isFinished(potentialVisit.lts))
+            if (potentialVisit.lts() != prevLts)
             {
-                if (potentialVisit.lts != prevLts)
-                {
-                    if (prevLts != -1)
-                        stateBuilder.endLts(prevLts);
-                    stateBuilder.beginLts(potentialVisit.lts);
-                    prevLts = potentialVisit.lts;
-                }
-                Operations.Operation op = replay.replay(potentialVisit.lts, potentialVisit.opId);
-                stateBuilder.operation(op);
+                if (prevLts != -1)
+                    stateBuilder.endLts(prevLts);
+                stateBuilder.beginLts(potentialVisit.lts());
+                prevLts = potentialVisit.lts();
             }
+
+            stateBuilder.operation(potentialVisit);
         }
 
         // Close last open LTS
@@ -85,7 +79,7 @@ public class QuiescentChecker implements Model
             stateBuilder.endLts(prevLts);
 
         partitionState.filter(select);
-        if (select.orderBy() == Operations.ClusteringOrderBy.DESC)
+        if (select.orderBy() == ClusteringOrderBy.DESC)
         {
             partitionState.reverse();
         }
@@ -94,7 +88,7 @@ public class QuiescentChecker implements Model
     }
 
     // TODO: reverse
-    public static void validate(ValueGenerators valueGenerators, PartitionState partitionState, List<ResultSetRow> actualRows)
+    public static void validate(ValueGenerators<Object[], Object[]> valueGenerators, PartitionState partitionState, List<ResultSetRow> actualRows)
     {
         Iterator<ResultSetRow> actual = actualRows.iterator();
         NavigableMap<Long, PartitionState.RowState> expectedRows = partitionState.rows();
@@ -148,7 +142,7 @@ public class QuiescentChecker implements Model
                                               "Found a row in the model that is not present in the resultset:" +
                                               "\nExpected: %s" +
                                               "\nActual: %s",
-                                              expectedRowState.toString(valueGenerators),
+                                              expectedRowState.toString(),
                                               actualRowState.toString(valueGenerators));
             }
 
@@ -158,7 +152,7 @@ public class QuiescentChecker implements Model
                                               "Returned row state doesn't match the one predicted by the model:" +
                                               "\nExpected: %s" +
                                               "\nActual:   %s.",
-                                              expectedRowState.toString(valueGenerators),
+                                              expectedRowState.toString(),
                                               actualRowState.toString(valueGenerators));
 
             if (!ltsEqual(expectedRowState.lts, actualRowState.lts))
@@ -167,7 +161,7 @@ public class QuiescentChecker implements Model
                                               "Timestamps in the row state don't match ones predicted by the model:" +
                                               "\nExpected: %s" +
                                               "\nActual:   %s.",
-                                              expectedRowState.toString(valueGenerators),
+                                              expectedRowState.toString(),
                                               actualRowState.toString(valueGenerators));
 
             if (partitionState.staticRow() != null || actualRowState.hasStaticColumns())
@@ -242,7 +236,7 @@ public class QuiescentChecker implements Model
                                           "Returned static row state doesn't match the one predicted by the model:" +
                                           "\nExpected: %s (%s)" +
                                           "\nActual:   %s (%s).",
-                                          descriptorsToString(staticRow.vds), staticRow.toString(valueGenerators),
+                                          descriptorsToString(staticRow.vds), staticRow.toString(),
                                           descriptorsToString(actualRowState.sds), actualRowState);
 
         if (!ltsEqual(staticRow.lts, actualRowState.slts))
@@ -251,7 +245,7 @@ public class QuiescentChecker implements Model
                                           "Timestamps in the static row state don't match ones predicted by the model:" +
                                           "\nExpected: %s (%s)" +
                                           "\nActual:   %s (%s).",
-                                          Arrays.toString(staticRow.lts), staticRow.toString(valueGenerators),
+                                          Arrays.toString(staticRow.lts), staticRow.toString(),
                                           Arrays.toString(actualRowState.slts), actualRowState);
     }
 
@@ -277,7 +271,7 @@ public class QuiescentChecker implements Model
         StringBuilder builder = new StringBuilder();
 
         for (PartitionState.RowState rowState : collection)
-            builder.append(rowState.toString(valueGenerators)).append("\n");
+            builder.append(rowState.toString()).append("\n");
         return builder.toString();
     }
 

test/harry/main/org/apache/cassandra/harry/model/TokenPlacementModel.java

@@ -981,14 +981,21 @@ public class TokenPlacementModel
         private final int dcIdx;
         private final int rackIdx;
         private final Lookup lookup;
+        private final String fqdn;
 
         public Node(int tokenIdx, int idx, int dcIdx, int rackIdx, Lookup lookup)
+        {
+            this(tokenIdx, idx, dcIdx, rackIdx, lookup, null);
+        }
+
+        public Node(int tokenIdx, int idx, int dcIdx, int rackIdx, Lookup lookup, String fqdn)
         {
             this.tokenIdx = tokenIdx;
             this.nodeIdx = idx;
             this.dcIdx = dcIdx;
             this.rackIdx = rackIdx;
             this.lookup = lookup;
+            this.fqdn = fqdn;
         }
 
         public String id()
@@ -996,6 +1003,16 @@ public class TokenPlacementModel
             return lookup.id(nodeIdx);
         }
 
+        public String fqdn()
+        {
+            return fqdn != null ? fqdn : id();
+        }
+
+        public Node overrideFQDN(String fqdn)
+        {
+            return new Node(tokenIdx, nodeIdx, dcIdx, rackIdx, lookup, fqdn);
+        }
+
         public int idx()
         {
             return nodeIdx;
@@ -1033,22 +1050,22 @@ public class TokenPlacementModel
 
         public Node withNewToken()
         {
-            return new Node(tokenIdx + 100_000, nodeIdx, dcIdx, rackIdx, lookup);
+            return new Node(tokenIdx + 100_000, nodeIdx, dcIdx, rackIdx, lookup, fqdn);
         }
 
         public Node withToken(int tokenIdx)
         {
-            return new Node(tokenIdx, nodeIdx, dcIdx, rackIdx, lookup);
+            return new Node(tokenIdx, nodeIdx, dcIdx, rackIdx, lookup, fqdn);
         }
 
         public Node overrideToken(long override)
         {
-            return new Node(tokenIdx, nodeIdx, dcIdx, rackIdx, lookup.forceToken(tokenIdx, override));
+            return new Node(tokenIdx, nodeIdx, dcIdx, rackIdx, lookup.forceToken(tokenIdx, override), fqdn);
         }
 
         public Node withNewRack(String newRack)
         {
-            return new Node(tokenIdx, nodeIdx, dcIdx, lookup.rackIdx(newRack), lookup);
+            return new Node(tokenIdx, nodeIdx, dcIdx, lookup.rackIdx(newRack), lookup, fqdn);
         }
 
         public Murmur3Partitioner.LongToken longToken()

test/harry/main/org/apache/cassandra/harry/op/ClusteringOrderBy.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.op;
+
+/**
+ * ClusteringOrder by should be understood in terms of how we're going to iterate through this partition
+ * (in other words, if first clustering component order is DESC, we'll iterate in ASC order)
+ */
+public enum ClusteringOrderBy
+{
+    ASC, DESC
+}

test/harry/main/org/apache/cassandra/harry/op/Kind.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.op;
+
+public enum Kind
+{
+    /**
+     * Custom operation such as flush
+     */
+    CUSTOM(false),
+
+    UPDATE(true),
+    INSERT(true),
+
+    DELETE_PARTITION(true),
+    DELETE_ROW(false),
+    DELETE_COLUMNS(true),
+    DELETE_RANGE(false),
+
+    SELECT_PARTITION(true),
+    SELECT_ROW(false),
+    SELECT_RANGE(true),
+    SELECT_CUSTOM(true);
+    public final boolean partititonLevel;
+
+    Kind(boolean partitionLevel)
+    {
+        this.partititonLevel = partitionLevel;
+    }
+}

test/harry/main/org/apache/cassandra/harry/op/Operations.java

@@ -28,11 +28,11 @@ import accord.utils.Invariants;
 import org.apache.cassandra.harry.ColumnSpec;
 import org.apache.cassandra.harry.MagicConstants;
 import org.apache.cassandra.harry.Relations;
-import org.apache.cassandra.harry.SchemaSpec;
 import org.apache.cassandra.harry.util.BitSet;
 
 public class Operations
 {
+    // TODO: remove lts from every op; leave only for descriptor
     public static class WriteOp extends PartitionOperation
     {
         private final long cd;
@@ -381,34 +381,6 @@ public class Operations
         }
     }
 
-    public enum Kind
-    {
-        /**
-         * Custom operation such as flush
-         */
-        CUSTOM(false),
-
-        UPDATE(true),
-        INSERT(true),
-
-        DELETE_PARTITION(true),
-        DELETE_ROW(false),
-        DELETE_COLUMNS(true),
-        DELETE_RANGE(false),
-
-        SELECT_PARTITION(true),
-        SELECT_ROW(false),
-        SELECT_RANGE(true),
-        SELECT_CUSTOM(true);
-        public final boolean partititonLevel;
-
-        Kind(boolean partitionLevel)
-        {
-            this.partititonLevel = partitionLevel;
-        }
-
-    }
-
     public static class CustomRunnableOperation implements Operation
     {
         public final long lts;
@@ -440,124 +412,4 @@ public class Operations
             return Kind.CUSTOM;
         }
     }
-
-    /**
-     * ClusteringOrder by should be understood in terms of how we're going to iterate through this partition
-     * (in other words, if first clustering component order is DESC, we'll iterate in ASC order)
-     */
-    public enum ClusteringOrderBy
-    {
-        ASC, DESC
-    }
-
-    public interface Selection
-    {
-        // TODO: allow expressions here
-        Collection<ColumnSpec<?>> columns();
-        boolean includeTimestamps();
-        boolean isWildcard();
-
-        boolean selects(ColumnSpec<?> column);
-        boolean selectsAllOf(List<ColumnSpec<?>> subSelection);
-        int indexOf(ColumnSpec<?> column);
-
-        static Selection fromBitSet(BitSet bitSet, SchemaSpec schema)
-        {
-            if (bitSet == MagicConstants.ALL_COLUMNS)
-            {
-                Map<ColumnSpec<?>, Integer> columns = new HashMap<>();
-                for (int i = 0; i < schema.allColumnInSelectOrder.size(); i++)
-                    columns.put(schema.allColumnInSelectOrder.get(i), i);
-                return new Wildcard(columns);
-            }
-            else
-            {
-                Invariants.require(schema.allColumnInSelectOrder.size() == bitSet.size());
-                Map<ColumnSpec<?>, Integer> columns = new HashMap<>();
-                for (int i = 0; i < schema.allColumnInSelectOrder.size(); i++)
-                {
-                    if (bitSet.isSet(i))
-                        columns.put(schema.allColumnInSelectOrder.get(i), i);
-                }
-                // TODO: timestamp
-                return new Columns(columns, false);
-            }
-        }
-    }
-
-    public static class Wildcard extends Columns
-    {
-        private Wildcard(Map<ColumnSpec<?>, Integer> columns)
-        {
-            super(columns, false);
-        }
-
-        @Override
-        public Collection<ColumnSpec<?>> columns()
-        {
-            return columns.keySet();
-        }
-
-        @Override
-        public boolean includeTimestamps()
-        {
-            return false;
-        }
-
-        @Override
-        public boolean isWildcard()
-        {
-            return true;
-        }
-    }
-
-    public static class Columns implements Selection
-    {
-        final Map<ColumnSpec<?>, Integer> columns;
-        final boolean includeTimestamp;
-
-        public Columns(Map<ColumnSpec<?>, Integer> columns, boolean includeTimestamp)
-        {
-            this.columns = columns;
-            this.includeTimestamp = includeTimestamp;
-        }
-
-        @Override
-        public Collection<ColumnSpec<?>> columns()
-        {
-            return columns.keySet();
-        }
-
-        @Override
-        public boolean includeTimestamps()
-        {
-            return includeTimestamp;
-        }
-
-        @Override
-        public boolean isWildcard()
-        {
-            return false;
-        }
-
-        public boolean selects(ColumnSpec<?> column)
-        {
-            return columns.containsKey(column);
-        }
-
-        public boolean selectsAllOf(List<ColumnSpec<?>> subSelection)
-        {
-            for (ColumnSpec<?> column : subSelection)
-            {
-                if (!selects(column))
-                    return false;
-            }
-            return true;
-        }
-
-        public int indexOf(ColumnSpec<?> column)
-        {
-            return columns.get(column);
-        }
-    }
 }

test/harry/main/org/apache/cassandra/harry/op/Selection.java

@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.op;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.MagicConstants;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.util.BitSet;
+
+public interface Selection
+{
+    // TODO: allow expressions here
+    Collection<ColumnSpec<?>> columns();
+
+    boolean includeTimestamps();
+
+    boolean isWildcard();
+
+    boolean selects(ColumnSpec<?> column);
+
+    boolean selectsAllOf(List<ColumnSpec<?>> subSelection);
+
+    int indexOf(ColumnSpec<?> column);
+
+    static Selection fromBitSet(BitSet bitSet, SchemaSpec schema)
+    {
+        if (bitSet == MagicConstants.ALL_COLUMNS)
+        {
+            Map<ColumnSpec<?>, Integer> columns = new HashMap<>();
+            for (int i = 0; i < schema.allColumnInSelectOrder.size(); i++)
+                columns.put(schema.allColumnInSelectOrder.get(i), i);
+            return new Wildcard(columns);
+        }
+        else
+        {
+            Invariants.require(schema.allColumnInSelectOrder.size() == bitSet.size());
+            Map<ColumnSpec<?>, Integer> columns = new HashMap<>();
+            for (int i = 0; i < schema.allColumnInSelectOrder.size(); i++)
+            {
+                if (bitSet.isSet(i))
+                    columns.put(schema.allColumnInSelectOrder.get(i), i);
+            }
+            // TODO: timestamp
+            return new Columns(columns, false);
+        }
+    }
+
+    class Columns implements Selection
+    {
+        final Map<ColumnSpec<?>, Integer> columns;
+        final boolean includeTimestamp;
+
+        public Columns(Map<ColumnSpec<?>, Integer> columns, boolean includeTimestamp)
+        {
+            this.columns = columns;
+            this.includeTimestamp = includeTimestamp;
+        }
+
+        @Override
+        public Collection<ColumnSpec<?>> columns()
+        {
+            return columns.keySet();
+        }
+
+        @Override
+        public boolean includeTimestamps()
+        {
+            return includeTimestamp;
+        }
+
+        @Override
+        public boolean isWildcard()
+        {
+            return false;
+        }
+
+        public boolean selects(ColumnSpec<?> column)
+        {
+            return columns.containsKey(column);
+        }
+
+        public boolean selectsAllOf(List<ColumnSpec<?>> subSelection)
+        {
+            for (ColumnSpec<?> column : subSelection)
+            {
+                if (!selects(column))
+                    return false;
+            }
+            return true;
+        }
+
+        public int indexOf(ColumnSpec<?> column)
+        {
+            return columns.get(column);
+        }
+    }
+
+    class Wildcard extends Columns
+    {
+        private Wildcard(Map<ColumnSpec<?>, Integer> columns)
+        {
+            super(columns, false);
+        }
+
+        @Override
+        public Collection<ColumnSpec<?>> columns()
+        {
+            return columns.keySet();
+        }
+
+        @Override
+        public boolean includeTimestamps()
+        {
+            return false;
+        }
+
+        @Override
+        public boolean isWildcard()
+        {
+            return true;
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/op/Visit.java

@@ -21,42 +21,50 @@ package org.apache.cassandra.harry.op;
 import java.util.HashSet;
 import java.util.Set;
 
-import org.junit.Assert;
-
+import accord.utils.Invariants;
 import org.apache.cassandra.harry.op.Operations.Operation;
 
 public class Visit
 {
     public final long lts;
+    // TODO: specialize single-op visits
     public final Operation[] operations;
-    public final Set<Long> visitedPartitions;
+    public final long[] visitedPartitions;
 
-    public final boolean selectOnly;
+    public final boolean validating;
     public final boolean hasCustom;
 
     public Visit(long lts, Operation[] operations)
     {
-        Assert.assertTrue(operations.length > 0);
+        Invariants.require(operations.length > 0);
         this.lts = lts;
         this.operations = operations;
-        this.visitedPartitions = new HashSet<>();
         boolean selectOnly = true;
         boolean hasCustom = false;
+        Set<Long> visitedPartitions = new HashSet<>();
         for (Operation operation : operations)
         {
-            if (operation.kind() == Operations.Kind.CUSTOM)
+            if (operation.kind() == Kind.CUSTOM)
                 hasCustom = true;
             if (selectOnly && !(operation instanceof Operations.SelectStatement))
                 selectOnly = false;
 
             if (operation instanceof Operations.PartitionOperation)
                 visitedPartitions.add(((Operations.PartitionOperation) operation).pd());
-
         }
-        this.selectOnly = selectOnly;
+        this.visitedPartitions = new long[visitedPartitions.size()];
+        int idx = 0;
+        for (Long partition : visitedPartitions)
+            this.visitedPartitions[idx++] = partition;
+        this.validating = selectOnly;
         this.hasCustom = hasCustom;
     }
 
+    public boolean validating()
+    {
+        return validating;
+    }
+
     public String toString()
     {
         if (operations.length == 1)

test/harry/main/org/apache/cassandra/harry/stress/ActivePartition.java

@@ -0,0 +1,744 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.IntSupplier;
+import java.util.function.LongConsumer;
+import java.util.function.LongFunction;
+import java.util.function.LongPredicate;
+import java.util.function.LongSupplier;
+import java.util.stream.Collectors;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.dsl.IndexedValueGenerators;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.gen.EntropySource;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.IndexGenerators;
+import org.apache.cassandra.harry.gen.InvertibleGenerator;
+import org.apache.cassandra.harry.gen.rng.JdkRandomEntropySource;
+import org.apache.cassandra.harry.gen.rng.PureRng;
+import org.apache.cassandra.harry.gen.rng.SeedableEntropySource;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+import org.apache.cassandra.harry.util.IteratorsUtil;
+import org.apache.cassandra.utils.LazyToString;
+
+import static org.apache.cassandra.harry.SchemaSpec.cumulativeEntropy;
+import static org.apache.cassandra.harry.SchemaSpec.forKeys;
+import static org.apache.cassandra.harry.dsl.HistoryBuilder.keyComparator;
+import static org.apache.cassandra.harry.dsl.IndexedValueGenerators.IndexedPartitionValues;
+import static org.apache.cassandra.harry.gen.InvertibleGenerator.fromType;
+
+/**
+ *
+ */
+public final class ActivePartition extends IndexedPartitionValues
+{
+    /**
+     * A helper class to convert between descriptors and indices _for partitions_
+     */
+    public enum DescriptorIndexBijection
+    {
+        INSTANCE;
+
+        private final long stream = 0xffeeddccbbaal;
+        private final PureRng rng = new PureRng.PCGFast(0xaabbccddeeffl);
+
+        public long toIdx(long pd)
+        {
+            return rng.sequenceNumber(pd, stream);
+        }
+
+        public long toPd(long idx)
+        {
+            return rng.randomNumber(idx, stream);
+        }
+
+    }
+
+    public final long idx;
+    public final long pd;
+
+    // TODO: document why two
+    private final HistoryBuilder.IndexedBijection<Object[]> cachingPkGen;
+    private final HistoryBuilder.IndexedBijection<Object[]> rawGen;
+    private final AtomicInteger refCount = new AtomicInteger(0);
+
+    private final LongConsumer cleanup;
+
+    ActivePartition(long pkIdx,
+                    long pd,
+                    HistoryBuilder.IndexedBijection<Object[]> cachingPkGen,
+                    HistoryBuilder.IndexedBijection<Object[]> rawGen,
+                    HistoryBuilder.IndexedBijection<Object[]> ckGen,
+                    List<HistoryBuilder.IndexedBijection<Object>> regularColumnGens,
+                    List<HistoryBuilder.IndexedBijection<Object>> staticColumnGens,
+                    List<Comparator<Object>> pkComparators,
+                    List<Comparator<Object>> ckComparators,
+                    List<Comparator<Object>> regularComparators,
+                    List<Comparator<Object>> staticComparators,
+                    LongConsumer cleanup)
+    {
+        super(ckGen, regularColumnGens, staticColumnGens, ckComparators, regularComparators, staticComparators,
+              IndexGenerators.uniform(ckGen),
+              IndexGenerators.uniform(regularColumnGens),
+              IndexGenerators.uniform(regularColumnGens));
+
+        this.cachingPkGen = cachingPkGen;
+        this.rawGen = rawGen;
+        Invariants.require(DescriptorIndexBijection.INSTANCE.toIdx(pd) == pkIdx);
+        Invariants.require(DescriptorIndexBijection.INSTANCE.toPd(pkIdx) == pd);
+        this.idx = pkIdx;
+        this.pd = pd;
+
+        this.cleanup = v -> {
+            cleanup.accept(v);
+            ckGen.discard();
+        };
+    }
+
+    public HistoryBuilder.IndexedBijection<Object[]> pkGen()
+    {
+        return cachingPkGen;
+    }
+
+    private static class ObjectWrapper
+    {
+        public final Object[] value;
+
+        private ObjectWrapper(Object[] value)
+        {
+            this.value = value;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (o == null || getClass() != o.getClass()) return false;
+            ObjectWrapper that = (ObjectWrapper) o;
+            return Objects.deepEquals(value, that.value);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Arrays.hashCode(value);
+        }
+    }
+
+    public void ref()
+    {
+        refCount.incrementAndGet();
+    }
+
+    public void deref()
+    {
+        int v = refCount.decrementAndGet();
+        Invariants.require(v >= 0);
+        if (v == 0)
+            cleanup.accept(pd);
+    }
+
+    /**
+     * It is easiest to generate an operation for a given partition based knowing what values partition may
+     * potentially hold.
+     *
+     * This class is _not_ thread safe
+     */
+    public static class Partitions extends IndexedValueGenerators
+    {
+        final SchemaSpec schema;
+        final Distribution rowPopulation;
+        final VisitGenerator.ColumnPopulation columnPopulation;
+        final List<ActivePartition> activePartitions;
+        final VisitedPartitions visitedPartitions;
+        final Map<Long, ActivePartition> partitionCache = new ConcurrentHashMap<>();
+        final AtomicLong nextPartitionIdx = new AtomicLong();
+        final RotationStrategy rotationStrategy;
+
+        final long minPartitionIdx;
+        final long maxPartitionIdx;
+        final long initialLts;
+
+        LongConsumer onRemove;
+
+        public Partitions(SchemaSpec schema,
+                          Distribution rowPopulation,
+                          VisitGenerator.ColumnPopulation columnPopulation,
+                          RotationStrategy rotationStrategy)
+        {
+            this(schema, rowPopulation, columnPopulation, rotationStrategy, 0, Long.MAX_VALUE, 0);
+        }
+
+        public Partitions(SchemaSpec schema,
+                          Distribution rowPopulation,
+                          VisitGenerator.ColumnPopulation columnPopulation,
+                          RotationStrategy rotationStrategy,
+                          long minPartitionIdx,
+                          long maxPartitionIdx)
+        {
+            this(schema, rowPopulation, columnPopulation, rotationStrategy, minPartitionIdx, maxPartitionIdx, 0);
+        }
+
+        public Partitions(SchemaSpec schema,
+                          Distribution rowPopulation,
+                          VisitGenerator.ColumnPopulation columnPopulation,
+                          RotationStrategy rotationStrategy,
+                          long minPartitionIdx,
+                          long maxPartitionIdx,
+                          long initialLts)
+        {
+            super(new PartitionKeyGen(schema));
+            Invariants.require(minPartitionIdx >= 0, "minPartitionIdx must be non-negative: %d", minPartitionIdx);
+            Invariants.require(maxPartitionIdx > minPartitionIdx, "maxPartitionIdx must be greater than minPartitionIdx: %d > %d", maxPartitionIdx, minPartitionIdx);
+            this.schema = schema;
+            this.rowPopulation = rowPopulation;
+            this.columnPopulation = columnPopulation;
+            this.activePartitions = new ArrayList<>(rotationStrategy.targetSize());
+            this.minPartitionIdx = minPartitionIdx;
+            this.maxPartitionIdx = maxPartitionIdx;
+            this.initialLts = initialLts;
+            this.nextPartitionIdx.set(minPartitionIdx);
+            this.visitedPartitions = new VisitedPartitions(minPartitionIdx);
+            this.onRemove = (pd_) -> {
+                Invariants.nonNull(partitionCache.remove(pd_));
+                pkGenInternal().cleanup(pd_);
+                long pdIdx = DescriptorIndexBijection.INSTANCE.toIdx(pd_);
+                visitedPartitions.add(pdIdx);
+            };
+            this.rotationStrategy = rotationStrategy;
+        }
+
+        /**
+         * Populates the active partitions by replaying all partition switches from LTS 0 up to
+         * {@code initialLts}, so that the active partitions and visited partitions are in the
+         * correct state for resuming from that LTS.
+         *
+         * When {@code initialLts} is 0, this simply creates the initial set of active partitions.
+         *
+         * During replay, we track only partition indices without creating full
+         * {@link ActivePartition} state. The actual partition objects are only created at the
+         * end, once the final set of active partition indices is known.
+         */
+        public void populate()
+        {
+            List<Long> activeIdxs = new ArrayList<>(rotationStrategy.targetSize());
+
+            for (int i = 0; i < rotationStrategy.targetSize(); i++)
+                activeIdxs.add(advanceNextPartitionIdx());
+
+            for (long lts = 0; lts < initialLts; lts++)
+            {
+                if (!rotationStrategy.shouldSwitch(lts))
+                    continue;
+
+                applyActions(lts,
+                             activeIdxs::size,
+                             this::advanceNextPartitionIdx,
+                             (pos, newIdx) -> { visitedPartitions.add(activeIdxs.get(pos)); activeIdxs.set(pos, newIdx); },
+                             (visitedPd) -> activeIdxs.contains(DescriptorIndexBijection.INSTANCE.toIdx(visitedPd)),
+                             (pos, visitedIdx) -> { visitedPartitions.add(activeIdxs.get(pos)); activeIdxs.set(pos, visitedIdx); },
+                             pos -> DescriptorIndexBijection.INSTANCE.toPd(activeIdxs.get(pos)),
+                             action -> {});
+            }
+
+            // Now inflate all the active partition objects from the final set of indices
+            for (long idx : activeIdxs)
+            {
+                ActivePartition activePartition = byIdx(idx);
+                activePartition.ref();
+                partitionCache.put(activePartition.pd, activePartition);
+                activePartitions.add(activePartition);
+            }
+        }
+
+        private long advanceNextPartitionIdx()
+        {
+            long idx = nextPartitionIdx.getAndIncrement();
+            Invariants.require(idx < maxPartitionIdx, "Exhausted partition index space: %d >= %d", idx, maxPartitionIdx);
+            return idx;
+        }
+
+        private void applyActions(long lts,
+                                  IntSupplier activeSize,
+                                  LongSupplier createNew,
+                                  BiConsumer<Integer, Long> replaceWithNew,
+                                  LongPredicate activePd,
+                                  BiConsumer<Integer, Long> replaceWithVisited,
+                                  java.util.function.IntToLongFunction pdAtPosition,
+                                  Consumer<RotationStrategy.PartitionAction> onAction)
+        {
+            RotationStrategy.PartitionAction[] actions = SeedableEntropySource.computeWithSeed(lts, rotationStrategy::generate);
+
+            for (int i = 0; i < actions.length; i++)
+            {
+                RotationStrategy.PartitionAction action = actions[i];
+                int size = activeSize.getAsInt();
+                switch (action)
+                {
+                    case REPLACE_WITH_NEW:
+                    {
+                        if (size == 0)
+                            continue;
+                        int remove = SeedableEntropySource.computeWithSeed(Util.hash(lts, i), rng -> rng.nextInt(size));
+                        // Skip eviction with probability proportional to log2 of partition size
+                        long candidatePd = pdAtPosition.applyAsLong(remove);
+                        int partitionSize = Math.toIntExact(rowPopulation.next(candidatePd));
+                        boolean evict = SeedableEntropySource.computeWithSeed(Util.hash(lts, i),
+                                                                              rng -> rng.nextInt(Math.max(1, Integer.highestOneBit(partitionSize))) == 0);
+                        if (!evict)
+                            continue;
+                        long newIdx = createNew.getAsLong();
+                        replaceWithNew.accept(remove, newIdx);
+                        break;
+                    }
+                    case REPLACE_WITH_VISITED:
+                    {
+                        if (size == 0)
+                            continue;
+                        int remove = SeedableEntropySource.computeWithSeed(Util.hash(lts, i), rng -> rng.nextInt(size));
+                        // Skip eviction with probability proportional to log2 of partition size
+                        long candidatePd = pdAtPosition.applyAsLong(remove);
+                        int partitionSize = Math.toIntExact(rowPopulation.next(candidatePd));
+                        boolean evict = SeedableEntropySource.computeWithSeed(Util.hash(lts, i),
+                                                                              rng -> rng.nextInt(Math.max(1, Integer.highestOneBit(partitionSize))) == 0);
+                        if (!evict)
+                            continue;
+                        long visitedIdx = visitedPartitions.getBySeed(Util.hash(lts, i));
+                        long visitedPd = visitedIdx < 0 ? -1 : DescriptorIndexBijection.INSTANCE.toPd(visitedIdx);
+                        if (visitedPd < 0 || activePd.test(visitedPd))
+                        {
+                            // No visited partitions available or picked one is still active; fall back to new
+                            long newIdx = createNew.getAsLong();
+                            replaceWithNew.accept(remove, newIdx);
+                        }
+                        else
+                        {
+                            replaceWithVisited.accept(remove, visitedIdx);
+                        }
+                        break;
+                    }
+                }
+                onAction.accept(action);
+            }
+        }
+
+        /**
+         * Add a callback to be triggered when partition is phased out.
+         *
+         * TODO (consider): This might start racing when we allow adding partitions back.
+         */
+        public void onRemove(LongConsumer consumer)
+        {
+            LongConsumer prev = this.onRemove;
+            this.onRemove = pd -> {
+                prev.accept(pd);
+                consumer.accept(pd);
+            };
+        }
+
+        // TODO: biased partition picker
+        public ActivePartition pick(EntropySource entropySource)
+        {
+            ActivePartition picked = activePartitions.get(entropySource.nextInt(activePartitions.size()));
+            Invariants.require(picked.refCount.get() > 0);
+            Invariants.require(partitionCache.containsKey(picked.pd));
+            return picked;
+        }
+
+        @Override
+        public Generator<Long> pkIdxGen()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        private PartitionKeyGen pkGenInternal()
+        {
+            return (PartitionKeyGen) super.pkGen();
+        }
+
+        private ActivePartition byIdx(long idx)
+        {
+            long pd = DescriptorIndexBijection.INSTANCE.toPd(idx);
+            ActivePartition partition = createActivePartition(idx, pd, schema, rowPopulation, columnPopulation, (HistoryBuilder.IndexedBijection<Object[]>) pkGen, onRemove);
+            pkGenInternal().ensure(pd, partition.rawGen::inflate);
+            return partition;
+        }
+
+        @Override
+        public ActivePartition forPd(long pd)
+        {
+            return Invariants.nonNull(partitionCache.get(pd));
+        }
+
+        public void maybeSwitchPartition(long lts, Consumer<RotationStrategy.PartitionAction> consumer)
+        {
+            if (!rotationStrategy.shouldSwitch(lts))
+                return;
+
+            applyActions(lts,
+                         activePartitions::size,
+                         () -> {
+                             long idx = advanceNextPartitionIdx();
+                             ActivePartition ap = byIdx(idx);
+                             ap.ref();
+                             partitionCache.put(ap.pd, ap);
+                             return idx;
+                         },
+                         (pos, newIdx) -> {
+                             ActivePartition next = Invariants.nonNull(partitionCache.get(DescriptorIndexBijection.INSTANCE.toPd(newIdx)));
+                             activePartitions.set(pos, next).deref();
+                         },
+                         partitionCache::containsKey,
+                         (pos, visitedIdx) -> {
+                             ActivePartition next = byIdx(visitedIdx);
+                             next.ref();
+                             partitionCache.put(next.pd, next);
+                             activePartitions.set(pos, next).deref();
+                         },
+                         pos -> activePartitions.get(pos).pd,
+                         consumer);
+        }
+    }
+
+    public static class PartitionKeyGen implements HistoryBuilder.IndexedBijection<Object[]>
+    {
+        final SchemaSpec schema;
+
+        final Map<ObjectWrapper, Long> deflate = new HashMap<>();
+        final Map<Long, Object[]> inflate = new HashMap<>();
+
+        public PartitionKeyGen(SchemaSpec schema)
+        {
+            this.schema = schema;
+        }
+
+        public void cleanup(long pd)
+        {
+            Object[] values = Invariants.nonNull(inflate.remove(pd));
+            Invariants.nonNull(deflate.remove(new ObjectWrapper(values)));
+        }
+
+        public void ensure(long pd, LongFunction<Object[]> value)
+        {
+            if (!inflate.containsKey(pd))
+            {
+                // TODO: need to extract pkgen from inside value descriptors
+                Object[] values = value.apply(pd);
+                inflate.put(pd, values);
+                deflate.put(new ObjectWrapper(values), pd);
+            }
+        }
+
+        @Override
+        public Object[] inflate(long pd)
+        {
+            return Invariants.nonNull(inflate.get(pd));
+        }
+
+        @Override
+        public long deflate(Object[] value)
+        {
+            return Invariants.nonNull(deflate.get(new ObjectWrapper(value)),
+                                      "Could not find deflate ", LazyToString.lazy(() -> Arrays.toString(value)));
+        }
+
+        @Override
+        public int byteSize()
+        {
+            return 0;
+        }
+
+        @Override
+        public int compare(long l, long r)
+        {
+            return 0;
+        }
+
+        @Override
+        public long idxFor(long pd)
+        {
+            return DescriptorIndexBijection.INSTANCE.toIdx(pd);
+        }
+
+        @Override
+        public long descriptorAt(long idx)
+        {
+            return DescriptorIndexBijection.INSTANCE.toPd(idx);
+        }
+    }
+
+    /**
+     * For _any_ partition, its characteristics are deterministic and depend on its pd. In other words, over the lifetime
+     * of partition, its max number of rows (and, therefore, possible values for its rows), _can not_ be changed. However,
+     * partition can get rotated in and out from active set at any point in time.
+     */
+    @SuppressWarnings("unchecked")
+    public static ActivePartition createActivePartition(long idx,
+                                                        long pd,
+                                                        SchemaSpec schema,
+                                                        Distribution rowPopulation,
+                                                        VisitGenerator.ColumnPopulation population,
+                                                        HistoryBuilder.IndexedBijection<Object[]> cachingPkGen,
+                                                        LongConsumer cleanup)
+    {
+        List<Comparator<Object>> pkComparators = new ArrayList<>();
+        List<Comparator<Object>> ckComparators = new ArrayList<>();
+        List<Comparator<Object>> regularComparators = new ArrayList<>();
+        List<Comparator<Object>> staticComparators = new ArrayList<>();
+
+        EntropySource rng = new JdkRandomEntropySource(pd);
+        for (int i = 0; i < schema.partitionKeys.size(); i++)
+            pkComparators.add((Comparator<Object>) schema.partitionKeys.get(i).type.comparator());
+        for (int i = 0; i < schema.clusteringKeys.size(); i++)
+            ckComparators.add((Comparator<Object>) schema.clusteringKeys.get(i).type.comparator());
+        for (int i = 0; i < schema.regularColumns.size(); i++)
+            regularComparators.add((Comparator<Object>) schema.regularColumns.get(i).type.comparator());
+        for (int i = 0; i < schema.staticColumns.size(); i++)
+            staticComparators.add((Comparator<Object>) schema.staticColumns.get(i).type.comparator());
+
+        Map<ColumnSpec<?>, HistoryBuilder.IndexedBijection<Object>> map = new HashMap<>();
+        for (ColumnSpec<?> column : IteratorsUtil.concat(schema.regularColumns, schema.staticColumns))
+        {
+            int populationPerColumn = Math.toIntExact(population.distribution(column.name).next(pd));
+            map.computeIfAbsent(column, (a) -> (HistoryBuilder.IndexedBijection<Object>) fromType(rng, populationPerColumn, column));
+        }
+
+        // As of now, we allow only single partition queries, and within the scope of the visit we can select
+        // values only from one partition, so we simply create an identity PK bijection to avoid lookups altogether.
+        HistoryBuilder.IndexedBijection<Object[]> rawPkGen = new HistoryBuilder.IndexedBijection<Object[]>() {
+            private Object[] value = null;
+
+            @Override
+            public Object[] inflate(long descriptor) {
+                Invariants.require(pd == descriptor, "Partition descriptor mismatch: %d != %d", pd, descriptor);
+                return ensureValue();
+            }
+
+            private Object[] ensureValue()
+            {
+                if (value == null)
+                    value = SeedableEntropySource.computeWithSeed(pd, forKeys(schema.partitionKeys)::generate);
+                return value;
+            }
+            @Override
+            public long deflate(Object[] value) {
+                Invariants.require(Arrays.equals(value, ensureValue()), "Partition key mismatch, %s != %s", ensureValue(), value);
+                // TODO (required): allow selecting only a subset of PK and CK
+                return pd;
+            }
+
+            @Override
+            public int byteSize() {
+                return Long.BYTES;
+            }
+
+            @Override
+            public int compare(long l, long r) {
+                throw new IllegalStateException("Not implemented");
+            }
+
+            @Override
+            public long idxFor(long pd) {
+                return DescriptorIndexBijection.INSTANCE.toIdx(pd);
+            }
+
+            @Override
+            public long descriptorAt(long idx) {
+                return DescriptorIndexBijection.INSTANCE.toPd(idx);
+            }
+        };
+
+        // TODO (required): at the moment, we generate the values for clusterings by pre-generating a set number of values, which
+        //                  doesn't give us enough control over the possible values. What we need to do instead is to allow
+        //                  generating a set number of values _per column_. For example, if ck1 has 5 unique value, and ck2 has
+        //                  5 unique values, for each ck1 we will have a value of ck2, so the number of possible values grows
+        //                  combinatorically.
+        int combinations = Math.toIntExact(rowPopulation.next(pd));
+        HistoryBuilder.IndexedBijection<Object[]> ckGenerator = new InvertibleGenerator<>(rng,
+                                                                                          cumulativeEntropy(schema.clusteringKeys),
+                                                                                          combinations,
+                                                                                          forKeys(schema.clusteringKeys),
+                                                                                          keyComparator(schema.clusteringKeys));
+
+        return new ActivePartition(idx,
+                                   pd,
+                                   cachingPkGen,
+                                   rawPkGen,
+                                   ckGenerator,
+                                   schema.regularColumns.stream()
+                                                        .map(map::get)
+                                                        .collect(Collectors.toList()),
+                                   schema.staticColumns.stream()
+                                                       .map(map::get)
+                                                       .collect(Collectors.toList()),
+                                   pkComparators,
+                                   ckComparators,
+                                   regularComparators,
+                                   staticComparators,
+                                   cleanup);
+    }
+
+    /**
+     * Tracks which partition indices have been visited. Maintains a contiguous range [minIdx, highIdxWatermark]
+     * and a min-heap of visited indices above the watermark. When indices added to the heap become consecutive
+     * with the watermark, the watermark is advanced.
+     *
+     * {@code getBySeed} picks a visited partition index uniformly at random using a deterministic seed,
+     * or returns -1 if no partitions have been visited.
+     */
+    public static class VisitedPartitions
+    {
+        private final long minIdx;
+        private long highIdxWatermark;
+        private long[] sorted;
+        private int sortedSize;
+
+        public VisitedPartitions(long minIdx)
+        {
+            Invariants.require(minIdx >= 0, "minIdx must be non-negative: %d", minIdx);
+            this.minIdx = minIdx;
+            this.highIdxWatermark = -1;
+            this.sorted = new long[16];
+            this.sortedSize = 0;
+        }
+
+        public synchronized void add(long idx)
+        {
+            Invariants.require(idx >= minIdx, "Partition index %d is below minimum %d", idx, minIdx);
+
+            if (highIdxWatermark >= 0 && idx <= highIdxWatermark)
+                return;
+
+            int pos = Arrays.binarySearch(sorted, 0, sortedSize, idx);
+            if (pos >= 0)
+                return; // already present
+
+            int insertPos = -pos - 1;
+            if (sortedSize == sorted.length)
+                sorted = Arrays.copyOf(sorted, sorted.length * 2);
+            System.arraycopy(sorted, insertPos, sorted, insertPos + 1, sortedSize - insertPos);
+            sorted[insertPos] = idx;
+            sortedSize++;
+
+            drain();
+        }
+
+        private void drain()
+        {
+            int removed = 0;
+            while (removed < sortedSize)
+            {
+                long top = sorted[removed];
+
+                if (highIdxWatermark == -1)
+                {
+                    if (top == minIdx)
+                    {
+                        removed++;
+                        highIdxWatermark = minIdx;
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                else if (top == highIdxWatermark + 1)
+                {
+                    removed++;
+                    highIdxWatermark = top;
+                }
+                else if (top <= highIdxWatermark)
+                {
+                    removed++;
+                }
+                else
+                {
+                    break;
+                }
+            }
+
+            if (removed > 0)
+            {
+                sortedSize -= removed;
+                System.arraycopy(sorted, removed, sorted, 0, sortedSize);
+            }
+        }
+
+        private synchronized long size()
+        {
+            long contiguous = highIdxWatermark >= 0 ? (highIdxWatermark - minIdx + 1) : 0;
+            return contiguous + sortedSize;
+        }
+
+        public synchronized long getBySeed(long seed)
+        {
+            long total = size();
+            if (total == 0)
+                return -1;
+
+            long chosen = SeedableEntropySource.computeWithSeed(seed, rng -> rng.nextLong(0, total));
+
+            long contiguous = highIdxWatermark >= 0 ? (highIdxWatermark - minIdx + 1) : 0;
+            if (chosen < contiguous)
+                return minIdx + chosen;
+
+            int heapIdx = Math.toIntExact(chosen - contiguous);
+            return sorted[heapIdx];
+        }
+    }
+
+    public static class TrackerWrapper implements DataTracker
+    {
+        private final DataTracker delegate;
+        private final Partitions partitions;
+
+        public TrackerWrapper(DataTracker delegate, Partitions partitions)
+        {
+            this.delegate = delegate;
+            this.partitions = partitions;
+
+        }
+
+        @Override
+        public void begin(Visit visit)
+        {
+            delegate.begin(visit);
+        }
+
+        @Override
+        public void end(Visit visit)
+        {
+            delegate.end(visit);
+            // Referencing happens before handing over to the worker
+            for (long pd : visit.visitedPartitions)
+                partitions.forPd(pd).deref();
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/ExternalClusterSut.java

@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import com.datastax.driver.core.*;
+import org.apache.cassandra.db.ConsistencyLevel;
+
+import java.util.List;
+
+import com.google.common.util.concurrent.MoreExecutors;
+
+public class ExternalClusterSut
+{
+    private final Session session;
+
+    public ExternalClusterSut(Session session)
+    {
+        this(session, 10);
+    }
+
+    public ExternalClusterSut(Session session, int threads)
+    {
+        this.session = session;
+    }
+
+    public Session session()
+    {
+        return session;
+    }
+
+    public Metadata metadata()
+    {
+        return this.session.getCluster().getMetadata();
+    }
+
+    public static ExternalClusterSut create(ConsistencyLevel cl, int port, String... contactPoints)
+    {
+        // TODO: close Cluster and Session!
+        return new ExternalClusterSut(Cluster.builder()
+                .withQueryOptions(new QueryOptions().setConsistencyLevel(toDriverCl(cl)))
+                .addContactPoints(contactPoints)
+                .withPort(port)
+                .withCredentials("cassandra", "cassandra")
+                .build()
+                .connect());
+    }
+
+    public boolean isShutdown()
+    {
+        return session.isClosed();
+    }
+
+    public void shutdown()
+    {
+        session.close();
+    }
+
+    // TODO: this is rather simplistic
+    public Object[][] execute(String statement, ConsistencyLevel cl, Object... bindings)
+    {
+        return resultSetToObjectArray(session.execute(statement, bindings));
+    }
+
+    private static final Object[][] EMPTY = new Object[0][];
+    public Object[][] execute(SimpleStatement statement)
+    {
+        ResultSetFuture future = session.executeAsync(statement);
+        return resultSetToObjectArray(future.getUninterruptibly());
+    }
+
+    public Object[][] execute(SimpleStatement statement, Runnable callback)
+    {
+        ResultSetFuture future = session.executeAsync(statement);
+        future.addListener(callback, MoreExecutors.directExecutor());
+        return resultSetToObjectArray(future.getUninterruptibly());
+    }
+
+    public static Object[][] resultSetToObjectArray(ResultSet rs)
+    {
+        List<Row> rows = rs.all();
+        if (rows.size() == 0)
+            return new Object[0][];
+        Object[][] results = new Object[rows.size()][];
+        for (int i = 0; i < results.length; i++)
+        {
+            Row row = rows.get(i);
+            ColumnDefinitions cds = row.getColumnDefinitions();
+            Object[] result = new Object[cds.size()];
+            for (int j = 0; j < cds.size(); j++)
+            {
+                if (!row.isNull(j))
+                    result[j] = row.getObject(j);
+            }
+            results[i] = result;
+        }
+        return results;
+    }
+
+    public static com.datastax.driver.core.ConsistencyLevel toDriverCl(ConsistencyLevel cl)
+    {
+        switch (cl)
+        {
+            case ONE:
+                return com.datastax.driver.core.ConsistencyLevel.ONE;
+            case ALL:
+                return com.datastax.driver.core.ConsistencyLevel.ALL;
+            case QUORUM:
+                return com.datastax.driver.core.ConsistencyLevel.QUORUM;
+        }
+        throw new IllegalArgumentException("Don't know a CL: " + cl);
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/HarryStress.java

@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.execution.LockingDataTracker;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.concurrent.Condition;
+
+import javax.annotation.Nullable;
+
+/**
+ * At any given point, we will have only N active partitions
+ *
+ * for every N partitions there should be its own invertible generator that will generate a partition key;
+ * however we need to somehow guarantee that later partition keys won't have same descriptors.
+ */
+public class HarryStress
+{
+    public static final Logger LOGGER = LoggerFactory.getLogger(HarryStress.class);
+
+    // VisitGenerator is a stateful component that generates operation specs. Operation specs are then used by
+    private final VisitGenerator visitGenerator;
+
+    private final ActivePartition.Partitions partitionFactory;
+    private final DataTracker.ReplayingDataTracker innerTracker;
+
+    private final Generator<VisitGenerator.VisitType> visitTypeGen;
+    private final Distribution visitSizeDistribution;
+    private final VisitGenerator.OpKindGenFactory operationKindGen;
+
+    private final List<StressWorker> workers;
+
+    private final EndCondition endCondition;
+    private final MetricCollector metrics = new MetricCollector();
+    private final int ratePerSecond;
+    private final @Nullable PrintStream metricsOut;
+    private final int reportIntervalSeconds;
+
+    public HarryStress(SchemaSpec schema,
+                       Distribution rowPopulation,
+                       VisitGenerator.ColumnPopulation columnPopulation,
+                       Generator<VisitGenerator.VisitType> visitTypeGen,
+                       Distribution visitSizeDistribution,
+                       VisitGenerator.OpKindGenFactory operationKindGen,
+                       RotationStrategy rotationStrategy,
+                       @Nullable PrintStream metricsOut,
+                       int reportIntervalSeconds,
+                       Supplier<BiFunction<CompiledStatement, Runnable, Object[][]>> sutFactory,
+                       int concurrency,
+                       int ratePerSecond,
+                       long minPartitionIdx,
+                       long maxPartitionIdx,
+                       long initialLts)
+    {
+        this.ratePerSecond = ratePerSecond;
+        this.metricsOut = metricsOut;
+        this.reportIntervalSeconds = reportIntervalSeconds;
+        this.visitTypeGen = visitTypeGen;
+        this.visitSizeDistribution = visitSizeDistribution;
+        this.operationKindGen = operationKindGen;
+        this.partitionFactory = new ActivePartition.Partitions(schema, rowPopulation, columnPopulation, rotationStrategy, minPartitionIdx, maxPartitionIdx, initialLts);
+        this.visitGenerator = new VisitGenerator(partitionFactory,
+                                                 visitTypeGen,
+                                                 visitSizeDistribution,
+                                                 operationKindGen,
+                                                 initialLts);
+
+        this.innerTracker = new DataTracker.SimpleDataTracker();
+
+        partitionFactory.onRemove(innerTracker::gc);
+        partitionFactory.populate();
+        DataTracker outerTracker = new ActivePartition.TrackerWrapper(new LockingDataTracker(innerTracker, Integer.MAX_VALUE, 1),
+                                                                      partitionFactory);
+
+        this.workers = new ArrayList<>();
+        this.endCondition = new EndCondition();
+
+        for (int i = 0; i < concurrency; i++)
+            workers.add(new StressWorker(i, innerTracker, outerTracker, partitionFactory, sutFactory, endCondition, this.ratePerSecond / concurrency));
+
+        metrics.partitionCount.set(partitionFactory.activePartitions.size());
+        metrics.activePartitionCount.set(partitionFactory.activePartitions.size());
+    }
+
+    private static class EndCondition implements Consumer<Throwable>
+    {
+        final List<Throwable> exceptions = new CopyOnWriteArrayList<>();
+        final Condition condition = Condition.newOneTimeCondition();
+
+        @Override
+        public void accept(Throwable e)
+        {
+            exceptions.add(e);
+            condition.signal();
+        }
+
+        public boolean awaitUntil(long nanoTimeDeadline) throws InterruptedException
+        {
+            return condition.awaitUntil(nanoTimeDeadline);
+        }
+
+        public Throwable maybeReportExceptions()
+        {
+            if (!exceptions.isEmpty())
+            {
+                RuntimeException ex = new RuntimeException("Caught exception while running bechmark");
+                for (Throwable exception : exceptions)
+                    ex.addSuppressed(exception);
+                return ex;
+            }
+            return null;
+        }
+    }
+
+    /**
+     * Replays the history [fromLts, toLts) into the model only (no SUT execution), so that reads over partitions whose
+     * data was loaded out-of-band (e.g. offline-generated SSTables produced from the same history) can be validated.
+     */
+    public void replay(long fromLts, long toLts)
+    {
+        VisitGenerator seed = new VisitGenerator(partitionFactory, visitTypeGen, visitSizeDistribution, operationKindGen, fromLts);
+        for (long lts = fromLts; lts < toLts; lts++)
+        {
+            Visit visit = seed.get();
+            innerTracker.begin(visit);
+            innerTracker.end(visit);
+        }
+    }
+
+    private Visit nextVisit()
+    {
+        Visit nextVisit = visitGenerator.get();
+        for (long pd : nextVisit.visitedPartitions)
+            partitionFactory.forPd(pd).ref();
+        return nextVisit;
+    }
+
+    // TODO: warm up
+    public void start(long maxIterations, long runUntil) throws Throwable
+    {
+        long now = Clock.Global.nanoTime();
+        long nextMetricsCollect = nextReport(now, reportIntervalSeconds);
+
+        List<PrintStream> metricOuts = new ArrayList<>();
+        metricOuts.add(System.out);
+        if (metricsOut != null)
+        {
+            metricOuts.add(metricsOut);
+            metricsOut.println(HEAD);
+        }
+
+        metrics.start(now);
+        Visit nextVisit = nextVisit();
+
+        while (true)
+        {
+            if (now > nextMetricsCollect)
+            {
+                // TODO (expected): this is potentially lossy, as worker may update metrics after we grab them
+                for (StressWorker worker : workers)
+                    metrics.merge(worker.resetMetrics());
+
+                metrics.end(now);
+                System.out.println(HEAD);
+                printRow(metrics, metricOuts);
+                metrics.start(now);
+                nextMetricsCollect = nextReport(Clock.Global.nanoTime(), reportIntervalSeconds);
+            }
+
+            for (StressWorker worker : workers)
+            {
+                int remaining = worker.getFreeSlots();
+                while (remaining-- > 0 && worker.offer(nextVisit))
+                {
+                    nextVisit = nextVisit();
+                    partitionFactory.maybeSwitchPartition(nextVisit.lts, action -> {});
+                }
+            }
+
+            if (endCondition.awaitUntil(now + TimeUnit.SECONDS.toNanos(1)))
+            {
+                System.out.println("Exiting early due to condition");
+                break; // errored out
+            }
+
+            now = Clock.Global.nanoTime();
+            if (nextVisit.lts >= maxIterations || now > runUntil)
+            {
+                break;
+            }
+        }
+
+        System.out.printf("Completed! %d%n", nextVisit.lts);
+
+        for (StressWorker worker : workers)
+            worker.shutdown();
+        for (StressWorker worker : workers)
+            worker.awaitTermination(1, TimeUnit.MINUTES);
+
+        Throwable t = endCondition.maybeReportExceptions();
+        if (t != null)
+            throw t;
+    }
+
+    private static long nextReport(long nowNanos, int reportIntervalSeconds)
+    {
+        Calendar calendar = Calendar.getInstance();
+        long nowMillis = calendar.getTimeInMillis();
+        int addSeconds = reportIntervalSeconds - (calendar.get(Calendar.SECOND) % reportIntervalSeconds);
+        calendar.add(Calendar.SECOND, addSeconds);
+        return nowNanos + TimeUnit.MILLISECONDS.toNanos(calendar.getTimeInMillis() - nowMillis);
+    }
+
+    public static final String HEADFORMAT = "%19s %10s %8s %8s %7s %8s %8s %8s %8s %8s %8s %8s %8s %8s";
+    public static final String ROWFORMAT =  "%tF %tT " + // time
+                                            "%10d " + // counts
+                                            "%8.0f " +
+                                            "%8d " +
+                                            "%7s " +
+                                            "%8d " +
+                                            "%8d " +    // count
+                                            "%8.0f " +  // rates
+                                            "%8.1f " +   // latency
+                                            "%8.1f " +
+                                            "%8.1f " +
+                                            "%8.1f " +
+                                            "%8.1f " +
+                                            "%8.1f";
+
+    public static final String[] HEADMETRICS = new String[]{ "time", "pcount", "pk/s", "pactive", "type", "count", "errors", "op/s","mean","med",".95",".99",".999","max"};
+    public static final String HEAD = String.format(HEADFORMAT, (Object[]) HEADMETRICS);
+
+    public static void main(String[] args)
+    {
+        System.out.println(HEAD);
+        printRow(new MetricCollector(), Collections.singletonList(System.out));
+        long waitUntil = nextReport(Clock.Global.nanoTime(), 30);
+        while (true)
+        {
+            long wait = waitUntil - Clock.Global.nanoTime();
+            if (wait <= 0)
+                break;
+
+            Uninterruptibles.sleepUninterruptibly(wait, TimeUnit.NANOSECONDS);
+        }
+        System.out.printf("%tT\n", Calendar.getInstance());
+    }
+
+    private static void printRow(MetricCollector metrics, List<PrintStream> outs)
+    {
+        Calendar calendar = Calendar.getInstance();
+        String reads = String.format(ROWFORMAT, calendar, calendar,
+                metrics.partitionCount.get(),
+                metrics.partitionRate(),
+                metrics.activePartitionCount.get(),
+                "read",
+                metrics.readCount(),
+                metrics.failedReads.getAndSet(0),
+                metrics.readRate(),
+                metrics.meanReadLatencyMs(),
+                metrics.medianReadLatencyMs(),
+                metrics.readLatencyAtPercentileMs(95.0),
+                metrics.readLatencyAtPercentileMs(99.0),
+                metrics.readLatencyAtPercentileMs(99.9),
+                metrics.maxReadLatencyMs());
+
+        String writes = String.format(ROWFORMAT, calendar, calendar,
+                metrics.partitionCount.get(),
+                metrics.partitionRate(),
+                metrics.activePartitionCount.get(),
+                "write",
+                metrics.writeCount(),
+                metrics.failedWrites.getAndSet(0),
+                metrics.writeRate(),
+                metrics.meanWriteLatencyMs(),
+                metrics.medianWriteLatencyMs(),
+                metrics.writeLatencyAtPercentileMs(95.0),
+                metrics.writeLatencyAtPercentileMs(99.0),
+                metrics.writeLatencyAtPercentileMs(99.9),
+                metrics.maxWriteLatencyMs());
+
+        for (PrintStream out : outs)
+        {
+            out.println(reads);
+            out.println(writes);
+            out.flush();
+        }
+    }
+
+}

test/harry/main/org/apache/cassandra/harry/stress/LevelledSStableGenerator.java

@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.execution.CQLVisitExecutor;
+import org.apache.cassandra.harry.gen.EntropySource;
+import org.apache.cassandra.harry.gen.ValueGenerators;
+import org.apache.cassandra.harry.gen.rng.SeedableEntropySource;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+import org.apache.cassandra.io.sstable.HarrySSTableWriter;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.ActiveRepairService;
+
+import static org.apache.cassandra.harry.stress.ActivePartition.createActivePartition;
+
+public class LevelledSStableGenerator
+{
+    private final SchemaSpec schema;
+    private final boolean disableCompression;
+    private final int sstableSizeMiB;
+    private final Distribution rowPopulation;
+    private final Distribution visitSizeDistribution;
+    private final VisitGenerator.OpKindGenFactory opKindGen;
+    private final VisitGenerator.ColumnPopulation columnPopulation;
+    private final ActivePartition.PartitionKeyGen pkGen;
+    private final TokenIndex tokenIndex;
+    private final SSTableLevelPicker levelPicker;
+    private final File directory;
+    private final long maxPartitions;
+    private final long repairedAtMillis;
+
+    public LevelledSStableGenerator(SchemaSpec schema,
+                                    Distribution rowPopulation,
+                                    VisitGenerator.ColumnPopulation columnPopulation,
+                                    Distribution visitSizeDistribution,
+                                    VisitGenerator.OpKindGenFactory operationKindGen,
+                                    boolean disableCompression,
+                                    int sstableSizeMiB,
+                                    SSTableLevelPicker levelPicker,
+                                    TokenIndex tokenIndex,
+                                    File directory)
+    {
+        this(schema, rowPopulation, columnPopulation, visitSizeDistribution, operationKindGen,
+             disableCompression, sstableSizeMiB, levelPicker, tokenIndex, directory, Long.MAX_VALUE, ActiveRepairService.UNREPAIRED_SSTABLE);
+    }
+
+    public LevelledSStableGenerator(SchemaSpec schema,
+                                    Distribution rowPopulation,
+                                    VisitGenerator.ColumnPopulation columnPopulation,
+                                    Distribution visitSizeDistribution,
+                                    VisitGenerator.OpKindGenFactory operationKindGen,
+                                    boolean disableCompression,
+                                    int sstableSizeMiB,
+                                    SSTableLevelPicker levelPicker,
+                                    TokenIndex tokenIndex,
+                                    File directory,
+                                    long maxPartitions)
+    {
+        this(schema, rowPopulation, columnPopulation, visitSizeDistribution, operationKindGen,
+             disableCompression, sstableSizeMiB, levelPicker, tokenIndex, directory, maxPartitions, ActiveRepairService.UNREPAIRED_SSTABLE);
+    }
+
+    public LevelledSStableGenerator(SchemaSpec schema,
+                                    Distribution rowPopulation,
+                                    VisitGenerator.ColumnPopulation columnPopulation,
+                                    Distribution visitSizeDistribution,
+                                    VisitGenerator.OpKindGenFactory operationKindGen,
+                                    boolean disableCompression,
+                                    int sstableSizeMiB,
+                                    SSTableLevelPicker levelPicker,
+                                    TokenIndex tokenIndex,
+                                    File directory,
+                                    long maxPartitions,
+                                    long repairedAtMillis)
+    {
+        this.schema = schema;
+        this.rowPopulation = rowPopulation;
+        this.columnPopulation = columnPopulation;
+        this.visitSizeDistribution = visitSizeDistribution;
+        this.opKindGen = operationKindGen;
+        this.disableCompression = disableCompression;
+        this.sstableSizeMiB = sstableSizeMiB;
+        this.levelPicker = levelPicker;
+        this.tokenIndex = tokenIndex;
+        this.directory = directory;
+        this.maxPartitions = maxPartitions;
+        this.repairedAtMillis = repairedAtMillis;
+        this.pkGen = new ActivePartition.PartitionKeyGen(schema);
+    }
+
+    public void generate(long minToken, long maxToken)
+    {
+        TokenIndex.EntryIterator iter = tokenIndex.range(minToken, maxToken);
+        HarrySSTableWriter[] writers = new HarrySSTableWriter[levelPicker.size()];
+        CQLVisitExecutor[] executors = new CQLVisitExecutor[levelPicker.size()];
+        // This is a bit of a hack: we do not create full blown Partitions, since we don't
+        // pick partitions in the same way we were picking them during "normal" generation.
+        CurrentPartition currentPartition = new CurrentPartition(pkGen);
+        for (int i = 0; i < writers.length; i++)
+        {
+            writers[i] = SSTableGenerator.newWriter(schema, disableCompression, sstableSizeMiB, directory, i, repairedAtMillis);
+            executors[i] = SSTableGenerator.createExecutor(schema, currentPartition, writers[i], null);
+        }
+        long counter = 0;
+        long lastChecking = System.currentTimeMillis();
+        while (iter.hasNext() && counter < maxPartitions)
+        {
+            counter++;
+            if (counter % 1000 == 0)
+            {
+                long now = System.currentTimeMillis();
+                System.out.println("Processed " + counter + " partitions " + (now - lastChecking) + "ms elapsed");
+                lastChecking = now;
+            }
+            long pd = iter.pd();
+            long[] ltss = iter.readLts();
+            ActivePartition partition = byPd(pd);
+            currentPartition.current = partition;
+            for (long lts : ltss)
+            {
+                Visit visit = VisitGenerator.mutatingVisit(lts, visitSizeDistribution, opKindGen, rng -> partition);
+                CQLVisitExecutor executor = executors[levelPicker.pick(lts)];
+                executor.execute(visit);
+            }
+            currentPartition.current = null;
+            pkGen.cleanup(pd);
+            iter.advance();
+        }
+
+        for (int i = 0; i < writers.length; i++)
+        {
+            try
+            {
+                writers[i].close();
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+        }
+    }
+
+    public static class SSTableLevelPicker
+    {
+        final long[] cdf;
+        final long total;
+
+        public SSTableLevelPicker(int... weights)
+        {
+            cdf = new long[weights.length];
+            cdf[0] = weights[0];
+            for (int i = 1; i < weights.length; i++)
+                cdf[i] = cdf[i - 1] + weights[i];
+            total = cdf[cdf.length - 1];
+        }
+
+        public int size()
+        {
+            return cdf.length;
+        }
+
+        public int pick(long lts)
+        {
+            EntropySource rng = SeedableEntropySource.entropySource(lts, 1);
+            long val = rng.nextLong(0, total);
+            for (int i = 0; i < cdf.length; i++)
+            {
+                if (val < cdf[i])
+                    return i;
+            }
+            return cdf.length - 1;
+        }
+    }
+
+    private ActivePartition byPd(long pd)
+    {
+        long idx = ActivePartition.DescriptorIndexBijection.INSTANCE.toIdx(pd);
+        ActivePartition partition = createActivePartition(idx, pd, schema, rowPopulation, columnPopulation, pkGen, (pd_) -> {});
+        pkGen.ensure(pd, d -> SeedableEntropySource.computeWithSeed(d, SchemaSpec.forKeys(schema.partitionKeys)::generate));
+        return partition;
+    }
+
+    static class CurrentPartition extends ValueGenerators<Object[], Object[]>
+    {
+        ActivePartition current;
+
+        CurrentPartition(HistoryBuilder.IndexedBijection<Object[]> pkGen)
+        {
+            super(pkGen);
+        }
+
+        @Override
+        public ActivePartition forPd(long pd)
+        {
+            if (current == null || current.pd != pd)
+                throw new IllegalStateException("No ActivePartition for pd=" + pd + "; expected pd=" + (current != null ? current.pd : "null"));
+            return current;
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/MetricCollector.java

@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.HdrHistogram.Histogram;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MetricCollector
+{
+    public static final Logger LOGGER = LoggerFactory.getLogger(MetricCollector.class);
+    private final Histogram reads = new Histogram(3);
+    private final Histogram writes = new Histogram(3);
+
+    // nanos
+    private long startNs = Long.MAX_VALUE;
+    private long endNs = Long.MIN_VALUE;
+
+    // discrete
+    public final AtomicLong activePartitionCount = new AtomicLong();
+    public final AtomicLong partitionCount = new AtomicLong();
+    public final AtomicLong failedReads = new AtomicLong();
+    public final AtomicLong failedWrites = new AtomicLong();
+
+    public void merge(StressWorker.WorkerMetrics metrics)
+    {
+        try
+        {
+            reads.add(metrics.reads);
+            writes.add(metrics.writes);
+            failedReads.addAndGet(metrics.failedReads.get());
+            failedWrites.addAndGet(metrics.failedWrites.get());
+        }
+        catch (Throwable t)
+        {
+            LOGGER.error("Could not merge histograms, reported values will be unreliable", t);
+
+        }
+    }
+
+    public synchronized double readRate()
+    {
+        return readCount() / ((endNs - startNs) * 0.000000001d);
+    }
+
+    public synchronized double writeRate()
+    {
+        return writeCount() / ((endNs - startNs) * 0.000000001d);
+    }
+
+    public synchronized double partitionRate()
+    {
+        return partitionCount.get() / ((endNs - startNs) * 0.000000001d);
+    }
+
+    public synchronized double meanReadLatencyMs()
+    {
+        return readLatencies().getMean() * 0.000001d;
+    }
+
+    public synchronized double maxReadLatencyMs()
+    {
+        return readLatencies().getMaxValue() * 0.000001d;
+    }
+
+    public synchronized double medianReadLatencyMs()
+    {
+        return readLatencies().getValueAtPercentile(50.0) * 0.000001d;
+    }
+
+    public synchronized double meanWriteLatencyMs()
+    {
+        return writeLatencies().getMean() * 0.000001d;
+    }
+
+    public synchronized double maxWriteLatencyMs()
+    {
+        return writeLatencies().getMaxValue() * 0.000001d;
+    }
+
+    public synchronized double medianWriteLatencyMs()
+    {
+        return writeLatencies().getValueAtPercentile(50.0) * 0.000001d;
+    }
+
+
+    /**
+     * @param percentile between 0.0 and 100.0
+     * @return latency in milliseconds at percentile
+     */
+    public synchronized double readLatencyAtPercentileMs(double percentile)
+    {
+        return readLatencies().getValueAtPercentile(percentile) * 0.000001d;
+    }
+
+    public synchronized double writeLatencyAtPercentileMs(double percentile)
+    {
+        return writeLatencies().getValueAtPercentile(percentile) * 0.000001d;
+    }
+
+    public synchronized long runTimeMs()
+    {
+        return (endNs - startNs) / 1000000;
+    }
+
+    public long end()
+    {
+        return endNs;
+    }
+
+    public long start()
+    {
+        return startNs;
+    }
+
+    private Histogram readLatencies()
+    {
+        return reads;
+    }
+
+    private Histogram writeLatencies()
+    {
+        return writes;
+    }
+
+    public synchronized long readCount()
+    {
+        return readLatencies().getTotalCount() + failedReads.get();
+    }
+
+    public synchronized long writeCount()
+    {
+        return writeLatencies().getTotalCount() + failedWrites.get();
+    }
+
+
+    public void start(long started)
+    {
+        this.startNs = started;
+        readLatencies().reset();
+        writeLatencies().reset();
+    }
+
+    public void end(long ended)
+    {
+        this.endNs = ended;
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/ReplicaLocator.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import com.datastax.driver.core.CodecRegistry;
+import com.datastax.driver.core.ColumnMetadata;
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.Host;
+import com.datastax.driver.core.KeyspaceMetadata;
+import com.datastax.driver.core.Metadata;
+import com.datastax.driver.core.ProtocolVersion;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.TableMetadata;
+import com.datastax.driver.core.TypeCodec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.gen.rng.SeedableEntropySource;
+
+import static org.apache.cassandra.harry.SchemaSpec.forKeys;
+import static org.apache.cassandra.harry.util.ByteUtils.putShortLength;
+
+public class ReplicaLocator
+{
+    @SuppressWarnings("unchecked")
+    public static Set<Host> getReplicaHosts(Session session, String keyspace, String table, Object[] pk)
+    {
+        Metadata metadata = session.getCluster().getMetadata();
+        CodecRegistry codecRegistry = session.getCluster().getConfiguration().getCodecRegistry();
+        ProtocolVersion protocolVersion = session.getCluster().getConfiguration()
+                                                 .getProtocolOptions().getProtocolVersion();
+
+        KeyspaceMetadata ksm = metadata.getKeyspace(keyspace);
+        if (ksm == null)
+            throw new IllegalArgumentException(String.format("Keyspace not found: %s", keyspace));
+
+        TableMetadata tm = ksm.getTable(table);
+        if (tm == null)
+            throw new IllegalArgumentException(String.format("Table not found: %s.%s", keyspace, table));
+
+        List<ColumnMetadata> pkColumns = tm.getPartitionKey();
+        if (pk.length != pkColumns.size())
+            throw new IllegalArgumentException(String.format("Expected %d partition key values but got %d",
+                                                             pkColumns.size(), pk.length));
+
+        ByteBuffer[] components = new ByteBuffer[pk.length];
+        for (int i = 0; i < pk.length; i++)
+        {
+            DataType type = pkColumns.get(i).getType();
+            TypeCodec codec = codecRegistry.codecFor(type);
+            components[i] = codec.serialize(pk[i], protocolVersion);
+        }
+
+        ByteBuffer routingKey = compose(components);
+
+        return metadata.getReplicas(keyspace, routingKey);
+    }
+
+    static ByteBuffer compose(ByteBuffer... buffers) {
+        if (buffers.length == 1) {
+            return buffers[0];
+        } else {
+            int totalLength = 0;
+
+            for(ByteBuffer bb : buffers) {
+                totalLength += 2 + bb.remaining() + 1;
+            }
+
+            ByteBuffer out = ByteBuffer.allocate(totalLength);
+
+            for(ByteBuffer buffer : buffers) {
+                ByteBuffer bb = buffer.duplicate();
+                putShortLength(out, bb.remaining());
+                out.put(bb);
+                out.put((byte)0);
+            }
+
+            out.flip();
+            return out;
+        }
+    }
+
+    public static List<InetSocketAddress> getReplicas(Session session, SchemaSpec schema, long pd)
+    {
+       Object[] pk = SeedableEntropySource.computeWithSeed(pd, forKeys(schema.partitionKeys)::generate);
+       return getReplicas(session, schema.keyspace, schema.table, pk);
+
+    }
+    public static List<InetSocketAddress> getReplicas(Session session, String keyspace, String table, Object[] pk)
+    {
+        Set<Host> replicas = getReplicaHosts(session, keyspace, table, pk);
+        List<InetSocketAddress> addresses = new ArrayList<>(replicas.size());
+        for (Host host : replicas)
+            addresses.add(host.getBroadcastSocketAddress());
+        return addresses;
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/RotationStrategy.java

@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import java.util.Arrays;
+
+import org.apache.cassandra.harry.gen.EntropySource;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.Generators;
+
+/**
+ * Rotation strategy is a way to control a number of active partitions.
+ *
+ * Rotation strategy has a target size, but will vary the number of partition to maintain it within a distribution over time.
+ */
+public interface RotationStrategy extends Generator<RotationStrategy.PartitionAction[]>
+{
+    int targetSize();
+
+    /**
+     * Determines whether a partition switch should occur at the given logical timestamp.
+     * Implementations track the last LTS at which a switch occurred and compare against
+     * the configured interval.
+     */
+    boolean shouldSwitch(long lts);
+
+    // TODO (required): control a total number of partitions
+    enum PartitionAction
+    {
+        REPLACE_WITH_NEW,      // Replace partition: rotate current one out, and pick a new partition in its place
+        REPLACE_WITH_VISITED,  // Replace partition: rotate current one out, and pick an already visited partition in its place
+    }
+
+    /**
+     * A trivial random rotation strategy, would attempt to keep the size close to the target, but
+     * has no bias or gives any guarantees.
+     */
+    class RandomRotationStrategy implements RotationStrategy
+    {
+        private static final Generator<PartitionAction> gen = Generators.pick(PartitionAction.REPLACE_WITH_VISITED, PartitionAction.REPLACE_WITH_NEW);
+        private final PartitionAction[] EMPTY = new PartitionAction[] {};
+        private final int targetSize;
+        private final int switchInterval;
+        private long lastSwitchLts = -1;
+
+        public RandomRotationStrategy(int targetSize)
+        {
+            this(targetSize, 500);
+        }
+
+        public RandomRotationStrategy(int targetSize, int switchInterval)
+        {
+            this.targetSize = targetSize;
+            this.switchInterval = switchInterval;
+        }
+
+        @Override
+        public int targetSize()
+        {
+            return targetSize;
+        }
+
+        @Override
+        public boolean shouldSwitch(long lts)
+        {
+            if (lastSwitchLts < 0 || lts - lastSwitchLts >= switchInterval)
+            {
+                lastSwitchLts = lts;
+                return true;
+            }
+            return false;
+        }
+
+        @Override
+        public PartitionAction[] generate(EntropySource rng)
+        {
+            // TODO (required): make configurable
+//            if (rng.nextBoolean())
+//                return EMPTY;
+
+            PartitionAction[] actions = new PartitionAction[rng.nextInt(5, 10)];
+            for (int i = 0; i < actions.length; i++)
+                actions[i] = gen.generate(rng);
+            return actions;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("random(target=%d, switchInterval=%d)", targetSize, switchInterval);
+        }
+    }
+
+    class FixedRotationStrategy implements RotationStrategy
+    {
+        private final int replaceWithNew;
+        private final int replaceWithVisited;
+        private final int targetSize;
+        private final int switchInterval;
+        private long lastSwitchLts = -1;
+
+        public FixedRotationStrategy(int targetSize, int replaceWithNew, int replaceWithVisited)
+        {
+            this(targetSize, replaceWithNew, replaceWithVisited, 500);
+        }
+
+        public FixedRotationStrategy(int targetSize, int replaceWithNew, int replaceWithVisited, int switchInterval)
+        {
+            this.replaceWithNew = replaceWithNew;
+            this.replaceWithVisited = replaceWithVisited;
+            this.targetSize = targetSize;
+            this.switchInterval = switchInterval;
+        }
+
+        @Override
+        public int targetSize()
+        {
+            return targetSize;
+        }
+
+        @Override
+        public boolean shouldSwitch(long lts)
+        {
+            if (lastSwitchLts < 0 || lts - lastSwitchLts >= switchInterval)
+            {
+                lastSwitchLts = lts;
+                return true;
+            }
+            return false;
+        }
+
+        @Override
+        public PartitionAction[] generate(EntropySource rng)
+        {
+            PartitionAction[] actions = new PartitionAction[replaceWithNew + replaceWithVisited];
+            Arrays.fill(actions, 0, replaceWithNew, PartitionAction.REPLACE_WITH_NEW);
+            Arrays.fill(actions, replaceWithNew, replaceWithNew + replaceWithVisited, PartitionAction.REPLACE_WITH_VISITED);
+            return actions;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("fixed(target=%d, replaceWithNew=%d, replaceWithVisited=%d, switchInterval=%d)",
+                                 targetSize, replaceWithNew, replaceWithVisited, switchInterval);
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/SSTableGenerator.java

@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.harry.stress;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.function.Consumer;
+
+import com.google.common.io.Files;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.execution.CQLVisitExecutor;
+import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.execution.QueryBuildingVisitExecutor;
+import org.apache.cassandra.harry.execution.ResultSetRow;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.ValueGenerators;
+import org.apache.cassandra.harry.model.Model;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+import org.apache.cassandra.io.sstable.HarrySSTableWriter;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Generates SSTables using the same visit generation machinery as {@link HarryStress}.
+ * Visits are compiled into CQL statements and fed into {@link HarrySSTableWriter} to produce
+ * SSTables on disk. The writer automatically flushes to a new SSTable when the configured
+ * size threshold is reached.
+ */
+public class SSTableGenerator
+{
+    public static final Logger LOGGER = LoggerFactory.getLogger(SSTableGenerator.class);
+
+    private final SchemaSpec schema;
+    private final VisitGenerator visitGenerator;
+    private final ActivePartition.Partitions partitionFactory;
+    private final boolean disableCompression;
+    private final int sstableSizeMiB;
+    private final int sstableLevel;
+    private final long repairedAtMillis;
+
+    public SSTableGenerator(SchemaSpec schema,
+                            Distribution rowPopulation,
+                            VisitGenerator.ColumnPopulation columnPopulation,
+                            Generator<VisitGenerator.VisitType> visitTypeGen,
+                            Distribution visitSizeDistribution,
+                            VisitGenerator.OpKindGenFactory operationKindGen,
+                            RotationStrategy rotationStrategy,
+                            boolean disableCompression,
+                            int sstableSizeMiB,
+                            long minPartitionIdx,
+                            long maxPartitionIdx,
+                            long initialLts)
+    {
+        this(schema, rowPopulation, columnPopulation, visitTypeGen, visitSizeDistribution,
+             operationKindGen, rotationStrategy, disableCompression, sstableSizeMiB,
+             minPartitionIdx, maxPartitionIdx, initialLts, 0, ActiveRepairService.UNREPAIRED_SSTABLE);
+    }
+
+    public SSTableGenerator(SchemaSpec schema,
+                            Distribution rowPopulation,
+                            VisitGenerator.ColumnPopulation columnPopulation,
+                            Generator<VisitGenerator.VisitType> visitTypeGen,
+                            Distribution visitSizeDistribution,
+                            VisitGenerator.OpKindGenFactory operationKindGen,
+                            RotationStrategy rotationStrategy,
+                            boolean disableCompression,
+                            int sstableSizeMiB,
+                            long minPartitionIdx,
+                            long maxPartitionIdx,
+                            long initialLts,
+                            int sstableLevel)
+    {
+        this(schema, rowPopulation, columnPopulation, visitTypeGen, visitSizeDistribution,
+             operationKindGen, rotationStrategy, disableCompression, sstableSizeMiB,
+             minPartitionIdx, maxPartitionIdx, initialLts, sstableLevel, ActiveRepairService.UNREPAIRED_SSTABLE);
+    }
+
+    public SSTableGenerator(SchemaSpec schema,
+                            Distribution rowPopulation,
+                            VisitGenerator.ColumnPopulation columnPopulation,
+                            Generator<VisitGenerator.VisitType> visitTypeGen,
+                            Distribution visitSizeDistribution,
+                            VisitGenerator.OpKindGenFactory operationKindGen,
+                            RotationStrategy rotationStrategy,
+                            boolean disableCompression,
+                            int sstableSizeMiB,
+                            long minPartitionIdx,
+                            long maxPartitionIdx,
+                            long initialLts,
+                            int sstableLevel,
+                            long repairedAtMillis)
+    {
+        this.schema = schema;
+        this.disableCompression = disableCompression;
+        this.sstableSizeMiB = sstableSizeMiB;
+        this.sstableLevel = sstableLevel;
+        this.repairedAtMillis = repairedAtMillis;
+        this.partitionFactory = new ActivePartition.Partitions(schema, rowPopulation, columnPopulation, rotationStrategy, minPartitionIdx, maxPartitionIdx, initialLts);
+        this.visitGenerator = new VisitGenerator(partitionFactory,
+                                                 visitTypeGen,
+                                                 visitSizeDistribution,
+                                                 operationKindGen,
+                                                 initialLts);
+        partitionFactory.populate();
+    }
+
+    public void generate(File directory, long totalVisits, java.io.File progressFile)
+    {
+        generate(directory, totalVisits, visit -> {}, progressFile);
+    }
+
+    public void generate(File directory, long totalVisits, Consumer<Visit> onVisit, java.io.File progressFile)
+    {
+        HarrySSTableWriter writer = newWriter(directory);
+        CQLVisitExecutor executor = createExecutor(writer, progressFile);
+
+        for (long i = 0; i < totalVisits; i++)
+        {
+            Visit visit = visitGenerator.get();
+
+            for (long pd : visit.visitedPartitions)
+                partitionFactory.forPd(pd).ref();
+
+            try
+            {
+                if (!visit.validating())
+                    executor.execute(visit);
+
+                onVisit.accept(visit);
+
+                partitionFactory.maybeSwitchPartition(visit.lts, action -> {});
+            }
+            finally
+            {
+                for (long pd : visit.visitedPartitions)
+                    partitionFactory.forPd(pd).deref();
+            }
+        }
+
+        closeWriter(writer);
+    }
+
+    private HarrySSTableWriter newWriter(File directory)
+    {
+        return newWriter(directory, sstableLevel, repairedAtMillis);
+    }
+
+    private HarrySSTableWriter newWriter(File directory, int level)
+    {
+        return newWriter(schema, disableCompression, sstableSizeMiB, directory, level, ActiveRepairService.UNREPAIRED_SSTABLE);
+    }
+
+    private HarrySSTableWriter newWriter(File directory, int level, long repairedAtMillis)
+    {
+        return newWriter(schema, disableCompression, sstableSizeMiB, directory, level, repairedAtMillis);
+    }
+
+    public static HarrySSTableWriter newWriter(SchemaSpec schema, boolean disableCompression, int sstableSizeMiB, File directory, int level)
+    {
+        return newWriter(schema, disableCompression, sstableSizeMiB, directory, level, ActiveRepairService.UNREPAIRED_SSTABLE);
+    }
+
+    public static HarrySSTableWriter newWriter(SchemaSpec schema, boolean disableCompression, int sstableSizeMiB, File directory, int level, long repairedAtMillis)
+    {
+        try
+        {
+            String tableCql = schema.compile();
+            if (disableCompression)
+            {
+                String noSemicolon = tableCql.endsWith(";") ? tableCql.substring(0, tableCql.length() - 1) : tableCql;
+                if (noSemicolon.contains(" WITH "))
+                    tableCql = noSemicolon + " AND compression = {'enabled': 'false'};";
+                else
+                    tableCql = noSemicolon + " WITH compression = {'enabled': 'false'};";
+            }
+            return HarrySSTableWriter.builder()
+                                     .forTable(tableCql)
+                                     .inDirectory(directory)
+                                     .withMaxSSTableSizeInMiB(sstableSizeMiB)
+                                     .withSSTableLevel(level)
+                                     .withRepairedAtMillis(repairedAtMillis)
+                                     .build();
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException("Failed to create SSTable writer", e);
+        }
+    }
+
+    private CQLVisitExecutor createExecutor(HarrySSTableWriter writer, java.io.File progressFile)
+    {
+        return createExecutor(schema, partitionFactory, writer, progressFile);
+    }
+
+    public static CQLVisitExecutor createExecutor(SchemaSpec schema, ValueGenerators<Object[], Object[]> valueGenerators, HarrySSTableWriter writer, java.io.File progressFile)
+    {
+        DataTracker tracker = new DataTracker.NoOpDataTracker();
+        QueryBuildingVisitExecutor queryBuilder = new QueryBuildingVisitExecutor(schema,
+                                                                                 QueryBuildingVisitExecutor.WrapQueries.EMPTY,
+                                                                                 valueGenerators);
+        return new CQLVisitExecutor(schema, tracker, Model.NO_OP, queryBuilder)
+        {
+            long lts;
+            {
+                writer.setListener(file -> {
+                    System.out.println(String.format("Written to %d: %s", lts, file));
+                    if (progressFile != null)
+                    {
+                        try { Files.write(ByteBufferUtil.getArray(ByteBufferUtil.bytes(lts)), progressFile); }
+                        catch (IOException e) { throw new RuntimeException(e); }
+                    }
+                });
+            }
+            @Override
+            protected void executeMutatingVisit(Visit visit, CompiledStatement statement)
+            {
+                lts = visit.lts;
+                try
+                {
+                    writer.addRow(statement.cql(), statement.bindings());
+                }
+                catch (IOException e)
+                {
+                    throw new UncheckedIOException(e);
+                }
+            }
+
+            @Override
+            protected void executeValidatingVisit(Visit visit, List<Operations.SelectStatement> selects, CompiledStatement compiledStatement)
+            {
+            }
+
+            @Override
+            protected List<ResultSetRow> executeWithResult(Visit visit, CompiledStatement statement)
+            {
+                throw new UnsupportedOperationException("SSTable generation does not support reads");
+            }
+
+            @Override
+            protected void executeWithoutResult(Visit visit, CompiledStatement statement)
+            {
+                executeMutatingVisit(visit, statement);
+            }
+
+            @Override
+            public void execute(Visit visit)
+            {
+                if (visit.visitedPartitions.length > 1)
+                    throw new IllegalStateException("SSTable generator does not support batch statements across multiple partitions");
+
+                super.execute(visit);
+            }
+        };
+    }
+
+    private static void closeWriter(HarrySSTableWriter writer)
+    {
+        try
+        {
+            writer.close();
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException("Failed to close SSTable writer", e);
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/StressWorker.java

@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import org.HdrHistogram.Histogram;
+import org.apache.cassandra.concurrent.ExecutorFactory;
+import org.apache.cassandra.concurrent.InfiniteLoopExecutor;
+import org.apache.cassandra.concurrent.Interruptible;
+import org.apache.cassandra.harry.execution.CQLVisitExecutor;
+import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
+import org.apache.cassandra.harry.execution.QueryBuildingVisitExecutor;
+import org.apache.cassandra.harry.execution.ResultSetRow;
+import org.apache.cassandra.harry.model.Model;
+import org.apache.cassandra.harry.model.QuiescentChecker;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.utils.Clock;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+
+public class StressWorker implements Interruptible
+{
+    public static class WorkerMetrics
+    {
+        final Histogram reads, writes;
+        final AtomicInteger failedReads = new AtomicInteger(), failedWrites = new AtomicInteger();
+
+        public WorkerMetrics(Histogram reads, Histogram writes)
+        {
+            this.reads = reads;
+            this.writes = writes;
+        }
+    }
+    private final LinkedBlockingQueue<Visit> tasks;
+    private final int capacity;
+
+    private final CQLVisitExecutor executor;
+    private final Interruptible loop;
+    private final AtomicReference<WorkerMetrics> metrics = new AtomicReference<>();
+
+    public StressWorker(int idx,
+                        Model.PartialReplay replay,
+                        DataTracker tracker,
+                        ActivePartition.Partitions partitionFactory,
+                        Supplier<BiFunction<CompiledStatement, Runnable, Object[][]>> sutFactory,
+                        Consumer<Throwable> onException,
+                        int queueCapacity)
+    {
+        resetMetrics();
+        Function<CompiledStatement, Object[][]> sut = new Function<>()
+        {
+            final BiFunction<CompiledStatement, Runnable, Object[][]> delegate = sutFactory.get();
+
+            @Override
+            public Object[][] apply(CompiledStatement compiledStatement)
+            {
+                long start = Clock.Global.nanoTime();
+                return delegate.apply(compiledStatement, () -> {
+                    long end = Clock.Global.nanoTime();
+                    WorkerMetrics metrics1 = StressWorker.this.metrics.get();
+                    (compiledStatement.validating ? metrics1.reads : metrics1.writes).recordValue(end - start);
+                });
+            }
+        };
+
+        this.tasks = new LinkedBlockingQueue<>(queueCapacity);
+        this.capacity = queueCapacity;
+        this.executor = new CQLVisitExecutor(partitionFactory.schema,
+                                             tracker,
+                                             new QuiescentChecker(partitionFactory, replay),
+                                             new QueryBuildingVisitExecutor(partitionFactory.schema, QueryBuildingVisitExecutor.WrapQueries.EMPTY, partitionFactory))
+        {
+            @Override
+            protected List<ResultSetRow> executeWithResult(Visit visit, CompiledStatement statement)
+            {
+                Object[][] result = sut.apply(statement);
+                if (result == null)
+                    return new ArrayList<>();
+                return InJvmDTestVisitExecutor.rowsToResultSet(schema, partitionFactory,
+                                                               (Operations.SelectStatement) visit.operations[0], result);
+            }
+
+            @Override
+            protected void executeWithoutResult(Visit visit, CompiledStatement statement)
+            {
+                sut.apply(statement);
+            }
+        };
+        this.loop = executorFactory().infiniteLoop("visit-executor" + idx, state -> {
+            try
+            {
+                switch (state)
+                {
+                    case NORMAL:
+                        Visit visit = tasks.take();
+                        try
+                        {
+                            executor.execute(visit);
+                        }
+                        catch (Throwable e)
+                        {
+                            if (e.getClass().toString().contains("InterruptedException"))
+                                return;
+                            WorkerMetrics ms = metrics.get();
+                            (visit.validating ? ms.failedReads : ms.failedWrites).incrementAndGet();
+                            onException.accept(e);
+                        }
+                        break;
+                    case INTERRUPTED:
+                    case SHUTTING_DOWN:
+                        break;
+                }
+            }
+            catch (Throwable e)
+            {
+                onException.accept(e);
+            }
+        }, SAFE, ExecutorFactory.SystemThreadTag.DAEMON, InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED);
+    }
+
+    public int getFreeSlots()
+    {
+        return capacity - tasks.size();
+    }
+
+    public WorkerMetrics resetMetrics()
+    {
+        return metrics.getAndSet(new WorkerMetrics(new Histogram(3), new Histogram(3)));
+    }
+
+    public boolean offer(Visit visit)
+    {
+        return tasks.offer(visit);
+    }
+
+    @Override
+    public void interrupt()
+    {
+        loop.interrupt();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return loop.isTerminated();
+    }
+
+    @Override
+    public void shutdown()
+    {
+        loop.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return loop.shutdownNow();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
+    {
+        return loop.awaitTermination(timeout, units);
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/TokenIndex.java

@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
+/**
+ * Indexed reader over a merged token data file and its companion {@code .idx} file,
+ * both produced by {@link TokenIndexGenerator#merge}.
+ *
+ * <p>Data file format (sorted by token, then pd): {@code [token:8][pd:8][count:4][lts:8 * count]} repeated.
+ * <p>Index file format (sorted by token): {@code [token:8][offset:8]} repeated, fixed 16-byte entries.
+ *
+ * <p>The index is binary-searchable because entries are fixed-size and token-sorted.
+ * After locating the start of a range in the index, entries are read sequentially from the data file.
+ */
+public class TokenIndex implements Closeable
+{
+    private final FileInputStream dataFis;
+    private final FileInputStream idxFis;
+    private final FileChannel dataChannel;
+    private final FileChannel idxChannel;
+    private final long entryCount;
+    private final ByteBuffer idxBuf = ByteBuffer.allocate(16); // token(8) + offset(8)
+
+    public TokenIndex(File dataFile, File idxFile)
+    {
+        try
+        {
+            this.dataFis = new FileInputStream(dataFile);
+            this.idxFis = new FileInputStream(idxFile);
+            this.dataChannel = dataFis.getChannel();
+            this.idxChannel = idxFis.getChannel();
+            this.entryCount = idxChannel.size() / 16;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    /** Convenience overload accepting Cassandra's {@link org.apache.cassandra.io.util.File}. */
+    public TokenIndex(org.apache.cassandra.io.util.File dataFile, org.apache.cassandra.io.util.File idxFile)
+    {
+        this(dataFile.toJavaIOFile(), idxFile.toJavaIOFile());
+    }
+
+    public long entryCount()
+    {
+        return entryCount;
+    }
+
+    public long lookup(long token)
+    {
+        long idx = lowerBound(token);
+        if (idx >= entryCount)
+            return -1;
+        readIdxEntry(idx);
+        long foundToken = idxBuf.getLong();
+        if (foundToken != token)
+            return -1;
+        return idxBuf.getLong();
+    }
+
+    public EntryIterator range(long minToken, long maxToken)
+    {
+        long startIdx = lowerBound(minToken);
+        if (startIdx >= entryCount)
+            return new EntryIterator(dataChannel, 0, maxToken, false);
+        readIdxEntry(startIdx);
+        idxBuf.getLong(); // skip token
+        long dataOffset = idxBuf.getLong();
+        return new EntryIterator(dataChannel, dataOffset, maxToken, true);
+    }
+
+    private long lowerBound(long target)
+    {
+        long lo = 0, hi = entryCount;
+        while (lo < hi)
+        {
+            long mid = lo + (hi - lo) / 2;
+            if (readIdxToken(mid) < target)
+                lo = mid + 1;
+            else
+                hi = mid;
+        }
+        return lo;
+    }
+
+    private long readIdxToken(long index)
+    {
+        readIdxEntry(index);
+        return idxBuf.getLong();
+    }
+
+    private void readIdxEntry(long index)
+    {
+        idxBuf.clear();
+        long pos = index * 16;
+        while (idxBuf.hasRemaining())
+        {
+            try
+            {
+                int n = idxChannel.read(idxBuf, pos + idxBuf.position());
+                if (n < 0)
+                    throw new IllegalStateException("Unexpected EOF in index at entry " + index);
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+        idxBuf.flip();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        try
+        {
+            dataChannel.close();
+        }
+        finally
+        {
+            try
+            {
+                dataFis.close();
+            }
+            finally
+            {
+                try
+                {
+                    idxChannel.close();
+                }
+                finally
+                {
+                    idxFis.close();
+                }
+            }
+        }
+    }
+
+    public static class EntryIterator
+    {
+        private final FileChannel channel;
+        private final long maxToken;
+        private final ByteBuffer headerBuf = ByteBuffer.allocate(20); // token(8) + pd(8) + count(4)
+        private long currentToken;
+        private long currentPd;
+        private int currentLtsCount;
+        private long ltsDataOffset;
+        private boolean hasMore;
+
+        EntryIterator(FileChannel channel, long startOffset, long maxToken, boolean hasData)
+        {
+            this.channel = channel;
+            this.maxToken = maxToken;
+            if (hasData)
+            {
+                try
+                {
+                    channel.position(startOffset);
+                }
+                catch (IOException e)
+                {
+                    throw new UncheckedIOException(e);
+                }
+                readHeader();
+            }
+            else
+            {
+                this.hasMore = false;
+            }
+        }
+
+        public boolean hasNext()
+        {
+            return hasMore;
+        }
+
+        public long token()
+        {
+            return currentToken;
+        }
+
+        public long pd()
+        {
+            return currentPd;
+        }
+
+        public int ltsCount()
+        {
+            return currentLtsCount;
+        }
+
+        public long[] readLts()
+        {
+            try
+            {
+                ByteBuffer buf = ByteBuffer.allocate(currentLtsCount * Long.BYTES);
+                long filePos = ltsDataOffset;
+                while (buf.hasRemaining())
+                {
+                    int n = channel.read(buf, filePos);
+                    if (n < 0)
+                        throw new IOException("Unexpected EOF reading LTS data");
+                    filePos += n;
+                }
+                buf.flip();
+                long[] lts = new long[currentLtsCount];
+                for (int i = 0; i < currentLtsCount; i++)
+                    lts[i] = buf.getLong();
+                return lts;
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        public void advance()
+        {
+            try
+            {
+                channel.position(ltsDataOffset + (long) currentLtsCount * Long.BYTES);
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+            readHeader();
+        }
+
+        private void readHeader()
+        {
+            headerBuf.clear();
+            try
+            {
+                while (headerBuf.hasRemaining())
+                {
+                    int read = channel.read(headerBuf);
+                    if (read < 0)
+                    {
+                        hasMore = false;
+                        return;
+                    }
+                }
+                headerBuf.flip();
+                currentToken = headerBuf.getLong();
+                currentPd = headerBuf.getLong();
+                currentLtsCount = headerBuf.getInt();
+                ltsDataOffset = channel.position();
+                hasMore = currentToken <= maxToken;
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/TokenIndexGenerator.java

@@ -0,0 +1,451 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import java.io.BufferedOutputStream;
+import java.io.Closeable;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.HashMap;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.rng.SeedableEntropySource;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+import org.apache.cassandra.harry.util.ByteUtils;
+import org.apache.cassandra.harry.util.TokenUtil;
+
+/**
+ * In order to generate SSTables per level, we ideally need to know partitions ahead of time, since otherwise
+ * we need to keep SSTableWriters open, which is not feasible. To do this, we iterate through all LTS that will
+ * be visited, and generate a mapping of Token / partition descriptor to set of LTS that will be visited for that
+ * partition.
+ *
+ * Token index files are sorted in memory and flushed on disk in token order. After all LTS were replayed, we
+ * merge-iterate through individual token index files and create one big merged file, in token order.
+ */
+public class TokenIndexGenerator
+{
+    static class TokenAndPd implements Comparable<TokenAndPd>
+    {
+        final long token;
+        final long pd;
+
+        TokenAndPd(long token, long pd)
+        {
+            this.token = token;
+            this.pd = pd;
+        }
+
+        @Override
+        public int compareTo(TokenAndPd o)
+        {
+            int cmp = Long.compare(token, o.token);
+            return cmp != 0 ? cmp : Long.compare(pd, o.pd);
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (!(o instanceof TokenAndPd)) return false;
+            TokenAndPd that = (TokenAndPd) o;
+            return token == that.token && pd == that.pd;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Long.hashCode(token) * 31 + Long.hashCode(pd);
+        }
+    }
+
+    public static void generate(File dir,
+                                SchemaSpec schema,
+                                RotationStrategy rotationStrategy,
+                                Distribution rowPopulation,
+                                Generator<VisitGenerator.VisitType> visitTypeGen,
+                                Distribution visitSizeDistribution,
+                                long initialLts,
+                                long visits) throws IOException
+    {
+        System.out.println("******************** SStable Level Generator ********************");
+        System.out.println(String.format("  Output Directory: %s", dir));
+        System.out.println(String.format("  Initial LTS: %d", initialLts));
+        System.out.println(String.format("  Visits: %d", visits));
+        System.out.println(String.format("  Rotation: %s", rotationStrategy));
+        System.out.println();
+        int fileIdx = 0;
+
+        long nextPartitionIdx = 0;
+
+        class TokenCache
+        {
+            private final HashMap<Long, Long> map = new HashMap<>();
+            private final long[] ring;
+            private int pos = 0;
+            private boolean full = false;
+
+            TokenCache(int capacity)
+            {
+                this.ring = new long[capacity];
+            }
+
+            long tokenFor(long partitionIdx)
+            {
+                Long cached = map.get(partitionIdx);
+                if (cached != null)
+                    return cached;
+
+                long pd = ActivePartition.DescriptorIndexBijection.INSTANCE.toPd(partitionIdx);
+                Object[] pkValues = SeedableEntropySource.computeWithSeed(pd, SchemaSpec.forKeys(schema.partitionKeys)::generate);
+                long token = TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(pkValues)));
+
+                if (full)
+                    map.remove(ring[pos]);
+                ring[pos] = partitionIdx;
+                pos = (pos + 1) % ring.length;
+                if (pos == 0) full = true;
+                map.put(partitionIdx, token);
+                return token;
+            }
+        }
+
+        TokenCache tokenCache = new TokenCache(10_000);
+        ActivePartition.VisitedPartitions visitedPartitions = new ActivePartition.VisitedPartitions(0);
+        long[] activePartitionIdxs = new long[rotationStrategy.targetSize()];
+        Set<Long> activePartitionIdxSet = new HashSet<>(activePartitionIdxs.length);
+        Map<Long, Integer> ltsPerPd = new HashMap<>();
+        Map<Long, Integer> sizePerPd = new HashMap<>();
+
+        // Initial active set fill. Do NOT pre-add to visitedPartitions: ActivePartition.Partitions.populate()
+        // only adds an idx to visitedPartitions when it's evicted by a rotation, never at fill time.
+        for (int i = 0; i < rotationStrategy.targetSize(); i++)
+        {
+            long idx = nextPartitionIdx++;
+            long pd = ActivePartition.DescriptorIndexBijection.INSTANCE.toPd(idx);
+            activePartitionIdxs[i] = idx;
+            activePartitionIdxSet.add(idx);
+            sizePerPd.putIfAbsent(pd, Math.toIntExact(rowPopulation.next(pd)));
+        }
+
+        TreeMap<TokenAndPd, ArrayList<Long>> visitsPerToken = new TreeMap<>();
+        AtomicInteger cnt = new AtomicInteger();
+        List<File> files = new ArrayList<>();
+        for (long lts = 0; lts < initialLts + visits; lts++)
+        {
+            if (lts % 100_000 == 0)
+                System.out.println("Seen " + lts);
+
+            // Visit at lts is generated against state from rotations [< lts] (matching VisitGenerator.inflate +
+            // HarryStress.start, where visit(L) is generated before maybeSwitchPartition(L) is called).
+            if (lts >= initialLts)
+            {
+                VisitGenerator.VisitType visitType = SeedableEntropySource.computeWithSeed(lts, visitTypeGen::generate);
+                if (visitType == VisitGenerator.VisitType.MUTATE)
+                {
+                    long visitSize = visitSizeDistribution.next(lts);
+                    Invariants.require(visitSize > 0);
+                    for (int op = 0; op < visitSize; op++)
+                    {
+                        // If you are changing choosing here, make sure to also change VisitGeneratort#mutationVisit
+                        int slot = SeedableEntropySource.computeWithSeed(lts, ~op, r -> r.nextInt(activePartitionIdxs.length));
+                        long partitionIdx = activePartitionIdxs[slot];
+                        long pd = ActivePartition.DescriptorIndexBijection.INSTANCE.toPd(partitionIdx);
+                        long token = tokenCache.tokenFor(partitionIdx);
+                        visitsPerToken.computeIfAbsent(new TokenAndPd(token, pd), k -> new ArrayList<>(10))
+                                      .add(lts);
+                        ltsPerPd.merge(pd, 1, Integer::sum);
+                        cnt.incrementAndGet();
+                    }
+                }
+            }
+
+            // Rotation. Skip at lts==initialLts: populate covers [0, initialLts) and HarryStress.start's first
+            // maybeSwitchPartition is at initialLts+1, so shouldSwitch(initialLts) is never called by runtime.
+            if (lts != initialLts && rotationStrategy.shouldSwitch(lts))
+            {
+                RotationStrategy.PartitionAction[] actions = SeedableEntropySource.computeWithSeed(lts, rotationStrategy::generate);
+                for (int i = 0; i < actions.length; i++)
+                {
+                    RotationStrategy.PartitionAction action = actions[i];
+                    int remove = SeedableEntropySource.computeWithSeed(Util.hash(lts, i), r -> r.nextInt(activePartitionIdxs.length));
+                    long toVisitedIdx = activePartitionIdxs[remove];
+                    long toVisitedPd = ActivePartition.DescriptorIndexBijection.INSTANCE.toPd(toVisitedIdx);
+                    int partitionSize = Math.toIntExact(rowPopulation.next(toVisitedPd));
+                    sizePerPd.putIfAbsent(toVisitedPd, partitionSize);
+                    boolean evict = SeedableEntropySource.computeWithSeed(Util.hash(lts, i), r -> r.nextInt(Math.max(1, Integer.highestOneBit(partitionSize))) == 0);
+                    if (!evict) continue;
+                    switch (action)
+                    {
+                        case REPLACE_WITH_NEW:
+                        {
+                            long newIdx = nextPartitionIdx++;
+                            long newPd = ActivePartition.DescriptorIndexBijection.INSTANCE.toPd(newIdx);
+                            sizePerPd.putIfAbsent(newPd, Math.toIntExact(rowPopulation.next(newPd)));
+                            activePartitionIdxSet.remove(toVisitedIdx);
+                            activePartitionIdxs[remove] = newIdx;
+                            activePartitionIdxSet.add(newIdx);
+                            break;
+                        }
+                        case REPLACE_WITH_VISITED:
+                        {
+                            long toRevisitIdx = visitedPartitions.getBySeed(Util.hash(lts, i));
+                            if (toRevisitIdx < 0 || activePartitionIdxSet.contains(toRevisitIdx))
+                            {
+                                // picked one is still active; fall back to new
+                                long newIdx = nextPartitionIdx++;
+                                long newPd = ActivePartition.DescriptorIndexBijection.INSTANCE.toPd(newIdx);
+                                sizePerPd.putIfAbsent(newPd, Math.toIntExact(rowPopulation.next(newPd)));
+                                activePartitionIdxSet.remove(activePartitionIdxs[remove]);
+                                activePartitionIdxs[remove] = newIdx;
+                                activePartitionIdxSet.add(newIdx);
+                            }
+                            else
+                            {
+                                activePartitionIdxSet.remove(toVisitedIdx);
+                                activePartitionIdxs[remove] = toRevisitIdx;
+                                activePartitionIdxSet.add(toRevisitIdx);
+                            }
+                            break;
+                        }
+                    }
+                    visitedPartitions.add(toVisitedIdx);
+                }
+            }
+
+            if (visitsPerToken.size() > 100000 || lts == initialLts + visits - 1)
+            {
+                File currentFile = new File(dir, "tokens_" + fileIdx++);
+                System.out.println("Writing " + currentFile);
+                currentFile.createNewFile();
+                files.add(currentFile);
+                try (FileOutputStream s = new FileOutputStream(currentFile);
+                     BufferedOutputStream os = new BufferedOutputStream(s);
+                     DataOutputStream dos = new DataOutputStream(os))
+                {
+                    for (Map.Entry<TokenAndPd, ArrayList<Long>> entry : visitsPerToken.entrySet())
+                    {
+                        dos.writeLong(entry.getKey().token);
+                        dos.writeLong(entry.getKey().pd);
+                        dos.writeInt(entry.getValue().size());
+                        for (Long l : entry.getValue())
+                            dos.writeLong(l);
+                    }
+                }
+                visitsPerToken.clear();
+            }
+        }
+        merge(files, new File(dir, "merged_tokens"));
+
+        printHistogram("LTS per partition", ltsPerPd.values());
+        printHistogram("Partition size", sizePerPd.values());
+    }
+
+    private static final int[] BOUNDARIES = { 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 100000 };
+
+    static void printHistogram(String label, Collection<Integer> values)
+    {
+        int[] buckets = new int[BOUNDARIES.length + 1];
+        for (int v : values)
+        {
+            int b = 0;
+            while (b < BOUNDARIES.length && v >= BOUNDARIES[b])
+                b++;
+            buckets[b]++;
+        }
+        System.out.println(String.format("\n%s distribution (%d entries):", label, values.size()));
+        System.out.println(String.format("  [0-%d): %d", BOUNDARIES[0], buckets[0]));
+        for (int i = 1; i < BOUNDARIES.length; i++)
+        {
+            if (buckets[i] > 0)
+                System.out.println(String.format("  [%d-%d): %d", BOUNDARIES[i - 1], BOUNDARIES[i], buckets[i]));
+        }
+        System.out.println(String.format("  [%d+): %d", BOUNDARIES[BOUNDARIES.length - 1], buckets[BOUNDARIES.length]));
+    }
+
+    /**
+     * Merges token files produced by {@link #generate}.
+     * If the same (token, pd) appears in multiple files, LTS bytes are concatenated in file index order.
+     * Also writes an.idx file with [token:8][offset:8] entries (one per unique token).
+     */
+    public static void merge(List<File> inputFiles, File outputFile) throws IOException
+    {
+        class PeekingIter implements Comparable<PeekingIter>, Closeable
+        {
+            private final FileInputStream fis;
+            private final FileChannel channel;
+            private final int fileIndex;
+            private final ByteBuffer headerBuf = ByteBuffer.allocate(20); // token(8) + pd(8) + count(4)
+            long currentToken;
+            long currentPd;
+            int currentLtsCount;
+            boolean hasMore;
+
+            PeekingIter(FileInputStream fis, int fileIndex) throws IOException
+            {
+                this.fis = fis;
+                this.channel = fis.getChannel();
+                this.fileIndex = fileIndex;
+                advance();
+            }
+
+            void advance() throws IOException
+            {
+                headerBuf.clear();
+                while (headerBuf.hasRemaining())
+                {
+                    int read = channel.read(headerBuf);
+                    if (read < 0)
+                    {
+                        hasMore = false;
+                        return;
+                    }
+                }
+                headerBuf.flip();
+                currentToken = headerBuf.getLong();
+                currentPd = headerBuf.getLong();
+                currentLtsCount = headerBuf.getInt();
+                hasMore = true;
+            }
+
+            void appendTo(FileChannel out) throws IOException
+            {
+                long bytes = (long) currentLtsCount * Long.BYTES;
+                long pos = channel.position();
+                long remaining = bytes;
+                while (remaining > 0)
+                {
+                    long transferred = channel.transferTo(pos, remaining, out);
+                    pos += transferred;
+                    remaining -= transferred;
+                }
+                channel.position(pos);
+            }
+
+            @Override
+            public int compareTo(PeekingIter other)
+            {
+                int cmp = Long.compare(this.currentToken, other.currentToken);
+                if (cmp != 0) return cmp;
+                cmp = Long.compare(this.currentPd, other.currentPd);
+                if (cmp != 0) return cmp;
+                return Integer.compare(this.fileIndex, other.fileIndex);
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+                channel.close();
+                fis.close();
+            }
+        }
+
+        ArrayList<PeekingIter> readers = new ArrayList<>();
+        PriorityQueue<PeekingIter> pq = new PriorityQueue<>();
+        for (int i = 0; i < inputFiles.size(); i++)
+        {
+            @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") FileInputStream fis = new FileInputStream(inputFiles.get(i));
+            PeekingIter reader = new PeekingIter(fis, i);
+            readers.add(reader);
+            if (reader.hasMore)
+                pq.add(reader);
+        }
+
+        System.out.println("Writing merged results to " + outputFile);
+        try (FileOutputStream fos = new FileOutputStream(outputFile);
+             FileChannel out = fos.getChannel();
+             FileOutputStream idxFos = new FileOutputStream(outputFile.getPath() + ".idx");
+             FileChannel idx = idxFos.getChannel())
+        {
+            ByteBuffer headerBuf = ByteBuffer.allocate(20); // token(8) + pd(8) + count(4)
+            ByteBuffer idxBuf = ByteBuffer.allocate(16);    // token(8) + offset(8)
+            long lastIndexedToken = Long.MIN_VALUE;
+            boolean firstEntry = true;
+
+            while (!pq.isEmpty())
+            {
+                long token = pq.peek().currentToken;
+                long pd = pq.peek().currentPd;
+
+                // Write index entry only for first occurrence of each token
+                if (firstEntry || token != lastIndexedToken)
+                {
+                    long offset = out.position();
+                    idxBuf.clear();
+                    idxBuf.putLong(token);
+                    idxBuf.putLong(offset);
+                    idxBuf.flip();
+                    while (idxBuf.hasRemaining())
+                        idx.write(idxBuf);
+                    lastIndexedToken = token;
+                    firstEntry = false;
+                }
+
+                // Gather all readers at this (token, pd); PQ orders by (token, pd, fileIndex)
+                int totalCount = 0;
+                ArrayList<PeekingIter> batch = new ArrayList<>();
+                while (!pq.isEmpty() && pq.peek().currentToken == token && pq.peek().currentPd == pd)
+                {
+                    PeekingIter r = pq.poll();
+                    totalCount += r.currentLtsCount;
+                    batch.add(r);
+                }
+
+                // Write merged header
+                headerBuf.clear();
+                headerBuf.putLong(token);
+                headerBuf.putLong(pd);
+                headerBuf.putInt(totalCount);
+                headerBuf.flip();
+                while (headerBuf.hasRemaining())
+                    out.write(headerBuf);
+
+                // Transfer LTS bytes from each reader directly to output
+                for (PeekingIter r : batch)
+                {
+                    r.appendTo(out);
+                    r.advance();
+                    if (r.hasMore)
+                        pq.add(r);
+                }
+            }
+        }
+        finally
+        {
+            for (PeekingIter r : readers)
+                r.close();
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/Util.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+public class Util
+{
+    // Cantor pairing
+    public static long hash(long v1, int v2) {
+        return ((v1 + v2) * (v1 + v2 + 1) / 2) + v2;
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/VisitGenerator.java

@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress;
+
+import accord.utils.UnhandledEnum;
+import org.apache.cassandra.harry.MagicConstants;
+import org.apache.cassandra.harry.Relations;
+import org.apache.cassandra.harry.dsl.SingleOperationVisitBuilder;
+import org.apache.cassandra.harry.gen.EntropySource;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.Generators;
+import org.apache.cassandra.harry.gen.Surjections;
+import org.apache.cassandra.harry.gen.rng.SeedableEntropySource;
+import org.apache.cassandra.harry.op.ClusteringOrderBy;
+import org.apache.cassandra.harry.op.Kind;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import static org.apache.cassandra.harry.op.Operations.Operation;
+import static org.apache.cassandra.harry.op.Operations.PartitionOperation;
+
+/**
+ * A stateful generator for operation steps.
+ *
+ * Generator maintains a list of active partitions. On each logical timestamp, a subset of these
+ * partitions is chosen for visit.
+ */
+public class VisitGenerator implements Surjections.Surjection<Visit>, Supplier<Visit>
+{
+    final ActivePartition.Partitions partitions;
+    final Generator<VisitType> visitTypeGen;
+
+    final Distribution visitSizeDistribution;
+    final OpKindGenFactory operationKindGen;
+
+    // The target number of inserts to split a partition into; if more than one, the partition will be placed in the revisit set
+    private final AtomicLong lts = new AtomicLong();
+
+    public VisitGenerator(ActivePartition.Partitions partitions,
+                          Generator<VisitType> visitTypeGen,
+                          Distribution visitSizeDistribution,
+                          OpKindGenFactory operationKindGen)
+    {
+        this(partitions, visitTypeGen, visitSizeDistribution, operationKindGen, 0);
+    }
+
+    public VisitGenerator(ActivePartition.Partitions partitions,
+                          Generator<VisitType> visitTypeGen,
+                          Distribution visitSizeDistribution,
+                          OpKindGenFactory operationKindGen,
+                          long initialLts)
+    {
+        this.partitions = partitions;
+        this.visitTypeGen = visitTypeGen;
+        this.visitSizeDistribution = visitSizeDistribution;
+        this.operationKindGen = operationKindGen;
+        this.lts.set(initialLts);
+    }
+
+    public interface ColumnPopulation
+    {
+        Distribution distribution(String column);
+    }
+
+    public interface OpKindGenFactory
+    {
+        Generator<Kind> forType(VisitType type);
+    }
+
+    @Override
+    public Visit inflate(long lts)
+    {
+        Visit visit;
+        VisitType visitType = SeedableEntropySource.computeWithSeed(lts, visitTypeGen::generate);
+        switch (visitType)
+        {
+            case MUTATE:
+                visit = mutatingVisit(lts);
+                break;
+            case VALIDATE:
+                visit = validatingVisit(lts);
+                break;
+            default:
+                throw UnhandledEnum.unknown(visitType);
+        }
+        return visit;
+    }
+
+    @Override
+    public Visit get()
+    {
+        return inflate(this.lts.getAndIncrement());
+    }
+
+    private Visit mutatingVisit(long lts)
+    {
+        return mutatingVisit(lts, visitSizeDistribution, operationKindGen, partitions::pick);
+    }
+
+    public static Visit mutatingVisit(long lts, Distribution visitSizeDistribution, OpKindGenFactory operationKindGen, Function<EntropySource, ActivePartition> partitionPicker)
+    {
+        Operation[] operations = new Operation[Math.toIntExact(visitSizeDistribution.next(lts))];
+        for (int op = 0; op < operations.length; op++)
+        {
+            // If you are changing choosing here, make sure to also change TokenIndexGenerator#generate
+            ActivePartition partition = SeedableEntropySource.computeWithSeed(lts, ~op, partitionPicker);
+            operations[op] = SeedableEntropySource.computeWithSeed(lts, op, rng -> {
+                Kind kind = operationKindGen.forType(VisitType.MUTATE).generate(rng);
+                return mutatingOperation(partition, kind, lts, rng);
+            });
+        }
+        return new Visit(lts, operations);
+    }
+
+    static PartitionOperation mutatingOperation(ActivePartition partition, Kind kind, long lts, EntropySource rng)
+    {
+        // TODO: more sophisticated picking
+        int cdIdx = rng.nextInt(partition.ckPopulation());
+
+        int[] valueIdxs = new int[partition.regularColumnCount()];
+        for (int i = 0; i < valueIdxs.length; i++)
+            valueIdxs[i] = rng.nextInt(partition.regularPopulation(i));
+        int[] sValueIdxs = new int[partition.staticColumnCount()];
+        for (int i = 0; i < sValueIdxs.length; i++)
+            sValueIdxs[i] = rng.nextInt(partition.staticColumnCount());
+        return SingleOperationVisitBuilder.write(partition.pkGen(),
+                                                 partition,
+                                                 lts,
+                                                 partition.idx,
+                                                 cdIdx,
+                                                 valueIdxs,
+                                                 sValueIdxs,
+                                                 kind);
+    }
+
+    private Visit validatingVisit(long lts)
+    {
+        ActivePartition partition = SeedableEntropySource.computeWithSeed(lts, ~0, partitions::pick);
+        return SeedableEntropySource.computeWithSeed(lts, rng -> {
+            Kind kind = operationKindGen.forType(VisitType.VALIDATE).generate(rng);
+            return new Visit(lts, new Operation[] { validatingOperation(partition, kind, lts, rng) });
+        });
+    }
+
+    private static PartitionOperation validatingOperation(ActivePartition partition, Kind kind, long lts, EntropySource rng)
+    {
+        switch (kind)
+        {
+            case CUSTOM:
+            case UPDATE:
+            case INSERT:
+            case DELETE_PARTITION:
+            case DELETE_ROW:
+            case DELETE_COLUMNS:
+            case DELETE_RANGE:
+                throw new IllegalArgumentException("Validating operation can only be SELECT");
+            case SELECT_PARTITION:
+                return new Operations.SelectPartition(lts, partition.pd, ClusteringOrderBy.ASC);
+            case SELECT_ROW:
+                // ckIdxGen yields a clustering index; convert it to the clustering descriptor (cd) the SelectRow
+                // expects, exactly as SELECT_RANGE does for its bound below.
+                return new Operations.SelectRow(lts, partition.pd, partition.ckGen().descriptorAt(partition.ckIdxGen.generate(rng)));
+            case SELECT_RANGE:
+                long lowerBoundIdx = partition.ckIdxGen.generate(rng);
+                long lowerBoundCd = partition.ckGen().descriptorAt(lowerBoundIdx);
+
+                int nonEqFrom = rng.nextInt(partition.ckColumnCount());
+                boolean includeBound = rng.nextBoolean();
+
+                Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[partition.ckColumnCount()];
+                for (int i = 0; i < Math.min(nonEqFrom + 1, partition.ckColumnCount()); i++)
+                {
+                    if (i < nonEqFrom)
+                        lowerBoundRelations[i] = Relations.RelationKind.EQ;
+                    else
+                        lowerBoundRelations[i] = includeBound ? Relations.RelationKind.GTE : Relations.RelationKind.GT;
+                }
+
+                return new Operations.SelectRange(lts, partition.pd,
+                                                  lowerBoundCd, MagicConstants.UNSET_DESCR,
+                                                  lowerBoundRelations, null);
+            case SELECT_CUSTOM:
+            default:
+                throw new UnsupportedOperationException(kind + " is not supported");
+        }
+    }
+
+    // TODO (required) The percent of a given rows columns to populate
+    public enum VisitType
+    {
+        MUTATE,
+        VALIDATE
+        // TODO: custom actions: repair, expand/shrink cluster
+        // TODO: TCM actions, such as ALTER TABLE, etc, too?
+    }
+
+    public static class RandomOpKindGenFactory implements OpKindGenFactory
+    {
+        public static Kind[] VALIDATE_KINDS = new Kind[] { Kind.SELECT_PARTITION, Kind.SELECT_ROW, Kind.SELECT_RANGE };
+        public static Kind[] MUTATE_KINDS = new Kind[] { Kind.INSERT, Kind.UPDATE };
+
+        private final Generator<Kind> mutateGen = Generators.pick(MUTATE_KINDS);
+        private final Generator<Kind> validateGen = Generators.pick(VALIDATE_KINDS);
+
+        @Override
+        public Generator<Kind> forType(VisitGenerator.VisitType type)
+        {
+            switch (type)
+            {
+                case MUTATE: return mutateGen;
+                case VALIDATE: return validateGen;
+                default: throw new AssertionError();
+            }
+        }
+    }
+
+    public static class WeightedOpKindGenFactory implements OpKindGenFactory
+    {
+        private final Generator<Kind> mutateGen;
+        private final Generator<Kind> validateGen;
+
+        public WeightedOpKindGenFactory(Map<Kind, Integer> mutateWeights, Map<Kind, Integer> validateWeights)
+        {
+            this.mutateGen = Generators.weighted(mutateWeights);
+            this.validateGen = Generators.weighted(validateWeights);
+        }
+
+        @Override
+        public Generator<Kind> forType(VisitType type)
+        {
+            switch (type)
+            {
+                case MUTATE: return mutateGen;
+                case VALIDATE: return validateGen;
+                default: throw new AssertionError();
+            }
+        }
+
+        public static Builder builder()
+        {
+            return new Builder();
+        }
+
+        public static class Builder
+        {
+            private final Map<Kind, Integer> mutateWeights = new LinkedHashMap<>();
+            private final Map<Kind, Integer> validateWeights = new LinkedHashMap<>();
+
+            public Builder()
+            {
+                mutateWeights.put(Kind.INSERT, 1);
+                mutateWeights.put(Kind.UPDATE, 1);
+                validateWeights.put(Kind.SELECT_PARTITION, 1);
+                validateWeights.put(Kind.SELECT_ROW, 1);
+                validateWeights.put(Kind.SELECT_RANGE, 1);
+            }
+
+            public Builder mutate(Kind kind, int weight)
+            {
+                mutateWeights.put(kind, weight);
+                return this;
+            }
+
+            public Builder validate(Kind kind, int weight)
+            {
+                validateWeights.put(kind, weight);
+                return this;
+            }
+
+            public WeightedOpKindGenFactory build()
+            {
+                return new WeightedOpKindGenFactory(mutateWeights, validateWeights);
+            }
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/config/StressSchemaConfig.java

@@ -0,0 +1,425 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.harry.stress.config;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cassandra.harry.gen.TypeAdapters;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.Constructor;
+
+import org.apache.cassandra.config.YamlConfigurationLoader;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.gen.SchemaGenerators;
+import org.apache.cassandra.harry.gen.EntropySource;
+import org.apache.cassandra.harry.gen.rng.JdkRandomEntropySource;
+import org.apache.cassandra.harry.stress.RotationStrategy;
+import org.apache.cassandra.harry.stress.VisitGenerator;
+import org.apache.cassandra.harry.stress.distribution.Distribution;
+import org.apache.cassandra.harry.stress.distribution.Distributions;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized;
+
+public class StressSchemaConfig
+{
+    private final SchemaSpec schema;
+    private final Distribution rowPopulation;
+    private final Map<String, Distribution> columnPopulations;
+    private final Map<String, Distribution> valueSizeDistribution;
+    private final RotationConfig rotation;
+
+    private StressSchemaConfig(SchemaSpec schema, Distribution rowPopulation, Map<String, Distribution> columnPopulations, Map<String, Distribution> valueSizeDistribution, RotationConfig rotation)
+    {
+        this.schema = schema;
+        this.rowPopulation = rowPopulation;
+        this.columnPopulations = columnPopulations;
+        this.valueSizeDistribution = valueSizeDistribution;
+        this.rotation = rotation;
+    }
+
+    public SchemaSpec schema()
+    {
+        return schema;
+    }
+
+    public Distribution rowPopulation()
+    {
+        return rowPopulation;
+    }
+
+    public VisitGenerator.ColumnPopulation columnPopulation()
+    {
+        return column -> columnPopulations.getOrDefault(column, Distributions.fixed(100));
+    }
+
+    /**
+     * A fresh {@link RotationStrategy} built from the {@code rotation:} section of the config. Returns a new instance
+     * on every call because rotation strategies are stateful, so a single config can hand identical-but-independent
+     * strategies to, e.g., offline generation and a validation replay.
+     */
+    public RotationStrategy rotationStrategy()
+    {
+        return rotation.build();
+    }
+
+    public static StressSchemaConfig load(Path yamlFile) throws IOException
+    {
+        try (InputStream is = Files.newInputStream(yamlFile))
+        {
+            return load(is);
+        }
+    }
+
+    public static StressSchemaConfig load(URI uri) throws IOException
+    {
+        try (InputStream is = uri.toURL().openStream())
+        {
+            return load(is);
+        }
+    }
+
+    public static StressSchemaConfig load(InputStream is)
+    {
+        Constructor constructor = new Constructor(StressSchemaYaml.class, YamlConfigurationLoader.getDefaultLoaderOptions());
+        Yaml yaml = new Yaml(constructor);
+        StressSchemaYaml parsed = yaml.loadAs(is, StressSchemaYaml.class);
+        return fromYaml(parsed);
+    }
+
+    static StressSchemaConfig fromYaml(StressSchemaYaml yaml)
+    {
+        if (yaml.keyspace == null)
+            throw new IllegalArgumentException("keyspace is required");
+        if (yaml.table == null)
+            throw new IllegalArgumentException("table is required");
+
+        Map<String, ColumnConfig> populationSpecs = parseColumnSpecs(yaml.columnspec);
+
+        Map<String, Distribution> populations = new HashMap<>();
+        Map<String, Distribution> valueSizeDistributions = new HashMap<>();
+        for (Map.Entry<String, ColumnConfig> e : populationSpecs.entrySet())
+        {
+            populations.put(e.getKey(), parseDistribution(e.getValue().population, Distributions.uniformRandom(1, 10)));
+            valueSizeDistributions.put(e.getKey(), parseDistribution(e.getValue().size, null));
+        }
+        Distribution rowPopulation = parseDistribution(yaml.rows, Distributions.uniformRandom(1, 10));
+
+        SchemaSpec schema;
+        if (yaml.table_definition != null)
+            schema = parseTableDefinition(yaml.keyspace, yaml.table, yaml.table_definition, valueSizeDistributions);
+        else
+            schema = generateRandomSchema(yaml.keyspace, yaml.table, yaml.seed);
+        return new StressSchemaConfig(schema, rowPopulation, populations, valueSizeDistributions, RotationConfig.fromYaml(yaml.rotation));
+    }
+
+    private static Map<String, ColumnConfig> parseColumnSpecs(List<Map<String, Object>> columnspec)
+    {
+        Map<String, ColumnConfig> columnConfigs = new HashMap<>();
+        if (columnspec != null)
+        {
+            for (Map<String, Object> spec : columnspec)
+            {
+                Map<String, Object> entry = new HashMap<>(spec);
+                lowerCaseKeys(entry);
+                String name = (String) entry.get("name");
+                if (name == null)
+                    throw new IllegalArgumentException("Missing 'name' in columnspec entry");
+                String population = (String) entry.get("population");
+                String size = (String) entry.get("size");
+                if (population != null || size != null)
+                    columnConfigs.put(name, new ColumnConfig(size, population));
+            }
+        }
+        return columnConfigs;
+    }
+
+    private static class ColumnConfig
+    {
+        final String size;
+        final String population;
+
+        ColumnConfig(String size, String population)
+        {
+            this.size = size;
+            this.population = population;
+        }
+    }
+
+    private static SchemaSpec parseTableDefinition(String keyspace, String table, String tableCql, Map<String, Distribution> valueSizeDistribution)
+    {
+        TableMetadata metadata = CreateTableStatement.parse(tableCql, keyspace).build();
+
+        List<ColumnSpec<?>> pks = new ArrayList<>();
+        for (ColumnMetadata cm : metadata.partitionKeyColumns())
+        {
+            pks.add(ColumnSpec.pk(cm.name.toString(),
+                                  resolveServerType(cm.type),
+                                  TypeAdapters.forValues(cm.type, valueSizeDistribution.get(cm.name.toString()))));
+        }
+
+        List<ColumnSpec<?>> cks = new ArrayList<>();
+        for (ColumnMetadata cm : metadata.clusteringColumns())
+        {
+            cks.add(ColumnSpec.ck(cm.name.toString(),
+                                  resolveServerType(cm.type.unwrap()),
+                                  TypeAdapters.forValues(cm.type.unwrap(), valueSizeDistribution.get(cm.name.toString())),
+                                  cm.type.isReversed()));
+        }
+
+        List<ColumnSpec<?>> regulars = new ArrayList<>();
+        List<ColumnSpec<?>> statics = new ArrayList<>();
+        Iterator<ColumnMetadata> it = metadata.allColumnsInSelectOrder();
+        while (it.hasNext())
+        {
+            ColumnMetadata cm = it.next();
+            if (cm.isRegular())
+            {
+                regulars.add(ColumnSpec.regularColumn(cm.name.toString(),
+                                                      resolveServerType(cm.type),
+                                                      TypeAdapters.forValues(cm.type, valueSizeDistribution.get(cm.name.toString()))));
+            }
+            else if (cm.isStatic())
+            {
+                statics.add(ColumnSpec.staticColumn(cm.name.toString(),
+                                                    resolveServerType(cm.type),
+                                                    TypeAdapters.forValues(cm.type, valueSizeDistribution.get(cm.name.toString()))));
+            }
+        }
+
+        SchemaSpec.Options options = SchemaSpec.optionsBuilder()
+                                               .ifNotExists(true)
+                                               .addWriteTimestamps(true)
+                                               // carry the compaction strategy (e.g. LeveledCompactionStrategy) through from the CQL
+                                               .compactionStrategy(metadata.params.compaction.klass().getSimpleName());
+
+        return new SchemaSpec(keyspace, table, pks, cks, regulars, statics, options);
+    }
+
+    private static SchemaSpec generateRandomSchema(String keyspace, String table, Long seed)
+    {
+        long s = seed != null ? seed : System.nanoTime();
+        EntropySource rng = new JdkRandomEntropySource(s);
+        return SchemaGenerators.schemaSpecGen(keyspace, table, 100).generate(rng);
+    }
+
+    @SuppressWarnings("rawtypes")
+    static ColumnSpec.DataType resolveServerType(AbstractType<?> serverType)
+    {
+        if (serverType instanceof ReversedType)
+            serverType = ((ReversedType<?>) serverType).baseType;
+
+        if (serverType instanceof AsciiType)       return ColumnSpec.asciiType;
+        if (serverType instanceof UTF8Type)        return ColumnSpec.textType;
+        if (serverType instanceof LongType)        return ColumnSpec.int64Type;
+        if (serverType instanceof Int32Type)       return ColumnSpec.int32Type;
+        if (serverType instanceof ShortType)       return ColumnSpec.int16Type;
+        if (serverType instanceof ByteType)        return ColumnSpec.int8Type;
+        if (serverType instanceof BooleanType)     return ColumnSpec.booleanType;
+        if (serverType instanceof FloatType)       return ColumnSpec.floatType;
+        if (serverType instanceof DoubleType)      return ColumnSpec.doubleType;
+        if (serverType instanceof BytesType)       return ColumnSpec.blobType;
+        if (serverType instanceof UUIDType)        return ColumnSpec.uuidType;
+        if (serverType instanceof TimeUUIDType)    return ColumnSpec.timeUuidType;
+        if (serverType instanceof TimestampType)   return ColumnSpec.timestampType;
+        if (serverType instanceof IntegerType)     return ColumnSpec.varintType;
+        if (serverType instanceof DecimalType)     return ColumnSpec.decimalType;
+        if (serverType instanceof InetAddressType) return ColumnSpec.inetType;
+        if (serverType instanceof TimeType)        return ColumnSpec.timeType;
+
+        throw new IllegalArgumentException("Unsupported column type: " + serverType.asCQL3Type());
+    }
+
+    public static Distribution parseDistribution(String spec, Distribution onNull)
+    {
+        if (spec == null) return onNull;
+        spec = spec.trim();
+        String lower = toLowerCaseLocalized(spec);
+
+        int parenOpen = lower.indexOf('(');
+        if (parenOpen < 0 || !lower.endsWith(")"))
+            throw new IllegalArgumentException("Invalid distribution spec: " + spec);
+
+        String name = lower.substring(0, parenOpen);
+        String args = spec.substring(parenOpen + 1, spec.length() - 1).trim();
+
+        switch (name)
+        {
+            case "fixed":
+                return Distributions.fixed(parseLong(args));
+            case "uniform":
+            {
+                long[] range = parseRange(args);
+                return Distributions.uniformRandom(range[0], range[1]);
+            }
+            case "cdf":
+            {
+                String[] vs = args.split(",");
+                float[] chances = new float[vs.length - 1];
+                long[] bounds = new long[vs.length];
+                for (int i = 0 ; i < vs.length ; ++i)
+                {
+                    String[] v = vs[i].split(":");
+                    if (i == 0 && chances[i] != 0f)
+                        throw new IllegalArgumentException("Must specify a mapping for 0; first is " + vs[i]);
+
+                    if (i < vs.length - 1) chances[i] = Float.parseFloat(v[0]);
+                    else if (Float.parseFloat(v[0]) != 1f)
+                        throw new IllegalArgumentException("Must specify a mapping for 1; last is " + vs[i]);
+
+
+                    bounds[i] = Long.parseLong(v[1]);
+                }
+                return new Distributions.CDF(chances, bounds);
+            }
+            case "gaussian":
+            case "gauss":
+            case "normal":
+            case "norm":
+            {
+                String[] parts = args.split(",");
+                long[] range = parseRange(parts[0].trim());
+                // TODO: use a proper gaussian distribution once available
+                return Distributions.fixed(range[0] + (range[1] - range[0]) / 2);
+            }
+            default:
+                throw new IllegalArgumentException("Unsupported distribution type: " + name +
+                                                   ". Supported: fixed, uniform, gaussian");
+        }
+    }
+
+    static long[] parseRange(String rangeSpec)
+    {
+        String[] bounds = rangeSpec.split("\\.\\.+");
+        if (bounds.length != 2)
+            throw new IllegalArgumentException("Expected range in form min..max, got: " + rangeSpec);
+        return new long[]{ parseLong(bounds[0].trim()), parseLong(bounds[1].trim()) };
+    }
+
+    static long parseLong(String value)
+    {
+        value = toLowerCaseLocalized(value.trim());
+        long multiplier = 1;
+        if (value.endsWith("b"))
+        {
+            multiplier = 1_000_000_000L;
+            value = value.substring(0, value.length() - 1);
+        }
+        else if (value.endsWith("m"))
+        {
+            multiplier = 1_000_000L;
+            value = value.substring(0, value.length() - 1);
+        }
+        else if (value.endsWith("k"))
+        {
+            multiplier = 1_000L;
+            value = value.substring(0, value.length() - 1);
+        }
+        return Long.parseLong(value) * multiplier;
+    }
+
+    private static void lowerCaseKeys(Map<String, Object> map)
+    {
+        List<String> keys = new ArrayList<>(map.keySet());
+        for (String key : keys)
+        {
+            String lower = toLowerCaseLocalized(key);
+            if (!lower.equals(key))
+            {
+                Object val = map.remove(key);
+                map.put(lower, val);
+            }
+        }
+    }
+
+    /** Parsed, default-applied rotation parameters; {@link #build()} yields a fresh (stateful) strategy per call. */
+    static final class RotationConfig
+    {
+        final String strategy;
+        final int target;
+        final int replaceWithNew;
+        final int replaceWithVisited;
+        final int switchInterval;
+
+        RotationConfig(String strategy, int target, int replaceWithNew, int replaceWithVisited, int switchInterval)
+        {
+            this.strategy = strategy;
+            this.target = target;
+            this.replaceWithNew = replaceWithNew;
+            this.replaceWithVisited = replaceWithVisited;
+            this.switchInterval = switchInterval;
+        }
+
+        static RotationConfig fromYaml(RotationYaml yaml)
+        {
+            if (yaml == null)
+                return new RotationConfig("fixed", 2000, 0, 0, 500);
+            return new RotationConfig(yaml.strategy != null ? toLowerCaseLocalized(yaml.strategy.trim()) : "fixed",
+                                      yaml.target != null ? yaml.target : 2000,
+                                      yaml.replace_with_new != null ? yaml.replace_with_new : 0,
+                                      yaml.replace_with_visited != null ? yaml.replace_with_visited : 0,
+                                      yaml.partition_switch_interval != null ? yaml.partition_switch_interval : 500);
+        }
+
+        RotationStrategy build()
+        {
+            switch (strategy)
+            {
+                case "fixed":
+                    return new RotationStrategy.FixedRotationStrategy(target, replaceWithNew, replaceWithVisited, switchInterval);
+                case "random":
+                    return new RotationStrategy.RandomRotationStrategy(target, switchInterval);
+                default:
+                    throw new IllegalArgumentException("Unknown rotation strategy: " + strategy + ". Expected 'fixed' or 'random'.");
+            }
+        }
+    }
+
+    public static class StressSchemaYaml
+    {
+        public String keyspace;
+        public String table;
+        public String table_definition;
+        public Long seed;
+        public List<Map<String, Object>> columnspec;
+        public String rows;
+        public RotationYaml rotation;
+    }
+
+    public static class RotationYaml
+    {
+        public String strategy;
+        public Integer target;
+        public Integer replace_with_new;
+        public Integer replace_with_visited;
+        public Integer partition_switch_interval;
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/distribution/Distribution.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress.distribution;
+
+public interface Distribution
+{
+    long next(long seed);
+    // TODO: nextInt
+    double nextDouble(long seed);
+
+    long min();
+    long max();
+}

test/harry/main/org/apache/cassandra/harry/stress/distribution/DistributionFactory.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress.distribution;
+
+
+import java.io.Serializable;
+
+import org.apache.cassandra.harry.gen.EntropySource;
+
+public interface DistributionFactory extends Serializable
+{
+    Distribution get(EntropySource rng);
+    String getConfigAsString();
+}

test/harry/main/org/apache/cassandra/harry/stress/distribution/Distributions.java

@@ -0,0 +1,782 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress.distribution;
+
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.math3.distribution.AbstractRealDistribution;
+import org.apache.commons.math3.distribution.ExponentialDistribution;
+import org.apache.commons.math3.distribution.NormalDistribution;
+import org.apache.commons.math3.distribution.UniformRealDistribution;
+import org.apache.commons.math3.distribution.WeibullDistribution;
+import org.apache.commons.math3.random.RandomGenerator;
+
+import org.apache.cassandra.harry.gen.EntropySource;
+import org.apache.cassandra.harry.gen.rng.SeedableEntropySource;
+
+public class Distributions
+{
+    public static Distribution fixed(long v)
+    {
+        return new Fixed(v);
+    }
+
+    public static Distribution exp(EntropySource rng, long min, long max, double mean)
+    {
+        return new Exp(rng, min, max, mean);
+    }
+
+    public static Distribution extreme(EntropySource rng, long min, long max, double shape, double scale)
+    {
+        return new Extreme(rng, min, max, shape, scale);
+    }
+
+    public static Distribution offset(Distribution distribution, long offset)
+    {
+        return new OffsetDistribution(distribution, offset);
+    }
+
+    public static Distribution quantizedExtreme(EntropySource rng, long min, long max, double shape, double scale, int quantas)
+    {
+        return new Quantized(new Extreme(rng, min, max, shape, scale), quantas);
+    }
+
+    public static Distribution gaussian(EntropySource rng, long min, long max, double mean, double stdev)
+    {
+        return new Gaussian(rng, min, max, mean, stdev);
+    }
+
+    public static Distribution binomial(EntropySource rng, int n, double p)
+    {
+        return new Binomial(n, p);
+    }
+
+    public static Distribution zipf(EntropySource rng, int total, double skew)
+    {
+        return new Zipf(total, skew);
+    }
+
+    public static Distribution biased(EntropySource rng, long min, long max, double bias)
+    {
+        return new Biased(min, max, bias);
+    }
+
+    public static Distribution uniform(EntropySource rng, long min, long max)
+    {
+        return new Uniform(rng, min, max);
+    }
+
+    public static Distribution uniformRandom(long min, long max)
+    {
+        return new UniformRandom(min, max);
+    }
+
+    public static Distribution sequence(long start, long end)
+    {
+        return new Sequence(start, end);
+    }
+
+    public static Distribution invert(Distribution distribution)
+    {
+        if (distribution instanceof Inverted)
+            return ((Inverted) distribution).wrapped;
+        return new Inverted(distribution);
+    }
+
+    public static class Fixed implements Distribution
+    {
+        private final long value;
+
+        public Fixed(long value)
+        {
+            this.value = value;
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return value;
+        }
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            return (double) value;
+        }
+
+        @Override
+        public long min()
+        {
+            return value;
+        }
+
+        @Override
+        public long max()
+        {
+            return value;
+        }
+    }
+
+    public static class Exp extends ApacheAdaptor
+    {
+        private final long min;
+        private final long max;
+
+        public Exp(EntropySource rng, long min, long max, double mean)
+        {
+            super(new ExponentialDistribution(new EntropySourceAdapter(rng), mean, ExponentialDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY), min, max);
+            this.min = min;
+            this.max = max;
+        }
+
+        @Override
+        public long min()
+        {
+            return min;
+        }
+
+        @Override
+        public long max()
+        {
+            return max;
+        }
+    }
+
+    public static class Extreme extends ApacheAdaptor
+    {
+        private final long min;
+        private final long max;
+
+        public Extreme(EntropySource rng, long min, long max, double shape, double scale)
+        {
+            super(new WeibullDistribution(new EntropySourceAdapter(rng), shape, scale, WeibullDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY), min, max);
+            this.min = min;
+            this.max = max;
+        }
+
+        @Override
+        public long min()
+        {
+            return min;
+        }
+
+        @Override
+        public long max()
+        {
+            return max;
+        }
+    }
+
+    public static class OffsetDistribution implements Distribution
+    {
+        private final Distribution base;
+        private final long offset;
+
+        public OffsetDistribution(Distribution base, long offset)
+        {
+            this.base = base;
+            this.offset = offset;
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return base.next(seed) + offset;
+        }
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            return base.nextDouble(seed) + offset;
+        }
+
+        @Override
+        public long min()
+        {
+            return base.min() + offset;
+        }
+
+        @Override
+        public long max()
+        {
+            return base.max() + offset;
+        }
+    }
+
+    public static class Quantized implements Distribution
+    {
+        final Distribution delegate;
+        final long[] bounds;
+
+        public Quantized(ApacheAdaptor delegate, int quantas)
+        {
+            this.delegate = delegate;
+            this.bounds = new long[quantas + 1];
+            bounds[0] = delegate.min;
+            bounds[quantas] = delegate.max + 1;
+            for (int i = 1; i < quantas; i++)
+                bounds[i] = delegate.inverseCumProb(i / (double) quantas);
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return SeedableEntropySource.computeWithSeed(seed, (rng) -> {
+                int quanta = quanta(delegate.next(seed));
+                return bounds[quanta] + (long) (rng.nextDouble() * ((bounds[quanta + 1] - bounds[quanta])));
+            });
+        }
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public long min()
+        {
+            return delegate.min();
+        }
+
+        @Override
+        public long max()
+        {
+            return delegate.max();
+        }
+
+        int quanta(long val)
+        {
+            int i = Arrays.binarySearch(bounds, val);
+            if (i < 0)
+                return -2 - i;
+            return i - 1;
+        }
+    }
+
+    public static class Gaussian extends ApacheAdaptor
+    {
+        public Gaussian(EntropySource rng, long min, long max, double mean, double stdev)
+        {
+            super(new NormalDistribution(new EntropySourceAdapter(rng), mean, stdev, NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY), min, max);
+        }
+    }
+
+    // Models the number of successes in n independent trials
+    public static class Binomial implements Distribution
+    {
+        private final int n;
+        private final double p;
+        private final long min;
+        private final long max;
+
+        public Binomial(int n, double p)
+        {
+            // number of trials
+            this.n = n;
+            // probability of success
+            this.p = p;
+            this.min = 0;
+            // Max would be when all trials succeed
+            this.max = n;
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return SeedableEntropySource.computeWithSeed(seed, rng -> {
+                long successes = 0;
+                for (int i = 0; i < n; i++)
+                {
+                    if (rng.nextDouble() < p)
+                    {
+                        successes++;
+                    }
+                }
+                return successes;
+            });
+        }
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            return (double) next(seed) / n;
+        }
+
+        @Override
+        public long min()
+        {
+            return min;
+        }
+
+        @Override
+        public long max()
+        {
+            return max;
+        }
+    }
+
+    public static class Zipf implements Distribution
+    {
+        private final int n;
+        private final double[] probabilities;
+        private final long min;
+        private final long max;
+
+        public Zipf(int total, double skew)
+        {
+            if (total <= 0) throw new IllegalArgumentException("n must be positive");
+            if (skew <= 0) throw new IllegalArgumentException("s must be positive");
+
+            this.n = total;
+            this.min = 1;
+            this.max = total;
+
+            this.probabilities = new double[total];
+            double sum = 0;
+            for (int i = 0; i < total; i++)
+            {
+                probabilities[i] = 1.0 / Math.pow(i + 1, skew);
+                sum += probabilities[i];
+            }
+
+            // normalize
+            for (int i = 0; i < total; i++)
+            {
+                probabilities[i] /= sum;
+                if (i > 0)
+                    probabilities[i] += probabilities[i - 1];
+            }
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return SeedableEntropySource.computeWithSeed(seed, rng -> {
+                double rand = rng.nextDouble();
+                long index = Arrays.binarySearch(probabilities, rand);
+                if (index < 0)
+                    index = -(index + 1);
+                return index + 1;
+            });
+        }
+
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            return (double) next(seed) / n;
+        }
+
+        @Override
+        public long min()
+        {
+            return min;
+        }
+
+        @Override
+        public long max()
+        {
+            return max;
+        }
+    }
+
+    public static class Biased implements Distribution
+    {
+        private final long min;
+        private final long max;
+        private final double bias;
+
+        public Biased(long min, long max, double bias)
+        {
+            this.min = min;
+            this.max = max;
+            this.bias = bias;
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return SeedableEntropySource.computeWithSeed(seed, rng -> {
+                // TODO (desired): try and compare with a variant without exponentiation
+                // f(x)=ax^p+b
+                double v = rng.nextDouble();
+                v = Math.pow(v, bias);
+                return min + (long) (v * (max - min));
+            });
+        }
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            return SeedableEntropySource.computeWithSeed(seed, rng -> {
+                double raw = rng.nextDouble();
+                return Math.pow(raw, bias);
+            });
+        }
+
+        @Override
+        public long min()
+        {
+            return min;
+        }
+
+        @Override
+        public long max()
+        {
+            return max;
+        }
+    }
+
+    public static class Uniform extends ApacheAdaptor
+    {
+        public Uniform(EntropySource rng, long min, long max)
+        {
+            super(new UniformRealDistribution(new EntropySourceAdapter(rng), min, max + 1), min, max);
+        }
+    }
+
+    /**
+     * A lightweight uniform random distribution that derives values purely from the seed,
+     * without requiring a stateful EntropySource or Apache Commons Math.
+     */
+    public static class UniformRandom implements Distribution
+    {
+        private final long min;
+        private final long max;
+        private final long range;
+
+        public UniformRandom(long min, long max)
+        {
+            if (min > max)
+                throw new IllegalArgumentException("min (" + min + ") must be <= max (" + max + ")");
+            this.min = min;
+            this.max = max;
+            this.range = max - min + 1;
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return SeedableEntropySource.computeWithSeed(seed, rng -> min + Math.abs(rng.next() % range));
+        }
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            return SeedableEntropySource.computeWithSeed(seed, rng -> min + rng.nextDouble() * (max - min));
+        }
+
+        @Override
+        public long min()
+        {
+            return min;
+        }
+
+        @Override
+        public long max()
+        {
+            return max;
+        }
+    }
+
+    public static class CDF implements Distribution
+    {
+        private final float[] chances;
+        private final long[] bounds;
+
+        public CDF(float[] chances, long[] bounds)
+        {
+            this.chances = chances;
+            this.bounds = bounds;
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return SeedableEntropySource.computeWithSeed(seed, rng -> {
+                float f = rng.nextFloat();
+                int i = Arrays.binarySearch(chances, f);
+                if (i < 0) i = -1 - i;
+                if (i == 0)
+                    return bounds[0];
+
+                return rng.nextLong(bounds[i - 1], bounds[i] + 1);
+            });
+        }
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            return next(seed);
+        }
+
+        @Override
+        public long min()
+        {
+            return bounds[0];
+        }
+
+        @Override
+        public long max()
+        {
+            return bounds[bounds.length - 1];
+        }
+    }
+
+    public static abstract class ApacheAdaptor implements Distribution
+    {
+        final AbstractRealDistribution delegate;
+        final long min, max, delta;
+
+        public ApacheAdaptor(AbstractRealDistribution delegate, long min, long max)
+        {
+            this.delegate = delegate;
+            this.min = min;
+            this.max = max;
+            this.delta = max - min;
+        }
+
+        public void setSeed(long seed)
+        {
+            delegate.reseedRandomGenerator(seed);
+        }
+
+        @Override
+        public synchronized long next(long seed)
+        {
+            delegate.reseedRandomGenerator(seed);
+            return offset(min, delta, delegate.sample());
+        }
+
+        @Override
+        public synchronized double nextDouble(long seed)
+        {
+            delegate.reseedRandomGenerator(seed);
+            return offsetDouble(min, delta, delegate.sample());
+        }
+
+        public long inverseCumProb(double cumProb)
+        {
+            return offset(min, delta, delegate.inverseCumulativeProbability(cumProb));
+        }
+
+        private long offset(long min, long delta, double val)
+        {
+            long r = (long) val;
+            if (r < 0)
+                r = 0;
+            if (r > delta)
+                r = delta;
+            return min + r;
+        }
+
+        private double offsetDouble(long min, long delta, double r)
+        {
+            if (r < 0)
+                r = 0;
+            if (r > delta)
+                r = delta;
+            return min + r;
+        }
+
+        @Override
+        public long min()
+        {
+            return min;
+        }
+
+        @Override
+        public long max()
+        {
+            return max;
+        }
+    }
+
+    public static class Sequence implements Distribution
+    {
+        private final long start;
+        private final long totalCount;
+        private final AtomicLong next = new AtomicLong();
+
+        public Sequence(long start, long end)
+        {
+            if (start > end)
+                throw new IllegalStateException();
+            this.start = start;
+            this.totalCount = 1 + end - start;
+        }
+
+        private long nextWithWrap()
+        {
+            long next = this.next.getAndIncrement();
+            return start + (next % totalCount);
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return nextWithWrap();
+        }
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            return nextWithWrap();
+        }
+
+        @Override
+        public long min()
+        {
+            return start;
+        }
+
+        @Override
+        public long max()
+        {
+            return start + totalCount;
+        }
+    }
+
+    public static class Inverted implements Distribution
+    {
+        final Distribution wrapped;
+        final long min;
+        final long max;
+
+        public Inverted(Distribution wrapped)
+        {
+            this.wrapped = wrapped;
+            this.min = wrapped.min();
+            this.max = wrapped.max();
+        }
+
+        @Override
+        public long next(long seed)
+        {
+            return max - (wrapped.next(seed) - min);
+        }
+
+        @Override
+        public double nextDouble(long seed)
+        {
+            return max - (wrapped.nextDouble(seed) - min);
+        }
+
+        @Override
+        public long min()
+        {
+            return min;
+        }
+
+        @Override
+        public long max()
+        {
+            return max;
+        }
+    }
+
+    public static class EntropySourceAdapter implements RandomGenerator
+    {
+        private final EntropySource entropySource;
+
+        public EntropySourceAdapter(EntropySource entropySource)
+        {
+            this.entropySource = entropySource;
+        }
+
+        @Override
+        public void setSeed(int seed)
+        {
+            entropySource.seed(seed);
+        }
+
+        @Override
+        public void setSeed(int[] seed)
+        {
+            // Convert the int array to a long seed
+            long longSeed = 0;
+            for (int s : seed)
+            {
+                longSeed = longSeed * 31 + s;
+            }
+            entropySource.seed(longSeed);
+        }
+
+        @Override
+        public void setSeed(long seed)
+        {
+            entropySource.seed(seed);
+        }
+
+        @Override
+        public void nextBytes(byte[] bytes)
+        {
+            for (int i = 0; i < bytes.length; i++)
+            {
+                bytes[i] = (byte) entropySource.nextInt(256);
+            }
+        }
+
+        @Override
+        public int nextInt()
+        {
+            return entropySource.nextInt();
+        }
+
+        @Override
+        public int nextInt(int n)
+        {
+            return entropySource.nextInt(n);
+        }
+
+        @Override
+        public long nextLong()
+        {
+            return entropySource.next();
+        }
+
+        @Override
+        public boolean nextBoolean()
+        {
+            return entropySource.nextBoolean();
+        }
+
+        @Override
+        public float nextFloat()
+        {
+            return entropySource.nextFloat();
+        }
+
+        @Override
+        public double nextDouble()
+        {
+            return entropySource.nextDouble();
+        }
+
+        @Override
+        public double nextGaussian()
+        {
+            double u = nextDouble();
+            double v = nextDouble();
+            return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
+        }
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/distribution/RatioDistribution.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress.distribution;
+
+public class RatioDistribution
+{
+
+    final Distribution distribution;
+    final double divisor;
+
+    public RatioDistribution(Distribution distribution, double divisor)
+    {
+        this.distribution = distribution;
+        this.divisor = divisor;
+    }
+
+    // yields a value between 0 and 1
+    public double next()
+    {
+        throw new IllegalStateException();
+//        return Math.max(0f, Math.min(1f, distribution.nextDouble() / divisor));
+    }
+
+    public double min()
+    {
+        return Math.min(1d, distribution.min() / divisor);
+    }
+
+    public double max()
+    {
+        return Math.min(1d, distribution.max() / divisor);
+    }
+}

test/harry/main/org/apache/cassandra/harry/stress/distribution/RatioDistributionFactory.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress.distribution;
+
+
+import java.io.Serializable;
+
+import org.apache.cassandra.harry.gen.EntropySource;
+
+public interface RatioDistributionFactory extends Serializable
+{
+
+    RatioDistribution get(EntropySource rng);
+    String getConfigAsString();
+
+}

test/harry/main/org/apache/cassandra/harry/stress/test/HarryCcmStressTest.java

@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.stress.test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.QueryOptions;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.checker.TestHelper;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.gen.Generators;
+import org.apache.cassandra.harry.stress.ExternalClusterSut;
+import org.apache.cassandra.harry.stress.HarryStress;
+import org.apache.cassandra.harry.stress.RotationStrategy;
+import org.apache.cassandra.harry.stress.VisitGenerator;
+import org.apache.cassandra.harry.stress.distribution.Distributions;
+import org.apache.cassandra.service.consensus.TransactionalMode;
+
+public class HarryCcmStressTest
+{
+    public static final Logger LOGGER = LoggerFactory.getLogger(HarryStressTest.class);
+
+    @Test
+    public void stressTest() throws Throwable
+    {
+        main();
+    }
+
+    public static Generator<SchemaSpec>  trivialSchema(String ks, String table, SchemaSpec.Options options)
+    {
+        return (rng) -> {
+            List<ColumnSpec<?>> pks = new ArrayList<>();
+            for (int i = 0; i < 2; i++)
+                pks.add(ColumnSpec.pk("pk" + i, ColumnSpec.asciiType, Generators.ascii(10, 20)));
+
+            List<ColumnSpec<?>> cks = new ArrayList<>();
+            for (int i = 0; i < 2; i++)
+                cks.add(ColumnSpec.ck("ck" + i, ColumnSpec.asciiType, Generators.ascii(10, 20), false));
+
+            List<ColumnSpec<?>> regularColumns = new ArrayList<>();
+            for (int i = 0; i < 2; i++)
+                regularColumns.add(ColumnSpec.regularColumn("regular" + i, ColumnSpec.asciiType, Generators.ascii(10, 20)));
+
+            List<ColumnSpec<?>> staticColumns = new ArrayList<>();
+            for (int i = 0; i < 2; i++)
+                staticColumns.add(ColumnSpec.staticColumn("static" + i, ColumnSpec.asciiType, Generators.ascii(10, 200)));
+
+            return new SchemaSpec(ks, table,
+                                  pks, cks, regularColumns, staticColumns,
+                                  options);
+        };
+    }
+
+    public static void main(String... args)
+    {
+        TestHelper.withRandom(1, rng -> {
+            SchemaSpec schema = trivialSchema("ks" + System.currentTimeMillis(), "tbl",
+                                              SchemaSpec.optionsBuilder()
+                                                        .withTransactionalMode(TransactionalMode.full)
+                                                        .addWriteTimestamps(false)
+                                                        .build())
+                                .generate(rng);
+            // TODO: move
+            {
+                Session sut = new Cluster.Builder()
+                              .addContactPoint("127.0.0.1")
+                              .withPort(9042)
+                              .withQueryOptions(new QueryOptions().setConsistencyLevel(com.datastax.driver.core.ConsistencyLevel.QUORUM))
+                              .build()
+                              .connect();
+                sut.execute(String.format("CREATE KEYSPACE %s WITH replication = { 'class' : 'org.apache.cassandra.locator.NetworkTopologyStrategy', 'datacenter1': '1' };",
+                                          ////            cluster.schemaChange(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};",
+//                                               "ks"));
+
+                                          schema.keyspace));
+//                sut.execute(String.format("create keyspace %s with replication = {'class':'SimpleStrategy', 'replication_factor':1}",
+//                                          schema.keyspace));
+
+                sut.execute(schema.compile());
+            }
+
+            HarryStress visitGenerator = new HarryStress(schema,
+                                                         Distributions.fixed(1),
+                                                         column -> Distributions.fixed(100),
+                                                         Generators.pick(VisitGenerator.VisitType.values()),
+                                                         Distributions.fixed(1),
+                                                         new VisitGenerator.RandomOpKindGenFactory(),
+                                                         new RotationStrategy.RandomRotationStrategy(100),
+                                                         null, 30,
+                                                         () -> {
+                                                             Session sut = new Cluster.Builder()
+                                                                           .addContactPoint("127.0.0.1")
+                                                                           .withPort(9042)
+                                                                           .withQueryOptions(new QueryOptions().setConsistencyLevel(com.datastax.driver.core.ConsistencyLevel.QUORUM))
+                                                                           .build()
+                                                                           .connect();
+
+                                                             return (statement, run) -> {
+                                                                 while (true)
+                                                                 {
+                                                                     try
+                                                                     {
+                                                                         Object[][] rs = ExternalClusterSut.resultSetToObjectArray(sut.execute(statement.cql(),
+                                                                                                                                      statement.bindings()));
+                                                                         if (run != null)
+                                                                            run.run();
+                                                                         return rs;
+                                                                     }
+                                                                     catch (Throwable t)
+                                                                     {
+                                                                         t.printStackTrace();
+                                                                         LOGGER.error("Failed to execute statement, sleeping before retrying", t);
+                                                                         sleepUninterruptibly(100);
+                                                                     }
+                                                                 }
+                                                             };
+                                                         },
+                                                         2,
+                                                         10_000,
+                                                         0,
+                                                         Long.MAX_VALUE,
+                                                         0);
+
+            visitGenerator.start(Long.MAX_VALUE, Long.MAX_VALUE);
+        });
+    }
+
+    private static void sleepUninterruptibly(long millis)
+    {
+        try
+        {
+            Thread.sleep(millis);
+        }
+        catch (InterruptedException e)
+        {
+            return;
+        }
+    }
+}
+