Merge branch 'cassandra-2.1' into trunk

Conflicts:
	tools/stress/src/org/apache/cassandra/stress/Operation.java
	tools/stress/src/org/apache/cassandra/stress/operations/CqlIndexedRangeSlicer.java
	tools/stress/src/org/apache/cassandra/stress/operations/CqlRangeSlicer.java
	tools/stress/src/org/apache/cassandra/stress/operations/ThriftInserter.java
	tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterAdder.java
	tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterGetter.java
	tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlInserter.java
	tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlOperation.java
	tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlReader.java
	tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java

CHANGES.txt

@@ -19,6 +19,7 @@
  * Optimise NativeCell comparisons (CASSANDRA-6755)
 
 2.1.0-rc3
+ * Introduce CQL support for stress tool (CASSANDRA-6146)
  * Fix ClassCastException processing expired messages (CASSANDRA-7496)
  * Fix prepared marker for collections inside UDT (CASSANDRA-7472)
  * Remove left-over populate_io_cache_on_flush and replicate_on_write

src/java/org/apache/cassandra/thrift/ThriftConversion.java

@@ -51,6 +51,25 @@ public class ThriftConversion
         throw new AssertionError();
     }
 
+    public static ConsistencyLevel toThrift(org.apache.cassandra.db.ConsistencyLevel cl)
+    {
+        switch (cl)
+        {
+            case ANY: return ConsistencyLevel.ANY;
+            case ONE: return ConsistencyLevel.ONE;
+            case TWO: return ConsistencyLevel.TWO;
+            case THREE: return ConsistencyLevel.THREE;
+            case QUORUM: return ConsistencyLevel.QUORUM;
+            case ALL: return ConsistencyLevel.ALL;
+            case LOCAL_QUORUM: return ConsistencyLevel.LOCAL_QUORUM;
+            case EACH_QUORUM: return ConsistencyLevel.EACH_QUORUM;
+            case SERIAL: return ConsistencyLevel.SERIAL;
+            case LOCAL_SERIAL: return ConsistencyLevel.LOCAL_SERIAL;
+            case LOCAL_ONE: return ConsistencyLevel.LOCAL_ONE;
+        }
+        throw new AssertionError();
+    }
+
     // We never return, but returning a RuntimeException allows to write "throw rethrow(e)" without java complaining
     // for methods that have a return value.
     public static RuntimeException rethrow(RequestExecutionException e) throws UnavailableException, TimedOutException

src/java/org/apache/cassandra/utils/UUIDGen.java

@@ -25,6 +25,8 @@ import java.util.Collection;
 import java.util.Random;
 import java.util.UUID;
 
+import com.google.common.annotations.VisibleForTesting;
+
 
 /**
  * The goods are here: www.ietf.org/rfc/rfc4122.txt.
@@ -80,6 +82,12 @@ public class UUIDGen
         return new UUID(createTime(fromUnixTimestamp(when)), clockSeqAndNode);
     }
 
+    @VisibleForTesting
+    public static UUID getTimeUUID(long when, long clockSeqAndNode)
+    {
+        return new UUID(createTime(fromUnixTimestamp(when)), clockSeqAndNode);
+    }
+
     /** creates a type 1 uuid from raw bytes. */
     public static UUID getUUID(ByteBuffer raw)
     {

tools/bin/cassandra-stress

@@ -42,4 +42,4 @@ if [ "x$JAVA" = "x" ]; then
     exit 1
 fi
 
-$JAVA -server -cp $CLASSPATH org.apache.cassandra.stress.Stress $@
+$JAVA -server -ea -cp $CLASSPATH org.apache.cassandra.stress.Stress $@

tools/cqlstress-counter-example.yaml

@@ -0,0 +1,85 @@
+#
+# This is an example YAML profile for cassandra-stress
+#
+# insert data
+# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1)
+#
+# read, using query simple1:
+# cassandra-stress profile=/home/jake/stress1.yaml ops(simple1=1)
+#
+# mixed workload (90/10)
+# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1,simple1=9)
+
+
+#
+# Keyspace info
+#
+keyspace: stresscql
+
+#
+# The CQL for creating a keyspace (optional if it already exists)
+#
+keyspace_definition: |
+  CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
+
+#
+# Table info
+#
+table: counttest
+
+#
+# The CQL for creating a table you wish to stress (optional if it already exists)
+#
+table_definition: |
+  CREATE TABLE counttest (
+        name text PRIMARY KEY,
+        count counter
+  ) WITH comment='A table of many types to test wide rows'
+
+#
+# Optional meta information on the generated columns in the above table
+# The min and max only apply to text and blob types
+# The distribution field represents the total unique population
+# distribution of that column across rows.  Supported types are
+# 
+#      EXP(min..max)                        An exponential distribution over the range [min..max]
+#      EXTREME(min..max,shape)              An extreme value (Weibull) distribution over the range [min..max]
+#      GAUSSIAN(min..max,stdvrng)           A gaussian/normal distribution, where mean=(min+max)/2, and stdev is (mean-min)/stdvrng
+#      GAUSSIAN(min..max,mean,stdev)        A gaussian/normal distribution, with explicitly defined mean and stdev
+#      UNIFORM(min..max)                    A uniform distribution over the range [min, max]
+#      FIXED(val)                           A fixed distribution, always returning the same value
+#      Aliases: extr, gauss, normal, norm, weibull
+#
+#      If preceded by ~, the distribution is inverted
+#
+# Defaults for all columns are size: uniform(1..256), identity: uniform(1..1024)
+#
+
+columnspec:
+  - name: name
+    clustering: uniform(1..100)
+    size: uniform(1..4)
+  - name: count
+    population: fixed(1)
+
+insert:
+  partitions: fixed(1)            # number of unique partitions to update in a single operation
+                                  # if perbatch < 1, multiple batches will be used but all partitions will
+                                  # occur in all batches (unless already finished); only the row counts will vary
+  pervisit: fixed(1)/1            # ratio of rows each partition should update in a single visit to the partition,
+                                  # as a proportion of the total possible for the partition
+  perbatch: fixed(1)/1            # number of rows each partition should update in a single batch statement,
+                                  # as a proportion of the proportion we are inserting this visit
+                                  # (i.e. compounds with (and capped by) pervisit)
+  batchtype: UNLOGGED             # type of batch to use
+
+#
+# A list of queries you wish to run against the schema
+#
+queries:
+   simple1: select * from counttest where name = ?
+
+#
+# In order to generate data consistently we need something to generate a unique key for this schema profile.
+#
+seed: changing this string changes the generated data. its hashcode is used as the random seed.

tools/cqlstress-example.yaml

@@ -0,0 +1,99 @@
+#
+# This is an example YAML profile for cassandra-stress
+#
+# insert data
+# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1)
+#
+# read, using query simple1:
+# cassandra-stress profile=/home/jake/stress1.yaml ops(simple1=1)
+#
+# mixed workload (90/10)
+# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1,simple1=9)
+
+
+#
+# Keyspace info
+#
+keyspace: stresscql
+
+#
+# The CQL for creating a keyspace (optional if it already exists)
+#
+keyspace_definition: |
+  CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
+
+#
+# Table info
+#
+table: typestest
+
+#
+# The CQL for creating a table you wish to stress (optional if it already exists)
+#
+table_definition: |
+  CREATE TABLE typestest (
+        name text,
+        choice boolean,
+        date timestamp,
+        address inet,
+        dbl double,
+        lval bigint,
+        ival int,
+        uid timeuuid,
+        value blob,
+        PRIMARY KEY((name,choice), date, address, dbl, lval, ival, uid)
+  ) WITH COMPACT STORAGE 
+    AND compaction = { 'class':'LeveledCompactionStrategy' }
+    AND comment='A table of many types to test wide rows'
+
+#
+# Optional meta information on the generated columns in the above table
+# The min and max only apply to text and blob types
+# The distribution field represents the total unique population
+# distribution of that column across rows.  Supported types are
+# 
+#      EXP(min..max)                        An exponential distribution over the range [min..max]
+#      EXTREME(min..max,shape)              An extreme value (Weibull) distribution over the range [min..max]
+#      GAUSSIAN(min..max,stdvrng)           A gaussian/normal distribution, where mean=(min+max)/2, and stdev is (mean-min)/stdvrng
+#      GAUSSIAN(min..max,mean,stdev)        A gaussian/normal distribution, with explicitly defined mean and stdev
+#      UNIFORM(min..max)                    A uniform distribution over the range [min, max]
+#      FIXED(val)                           A fixed distribution, always returning the same value
+#      Aliases: extr, gauss, normal, norm, weibull
+#
+#      If preceded by ~, the distribution is inverted
+#
+# Defaults for all columns are size: uniform(1..256), identity: uniform(1..1024)
+#
+columnspec:
+  - name: name
+    size: uniform(1..10)
+    population: uniform(1..1M)     # the range of unique values to select for the field (default is 100Billion)
+  - name: choice
+  - name: date
+    cluster: uniform(1..4)
+  - name: lval
+    population: gaussian(1..1000)
+    cluster: uniform(1..4)
+
+insert:
+  partitions: uniform(1..50)      # number of unique partitions to update in a single operation
+                                  # if perbatch < 1, multiple batches will be used but all partitions will
+                                  # occur in all batches (unless already finished); only the row counts will vary
+  pervisit: uniform(1..10)/10     # ratio of rows each partition should update in a single visit to the partition,
+                                  # as a proportion of the total possible for the partition
+  perbatch: ~exp(1..3)/4          # number of rows each partition should update in a single batch statement,
+                                  # as a proportion of the proportion we are inserting this visit
+                                  # (i.e. compounds with (and capped by) pervisit)
+  batchtype: UNLOGGED             # type of batch to use
+
+#
+# A list of queries you wish to run against the schema
+#
+queries:
+   simple1: select * from typestest where name = ? and choice = ? LIMIT 100
+   range1: select * from typestest where name = ? and choice = ? and date >= ? LIMIT 100
+
+#
+# In order to generate data consistently we need something to generate a unique key for this schema profile.
+#
+seed: changing this string changes the generated data. its hashcode is used as the random seed.

tools/cqlstress-insanity-example.yaml

@@ -0,0 +1,102 @@
+#
+# This is an example YAML profile for cassandra-stress
+#
+# insert data
+# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1)
+#
+# read, using query simple1:
+# cassandra-stress profile=/home/jake/stress1.yaml ops(simple1=1)
+#
+# mixed workload (90/10)
+# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1,simple1=9)
+
+
+#
+# Keyspace info
+#
+keyspace: stresscql
+
+#
+# The CQL for creating a keyspace (optional if it already exists)
+#
+keyspace_definition: |
+  CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
+
+#
+# Table info
+#
+table: insanitytest
+
+#
+# The CQL for creating a table you wish to stress (optional if it already exists)
+#
+table_definition: |
+  CREATE TABLE insanitytest (
+        name text,
+        choice boolean,
+        date timestamp,
+        address inet,
+        dbl double,
+        lval bigint,
+        fval float,
+        ival int,
+        uid timeuuid,
+        dates list<timestamp>,
+        inets set<inet>,
+        value blob,
+        PRIMARY KEY((name, choice), date)
+  ) WITH compaction = { 'class':'LeveledCompactionStrategy' }
+    AND comment='A table of many types to test wide rows and collections'
+
+#
+# Optional meta information on the generated columns in the above table
+# The min and max only apply to text and blob types
+# The distribution field represents the total unique population
+# distribution of that column across rows.  Supported types are
+# 
+#      EXP(min..max)                        An exponential distribution over the range [min..max]
+#      EXTREME(min..max,shape)              An extreme value (Weibull) distribution over the range [min..max]
+#      GAUSSIAN(min..max,stdvrng)           A gaussian/normal distribution, where mean=(min+max)/2, and stdev is (mean-min)/stdvrng
+#      GAUSSIAN(min..max,mean,stdev)        A gaussian/normal distribution, with explicitly defined mean and stdev
+#      UNIFORM(min..max)                    A uniform distribution over the range [min, max]
+#      FIXED(val)                           A fixed distribution, always returning the same value
+#      Aliases: extr, gauss, normal, norm, weibull
+#
+#      If preceded by ~, the distribution is inverted
+#
+# Defaults for all columns are size: uniform(1..256), population: uniform(1..100B)
+#
+columnspec:
+  - name: name
+    clustering: uniform(1..4)
+  - name: date
+    clustering: gaussian(1..20)
+  - name: lval
+    population: fixed(1)
+  - name: dates
+    clustering: uniform(1..100)
+  - name: inets
+    clustering: uniform(1..200)
+  - name: value
+
+insert:
+  partitions: fixed(1)            # number of unique partitions to update in a single operation
+                                  # if perbatch < 1, multiple batches will be used but all partitions will
+                                  # occur in all batches (unless already finished); only the row counts will vary
+  pervisit: uniform(1..10)/100K   # ratio of rows each partition should update in a single visit to the partition,
+                                  # as a proportion of the total possible for the partition
+  perbatch: fixed(1)/1            # number of rows each partition should update in a single batch statement,
+                                  # as a proportion of the proportion we are inserting this visit
+                                  # (i.e. compounds with (and capped by) pervisit)
+  batchtype: UNLOGGED             # type of batch to use
+
+#
+# A list of queries you wish to run against the schema
+#
+queries:
+   simple1: select * from insanitytest where name = ? and choice = ? LIMIT 100
+
+#
+# In order to generate data consistently we need something to generate a unique key for this schema profile.
+#
+seed: changing this string changes the generated data. its hashcode is used as the random seed.

tools/stress/src/org/apache/cassandra/stress/Operation.java

@@ -18,199 +18,47 @@
 package org.apache.cassandra.stress;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.EnumMap;
 import java.util.List;
-import java.util.concurrent.ThreadLocalRandom;
 
-import org.apache.cassandra.stress.generatedata.Distribution;
-import org.apache.cassandra.stress.generatedata.KeyGen;
-import org.apache.cassandra.stress.generatedata.RowGen;
+import org.apache.cassandra.stress.generate.Distribution;
+import org.apache.cassandra.stress.generate.Partition;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
 import org.apache.cassandra.stress.settings.*;
 import org.apache.cassandra.stress.util.JavaDriverClient;
 import org.apache.cassandra.stress.util.ThriftClient;
 import org.apache.cassandra.stress.util.Timer;
-import org.apache.cassandra.thrift.ColumnParent;
 import org.apache.cassandra.thrift.InvalidRequestException;
-import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.thrift.SliceRange;
 import org.apache.cassandra.transport.SimpleClient;
-import org.apache.cassandra.utils.ByteBufferUtil;
 
 public abstract class Operation
 {
-    public final long index;
-    protected final State state;
+    public final StressSettings settings;
+    public final Timer timer;
+    public final PartitionGenerator generator;
+    public final Distribution partitionCount;
 
-    public Operation(State state, long idx)
+    protected List<Partition> partitions;
+
+    public Operation(Timer timer, PartitionGenerator generator, StressSettings settings, Distribution partitionCount)
     {
-        index = idx;
-        this.state = state;
+        this.generator = generator;
+        this.timer = timer;
+        this.settings = settings;
+        this.partitionCount = partitionCount;
     }
 
     public static interface RunOp
     {
         public boolean run() throws Exception;
-        public String key();
-        public int keyCount();
+        public int partitionCount();
+        public int rowCount();
     }
 
-    // one per thread!
-    public static final class State
+    protected void setPartitions(List<Partition> partitions)
     {
-
-        public final StressSettings settings;
-        public final Timer timer;
-        public final Command type;
-        public final KeyGen keyGen;
-        public final RowGen rowGen;
-        public final Distribution counteradd;
-        public final List<ColumnParent> columnParents;
-        public final StressMetrics metrics;
-        public final SettingsCommandMixed.CommandSelector commandSelector;
-        private final EnumMap<Command, State> substates;
-        private Object cqlCache;
-
-        public State(Command type, StressSettings settings, StressMetrics metrics)
-        {
-            this.type = type;
-            this.timer = metrics.getTiming().newTimer();
-            if (type == Command.MIXED)
-            {
-                commandSelector = ((SettingsCommandMixed) settings.command).selector();
-                substates = new EnumMap<>(Command.class);
-            }
-            else
-            {
-                commandSelector = null;
-                substates = null;
-            }
-            counteradd = settings.command.add.get();
-            this.settings = settings;
-            this.keyGen = settings.keys.newKeyGen();
-            this.rowGen = settings.columns.newRowGen();
-            this.metrics = metrics;
-            this.columnParents = columnParents(type, settings);
-        }
-
-        private State(Command type, State copy)
-        {
-            this.type = type;
-            this.timer = copy.timer;
-            this.rowGen = copy.rowGen;
-            this.keyGen = copy.keyGen;
-            this.columnParents = columnParents(type, copy.settings);
-            this.metrics = copy.metrics;
-            this.settings = copy.settings;
-            this.counteradd = copy.counteradd;
-            this.substates = null;
-            this.commandSelector = null;
-        }
-
-        private List<ColumnParent> columnParents(Command type, StressSettings settings)
-        {
-            if (!settings.columns.useSuperColumns)
-                return Collections.singletonList(new ColumnParent(type.table));
-            else
-            {
-                ColumnParent[] cp = new ColumnParent[settings.columns.superColumns];
-                for (int i = 0 ; i < cp.length ; i++)
-                    cp[i] = new ColumnParent(type.supertable).setSuper_column(ByteBufferUtil.bytes("S" + i));
-                return Arrays.asList(cp);
-            }
-        }
-
-
-
-        public boolean isCql3()
-        {
-            return settings.mode.cqlVersion == CqlVersion.CQL3;
-        }
-        public Object getCqlCache()
-        {
-            return cqlCache;
-        }
-        public void storeCqlCache(Object val)
-        {
-            cqlCache = val;
-        }
-
-        public State substate(Command command)
-        {
-            assert type == Command.MIXED;
-            State substate = substates.get(command);
-            if (substate == null)
-            {
-                substates.put(command, substate = new State(command, this));
-            }
-            return substate;
-        }
-
-    }
-
-    protected ByteBuffer getKey()
-    {
-        return state.keyGen.getKeys(1, index).get(0);
-    }
-
-    protected List<ByteBuffer> getKeys(int count)
-    {
-        return state.keyGen.getKeys(count, index);
-    }
-
-    protected List<ByteBuffer> generateColumnValues(ByteBuffer key)
-    {
-        return state.rowGen.generate(index, key);
-    }
-
-    private int sliceStart(int count)
-    {
-        if (count == state.settings.columns.maxColumnsPerKey)
-            return 0;
-        return 1 + ThreadLocalRandom.current().nextInt(state.settings.columns.maxColumnsPerKey - count);
-    }
-
-    protected SlicePredicate slicePredicate()
-    {
-        final SlicePredicate predicate = new SlicePredicate();
-        if (state.settings.columns.slice)
-        {
-            int count = state.rowGen.count(index);
-            int start = sliceStart(count);
-            predicate.setSlice_range(new SliceRange()
-                                     .setStart(state.settings.columns.names.get(start))
-                                     .setFinish(new byte[] {})
-                                     .setReversed(false)
-                                     .setCount(count)
-            );
-        }
-        else
-            predicate.setColumn_names(randomNames());
-        return predicate;
-    }
-
-    protected List<ByteBuffer> randomNames()
-    {
-        int count = state.rowGen.count(index);
-        List<ByteBuffer> src = state.settings.columns.names;
-        if (count == src.size())
-            return src;
-        ThreadLocalRandom rnd = ThreadLocalRandom.current();
-        List<ByteBuffer> r = new ArrayList<>();
-        int c = 0, o = 0;
-        while (c < count && count + o < src.size())
-        {
-            int leeway = src.size() - (count + o);
-            int spreadover = count - c;
-            o += Math.round(rnd.nextDouble() * (leeway / (double) spreadover));
-            r.add(src.get(o + c++));
-        }
-        while (c < count)
-            r.add(src.get(o + c++));
-        return r;
+        this.partitions = partitions;
     }
 
     /**
@@ -230,13 +78,13 @@ public abstract class Operation
 
     public void timeWithRetry(RunOp run) throws IOException
     {
-        state.timer.start();
+        timer.start();
 
         boolean success = false;
         String exceptionMessage = null;
 
         int tries = 0;
-        for (; tries < state.settings.command.tries; tries++)
+        for (; tries < settings.command.tries; tries++)
         {
             try
             {
@@ -245,7 +93,7 @@ public abstract class Operation
             }
             catch (Exception e)
             {
-                switch (state.settings.log.level)
+                switch (settings.log.level)
                 {
                     case MINIMAL:
                         break;
@@ -265,15 +113,13 @@ public abstract class Operation
             }
         }
 
-        state.timer.stop(run.keyCount());
+        timer.stop(run.partitionCount(), run.rowCount());
 
         if (!success)
         {
-            error(String.format("Operation [%d] x%d key %s (0x%s) %s%n",
-                    index,
+            error(String.format("Operation x%d on key(s) %s: %s%n",
                     tries,
-                    run.key(),
-                    ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(run.key())),
+                    key(),
                     (exceptionMessage == null)
                         ? "Data returned was not validated"
                         : "Error executing: " + exceptionMessage));
@@ -281,6 +127,14 @@ public abstract class Operation
 
     }
 
+    private String key()
+    {
+        List<String> keys = new ArrayList<>();
+        for (Partition partition : partitions)
+            keys.add(partition.getKeyAsString());
+        return keys.toString();
+    }
+
     protected String getExceptionMessage(Exception e)
     {
         String className = e.getClass().getSimpleName();
@@ -290,9 +144,9 @@ public abstract class Operation
 
     protected void error(String message) throws IOException
     {
-        if (!state.settings.command.ignoreErrors)
+        if (!settings.command.ignoreErrors)
             throw new IOException(message);
-        else if (state.settings.log.level.compareTo(SettingsLog.Level.MINIMAL) > 0)
+        else if (settings.log.level.compareTo(SettingsLog.Level.MINIMAL) > 0)
             System.err.println(message);
     }
 

tools/stress/src/org/apache/cassandra/stress/StressAction.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.CountDownLatch;
@@ -29,10 +30,15 @@ import java.util.concurrent.atomic.AtomicLong;
 
 import com.google.common.util.concurrent.RateLimiter;
 import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.cassandra.stress.operations.*;
+
+import org.apache.cassandra.stress.generate.Partition;
+import org.apache.cassandra.stress.generate.SeedGenerator;
+import org.apache.cassandra.stress.operations.OpDistribution;
+import org.apache.cassandra.stress.operations.OpDistributionFactory;
 import org.apache.cassandra.stress.settings.*;
 import org.apache.cassandra.stress.util.JavaDriverClient;
 import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
 import org.apache.cassandra.transport.SimpleClient;
 
 public class StressAction implements Runnable
@@ -52,7 +58,8 @@ public class StressAction implements Runnable
         // creating keyspace and column families
         settings.maybeCreateKeyspaces();
 
-        warmup(settings.command.type, settings.command);
+        if (!settings.command.noWarmup)
+            warmup(settings.command.getFactory(settings));
 
         output.println("Sleeping 2s...");
         Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
@@ -61,7 +68,7 @@ public class StressAction implements Runnable
         if (settings.rate.auto)
             success = runAuto();
         else
-            success = null != run(settings.command.type, settings.rate.threadCount, settings.command.count, output);
+            success = null != run(settings.command.getFactory(settings), settings.rate.threadCount, settings.command.count, output);
 
         if (success)
             output.println("END");
@@ -72,33 +79,18 @@ public class StressAction implements Runnable
     }
 
     // type provided separately to support recursive call for mixed command with each command type it is performing
-    private void warmup(Command type, SettingsCommand command)
+    private void warmup(OpDistributionFactory operations)
     {
         // warmup - do 50k iterations; by default hotspot compiles methods after 10k invocations
         PrintStream warmupOutput = new PrintStream(new OutputStream() { @Override public void write(int b) throws IOException { } } );
-        int iterations;
-        switch (type.category)
+        int iterations = 50000 * settings.node.nodes.size();
+        for (OpDistributionFactory single : operations.each())
         {
-            case BASIC:
-                iterations = 50000;
-                break;
-            case MIXED:
-                for (Command subtype : ((SettingsCommandMixed) command).getCommands())
-                    warmup(subtype, command);
-                return;
-            case MULTI:
-                int keysAtOnce = command.keysAtOnce;
-                iterations = Math.min(50000, (int) Math.ceil(500000d / keysAtOnce));
-                break;
-            default:
-                throw new IllegalStateException();
+            // we need to warm up all the nodes in the cluster ideally, but we may not be the only stress instance;
+            // so warm up all the nodes we're speaking to only.
+            output.println(String.format("Warming up %s with %d iterations...", single.desc(), iterations));
+            run(single, 20, iterations, warmupOutput);
         }
-
-        // we need to warm up all the nodes in the cluster ideally, but we may not be the only stress instance;
-        // so warm up all the nodes we're speaking to only.
-        iterations *= settings.node.nodes.size();
-        output.println(String.format("Warming up %s with %d iterations...", type, iterations));
-        run(type, 20, iterations, warmupOutput);
     }
 
     // TODO : permit varying more than just thread count
@@ -113,7 +105,7 @@ public class StressAction implements Runnable
         {
             output.println(String.format("Running with %d threadCount", threadCount));
 
-            StressMetrics result = run(settings.command.type, threadCount, settings.command.count, output);
+            StressMetrics result = run(settings.command.getFactory(settings), threadCount, settings.command.count, output);
             if (result == null)
                 return false;
             results.add(result);
@@ -170,13 +162,13 @@ public class StressAction implements Runnable
         return improvement / count;
     }
 
-    private StressMetrics run(Command type, int threadCount, long opCount, PrintStream output)
+    private StressMetrics run(OpDistributionFactory operations, int threadCount, long opCount, PrintStream output)
     {
 
         output.println(String.format("Running %s with %d threads %s",
-                type.toString(),
-                threadCount,
-                opCount > 0 ? " for " + opCount + " iterations" : "until stderr of mean < " + settings.command.targetUncertainty));
+                                     operations.desc(),
+                                     threadCount,
+                                     opCount > 0 ? " for " + opCount + " iterations" : "until stderr of mean < " + settings.command.targetUncertainty));
         final WorkQueue workQueue;
         if (opCount < 0)
             workQueue = new ContinuousWorkQueue(50);
@@ -193,7 +185,7 @@ public class StressAction implements Runnable
         final CountDownLatch done = new CountDownLatch(threadCount);
         final Consumer[] consumers = new Consumer[threadCount];
         for (int i = 0; i < threadCount; i++)
-            consumers[i] = new Consumer(type, done, workQueue, metrics, rateLimiter);
+            consumers[i] = new Consumer(operations, done, workQueue, metrics, rateLimiter);
 
         // starting worker threadCount
         for (int i = 0; i < threadCount; i++)
@@ -236,18 +228,24 @@ public class StressAction implements Runnable
     private class Consumer extends Thread
     {
 
-        private final Operation.State state;
+        private final OpDistribution operations;
+        private final StressMetrics metrics;
+        private final Timer timer;
+        private final SeedGenerator seedGenerator;
         private final RateLimiter rateLimiter;
         private volatile boolean success = true;
         private final WorkQueue workQueue;
         private final CountDownLatch done;
 
-        public Consumer(Command type, CountDownLatch done, WorkQueue workQueue, StressMetrics metrics, RateLimiter rateLimiter)
+        public Consumer(OpDistributionFactory operations, CountDownLatch done, WorkQueue workQueue, StressMetrics metrics, RateLimiter rateLimiter)
         {
             this.done = done;
             this.rateLimiter = rateLimiter;
             this.workQueue = workQueue;
-            this.state = new Operation.State(type, settings, metrics);
+            this.metrics = metrics;
+            this.timer = metrics.getTiming().newTimer();
+            this.seedGenerator = settings.keys.newSeedGenerator();
+            this.operations = operations.get(timer);
         }
 
         public void run()
@@ -269,63 +267,89 @@ public class StressAction implements Runnable
                         sclient = settings.getSimpleNativeClient();
                         break;
                     case THRIFT:
-                        tclient = settings.getThriftClient();
-                        break;
                     case THRIFT_SMART:
-                        tclient = settings.getSmartThriftClient();
+                        tclient = settings.getThriftClient();
                         break;
                     default:
                         throw new IllegalStateException();
                 }
 
-                Work work;
-                while ( null != (work = workQueue.poll()) )
+                int maxBatchSize = operations.maxBatchSize();
+                Work work = workQueue.poll();
+                Partition[] partitions = new Partition[maxBatchSize];
+                int workDone = 0;
+                while (work != null)
                 {
 
-                    if (rateLimiter != null)
-                        rateLimiter.acquire(work.count);
+                    Operation op = operations.next();
+                    op.generator.reset();
+                    int batchSize = Math.max(1, (int) op.partitionCount.next());
+                    int partitionCount = 0;
 
-                    for (int i = 0 ; i < work.count ; i++)
+                    while (partitionCount < batchSize)
                     {
-                        try
+                        int count = Math.min((work.count - workDone), batchSize - partitionCount);
+                        for (int i = 0 ; i < count ; i++)
                         {
-                            Operation op = createOperation(state, i + work.offset);
-                            switch (settings.mode.api)
-                            {
-                                case JAVA_DRIVER_NATIVE:
-                                    op.run(jclient);
-                                    break;
-                                case SIMPLE_NATIVE:
-                                    op.run(sclient);
-                                    break;
-                                case THRIFT:
-                                case THRIFT_SMART:
-                                default:
-                                    op.run(tclient);
-                            }
-                        } catch (Exception e)
+                            long seed = seedGenerator.next(work.offset + workDone + i);
+                            partitions[partitionCount + i] = op.generator.generate(seed);
+                        }
+                        workDone += count;
+                        partitionCount += count;
+                        if (workDone == work.count)
                         {
-                            if (output == null)
+                            workDone = 0;
+                            work = workQueue.poll();
+                            if (work == null)
                             {
-                                System.err.println(e.getMessage());
-                                success = false;
-                                System.exit(-1);
+                                if (partitionCount == 0)
+                                    return;
+                                break;
                             }
-
-                            e.printStackTrace(output);
-                            success = false;
-                            workQueue.stop();
-                            state.metrics.cancel();
-                            return;
+                            if (rateLimiter != null)
+                                rateLimiter.acquire(work.count);
                         }
                     }
-                }
 
+                    op.setPartitions(Arrays.asList(partitions).subList(0, partitionCount));
+
+                    try
+                    {
+                        switch (settings.mode.api)
+                        {
+                            case JAVA_DRIVER_NATIVE:
+                                op.run(jclient);
+                                break;
+                            case SIMPLE_NATIVE:
+                                op.run(sclient);
+                                break;
+                            case THRIFT:
+                            case THRIFT_SMART:
+                            default:
+                                op.run(tclient);
+                        }
+                    }
+                    catch (Exception e)
+                    {
+                        if (output == null)
+                        {
+                            System.err.println(e.getMessage());
+                            success = false;
+                            System.exit(-1);
+                        }
+
+                        e.printStackTrace(output);
+                        success = false;
+                        workQueue.stop();
+                        metrics.cancel();
+                        return;
+                    }
+                }
             }
             finally
             {
                 done.countDown();
-                state.timer.close();
+                timer.close();
             }
 
         }
@@ -443,106 +467,4 @@ public class StressAction implements Runnable
 
     }
 
-    private Operation createOperation(Operation.State state, long index)
-    {
-        return createOperation(state.type, state, index);
-    }
-    private Operation createOperation(Command type, Operation.State state, long index)
-    {
-        switch (type)
-        {
-            case READ:
-                switch(state.settings.mode.style)
-                {
-                    case THRIFT:
-                        return new ThriftReader(state, index);
-                    case CQL:
-                    case CQL_PREPARED:
-                        return new CqlReader(state, index);
-                    default:
-                        throw new UnsupportedOperationException();
-                }
-
-
-            case COUNTER_READ:
-                switch(state.settings.mode.style)
-                {
-                    case THRIFT:
-                        return new ThriftCounterGetter(state, index);
-                    case CQL:
-                    case CQL_PREPARED:
-                        return new CqlCounterGetter(state, index);
-                    default:
-                        throw new UnsupportedOperationException();
-                }
-
-            case WRITE:
-                switch(state.settings.mode.style)
-                {
-                    case THRIFT:
-                        return new ThriftInserter(state, index);
-                    case CQL:
-                    case CQL_PREPARED:
-                        return new CqlInserter(state, index);
-                    default:
-                        throw new UnsupportedOperationException();
-                }
-
-            case COUNTER_WRITE:
-                switch(state.settings.mode.style)
-                {
-                    case THRIFT:
-                        return new ThriftCounterAdder(state, index);
-                    case CQL:
-                    case CQL_PREPARED:
-                        return new CqlCounterAdder(state, index);
-                    default:
-                        throw new UnsupportedOperationException();
-                }
-
-            case RANGE_SLICE:
-                switch(state.settings.mode.style)
-                {
-                    case THRIFT:
-                        return new ThriftRangeSlicer(state, index);
-                    case CQL:
-                    case CQL_PREPARED:
-                        return new CqlRangeSlicer(state, index);
-                    default:
-                        throw new UnsupportedOperationException();
-                }
-
-            case INDEXED_RANGE_SLICE:
-                switch(state.settings.mode.style)
-                {
-                    case THRIFT:
-                        return new ThriftIndexedRangeSlicer(state, index);
-                    case CQL:
-                    case CQL_PREPARED:
-                        return new CqlIndexedRangeSlicer(state, index);
-                    default:
-                        throw new UnsupportedOperationException();
-                }
-
-            case READ_MULTI:
-                switch(state.settings.mode.style)
-                {
-                    case THRIFT:
-                        return new ThriftMultiGetter(state, index);
-                    case CQL:
-                    case CQL_PREPARED:
-                        return new CqlMultiGetter(state, index);
-                    default:
-                        throw new UnsupportedOperationException();
-                }
-
-            case MIXED:
-                Command subcommand = state.commandSelector.next();
-                return createOperation(subcommand, state.substate(subcommand), index);
-
-        }
-
-        throw new UnsupportedOperationException();
-    }
-
 }

tools/stress/src/org/apache/cassandra/stress/StressMetrics.java

@@ -127,20 +127,21 @@ public class StressMetrics
 
     // PRINT FORMATTING
 
-    public static final String HEADFORMAT = "%-10s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%7s,%9s";
-    public static final String ROWFORMAT =  "%-10d,%8.0f,%8.0f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%7.1f,%9.5f";
+    public static final String HEADFORMAT = "%-10s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%7s,%9s";
+    public static final String ROWFORMAT =  "%-10d,%8.0f,%8.0f,%8.0f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%7.1f,%9.5f";
 
     private static void printHeader(String prefix, PrintStream output)
     {
-        output.println(prefix + String.format(HEADFORMAT, "ops","op/s", "key/s","mean","med",".95",".99",".999","max","time","stderr"));
+        output.println(prefix + String.format(HEADFORMAT, "partitions","op/s", "pk/s", "row/s","mean","med",".95",".99",".999","max","time","stderr"));
     }
 
     private static void printRow(String prefix, TimingInterval interval, TimingInterval total, Uncertainty opRateUncertainty, PrintStream output)
     {
         output.println(prefix + String.format(ROWFORMAT,
-                total.operationCount,
+                total.partitionCount,
                 interval.realOpRate(),
-                interval.keyRate(),
+                interval.partitionRate(),
+                interval.rowRate(),
                 interval.meanLatency(),
                 interval.medianLatency(),
                 interval.rankLatency(0.95f),
@@ -156,9 +157,9 @@ public class StressMetrics
         output.println("\n");
         output.println("Results:");
         TimingInterval history = timing.getHistory();
-        output.println(String.format("real op rate              : %.0f", history.realOpRate()));
-        output.println(String.format("adjusted op rate stderr   : %.0f", opRateUncertainty.getUncertainty()));
-        output.println(String.format("key rate                  : %.0f", history.keyRate()));
+        output.println(String.format("op rate                   : %.0f", history.realOpRate()));
+        output.println(String.format("partition rate            : %.0f", history.partitionRate()));
+        output.println(String.format("row rate                  : %.0f", history.rowRate()));
         output.println(String.format("latency mean              : %.1f", history.meanLatency()));
         output.println(String.format("latency median            : %.1f", history.medianLatency()));
         output.println(String.format("latency 95th percentile   : %.1f", history.rankLatency(.95f)));

tools/stress/src/org/apache/cassandra/stress/StressProfile.java

@@ -0,0 +1,504 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress;
+
+
+import com.datastax.driver.core.*;
+import com.datastax.driver.core.exceptions.AlreadyExistsException;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.statements.CreateKeyspaceStatement;
+import org.apache.cassandra.exceptions.RequestValidationException;
+
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.generate.RatioDistributionFactory;
+import org.apache.cassandra.stress.generate.values.Booleans;
+import org.apache.cassandra.stress.generate.values.Bytes;
+import org.apache.cassandra.stress.generate.values.Generator;
+import org.apache.cassandra.stress.generate.values.Dates;
+import org.apache.cassandra.stress.generate.values.Doubles;
+import org.apache.cassandra.stress.generate.values.Floats;
+import org.apache.cassandra.stress.generate.values.GeneratorConfig;
+import org.apache.cassandra.stress.generate.values.Inets;
+import org.apache.cassandra.stress.generate.values.Integers;
+import org.apache.cassandra.stress.generate.values.Lists;
+import org.apache.cassandra.stress.generate.values.Longs;
+import org.apache.cassandra.stress.generate.values.Sets;
+import org.apache.cassandra.stress.generate.values.Strings;
+import org.apache.cassandra.stress.generate.values.TimeUUIDs;
+import org.apache.cassandra.stress.generate.values.UUIDs;
+import org.apache.cassandra.stress.operations.userdefined.SchemaInsert;
+import org.apache.cassandra.stress.operations.userdefined.SchemaQuery;
+import org.apache.cassandra.stress.settings.OptionDistribution;
+import org.apache.cassandra.stress.settings.OptionRatioDistribution;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.settings.ValidationType;
+import org.apache.cassandra.stress.util.JavaDriverClient;
+import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
+import org.apache.cassandra.thrift.Compression;
+import org.apache.cassandra.thrift.ThriftConversion;
+import org.apache.thrift.TException;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.Constructor;
+import org.yaml.snakeyaml.error.YAMLException;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOError;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+public class StressProfile implements Serializable
+{
+    private String keyspaceCql;
+    private String tableCql;
+    private String seedStr;
+
+    public String keyspaceName;
+    public String tableName;
+    private Map<String, GeneratorConfig> columnConfigs;
+    private Map<String, String> queries;
+    private Map<String, String> insert;
+
+    transient volatile TableMetadata tableMetaData;
+
+    transient volatile GeneratorFactory generatorFactory;
+
+    transient volatile BatchStatement.Type batchType;
+    transient volatile DistributionFactory partitions;
+    transient volatile RatioDistributionFactory pervisit;
+    transient volatile RatioDistributionFactory perbatch;
+    transient volatile PreparedStatement insertStatement;
+    transient volatile Integer thriftInsertId;
+
+    transient volatile Map<String, PreparedStatement> queryStatements;
+    transient volatile Map<String, Integer> thriftQueryIds;
+
+    private void init(StressYaml yaml) throws RequestValidationException
+    {
+        keyspaceName = yaml.keyspace;
+        keyspaceCql = yaml.keyspace_definition;
+        tableName = yaml.table;
+        tableCql = yaml.table_definition;
+        seedStr = yaml.seed;
+        queries = yaml.queries;
+        insert = yaml.insert;
+
+        assert keyspaceName != null : "keyspace name is required in yaml file";
+        assert tableName != null : "table name is required in yaml file";
+        assert queries != null : "queries map is required in yaml file";
+
+        if (keyspaceCql != null && keyspaceCql.length() > 0)
+        {
+            String name = ((CreateKeyspaceStatement) QueryProcessor.parseStatement(keyspaceCql)).keyspace();
+            assert name.equalsIgnoreCase(keyspaceName) : "Name in keyspace_definition doesn't match keyspace property: '" + name + "' != '" + keyspaceName + "'";
+        }
+        else
+        {
+            keyspaceCql = null;
+        }
+
+        if (tableCql != null && tableCql.length() > 0)
+        {
+            String name = CFMetaData.compile(tableCql, keyspaceName).cfName;
+            assert name.equalsIgnoreCase(tableName) : "Name in table_definition doesn't match table property: '" + name + "' != '" + tableName + "'";
+        }
+        else
+        {
+            tableCql = null;
+        }
+
+        columnConfigs = new HashMap<>();
+        for (Map<String,Object> spec : yaml.columnspec)
+        {
+            lowerCase(spec);
+            String name = (String) spec.remove("name");
+            DistributionFactory population = !spec.containsKey("population") ? null : OptionDistribution.get((String) spec.remove("population"));
+            DistributionFactory size = !spec.containsKey("size") ? null : OptionDistribution.get((String) spec.remove("size"));
+            DistributionFactory clustering = !spec.containsKey("cluster") ? null : OptionDistribution.get((String) spec.remove("cluster"));
+
+            if (!spec.isEmpty())
+                throw new IllegalArgumentException("Unrecognised option(s) in column spec: " + spec);
+            if (name == null)
+                throw new IllegalArgumentException("Missing name argument in column spec");
+
+            GeneratorConfig config = new GeneratorConfig(yaml.seed + name, clustering, size, population);
+            columnConfigs.put(name, config);
+        }
+    }
+
+    public void maybeCreateSchema(StressSettings settings)
+    {
+        JavaDriverClient client = settings.getJavaDriverClient(false);
+
+        if (keyspaceCql != null)
+        {
+            try
+            {
+                client.execute(keyspaceCql, org.apache.cassandra.db.ConsistencyLevel.ONE);
+            }
+            catch (AlreadyExistsException e)
+            {
+            }
+        }
+
+        client.execute("use "+keyspaceName, org.apache.cassandra.db.ConsistencyLevel.ONE);
+
+        if (tableCql != null)
+        {
+            try
+            {
+                client.execute(tableCql, org.apache.cassandra.db.ConsistencyLevel.ONE);
+            }
+            catch (AlreadyExistsException e)
+            {
+            }
+
+            System.out.println(String.format("Created schema. Sleeping %ss for propagation.", settings.node.nodes.size()));
+            Uninterruptibles.sleepUninterruptibly(settings.node.nodes.size(), TimeUnit.SECONDS);
+        }
+
+        maybeLoadSchemaInfo(settings);
+    }
+
+
+    private void maybeLoadSchemaInfo(StressSettings settings)
+    {
+        if (tableMetaData == null)
+        {
+            JavaDriverClient client = settings.getJavaDriverClient();
+
+            synchronized (client)
+            {
+
+                if (tableMetaData != null)
+                    return;
+
+                TableMetadata metadata = client.getCluster()
+                                               .getMetadata()
+                                               .getKeyspace(keyspaceName)
+                                               .getTable(tableName);
+
+                //Fill in missing column configs
+                for (ColumnMetadata col : metadata.getColumns())
+                {
+                    if (columnConfigs.containsKey(col.getName()))
+                        continue;
+
+                    columnConfigs.put(col.getName(), new GeneratorConfig(seedStr + col.getName(), null, null, null));
+                }
+
+                tableMetaData = metadata;
+            }
+        }
+    }
+
+    public SchemaQuery getQuery(String name, Timer timer, PartitionGenerator generator, StressSettings settings)
+    {
+        if (queryStatements == null)
+        {
+            synchronized (this)
+            {
+                if (queryStatements == null)
+                {
+                    try
+                    {
+                        JavaDriverClient jclient = settings.getJavaDriverClient();
+                        ThriftClient tclient = settings.getThriftClient();
+                        Map<String, PreparedStatement> stmts = new HashMap<>();
+                        Map<String, Integer> tids = new HashMap<>();
+                        for (Map.Entry<String, String> e : queries.entrySet())
+                        {
+                            stmts.put(e.getKey().toLowerCase(), jclient.prepare(e.getValue()));
+                            tids.put(e.getKey().toLowerCase(), tclient.prepare_cql3_query(e.getValue(), Compression.NONE));
+                        }
+                        thriftQueryIds = tids;
+                        queryStatements = stmts;
+                    }
+                    catch (TException e)
+                    {
+                        throw new RuntimeException(e);
+                    }
+                }
+            }
+        }
+
+        // TODO validation
+        name = name.toLowerCase();
+        return new SchemaQuery(timer, generator, settings, thriftQueryIds.get(name), queryStatements.get(name), ThriftConversion.fromThrift(settings.command.consistencyLevel), ValidationType.NOT_FAIL);
+    }
+
+    public SchemaInsert getInsert(Timer timer, PartitionGenerator generator, StressSettings settings)
+    {
+        if (insertStatement == null)
+        {
+            synchronized (this)
+            {
+                if (insertStatement == null)
+                {
+                    maybeLoadSchemaInfo(settings);
+
+                    Set<ColumnMetadata> keyColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getPrimaryKey());
+
+                    //Non PK Columns
+                    StringBuilder sb = new StringBuilder();
+
+                    sb.append("UPDATE \"").append(tableName).append("\" SET ");
+
+                    //PK Columns
+                    StringBuilder pred = new StringBuilder();
+                    pred.append(" WHERE ");
+
+                    boolean firstCol = true;
+                    boolean firstPred = true;
+                    for (ColumnMetadata c : tableMetaData.getColumns())
+                    {
+
+                        if (keyColumns.contains(c))
+                        {
+                            if (firstPred)
+                                firstPred = false;
+                            else
+                                pred.append(" AND ");
+
+                            pred.append(c.getName()).append(" = ?");
+                        }
+                        else
+                        {
+                            if (firstCol)
+                                firstCol = false;
+                            else
+                                sb.append(",");
+
+                            sb.append(c.getName()).append(" = ");
+
+                            switch (c.getType().getName())
+                            {
+                                case SET:
+                                case LIST:
+                                case COUNTER:
+                                    sb.append(c.getName()).append(" + ?");
+                                    break;
+                                default:
+                                    sb.append("?");
+                                    break;
+                            }
+                        }
+                    }
+
+                    //Put PK predicates at the end
+                    sb.append(pred);
+
+                    if (insert == null)
+                        insert = new HashMap<>();
+                    lowerCase(insert);
+
+                    partitions = OptionDistribution.get(!insert.containsKey("partitions") ? "fixed(1)" : insert.remove("partitions"));
+                    pervisit = OptionRatioDistribution.get(!insert.containsKey("pervisit") ? "fixed(1)/1" : insert.remove("pervisit"));
+                    perbatch = OptionRatioDistribution.get(!insert.containsKey("perbatch") ? "fixed(1)/1" : insert.remove("perbatch"));
+                    batchType = !insert.containsKey("batchtype") ? BatchStatement.Type.UNLOGGED : BatchStatement.Type.valueOf(insert.remove("batchtype"));
+                    if (!insert.isEmpty())
+                        throw new IllegalArgumentException("Unrecognised insert option(s): " + insert);
+
+                    if (generator.maxRowCount > 100 * 1000 * 1000)
+                        System.err.printf("WARNING: You have defined a schema that permits very large partitions (%.0f max rows (>100M))\n", generator.maxRowCount);
+                    if (perbatch.get().max() * pervisit.get().max() * partitions.get().maxValue() * generator.maxRowCount > 100000)
+                        System.err.printf("WARNING: You have defined a schema that permits very large batches (%.0f max rows (>100K)). This may OOM this stress client, or the server.\n",
+                                           perbatch.get().max() * pervisit.get().max() * partitions.get().maxValue() * generator.maxRowCount);
+
+                    JavaDriverClient client = settings.getJavaDriverClient();
+                    String query = sb.toString();
+                    try
+                    {
+                        thriftInsertId = settings.getThriftClient().prepare_cql3_query(query, Compression.NONE);
+                    }
+                    catch (TException e)
+                    {
+                        throw new RuntimeException(e);
+                    }
+                    insertStatement = client.prepare(query);
+                }
+            }
+        }
+
+        return new SchemaInsert(timer, generator, settings, partitions.get(), pervisit.get(), perbatch.get(), thriftInsertId, insertStatement, ThriftConversion.fromThrift(settings.command.consistencyLevel), batchType);
+    }
+
+    public PartitionGenerator newGenerator(StressSettings settings)
+    {
+        if (generatorFactory == null)
+        {
+            synchronized (this)
+            {
+                maybeLoadSchemaInfo(settings);
+                if (generatorFactory == null)
+                    generatorFactory = new GeneratorFactory();
+            }
+        }
+
+        return generatorFactory.newGenerator();
+    }
+
+    private class GeneratorFactory
+    {
+        final List<ColumnInfo> partitionKeys = new ArrayList<>();
+        final List<ColumnInfo> clusteringColumns = new ArrayList<>();
+        final List<ColumnInfo> valueColumns = new ArrayList<>();
+
+        private GeneratorFactory()
+        {
+            Set<ColumnMetadata> keyColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getPrimaryKey());
+
+            for (ColumnMetadata metadata : tableMetaData.getPartitionKey())
+                partitionKeys.add(new ColumnInfo(metadata.getName(), metadata.getType(), columnConfigs.get(metadata.getName())));
+            for (ColumnMetadata metadata : tableMetaData.getClusteringColumns())
+                clusteringColumns.add(new ColumnInfo(metadata.getName(), metadata.getType(), columnConfigs.get(metadata.getName())));
+            for (ColumnMetadata metadata : tableMetaData.getColumns())
+                if (!keyColumns.contains(metadata))
+                    valueColumns.add(new ColumnInfo(metadata.getName(), metadata.getType(), columnConfigs.get(metadata.getName())));
+        }
+
+        PartitionGenerator newGenerator()
+        {
+            return new PartitionGenerator(get(partitionKeys), get(clusteringColumns), get(valueColumns));
+        }
+
+        List<Generator> get(List<ColumnInfo> columnInfos)
+        {
+            List<Generator> result = new ArrayList<>();
+            for (ColumnInfo columnInfo : columnInfos)
+                result.add(columnInfo.getGenerator());
+            return result;
+        }
+    }
+
+    static class ColumnInfo
+    {
+        final String name;
+        final DataType type;
+        final GeneratorConfig config;
+
+        ColumnInfo(String name, DataType type, GeneratorConfig config)
+        {
+            this.name = name;
+            this.type = type;
+            this.config = config;
+        }
+
+        Generator getGenerator()
+        {
+            return getGenerator(name, type, config);
+        }
+
+        static Generator getGenerator(final String name, final DataType type, GeneratorConfig config)
+        {
+            switch (type.getName())
+            {
+                case ASCII:
+                case TEXT:
+                case VARCHAR:
+                    return new Strings(name, config);
+                case BIGINT:
+                case COUNTER:
+                    return new Longs(name, config);
+                case BLOB:
+                    return new Bytes(name, config);
+                case BOOLEAN:
+                    return new Booleans(name, config);
+                case DECIMAL:
+                case DOUBLE:
+                    return new Doubles(name, config);
+                case FLOAT:
+                    return new Floats(name, config);
+                case INET:
+                    return new Inets(name, config);
+                case INT:
+                case VARINT:
+                    return new Integers(name, config);
+                case TIMESTAMP:
+                    return new Dates(name, config);
+                case UUID:
+                    return new UUIDs(name, config);
+                case TIMEUUID:
+                    return new TimeUUIDs(name, config);
+                case SET:
+                    return new Sets(name, getGenerator(name, type.getTypeArguments().get(0), config), config);
+                case LIST:
+                    return new Lists(name, getGenerator(name, type.getTypeArguments().get(0), config), config);
+                default:
+                    throw new UnsupportedOperationException();
+            }
+        }
+    }
+
+    public static StressProfile load(File file) throws IOError
+    {
+        try
+        {
+            byte[] profileBytes = Files.readAllBytes(Paths.get(file.toURI()));
+
+            Constructor constructor = new Constructor(StressYaml.class);
+
+            Yaml yaml = new Yaml(constructor);
+
+            StressYaml profileYaml = yaml.loadAs(new ByteArrayInputStream(profileBytes), StressYaml.class);
+
+            StressProfile profile = new StressProfile();
+            profile.init(profileYaml);
+
+            return profile;
+        }
+        catch (YAMLException | IOException | RequestValidationException e)
+        {
+            throw new IOError(e);
+        }
+    }
+
+    static <V> void lowerCase(Map<String, V> map)
+    {
+        List<Map.Entry<String, V>> reinsert = new ArrayList<>();
+        Iterator<Map.Entry<String, V>> iter = map.entrySet().iterator();
+        while (iter.hasNext())
+        {
+            Map.Entry<String, V> e = iter.next();
+            if (!e.getKey().toLowerCase().equalsIgnoreCase(e.getKey()))
+            {
+                reinsert.add(e);
+                iter.remove();
+            }
+        }
+        for (Map.Entry<String, V> e : reinsert)
+            map.put(e.getKey().toLowerCase(), e.getValue());
+    }
+}

tools/stress/src/org/apache/cassandra/stress/StressYaml.java

@@ -1,6 +1,5 @@
-package org.apache.cassandra.stress.generatedata;
 /*
- * 
+ *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -8,32 +7,32 @@ package org.apache.cassandra.stress.generatedata;
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
- * 
+ *
  */
+package org.apache.cassandra.stress;
 
-
-import java.nio.ByteBuffer;
 import java.util.List;
+import java.util.Map;
 
-public abstract class DataGen
+public class StressYaml
 {
+    public String seed;
+    public String keyspace;
+    public String keyspace_definition;
+    public String table;
+    public String table_definition;
 
-    public abstract void generate(ByteBuffer fill, long index, ByteBuffer seed);
-    public abstract boolean isDeterministic();
-
-    public void generate(List<ByteBuffer> fills, long index, ByteBuffer seed)
-    {
-        for (ByteBuffer fill : fills)
-            generate(fill, index++, seed);
-    }
+    public List<Map<String,Object>> columnspec;
+    public Map<String,String> queries;
+    public Map<String,String> insert;
 
 }

tools/stress/src/org/apache/cassandra/stress/generate/Distribution.java

@@ -0,0 +1,57 @@
+package org.apache.cassandra.stress.generate;
+/*
+ * 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * 
+ */
+
+
+import java.io.Serializable;
+
+public abstract class Distribution implements Serializable
+{
+
+    public abstract long next();
+    public abstract double nextDouble();
+    public abstract long inverseCumProb(double cumProb);
+    public abstract void setSeed(long seed);
+
+    public long maxValue()
+    {
+        return inverseCumProb(1d);
+    }
+
+    public long minValue()
+    {
+        return inverseCumProb(0d);
+    }
+
+    // approximation of the average; slightly costly to calculate, so should not be invoked frequently
+    public long average()
+    {
+        double sum = 0;
+        int count = 0;
+        for (float d = 0 ; d <= 1.0d ; d += 0.02d)
+        {
+            sum += inverseCumProb(d);
+            count += 1;
+        }
+        return (long) (sum / count);
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/generate/DistributionBoundApache.java

@@ -1,4 +1,4 @@
-package org.apache.cassandra.stress.generatedata;
+package org.apache.cassandra.stress.generate;
 /*
  * 
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -42,12 +42,22 @@ public class DistributionBoundApache extends Distribution
         return bound(min, max, delegate.sample());
     }
 
+    public double nextDouble()
+    {
+        return boundDouble(min, max, delegate.sample());
+    }
+
     @Override
     public long inverseCumProb(double cumProb)
     {
         return bound(min, max, delegate.inverseCumulativeProbability(cumProb));
     }
 
+    public void setSeed(long seed)
+    {
+        delegate.reseedRandomGenerator(seed);
+    }
+
     private static long bound(long min, long max, double val)
     {
         long r = (long) val;
@@ -60,4 +70,15 @@ public class DistributionBoundApache extends Distribution
         throw new IllegalStateException();
     }
 
+    private static double boundDouble(long min, long max, double r)
+    {
+        if ((r >= min) & (r <= max))
+            return r;
+        if (r < min)
+            return min;
+        if (r > max)
+            return max;
+        throw new IllegalStateException();
+    }
+
 }

tools/stress/src/org/apache/cassandra/stress/generate/DistributionFactory.java

@@ -1,4 +1,4 @@
-package org.apache.cassandra.stress.generatedata;
+package org.apache.cassandra.stress.generate;
 /*
  * 
  * Licensed to the Apache Software Foundation (ASF) under one

tools/stress/src/org/apache/cassandra/stress/generate/DistributionFixed.java

@@ -1,4 +1,4 @@
-package org.apache.cassandra.stress.generatedata;
+package org.apache.cassandra.stress.generate;
 /*
  * 
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -37,10 +37,18 @@ public class DistributionFixed extends Distribution
         return key;
     }
 
+    public double nextDouble()
+    {
+        return key;
+    }
+
     @Override
     public long inverseCumProb(double cumProb)
     {
         return key;
     }
 
+    public void setSeed(long seed)
+    {
+    }
 }

tools/stress/src/org/apache/cassandra/stress/generate/DistributionInverted.java

@@ -0,0 +1,37 @@
+package org.apache.cassandra.stress.generate;
+
+public class DistributionInverted extends Distribution
+{
+
+    final Distribution wrapped;
+    final long min;
+    final long max;
+
+    public DistributionInverted(Distribution wrapped)
+    {
+        this.wrapped = wrapped;
+        this.min = wrapped.minValue();
+        this.max = wrapped.maxValue();
+    }
+
+    public long next()
+    {
+        return max - (wrapped.next() - min);
+    }
+
+    public double nextDouble()
+    {
+        return max - (wrapped.nextDouble() - min);
+    }
+
+    public long inverseCumProb(double cumProb)
+    {
+        return max - (wrapped.inverseCumProb(cumProb) - min);
+    }
+
+    public void setSeed(long seed)
+    {
+        wrapped.setSeed(seed);
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/generate/DistributionOffsetApache.java

@@ -1,4 +1,4 @@
-package org.apache.cassandra.stress.generatedata;
+package org.apache.cassandra.stress.generate;
 /*
  * 
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -36,12 +36,22 @@ public class DistributionOffsetApache extends Distribution
         this.delta = max - min;
     }
 
+    public void setSeed(long seed)
+    {
+        delegate.reseedRandomGenerator(seed);
+    }
+
     @Override
     public long next()
     {
         return offset(min, delta, delegate.sample());
     }
 
+    public double nextDouble()
+    {
+        return offsetDouble(min, delta, delegate.sample());
+    }
+
     @Override
     public long inverseCumProb(double cumProb)
     {
@@ -58,4 +68,13 @@ public class DistributionOffsetApache extends Distribution
         return min + r;
     }
 
+    private double offsetDouble(long min, long delta, double r)
+    {
+        if (r < 0)
+            r = 0;
+        if (r > delta)
+            r = delta;
+        return min + r;
+    }
+
 }

tools/stress/src/org/apache/cassandra/stress/generate/Partition.java

@@ -0,0 +1,343 @@
+package org.apache.cassandra.stress.generate;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Queue;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.stress.generate.values.Generator;
+
+// a partition is re-used to reduce garbage generation, as is its internal RowIterator
+public class Partition
+{
+
+    private long idseed;
+    private final Object[] partitionKey;
+    private final PartitionGenerator generator;
+    private final RowIterator iterator;
+
+    public Partition(PartitionGenerator generator)
+    {
+        this.generator = generator;
+        this.partitionKey = new Object[generator.partitionKey.size()];
+        if (generator.clusteringComponents.size() > 0)
+            iterator = new MultiRowIterator();
+        else
+            iterator = new SingleRowIterator();
+    }
+
+    void setSeed(long seed)
+    {
+        long idseed = 0;
+        for (int i = 0 ; i < partitionKey.length ; i++)
+        {
+            Generator generator = this.generator.partitionKey.get(i);
+            // set the partition key seed based on the current work item we're processing
+            generator.setSeed(seed);
+            Object key = generator.generate();
+            partitionKey[i] = key;
+            // then contribute this value to the data seed
+            idseed = seed(key, generator.type, idseed);
+        }
+        this.idseed = idseed;
+    }
+
+    public RowIterator iterator(double useChance)
+    {
+        iterator.reset(useChance, 0);
+        return iterator;
+    }
+
+    public RowIterator iterator(int targetCount)
+    {
+        iterator.reset(Double.NaN, targetCount);
+        return iterator;
+    }
+
+    class SingleRowIterator extends RowIterator
+    {
+        boolean done;
+
+        void reset(double useChance, int targetCount)
+        {
+            done = false;
+        }
+
+        public Iterable<Row> batch(double ratio)
+        {
+            if (done)
+                return Collections.emptyList();
+            for (int i = 0 ; i < row.row.length ; i++)
+            {
+                Generator gen = generator.valueComponents.get(i);
+                gen.setSeed(idseed);
+                row.row[i] = gen.generate();
+            }
+            done = true;
+            return Collections.singleton(row);
+        }
+
+        public boolean done()
+        {
+            return done;
+        }
+    }
+
+    public abstract class RowIterator
+    {
+        // we reuse the row object to save garbage
+        final Row row = new Row(partitionKey, new Object[generator.clusteringComponents.size() + generator.valueComponents.size()]);
+
+        public abstract Iterable<Row> batch(double ratio);
+        abstract void reset(double useChance, int targetCount);
+
+        public abstract boolean done();
+
+        public Partition partition()
+        {
+            return Partition.this;
+        }
+    }
+
+    // permits iterating a random subset of the procedurally generated rows in this partition;  this is the only mechanism for visiting rows
+    // we maintain a stack of clustering components and their seeds; for each clustering component we visit, we generate all values it takes at that level,
+    // and then, using the average (total) number of children it takes we randomly choose whether or not we visit its children;
+    // if we do, we generate all possible values the children can take, and repeat the process. So at any one time we are using space proportional
+    // to C.N, where N is the average number of values each clustering component takes, as opposed to N^C total values in the partition.
+    class MultiRowIterator extends RowIterator
+    {
+
+        // probability any single row will be generated in this iteration
+        double useChance;
+        double expectedRowCount;
+
+        // the current seed in use at any given level; used to save recalculating it for each row, so we only need to recalc
+        // from prior row
+        final long[] clusteringSeeds = new long[generator.clusteringComponents.size()];
+        // the components remaining to be visited for each level of the current stack
+        final Queue<Object>[] clusteringComponents = new ArrayDeque[generator.clusteringComponents.size()];
+
+        // we want our chance of selection to be applied uniformly, so we compound the roll we make at each level
+        // so that we know with what chance we reached there, and we adjust our roll at that level by that amount
+        double[] chancemodifier = new double[generator.clusteringComponents.size()];
+        double[] rollmodifier = new double[generator.clusteringComponents.size()];
+
+        // reusable set for generating unique clustering components
+        final Set<Object> unique = new HashSet<>();
+        final Random random = new Random();
+
+        MultiRowIterator()
+        {
+            for (int i = 0 ; i < clusteringComponents.length ; i++)
+                clusteringComponents[i] = new ArrayDeque<>();
+            rollmodifier[0] = 1f;
+            chancemodifier[0] = generator.clusteringChildAverages[0];
+        }
+
+        void reset(double useChance, int targetCount)
+        {
+            generator.clusteringComponents.get(0).setSeed(idseed);
+            int firstComponentCount = (int) generator.clusteringComponents.get(0).clusteringDistribution.next();
+            this.expectedRowCount = firstComponentCount * generator.clusteringChildAverages[0];
+            if (Double.isNaN(useChance))
+                useChance = Math.max(0d, Math.min(1d, targetCount / expectedRowCount));
+
+            for (Queue<?> q : clusteringComponents)
+                q.clear();
+
+            this.useChance = useChance;
+            clusteringSeeds[0] = idseed;
+            clusteringComponents[0].add(this);
+            fill(clusteringComponents[0], firstComponentCount, generator.clusteringComponents.get(0));
+            advance(0, 1f);
+        }
+
+        void fill(int component)
+        {
+            long seed = clusteringSeeds[component - 1];
+            Generator gen = generator.clusteringComponents.get(component);
+            gen.setSeed(seed);
+            clusteringSeeds[component] = seed(clusteringComponents[component - 1].peek(), generator.clusteringComponents.get(component - 1).type, seed);
+            fill(clusteringComponents[component], (int) gen.clusteringDistribution.next(), gen);
+        }
+
+        void fill(Queue<Object> queue, int count, Generator generator)
+        {
+            if (count == 1)
+            {
+                queue.add(generator.generate());
+            }
+            else
+            {
+                unique.clear();
+                for (int i = 0 ; i < count ; i++)
+                {
+                    Object next = generator.generate();
+                    if (unique.add(next))
+                        queue.add(next);
+                }
+            }
+        }
+
+        private boolean advance(double continueChance)
+        {
+            // we always start at the leaf level
+            int depth = clusteringComponents.length - 1;
+            // fill the row with the position we *were* at (unless pre-start)
+            for (int i = clusteringSeeds.length ; i < row.row.length ; i++)
+            {
+                Generator gen = generator.valueComponents.get(i - clusteringSeeds.length);
+                long seed = clusteringSeeds[depth];
+                seed = seed(clusteringComponents[depth].peek(), generator.clusteringComponents.get(depth).type, seed);
+                gen.setSeed(seed);
+                row.row[i] = gen.generate();
+            }
+            clusteringComponents[depth].poll();
+
+            return advance(depth, continueChance);
+        }
+
+        private boolean advance(int depth, double continueChance)
+        {
+            // advance the leaf component
+            clusteringComponents[depth].poll();
+            while (true)
+            {
+                if (clusteringComponents[depth].isEmpty())
+                {
+                    if (depth == 0)
+                        return false;
+                    depth--;
+                    clusteringComponents[depth].poll();
+                    continue;
+                }
+
+                // the chance of descending is the uniform use chance, multiplied by the number of children
+                // we would on average generate (so if we have a 0.1 use chance, but should generate 10 children
+                // then we will always descend), multiplied by 1/(compound roll), where (compound roll) is the
+                // chance with which we reached this depth, i.e. if we already beat 50/50 odds, we double our
+                // chance of beating this next roll
+                double thischance = useChance * chancemodifier[depth];
+                if (thischance > 0.999f || thischance >= random.nextDouble())
+                {
+                    row.row[depth] = clusteringComponents[depth].peek();
+                    depth++;
+                    if (depth == clusteringComponents.length)
+                        break;
+                    rollmodifier[depth] = rollmodifier[depth - 1] / Math.min(1d, thischance);
+                    chancemodifier[depth] = generator.clusteringChildAverages[depth] * rollmodifier[depth];
+                    fill(depth);
+                    continue;
+                }
+
+                clusteringComponents[depth].poll();
+            }
+
+            return continueChance >= 1.0d || continueChance >= random.nextDouble();
+        }
+
+        public Iterable<Row> batch(final double ratio)
+        {
+            final double continueChance = 1d - (Math.pow(ratio, expectedRowCount * useChance));
+            return new Iterable<Row>()
+            {
+                public Iterator<Row> iterator()
+                {
+                    return new Iterator<Row>()
+                    {
+                        boolean hasNext = true;
+                        public boolean hasNext()
+                        {
+                            return hasNext;
+                        }
+
+                        public Row next()
+                        {
+                            hasNext = advance(continueChance);
+                            return row;
+                        }
+
+                        public void remove()
+                        {
+                            throw new UnsupportedOperationException();
+                        }
+                    };
+                }
+            };
+        }
+
+        public boolean done()
+        {
+            return clusteringComponents[0].isEmpty();
+        }
+
+        public Partition partition()
+        {
+            return Partition.this;
+        }
+    }
+
+    public String getKeyAsString()
+    {
+        StringBuilder sb = new StringBuilder();
+        int i = 0;
+        for (Object key : partitionKey)
+        {
+            if (i > 0)
+                sb.append("|");
+            AbstractType type = generator.partitionKey.get(i++).type;
+            sb.append(type.getString(type.decompose(key)));
+        }
+        return sb.toString();
+    }
+
+    static long seed(Object object, AbstractType type, long seed)
+    {
+        if (object instanceof ByteBuffer)
+        {
+            ByteBuffer buf = (ByteBuffer) object;
+            for (int i = buf.position() ; i < buf.limit() ; i++)
+                seed = (31 * seed) + buf.get(i);
+            return seed;
+        }
+        else if (object instanceof String)
+        {
+            String str = (String) object;
+            for (int i = 0 ; i < str.length() ; i++)
+                seed = (31 * seed) + str.charAt(i);
+            return seed;
+        }
+        else if (object instanceof Number)
+        {
+            return (seed * 31) + ((Number) object).longValue();
+        }
+        else if (object instanceof UUID)
+        {
+            return seed * 31 + (((UUID) object).getLeastSignificantBits() ^ ((UUID) object).getMostSignificantBits());
+        }
+        else
+        {
+            return seed(type.decompose(object), BytesType.instance, seed);
+        }
+    }
+
+    public Object getPartitionKey(int i)
+    {
+        return partitionKey[i];
+    }
+
+    // used for thrift smart routing - if it's a multi-part key we don't try to route correctly right now
+    public ByteBuffer getToken()
+    {
+        return generator.partitionKey.get(0).type.decompose(partitionKey[0]);
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/generate/PartitionGenerator.java

@@ -0,0 +1,80 @@
+package org.apache.cassandra.stress.generate;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.stress.generate.values.Generator;
+
+public class PartitionGenerator
+{
+
+    public final double maxRowCount;
+    final List<Generator> partitionKey;
+    final List<Generator> clusteringComponents;
+    final List<Generator> valueComponents;
+    final int[] clusteringChildAverages;
+
+    private final Map<String, Integer> indexMap;
+
+    final List<Partition> recyclable = new ArrayList<>();
+    int partitionsInUse = 0;
+
+    public void reset()
+    {
+        partitionsInUse = 0;
+    }
+
+    public PartitionGenerator(List<Generator> partitionKey, List<Generator> clusteringComponents, List<Generator> valueComponents)
+    {
+        this.partitionKey = partitionKey;
+        this.clusteringComponents = clusteringComponents;
+        this.valueComponents = valueComponents;
+        this.clusteringChildAverages = new int[clusteringComponents.size()];
+        for (int i = clusteringChildAverages.length - 1 ; i >= 0 ; i--)
+            clusteringChildAverages[i] = (int) (i < (clusteringChildAverages.length - 1) ? clusteringComponents.get(i + 1).clusteringDistribution.average() * clusteringChildAverages[i + 1] : 1);
+        double maxRowCount = 1d;
+        for (Generator component : clusteringComponents)
+            maxRowCount *= component.clusteringDistribution.maxValue();
+        this.maxRowCount = maxRowCount;
+        this.indexMap = new HashMap<>();
+        int i = 0;
+        for (Generator generator : partitionKey)
+            indexMap.put(generator.name, --i);
+        i = 0;
+        for (Generator generator : Iterables.concat(clusteringComponents, valueComponents))
+            indexMap.put(generator.name, i++);
+    }
+
+    public int indexOf(String name)
+    {
+        Integer i = indexMap.get(name);
+        if (i == null)
+            throw new NoSuchElementException();
+        return i;
+    }
+
+    public Partition generate(long seed)
+    {
+        if (recyclable.size() <= partitionsInUse || recyclable.get(partitionsInUse) == null)
+            recyclable.add(new Partition(this));
+
+        Partition partition = recyclable.get(partitionsInUse++);
+        partition.setSeed(seed);
+        return partition;
+    }
+
+    public ByteBuffer convert(int c, Object v)
+    {
+        if (c < 0)
+            return partitionKey.get(-1-c).type.decompose(v);
+        if (c < clusteringComponents.size())
+            return clusteringComponents.get(c).type.decompose(v);
+        return valueComponents.get(c - clusteringComponents.size()).type.decompose(v);
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/RatioDistribution.java

@@ -0,0 +1,25 @@
+package org.apache.cassandra.stress.generate;
+
+public class RatioDistribution
+{
+
+    final Distribution distribution;
+    final double divisor;
+
+    public RatioDistribution(Distribution distribution, double divisor)
+    {
+        this.distribution = distribution;
+        this.divisor = divisor;
+    }
+
+    // yields a value between 0 and 1
+    public double next()
+    {
+        return Math.max(0f, Math.min(1f, distribution.nextDouble() / divisor));
+    }
+
+    public double max()
+    {
+        return Math.min(1d, distribution.maxValue() / divisor);
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/RatioDistributionFactory.java

@@ -1,4 +1,4 @@
-package org.apache.cassandra.stress.generatedata;
+package org.apache.cassandra.stress.generate;
 /*
  * 
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -23,8 +23,9 @@ package org.apache.cassandra.stress.generatedata;
 
 import java.io.Serializable;
 
-public interface DataGenFactory extends Serializable
+public interface RatioDistributionFactory extends Serializable
 {
-    DataGen get();
-}
 
+    RatioDistribution get();
+
+}

tools/stress/src/org/apache/cassandra/stress/generate/Row.java

@@ -0,0 +1,22 @@
+package org.apache.cassandra.stress.generate;
+
+public class Row
+{
+
+    final Object[] partitionKey;
+    final Object[] row;
+
+    public Row(Object[] partitionKey, Object[] row)
+    {
+        this.partitionKey = partitionKey;
+        this.row = row;
+    }
+
+    public Object get(int column)
+    {
+        if (column < 0)
+            return partitionKey[-1-column];
+        return row[column];
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/generate/SeedGenerator.java

@@ -0,0 +1,8 @@
+package org.apache.cassandra.stress.generate;
+
+public interface SeedGenerator
+{
+
+    long next(long workIndex);
+
+}

tools/stress/src/org/apache/cassandra/stress/generate/SeedRandomGenerator.java

@@ -0,0 +1,33 @@
+package org.apache.cassandra.stress.generate;
+
+public class SeedRandomGenerator implements SeedGenerator
+{
+
+    final Distribution distribution;
+    final Distribution clustering;
+
+    private long next;
+    private int count;
+
+    public SeedRandomGenerator(Distribution distribution, Distribution clustering)
+    {
+        this.distribution = distribution;
+        this.clustering = clustering;
+    }
+
+    public long next(long workIndex)
+    {
+        if (count == 0)
+        {
+            next = distribution.next();
+            count = (int) clustering.next();
+        }
+        long result = next;
+        count--;
+        if (next == distribution.maxValue())
+            next = distribution.minValue();
+        else
+            next++;
+        return result;
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/SeedSeriesGenerator.java

@@ -0,0 +1,21 @@
+package org.apache.cassandra.stress.generate;
+
+public class SeedSeriesGenerator implements SeedGenerator
+{
+
+    final long min;
+    final long count;
+
+    public SeedSeriesGenerator(long min, long max)
+    {
+        if (min > max)
+            throw new IllegalStateException();
+        this.min = min;
+        this.count = 1 + max - min;
+    }
+
+    public long next(long workIndex)
+    {
+        return min + (workIndex % count);
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/Booleans.java

@@ -0,0 +1,37 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+import org.apache.cassandra.db.marshal.BooleanType;
+
+public class Booleans extends Generator<Boolean>
+{
+    public Booleans(String name, GeneratorConfig config)
+    {
+        super(BooleanType.instance, config, name);
+    }
+
+    @Override
+    public Boolean generate()
+    {
+        return identityDistribution.next() % 1 == 0;
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/Bytes.java

@@ -0,0 +1,54 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+import org.apache.cassandra.db.marshal.BytesType;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Random;
+
+public class Bytes extends Generator<ByteBuffer>
+{
+    private final byte[] bytes;
+    private final Random rand = new Random();
+
+    public Bytes(String name, GeneratorConfig config)
+    {
+        super(BytesType.instance, config, name);
+        bytes = new byte[(int) sizeDistribution.maxValue()];
+    }
+
+    @Override
+    public ByteBuffer generate()
+    {
+        long seed = identityDistribution.next();
+        sizeDistribution.setSeed(seed);
+        rand.setSeed(~seed);
+        int size = (int) sizeDistribution.next();
+        for (int i = 0; i < size; )
+            for (int v = rand.nextInt(),
+                 n = Math.min(size - i, Integer.SIZE/Byte.SIZE);
+                 n-- > 0; v >>= Byte.SIZE)
+                bytes[i++] = (byte)v;
+        return ByteBuffer.wrap(Arrays.copyOf(bytes, size));
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/Dates.java

@@ -1,6 +1,5 @@
-package org.apache.cassandra.stress.generatedata;
 /*
- * 
+ *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -8,41 +7,40 @@ package org.apache.cassandra.stress.generatedata;
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
- * 
+ *
  */
+package org.apache.cassandra.stress.generate.values;
 
+import java.util.Date;
 
-public class DataGenHexFromOpIndex extends DataGenHex
+import org.apache.cassandra.db.marshal.DateType;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.settings.OptionDistribution;
+
+public class Dates extends Generator<Date>
 {
-
-    final long minKey;
-    final long maxKey;
-
-    public DataGenHexFromOpIndex(long minKey, long maxKey)
+    public Dates(String name, GeneratorConfig config)
     {
-        this.minKey = minKey;
-        this.maxKey = maxKey;
+        super(DateType.instance, config, name);
     }
 
     @Override
-    public boolean isDeterministic()
+    public Date generate()
     {
-        return true;
+        return new Date(identityDistribution.next());
     }
 
-    @Override
-    long next(long operationIndex)
+    DistributionFactory defaultIdentityDistribution()
     {
-        long range = maxKey + 1 - minKey;
-        return Math.abs((operationIndex % range) + minKey);
+        return OptionDistribution.get("uniform(1.." + Long.toString(50L*365L*24L*60L*60L*1000L) + ")");
     }
 }

tools/stress/src/org/apache/cassandra/stress/generate/values/Doubles.java

@@ -0,0 +1,37 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+import org.apache.cassandra.db.marshal.DoubleType;
+
+public class Doubles extends Generator<Double>
+{
+    public Doubles(String name, GeneratorConfig config)
+    {
+        super(DoubleType.instance, config, name);
+    }
+
+    @Override
+    public Double generate()
+    {
+        return identityDistribution.nextDouble();
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/Floats.java

@@ -0,0 +1,37 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+import org.apache.cassandra.db.marshal.FloatType;
+
+public class Floats extends Generator<Float>
+{
+    public Floats(String name, GeneratorConfig config)
+    {
+        super(FloatType.instance, config, name);
+    }
+
+    @Override
+    public Float generate()
+    {
+        return (float) identityDistribution.nextDouble();
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/Generator.java

@@ -0,0 +1,50 @@
+package org.apache.cassandra.stress.generate.values;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.stress.generate.Distribution;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.settings.OptionDistribution;
+
+public abstract class Generator<T>
+{
+
+    public final String name;
+    public final AbstractType<T> type;
+    final long salt;
+    final Distribution identityDistribution;
+    final Distribution sizeDistribution;
+    public final Distribution clusteringDistribution;
+
+    public Generator(AbstractType<T> type, GeneratorConfig config, String name)
+    {
+        this.type = type;
+        this.name = name;
+        this.salt = config.salt;
+        this.identityDistribution = config.getIdentityDistribution(defaultIdentityDistribution());
+        this.sizeDistribution = config.getSizeDistribution(defaultSizeDistribution());
+        this.clusteringDistribution = config.getClusteringDistribution(defaultClusteringDistribution());
+    }
+
+    public void setSeed(long seed)
+    {
+        identityDistribution.setSeed(seed ^ salt);
+        clusteringDistribution.setSeed(seed ^ ~salt);
+    }
+
+    public abstract T generate();
+
+    DistributionFactory defaultIdentityDistribution()
+    {
+        return OptionDistribution.get("uniform(1..100B)");
+    }
+
+    DistributionFactory defaultSizeDistribution()
+    {
+        return OptionDistribution.get("uniform(4..8)");
+    }
+
+    DistributionFactory defaultClusteringDistribution()
+    {
+        return OptionDistribution.get("fixed(1)");
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/GeneratorConfig.java

@@ -0,0 +1,68 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+import org.apache.cassandra.stress.generate.Distribution;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.MurmurHash;
+
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Map;
+
+public class GeneratorConfig implements Serializable
+{
+    public final long salt;
+
+    private final DistributionFactory clusteringDistributions;
+    private final DistributionFactory sizeDistributions;
+    private final DistributionFactory identityDistributions;
+
+    public GeneratorConfig(String seedStr, DistributionFactory clusteringDistributions, DistributionFactory sizeDistributions, DistributionFactory identityDistributions)
+    {
+        this.clusteringDistributions = clusteringDistributions;
+        this.sizeDistributions = sizeDistributions;
+        this.identityDistributions = identityDistributions;
+        ByteBuffer buf = ByteBufferUtil.bytes(seedStr);
+        long[] hash = new long[2];
+        MurmurHash.hash3_x64_128(buf, buf.position(), buf.remaining(), 0, hash);
+        salt = hash[0];
+    }
+
+    Distribution getClusteringDistribution(DistributionFactory deflt)
+    {
+        return (clusteringDistributions == null ? deflt : clusteringDistributions).get();
+    }
+
+    Distribution getIdentityDistribution(DistributionFactory deflt)
+    {
+        return (identityDistributions == null ? deflt : identityDistributions).get();
+    }
+
+    Distribution getSizeDistribution(DistributionFactory deflt)
+    {
+        return (sizeDistributions == null ? deflt : sizeDistributions).get();
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/HexBytes.java

@@ -0,0 +1,56 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.cassandra.db.marshal.BytesType;
+
+public class HexBytes extends Generator<ByteBuffer>
+{
+    private final byte[] bytes;
+
+    public HexBytes(String name, GeneratorConfig config)
+    {
+        super(BytesType.instance, config, name);
+        bytes = new byte[(int) sizeDistribution.maxValue()];
+    }
+
+    @Override
+    public ByteBuffer generate()
+    {
+        long seed = identityDistribution.next();
+        sizeDistribution.setSeed(seed);
+        int size = (int) sizeDistribution.next();
+        for (int i = 0 ; i < size ; i +=16)
+        {
+            long value = identityDistribution.next();
+            for (int j = 0 ; j < 16 && i + j < size ; j++)
+            {
+                int v = (int) (value & 15);
+                bytes[i + j] = (byte) ((v < 10 ? '0' : 'A') + v);
+                value >>>= 4;
+            }
+        }
+        return ByteBuffer.wrap(Arrays.copyOf(bytes, size));
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/HexStrings.java

@@ -0,0 +1,55 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+import java.util.Random;
+
+import org.apache.cassandra.db.marshal.UTF8Type;
+
+public class HexStrings extends Generator<String>
+{
+    private final char[] chars;
+
+    public HexStrings(String name, GeneratorConfig config)
+    {
+        super(UTF8Type.instance, config, name);
+        chars = new char[(int) sizeDistribution.maxValue()];
+    }
+
+    @Override
+    public String generate()
+    {
+        long seed = identityDistribution.next();
+        sizeDistribution.setSeed(seed);
+        int size = (int) sizeDistribution.next();
+        for (int i = 0 ; i < size ; i +=16)
+        {
+            long value = identityDistribution.next();
+            for (int j = 0 ; j < 16 && i + j < size ; j++)
+            {
+                int v = (int) (value & 15);
+                chars[i + j] = (char) ((v < 10 ? '0' : 'A') + v);
+                value >>>= 4;
+            }
+        }
+        return new String(chars, 0, size);
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/Inets.java

@@ -0,0 +1,57 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import org.apache.cassandra.db.marshal.InetAddressType;
+
+
+public class Inets extends Generator<InetAddress>
+{
+    final byte[] buf;
+    public Inets(String name, GeneratorConfig config)
+    {
+        super(InetAddressType.instance, config, name);
+        buf = new byte[4];
+    }
+
+    @Override
+    public InetAddress generate()
+    {
+        int val = (int) identityDistribution.next();
+
+        buf[0] = (byte)(val >>> 24);
+        buf[1] = (byte)(val >>> 16);
+        buf[2] = (byte)(val >>> 8);
+        buf[3] = (byte)val;
+
+        try
+        {
+            return InetAddress.getByAddress(buf);
+        }
+        catch (UnknownHostException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/Integers.java

@@ -1,6 +1,5 @@
-package org.apache.cassandra.stress.generatedata;
 /*
- * 
+ *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -8,38 +7,32 @@ package org.apache.cassandra.stress.generatedata;
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
- * 
+ *
  */
+package org.apache.cassandra.stress.generate.values;
 
+import org.apache.cassandra.db.marshal.Int32Type;
 
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-public class DataGenBytesRandom extends DataGen
+public class Integers extends Generator<Integer>
 {
 
-    private final Random rnd = new Random();
-
-    @Override
-    public void generate(ByteBuffer fill, long index, ByteBuffer seed)
+    public Integers(String name, GeneratorConfig config)
     {
-        fill.clear();
-        rnd.nextBytes(fill.array());
+        super(Int32Type.instance, config, name);
     }
 
     @Override
-    public boolean isDeterministic()
+    public Integer generate()
     {
-        return false;
+        return (int) identityDistribution.next();
     }
-
 }

tools/stress/src/org/apache/cassandra/stress/generate/values/Lists.java

@@ -1,4 +1,3 @@
-package org.apache.cassandra.stress.operations;
 /*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -19,35 +18,38 @@ package org.apache.cassandra.stress.operations;
  * under the License.
  *
  */
+package org.apache.cassandra.stress.generate.values;
 
 
-import java.nio.ByteBuffer;
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.List;
 
-public class CqlRangeSlicer extends CqlOperation<Integer>
+import org.apache.cassandra.db.marshal.ListType;
+
+public class Lists extends Generator<List>
 {
-    public CqlRangeSlicer(State state, long idx)
+    final Generator valueType;
+    final Object[] buffer;
+
+    public Lists(String name, Generator valueType, GeneratorConfig config)
     {
-        super(state, idx);
+        super(ListType.getInstance(valueType.type), config, name);
+        this.valueType = valueType;
+        buffer = new Object[(int) sizeDistribution.maxValue()];
+    }
+
+    public void setSeed(long seed)
+    {
+        super.setSeed(seed);
+        valueType.setSeed(seed * 31);
     }
 
     @Override
-    protected List<Object> getQueryParameters(byte[] key)
+    public List generate()
     {
-        return Collections.<Object>singletonList(ByteBuffer.wrap(key));
+        int size = (int) sizeDistribution.next();
+        for (int i = 0 ; i < size ; i++)
+            buffer[i] = valueType.generate();
+        return com.google.common.collect.Lists.newArrayList(Arrays.copyOf(buffer, size));
     }
-
-    @Override
-    protected String buildQuery()
-    {
-        return "SELECT FIRST " + state.settings.columns.maxColumnsPerKey + " ''..'' FROM " + wrapInQuotes(state.type.table) + " WHERE KEY > ?";
-    }
-
-    @Override
-    protected CqlRunOp<Integer> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, String keyid, ByteBuffer key)
-    {
-        return new CqlRunOpTestNonEmpty(client, query, queryId, params, keyid, key);
-    }
-
 }

tools/stress/src/org/apache/cassandra/stress/generate/values/Longs.java

@@ -1,6 +1,5 @@
-package org.apache.cassandra.stress.generatedata;
 /*
- * 
+ *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -8,33 +7,31 @@ package org.apache.cassandra.stress.generatedata;
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
- * 
+ *
  */
+package org.apache.cassandra.stress.generate.values;
 
+import org.apache.cassandra.db.marshal.LongType;
 
-public abstract class Distribution
+public class Longs extends Generator<Long>
 {
-
-    public abstract long next();
-    public abstract long inverseCumProb(double cumProb);
-
-    public long maxValue()
+    public Longs(String name, GeneratorConfig config)
     {
-        return inverseCumProb(1d);
+        super(LongType.instance, config, name);
     }
 
-    public long minValue()
+    @Override
+    public Long generate()
     {
-        return inverseCumProb(0d);
+        return identityDistribution.next();
     }
-
 }

tools/stress/src/org/apache/cassandra/stress/generate/values/Sets.java

@@ -1,6 +1,5 @@
-package org.apache.cassandra.stress.generatedata;
 /*
- * 
+ *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -8,47 +7,48 @@ package org.apache.cassandra.stress.generatedata;
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
- * 
+ *
  */
+package org.apache.cassandra.stress.generate.values;
 
 
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashSet;
+import java.util.Set;
 
-public class KeyGen
+import org.apache.cassandra.db.marshal.SetType;
+
+public class Sets extends Generator<Set>
 {
+    final Generator valueType;
 
-    final DataGen dataGen;
-    final int keySize;
-    final List<ByteBuffer> keyBuffers = new ArrayList<>();
-
-    public KeyGen(DataGen dataGen, int keySize)
+    public Sets(String name, Generator valueType, GeneratorConfig config)
     {
-        this.dataGen = dataGen;
-        this.keySize = keySize;
+        super(SetType.getInstance(valueType.type), config, name);
+        this.valueType = valueType;
     }
 
-    public List<ByteBuffer> getKeys(int n, long index)
+    public void setSeed(long seed)
     {
-        while (keyBuffers.size() < n)
-            keyBuffers.add(ByteBuffer.wrap(new byte[keySize]));
-        dataGen.generate(keyBuffers, index, null);
-        return keyBuffers;
+        super.setSeed(seed);
+        valueType.setSeed(seed * 31);
     }
 
-    public boolean isDeterministic()
+    @Override
+    public Set generate()
     {
-        return dataGen.isDeterministic();
+        final Set set = new HashSet();
+        int size = (int) sizeDistribution.next();
+        for (int i = 0 ; i < size ; i++)
+            set.add(valueType.generate());
+        return set;
     }
-
 }

tools/stress/src/org/apache/cassandra/stress/generate/values/Strings.java

@@ -0,0 +1,49 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+import java.util.Random;
+
+import org.apache.cassandra.db.marshal.UTF8Type;
+
+public class Strings extends Generator<String>
+{
+    private final char[] chars;
+    private final Random rnd = new Random();
+
+    public Strings(String name, GeneratorConfig config)
+    {
+        super(UTF8Type.instance, config, name);
+        chars = new char[(int) sizeDistribution.maxValue()];
+    }
+
+    @Override
+    public String generate()
+    {
+        long seed = identityDistribution.next();
+        sizeDistribution.setSeed(seed);
+        rnd.setSeed(~seed);
+        int size = (int) sizeDistribution.next();
+        for (int i = 0 ; i < size ; i++)
+            chars[i] = (char) (32 +rnd.nextInt(128-32));
+        return new String(chars, 0, size);
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/TimeUUIDs.java

@@ -0,0 +1,51 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.generate.values;
+
+
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.utils.UUIDGen;
+
+import java.util.UUID;
+
+public class TimeUUIDs extends Generator<UUID>
+{
+    final Dates dateGen;
+    final long clockSeqAndNode;
+
+    public TimeUUIDs(String name, GeneratorConfig config)
+    {
+        super(TimeUUIDType.instance, config, name);
+        dateGen = new Dates(name, config);
+        clockSeqAndNode = config.salt;
+    }
+
+    public void setSeed(long seed)
+    {
+        dateGen.setSeed(seed);
+    }
+
+    @Override
+    public UUID generate()
+    {
+        return UUIDGen.getTimeUUID(dateGen.generate().getTime(), clockSeqAndNode);
+    }
+}

tools/stress/src/org/apache/cassandra/stress/generate/values/UUIDs.java

@@ -1,4 +1,3 @@
-package org.apache.cassandra.stress.operations;
 /*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -19,24 +18,22 @@ package org.apache.cassandra.stress.operations;
  * under the License.
  *
  */
+package org.apache.cassandra.stress.generate.values;
 
+import java.util.UUID;
 
-import java.io.IOException;
+import org.apache.cassandra.db.marshal.UUIDType;
 
-import org.apache.cassandra.stress.Operation;
-import org.apache.cassandra.stress.util.ThriftClient;
-
-public class CqlMultiGetter extends Operation
+public class UUIDs extends Generator<UUID>
 {
-    public CqlMultiGetter(State state, long idx)
+    public UUIDs(String name, GeneratorConfig config)
     {
-        super(state, idx);
-        throw new RuntimeException("Multiget is not implemented for CQL");
+        super(UUIDType.instance, config, name);
     }
 
     @Override
-    public void run(ThriftClient client) throws IOException
+    public UUID generate()
     {
+        return new UUID(identityDistribution.next(), identityDistribution.next());
     }
-
 }

tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHex.java

@@ -1,60 +0,0 @@
-package org.apache.cassandra.stress.generatedata;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-
-import java.nio.ByteBuffer;
-
-public abstract class DataGenHex extends DataGen
-{
-
-    abstract long next(long operationIndex);
-
-    @Override
-    public final void generate(ByteBuffer fill, long operationIndex, ByteBuffer seed)
-    {
-        fill.clear();
-        fillKeyStringBytes(next(operationIndex), fill.array());
-    }
-
-    public static void fillKeyStringBytes(long key, byte[] fill)
-    {
-        int ub = fill.length - 1;
-        int offset = 0;
-        while (key != 0)
-        {
-            int digit = ((int) key) & 15;
-            key >>>= 4;
-            fill[ub - offset++] = digit(digit);
-        }
-        while (offset < fill.length)
-            fill[ub - offset++] = '0';
-    }
-
-    // needs to be UTF-8, but for these chars there is no difference
-    private static byte digit(int num)
-    {
-        if (num < 10)
-            return (byte)('0' + num);
-        return (byte)('A' + (num - 10));
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHexFromDistribution.java

@@ -1,66 +0,0 @@
-package org.apache.cassandra.stress.generatedata;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-
-import org.apache.commons.math3.distribution.NormalDistribution;
-import org.apache.commons.math3.distribution.UniformRealDistribution;
-
-public class DataGenHexFromDistribution extends DataGenHex
-{
-
-    final Distribution distribution;
-
-    public DataGenHexFromDistribution(Distribution distribution)
-    {
-        this.distribution = distribution;
-    }
-
-    @Override
-    public boolean isDeterministic()
-    {
-        return false;
-    }
-
-    @Override
-    long next(long operationIndex)
-    {
-        return distribution.next();
-    }
-
-    public static DataGenHex buildGaussian(long minKey, long maxKey, double stdevsToLimit)
-    {
-        double midRange = (maxKey + minKey) / 2d;
-        double halfRange = (maxKey - minKey) / 2d;
-        return new DataGenHexFromDistribution(new DistributionBoundApache(new NormalDistribution(midRange, halfRange / stdevsToLimit), minKey, maxKey));
-    }
-
-    public static DataGenHex buildGaussian(long minKey, long maxKey, double mean, double stdev)
-    {
-        return new DataGenHexFromDistribution(new DistributionBoundApache(new NormalDistribution(mean, stdev), minKey, maxKey));
-    }
-
-    public static DataGenHex buildUniform(long minKey, long maxKey)
-    {
-        return new DataGenHexFromDistribution(new DistributionBoundApache(new UniformRealDistribution(minKey, maxKey), minKey, maxKey));
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringDictionary.java

@@ -1,107 +0,0 @@
-package org.apache.cassandra.stress.generatedata;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.math3.distribution.EnumeratedDistribution;
-import org.apache.commons.math3.util.Pair;
-
-import static com.google.common.base.Charsets.UTF_8;
-
-public class DataGenStringDictionary extends DataGen
-{
-
-    private final byte space = ' ';
-    private final EnumeratedDistribution<byte[]> words;
-
-    public DataGenStringDictionary(EnumeratedDistribution<byte[]> wordDistribution)
-    {
-        words = wordDistribution;
-    }
-
-    @Override
-    public void generate(ByteBuffer fill, long index, ByteBuffer seed)
-    {
-        fill(fill);
-    }
-
-    @Override
-    public void generate(List<ByteBuffer> fills, long index, ByteBuffer seed)
-    {
-        for (int i = 0 ; i < fills.size() ; i++)
-            fill(fills.get(0));
-    }
-
-    private void fill(ByteBuffer fill)
-    {
-        fill.clear();
-        byte[] trg = fill.array();
-        int i = 0;
-        while (i < trg.length)
-        {
-            if (i > 0)
-                trg[i++] = space;
-            byte[] src = words.sample();
-            System.arraycopy(src, 0, trg, i, Math.min(src.length, trg.length - i));
-            i += src.length;
-        }
-    }
-
-    @Override
-    public boolean isDeterministic()
-    {
-        return false;
-    }
-
-    public static DataGenFactory getFactory(File file) throws IOException
-    {
-        final List<Pair<byte[], Double>> words = new ArrayList<>();
-        try (final BufferedReader reader = new BufferedReader(new FileReader(file)))
-        {
-            String line;
-            while ( null != (line = reader.readLine()) )
-            {
-                String[] pair = line.split(" +");
-                if (pair.length != 2)
-                    throw new IllegalArgumentException("Invalid record in dictionary: \"" + line + "\"");
-                words.add(new Pair<>(pair[1].getBytes(UTF_8), Double.parseDouble(pair[0])));
-            }
-            final EnumeratedDistribution<byte[]> dist = new EnumeratedDistribution<byte[]>(words);
-            return new DataGenFactory()
-            {
-                @Override
-                public DataGen get()
-                {
-                    return new DataGenStringDictionary(dist);
-                }
-            };
-        }
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringRepeats.java

@@ -1,90 +0,0 @@
-package org.apache.cassandra.stress.generatedata;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.cassandra.utils.FBUtilities;
-
-import static com.google.common.base.Charsets.UTF_8;
-
-public class DataGenStringRepeats extends DataGen
-{
-
-    private static final ConcurrentHashMap<Integer, ConcurrentHashMap<Long, byte[]>> CACHE_LOOKUP = new ConcurrentHashMap<>();
-
-    private final ConcurrentHashMap<Long, byte[]> cache;
-    private final int repeatFrequency;
-    public DataGenStringRepeats(int repeatFrequency)
-    {
-        if (!CACHE_LOOKUP.containsKey(repeatFrequency))
-            CACHE_LOOKUP.putIfAbsent(repeatFrequency, new ConcurrentHashMap<Long, byte[]>());
-        cache = CACHE_LOOKUP.get(repeatFrequency);
-        this.repeatFrequency = repeatFrequency;
-    }
-
-    @Override
-    public void generate(ByteBuffer fill, long index, ByteBuffer seed)
-    {
-        fill(fill, index, 0, seed);
-    }
-
-    @Override
-    public void generate(List<ByteBuffer> fills, long index, ByteBuffer seed)
-    {
-        for (int i = 0 ; i < fills.size() ; i++)
-        {
-            fill(fills.get(i), index, i, seed);
-        }
-    }
-
-    private void fill(ByteBuffer fill, long index, int column, ByteBuffer seed)
-    {
-        fill.clear();
-        byte[] trg = fill.array();
-        byte[] src = getData(index, column, seed);
-        for (int j = 0 ; j < trg.length ; j += src.length)
-            System.arraycopy(src, 0, trg, j, Math.min(src.length, trg.length - j));
-    }
-
-    private byte[] getData(long index, int column, ByteBuffer seed)
-    {
-        final long key = ((long)column * repeatFrequency) + ((seed == null ? index : Math.abs(seed.hashCode())) % repeatFrequency);
-        byte[] r = cache.get(key);
-        if (r != null)
-            return r;
-        MessageDigest md = FBUtilities.threadLocalMD5Digest();
-        r = md.digest(Long.toString(key).getBytes(UTF_8));
-        cache.putIfAbsent(key, r);
-        return r;
-    }
-
-    @Override
-    public boolean isDeterministic()
-    {
-        return true;
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionSeqBatch.java

@@ -1,68 +0,0 @@
-package org.apache.cassandra.stress.generatedata;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-
-public class DistributionSeqBatch extends DataGenHex
-{
-
-    final Distribution delegate;
-    final int batchSize;
-    final long maxKey;
-
-    private int batchIndex;
-    private long batchKey;
-
-    // object must be published safely if passed between threadCount, due to batchIndex not being volatile. various
-    // hacks possible, but not ideal. don't want to use volatile as object intended for single threaded use.
-    public DistributionSeqBatch(int batchSize, long maxKey, Distribution delegate)
-    {
-        this.batchIndex = batchSize;
-        this.batchSize = batchSize;
-        this.maxKey = maxKey;
-        this.delegate = delegate;
-    }
-
-    @Override
-    long next(long operationIndex)
-    {
-        if (batchIndex >= batchSize)
-        {
-            batchKey = delegate.next();
-            batchIndex = 0;
-        }
-        long r = batchKey + batchIndex++;
-        if (r > maxKey)
-        {
-            batchKey = delegate.next();
-            batchIndex = 1;
-            r = batchKey;
-        }
-        return r;
-    }
-
-    @Override
-    public boolean isDeterministic()
-    {
-        return false;
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/generatedata/RowGen.java

@@ -1,53 +0,0 @@
-package org.apache.cassandra.stress.generatedata;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-
-import java.nio.ByteBuffer;
-import java.util.List;
-
-/**
- * Generates a row of data, by constructing one byte buffers per column according to some algorithm
- * and delegating the work of populating the values of those byte buffers to the provided data generator
- */
-public abstract class RowGen
-{
-
-    final DataGen dataGen;
-    protected RowGen(DataGen dataGenerator)
-    {
-        this.dataGen = dataGenerator;
-    }
-
-    public List<ByteBuffer> generate(long operationIndex, ByteBuffer key)
-    {
-        List<ByteBuffer> fill = getColumns(operationIndex);
-        dataGen.generate(fill, operationIndex, key);
-        return fill;
-    }
-
-    // these byte[] may be re-used
-    abstract List<ByteBuffer> getColumns(long operationIndex);
-    abstract public int count(long operationIndex);
-
-    abstract public boolean isDeterministic();
-
-}

tools/stress/src/org/apache/cassandra/stress/generatedata/RowGenDistributedSize.java

@@ -1,116 +0,0 @@
-package org.apache.cassandra.stress.generatedata;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-public class RowGenDistributedSize extends RowGen
-{
-
-    // TODO - make configurable
-    static final int MAX_SINGLE_CACHE_SIZE = 16 * 1024;
-
-    final Distribution countDistribution;
-    final Distribution sizeDistribution;
-
-    final TreeMap<Integer, ByteBuffer> cache = new TreeMap<>();
-
-    // array re-used for returning columns
-    final ByteBuffer[] ret;
-    final int[] sizes;
-
-    final boolean isDeterministic;
-
-    public RowGenDistributedSize(DataGen dataGenerator, Distribution countDistribution, Distribution sizeDistribution)
-    {
-        super(dataGenerator);
-        this.countDistribution = countDistribution;
-        this.sizeDistribution = sizeDistribution;
-        ret = new ByteBuffer[(int) countDistribution.maxValue()];
-        sizes = new int[ret.length];
-        // TODO: should keep it deterministic in event that count distribution is not, but size and dataGen are, so that
-        // we simply need to generate the correct selection of columns
-        this.isDeterministic = dataGen.isDeterministic() && countDistribution.maxValue() == countDistribution.minValue()
-            && sizeDistribution.minValue() == sizeDistribution.maxValue();
-    }
-
-    ByteBuffer getBuffer(int size)
-    {
-        if (size >= MAX_SINGLE_CACHE_SIZE)
-            return ByteBuffer.allocate(size);
-        Map.Entry<Integer, ByteBuffer> found = cache.ceilingEntry(size);
-        if (found == null)
-        {
-            // remove the next entry down, and replace it with a cache of this size
-            Integer del = cache.lowerKey(size);
-            if (del != null)
-                cache.remove(del);
-            return ByteBuffer.allocate(size);
-        }
-        ByteBuffer r = found.getValue();
-        cache.remove(found.getKey());
-        return r;
-    }
-
-    @Override
-    List<ByteBuffer> getColumns(long operationIndex)
-    {
-        int i = 0;
-        int count = (int) countDistribution.next();
-        while (i < count)
-        {
-            int columnSize = (int) sizeDistribution.next();
-            sizes[i] = columnSize;
-            ret[i] = getBuffer(columnSize);
-            i++;
-        }
-        while (i < ret.length && ret[i] != null)
-            ret[i] = null;
-        i = 0;
-        while (i < count)
-        {
-            ByteBuffer b = ret[i];
-            cache.put(b.capacity(), b);
-            b.position(b.capacity() - sizes[i]);
-            ret[i] = b.slice();
-            b.position(0);
-            i++;
-        }
-        return Arrays.asList(ret).subList(0, count);
-    }
-
-    public int count(long operationIndex)
-    {
-        return (int) countDistribution.next();
-    }
-
-    @Override
-    public boolean isDeterministic()
-    {
-        return isDeterministic;
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/operations/CqlIndexedRangeSlicer.java

@@ -1,115 +0,0 @@
-package org.apache.cassandra.stress.operations;
-/*
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied.  See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*/
-
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.cassandra.utils.FBUtilities;
-
-public class CqlIndexedRangeSlicer extends CqlOperation<byte[][]>
-{
-
-    volatile boolean acceptNoResults = false;
-
-    public CqlIndexedRangeSlicer(State state, long idx)
-    {
-        super(state, idx);
-    }
-
-    @Override
-    protected List<Object> getQueryParameters(byte[] key)
-    {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    protected String buildQuery()
-    {
-        StringBuilder query = new StringBuilder("SELECT");
-        query.append(wrapInQuotes("key"));
-        query.append(" FROM ");
-        query.append(wrapInQuotes(state.type.table));
-
-        final String columnName = (state.settings.columns.namestrs.get(1));
-        query.append(" WHERE ").append(columnName).append("=?")
-                .append(" AND KEY > ? LIMIT ").append(state.settings.command.keysAtOnce);
-        return query.toString();
-    }
-
-    @Override
-    protected void run(CqlOperation.ClientWrapper client) throws IOException
-    {
-        acceptNoResults = false;
-        final List<ByteBuffer> columns = generateColumnValues(getKey());
-        final ByteBuffer value = columns.get(1); // only C1 column is indexed
-        byte[] minKey = new byte[0];
-        int rowCount;
-        do
-        {
-            List<Object> params = Arrays.<Object>asList(value, ByteBuffer.wrap(minKey));
-            CqlRunOp<byte[][]> op = run(client, params, value, new String(value.array()));
-            byte[][] keys = op.result;
-            rowCount = keys.length;
-            minKey = getNextMinKey(minKey, keys);
-            acceptNoResults = true;
-        } while (rowCount > 0);
-    }
-
-    private final class IndexedRangeSliceRunOp extends CqlRunOpFetchKeys
-    {
-
-        protected IndexedRangeSliceRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, String keyid, ByteBuffer key)
-        {
-            super(client, query, queryId, params, keyid, key);
-        }
-
-        @Override
-        public boolean validate(byte[][] result)
-        {
-            return acceptNoResults || result.length > 0;
-        }
-    }
-
-    @Override
-    protected CqlRunOp<byte[][]> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, String keyid, ByteBuffer key)
-    {
-        return new IndexedRangeSliceRunOp(client, query, queryId, params, keyid, key);
-    }
-
-    private static byte[] getNextMinKey(byte[] cur, byte[][] keys)
-    {
-        // find max
-        for (byte[] key : keys)
-            if (FBUtilities.compareUnsigned(cur, key) < 0)
-                cur = key;
-
-        // increment
-        for (int i = 0 ; i < cur.length ; i++)
-            if (++cur[i] != 0)
-                break;
-        return cur;
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/operations/FixedOpDistribution.java

@@ -0,0 +1,25 @@
+package org.apache.cassandra.stress.operations;
+
+import org.apache.cassandra.stress.Operation;
+
+public class FixedOpDistribution implements OpDistribution
+{
+
+    final Operation operation;
+
+    public FixedOpDistribution(Operation operation)
+    {
+        this.operation = operation;
+    }
+
+    public Operation next()
+    {
+        return operation;
+    }
+
+    public int maxBatchSize()
+    {
+        return (int) operation.partitionCount.maxValue();
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/operations/OpDistribution.java

@@ -0,0 +1,11 @@
+package org.apache.cassandra.stress.operations;
+
+import org.apache.cassandra.stress.Operation;
+
+public interface OpDistribution
+{
+
+    Operation next();
+    public int maxBatchSize();
+
+}

tools/stress/src/org/apache/cassandra/stress/operations/OpDistributionFactory.java

@@ -0,0 +1,12 @@
+package org.apache.cassandra.stress.operations;
+
+import org.apache.cassandra.stress.util.Timer;
+
+public interface OpDistributionFactory
+{
+
+    public OpDistribution get(Timer timer);
+    public String desc();
+    Iterable<OpDistributionFactory> each();
+
+}

tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistribution.java

@@ -0,0 +1,41 @@
+package org.apache.cassandra.stress.operations;
+
+import org.apache.commons.math3.distribution.EnumeratedDistribution;
+import org.apache.commons.math3.util.Pair;
+
+import org.apache.cassandra.stress.Operation;
+import org.apache.cassandra.stress.generate.Distribution;
+
+public class SampledOpDistribution implements OpDistribution
+{
+
+    final EnumeratedDistribution<Operation> operations;
+    final Distribution clustering;
+    private Operation cur;
+    private long remaining;
+
+    public SampledOpDistribution(EnumeratedDistribution<Operation> operations, Distribution clustering)
+    {
+        this.operations = operations;
+        this.clustering = clustering;
+    }
+
+    public int maxBatchSize()
+    {
+        int max = 1;
+        for (Pair<Operation, Double> pair : operations.getPmf())
+            max = Math.max(max, (int) pair.getFirst().partitionCount.maxValue());
+        return max;
+    }
+
+    public Operation next()
+    {
+        while (remaining == 0)
+        {
+            remaining = clustering.next();
+            cur = operations.sample();
+        }
+        remaining--;
+        return cur;
+    }
+}

tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistributionFactory.java

@@ -0,0 +1,72 @@
+package org.apache.cassandra.stress.operations;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.math3.distribution.EnumeratedDistribution;
+import org.apache.commons.math3.util.Pair;
+
+import org.apache.cassandra.stress.Operation;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.util.Timer;
+
+public abstract class SampledOpDistributionFactory<T> implements OpDistributionFactory
+{
+
+    final List<Pair<T, Double>> ratios;
+    final DistributionFactory clustering;
+    protected SampledOpDistributionFactory(List<Pair<T, Double>> ratios, DistributionFactory clustering)
+    {
+        this.ratios = ratios;
+        this.clustering = clustering;
+    }
+
+    protected abstract Operation get(Timer timer, PartitionGenerator generator, T key);
+    protected abstract PartitionGenerator newGenerator();
+
+    public OpDistribution get(Timer timer)
+    {
+        PartitionGenerator generator = newGenerator();
+        List<Pair<Operation, Double>> operations = new ArrayList<>();
+        for (Pair<T, Double> ratio : ratios)
+            operations.add(new Pair<>(get(timer, generator, ratio.getFirst()), ratio.getSecond()));
+        return new SampledOpDistribution(new EnumeratedDistribution<>(operations), clustering.get());
+    }
+
+    public String desc()
+    {
+        List<T> keys = new ArrayList<>();
+        for (Pair<T, Double> p : ratios)
+            keys.add(p.getFirst());
+        return keys.toString();
+    }
+
+    public Iterable<OpDistributionFactory> each()
+    {
+        List<OpDistributionFactory> out = new ArrayList<>();
+        for (final Pair<T, Double> ratio : ratios)
+        {
+            out.add(new OpDistributionFactory()
+            {
+                public OpDistribution get(Timer timer)
+                {
+                    return new FixedOpDistribution(SampledOpDistributionFactory.this.get(timer, newGenerator(), ratio.getFirst()));
+                }
+
+                public String desc()
+                {
+                    return ratio.getFirst().toString();
+                }
+
+                public Iterable<OpDistributionFactory> each()
+                {
+                    return Collections.<OpDistributionFactory>singleton(this);
+                }
+            });
+        }
+        return out;
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/operations/ThriftIndexedRangeSlicer.java

@@ -1,114 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.cassandra.stress.operations;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.cassandra.stress.Operation;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.*;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-
-public class ThriftIndexedRangeSlicer extends Operation
-{
-    public ThriftIndexedRangeSlicer(State state, long index)
-    {
-        super(state, index);
-        if (!state.rowGen.isDeterministic() || !state.keyGen.isDeterministic())
-            throw new IllegalStateException("Only run with a isDeterministic row/key generator");
-        if (state.settings.columns.useSuperColumns || state.columnParents.size() != 1)
-            throw new IllegalStateException("Does not support super columns");
-        if (state.settings.columns.useTimeUUIDComparator)
-            throw new IllegalStateException("Does not support TimeUUID column names");
-    }
-
-    public void run(final ThriftClient client) throws IOException
-    {
-
-        final SlicePredicate predicate = new SlicePredicate()
-                .setSlice_range(new SliceRange(ByteBufferUtil.EMPTY_BYTE_BUFFER,
-                        ByteBufferUtil.EMPTY_BYTE_BUFFER,
-                        false, state.settings.columns.maxColumnsPerKey));
-        final List<ByteBuffer> columns = generateColumnValues(getKey());
-        final ColumnParent parent = state.columnParents.get(0);
-
-        final ByteBuffer columnName = state.settings.columns.names.get(1);
-        final ByteBuffer value = columns.get(1); // only C1 column is indexed
-
-        IndexExpression expression = new IndexExpression(columnName, IndexOperator.EQ, value);
-        byte[] minKey = new byte[0];
-        final List<KeySlice>[] results = new List[1];
-        do
-        {
-
-            final boolean first = minKey.length == 0;
-            final IndexClause clause = new IndexClause(Arrays.asList(expression),
-                                                 ByteBuffer.wrap(minKey),
-                                                state.settings.command.keysAtOnce);
-
-            timeWithRetry(new RunOp()
-            {
-                @Override
-                public boolean run() throws Exception
-                {
-                    results[0] = client.get_indexed_slices(parent, clause, predicate, state.settings.command.consistencyLevel);
-                    return !first || results[0].size() > 0;
-                }
-
-                @Override
-                public String key()
-                {
-                    return new String(value.array());
-                }
-
-                @Override
-                public int keyCount()
-                {
-                    return results[0].size();
-                }
-            });
-
-            minKey = getNextMinKey(minKey, results[0]);
-
-        } while (results[0].size() > 0);
-    }
-
-    /**
-     * Get maximum key from keySlice list
-     * @param slices list of the KeySlice objects
-     * @return maximum key value of the list
-     */
-    private static byte[] getNextMinKey(byte[] cur, List<KeySlice> slices)
-    {
-        // find max
-        for (KeySlice slice : slices)
-            if (FBUtilities.compareUnsigned(cur, slice.getKey()) < 0)
-                cur = slice.getKey();
-
-        // increment
-        for (int i = 0 ; i < cur.length ; i++)
-            if (++cur[i] != 0)
-                break;
-        return cur;
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/operations/ThriftInserter.java

@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.stress.operations;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.cassandra.db.marshal.TimeUUIDType;
-import org.apache.cassandra.stress.Operation;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.*;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.UUIDGen;
-
-public final class ThriftInserter extends Operation
-{
-
-    public ThriftInserter(State state, long index)
-    {
-        super(state, index);
-    }
-
-    public void run(final ThriftClient client) throws IOException
-    {
-        final ByteBuffer key = getKey();
-        final List<Column> columns = generateColumns(key);
-
-        Map<String, List<Mutation>> row;
-        if (!state.settings.columns.useSuperColumns)
-        {
-            List<Mutation> mutations = new ArrayList<>(columns.size());
-            for (Column c : columns)
-            {
-                ColumnOrSuperColumn column = new ColumnOrSuperColumn().setColumn(c);
-                mutations.add(new Mutation().setColumn_or_supercolumn(column));
-            }
-            row = Collections.singletonMap(state.type.table, mutations);
-        }
-        else
-        {
-            List<Mutation> mutations = new ArrayList<>(state.columnParents.size());
-            for (ColumnParent parent : state.columnParents)
-            {
-                final SuperColumn s = new SuperColumn(parent.bufferForSuper_column(), columns);
-                final ColumnOrSuperColumn cosc = new ColumnOrSuperColumn().setSuper_column(s);
-                mutations.add(new Mutation().setColumn_or_supercolumn(cosc));
-            }
-            row = Collections.singletonMap(state.settings.command.type.supertable, mutations);
-        }
-
-        final Map<ByteBuffer, Map<String, List<Mutation>>> record = Collections.singletonMap(key, row);
-
-        timeWithRetry(new RunOp()
-        {
-            @Override
-            public boolean run() throws Exception
-            {
-                client.batch_mutate(record, state.settings.command.consistencyLevel);
-                return true;
-            }
-
-            @Override
-            public String key()
-            {
-                return new String(key.array());
-            }
-
-            @Override
-            public int keyCount()
-            {
-                return 1;
-            }
-        });
-    }
-
-    protected List<Column> generateColumns(ByteBuffer key)
-    {
-        final List<ByteBuffer> values = generateColumnValues(key);
-        final List<Column> columns = new ArrayList<>(values.size());
-
-        if (state.settings.columns.useTimeUUIDComparator)
-            for (int i = 0 ; i < values.size() ; i++)
-                new Column(TimeUUIDType.instance.decompose(UUIDGen.getTimeUUID()));
-        else
-            // TODO : consider randomly allocating column names in case where have fewer than max columns
-            // but need to think about implications for indexes / indexed range slicer / other knock on effects
-            for (int i = 0 ; i < values.size() ; i++)
-                columns.add(new Column(state.settings.columns.names.get(i)));
-
-        for (int i = 0 ; i < values.size() ; i++)
-            columns.get(i)
-                    .setValue(values.get(i))
-                    .setTimestamp(state.settings.columns.timestamp != null
-                                  ? Long.parseLong(state.settings.columns.timestamp)
-                                  : FBUtilities.timestampMicros());
-
-        return columns;
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/operations/ThriftMultiGetter.java

@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.stress.operations;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.cassandra.stress.Operation;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.ColumnParent;
-import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.thrift.SliceRange;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-
-public final class ThriftMultiGetter extends Operation
-{
-
-    public ThriftMultiGetter(State state, long index)
-    {
-        super(state, index);
-    }
-
-    public void run(final ThriftClient client) throws IOException
-    {
-
-        final SlicePredicate predicate = new SlicePredicate().setSlice_range(
-                new SliceRange(
-                        ByteBufferUtil.EMPTY_BYTE_BUFFER,
-                        ByteBufferUtil.EMPTY_BYTE_BUFFER,
-                        false,
-                        state.settings.columns.maxColumnsPerKey
-                )
-        );
-
-        final List<ByteBuffer> keys = getKeys(state.settings.command.keysAtOnce);
-
-        for (final ColumnParent parent : state.columnParents)
-        {
-            timeWithRetry(new RunOp()
-            {
-                int count;
-                @Override
-                public boolean run() throws Exception
-                {
-                    return (count = client.multiget_slice(keys, parent, predicate, state.settings.command.consistencyLevel).size()) != 0;
-                }
-
-                @Override
-                public String key()
-                {
-                    return keys.toString();
-                }
-
-                @Override
-                public int keyCount()
-                {
-                    return count;
-                }
-            });
-        }
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/operations/ThriftRangeSlicer.java

@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.stress.operations;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.cassandra.stress.Operation;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.ColumnParent;
-import org.apache.cassandra.thrift.KeyRange;
-import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.thrift.SliceRange;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-public final class ThriftRangeSlicer extends Operation
-{
-
-    public ThriftRangeSlicer(State state, long index)
-    {
-        super(state, index);
-    }
-
-    @Override
-    public void run(final ThriftClient client) throws IOException
-    {
-        final SlicePredicate predicate = new SlicePredicate()
-                .setSlice_range(
-                        new SliceRange(
-                                ByteBufferUtil.EMPTY_BYTE_BUFFER,
-                                ByteBufferUtil.EMPTY_BYTE_BUFFER,
-                                false,
-                                state.settings.columns.maxColumnsPerKey
-                        )
-                );
-
-        final ByteBuffer start = getKey();
-        final KeyRange range =
-                new KeyRange(state.settings.columns.maxColumnsPerKey)
-                        .setStart_key(start)
-                        .setEnd_key(ByteBufferUtil.EMPTY_BYTE_BUFFER)
-                        .setCount(state.settings.command.keysAtOnce);
-
-        for (final ColumnParent parent : state.columnParents)
-        {
-            timeWithRetry(new RunOp()
-            {
-                private int count = 0;
-                @Override
-                public boolean run() throws Exception
-                {
-                    return (count = client.get_range_slices(parent, predicate, range, state.settings.command.consistencyLevel).size()) != 0;
-                }
-
-                @Override
-                public String key()
-                {
-                    return new String(range.bufferForStart_key().array());
-                }
-
-                @Override
-                public int keyCount()
-                {
-                    return count;
-                }
-            });
-        }
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/operations/ThriftReader.java

@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.stress.operations;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.cassandra.stress.Operation;
-import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.thrift.ColumnOrSuperColumn;
-import org.apache.cassandra.thrift.ColumnParent;
-import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.thrift.SuperColumn;
-
-public final class ThriftReader extends Operation
-{
-
-    public ThriftReader(State state, long index)
-    {
-        super(state, index);
-    }
-
-    public void run(final ThriftClient client) throws IOException
-    {
-        final SlicePredicate predicate = slicePredicate();
-        final ByteBuffer key = getKey();
-        final List<ByteBuffer> expect = state.rowGen.isDeterministic() ? generateColumnValues(key) : null;
-        for (final ColumnParent parent : state.columnParents)
-        {
-            timeWithRetry(new RunOp()
-            {
-                @Override
-                public boolean run() throws Exception
-                {
-                    List<ColumnOrSuperColumn> row = client.get_slice(key, parent, predicate, state.settings.command.consistencyLevel);
-                    if (expect == null)
-                        return !row.isEmpty();
-                    if (row == null)
-                        return false;
-                    if (!state.settings.columns.useSuperColumns)
-                    {
-                        if (row.size() != expect.size())
-                            return false;
-                        for (int i = 0 ; i < row.size() ; i++)
-                            if (!row.get(i).getColumn().bufferForValue().equals(expect.get(i)))
-                                return false;
-                    }
-                    else
-                    {
-                        for (ColumnOrSuperColumn col : row)
-                        {
-                            SuperColumn superColumn = col.getSuper_column();
-                            if (superColumn.getColumns().size() != expect.size())
-                                return false;
-                            for (int i = 0 ; i < expect.size() ; i++)
-                                if (!superColumn.getColumns().get(i).bufferForValue().equals(expect.get(i)))
-                                    return false;
-                        }
-                    }
-                    return true;
-                }
-
-                @Override
-                public String key()
-                {
-                    return new String(key.array());
-                }
-
-                @Override
-                public int keyCount()
-                {
-                    return 1;
-                }
-            });
-        }
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterAdder.java

@@ -1,4 +1,4 @@
-package org.apache.cassandra.stress.operations;
+package org.apache.cassandra.stress.operations.predefined;
 /*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -25,11 +25,21 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.cassandra.stress.generate.Distribution;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.Command;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.util.Timer;
+
 public class CqlCounterAdder extends CqlOperation<Integer>
 {
-    public CqlCounterAdder(State state, long idx)
+
+    final Distribution counteradd;
+    public CqlCounterAdder(DistributionFactory counteradd, Timer timer, PartitionGenerator generator, StressSettings settings)
     {
-        super(state, idx);
+        super(Command.COUNTER_WRITE, timer, generator, settings);
+        this.counteradd = counteradd.get();
     }
 
     @Override
@@ -38,12 +48,12 @@ public class CqlCounterAdder extends CqlOperation<Integer>
         StringBuilder query = new StringBuilder("UPDATE \"Counter3\" SET ");
 
         // TODO : increment distribution subset of columns
-        for (int i = 0; i < state.settings.columns.maxColumnsPerKey; i++)
+        for (int i = 0; i < settings.columns.maxColumnsPerKey; i++)
         {
             if (i > 0)
                 query.append(",");
 
-            String name = state.settings.columns.namestrs.get(i);
+            String name = settings.columns.namestrs.get(i);
             query.append(name).append("=").append(name).append("+?");
         }
         query.append(" WHERE KEY=?");
@@ -54,15 +64,15 @@ public class CqlCounterAdder extends CqlOperation<Integer>
     protected List<Object> getQueryParameters(byte[] key)
     {
         final List<Object> list = new ArrayList<>();
-        for (int i = 0; i < state.settings.columns.maxColumnsPerKey; i++)
-            list.add(state.counteradd.next());
+        for (int i = 0; i < settings.columns.maxColumnsPerKey; i++)
+            list.add(counteradd.next());
         list.add(ByteBuffer.wrap(key));
         return list;
     }
 
     @Override
-    protected CqlRunOp<Integer> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, String keyid, ByteBuffer key)
+    protected CqlRunOp<Integer> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, ByteBuffer key)
     {
-        return new CqlRunOpAlwaysSucceed(client, query, queryId, params, keyid, key, 1);
+        return new CqlRunOpAlwaysSucceed(client, query, queryId, params, key, 1);
     }
 }

tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterGetter.java

@@ -1,4 +1,4 @@
-package org.apache.cassandra.stress.operations;
+package org.apache.cassandra.stress.operations.predefined;
 /*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -25,12 +25,17 @@ import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.Command;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.util.Timer;
+
 public class CqlCounterGetter extends CqlOperation<Integer>
 {
 
-    public CqlCounterGetter(State state, long idx)
+    public CqlCounterGetter(Timer timer, PartitionGenerator generator, StressSettings settings)
     {
-        super(state, idx);
+        super(Command.COUNTER_READ, timer, generator, settings);
     }
 
     @Override
@@ -42,18 +47,13 @@ public class CqlCounterGetter extends CqlOperation<Integer>
     @Override
     protected String buildQuery()
     {
-        StringBuilder query = new StringBuilder("SELECT *");
-
-        // TODO: obey slice/noslice option (instead of always slicing)
-        query.append(" FROM ").append(wrapInQuotes(state.type.table));
-
-        return query.append(" WHERE KEY=?").toString();
+        return "SELECT * FROM " + wrapInQuotes(type.table) + " WHERE KEY=?";
     }
 
     @Override
-    protected CqlRunOp<Integer> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, String keyid, ByteBuffer key)
+    protected CqlRunOp<Integer> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, ByteBuffer key)
     {
-        return new CqlRunOpTestNonEmpty(client, query, queryId, params, keyid, key);
+        return new CqlRunOpTestNonEmpty(client, query, queryId, params, key);
     }
 
 }

tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlInserter.java

@@ -1,4 +1,4 @@
-package org.apache.cassandra.stress.operations;
+package org.apache.cassandra.stress.operations.predefined;
 /*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -25,44 +25,35 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.Command;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.util.Timer;
 import org.apache.cassandra.utils.UUIDGen;
 
 public class CqlInserter extends CqlOperation<Integer>
 {
 
-    public CqlInserter(State state, long idx)
+    public CqlInserter(Timer timer, PartitionGenerator generator, StressSettings settings)
     {
-        super(state, idx);
-        if (state.settings.columns.useTimeUUIDComparator)
-            throw new IllegalStateException("Cannot use TimeUUID Comparator with CQL");
+        super(Command.WRITE, timer, generator, settings);
     }
 
     @Override
     protected String buildQuery()
     {
-        StringBuilder query = new StringBuilder("UPDATE ").append(wrapInQuotes(state.type.table));
-        if (state.settings.columns.timestamp != null)
-            query.append(" USING TIMESTAMP ").append(state.settings.columns.timestamp);
+        StringBuilder query = new StringBuilder("UPDATE ").append(wrapInQuotes(type.table));
+        if (settings.columns.timestamp != null)
+            query.append(" USING TIMESTAMP ").append(settings.columns.timestamp);
 
         query.append(" SET ");
 
-        for (int i = 0 ; i < state.settings.columns.maxColumnsPerKey; i++)
+        for (int i = 0 ; i < settings.columns.maxColumnsPerKey ; i++)
         {
             if (i > 0)
                 query.append(',');
 
-            if (state.settings.columns.useTimeUUIDComparator)
-            {
-                if (state.isCql3())
-                    throw new UnsupportedOperationException("Cannot use UUIDs in column names with CQL3");
-
-                query.append(wrapInQuotes(UUIDGen.getTimeUUID().toString()))
-                        .append(" = ?");
-            }
-            else
-            {
-                query.append(wrapInQuotes("C" + i)).append(" = ?");
-            }
+            query.append(wrapInQuotes(settings.columns.namestrs.get(i))).append(" = ?");
         }
 
         query.append(" WHERE KEY=?");
@@ -73,15 +64,15 @@ public class CqlInserter extends CqlOperation<Integer>
     protected List<Object> getQueryParameters(byte[] key)
     {
         final ArrayList<Object> queryParams = new ArrayList<>();
-        final List<ByteBuffer> values = generateColumnValues(ByteBuffer.wrap(key));
+        List<ByteBuffer> values = getColumnValues();
         queryParams.addAll(values);
         queryParams.add(ByteBuffer.wrap(key));
         return queryParams;
     }
 
     @Override
-    protected CqlRunOp<Integer> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, String keyid, ByteBuffer key)
+    protected CqlRunOp<Integer> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, ByteBuffer key)
     {
-        return new CqlRunOpAlwaysSucceed(client, query, queryId, params, keyid, key, 1);
+        return new CqlRunOpAlwaysSucceed(client, query, queryId, params, key, 1);
     }
 }

tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlOperation.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.cassandra.stress.operations;
+package org.apache.cassandra.stress.operations.predefined;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -29,10 +29,15 @@ import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
 import com.google.common.base.Function;
 import org.apache.cassandra.stress.Operation;
+import org.apache.cassandra.stress.StressMetrics;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.Command;
 import org.apache.cassandra.stress.settings.ConnectionStyle;
 import org.apache.cassandra.stress.settings.CqlVersion;
+import org.apache.cassandra.stress.settings.StressSettings;
 import org.apache.cassandra.stress.util.JavaDriverClient;
 import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
 import org.apache.cassandra.thrift.Compression;
 import org.apache.cassandra.thrift.CqlResult;
 import org.apache.cassandra.thrift.CqlRow;
@@ -42,31 +47,27 @@ import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.thrift.TException;
 
-public abstract class CqlOperation<V> extends Operation
+public abstract class CqlOperation<V> extends PredefinedOperation
 {
 
     protected abstract List<Object> getQueryParameters(byte[] key);
     protected abstract String buildQuery();
-    protected abstract CqlRunOp<V> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, String id, ByteBuffer key);
+    protected abstract CqlRunOp<V> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, ByteBuffer key);
 
-    public CqlOperation(State state, long idx)
+    public CqlOperation(Command type, Timer timer, PartitionGenerator generator, StressSettings settings)
     {
-        super(state, idx);
-        if (state.settings.columns.useSuperColumns)
-            throw new IllegalStateException("Super columns are not implemented for CQL");
-        if (state.settings.columns.variableColumnCount)
+        super(type, timer, generator, settings);
+        if (settings.columns.variableColumnCount)
             throw new IllegalStateException("Variable column counts are not implemented for CQL");
-        if (state.settings.columns.useTimeUUIDComparator)
-            throw new IllegalStateException("Cannot use TimeUUID Comparator with CQL");
     }
 
-    protected CqlRunOp<V> run(final ClientWrapper client, final List<Object> queryParams, final ByteBuffer key, final String keyid) throws IOException
+    protected CqlRunOp<V> run(final ClientWrapper client, final List<Object> queryParams, final ByteBuffer key) throws IOException
     {
         final CqlRunOp<V> op;
-        if (state.settings.mode.style == ConnectionStyle.CQL_PREPARED)
+        if (settings.mode.style == ConnectionStyle.CQL_PREPARED)
         {
             final Object id;
-            Object idobj = state.getCqlCache();
+            Object idobj = getCqlCache();
             if (idobj == null)
             {
                 try
@@ -76,23 +77,23 @@ public abstract class CqlOperation<V> extends Operation
                 {
                     throw new RuntimeException(e);
                 }
-                state.storeCqlCache(id);
+                storeCqlCache(id);
             }
             else
                 id = idobj;
 
-            op = buildRunOp(client, null, id, queryParams, keyid, key);
+            op = buildRunOp(client, null, id, queryParams, key);
         }
         else
         {
             final String query;
-            Object qobj = state.getCqlCache();
+            Object qobj = getCqlCache();
             if (qobj == null)
-                state.storeCqlCache(query = buildQuery());
+                storeCqlCache(query = buildQuery());
             else
                 query = qobj.toString();
 
-            op = buildRunOp(client, query, null, queryParams, keyid, key);
+            op = buildRunOp(client, query, null, queryParams, key);
         }
 
         timeWithRetry(op);
@@ -103,7 +104,7 @@ public abstract class CqlOperation<V> extends Operation
     {
         final byte[] key = getKey().array();
         final List<Object> queryParams = getQueryParameters(key);
-        run(client, queryParams, ByteBuffer.wrap(key), new String(key));
+        run(client, queryParams, ByteBuffer.wrap(key));
     }
 
     // Classes to process Cql results
@@ -114,9 +115,9 @@ public abstract class CqlOperation<V> extends Operation
 
         final int keyCount;
 
-        protected CqlRunOpAlwaysSucceed(ClientWrapper client, String query, Object queryId, List<Object> params, String id, ByteBuffer key, int keyCount)
+        protected CqlRunOpAlwaysSucceed(ClientWrapper client, String query, Object queryId, List<Object> params, ByteBuffer key, int keyCount)
         {
-            super(client, query, queryId, RowCountHandler.INSTANCE, params, id, key);
+            super(client, query, queryId, RowCountHandler.INSTANCE, params, key);
             this.keyCount = keyCount;
         }
 
@@ -127,7 +128,13 @@ public abstract class CqlOperation<V> extends Operation
         }
 
         @Override
-        public int keyCount()
+        public int partitionCount()
+        {
+            return keyCount;
+        }
+
+        @Override
+        public int rowCount()
         {
             return keyCount;
         }
@@ -137,9 +144,9 @@ public abstract class CqlOperation<V> extends Operation
     protected final class CqlRunOpTestNonEmpty extends CqlRunOp<Integer>
     {
 
-        protected CqlRunOpTestNonEmpty(ClientWrapper client, String query, Object queryId, List<Object> params, String id, ByteBuffer key)
+        protected CqlRunOpTestNonEmpty(ClientWrapper client, String query, Object queryId, List<Object> params, ByteBuffer key)
         {
-            super(client, query, queryId, RowCountHandler.INSTANCE, params, id, key);
+            super(client, query, queryId, RowCountHandler.INSTANCE, params, key);
         }
 
         @Override
@@ -149,7 +156,13 @@ public abstract class CqlOperation<V> extends Operation
         }
 
         @Override
-        public int keyCount()
+        public int partitionCount()
+        {
+            return result;
+        }
+
+        @Override
+        public int rowCount()
         {
             return result;
         }
@@ -159,17 +172,22 @@ public abstract class CqlOperation<V> extends Operation
     protected abstract class CqlRunOpFetchKeys extends CqlRunOp<byte[][]>
     {
 
-        protected CqlRunOpFetchKeys(ClientWrapper client, String query, Object queryId, List<Object> params, String id, ByteBuffer key)
+        protected CqlRunOpFetchKeys(ClientWrapper client, String query, Object queryId, List<Object> params, ByteBuffer key)
         {
-            super(client, query, queryId, KeysHandler.INSTANCE, params, id, key);
+            super(client, query, queryId, KeysHandler.INSTANCE, params, key);
         }
 
         @Override
-        public int keyCount()
+        public int partitionCount()
         {
             return result.length;
         }
 
+        @Override
+        public int rowCount()
+        {
+            return result.length;
+        }
     }
 
     protected final class CqlRunOpMatchResults extends CqlRunOp<ByteBuffer[][]>
@@ -178,14 +196,20 @@ public abstract class CqlOperation<V> extends Operation
         final List<List<ByteBuffer>> expect;
 
         // a null value for an item in expect means we just check the row is present
-        protected CqlRunOpMatchResults(ClientWrapper client, String query, Object queryId, List<Object> params, String id, ByteBuffer key, List<List<ByteBuffer>> expect)
+        protected CqlRunOpMatchResults(ClientWrapper client, String query, Object queryId, List<Object> params, ByteBuffer key, List<List<ByteBuffer>> expect)
         {
-            super(client, query, queryId, RowsHandler.INSTANCE, params, id, key);
+            super(client, query, queryId, RowsHandler.INSTANCE, params, key);
             this.expect = expect;
         }
 
         @Override
-        public int keyCount()
+        public int partitionCount()
+        {
+            return result == null ? 0 : result.length;
+        }
+
+        @Override
+        public int rowCount()
         {
             return result == null ? 0 : result.length;
         }
@@ -209,19 +233,17 @@ public abstract class CqlOperation<V> extends Operation
         final String query;
         final Object queryId;
         final List<Object> params;
-        final String id;
         final ByteBuffer key;
         final ResultHandler<V> handler;
         V result;
 
-        private CqlRunOp(ClientWrapper client, String query, Object queryId, ResultHandler<V> handler, List<Object> params, String id, ByteBuffer key)
+        private CqlRunOp(ClientWrapper client, String query, Object queryId, ResultHandler<V> handler, List<Object> params, ByteBuffer key)
         {
             this.client = client;
             this.query = query;
             this.queryId = queryId;
             this.handler = handler;
             this.params = params;
-            this.id = id;
             this.key = key;
         }
 
@@ -233,12 +255,6 @@ public abstract class CqlOperation<V> extends Operation
             : validate(result = client.execute(query, key, params, handler));
         }
 
-        @Override
-        public String key()
-        {
-            return id;
-        }
-
         public abstract boolean validate(V result);
 
     }
@@ -299,7 +315,7 @@ public abstract class CqlOperation<V> extends Operation
         public <V> V execute(String query, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler)
         {
             String formattedQuery = formatCqlQuery(query, queryParams);
-            return handler.javaDriverHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(state.settings.command.consistencyLevel)));
+            return handler.javaDriverHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(settings.command.consistencyLevel)));
         }
 
         @Override
@@ -309,7 +325,7 @@ public abstract class CqlOperation<V> extends Operation
                     client.executePrepared(
                             (PreparedStatement) preparedStatementId,
                             queryParams,
-                            ThriftConversion.fromThrift(state.settings.command.consistencyLevel)));
+                            ThriftConversion.fromThrift(settings.command.consistencyLevel)));
         }
 
         @Override
@@ -331,7 +347,7 @@ public abstract class CqlOperation<V> extends Operation
         public <V> V execute(String query, ByteBuffer key, List<Object> queryParams, ResultHandler<V> handler)
         {
             String formattedQuery = formatCqlQuery(query, queryParams);
-            return handler.thriftHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(state.settings.command.consistencyLevel)));
+            return handler.thriftHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(settings.command.consistencyLevel)));
         }
 
         @Override
@@ -341,7 +357,7 @@ public abstract class CqlOperation<V> extends Operation
                     client.executePrepared(
                             (byte[]) preparedStatementId,
                             toByteBufferParams(queryParams),
-                            ThriftConversion.fromThrift(state.settings.command.consistencyLevel)));
+                            ThriftConversion.fromThrift(settings.command.consistencyLevel)));
         }
 
         @Override
@@ -365,7 +381,7 @@ public abstract class CqlOperation<V> extends Operation
         {
             String formattedQuery = formatCqlQuery(query, queryParams);
             return handler.simpleNativeHandler().apply(
-                    client.execute_cql3_query(formattedQuery, key, Compression.NONE, state.settings.command.consistencyLevel)
+                    client.execute_cql3_query(formattedQuery, key, Compression.NONE, settings.command.consistencyLevel)
             );
         }
 
@@ -374,7 +390,7 @@ public abstract class CqlOperation<V> extends Operation
         {
             Integer id = (Integer) preparedStatementId;
             return handler.simpleNativeHandler().apply(
-                    client.execute_prepared_cql3_query(id, key, toByteBufferParams(queryParams), state.settings.command.consistencyLevel)
+                    client.execute_prepared_cql3_query(id, key, toByteBufferParams(queryParams), settings.command.consistencyLevel)
             );
         }
 

tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlReader.java

@@ -1,4 +1,4 @@
-package org.apache.cassandra.stress.operations;
+package org.apache.cassandra.stress.operations.predefined;
 /*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -27,14 +27,18 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.Command;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.util.Timer;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 public class CqlReader extends CqlOperation<ByteBuffer[][]>
 {
 
-    public CqlReader(State state, long idx)
+    public CqlReader(Timer timer, PartitionGenerator generator, StressSettings settings)
     {
-        super(state, idx);
+        super(Command.READ, timer, generator, settings);
     }
 
     @Override
@@ -42,29 +46,21 @@ public class CqlReader extends CqlOperation<ByteBuffer[][]>
     {
         StringBuilder query = new StringBuilder("SELECT ");
 
-        if (state.settings.columns.slice)
+        if (settings.columns.slice)
         {
             query.append("*");
         }
         else
         {
-            try
+            for (int i = 0; i < settings.columns.maxColumnsPerKey ; i++)
             {
-                for (int i = 0; i < state.settings.columns.names.size() ; i++)
-                {
-                    if (i > 0)
-                        query.append(",");
-                    query.append(wrapInQuotes(ByteBufferUtil.string(state.settings.columns.names.get(i))));
-                }
-            }
-            catch (CharacterCodingException e)
-            {
-                throw new IllegalStateException(e);
+                if (i > 0)
+                    query.append(",");
+                query.append(wrapInQuotes(settings.columns.namestrs.get(i)));
             }
         }
 
-        query.append(" FROM ").append(wrapInQuotes(state.type.table));
-
+        query.append(" FROM ").append(wrapInQuotes(type.table));
         query.append(" WHERE KEY=?");
         return query.toString();
     }
@@ -76,10 +72,10 @@ public class CqlReader extends CqlOperation<ByteBuffer[][]>
     }
 
     @Override
-    protected CqlRunOp<ByteBuffer[][]> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, String keyid, ByteBuffer key)
+    protected CqlRunOp<ByteBuffer[][]> buildRunOp(ClientWrapper client, String query, Object queryId, List<Object> params, ByteBuffer key)
     {
-        List<ByteBuffer> expectRow = state.rowGen.isDeterministic() ? generateColumnValues(key) : null;
-        return new CqlRunOpMatchResults(client, query, queryId, params, keyid, key, Arrays.asList(expectRow));
+        List<ByteBuffer> expectRow = getColumnValues();
+        return new CqlRunOpMatchResults(client, query, queryId, params, key, Arrays.asList(expectRow));
     }
 
 }

tools/stress/src/org/apache/cassandra/stress/operations/predefined/PredefinedOperation.java

@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.stress.operations.predefined;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.apache.cassandra.stress.Operation;
+import org.apache.cassandra.stress.StressMetrics;
+import org.apache.cassandra.stress.generate.Distribution;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.DistributionFixed;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.generate.Row;
+import org.apache.cassandra.stress.settings.Command;
+import org.apache.cassandra.stress.settings.CqlVersion;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.util.Timer;
+import org.apache.cassandra.thrift.ColumnParent;
+import org.apache.cassandra.thrift.SlicePredicate;
+import org.apache.cassandra.thrift.SliceRange;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public abstract class PredefinedOperation extends Operation
+{
+    public final Command type;
+    private final Distribution columnCount;
+    private Object cqlCache;
+
+    public PredefinedOperation(Command type, Timer timer, PartitionGenerator generator, StressSettings settings)
+    {
+        super(timer, generator, settings, new DistributionFixed(1));
+        this.type = type;
+        this.columnCount = settings.columns.countDistribution.get();
+    }
+
+    public boolean isCql3()
+    {
+        return settings.mode.cqlVersion == CqlVersion.CQL3;
+    }
+    public Object getCqlCache()
+    {
+        return cqlCache;
+    }
+    public void storeCqlCache(Object val)
+    {
+        cqlCache = val;
+    }
+
+    protected ByteBuffer getKey()
+    {
+        return (ByteBuffer) partitions.get(0).getPartitionKey(0);
+    }
+
+    final class ColumnSelection
+    {
+        final int[] indices;
+        final int lb, ub;
+        private ColumnSelection(int[] indices, int lb, int ub)
+        {
+            this.indices = indices;
+            this.lb = lb;
+            this.ub = ub;
+        }
+
+        public <V> List<V> select(List<V> in)
+        {
+            List<V> out = new ArrayList<>();
+            if (indices != null)
+            {
+                for (int i : indices)
+                    out.add(in.get(i));
+            }
+            else
+            {
+                out.addAll(in.subList(lb, ub));
+            }
+            return out;
+        }
+
+        int count()
+        {
+            return indices != null ? indices.length : ub - lb;
+        }
+
+        SlicePredicate predicate()
+        {
+            final SlicePredicate predicate = new SlicePredicate();
+            if (indices == null)
+            {
+                predicate.setSlice_range(new SliceRange()
+                                         .setStart(settings.columns.names.get(lb))
+                                         .setFinish(new byte[] {})
+                                         .setReversed(false)
+                                         .setCount(count())
+                );
+            }
+            else
+                predicate.setColumn_names(select(settings.columns.names));
+            return predicate;
+
+        }
+    }
+
+    public String toString()
+    {
+        return type.toString();
+    }
+
+    ColumnSelection select()
+    {
+        if (settings.columns.slice)
+        {
+            int count = (int) columnCount.next();
+            int start;
+            if (count == settings.columns.maxColumnsPerKey)
+                start = 0;
+            else
+                start = 1 + ThreadLocalRandom.current().nextInt(settings.columns.maxColumnsPerKey - count);
+            return new ColumnSelection(null, start, start + count);
+        }
+
+        int count = (int) columnCount.next();
+        int totalCount = settings.columns.names.size();
+        if (count == settings.columns.names.size())
+            return new ColumnSelection(null, 0, count);
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+        int[] indices = new int[count];
+        int c = 0, o = 0;
+        while (c < count && count + o < totalCount)
+        {
+            int leeway = totalCount - (count + o);
+            int spreadover = count - c;
+            o += Math.round(rnd.nextDouble() * (leeway / (double) spreadover));
+            indices[c] = o + c;
+            c++;
+        }
+        while (c < count)
+        {
+            indices[c] = o + c;
+            c++;
+        }
+        return new ColumnSelection(indices, 0, 0);
+    }
+
+    protected List<ByteBuffer> getColumnValues()
+    {
+        return getColumnValues(new ColumnSelection(null, 0, settings.columns.names.size()));
+    }
+
+    protected List<ByteBuffer> getColumnValues(ColumnSelection columns)
+    {
+        Row row = partitions.get(0).iterator(1).batch(1f).iterator().next();
+        ByteBuffer[] r = new ByteBuffer[columns.count()];
+        int c = 0;
+        if (columns.indices != null)
+            for (int i : columns.indices)
+                r[c++] = (ByteBuffer) row.get(i);
+        else
+            for (int i = columns.lb ; i < columns.ub ; i++)
+                r[c++] = (ByteBuffer) row.get(i);
+        return Arrays.asList(r);
+    }
+
+    public static Operation operation(Command type, Timer timer, PartitionGenerator generator, StressSettings settings, DistributionFactory counteradd)
+    {
+        switch (type)
+        {
+            case READ:
+                switch(settings.mode.style)
+                {
+                    case THRIFT:
+                        return new ThriftReader(timer, generator, settings);
+                    case CQL:
+                    case CQL_PREPARED:
+                        return new CqlReader(timer, generator, settings);
+                    default:
+                        throw new UnsupportedOperationException();
+                }
+
+
+            case COUNTER_READ:
+                switch(settings.mode.style)
+                {
+                    case THRIFT:
+                        return new ThriftCounterGetter(timer, generator, settings);
+                    case CQL:
+                    case CQL_PREPARED:
+                        return new CqlCounterGetter(timer, generator, settings);
+                    default:
+                        throw new UnsupportedOperationException();
+                }
+
+            case WRITE:
+
+                switch(settings.mode.style)
+                {
+                    case THRIFT:
+                        return new ThriftInserter(timer, generator, settings);
+                    case CQL:
+                    case CQL_PREPARED:
+                        return new CqlInserter(timer, generator, settings);
+                    default:
+                        throw new UnsupportedOperationException();
+                }
+
+            case COUNTER_WRITE:
+                switch(settings.mode.style)
+                {
+                    case THRIFT:
+                        return new ThriftCounterAdder(counteradd, timer, generator, settings);
+                    case CQL:
+                    case CQL_PREPARED:
+                        return new CqlCounterAdder(counteradd, timer, generator, settings);
+                    default:
+                        throw new UnsupportedOperationException();
+                }
+
+        }
+
+        throw new UnsupportedOperationException();
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterAdder.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.stress.operations;
+package org.apache.cassandra.stress.operations.predefined;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -23,48 +23,39 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
 
-import org.apache.cassandra.stress.Operation;
+import org.apache.cassandra.stress.generate.Distribution;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.Command;
+import org.apache.cassandra.stress.settings.StressSettings;
 import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
 import org.apache.cassandra.thrift.*;
 
-public class ThriftCounterAdder extends Operation
+public class ThriftCounterAdder extends PredefinedOperation
 {
-    public ThriftCounterAdder(State state, long index)
+
+    final Distribution counteradd;
+    public ThriftCounterAdder(DistributionFactory counteradd, Timer timer, PartitionGenerator generator, StressSettings settings)
     {
-        super(state, index);
+        super(Command.COUNTER_WRITE, timer, generator, settings);
+        this.counteradd = counteradd.get();
     }
 
     public void run(final ThriftClient client) throws IOException
     {
         List<CounterColumn> columns = new ArrayList<>();
-        for (ByteBuffer name : randomNames())
-            columns.add(new CounterColumn(name, state.counteradd.next()));
+        for (ByteBuffer name : select().select(settings.columns.names))
+            columns.add(new CounterColumn(name, counteradd.next()));
 
-        Map<String, List<Mutation>> row;
-        if (state.settings.columns.useSuperColumns)
+        List<Mutation> mutations = new ArrayList<>(columns.size());
+        for (CounterColumn c : columns)
         {
-            List<Mutation> mutations = new ArrayList<>();
-            for (ColumnParent parent : state.columnParents)
-            {
-                CounterSuperColumn csc = new CounterSuperColumn(ByteBuffer.wrap(parent.getSuper_column()), columns);
-                ColumnOrSuperColumn cosc = new ColumnOrSuperColumn().setCounter_super_column(csc);
-                mutations.add(new Mutation().setColumn_or_supercolumn(cosc));
-            }
-            row = Collections.singletonMap(state.type.supertable, mutations);
-        }
-        else
-        {
-            List<Mutation> mutations = new ArrayList<>(columns.size());
-            for (CounterColumn c : columns)
-            {
-                ColumnOrSuperColumn cosc = new ColumnOrSuperColumn().setCounter_column(c);
-                mutations.add(new Mutation().setColumn_or_supercolumn(cosc));
-            }
-            row = Collections.singletonMap(state.type.table, mutations);
+            ColumnOrSuperColumn cosc = new ColumnOrSuperColumn().setCounter_column(c);
+            mutations.add(new Mutation().setColumn_or_supercolumn(cosc));
         }
+        Map<String, List<Mutation>> row = Collections.singletonMap(type.table, mutations);
 
         final ByteBuffer key = getKey();
         final Map<ByteBuffer, Map<String, List<Mutation>>> record = Collections.singletonMap(key, row);
@@ -74,18 +65,18 @@ public class ThriftCounterAdder extends Operation
             @Override
             public boolean run() throws Exception
             {
-                client.batch_mutate(record, state.settings.command.consistencyLevel);
+                client.batch_mutate(record, settings.command.consistencyLevel);
                 return true;
             }
 
             @Override
-            public String key()
+            public int partitionCount()
             {
-                return new String(key.array());
+                return 1;
             }
 
             @Override
-            public int keyCount()
+            public int rowCount()
             {
                 return 1;
             }

tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterGetter.java

@@ -15,54 +15,52 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.stress.operations;
+package org.apache.cassandra.stress.operations.predefined;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
 
-import org.apache.cassandra.stress.Operation;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.Command;
+import org.apache.cassandra.stress.settings.StressSettings;
 import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
 import org.apache.cassandra.thrift.ColumnParent;
 import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.thrift.SliceRange;
 
-public class ThriftCounterGetter extends Operation
+public class ThriftCounterGetter extends PredefinedOperation
 {
-    public ThriftCounterGetter(State state, long index)
+    public ThriftCounterGetter(Timer timer, PartitionGenerator generator, StressSettings settings)
     {
-        super(state, index);
+        super(Command.COUNTER_READ, timer, generator, settings);
     }
 
     public void run(final ThriftClient client) throws IOException
     {
-        final SlicePredicate predicate = slicePredicate();
+        final SlicePredicate predicate = select().predicate();
         final ByteBuffer key = getKey();
-        for (final ColumnParent parent : state.columnParents)
+        timeWithRetry(new RunOp()
         {
-
-            timeWithRetry(new RunOp()
+            @Override
+            public boolean run() throws Exception
             {
-                @Override
-                public boolean run() throws Exception
-                {
-                    List<?> r = client.get_slice(key, parent, predicate, state.settings.command.consistencyLevel);
-                    return r != null && r.size() > 0;
-                }
+                List<?> r = client.get_slice(key, new ColumnParent(type.table), predicate, settings.command.consistencyLevel);
+                return r != null && r.size() > 0;
+            }
 
-                @Override
-                public String key()
-                {
-                    return new String(key.array());
-                }
+            @Override
+            public int partitionCount()
+            {
+                return 1;
+            }
 
-                @Override
-                public int keyCount()
-                {
-                    return 1;
-                }
-            });
-        }
+            @Override
+            public int rowCount()
+            {
+                return 1;
+            }
+        });
     }
 
 }

tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftInserter.java

@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.stress.operations.predefined;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cassandra.db.marshal.TimeUUIDType;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.Command;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
+import org.apache.cassandra.thrift.*;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+public final class ThriftInserter extends PredefinedOperation
+{
+
+    public ThriftInserter(Timer timer, PartitionGenerator generator, StressSettings settings)
+    {
+        super(Command.WRITE, timer, generator, settings);
+    }
+
+    public void run(final ThriftClient client) throws IOException
+    {
+        final ByteBuffer key = getKey();
+        final List<Column> columns = getColumns();
+
+        List<Mutation> mutations = new ArrayList<>(columns.size());
+        for (Column c : columns)
+        {
+            ColumnOrSuperColumn column = new ColumnOrSuperColumn().setColumn(c);
+            mutations.add(new Mutation().setColumn_or_supercolumn(column));
+        }
+        Map<String, List<Mutation>> row = Collections.singletonMap(type.table, mutations);
+
+        final Map<ByteBuffer, Map<String, List<Mutation>>> record = Collections.singletonMap(key, row);
+
+        timeWithRetry(new RunOp()
+        {
+            @Override
+            public boolean run() throws Exception
+            {
+                client.batch_mutate(record, settings.command.consistencyLevel);
+                return true;
+            }
+
+            @Override
+            public int partitionCount()
+            {
+                return 1;
+            }
+
+            @Override
+            public int rowCount()
+            {
+                return 1;
+            }
+        });
+    }
+
+    protected List<Column> getColumns()
+    {
+        final ColumnSelection selection = select();
+        final List<ByteBuffer> values = getColumnValues(selection);
+        final List<Column> columns = new ArrayList<>(values.size());
+        final List<ByteBuffer> names = select().select(settings.columns.names);
+        for (int i = 0 ; i < values.size() ; i++)
+            columns.add(new Column(names.get(i))
+                        .setValue(values.get(i))
+                        .setTimestamp(settings.columns.timestamp != null
+                                      ? Long.parseLong(settings.columns.timestamp)
+                                      : FBUtilities.timestampMicros()));
+        return columns;
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftReader.java

@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.stress.operations.predefined;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.Command;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
+import org.apache.cassandra.thrift.ColumnOrSuperColumn;
+import org.apache.cassandra.thrift.ColumnParent;
+import org.apache.cassandra.thrift.SlicePredicate;
+import org.apache.cassandra.thrift.SuperColumn;
+
+public final class ThriftReader extends PredefinedOperation
+{
+
+    public ThriftReader(Timer timer, PartitionGenerator generator, StressSettings settings)
+    {
+        super(Command.READ, timer, generator, settings);
+    }
+
+    public void run(final ThriftClient client) throws IOException
+    {
+        final ColumnSelection select = select();
+        final ByteBuffer key = getKey();
+        final List<ByteBuffer> expect = getColumnValues(select);
+        timeWithRetry(new RunOp()
+        {
+            @Override
+            public boolean run() throws Exception
+            {
+                List<ColumnOrSuperColumn> row = client.get_slice(key, new ColumnParent(type.table), select.predicate(), settings.command.consistencyLevel);
+                if (expect == null)
+                    return !row.isEmpty();
+                if (row == null)
+                    return false;
+                if (row.size() != expect.size())
+                    return false;
+                for (int i = 0 ; i < row.size() ; i++)
+                    if (!row.get(i).getColumn().bufferForValue().equals(expect.get(i)))
+                        return false;
+                return true;
+            }
+
+            @Override
+            public int partitionCount()
+            {
+                return 1;
+            }
+
+            @Override
+            public int rowCount()
+            {
+                return 1;
+            }
+        });
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaInsert.java

@@ -0,0 +1,144 @@
+package org.apache.cassandra.stress.operations.userdefined;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.datastax.driver.core.BatchStatement;
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.Statement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.stress.generate.Distribution;
+import org.apache.cassandra.stress.generate.Partition;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.generate.RatioDistribution;
+import org.apache.cassandra.stress.generate.Row;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.settings.ValidationType;
+import org.apache.cassandra.stress.util.JavaDriverClient;
+import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
+
+public class SchemaInsert extends SchemaStatement
+{
+
+    private final BatchStatement.Type batchType;
+    private final RatioDistribution perVisit;
+    private final RatioDistribution perBatch;
+
+    public SchemaInsert(Timer timer, PartitionGenerator generator, StressSettings settings, Distribution partitionCount, RatioDistribution perVisit, RatioDistribution perBatch, Integer thriftId, PreparedStatement statement, ConsistencyLevel cl, BatchStatement.Type batchType)
+    {
+        super(timer, generator, settings, partitionCount, statement, thriftId, cl, ValidationType.NOT_FAIL);
+        this.batchType = batchType;
+        this.perVisit = perVisit;
+        this.perBatch = perBatch;
+    }
+
+    private class JavaDriverRun extends Runner
+    {
+        final JavaDriverClient client;
+
+        private JavaDriverRun(JavaDriverClient client)
+        {
+            this.client = client;
+        }
+
+        public boolean run() throws Exception
+        {
+            Partition.RowIterator[] iterators = new Partition.RowIterator[partitions.size()];
+            for (int i = 0 ; i < iterators.length ; i++)
+                iterators[i] = partitions.get(i).iterator(perVisit.next());
+            List<BoundStatement> stmts = new ArrayList<>();
+            partitionCount = partitions.size();
+
+            boolean done;
+            do
+            {
+                done = true;
+                stmts.clear();
+                for (Partition.RowIterator iterator : iterators)
+                {
+                    if (iterator.done())
+                        continue;
+
+                    for (Row row : iterator.batch(perBatch.next()))
+                        stmts.add(bindRow(row));
+
+                    done &= iterator.done();
+                }
+
+                rowCount += stmts.size();
+
+                Statement stmt;
+                if (stmts.size() == 1)
+                {
+                    stmt = stmts.get(0);
+                }
+                else
+                {
+                    BatchStatement batch = new BatchStatement(batchType);
+                    batch.setConsistencyLevel(JavaDriverClient.from(cl));
+                    batch.addAll(stmts);
+                    stmt = batch;
+                }
+                validate(client.getSession().execute(stmt));
+
+            } while (!done);
+
+            return true;
+        }
+    }
+
+    private class ThriftRun extends Runner
+    {
+        final ThriftClient client;
+
+        private ThriftRun(ThriftClient client)
+        {
+            this.client = client;
+        }
+
+        public boolean run() throws Exception
+        {
+            Partition.RowIterator[] iterators = new Partition.RowIterator[partitions.size()];
+            for (int i = 0 ; i < iterators.length ; i++)
+                iterators[i] = partitions.get(i).iterator(perVisit.next());
+            partitionCount = partitions.size();
+
+            boolean done;
+            do
+            {
+                done = true;
+                for (Partition.RowIterator iterator : iterators)
+                {
+                    if (iterator.done())
+                        continue;
+
+                    for (Row row : iterator.batch(perBatch.next()))
+                    {
+                        validate(client.execute_prepared_cql3_query(thriftId, iterator.partition().getToken(), thriftRowArgs(row), settings.command.consistencyLevel));
+                        rowCount += 1;
+                    }
+
+                    done &= iterator.done();
+                }
+            } while (!done);
+
+            return true;
+        }
+    }
+
+    @Override
+    public void run(JavaDriverClient client) throws IOException
+    {
+        timeWithRetry(new JavaDriverRun(client));
+    }
+
+    @Override
+    public void run(ThriftClient client) throws IOException
+    {
+        timeWithRetry(new ThriftRun(client));
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaQuery.java

@@ -0,0 +1,86 @@
+package org.apache.cassandra.stress.operations.userdefined;
+
+import java.io.IOException;
+
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.settings.OptionDistribution;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.settings.ValidationType;
+import org.apache.cassandra.stress.util.JavaDriverClient;
+import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
+import org.apache.cassandra.thrift.CqlResult;
+import org.apache.cassandra.thrift.ThriftConversion;
+
+public class SchemaQuery extends SchemaStatement
+{
+
+    public SchemaQuery(Timer timer, PartitionGenerator generator, StressSettings settings, Integer thriftId, PreparedStatement statement, ConsistencyLevel cl, ValidationType validationType)
+    {
+        super(timer, generator, settings, OptionDistribution.get("fixed(1)").get(), statement, thriftId, cl, validationType);
+    }
+
+    int execute(JavaDriverClient client) throws Exception
+    {
+        return client.getSession().execute(bindRandom(partitions.get(0))).all().size();
+    }
+
+    int execute(ThriftClient client) throws Exception
+    {
+        return client.execute_prepared_cql3_query(thriftId, partitions.get(0).getToken(), thriftRandomArgs(partitions.get(0)), ThriftConversion.toThrift(cl)).getRowsSize();
+    }
+
+    private class JavaDriverRun extends Runner
+    {
+        final JavaDriverClient client;
+
+        private JavaDriverRun(JavaDriverClient client)
+        {
+            this.client = client;
+        }
+
+        public boolean run() throws Exception
+        {
+            ResultSet rs = client.getSession().execute(bindRandom(partitions.get(0)));
+            validate(rs);
+            rowCount = rs.all().size();
+            partitionCount = Math.min(1, rowCount);
+            return true;
+        }
+    }
+
+    private class ThriftRun extends Runner
+    {
+        final ThriftClient client;
+
+        private ThriftRun(ThriftClient client)
+        {
+            this.client = client;
+        }
+
+        public boolean run() throws Exception
+        {
+            CqlResult rs = client.execute_prepared_cql3_query(thriftId, partitions.get(0).getToken(), thriftRandomArgs(partitions.get(0)), ThriftConversion.toThrift(cl));
+            validate(rs);
+            rowCount = rs.getRowsSize();
+            partitionCount = Math.min(1, rowCount);
+            return true;
+        }
+    }
+
+    @Override
+    public void run(JavaDriverClient client) throws IOException
+    {
+        timeWithRetry(new JavaDriverRun(client));
+    }
+
+    @Override
+    public void run(ThriftClient client) throws IOException
+    {
+        timeWithRetry(new ThriftRun(client));
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaStatement.java

@@ -0,0 +1,164 @@
+package org.apache.cassandra.stress.operations.userdefined;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.ColumnDefinitions;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.stress.Operation;
+import org.apache.cassandra.stress.generate.Distribution;
+import org.apache.cassandra.stress.generate.Partition;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.generate.Row;
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.apache.cassandra.stress.settings.ValidationType;
+import org.apache.cassandra.stress.util.JavaDriverClient;
+import org.apache.cassandra.stress.util.ThriftClient;
+import org.apache.cassandra.stress.util.Timer;
+import org.apache.cassandra.thrift.CqlResult;
+import org.apache.cassandra.transport.SimpleClient;
+
+public abstract class SchemaStatement extends Operation
+{
+
+    final PartitionGenerator generator;
+    private final PreparedStatement statement;
+    final Integer thriftId;
+    final ConsistencyLevel cl;
+    final ValidationType validationType;
+    private final int[] argumentIndex;
+    private final Object[] bindBuffer;
+    private final Object[][] randomBuffer;
+    private final Random random = new Random();
+
+    public SchemaStatement(Timer timer, PartitionGenerator generator, StressSettings settings, Distribution partitionCount,
+                           PreparedStatement statement, Integer thriftId, ConsistencyLevel cl, ValidationType validationType)
+    {
+        super(timer, generator, settings, partitionCount);
+        this.generator = generator;
+        this.statement = statement;
+        this.thriftId = thriftId;
+        this.cl = cl;
+        this.validationType = validationType;
+        argumentIndex = new int[statement.getVariables().size()];
+        bindBuffer = new Object[argumentIndex.length];
+        randomBuffer = new Object[argumentIndex.length][argumentIndex.length];
+        int i = 0;
+        for (ColumnDefinitions.Definition definition : statement.getVariables())
+            argumentIndex[i++] = generator.indexOf(definition.getName());
+    }
+
+    private int filLRandom(Partition partition)
+    {
+        int c = 0;
+        for (Row row : partition.iterator(randomBuffer.length).batch(1f))
+        {
+            Object[] randomRow = randomBuffer[c++];
+            for (int i = 0 ; i < argumentIndex.length ; i++)
+                randomRow[i] = row.get(argumentIndex[i]);
+            if (c >= randomBuffer.length)
+                break;
+        }
+        return c;
+    }
+
+    BoundStatement bindRandom(Partition partition)
+    {
+        int c = filLRandom(partition);
+        for (int i = 0 ; i < argumentIndex.length ; i++)
+        {
+            int argIndex = argumentIndex[i];
+            bindBuffer[i] = randomBuffer[argIndex < 0 ? 0 : random.nextInt(c)][i];
+        }
+        return statement.bind(bindBuffer);
+    }
+
+    BoundStatement bindRow(Row row)
+    {
+        for (int i = 0 ; i < argumentIndex.length ; i++)
+            bindBuffer[i] = row.get(argumentIndex[i]);
+        return statement.bind(bindBuffer);
+    }
+
+    List<ByteBuffer> thriftRowArgs(Row row)
+    {
+        List<ByteBuffer> args = new ArrayList<>();
+        for (int i : argumentIndex)
+            args.add(generator.convert(i, row.get(i)));
+        return args;
+    }
+
+    List<ByteBuffer> thriftRandomArgs(Partition partition)
+    {
+        List<ByteBuffer> args = new ArrayList<>();
+        int c = filLRandom(partition);
+        for (int i = 0 ; i < argumentIndex.length ; i++)
+        {
+            int argIndex = argumentIndex[i];
+            args.add(generator.convert(argIndex, randomBuffer[argIndex < 0 ? 0 : random.nextInt(c)][i]));
+        }
+        return args;
+    }
+
+    void validate(ResultSet rs)
+    {
+        switch (validationType)
+        {
+            case NOT_FAIL:
+                return;
+            case NON_ZERO:
+                if (rs.all().size() == 0)
+                    throw new IllegalStateException("Expected non-zero results");
+                break;
+            default:
+                throw new IllegalStateException("Unsupported validation type");
+        }
+    }
+
+    void validate(CqlResult rs)
+    {
+        switch (validationType)
+        {
+            case NOT_FAIL:
+                return;
+            case NON_ZERO:
+                if (rs.getRowsSize() == 0)
+                    throw new IllegalStateException("Expected non-zero results");
+                break;
+            default:
+                throw new IllegalStateException("Unsupported validation type");
+        }
+    }
+
+    @Override
+    public void run(SimpleClient client) throws IOException
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    abstract class Runner implements RunOp
+    {
+        int partitionCount;
+        int rowCount;
+
+        @Override
+        public int partitionCount()
+        {
+            return partitionCount;
+        }
+
+        @Override
+        public int rowCount()
+        {
+            return rowCount;
+        }
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/settings/Command.java

@@ -41,19 +41,6 @@ public enum Command
             "Interleaving of any basic commands, with configurable ratio and distribution - the cluster must first be populated by a write test",
             CommandCategory.MIXED
     ),
-    RANGE_SLICE(false, "Standard1", "Super1",
-            "Range slice queries - the cluster must first be populated by a write test",
-            CommandCategory.MULTI
-    ),
-    INDEXED_RANGE_SLICE(false, "Standard1", "Super1",
-            "Range slice queries through a secondary index. The cluster must first be populated by a write test, with indexing enabled.",
-            CommandCategory.BASIC
-    ),
-    READ_MULTI(false, "Standard1", "Super1",
-            "multi_read",
-            "Multiple concurrent reads fetching multiple rows at once. The cluster must first be populated by a write test.",
-            CommandCategory.MULTI
-    ),
     COUNTER_WRITE(true, "Counter1", "SuperCounter1",
             "counter_add",
             "Multiple concurrent updates of counters.",
@@ -64,6 +51,10 @@ public enum Command
             "Multiple concurrent reads of counters. The cluster must first be populated by a counterwrite test.",
             CommandCategory.BASIC
     ),
+    USER(true, null, null,
+          "Interleaving of user provided queries, with configurable ratio and distribution",
+          CommandCategory.USER
+    ),
 
     HELP(false, null, null, "-?", "Print help for a command or option", null),
     PRINT(false, null, null, "Inspect the output of a distribution definition", null),
@@ -136,11 +127,12 @@ public enum Command
         }
         switch (category)
         {
+            case USER:
+                return SettingsCommandUser.helpPrinter();
             case BASIC:
-            case MULTI:
-                return SettingsCommand.helpPrinter(this);
+                return SettingsCommandPreDefined.helpPrinter(this);
             case MIXED:
-                return SettingsCommandMixed.helpPrinter();
+                return SettingsCommandPreDefinedMixed.helpPrinter();
         }
         throw new AssertionError();
     }

tools/stress/src/org/apache/cassandra/stress/settings/CommandCategory.java

@@ -24,6 +24,6 @@ package org.apache.cassandra.stress.settings;
 public enum CommandCategory
 {
     BASIC,
-    MULTI,
-    MIXED
+    MIXED,
+    USER
 }

tools/stress/src/org/apache/cassandra/stress/settings/OptionAnyProbabilities.java

@@ -0,0 +1,78 @@
+package org.apache.cassandra.stress.settings;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.math3.util.Pair;
+
+public final class OptionAnyProbabilities extends OptionMulti
+{
+    public OptionAnyProbabilities(String name, String description)
+    {
+        super(name, description, false);
+    }
+
+    final CollectRatios ratios = new CollectRatios();
+
+    private static final class CollectRatios extends Option
+    {
+        Map<String, Double> options = new LinkedHashMap<>();
+
+        boolean accept(String param)
+        {
+            String[] args = param.split("=");
+            if (args.length == 2 && args[1].length() > 0 && args[0].length() > 0)
+            {
+                if (options.put(args[0], Double.parseDouble(args[1])) != null)
+                    throw new IllegalArgumentException(args[0] + " set twice");
+                return true;
+            }
+            return false;
+        }
+
+        boolean happy()
+        {
+            return !options.isEmpty();
+        }
+
+        String shortDisplay()
+        {
+            return null;
+        }
+
+        String longDisplay()
+        {
+            return null;
+        }
+
+        List<String> multiLineDisplay()
+        {
+            return Collections.emptyList();
+        }
+
+        boolean setByUser()
+        {
+            return !options.isEmpty();
+        }
+    }
+
+
+    @Override
+    public List<? extends Option> options()
+    {
+        return Arrays.asList(ratios);
+    }
+
+    List<Pair<String, Double>> ratios()
+    {
+        List<Pair<String, Double>> ratiosOut = new ArrayList<>();
+        for (Map.Entry<String, Double> e : ratios.options.entrySet())
+            ratiosOut.add(new Pair<String, Double>(e.getKey(), e.getValue()));
+        return ratiosOut;
+    }
+}
+

tools/stress/src/org/apache/cassandra/stress/settings/OptionDataGen.java

@@ -1,203 +0,0 @@
-package org.apache.cassandra.stress.settings;
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-
-
-import java.io.File;
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.cassandra.stress.generatedata.*;
-
-/**
- * For selecting a data generator
- */
-class OptionDataGen extends Option
-{
-
-    private static final Pattern FULL = Pattern.compile("([A-Z]+)\\(([^)]+)\\)", Pattern.CASE_INSENSITIVE);
-    private static final Pattern ARGS = Pattern.compile("[^,]+");
-
-    final String prefix;
-    private DataGenFactory factory;
-    private final DataGenFactory defaultFactory;
-
-    public OptionDataGen(String prefix, String defaultSpec)
-    {
-        this.prefix = prefix;
-        this.defaultFactory = defaultSpec == null ? null : get(defaultSpec);
-    }
-
-    @Override
-    public boolean accept(String param)
-    {
-        if (!param.toLowerCase().startsWith(prefix))
-            return false;
-        factory = get(param.substring(prefix.length()));
-        return true;
-    }
-
-    private static DataGenFactory get(String spec)
-    {
-        Matcher m = FULL.matcher(spec);
-        if (!m.matches())
-            throw new IllegalArgumentException("Illegal data generator specification: " + spec);
-        String name = m.group(1);
-        Impl impl = LOOKUP.get(name.toLowerCase());
-        if (impl == null)
-            throw new IllegalArgumentException("Illegal data generator type: " + name);
-        List<String> params = new ArrayList<>();
-        m = ARGS.matcher(m.group(2));
-        while (m.find())
-            params.add(m.group());
-        return impl.getFactory(params);
-    }
-
-    public DataGenFactory get()
-    {
-        return factory != null ? factory : defaultFactory;
-    }
-
-    @Override
-    public boolean happy()
-    {
-        return factory != null || defaultFactory != null;
-    }
-
-    public boolean setByUser()
-    {
-        return factory != null;
-    }
-
-    @Override
-    public String shortDisplay()
-    {
-        return prefix + "ALG()";
-    }
-
-    public String longDisplay()
-    {
-        return shortDisplay() + ": Specify a data generator from:";
-    }
-
-    @Override
-    public List<String> multiLineDisplay()
-    {
-        return Arrays.asList(
-                GroupedOptions.formatMultiLine("RANDOM()", "Completely random byte generation"),
-                GroupedOptions.formatMultiLine("REPEAT(<freq>)", "An MD5 hash of (opIndex % freq) combined with the column index"),
-                GroupedOptions.formatMultiLine("DICT(<file>)","Random words from a dictionary; the file should be in the format \"<freq> <word>\"")
-        );
-    }
-
-    private static final Map<String, Impl> LOOKUP;
-    static
-    {
-        final Map<String, Impl> lookup = new HashMap<>();
-        lookup.put("random", new RandomImpl());
-        lookup.put("rand", new RandomImpl());
-        lookup.put("rnd", new RandomImpl());
-        lookup.put("repeat", new RepeatImpl());
-        lookup.put("dict", new DictionaryImpl());
-        lookup.put("dictionary", new DictionaryImpl());
-        LOOKUP = lookup;
-    }
-
-    private static interface Impl
-    {
-        public DataGenFactory getFactory(List<String> params);
-    }
-
-    private static final class RandomImpl implements Impl
-    {
-        @Override
-        public DataGenFactory getFactory(List<String> params)
-        {
-            if (params.size() != 0)
-                throw new IllegalArgumentException("Invalid parameter list for random generator: " + params);
-            return new RandomFactory();
-        }
-    }
-
-    private static final class RepeatImpl implements Impl
-    {
-
-        @Override
-        public DataGenFactory getFactory(List<String> params)
-        {
-            if (params.size() != 1)
-                throw new IllegalArgumentException("Invalid parameter list for repeating generator: " + params);
-            try
-            {
-                int repeatFrequency = Integer.parseInt(params.get(0));
-                return new RepeatsFactory(repeatFrequency);
-            } catch (Exception _)
-            {
-                throw new IllegalArgumentException("Invalid parameter list for repeating generator: " + params);
-            }
-        }
-    }
-
-    private static final class DictionaryImpl implements Impl
-    {
-
-        @Override
-        public DataGenFactory getFactory(List<String> params)
-        {
-            if (params.size() != 1)
-                throw new IllegalArgumentException("Invalid parameter list for dictionary generator: " + params);
-            try
-            {
-                final File file = new File(params.get(0));
-                return DataGenStringDictionary.getFactory(file);
-            } catch (Exception e)
-            {
-                throw new IllegalArgumentException("Invalid parameter list for dictionary generator: " + params, e);
-            }
-        }
-    }
-
-    private static final class RandomFactory implements DataGenFactory
-    {
-        @Override
-        public DataGen get()
-        {
-            return new DataGenBytesRandom();
-        }
-    }
-
-    private static final class RepeatsFactory implements DataGenFactory
-    {
-        final int frequency;
-        private RepeatsFactory(int frequency)
-        {
-            this.frequency = frequency;
-        }
-
-        @Override
-        public DataGen get()
-        {
-            return new DataGenStringRepeats(frequency);
-        }
-    }
-
-}

tools/stress/src/org/apache/cassandra/stress/settings/OptionDistribution.java

@@ -25,19 +25,20 @@ import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.cassandra.stress.generatedata.*;
+import org.apache.cassandra.stress.generate.*;
 import org.apache.commons.math3.distribution.ExponentialDistribution;
 import org.apache.commons.math3.distribution.NormalDistribution;
 import org.apache.commons.math3.distribution.UniformRealDistribution;
 import org.apache.commons.math3.distribution.WeibullDistribution;
+import org.apache.commons.math3.random.JDKRandomGenerator;
 
 /**
  * For selecting a mathematical distribution
  */
-class OptionDistribution extends Option
+public class OptionDistribution extends Option
 {
 
-    private static final Pattern FULL = Pattern.compile("([A-Z]+)\\((.+)\\)", Pattern.CASE_INSENSITIVE);
+    private static final Pattern FULL = Pattern.compile("(~?)([A-Z]+)\\((.+)\\)", Pattern.CASE_INSENSITIVE);
     private static final Pattern ARGS = Pattern.compile("[^,]+");
 
     final String prefix;
@@ -61,20 +62,22 @@ class OptionDistribution extends Option
         return true;
     }
 
-    private static DistributionFactory get(String spec)
+    public static DistributionFactory get(String spec)
     {
         Matcher m = FULL.matcher(spec);
         if (!m.matches())
             throw new IllegalArgumentException("Illegal distribution specification: " + spec);
-        String name = m.group(1);
+        boolean inverse = m.group(1).equals("~");
+        String name = m.group(2);
         Impl impl = LOOKUP.get(name.toLowerCase());
         if (impl == null)
             throw new IllegalArgumentException("Illegal distribution type: " + name);
         List<String> params = new ArrayList<>();
-        m = ARGS.matcher(m.group(2));
+        m = ARGS.matcher(m.group(3));
         while (m.find())
             params.add(m.group());
-        return impl.getFactory(params);
+        DistributionFactory factory = impl.getFactory(params);
+        return inverse ? new InverseFactory(factory) : factory;
     }
 
     public DistributionFactory get()
@@ -103,6 +106,7 @@ class OptionDistribution extends Option
                 GroupedOptions.formatMultiLine("GAUSSIAN(min..max,mean,stdev)", "A gaussian/normal distribution, with explicitly defined mean and stdev"),
                 GroupedOptions.formatMultiLine("UNIFORM(min..max)", "A uniform distribution over the range [min, max]"),
                 GroupedOptions.formatMultiLine("FIXED(val)", "A fixed distribution, always returning the same value"),
+                "Preceding the name with ~ will invert the distribution, e.g. ~exp(1..10) will yield 10 most, instead of least, often",
                 "Aliases: extr, gauss, normal, norm, weibull"
         );
     }
@@ -142,6 +146,23 @@ class OptionDistribution extends Option
         public DistributionFactory getFactory(List<String> params);
     }
 
+    public static long parseLong(String value)
+    {
+        long multiplier = 1;
+        value = value.trim().toLowerCase();
+        switch (value.charAt(value.length() - 1))
+        {
+            case 'b':
+                multiplier *= 1000;
+            case 'm':
+                multiplier *= 1000;
+            case 'k':
+                multiplier *= 1000;
+                value = value.substring(0, value.length() - 1);
+        }
+        return Long.parseLong(value) * multiplier;
+    }
+
     private static final class GaussianImpl implements Impl
     {
 
@@ -153,8 +174,8 @@ class OptionDistribution extends Option
             try
             {
                 String[] bounds = params.get(0).split("\\.\\.+");
-                final long min = Long.parseLong(bounds[0]);
-                final long max = Long.parseLong(bounds[1]);
+                final long min = parseLong(bounds[0]);
+                final long max = parseLong(bounds[1]);
                 final double mean, stdev;
                 if (params.size() == 3)
                 {
@@ -185,8 +206,8 @@ class OptionDistribution extends Option
             try
             {
                 String[] bounds = params.get(0).split("\\.\\.+");
-                final long min = Long.parseLong(bounds[0]);
-                final long max = Long.parseLong(bounds[1]);
+                final long min = parseLong(bounds[0]);
+                final long max = parseLong(bounds[1]);
                 ExponentialDistribution findBounds = new ExponentialDistribution(1d);
                 // max probability should be roughly equal to accuracy of (max-min) to ensure all values are visitable,
                 // over entire range, but this results in overly skewed distribution, so take sqrt
@@ -209,8 +230,8 @@ class OptionDistribution extends Option
             try
             {
                 String[] bounds = params.get(0).split("\\.\\.+");
-                final long min = Long.parseLong(bounds[0]);
-                final long max = Long.parseLong(bounds[1]);
+                final long min = parseLong(bounds[0]);
+                final long max = parseLong(bounds[1]);
                 final double shape = Double.parseDouble(params.get(1));
                 WeibullDistribution findBounds = new WeibullDistribution(shape, 1d);
                 // max probability should be roughly equal to accuracy of (max-min) to ensure all values are visitable,
@@ -235,8 +256,8 @@ class OptionDistribution extends Option
             try
             {
                 String[] bounds = params.get(0).split("\\.\\.+");
-                final long min = Long.parseLong(bounds[0]);
-                final long max = Long.parseLong(bounds[1]);
+                final long min = parseLong(bounds[0]);
+                final long max = parseLong(bounds[1]);
                 return new UniformFactory(min, max);
             } catch (Exception _)
             {
@@ -255,7 +276,7 @@ class OptionDistribution extends Option
                 throw new IllegalArgumentException("Invalid parameter list for uniform distribution: " + params);
             try
             {
-                final long key = Long.parseLong(params.get(0));
+                final long key = parseLong(params.get(0));
                 return new FixedFactory(key);
             } catch (Exception _)
             {
@@ -264,6 +285,20 @@ class OptionDistribution extends Option
         }
     }
 
+    private static final class InverseFactory implements DistributionFactory
+    {
+        final DistributionFactory wrapped;
+        private InverseFactory(DistributionFactory wrapped)
+        {
+            this.wrapped = wrapped;
+        }
+
+        public Distribution get()
+        {
+            return new DistributionInverted(wrapped.get());
+        }
+    }
+
     // factories
 
     private static final class ExpFactory implements DistributionFactory
@@ -280,7 +315,7 @@ class OptionDistribution extends Option
         @Override
         public Distribution get()
         {
-            return new DistributionOffsetApache(new ExponentialDistribution(mean), min, max);
+            return new DistributionOffsetApache(new ExponentialDistribution(new JDKRandomGenerator(), mean, ExponentialDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY), min, max);
         }
     }
 
@@ -299,7 +334,7 @@ class OptionDistribution extends Option
         @Override
         public Distribution get()
         {
-            return new DistributionOffsetApache(new WeibullDistribution(shape, scale), min, max);
+            return new DistributionOffsetApache(new WeibullDistribution(new JDKRandomGenerator(), shape, scale, WeibullDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY), min, max);
         }
     }
 
@@ -318,7 +353,7 @@ class OptionDistribution extends Option
         @Override
         public Distribution get()
         {
-            return new DistributionBoundApache(new NormalDistribution(mean, stdev), min, max);
+            return new DistributionBoundApache(new NormalDistribution(new JDKRandomGenerator(), mean, stdev, NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY), min, max);
         }
     }
 
@@ -334,7 +369,7 @@ class OptionDistribution extends Option
         @Override
         public Distribution get()
         {
-            return new DistributionBoundApache(new UniformRealDistribution(min, max + 1), min, max);
+            return new DistributionBoundApache(new UniformRealDistribution(new JDKRandomGenerator(), min, max + 1), min, max);
         }
     }
 

tools/stress/src/org/apache/cassandra/stress/settings/OptionEnumProbabilities.java

@@ -0,0 +1,62 @@
+package org.apache.cassandra.stress.settings;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.math3.util.Pair;
+
+public final class OptionEnumProbabilities<T> extends OptionMulti
+{
+    final List<OptMatcher<T>> options;
+
+    public static class Opt<T>
+    {
+        final T option;
+        final String defaultValue;
+
+        public Opt(T option, String defaultValue)
+        {
+            this.option = option;
+            this.defaultValue = defaultValue;
+        }
+    }
+
+    private static final class OptMatcher<T> extends OptionSimple
+    {
+        final T opt;
+        OptMatcher(T opt, String defaultValue)
+        {
+            super(opt.toString().toLowerCase() + "=", "[0-9]+(\\.[0-9]+)?", defaultValue, "Performs this many " + opt + " operations out of total", false);
+            this.opt = opt;
+        }
+    }
+
+    public OptionEnumProbabilities(List<Opt<T>> universe, String name, String description)
+    {
+        super(name, description, false);
+        List<OptMatcher<T>> options = new ArrayList<>();
+        for (Opt<T> option : universe)
+            options.add(new OptMatcher<T>(option.option, option.defaultValue));
+        this.options = options;
+    }
+
+    @Override
+    public List<? extends Option> options()
+    {
+        return options;
+    }
+
+    List<Pair<T, Double>> ratios()
+    {
+        List<? extends Option> ratiosIn = setByUser() ? optionsSetByUser() : defaultOptions();
+        List<Pair<T, Double>> ratiosOut = new ArrayList<>();
+        for (Option opt : ratiosIn)
+        {
+            OptMatcher<T> optMatcher = (OptMatcher<T>) opt;
+            double d = Double.parseDouble(optMatcher.value());
+            ratiosOut.add(new Pair<>(optMatcher.opt, d));
+        }
+        return ratiosOut;
+    }
+}
+

tools/stress/src/org/apache/cassandra/stress/settings/OptionMulti.java

@@ -156,7 +156,8 @@ abstract class OptionMulti extends Option
             String[] args = param.split("=");
             if (args.length == 2 && args[1].length() > 0 && args[0].length() > 0)
             {
-                options.put(args[0], args[1]);
+                if (options.put(args[0], args[1]) != null)
+                    throw new IllegalArgumentException(args[0] + " set twice");
                 return true;
             }
             return false;

tools/stress/src/org/apache/cassandra/stress/settings/OptionRatioDistribution.java

@@ -0,0 +1,166 @@
+package org.apache.cassandra.stress.settings;
+/*
+ * 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * 
+ */
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.math3.distribution.ExponentialDistribution;
+import org.apache.commons.math3.distribution.NormalDistribution;
+import org.apache.commons.math3.distribution.UniformRealDistribution;
+import org.apache.commons.math3.distribution.WeibullDistribution;
+import org.apache.commons.math3.random.JDKRandomGenerator;
+
+import org.apache.cassandra.stress.generate.Distribution;
+import org.apache.cassandra.stress.generate.DistributionBoundApache;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.DistributionFixed;
+import org.apache.cassandra.stress.generate.DistributionInverted;
+import org.apache.cassandra.stress.generate.DistributionOffsetApache;
+import org.apache.cassandra.stress.generate.RatioDistribution;
+import org.apache.cassandra.stress.generate.RatioDistributionFactory;
+
+/**
+ * For selecting a mathematical distribution
+ */
+public class OptionRatioDistribution extends Option
+{
+
+    private static final Pattern FULL = Pattern.compile("(.*)/([0-9]+[KMB]?)", Pattern.CASE_INSENSITIVE);
+
+    final OptionDistribution delegate;
+    private double divisor;
+
+    private static final RatioDistribution DEFAULT = new RatioDistribution(new DistributionFixed(1), 1);
+
+    public OptionRatioDistribution(String prefix, String defaultSpec, String description)
+    {
+        delegate = new OptionDistribution(prefix, defaultSpec, description);
+    }
+
+    @Override
+    public boolean accept(String param)
+    {
+        Matcher m = FULL.matcher(param);
+        if (!m.matches() || !delegate.accept(m.group(1)))
+            return false;
+        divisor = OptionDistribution.parseLong(m.group(2));
+        return true;
+    }
+
+    public static RatioDistributionFactory get(String spec)
+    {
+        OptionRatioDistribution opt = new OptionRatioDistribution("", "", "");
+        if (!opt.accept(spec))
+            throw new IllegalArgumentException();
+        return opt.get();
+    }
+
+    public RatioDistributionFactory get()
+    {
+        return !delegate.setByUser() ? new DefaultFactory() : new DelegateFactory(delegate.get(), divisor);
+    }
+
+    @Override
+    public boolean happy()
+    {
+        return delegate.happy();
+    }
+
+    public String longDisplay()
+    {
+        return delegate.longDisplay();
+    }
+
+    @Override
+    public List<String> multiLineDisplay()
+    {
+        return Arrays.asList(
+                GroupedOptions.formatMultiLine("EXP(min..max)/divisor", "An exponential ratio distribution over the range [min..max]/divisor"),
+                GroupedOptions.formatMultiLine("EXTREME(min..max,shape)/divisor", "An extreme value (Weibull) ratio distribution over the range [min..max]/divisor"),
+                GroupedOptions.formatMultiLine("GAUSSIAN(min..max,stdvrng)/divisor", "A gaussian/normal ratio distribution, where mean=(min+max)/2, and stdev is ((mean-min)/stdvrng)/divisor"),
+                GroupedOptions.formatMultiLine("GAUSSIAN(min..max,mean,stdev)/divisor", "A gaussian/normal ratio distribution, with explicitly defined mean and stdev"),
+                GroupedOptions.formatMultiLine("UNIFORM(min..max)/divisor", "A uniform ratio distribution over the range [min, max]/divisor"),
+                GroupedOptions.formatMultiLine("FIXED(val)/divisor", "A fixed ratio distribution, always returning the same value"),
+                "Preceding the name with ~ will invert the distribution, e.g. ~exp(1..10)/10 will yield 0.1 least, instead of most, often",
+                "Aliases: extr, gauss, normal, norm, weibull"
+        );
+    }
+
+    boolean setByUser()
+    {
+        return delegate.setByUser();
+    }
+
+    @Override
+    public String shortDisplay()
+    {
+        return delegate.shortDisplay();
+    }
+
+    // factories
+
+    private static final class DefaultFactory implements RatioDistributionFactory
+    {
+        @Override
+        public RatioDistribution get()
+        {
+            return DEFAULT;
+        }
+    }
+
+    private static final class DelegateFactory implements RatioDistributionFactory
+    {
+        final DistributionFactory delegate;
+        final double divisor;
+
+        private DelegateFactory(DistributionFactory delegate, double divisor)
+        {
+            this.delegate = delegate;
+            this.divisor = divisor;
+        }
+
+        @Override
+        public RatioDistribution get()
+        {
+            return new RatioDistribution(delegate.get(), divisor);
+        }
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return delegate.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object that)
+    {
+        return super.equals(that) && ((OptionRatioDistribution) that).delegate.equals(this.delegate);
+    }
+
+}

tools/stress/src/org/apache/cassandra/stress/settings/OptionSimple.java

@@ -21,6 +21,7 @@ package org.apache.cassandra.stress.settings;
  */
 
 
+import java.io.Serializable;
 import java.util.Collections;
 import java.util.List;
 import java.util.regex.Pattern;
@@ -30,7 +31,7 @@ import com.google.common.base.Function;
 /**
  * For parsing a simple (sub)option for a command/major option
  */
-class OptionSimple extends Option
+class OptionSimple extends Option implements Serializable
 {
 
     final String displayPrefix;
@@ -41,7 +42,7 @@ class OptionSimple extends Option
     private final boolean required;
     private String value;
 
-    private static final class ValueMatcher implements Function<String, String>
+    private static final class ValueMatcher implements Function<String, String>, Serializable
     {
         final Pattern pattern;
         private ValueMatcher(Pattern pattern)

tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java

@@ -23,13 +23,15 @@ package org.apache.cassandra.stress.settings;
 
 import java.io.Serializable;
 import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.stress.generatedata.*;
+import org.apache.cassandra.stress.generate.*;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 /**
@@ -39,18 +41,14 @@ public class SettingsColumn implements Serializable
 {
 
     public final int maxColumnsPerKey;
-    public final List<ByteBuffer> names;
+    public transient final List<ByteBuffer> names;
     public final List<String> namestrs;
     public final String comparator;
     public final String timestamp;
-    public final boolean useTimeUUIDComparator;
-    public final int superColumns;
-    public final boolean useSuperColumns;
     public final boolean variableColumnCount;
     public final boolean slice;
-    private final DistributionFactory sizeDistribution;
-    private final DistributionFactory countDistribution;
-    private final DataGenFactory dataGenFactory;
+    public final DistributionFactory sizeDistribution;
+    public final DistributionFactory countDistribution;
 
     public SettingsColumn(GroupedOptions options)
     {
@@ -63,9 +61,6 @@ public class SettingsColumn implements Serializable
     public SettingsColumn(Options options, NameOptions name, CountOptions count)
     {
         sizeDistribution = options.size.get();
-        superColumns = Integer.parseInt(options.superColumns.value());
-        dataGenFactory = options.generator.get();
-        useSuperColumns = superColumns > 0;
         {
             timestamp = options.timestamp.value();
             comparator = options.comparator.value();
@@ -81,8 +76,6 @@ public class SettingsColumn implements Serializable
                 System.exit(1);
             }
 
-            useTimeUUIDComparator = parsed instanceof TimeUUIDType;
-
             if (!(parsed instanceof TimeUUIDType || parsed instanceof AsciiType || parsed instanceof UTF8Type))
             {
                 System.err.println("Currently supported types are: TimeUUIDType, AsciiType, UTF8Type.");
@@ -104,10 +97,13 @@ public class SettingsColumn implements Serializable
 
             final String[] names = name.name.value().split(",");
             this.names = new ArrayList<>(names.length);
-            this.namestrs = Arrays.asList(names);
 
             for (String columnName : names)
                 this.names.add(comparator.fromString(columnName));
+            Collections.sort(this.names, BytesType.instance);
+            this.namestrs = new ArrayList<>();
+            for (ByteBuffer columnName : this.names)
+                this.namestrs.add(comparator.getString(columnName));
 
             final int nameCount = this.names.size();
             countDistribution = new DistributionFactory()
@@ -125,23 +121,24 @@ public class SettingsColumn implements Serializable
             ByteBuffer[] names = new ByteBuffer[(int) countDistribution.get().maxValue()];
             String[] namestrs = new String[(int) countDistribution.get().maxValue()];
             for (int i = 0 ; i < names.length ; i++)
-            {
                 names[i] = ByteBufferUtil.bytes("C" + i);
-                namestrs[i] = "C" + i;
+            Arrays.sort(names, BytesType.instance);
+            try
+            {
+                for (int i = 0 ; i < names.length ; i++)
+                    namestrs[i] = ByteBufferUtil.string(names[i]);
             }
+            catch (CharacterCodingException e)
+            {
+                throw new RuntimeException(e);
+            }
+
             this.names = Arrays.asList(names);
             this.namestrs = Arrays.asList(namestrs);
         }
         maxColumnsPerKey = (int) countDistribution.get().maxValue();
         variableColumnCount = countDistribution.get().minValue() < maxColumnsPerKey;
-        // TODO: should warn that we always slice for useTimeUUIDComparator?
-        slice = options.slice.setByUser() || useTimeUUIDComparator;
-        // TODO: with useTimeUUIDCOmparator, should we still try to select a random start for reads if possible?
-    }
-
-    public RowGen newRowGen()
-    {
-        return new RowGenDistributedSize(dataGenFactory.get(), countDistribution.get(), sizeDistribution.get());
+        slice = options.slice.setByUser();
     }
 
     // Option Declarations
@@ -153,7 +150,6 @@ public class SettingsColumn implements Serializable
         final OptionSimple slice = new OptionSimple("slice", "", null, "If set, range slices will be used for reads, otherwise a names query will be", false);
         final OptionSimple timestamp = new OptionSimple("timestamp=", "[0-9]+", null, "If set, all columns will be written with the given timestamp", false);
         final OptionDistribution size = new OptionDistribution("size=", "FIXED(34)", "Cell size distribution");
-        final OptionDataGen generator = new OptionDataGen("data=", "REPEAT(50)");
     }
 
     private static final class NameOptions extends Options
@@ -163,7 +159,7 @@ public class SettingsColumn implements Serializable
         @Override
         public List<? extends Option> options()
         {
-            return Arrays.asList(name, slice, superColumns, comparator, timestamp, size, generator);
+            return Arrays.asList(name, slice, superColumns, comparator, timestamp, size);
         }
     }
 
@@ -174,7 +170,7 @@ public class SettingsColumn implements Serializable
         @Override
         public List<? extends Option> options()
         {
-            return Arrays.asList(count, slice, superColumns, comparator, timestamp, size, generator);
+            return Arrays.asList(count, slice, superColumns, comparator, timestamp, size);
         }
     }
 

tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommand.java

@@ -26,23 +26,24 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.cassandra.stress.generatedata.DistributionFactory;
+import org.apache.cassandra.stress.operations.OpDistributionFactory;
 import org.apache.cassandra.thrift.ConsistencyLevel;
 
 // Generic command settings - common to read/write/etc
-public class SettingsCommand implements Serializable
+public abstract class SettingsCommand implements Serializable
 {
 
     public final Command type;
     public final long count;
     public final int tries;
     public final boolean ignoreErrors;
+    public final boolean noWarmup;
     public final ConsistencyLevel consistencyLevel;
     public final double targetUncertainty;
     public final int minimumUncertaintyMeasurements;
     public final int maximumUncertaintyMeasurements;
-    public final DistributionFactory add;
-    public final int keysAtOnce;
+
+    public abstract OpDistributionFactory getFactory(StressSettings settings);
 
     public SettingsCommand(Command type, GroupedOptions options)
     {
@@ -58,8 +59,7 @@ public class SettingsCommand implements Serializable
         this.tries = Math.max(1, Integer.parseInt(options.retries.value()) + 1);
         this.ignoreErrors = options.ignoreErrors.setByUser();
         this.consistencyLevel = ConsistencyLevel.valueOf(options.consistencyLevel.value().toUpperCase());
-        this.keysAtOnce = Integer.parseInt(options.atOnce.value());
-        this.add = options.add.get();
+        this.noWarmup = options.noWarmup.setByUser();
         if (count != null)
         {
             this.count = Long.parseLong(count.count.value());
@@ -82,8 +82,8 @@ public class SettingsCommand implements Serializable
     {
         final OptionSimple retries = new OptionSimple("tries=", "[0-9]+", "9", "Number of tries to perform for each operation before failing", false);
         final OptionSimple ignoreErrors = new OptionSimple("ignore_errors", "", null, "Do not print/log errors", false);
+        final OptionSimple noWarmup = new OptionSimple("no_warmup", "", null, "Do not warmup the process", false);
         final OptionSimple consistencyLevel = new OptionSimple("cl=", "ONE|QUORUM|LOCAL_QUORUM|EACH_QUORUM|ALL|ANY", "ONE", "Consistency level to use", false);
-        final OptionDistribution add = new OptionDistribution("add=", "fixed(1)", "Distribution of value of counter increments");
         final OptionSimple atOnce = new OptionSimple("at-once=", "[0-9]+", "1000", "Number of keys per operation for multiget", false);
     }
 
@@ -93,7 +93,7 @@ public class SettingsCommand implements Serializable
         @Override
         public List<? extends Option> options()
         {
-            return Arrays.asList(count, retries, ignoreErrors, consistencyLevel, add, atOnce);
+            return Arrays.asList(count, retries, noWarmup, ignoreErrors, consistencyLevel, atOnce);
         }
     }
 
@@ -105,7 +105,7 @@ public class SettingsCommand implements Serializable
         @Override
         public List<? extends Option> options()
         {
-            return Arrays.asList(uncertainty, minMeasurements, maxMeasurements, retries, ignoreErrors, consistencyLevel, add, atOnce);
+            return Arrays.asList(uncertainty, minMeasurements, maxMeasurements, retries, noWarmup, ignoreErrors, consistencyLevel, atOnce);
         }
     }
 
@@ -127,48 +127,16 @@ public class SettingsCommand implements Serializable
                 switch (cmd.category)
                 {
                     case BASIC:
-                    case MULTI:
-                        return build(cmd, params);
+                        return SettingsCommandPreDefined.build(cmd, params);
                     case MIXED:
-                        return SettingsCommandMixed.build(params);
+                        return SettingsCommandPreDefinedMixed.build(params);
+                    case USER:
+                        return SettingsCommandUser.build(params);
                 }
             }
         }
         return null;
     }
 
-    static SettingsCommand build(Command type, String[] params)
-    {
-        GroupedOptions options = GroupedOptions.select(params, new Count(), new Uncertainty());
-        if (options == null)
-        {
-            printHelp(type);
-            System.out.println("Invalid " + type + " options provided, see output for valid options");
-            System.exit(1);
-        }
-        return new SettingsCommand(type, options);
-    }
-
-    static void printHelp(Command type)
-    {
-        printHelp(type.toString().toLowerCase());
-    }
-
-    static void printHelp(String type)
-    {
-        GroupedOptions.printOptions(System.out, type.toLowerCase(), new Uncertainty(), new Count());
-    }
-
-    static Runnable helpPrinter(final Command type)
-    {
-        return new Runnable()
-        {
-            @Override
-            public void run()
-            {
-                printHelp(type);
-            }
-        };
-    }
 }
 

tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefined.java

@@ -0,0 +1,145 @@
+package org.apache.cassandra.stress.settings;
+/*
+ * 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * 
+ */
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.generate.values.Bytes;
+import org.apache.cassandra.stress.generate.values.Generator;
+import org.apache.cassandra.stress.generate.values.GeneratorConfig;
+import org.apache.cassandra.stress.generate.values.HexBytes;
+import org.apache.cassandra.stress.operations.FixedOpDistribution;
+import org.apache.cassandra.stress.operations.OpDistribution;
+import org.apache.cassandra.stress.operations.OpDistributionFactory;
+import org.apache.cassandra.stress.operations.predefined.PredefinedOperation;
+import org.apache.cassandra.stress.util.Timer;
+
+// Settings unique to the mixed command type
+public class SettingsCommandPreDefined extends SettingsCommand
+{
+
+    public final DistributionFactory add;
+
+    public OpDistributionFactory getFactory(final StressSettings settings)
+    {
+        return new OpDistributionFactory()
+        {
+            public OpDistribution get(Timer timer)
+            {
+                return new FixedOpDistribution(PredefinedOperation.operation(type, timer, newGenerator(settings), settings, add));
+            }
+
+            public String desc()
+            {
+                return type.toString();
+            }
+
+            public Iterable<OpDistributionFactory> each()
+            {
+                return Collections.<OpDistributionFactory>singleton(this);
+            }
+        };
+    }
+
+    PartitionGenerator newGenerator(StressSettings settings)
+    {
+        List<String> names = settings.columns.namestrs;
+        List<Generator> partitionKey = Collections.<Generator>singletonList(new HexBytes("key",
+                                       new GeneratorConfig("randomstrkey", null,
+                                                           OptionDistribution.get("fixed(" + settings.keys.keySize + ")"), null)));
+
+        List<Generator> columns = new ArrayList<>();
+        for (int i = 0 ; i < settings.columns.maxColumnsPerKey ; i++)
+            columns.add(new Bytes(names.get(i), new GeneratorConfig("randomstr" + names.get(i), null, settings.columns.sizeDistribution, null)));
+        return new PartitionGenerator(partitionKey, Collections.<Generator>emptyList(), columns);
+    }
+
+    public SettingsCommandPreDefined(Command type, Options options)
+    {
+        super(type, options.parent);
+        add = options.add.get();
+    }
+
+    // Option Declarations
+
+    static class Options extends GroupedOptions
+    {
+        final SettingsCommand.Options parent;
+        protected Options(SettingsCommand.Options parent)
+        {
+            this.parent = parent;
+        }
+        final OptionDistribution add = new OptionDistribution("add=", "fixed(1)", "Distribution of value of counter increments");
+
+        @Override
+        public List<? extends Option> options()
+        {
+            final List<Option> options = new ArrayList<>();
+            options.addAll(parent.options());
+            options.add(add);
+            return options;
+        }
+
+    }
+
+    // CLI utility methods
+
+    public static SettingsCommandPreDefined build(Command type, String[] params)
+    {
+        GroupedOptions options = GroupedOptions.select(params,
+                new Options(new Uncertainty()),
+                new Options(new Count()));
+        if (options == null)
+        {
+            printHelp(type);
+            System.out.println("Invalid " + type + " options provided, see output for valid options");
+            System.exit(1);
+        }
+        return new SettingsCommandPreDefined(type, (Options) options);
+    }
+
+    static void printHelp(Command type)
+    {
+        printHelp(type.toString().toLowerCase());
+    }
+
+    static void printHelp(String type)
+    {
+        GroupedOptions.printOptions(System.out, type.toLowerCase(), new Uncertainty(), new Count());
+    }
+
+    static Runnable helpPrinter(final Command type)
+    {
+        return new Runnable()
+        {
+            @Override
+            public void run()
+            {
+                printHelp(type);
+            }
+        };
+    }
+}

tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefinedMixed.java

@@ -24,23 +24,27 @@ package org.apache.cassandra.stress.settings;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.cassandra.stress.generatedata.Distribution;
-import org.apache.cassandra.stress.generatedata.DistributionFactory;
+import org.apache.cassandra.stress.Operation;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.operations.OpDistributionFactory;
+import org.apache.cassandra.stress.operations.SampledOpDistributionFactory;
+import org.apache.cassandra.stress.operations.predefined.PredefinedOperation;
+import org.apache.cassandra.stress.util.Timer;
 
-import org.apache.commons.math3.distribution.EnumeratedDistribution;
 import org.apache.commons.math3.util.Pair;
 
 // Settings unique to the mixed command type
-public class SettingsCommandMixed extends SettingsCommand
+public class SettingsCommandPreDefinedMixed extends SettingsCommandPreDefined
 {
 
     // Ratios for selecting commands - index for each Command, NaN indicates the command is not requested
     private final List<Pair<Command, Double>> ratios;
     private final DistributionFactory clustering;
 
-    public SettingsCommandMixed(Options options)
+    public SettingsCommandPreDefinedMixed(Options options)
     {
-        super(Command.MIXED, options.parent);
+        super(Command.MIXED, options);
 
         clustering = options.clustering.get();
         ratios = options.probabilities.ratios();
@@ -48,60 +52,29 @@ public class SettingsCommandMixed extends SettingsCommand
             throw new IllegalArgumentException("Must specify at least one command with a non-zero ratio");
     }
 
-    public List<Command> getCommands()
+    public OpDistributionFactory getFactory(final StressSettings settings)
     {
-        final List<Command> r = new ArrayList<>();
-        for (Pair<Command, Double> p : ratios)
-            r.add(p.getFirst());
-        return r;
-    }
-
-    public CommandSelector selector()
-    {
-        return new CommandSelector(ratios, clustering.get());
-    }
-
-    // Class for randomly selecting the next command type
-
-    public static final class CommandSelector
-    {
-
-        final EnumeratedDistribution<Command> selector;
-        final Distribution count;
-        private Command cur;
-        private long remaining;
-
-        public CommandSelector(List<Pair<Command, Double>> ratios, Distribution count)
+        return new SampledOpDistributionFactory<Command>(ratios, clustering)
         {
-            selector = new EnumeratedDistribution<>(ratios);
-            this.count = count;
-        }
-
-        public Command next()
-        {
-            while (remaining == 0)
+            protected Operation get(Timer timer, PartitionGenerator generator, Command key)
             {
-                remaining = count.next();
-                cur = selector.sample();
+                return PredefinedOperation.operation(key, timer, generator, settings, add);
             }
-            remaining--;
-            return cur;
-        }
+
+            protected PartitionGenerator newGenerator()
+            {
+                return SettingsCommandPreDefinedMixed.this.newGenerator(settings);
+            }
+        };
     }
 
     // Option Declarations
 
-    static final class Probabilities extends OptionMulti
+    static class Options extends SettingsCommandPreDefined.Options
     {
-        // entry for each in Command.values()
-        final OptionSimple[] ratios;
-        final List<OptionSimple> grouping;
-
-        public Probabilities()
+        static List<OptionEnumProbabilities.Opt<Command>> probabilityOptions = new ArrayList<>();
+        static
         {
-            super("ratio", "Specify the ratios for operations to perform; e.g. (reads=2,writes=1) will perform 2 reads for each write", false);
-            OptionSimple[] ratios = new OptionSimple[Command.values().length];
-            List<OptionSimple> grouping = new ArrayList<>();
             for (Command command : Command.values())
             {
                 if (command.category == null)
@@ -118,45 +91,16 @@ public class SettingsCommandMixed extends SettingsCommand
                     default:
                         defaultValue = null;
                 }
-                OptionSimple ratio = new OptionSimple(command.toString().toLowerCase() +
-                        "=", "[0-9]+(\\.[0-9]+)?", defaultValue, "Performs this many " + command + " operations out of total", false);
-                ratios[command.ordinal()] = ratio;
-                grouping.add(ratio);
+                probabilityOptions.add(new OptionEnumProbabilities.Opt<>(command, defaultValue));
             }
-            this.grouping = grouping;
-            this.ratios = ratios;
         }
 
-        @Override
-        public List<? extends Option> options()
-        {
-            return grouping;
-        }
-
-        List<Pair<Command, Double>> ratios()
-        {
-            List<? extends Option> ratiosIn = setByUser() ? optionsSetByUser() : defaultOptions();
-            List<Pair<Command, Double>> ratiosOut = new ArrayList<>();
-            for (Option opt : ratiosIn)
-            {
-                OptionSimple ratioIn = (OptionSimple) opt;
-                Command command = Command.get(ratioIn.displayPrefix.substring(0, ratioIn.displayPrefix.length() - 1));
-                double d = Double.parseDouble(ratioIn.value());
-                ratiosOut.add(new Pair<>(command, d));
-            }
-            return ratiosOut;
-        }
-    }
-
-    static final class Options extends GroupedOptions
-    {
-        final SettingsCommand.Options parent;
         protected Options(SettingsCommand.Options parent)
         {
-            this.parent = parent;
+            super(parent);
         }
         final OptionDistribution clustering = new OptionDistribution("clustering=", "GAUSSIAN(1..10)", "Distribution clustering runs of operations of the same kind");
-        final Probabilities probabilities = new Probabilities();
+        final OptionEnumProbabilities probabilities = new OptionEnumProbabilities<>(probabilityOptions, "ratio", "Specify the ratios for operations to perform; e.g. (read=2,write=1) will perform 2 reads for each write");
 
         @Override
         public List<? extends Option> options()
@@ -164,7 +108,7 @@ public class SettingsCommandMixed extends SettingsCommand
             final List<Option> options = new ArrayList<>();
             options.add(clustering);
             options.add(probabilities);
-            options.addAll(parent.options());
+            options.addAll(super.options());
             return options;
         }
 
@@ -172,7 +116,7 @@ public class SettingsCommandMixed extends SettingsCommand
 
     // CLI utility methods
 
-    public static SettingsCommandMixed build(String[] params)
+    public static SettingsCommandPreDefinedMixed build(String[] params)
     {
         GroupedOptions options = GroupedOptions.select(params,
                 new Options(new SettingsCommand.Uncertainty()),
@@ -183,7 +127,7 @@ public class SettingsCommandMixed extends SettingsCommand
             System.out.println("Invalid MIXED options provided, see output for valid options");
             System.exit(1);
         }
-        return new SettingsCommandMixed((Options) options);
+        return new SettingsCommandPreDefinedMixed((Options) options);
     }
 
     public static void printHelp()

tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandUser.java

@@ -0,0 +1,135 @@
+package org.apache.cassandra.stress.settings;
+/*
+ * 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * 
+ */
+
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.math3.util.Pair;
+
+import org.apache.cassandra.stress.Operation;
+import org.apache.cassandra.stress.StressProfile;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.PartitionGenerator;
+import org.apache.cassandra.stress.operations.OpDistributionFactory;
+import org.apache.cassandra.stress.operations.SampledOpDistributionFactory;
+import org.apache.cassandra.stress.util.Timer;
+
+// Settings unique to the mixed command type
+public class SettingsCommandUser extends SettingsCommand
+{
+
+    // Ratios for selecting commands - index for each Command, NaN indicates the command is not requested
+    private final List<Pair<String, Double>> ratios;
+    private final DistributionFactory clustering;
+    public final StressProfile profile;
+
+    public SettingsCommandUser(Options options)
+    {
+        super(Command.USER, options.parent);
+
+        clustering = options.clustering.get();
+        ratios = options.ops.ratios();
+        profile = StressProfile.load(new File(options.profile.value()));
+
+        if (ratios.size() == 0)
+            throw new IllegalArgumentException("Must specify at least one command with a non-zero ratio");
+    }
+
+    public OpDistributionFactory getFactory(final StressSettings settings)
+    {
+        return new SampledOpDistributionFactory<String>(ratios, clustering)
+        {
+            protected Operation get(Timer timer, PartitionGenerator generator, String key)
+            {
+                if (key.equalsIgnoreCase("insert"))
+                    return profile.getInsert(timer, generator, settings);
+                return profile.getQuery(key, timer, generator, settings);
+            }
+
+            protected PartitionGenerator newGenerator()
+            {
+                return profile.newGenerator(settings);
+            }
+        };
+    }
+
+    static final class Options extends GroupedOptions
+    {
+        final SettingsCommand.Options parent;
+        protected Options(SettingsCommand.Options parent)
+        {
+            this.parent = parent;
+        }
+        final OptionDistribution clustering = new OptionDistribution("clustering=", "GAUSSIAN(1..10)", "Distribution clustering runs of operations of the same kind");
+        final OptionSimple profile = new OptionSimple("profile=", ".*", null, "Specify the path to a yaml cql3 profile", false);
+        final OptionAnyProbabilities ops = new OptionAnyProbabilities("ops", "Specify the ratios for inserts/queries to perform; e.g. ops(insert=2,<query1>=1) will perform 2 inserts for each query1");
+
+        @Override
+        public List<? extends Option> options()
+        {
+            final List<Option> options = new ArrayList<>();
+            options.add(clustering);
+            options.add(ops);
+            options.add(profile);
+            options.addAll(parent.options());
+            return options;
+        }
+
+    }
+
+    // CLI utility methods
+
+    public static SettingsCommandUser build(String[] params)
+    {
+        GroupedOptions options = GroupedOptions.select(params,
+                new Options(new Uncertainty()),
+                new Options(new Count()));
+        if (options == null)
+        {
+            printHelp();
+            System.out.println("Invalid USER options provided, see output for valid options");
+            System.exit(1);
+        }
+        return new SettingsCommandUser((Options) options);
+    }
+
+    public static void printHelp()
+    {
+        GroupedOptions.printOptions(System.out, "user",
+                                    new Options(new Uncertainty()),
+                                    new Options(new Count()));
+    }
+
+    public static Runnable helpPrinter()
+    {
+        return new Runnable()
+        {
+            @Override
+            public void run()
+            {
+                printHelp();
+            }
+        };
+    }
+}

tools/stress/src/org/apache/cassandra/stress/settings/SettingsKey.java

@@ -26,23 +26,25 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.cassandra.stress.generatedata.DataGenHexFromDistribution;
-import org.apache.cassandra.stress.generatedata.DataGenHexFromOpIndex;
-import org.apache.cassandra.stress.generatedata.DistributionFactory;
-import org.apache.cassandra.stress.generatedata.KeyGen;
+import org.apache.cassandra.stress.generate.DistributionFactory;
+import org.apache.cassandra.stress.generate.SeedGenerator;
+import org.apache.cassandra.stress.generate.SeedRandomGenerator;
+import org.apache.cassandra.stress.generate.SeedSeriesGenerator;
 
 // Settings for key generation
 public class SettingsKey implements Serializable
 {
 
-    private final int keySize;
+    final int keySize;
     private final DistributionFactory distribution;
+    private final DistributionFactory clustering;
     private final long[] range;
 
     public SettingsKey(DistributionOptions options)
     {
         this.keySize = Integer.parseInt(options.size.value());
         this.distribution = options.dist.get();
+        this.clustering = options.clustering.get();
         this.range = null;
     }
 
@@ -50,8 +52,9 @@ public class SettingsKey implements Serializable
     {
         this.keySize = Integer.parseInt(options.size.value());
         this.distribution = null;
+        this.clustering = null;
         String[] bounds = options.populate.value().split("\\.\\.+");
-        this.range = new long[] { Long.parseLong(bounds[0]), Long.parseLong(bounds[1]) };
+        this.range = new long[] { OptionDistribution.parseLong(bounds[0]), OptionDistribution.parseLong(bounds[1]) };
     }
 
     // Option Declarations
@@ -59,6 +62,7 @@ public class SettingsKey implements Serializable
     private static final class DistributionOptions extends GroupedOptions
     {
         final OptionDistribution dist;
+        final OptionDistribution clustering = new OptionDistribution("cluster=", "fixed(1)", "Keys are clustered in adjacent value runs of this size");
         final OptionSimple size = new OptionSimple("size=", "[0-9]+", "10", "Key size in bytes", false);
 
         public DistributionOptions(String defaultLimit)
@@ -69,7 +73,7 @@ public class SettingsKey implements Serializable
         @Override
         public List<? extends Option> options()
         {
-            return Arrays.asList(dist, size);
+            return Arrays.asList(dist, size, clustering);
         }
     }
 
@@ -80,7 +84,7 @@ public class SettingsKey implements Serializable
 
         public PopulateOptions(String defaultLimit)
         {
-            populate = new OptionSimple("populate=", "[0-9]+\\.\\.+[0-9]+",
+            populate = new OptionSimple("populate=", "[0-9]+\\.\\.+[0-9]+[MBK]?",
                     "1.." + defaultLimit,
                     "Populate all keys in sequence", true);
         }
@@ -92,12 +96,9 @@ public class SettingsKey implements Serializable
         }
     }
 
-    public KeyGen newKeyGen()
+    public SeedGenerator newSeedGenerator()
     {
-        return new KeyGen(range == null
-                            ? new DataGenHexFromDistribution(distribution.get())
-                            : new DataGenHexFromOpIndex(range[0], range[1]),
-                          keySize);
+        return range == null ? new SeedRandomGenerator(distribution.get(), clustering.get()) : new SeedSeriesGenerator(range[0], range[1]);
     }
 
     // CLI Utility Methods

tools/stress/src/org/apache/cassandra/stress/settings/SettingsMisc.java

@@ -27,7 +27,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.cassandra.stress.generatedata.Distribution;
+import org.apache.cassandra.stress.generate.Distribution;
 
 public class SettingsMisc implements Serializable
 {

tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java

@@ -140,8 +140,10 @@ public class SettingsMode implements Serializable
         String[] params = clArgs.remove("-mode");
         if (params == null)
         {
-            ThriftOptions opts = new ThriftOptions();
-            opts.smart.accept("smart");
+            Cql3NativeOptions opts = new Cql3NativeOptions();
+            opts.accept("cql3");
+            opts.accept("native");
+            opts.accept("prepared");
             return new SettingsMode(opts);
         }
 

tools/stress/src/org/apache/cassandra/stress/settings/SettingsSchema.java

@@ -25,40 +25,51 @@ import java.nio.ByteBuffer;
 import java.util.*;
 
 import org.apache.cassandra.thrift.*;
+import org.apache.cassandra.thrift.ConsistencyLevel;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 public class SettingsSchema implements Serializable
 {
 
-    public static final String DEFAULT_COMPARATOR = "AsciiType";
     public static final String DEFAULT_VALIDATOR  = "BytesType";
 
     private final String replicationStrategy;
     private final Map<String, String> replicationStrategyOptions;
 
-    private final IndexType indexType;
     private final String compression;
     private final String compactionStrategy;
     private final Map<String, String> compactionStrategyOptions;
     public final String keyspace;
 
-    public SettingsSchema(Options options)
+    public SettingsSchema(Options options, SettingsCommand command)
     {
+        if (command instanceof SettingsCommandUser)
+        {
+            if (options.compaction.setByUser() || options.keyspace.setByUser() || options.compression.setByUser() || options.replication.setByUser())
+                throw new IllegalArgumentException("Cannot provide command line schema settings if a user profile is provided");
+
+            keyspace = ((SettingsCommandUser) command).profile.keyspaceName;
+        }
+        else
+            keyspace = options.keyspace.value();
+
         replicationStrategy = options.replication.getStrategy();
         replicationStrategyOptions = options.replication.getOptions();
-        if (options.index.setByUser())
-            indexType = IndexType.valueOf(options.index.value().toUpperCase());
-        else
-            indexType = null;
         compression = options.compression.value();
         compactionStrategy = options.compaction.getStrategy();
         compactionStrategyOptions = options.compaction.getOptions();
-        keyspace = options.keyspace.value();
     }
 
     public void createKeySpaces(StressSettings settings)
     {
-        createKeySpacesThrift(settings);
+        if (!(settings.command instanceof SettingsCommandUser))
+        {
+            createKeySpacesThrift(settings);
+        }
+        else
+        {
+            ((SettingsCommandUser) settings.command).profile.maybeCreateSchema(settings);
+        }
     }
 
 
@@ -71,7 +82,7 @@ public class SettingsSchema implements Serializable
 
         // column family for standard columns
         CfDef standardCfDef = new CfDef(keyspace, "Standard1");
-        Map<String, String> compressionOptions = new HashMap<String, String>();
+        Map<String, String> compressionOptions = new HashMap<>();
         if (compression != null)
             compressionOptions.put("sstable_compression", compression);
 
@@ -80,28 +91,8 @@ public class SettingsSchema implements Serializable
                 .setDefault_validation_class(DEFAULT_VALIDATOR)
                 .setCompression_options(compressionOptions);
 
-        if (!settings.columns.useTimeUUIDComparator)
-        {
-            for (int i = 0; i < settings.columns.maxColumnsPerKey; i++)
-            {
-                standardCfDef.addToColumn_metadata(new ColumnDef(ByteBufferUtil.bytes("C" + i), "BytesType"));
-            }
-        }
-
-        if (indexType != null)
-        {
-            ColumnDef standardColumn = new ColumnDef(ByteBufferUtil.bytes("C1"), "BytesType");
-            standardColumn.setIndex_type(indexType).setIndex_name("Idx1");
-            standardCfDef.setColumn_metadata(Arrays.asList(standardColumn));
-        }
-
-        // column family with super columns
-        CfDef superCfDef = new CfDef(keyspace, "Super1")
-                .setColumn_type("Super");
-        superCfDef.setComparator_type(DEFAULT_COMPARATOR)
-                .setSubcomparator_type(comparator)
-                .setDefault_validation_class(DEFAULT_VALIDATOR)
-                .setCompression_options(compressionOptions);
+        for (int i = 0; i < settings.columns.names.size(); i++)
+            standardCfDef.addToColumn_metadata(new ColumnDef(settings.columns.names.get(i), "BytesType"));
 
         // column family for standard counters
         CfDef counterCfDef = new CfDef(keyspace, "Counter1")
@@ -109,13 +100,6 @@ public class SettingsSchema implements Serializable
                 .setDefault_validation_class("CounterColumnType")
                 .setCompression_options(compressionOptions);
 
-        // column family with counter super columns
-        CfDef counterSuperCfDef = new CfDef(keyspace, "SuperCounter1")
-                .setComparator_type(comparator)
-                .setDefault_validation_class("CounterColumnType")
-                .setColumn_type("Super")
-                .setCompression_options(compressionOptions);
-
         ksdef.setName(keyspace);
         ksdef.setStrategy_class(replicationStrategy);
 
@@ -127,19 +111,15 @@ public class SettingsSchema implements Serializable
         if (compactionStrategy != null)
         {
             standardCfDef.setCompaction_strategy(compactionStrategy);
-            superCfDef.setCompaction_strategy(compactionStrategy);
             counterCfDef.setCompaction_strategy(compactionStrategy);
-            counterSuperCfDef.setCompaction_strategy(compactionStrategy);
             if (!compactionStrategyOptions.isEmpty())
             {
                 standardCfDef.setCompaction_strategy_options(compactionStrategyOptions);
-                superCfDef.setCompaction_strategy_options(compactionStrategyOptions);
                 counterCfDef.setCompaction_strategy_options(compactionStrategyOptions);
-                counterSuperCfDef.setCompaction_strategy_options(compactionStrategyOptions);
             }
         }
 
-        ksdef.setCf_defs(new ArrayList<>(Arrays.asList(standardCfDef, superCfDef, counterCfDef, counterSuperCfDef)));
+        ksdef.setCf_defs(new ArrayList<>(Arrays.asList(standardCfDef, counterCfDef)));
 
         Cassandra.Client client = settings.getRawThriftClient(false);
 
@@ -191,24 +171,23 @@ public class SettingsSchema implements Serializable
     {
         final OptionReplication replication = new OptionReplication();
         final OptionCompaction compaction = new OptionCompaction();
-        final OptionSimple index = new OptionSimple("index=", "KEYS|CUSTOM|COMPOSITES", null, "Type of index to create on needed column families (KEYS)", false);
         final OptionSimple keyspace = new OptionSimple("keyspace=", ".*", "Keyspace1", "The keyspace name to use", false);
         final OptionSimple compression = new OptionSimple("compression=", ".*", null, "Specify the compression to use for sstable, default:no compression", false);
 
         @Override
         public List<? extends Option> options()
         {
-            return Arrays.asList(replication, index, keyspace, compaction, compression);
+            return Arrays.asList(replication, keyspace, compaction, compression);
         }
     }
 
     // CLI Utility Methods
 
-    public static SettingsSchema get(Map<String, String[]> clArgs)
+    public static SettingsSchema get(Map<String, String[]> clArgs, SettingsCommand command)
     {
         String[] params = clArgs.remove("-schema");
         if (params == null)
-            return new SettingsSchema(new Options());
+            return new SettingsSchema(new Options(), command);
 
         GroupedOptions options = GroupedOptions.select(params, new Options());
         if (options == null)
@@ -217,7 +196,7 @@ public class SettingsSchema implements Serializable
             System.out.println("Invalid -schema options provided, see output for valid options");
             System.exit(1);
         }
-        return new SettingsSchema((Options) options);
+        return new SettingsSchema((Options) options, command);
     }
 
     public static void printHelp()

tools/stress/src/org/apache/cassandra/stress/settings/SettingsTransport.java

@@ -111,7 +111,7 @@ public class SettingsTransport implements Serializable
 
     // Option Declarations
 
-    static class TOptions extends GroupedOptions
+    static class TOptions extends GroupedOptions implements Serializable
     {
         final OptionSimple factory = new OptionSimple("factory=", ".*", TFramedTransportFactory.class.getName(), "Fully-qualified ITransportFactory class name for creating a connection. Note: For Thrift over SSL, use org.apache.cassandra.thrift.SSLTransportFactory.", false);
         final OptionSimple trustStore = new OptionSimple("truststore=", ".*", null, "SSL: full path to truststore", false);

tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java

@@ -29,6 +29,7 @@ import org.apache.cassandra.config.EncryptionOptions;
 import org.apache.cassandra.stress.util.JavaDriverClient;
 import org.apache.cassandra.stress.util.SimpleThriftClient;
 import org.apache.cassandra.stress.util.SmartThriftClient;
+import org.apache.cassandra.stress.util.ThriftClient;
 import org.apache.cassandra.thrift.Cassandra;
 import org.apache.cassandra.thrift.InvalidRequestException;
 import org.apache.cassandra.transport.SimpleClient;
@@ -37,7 +38,6 @@ import org.apache.thrift.transport.TTransport;
 
 public class StressSettings implements Serializable
 {
-
     public final SettingsCommand command;
     public final SettingsRate rate;
     public final SettingsKey keys;
@@ -65,7 +65,24 @@ public class StressSettings implements Serializable
         this.sendToDaemon = sendToDaemon;
     }
 
-    public SmartThriftClient getSmartThriftClient()
+    private SmartThriftClient tclient;
+
+    /**
+     * Thrift client connection
+     * @return cassandra client connection
+     */
+    public synchronized ThriftClient getThriftClient()
+    {
+        if (mode.api != ConnectionAPI.THRIFT_SMART)
+            return getSimpleThriftClient();
+
+        if (tclient == null)
+            tclient = getSmartThriftClient();
+
+        return tclient;
+    }
+
+    private SmartThriftClient getSmartThriftClient()
     {
         Metadata metadata = getJavaDriverClient().getCluster().getMetadata();
         return new SmartThriftClient(this, schema.keyspace, metadata);
@@ -75,7 +92,7 @@ public class StressSettings implements Serializable
      * Thrift client connection
      * @return cassandra client connection
      */
-    public SimpleThriftClient getThriftClient()
+    private SimpleThriftClient getSimpleThriftClient()
     {
         return new SimpleThriftClient(getRawThriftClient(node.randomNode(), true));
     }
@@ -138,6 +155,11 @@ public class StressSettings implements Serializable
     private static volatile JavaDriverClient client;
 
     public JavaDriverClient getJavaDriverClient()
+    {
+        return getJavaDriverClient(true);
+    }
+
+    public JavaDriverClient getJavaDriverClient(boolean setKeyspace)
     {
         if (client != null)
             return client;
@@ -153,7 +175,9 @@ public class StressSettings implements Serializable
                 EncryptionOptions.ClientEncryptionOptions encOptions = transport.getEncryptionOptions();
                 JavaDriverClient c = new JavaDriverClient(currentNode, port.nativePort, encOptions);
                 c.connect(mode.compression());
-                c.execute("USE \"" + schema.keyspace + "\";", org.apache.cassandra.db.ConsistencyLevel.ONE);
+                if (setKeyspace)
+                    c.execute("USE \"" + schema.keyspace + "\";", org.apache.cassandra.db.ConsistencyLevel.ONE);
+
                 return client = c;
             }
         }
@@ -165,7 +189,7 @@ public class StressSettings implements Serializable
 
     public void maybeCreateKeyspaces()
     {
-        if (command.type == Command.WRITE || command.type == Command.COUNTER_WRITE)
+        if (command.type == Command.WRITE || command.type == Command.COUNTER_WRITE || command.type == Command.USER)
             schema.createKeySpaces(this);
 
     }
@@ -202,7 +226,7 @@ public class StressSettings implements Serializable
         SettingsLog log = SettingsLog.get(clArgs);
         SettingsMode mode = SettingsMode.get(clArgs);
         SettingsNode node = SettingsNode.get(clArgs);
-        SettingsSchema schema = SettingsSchema.get(clArgs);
+        SettingsSchema schema = SettingsSchema.get(clArgs, command);
         SettingsTransport transport = SettingsTransport.get(clArgs);
         if (!clArgs.isEmpty())
         {

tools/stress/src/org/apache/cassandra/stress/settings/ValidationType.java

@@ -0,0 +1,8 @@
+package org.apache.cassandra.stress.settings;
+
+public enum ValidationType
+{
+
+    NOT_FAIL, NON_ZERO, SUBSET, EQUAL
+
+}

tools/stress/src/org/apache/cassandra/stress/util/JavaDriverClient.java

@@ -18,6 +18,8 @@
 package org.apache.cassandra.stress.util;
 
 import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import javax.net.ssl.SSLContext;
 
 import com.datastax.driver.core.*;
@@ -40,6 +42,8 @@ public class JavaDriverClient
     private Cluster cluster;
     private Session session;
 
+    private static final ConcurrentMap<String, PreparedStatement> stmts = new ConcurrentHashMap<>();
+
     public JavaDriverClient(String host, int port)
     {
         this(host, port, new EncryptionOptions.ClientEncryptionOptions());
@@ -54,7 +58,18 @@ public class JavaDriverClient
 
     public PreparedStatement prepare(String query)
     {
-        return getSession().prepare(query);
+        PreparedStatement stmt = stmts.get(query);
+        if (stmt != null)
+            return stmt;
+        synchronized (stmts)
+        {
+            stmt = stmts.get(query);
+            if (stmt != null)
+                return stmt;
+            stmt = getSession().prepare(query);
+            stmts.put(query, stmt);
+        }
+        return stmt;
     }
 
     public void connect(ProtocolOptions.Compression compression) throws Exception
@@ -116,7 +131,7 @@ public class JavaDriverClient
      * @param cl
      * @return
      */
-    ConsistencyLevel from(org.apache.cassandra.db.ConsistencyLevel cl)
+    public static ConsistencyLevel from(org.apache.cassandra.db.ConsistencyLevel cl)
     {
         switch (cl)
         {

tools/stress/src/org/apache/cassandra/stress/util/SmartThriftClient.java

@@ -63,8 +63,10 @@ public class SmartThriftClient implements ThriftClient
             return r;
         r = queryIdCounter.incrementAndGet();
         if (queryIds.putIfAbsent(query, r) == null)
+        {
+            queryStrings.put(r, query);
             return r;
-        queryStrings.put(r, query);
+        }
         return queryIds.get(query);
     }
 

tools/stress/src/org/apache/cassandra/stress/util/Timer.java

@@ -43,7 +43,8 @@ public final class Timer
     private int opCount;
 
     // aggregate info
-    private int keyCount;
+    private long partitionCount;
+    private long rowCount;
     private long total;
     private long max;
     private long maxStart;
@@ -65,7 +66,7 @@ public final class Timer
         return 1 + (index >>> SAMPLE_SIZE_SHIFT);
     }
 
-    public void stop(int keys)
+    public void stop(long partitionCount, long rowCount)
     {
         maybeReport();
         long now = System.nanoTime();
@@ -79,7 +80,8 @@ public final class Timer
         }
         total += time;
         opCount += 1;
-        keyCount += keys;
+        this.partitionCount += partitionCount;
+        this.rowCount += rowCount;
         upToDateAsOf = now;
     }
 
@@ -94,11 +96,12 @@ public final class Timer
                 (       new SampleOfLongs(Arrays.copyOf(sample, index(opCount)), p(opCount)),
                         new SampleOfLongs(Arrays.copyOfRange(sample, index(opCount), Math.min(opCount, sample.length)), p(opCount) - 1)
                 );
-        final TimingInterval report = new TimingInterval(lastSnap, upToDateAsOf, max, maxStart, max, keyCount, total, opCount,
+        final TimingInterval report = new TimingInterval(lastSnap, upToDateAsOf, max, maxStart, max, partitionCount, rowCount, total, opCount,
                 SampleOfLongs.merge(rnd, sampleLatencies, Integer.MAX_VALUE));
         // reset counters
         opCount = 0;
-        keyCount = 0;
+        partitionCount = 0;
+        rowCount = 0;
         total = 0;
         max = 0;
         lastSnap = upToDateAsOf;
@@ -145,6 +148,4 @@ public final class Timer
             reportRequest = null;
         }
     }
-
-}
-
+}

tools/stress/src/org/apache/cassandra/stress/util/TimingInterval.java

@@ -39,7 +39,8 @@ public final class TimingInterval
     public final long totalLatency;
 
     // discrete
-    public final long keyCount;
+    public final long partitionCount;
+    public final long rowCount;
     public final long operationCount;
 
     final SampleOfLongs sample;
@@ -48,16 +49,17 @@ public final class TimingInterval
     {
         start = end = time;
         maxLatency = totalLatency = 0;
-        keyCount = operationCount = 0;
+        partitionCount = rowCount = operationCount = 0;
         pauseStart = pauseLength = 0;
         sample = new SampleOfLongs(new long[0], 1d);
     }
-    TimingInterval(long start, long end, long maxLatency, long pauseStart, long pauseLength, long keyCount, long totalLatency, long operationCount, SampleOfLongs sample)
+    TimingInterval(long start, long end, long maxLatency, long pauseStart, long pauseLength, long partitionCount, long rowCount, long totalLatency, long operationCount, SampleOfLongs sample)
     {
         this.start = start;
         this.end = Math.max(end, start);
         this.maxLatency = maxLatency;
-        this.keyCount = keyCount;
+        this.partitionCount = partitionCount;
+        this.rowCount = rowCount;
         this.totalLatency = totalLatency;
         this.operationCount = operationCount;
         this.pauseStart = pauseStart;
@@ -68,7 +70,7 @@ public final class TimingInterval
     // merge multiple timer intervals together
     static TimingInterval merge(Random rnd, List<TimingInterval> intervals, int maxSamples, long start)
     {
-        int operationCount = 0, keyCount = 0;
+        long operationCount = 0, partitionCount = 0, rowCount = 0;
         long maxLatency = 0, totalLatency = 0;
         List<SampleOfLongs> latencies = new ArrayList<>();
         long end = 0;
@@ -79,7 +81,8 @@ public final class TimingInterval
             operationCount += interval.operationCount;
             maxLatency = Math.max(interval.maxLatency, maxLatency);
             totalLatency += interval.totalLatency;
-            keyCount += interval.keyCount;
+            partitionCount += interval.partitionCount;
+            rowCount += interval.rowCount;
             latencies.addAll(Arrays.asList(interval.sample));
             if (interval.pauseLength > 0)
             {
@@ -89,7 +92,7 @@ public final class TimingInterval
         }
         if (pauseEnd < pauseStart)
             pauseEnd = pauseStart = 0;
-        return new TimingInterval(start, end, maxLatency, pauseStart, pauseEnd - pauseStart, keyCount, totalLatency, operationCount,
+        return new TimingInterval(start, end, maxLatency, pauseStart, pauseEnd - pauseStart, partitionCount, rowCount, totalLatency, operationCount,
                 SampleOfLongs.merge(rnd, latencies, maxSamples));
 
     }
@@ -104,9 +107,14 @@ public final class TimingInterval
         return operationCount / ((end - (start + pauseLength)) * 0.000000001d);
     }
 
-    public double keyRate()
+    public double partitionRate()
     {
-        return keyCount / ((end - start) * 0.000000001d);
+        return partitionCount / ((end - start) * 0.000000001d);
+    }
+
+    public double rowRate()
+    {
+        return rowCount / ((end - start) * 0.000000001d);
     }
 
     public double meanLatency()