From 75364296cb3749dfa6d4307cf055f1a18bcf7a9b Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Mon, 7 Jul 2014 18:03:59 +0100 Subject: [PATCH] Introduce CQL support for stress tool patch by Jake Luciani and Benedict Elliott Smith --- CHANGES.txt | 1 + .../cassandra/thrift/ThriftConversion.java | 19 + .../org/apache/cassandra/utils/UUIDGen.java | 8 + tools/bin/cassandra-stress | 2 +- tools/cqlstress-counter-example.yaml | 85 +++ tools/cqlstress-example.yaml | 99 ++++ tools/cqlstress-insanity-example.yaml | 102 ++++ .../apache/cassandra/stress/Operation.java | 218 ++------ .../apache/cassandra/stress/StressAction.java | 266 ++++----- .../cassandra/stress/StressMetrics.java | 17 +- .../cassandra/stress/StressProfile.java | 504 ++++++++++++++++++ .../DataGen.java => StressYaml.java} | 31 +- .../stress/generate/Distribution.java | 57 ++ .../DistributionBoundApache.java | 23 +- .../DistributionFactory.java | 2 +- .../DistributionFixed.java | 10 +- .../stress/generate/DistributionInverted.java | 37 ++ .../DistributionOffsetApache.java | 21 +- .../cassandra/stress/generate/Partition.java | 343 ++++++++++++ .../stress/generate/PartitionGenerator.java | 80 +++ .../stress/generate/RatioDistribution.java | 25 + .../RatioDistributionFactory.java} | 9 +- .../apache/cassandra/stress/generate/Row.java | 22 + .../stress/generate/SeedGenerator.java | 8 + .../stress/generate/SeedRandomGenerator.java | 33 ++ .../stress/generate/SeedSeriesGenerator.java | 21 + .../stress/generate/values/Booleans.java | 37 ++ .../stress/generate/values/Bytes.java | 54 ++ .../values/Dates.java} | 36 +- .../stress/generate/values/Doubles.java | 37 ++ .../stress/generate/values/Floats.java | 37 ++ .../stress/generate/values/Generator.java | 50 ++ .../generate/values/GeneratorConfig.java | 68 +++ .../stress/generate/values/HexBytes.java | 56 ++ .../stress/generate/values/HexStrings.java | 55 ++ .../stress/generate/values/Inets.java | 57 ++ .../values/Integers.java} | 29 +- .../stress/generate/values/Lists.java | 55 ++ .../values/Longs.java} | 27 +- .../KeyGen.java => generate/values/Sets.java} | 48 +- .../stress/generate/values/Strings.java | 49 ++ .../stress/generate/values/TimeUUIDs.java | 51 ++ .../values/UUIDs.java} | 19 +- .../stress/generatedata/DataGenHex.java | 60 --- .../DataGenHexFromDistribution.java | 66 --- .../generatedata/DataGenStringDictionary.java | 107 ---- .../generatedata/DataGenStringRepeats.java | 90 ---- .../generatedata/DistributionSeqBatch.java | 68 --- .../cassandra/stress/generatedata/RowGen.java | 53 -- .../generatedata/RowGenDistributedSize.java | 116 ---- .../operations/CqlIndexedRangeSlicer.java | 118 ---- .../stress/operations/CqlRangeSlicer.java | 59 -- .../operations/FixedOpDistribution.java | 25 + .../stress/operations/OpDistribution.java | 11 + .../operations/OpDistributionFactory.java | 12 + .../operations/SampledOpDistribution.java | 41 ++ .../SampledOpDistributionFactory.java | 72 +++ .../operations/ThriftIndexedRangeSlicer.java | 114 ---- .../stress/operations/ThriftInserter.java | 117 ---- .../stress/operations/ThriftMultiGetter.java | 80 --- .../stress/operations/ThriftRangeSlicer.java | 85 --- .../stress/operations/ThriftReader.java | 94 ---- .../{ => predefined}/CqlCounterAdder.java | 33 +- .../{ => predefined}/CqlCounterGetter.java | 25 +- .../{ => predefined}/CqlInserter.java | 39 +- .../{ => predefined}/CqlOperation.java | 116 ++-- .../{ => predefined}/CqlReader.java | 44 +- .../predefined/PredefinedOperation.java | 248 +++++++++ .../{ => predefined}/ThriftCounterAdder.java | 57 +- .../{ => predefined}/ThriftCounterGetter.java | 54 +- .../operations/predefined/ThriftInserter.java | 96 ++++ .../operations/predefined/ThriftReader.java | 79 +++ .../operations/userdefined/SchemaInsert.java | 144 +++++ .../operations/userdefined/SchemaQuery.java | 86 +++ .../userdefined/SchemaStatement.java | 164 ++++++ .../cassandra/stress/settings/Command.java | 24 +- .../stress/settings/CommandCategory.java | 4 +- .../settings/OptionAnyProbabilities.java | 78 +++ .../stress/settings/OptionDataGen.java | 203 ------- .../stress/settings/OptionDistribution.java | 75 ++- .../settings/OptionEnumProbabilities.java | 62 +++ .../stress/settings/OptionMulti.java | 3 +- .../settings/OptionRatioDistribution.java | 166 ++++++ .../stress/settings/OptionSimple.java | 5 +- .../stress/settings/SettingsColumn.java | 50 +- .../stress/settings/SettingsCommand.java | 58 +- .../settings/SettingsCommandPreDefined.java | 145 +++++ ...va => SettingsCommandPreDefinedMixed.java} | 114 +--- .../stress/settings/SettingsCommandUser.java | 135 +++++ .../stress/settings/SettingsKey.java | 27 +- .../stress/settings/SettingsMisc.java | 2 +- .../stress/settings/SettingsMode.java | 6 +- .../stress/settings/SettingsSchema.java | 77 +-- .../stress/settings/SettingsTransport.java | 2 +- .../stress/settings/StressSettings.java | 36 +- .../stress/settings/ValidationType.java | 8 + .../stress/util/JavaDriverClient.java | 19 +- .../stress/util/SmartThriftClient.java | 4 +- .../apache/cassandra/stress/util/Timer.java | 17 +- .../cassandra/stress/util/TimingInterval.java | 26 +- 100 files changed, 4349 insertions(+), 2378 deletions(-) create mode 100644 tools/cqlstress-counter-example.yaml create mode 100644 tools/cqlstress-example.yaml create mode 100644 tools/cqlstress-insanity-example.yaml create mode 100644 tools/stress/src/org/apache/cassandra/stress/StressProfile.java rename tools/stress/src/org/apache/cassandra/stress/{generatedata/DataGen.java => StressYaml.java} (66%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/Distribution.java rename tools/stress/src/org/apache/cassandra/stress/{generatedata => generate}/DistributionBoundApache.java (77%) rename tools/stress/src/org/apache/cassandra/stress/{generatedata => generate}/DistributionFactory.java (94%) rename tools/stress/src/org/apache/cassandra/stress/{generatedata => generate}/DistributionFixed.java (87%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/DistributionInverted.java rename tools/stress/src/org/apache/cassandra/stress/{generatedata => generate}/DistributionOffsetApache.java (79%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/Partition.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/PartitionGenerator.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/RatioDistribution.java rename tools/stress/src/org/apache/cassandra/stress/{generatedata/DataGenFactory.java => generate/RatioDistributionFactory.java} (86%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/Row.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/SeedGenerator.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/SeedRandomGenerator.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/SeedSeriesGenerator.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/Booleans.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/Bytes.java rename tools/stress/src/org/apache/cassandra/stress/{generatedata/DataGenHexFromOpIndex.java => generate/values/Dates.java} (57%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/Doubles.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/Floats.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/Generator.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/GeneratorConfig.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/HexBytes.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/HexStrings.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/Inets.java rename tools/stress/src/org/apache/cassandra/stress/{generatedata/DataGenBytesRandom.java => generate/values/Integers.java} (66%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/Lists.java rename tools/stress/src/org/apache/cassandra/stress/{generatedata/Distribution.java => generate/values/Longs.java} (70%) rename tools/stress/src/org/apache/cassandra/stress/{generatedata/KeyGen.java => generate/values/Sets.java} (53%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/Strings.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/generate/values/TimeUUIDs.java rename tools/stress/src/org/apache/cassandra/stress/{operations/CqlMultiGetter.java => generate/values/UUIDs.java} (66%) delete mode 100644 tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHex.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHexFromDistribution.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringDictionary.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringRepeats.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionSeqBatch.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/generatedata/RowGen.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/generatedata/RowGenDistributedSize.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/CqlIndexedRangeSlicer.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/CqlRangeSlicer.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/FixedOpDistribution.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/OpDistribution.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/OpDistributionFactory.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistribution.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistributionFactory.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/ThriftIndexedRangeSlicer.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/ThriftInserter.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/ThriftMultiGetter.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/ThriftRangeSlicer.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/ThriftReader.java rename tools/stress/src/org/apache/cassandra/stress/operations/{ => predefined}/CqlCounterAdder.java (61%) rename tools/stress/src/org/apache/cassandra/stress/operations/{ => predefined}/CqlCounterGetter.java (66%) rename tools/stress/src/org/apache/cassandra/stress/operations/{ => predefined}/CqlInserter.java (59%) rename tools/stress/src/org/apache/cassandra/stress/operations/{ => predefined}/CqlOperation.java (88%) rename tools/stress/src/org/apache/cassandra/stress/operations/{ => predefined}/CqlReader.java (58%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/predefined/PredefinedOperation.java rename tools/stress/src/org/apache/cassandra/stress/operations/{ => predefined}/ThriftCounterAdder.java (50%) rename tools/stress/src/org/apache/cassandra/stress/operations/{ => predefined}/ThriftCounterGetter.java (52%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftInserter.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftReader.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaInsert.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaQuery.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaStatement.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/OptionAnyProbabilities.java delete mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/OptionDataGen.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/OptionEnumProbabilities.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/OptionRatioDistribution.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefined.java rename tools/stress/src/org/apache/cassandra/stress/settings/{SettingsCommandMixed.java => SettingsCommandPreDefinedMixed.java} (51%) create mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandUser.java create mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/ValidationType.java diff --git a/CHANGES.txt b/CHANGES.txt index 491ad6d490..ff2f586e9e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.1.0-rc3 + * Introduce CQL support for stress tool (CASSANDRA-6146) * Fix ClassCastException processing expired messages (CASSANDRA-7496) * Fix prepared marker for collections inside UDT (CASSANDRA-7472) * Remove left-over populate_io_cache_on_flush and replicate_on_write diff --git a/src/java/org/apache/cassandra/thrift/ThriftConversion.java b/src/java/org/apache/cassandra/thrift/ThriftConversion.java index 9573b56ccb..0c75d2cb6a 100644 --- a/src/java/org/apache/cassandra/thrift/ThriftConversion.java +++ b/src/java/org/apache/cassandra/thrift/ThriftConversion.java @@ -51,6 +51,25 @@ public class ThriftConversion throw new AssertionError(); } + public static ConsistencyLevel toThrift(org.apache.cassandra.db.ConsistencyLevel cl) + { + switch (cl) + { + case ANY: return ConsistencyLevel.ANY; + case ONE: return ConsistencyLevel.ONE; + case TWO: return ConsistencyLevel.TWO; + case THREE: return ConsistencyLevel.THREE; + case QUORUM: return ConsistencyLevel.QUORUM; + case ALL: return ConsistencyLevel.ALL; + case LOCAL_QUORUM: return ConsistencyLevel.LOCAL_QUORUM; + case EACH_QUORUM: return ConsistencyLevel.EACH_QUORUM; + case SERIAL: return ConsistencyLevel.SERIAL; + case LOCAL_SERIAL: return ConsistencyLevel.LOCAL_SERIAL; + case LOCAL_ONE: return ConsistencyLevel.LOCAL_ONE; + } + throw new AssertionError(); + } + // We never return, but returning a RuntimeException allows to write "throw rethrow(e)" without java complaining // for methods that have a return value. public static RuntimeException rethrow(RequestExecutionException e) throws UnavailableException, TimedOutException diff --git a/src/java/org/apache/cassandra/utils/UUIDGen.java b/src/java/org/apache/cassandra/utils/UUIDGen.java index f3857447c8..53293b2a83 100644 --- a/src/java/org/apache/cassandra/utils/UUIDGen.java +++ b/src/java/org/apache/cassandra/utils/UUIDGen.java @@ -25,6 +25,8 @@ import java.util.Collection; import java.util.Random; import java.util.UUID; +import com.google.common.annotations.VisibleForTesting; + /** * The goods are here: www.ietf.org/rfc/rfc4122.txt. @@ -80,6 +82,12 @@ public class UUIDGen return new UUID(createTime(fromUnixTimestamp(when)), clockSeqAndNode); } + @VisibleForTesting + public static UUID getTimeUUID(long when, long clockSeqAndNode) + { + return new UUID(createTime(fromUnixTimestamp(when)), clockSeqAndNode); + } + /** creates a type 1 uuid from raw bytes. */ public static UUID getUUID(ByteBuffer raw) { diff --git a/tools/bin/cassandra-stress b/tools/bin/cassandra-stress index 39257cd53f..c855cf596f 100755 --- a/tools/bin/cassandra-stress +++ b/tools/bin/cassandra-stress @@ -42,4 +42,4 @@ if [ "x$JAVA" = "x" ]; then exit 1 fi -$JAVA -server -cp $CLASSPATH org.apache.cassandra.stress.Stress $@ +$JAVA -server -ea -cp $CLASSPATH org.apache.cassandra.stress.Stress $@ diff --git a/tools/cqlstress-counter-example.yaml b/tools/cqlstress-counter-example.yaml new file mode 100644 index 0000000000..a65080adcd --- /dev/null +++ b/tools/cqlstress-counter-example.yaml @@ -0,0 +1,85 @@ +# +# This is an example YAML profile for cassandra-stress +# +# insert data +# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1) +# +# read, using query simple1: +# cassandra-stress profile=/home/jake/stress1.yaml ops(simple1=1) +# +# mixed workload (90/10) +# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1,simple1=9) + + +# +# Keyspace info +# +keyspace: stresscql + +# +# The CQL for creating a keyspace (optional if it already exists) +# +keyspace_definition: | + CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; + +# +# Table info +# +table: counttest + +# +# The CQL for creating a table you wish to stress (optional if it already exists) +# +table_definition: | + CREATE TABLE counttest ( + name text PRIMARY KEY, + count counter + ) WITH comment='A table of many types to test wide rows' + +# +# Optional meta information on the generated columns in the above table +# The min and max only apply to text and blob types +# The distribution field represents the total unique population +# distribution of that column across rows. Supported types are +# +# EXP(min..max) An exponential distribution over the range [min..max] +# EXTREME(min..max,shape) An extreme value (Weibull) distribution over the range [min..max] +# GAUSSIAN(min..max,stdvrng) A gaussian/normal distribution, where mean=(min+max)/2, and stdev is (mean-min)/stdvrng +# GAUSSIAN(min..max,mean,stdev) A gaussian/normal distribution, with explicitly defined mean and stdev +# UNIFORM(min..max) A uniform distribution over the range [min, max] +# FIXED(val) A fixed distribution, always returning the same value +# Aliases: extr, gauss, normal, norm, weibull +# +# If preceded by ~, the distribution is inverted +# +# Defaults for all columns are size: uniform(1..256), identity: uniform(1..1024) +# + +columnspec: + - name: name + clustering: uniform(1..100) + size: uniform(1..4) + - name: count + population: fixed(1) + +insert: + partitions: fixed(1) # number of unique partitions to update in a single operation + # if perbatch < 1, multiple batches will be used but all partitions will + # occur in all batches (unless already finished); only the row counts will vary + pervisit: fixed(1)/1 # ratio of rows each partition should update in a single visit to the partition, + # as a proportion of the total possible for the partition + perbatch: fixed(1)/1 # number of rows each partition should update in a single batch statement, + # as a proportion of the proportion we are inserting this visit + # (i.e. compounds with (and capped by) pervisit) + batchtype: UNLOGGED # type of batch to use + +# +# A list of queries you wish to run against the schema +# +queries: + simple1: select * from counttest where name = ? + +# +# In order to generate data consistently we need something to generate a unique key for this schema profile. +# +seed: changing this string changes the generated data. its hashcode is used as the random seed. diff --git a/tools/cqlstress-example.yaml b/tools/cqlstress-example.yaml new file mode 100644 index 0000000000..a997529174 --- /dev/null +++ b/tools/cqlstress-example.yaml @@ -0,0 +1,99 @@ +# +# This is an example YAML profile for cassandra-stress +# +# insert data +# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1) +# +# read, using query simple1: +# cassandra-stress profile=/home/jake/stress1.yaml ops(simple1=1) +# +# mixed workload (90/10) +# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1,simple1=9) + + +# +# Keyspace info +# +keyspace: stresscql + +# +# The CQL for creating a keyspace (optional if it already exists) +# +keyspace_definition: | + CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; + +# +# Table info +# +table: typestest + +# +# The CQL for creating a table you wish to stress (optional if it already exists) +# +table_definition: | + CREATE TABLE typestest ( + name text, + choice boolean, + date timestamp, + address inet, + dbl double, + lval bigint, + ival int, + uid timeuuid, + value blob, + PRIMARY KEY((name,choice), date, address, dbl, lval, ival, uid) + ) WITH COMPACT STORAGE + AND compaction = { 'class':'LeveledCompactionStrategy' } + AND comment='A table of many types to test wide rows' + +# +# Optional meta information on the generated columns in the above table +# The min and max only apply to text and blob types +# The distribution field represents the total unique population +# distribution of that column across rows. Supported types are +# +# EXP(min..max) An exponential distribution over the range [min..max] +# EXTREME(min..max,shape) An extreme value (Weibull) distribution over the range [min..max] +# GAUSSIAN(min..max,stdvrng) A gaussian/normal distribution, where mean=(min+max)/2, and stdev is (mean-min)/stdvrng +# GAUSSIAN(min..max,mean,stdev) A gaussian/normal distribution, with explicitly defined mean and stdev +# UNIFORM(min..max) A uniform distribution over the range [min, max] +# FIXED(val) A fixed distribution, always returning the same value +# Aliases: extr, gauss, normal, norm, weibull +# +# If preceded by ~, the distribution is inverted +# +# Defaults for all columns are size: uniform(1..256), identity: uniform(1..1024) +# +columnspec: + - name: name + size: uniform(1..10) + population: uniform(1..1M) # the range of unique values to select for the field (default is 100Billion) + - name: choice + - name: date + cluster: uniform(1..4) + - name: lval + population: gaussian(1..1000) + cluster: uniform(1..4) + +insert: + partitions: uniform(1..50) # number of unique partitions to update in a single operation + # if perbatch < 1, multiple batches will be used but all partitions will + # occur in all batches (unless already finished); only the row counts will vary + pervisit: uniform(1..10)/10 # ratio of rows each partition should update in a single visit to the partition, + # as a proportion of the total possible for the partition + perbatch: ~exp(1..3)/4 # number of rows each partition should update in a single batch statement, + # as a proportion of the proportion we are inserting this visit + # (i.e. compounds with (and capped by) pervisit) + batchtype: UNLOGGED # type of batch to use + +# +# A list of queries you wish to run against the schema +# +queries: + simple1: select * from typestest where name = ? and choice = ? LIMIT 100 + range1: select * from typestest where name = ? and choice = ? and date >= ? LIMIT 100 + +# +# In order to generate data consistently we need something to generate a unique key for this schema profile. +# +seed: changing this string changes the generated data. its hashcode is used as the random seed. diff --git a/tools/cqlstress-insanity-example.yaml b/tools/cqlstress-insanity-example.yaml new file mode 100644 index 0000000000..e94c9c32e9 --- /dev/null +++ b/tools/cqlstress-insanity-example.yaml @@ -0,0 +1,102 @@ +# +# This is an example YAML profile for cassandra-stress +# +# insert data +# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1) +# +# read, using query simple1: +# cassandra-stress profile=/home/jake/stress1.yaml ops(simple1=1) +# +# mixed workload (90/10) +# cassandra-stress user profile=/home/jake/stress1.yaml ops(insert=1,simple1=9) + + +# +# Keyspace info +# +keyspace: stresscql + +# +# The CQL for creating a keyspace (optional if it already exists) +# +keyspace_definition: | + CREATE KEYSPACE stresscql WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; + +# +# Table info +# +table: insanitytest + +# +# The CQL for creating a table you wish to stress (optional if it already exists) +# +table_definition: | + CREATE TABLE insanitytest ( + name text, + choice boolean, + date timestamp, + address inet, + dbl double, + lval bigint, + fval float, + ival int, + uid timeuuid, + dates list, + inets set, + value blob, + PRIMARY KEY((name, choice), date) + ) WITH compaction = { 'class':'LeveledCompactionStrategy' } + AND comment='A table of many types to test wide rows and collections' + +# +# Optional meta information on the generated columns in the above table +# The min and max only apply to text and blob types +# The distribution field represents the total unique population +# distribution of that column across rows. Supported types are +# +# EXP(min..max) An exponential distribution over the range [min..max] +# EXTREME(min..max,shape) An extreme value (Weibull) distribution over the range [min..max] +# GAUSSIAN(min..max,stdvrng) A gaussian/normal distribution, where mean=(min+max)/2, and stdev is (mean-min)/stdvrng +# GAUSSIAN(min..max,mean,stdev) A gaussian/normal distribution, with explicitly defined mean and stdev +# UNIFORM(min..max) A uniform distribution over the range [min, max] +# FIXED(val) A fixed distribution, always returning the same value +# Aliases: extr, gauss, normal, norm, weibull +# +# If preceded by ~, the distribution is inverted +# +# Defaults for all columns are size: uniform(1..256), population: uniform(1..100B) +# +columnspec: + - name: name + clustering: uniform(1..4) + - name: date + clustering: gaussian(1..20) + - name: lval + population: fixed(1) + - name: dates + clustering: uniform(1..100) + - name: inets + clustering: uniform(1..200) + - name: value + +insert: + partitions: fixed(1) # number of unique partitions to update in a single operation + # if perbatch < 1, multiple batches will be used but all partitions will + # occur in all batches (unless already finished); only the row counts will vary + pervisit: uniform(1..10)/100K # ratio of rows each partition should update in a single visit to the partition, + # as a proportion of the total possible for the partition + perbatch: fixed(1)/1 # number of rows each partition should update in a single batch statement, + # as a proportion of the proportion we are inserting this visit + # (i.e. compounds with (and capped by) pervisit) + batchtype: UNLOGGED # type of batch to use + +# +# A list of queries you wish to run against the schema +# +queries: + simple1: select * from insanitytest where name = ? and choice = ? LIMIT 100 + +# +# In order to generate data consistently we need something to generate a unique key for this schema profile. +# +seed: changing this string changes the generated data. its hashcode is used as the random seed. diff --git a/tools/stress/src/org/apache/cassandra/stress/Operation.java b/tools/stress/src/org/apache/cassandra/stress/Operation.java index 87afb3d133..7831074e4a 100644 --- a/tools/stress/src/org/apache/cassandra/stress/Operation.java +++ b/tools/stress/src/org/apache/cassandra/stress/Operation.java @@ -18,203 +18,47 @@ package org.apache.cassandra.stress; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.EnumMap; import java.util.List; -import java.util.concurrent.ThreadLocalRandom; -import org.apache.cassandra.stress.generatedata.Distribution; -import org.apache.cassandra.stress.generatedata.KeyGen; -import org.apache.cassandra.stress.generatedata.RowGen; +import org.apache.cassandra.stress.generate.Distribution; +import org.apache.cassandra.stress.generate.Partition; +import org.apache.cassandra.stress.generate.PartitionGenerator; import org.apache.cassandra.stress.settings.*; import org.apache.cassandra.stress.util.JavaDriverClient; import org.apache.cassandra.stress.util.ThriftClient; import org.apache.cassandra.stress.util.Timer; -import org.apache.cassandra.thrift.ColumnParent; import org.apache.cassandra.thrift.InvalidRequestException; -import org.apache.cassandra.thrift.SlicePredicate; -import org.apache.cassandra.thrift.SliceRange; import org.apache.cassandra.transport.SimpleClient; -import org.apache.cassandra.utils.ByteBufferUtil; public abstract class Operation { - public final long index; - protected final State state; + public final StressSettings settings; + public final Timer timer; + public final PartitionGenerator generator; + public final Distribution partitionCount; - public Operation(State state, long idx) + protected List partitions; + + public Operation(Timer timer, PartitionGenerator generator, StressSettings settings, Distribution partitionCount) { - index = idx; - this.state = state; + this.generator = generator; + this.timer = timer; + this.settings = settings; + this.partitionCount = partitionCount; } public static interface RunOp { public boolean run() throws Exception; - public String key(); - public int keyCount(); + public int partitionCount(); + public int rowCount(); } - // one per thread! - public static final class State + protected void setPartitions(List partitions) { - - public final StressSettings settings; - public final Timer timer; - public final Command type; - public final KeyGen keyGen; - public final RowGen rowGen; - public final Distribution counteradd; - public final List columnParents; - public final StressMetrics metrics; - public final SettingsCommandMixed.CommandSelector commandSelector; - private final EnumMap substates; - private Object cqlCache; - - public State(Command type, StressSettings settings, StressMetrics metrics) - { - this.type = type; - this.timer = metrics.getTiming().newTimer(); - if (type == Command.MIXED) - { - commandSelector = ((SettingsCommandMixed) settings.command).selector(); - substates = new EnumMap<>(Command.class); - } - else - { - commandSelector = null; - substates = null; - } - counteradd = settings.command.add.get(); - this.settings = settings; - this.keyGen = settings.keys.newKeyGen(); - this.rowGen = settings.columns.newRowGen(); - this.metrics = metrics; - this.columnParents = columnParents(type, settings); - } - - private State(Command type, State copy) - { - this.type = type; - this.timer = copy.timer; - this.rowGen = copy.rowGen; - this.keyGen = copy.keyGen; - this.columnParents = columnParents(type, copy.settings); - this.metrics = copy.metrics; - this.settings = copy.settings; - this.counteradd = copy.counteradd; - this.substates = null; - this.commandSelector = null; - } - - private List columnParents(Command type, StressSettings settings) - { - if (!settings.columns.useSuperColumns) - return Collections.singletonList(new ColumnParent(type.table)); - else - { - ColumnParent[] cp = new ColumnParent[settings.columns.superColumns]; - for (int i = 0 ; i < cp.length ; i++) - cp[i] = new ColumnParent(type.supertable).setSuper_column(ByteBufferUtil.bytes("S" + i)); - return Arrays.asList(cp); - } - } - - - - public boolean isCql3() - { - return settings.mode.cqlVersion == CqlVersion.CQL3; - } - public boolean isCql2() - { - return settings.mode.cqlVersion == CqlVersion.CQL2; - } - public Object getCqlCache() - { - return cqlCache; - } - public void storeCqlCache(Object val) - { - cqlCache = val; - } - - public State substate(Command command) - { - assert type == Command.MIXED; - State substate = substates.get(command); - if (substate == null) - { - substates.put(command, substate = new State(command, this)); - } - return substate; - } - - } - - protected ByteBuffer getKey() - { - return state.keyGen.getKeys(1, index).get(0); - } - - protected List getKeys(int count) - { - return state.keyGen.getKeys(count, index); - } - - protected List generateColumnValues(ByteBuffer key) - { - return state.rowGen.generate(index, key); - } - - private int sliceStart(int count) - { - if (count == state.settings.columns.maxColumnsPerKey) - return 0; - return 1 + ThreadLocalRandom.current().nextInt(state.settings.columns.maxColumnsPerKey - count); - } - - protected SlicePredicate slicePredicate() - { - final SlicePredicate predicate = new SlicePredicate(); - if (state.settings.columns.slice) - { - int count = state.rowGen.count(index); - int start = sliceStart(count); - predicate.setSlice_range(new SliceRange() - .setStart(state.settings.columns.names.get(start)) - .setFinish(new byte[] {}) - .setReversed(false) - .setCount(count) - ); - } - else - predicate.setColumn_names(randomNames()); - return predicate; - } - - protected List randomNames() - { - int count = state.rowGen.count(index); - List src = state.settings.columns.names; - if (count == src.size()) - return src; - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - List r = new ArrayList<>(); - int c = 0, o = 0; - while (c < count && count + o < src.size()) - { - int leeway = src.size() - (count + o); - int spreadover = count - c; - o += Math.round(rnd.nextDouble() * (leeway / (double) spreadover)); - r.add(src.get(o + c++)); - } - while (c < count) - r.add(src.get(o + c++)); - return r; + this.partitions = partitions; } /** @@ -234,13 +78,13 @@ public abstract class Operation public void timeWithRetry(RunOp run) throws IOException { - state.timer.start(); + timer.start(); boolean success = false; String exceptionMessage = null; int tries = 0; - for (; tries < state.settings.command.tries; tries++) + for (; tries < settings.command.tries; tries++) { try { @@ -249,7 +93,7 @@ public abstract class Operation } catch (Exception e) { - switch (state.settings.log.level) + switch (settings.log.level) { case MINIMAL: break; @@ -269,15 +113,13 @@ public abstract class Operation } } - state.timer.stop(run.keyCount()); + timer.stop(run.partitionCount(), run.rowCount()); if (!success) { - error(String.format("Operation [%d] x%d key %s (0x%s) %s%n", - index, + error(String.format("Operation x%d on key(s) %s: %s%n", tries, - run.key(), - ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(run.key())), + key(), (exceptionMessage == null) ? "Data returned was not validated" : "Error executing: " + exceptionMessage)); @@ -285,6 +127,14 @@ public abstract class Operation } + private String key() + { + List keys = new ArrayList<>(); + for (Partition partition : partitions) + keys.add(partition.getKeyAsString()); + return keys.toString(); + } + protected String getExceptionMessage(Exception e) { String className = e.getClass().getSimpleName(); @@ -294,9 +144,9 @@ public abstract class Operation protected void error(String message) throws IOException { - if (!state.settings.command.ignoreErrors) + if (!settings.command.ignoreErrors) throw new IOException(message); - else if (state.settings.log.level.compareTo(SettingsLog.Level.MINIMAL) > 0) + else if (settings.log.level.compareTo(SettingsLog.Level.MINIMAL) > 0) System.err.println(message); } diff --git a/tools/stress/src/org/apache/cassandra/stress/StressAction.java b/tools/stress/src/org/apache/cassandra/stress/StressAction.java index 07ba1d87d6..2105a7295e 100644 --- a/tools/stress/src/org/apache/cassandra/stress/StressAction.java +++ b/tools/stress/src/org/apache/cassandra/stress/StressAction.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CountDownLatch; @@ -29,10 +30,15 @@ import java.util.concurrent.atomic.AtomicLong; import com.google.common.util.concurrent.RateLimiter; import com.google.common.util.concurrent.Uninterruptibles; -import org.apache.cassandra.stress.operations.*; + +import org.apache.cassandra.stress.generate.Partition; +import org.apache.cassandra.stress.generate.SeedGenerator; +import org.apache.cassandra.stress.operations.OpDistribution; +import org.apache.cassandra.stress.operations.OpDistributionFactory; import org.apache.cassandra.stress.settings.*; import org.apache.cassandra.stress.util.JavaDriverClient; import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; import org.apache.cassandra.transport.SimpleClient; public class StressAction implements Runnable @@ -52,7 +58,8 @@ public class StressAction implements Runnable // creating keyspace and column families settings.maybeCreateKeyspaces(); - warmup(settings.command.type, settings.command); + if (!settings.command.noWarmup) + warmup(settings.command.getFactory(settings)); output.println("Sleeping 2s..."); Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); @@ -61,7 +68,7 @@ public class StressAction implements Runnable if (settings.rate.auto) success = runAuto(); else - success = null != run(settings.command.type, settings.rate.threadCount, settings.command.count, output); + success = null != run(settings.command.getFactory(settings), settings.rate.threadCount, settings.command.count, output); if (success) output.println("END"); @@ -72,33 +79,18 @@ public class StressAction implements Runnable } // type provided separately to support recursive call for mixed command with each command type it is performing - private void warmup(Command type, SettingsCommand command) + private void warmup(OpDistributionFactory operations) { // warmup - do 50k iterations; by default hotspot compiles methods after 10k invocations PrintStream warmupOutput = new PrintStream(new OutputStream() { @Override public void write(int b) throws IOException { } } ); - int iterations; - switch (type.category) + int iterations = 50000 * settings.node.nodes.size(); + for (OpDistributionFactory single : operations.each()) { - case BASIC: - iterations = 50000; - break; - case MIXED: - for (Command subtype : ((SettingsCommandMixed) command).getCommands()) - warmup(subtype, command); - return; - case MULTI: - int keysAtOnce = command.keysAtOnce; - iterations = Math.min(50000, (int) Math.ceil(500000d / keysAtOnce)); - break; - default: - throw new IllegalStateException(); + // we need to warm up all the nodes in the cluster ideally, but we may not be the only stress instance; + // so warm up all the nodes we're speaking to only. + output.println(String.format("Warming up %s with %d iterations...", single.desc(), iterations)); + run(single, 20, iterations, warmupOutput); } - - // we need to warm up all the nodes in the cluster ideally, but we may not be the only stress instance; - // so warm up all the nodes we're speaking to only. - iterations *= settings.node.nodes.size(); - output.println(String.format("Warming up %s with %d iterations...", type, iterations)); - run(type, 20, iterations, warmupOutput); } // TODO : permit varying more than just thread count @@ -113,7 +105,7 @@ public class StressAction implements Runnable { output.println(String.format("Running with %d threadCount", threadCount)); - StressMetrics result = run(settings.command.type, threadCount, settings.command.count, output); + StressMetrics result = run(settings.command.getFactory(settings), threadCount, settings.command.count, output); if (result == null) return false; results.add(result); @@ -170,13 +162,13 @@ public class StressAction implements Runnable return improvement / count; } - private StressMetrics run(Command type, int threadCount, long opCount, PrintStream output) + private StressMetrics run(OpDistributionFactory operations, int threadCount, long opCount, PrintStream output) { output.println(String.format("Running %s with %d threads %s", - type.toString(), - threadCount, - opCount > 0 ? " for " + opCount + " iterations" : "until stderr of mean < " + settings.command.targetUncertainty)); + operations.desc(), + threadCount, + opCount > 0 ? " for " + opCount + " iterations" : "until stderr of mean < " + settings.command.targetUncertainty)); final WorkQueue workQueue; if (opCount < 0) workQueue = new ContinuousWorkQueue(50); @@ -193,7 +185,7 @@ public class StressAction implements Runnable final CountDownLatch done = new CountDownLatch(threadCount); final Consumer[] consumers = new Consumer[threadCount]; for (int i = 0; i < threadCount; i++) - consumers[i] = new Consumer(type, done, workQueue, metrics, rateLimiter); + consumers[i] = new Consumer(operations, done, workQueue, metrics, rateLimiter); // starting worker threadCount for (int i = 0; i < threadCount; i++) @@ -236,18 +228,24 @@ public class StressAction implements Runnable private class Consumer extends Thread { - private final Operation.State state; + private final OpDistribution operations; + private final StressMetrics metrics; + private final Timer timer; + private final SeedGenerator seedGenerator; private final RateLimiter rateLimiter; private volatile boolean success = true; private final WorkQueue workQueue; private final CountDownLatch done; - public Consumer(Command type, CountDownLatch done, WorkQueue workQueue, StressMetrics metrics, RateLimiter rateLimiter) + public Consumer(OpDistributionFactory operations, CountDownLatch done, WorkQueue workQueue, StressMetrics metrics, RateLimiter rateLimiter) { this.done = done; this.rateLimiter = rateLimiter; this.workQueue = workQueue; - this.state = new Operation.State(type, settings, metrics); + this.metrics = metrics; + this.timer = metrics.getTiming().newTimer(); + this.seedGenerator = settings.keys.newSeedGenerator(); + this.operations = operations.get(timer); } public void run() @@ -269,63 +267,89 @@ public class StressAction implements Runnable sclient = settings.getSimpleNativeClient(); break; case THRIFT: - tclient = settings.getThriftClient(); - break; case THRIFT_SMART: - tclient = settings.getSmartThriftClient(); + tclient = settings.getThriftClient(); break; default: throw new IllegalStateException(); } - Work work; - while ( null != (work = workQueue.poll()) ) + int maxBatchSize = operations.maxBatchSize(); + Work work = workQueue.poll(); + Partition[] partitions = new Partition[maxBatchSize]; + int workDone = 0; + while (work != null) { - if (rateLimiter != null) - rateLimiter.acquire(work.count); + Operation op = operations.next(); + op.generator.reset(); + int batchSize = Math.max(1, (int) op.partitionCount.next()); + int partitionCount = 0; - for (int i = 0 ; i < work.count ; i++) + while (partitionCount < batchSize) { - try + int count = Math.min((work.count - workDone), batchSize - partitionCount); + for (int i = 0 ; i < count ; i++) { - Operation op = createOperation(state, i + work.offset); - switch (settings.mode.api) - { - case JAVA_DRIVER_NATIVE: - op.run(jclient); - break; - case SIMPLE_NATIVE: - op.run(sclient); - break; - case THRIFT: - case THRIFT_SMART: - default: - op.run(tclient); - } - } catch (Exception e) + long seed = seedGenerator.next(work.offset + workDone + i); + partitions[partitionCount + i] = op.generator.generate(seed); + } + workDone += count; + partitionCount += count; + if (workDone == work.count) { - if (output == null) + workDone = 0; + work = workQueue.poll(); + if (work == null) { - System.err.println(e.getMessage()); - success = false; - System.exit(-1); + if (partitionCount == 0) + return; + break; } - - e.printStackTrace(output); - success = false; - workQueue.stop(); - state.metrics.cancel(); - return; + if (rateLimiter != null) + rateLimiter.acquire(work.count); } } - } + op.setPartitions(Arrays.asList(partitions).subList(0, partitionCount)); + + try + { + switch (settings.mode.api) + { + case JAVA_DRIVER_NATIVE: + op.run(jclient); + break; + case SIMPLE_NATIVE: + op.run(sclient); + break; + case THRIFT: + case THRIFT_SMART: + default: + op.run(tclient); + } + } + catch (Exception e) + { + if (output == null) + { + System.err.println(e.getMessage()); + success = false; + System.exit(-1); + } + + e.printStackTrace(output); + success = false; + workQueue.stop(); + metrics.cancel(); + return; + } + } } finally { done.countDown(); - state.timer.close(); + timer.close(); } } @@ -443,106 +467,4 @@ public class StressAction implements Runnable } - private Operation createOperation(Operation.State state, long index) - { - return createOperation(state.type, state, index); - } - private Operation createOperation(Command type, Operation.State state, long index) - { - switch (type) - { - case READ: - switch(state.settings.mode.style) - { - case THRIFT: - return new ThriftReader(state, index); - case CQL: - case CQL_PREPARED: - return new CqlReader(state, index); - default: - throw new UnsupportedOperationException(); - } - - - case COUNTER_READ: - switch(state.settings.mode.style) - { - case THRIFT: - return new ThriftCounterGetter(state, index); - case CQL: - case CQL_PREPARED: - return new CqlCounterGetter(state, index); - default: - throw new UnsupportedOperationException(); - } - - case WRITE: - switch(state.settings.mode.style) - { - case THRIFT: - return new ThriftInserter(state, index); - case CQL: - case CQL_PREPARED: - return new CqlInserter(state, index); - default: - throw new UnsupportedOperationException(); - } - - case COUNTER_WRITE: - switch(state.settings.mode.style) - { - case THRIFT: - return new ThriftCounterAdder(state, index); - case CQL: - case CQL_PREPARED: - return new CqlCounterAdder(state, index); - default: - throw new UnsupportedOperationException(); - } - - case RANGE_SLICE: - switch(state.settings.mode.style) - { - case THRIFT: - return new ThriftRangeSlicer(state, index); - case CQL: - case CQL_PREPARED: - return new CqlRangeSlicer(state, index); - default: - throw new UnsupportedOperationException(); - } - - case INDEXED_RANGE_SLICE: - switch(state.settings.mode.style) - { - case THRIFT: - return new ThriftIndexedRangeSlicer(state, index); - case CQL: - case CQL_PREPARED: - return new CqlIndexedRangeSlicer(state, index); - default: - throw new UnsupportedOperationException(); - } - - case READ_MULTI: - switch(state.settings.mode.style) - { - case THRIFT: - return new ThriftMultiGetter(state, index); - case CQL: - case CQL_PREPARED: - return new CqlMultiGetter(state, index); - default: - throw new UnsupportedOperationException(); - } - - case MIXED: - Command subcommand = state.commandSelector.next(); - return createOperation(subcommand, state.substate(subcommand), index); - - } - - throw new UnsupportedOperationException(); - } - } diff --git a/tools/stress/src/org/apache/cassandra/stress/StressMetrics.java b/tools/stress/src/org/apache/cassandra/stress/StressMetrics.java index 54a1e2c2d2..7e5c1b6310 100644 --- a/tools/stress/src/org/apache/cassandra/stress/StressMetrics.java +++ b/tools/stress/src/org/apache/cassandra/stress/StressMetrics.java @@ -127,20 +127,21 @@ public class StressMetrics // PRINT FORMATTING - public static final String HEADFORMAT = "%-10s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%7s,%9s"; - public static final String ROWFORMAT = "%-10d,%8.0f,%8.0f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%7.1f,%9.5f"; + public static final String HEADFORMAT = "%-10s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%7s,%9s"; + public static final String ROWFORMAT = "%-10d,%8.0f,%8.0f,%8.0f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%7.1f,%9.5f"; private static void printHeader(String prefix, PrintStream output) { - output.println(prefix + String.format(HEADFORMAT, "ops","op/s", "key/s","mean","med",".95",".99",".999","max","time","stderr")); + output.println(prefix + String.format(HEADFORMAT, "partitions","op/s", "pk/s", "row/s","mean","med",".95",".99",".999","max","time","stderr")); } private static void printRow(String prefix, TimingInterval interval, TimingInterval total, Uncertainty opRateUncertainty, PrintStream output) { output.println(prefix + String.format(ROWFORMAT, - total.operationCount, + total.partitionCount, interval.realOpRate(), - interval.keyRate(), + interval.partitionRate(), + interval.rowRate(), interval.meanLatency(), interval.medianLatency(), interval.rankLatency(0.95f), @@ -156,9 +157,9 @@ public class StressMetrics output.println("\n"); output.println("Results:"); TimingInterval history = timing.getHistory(); - output.println(String.format("real op rate : %.0f", history.realOpRate())); - output.println(String.format("adjusted op rate stderr : %.0f", opRateUncertainty.getUncertainty())); - output.println(String.format("key rate : %.0f", history.keyRate())); + output.println(String.format("op rate : %.0f", history.realOpRate())); + output.println(String.format("partition rate : %.0f", history.partitionRate())); + output.println(String.format("row rate : %.0f", history.rowRate())); output.println(String.format("latency mean : %.1f", history.meanLatency())); output.println(String.format("latency median : %.1f", history.medianLatency())); output.println(String.format("latency 95th percentile : %.1f", history.rankLatency(.95f))); diff --git a/tools/stress/src/org/apache/cassandra/stress/StressProfile.java b/tools/stress/src/org/apache/cassandra/stress/StressProfile.java new file mode 100644 index 0000000000..13f26a250a --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/StressProfile.java @@ -0,0 +1,504 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress; + + +import com.datastax.driver.core.*; +import com.datastax.driver.core.exceptions.AlreadyExistsException; + +import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.statements.CreateKeyspaceStatement; +import org.apache.cassandra.exceptions.RequestValidationException; + +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.generate.RatioDistributionFactory; +import org.apache.cassandra.stress.generate.values.Booleans; +import org.apache.cassandra.stress.generate.values.Bytes; +import org.apache.cassandra.stress.generate.values.Generator; +import org.apache.cassandra.stress.generate.values.Dates; +import org.apache.cassandra.stress.generate.values.Doubles; +import org.apache.cassandra.stress.generate.values.Floats; +import org.apache.cassandra.stress.generate.values.GeneratorConfig; +import org.apache.cassandra.stress.generate.values.Inets; +import org.apache.cassandra.stress.generate.values.Integers; +import org.apache.cassandra.stress.generate.values.Lists; +import org.apache.cassandra.stress.generate.values.Longs; +import org.apache.cassandra.stress.generate.values.Sets; +import org.apache.cassandra.stress.generate.values.Strings; +import org.apache.cassandra.stress.generate.values.TimeUUIDs; +import org.apache.cassandra.stress.generate.values.UUIDs; +import org.apache.cassandra.stress.operations.userdefined.SchemaInsert; +import org.apache.cassandra.stress.operations.userdefined.SchemaQuery; +import org.apache.cassandra.stress.settings.OptionDistribution; +import org.apache.cassandra.stress.settings.OptionRatioDistribution; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.settings.ValidationType; +import org.apache.cassandra.stress.util.JavaDriverClient; +import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; +import org.apache.cassandra.thrift.Compression; +import org.apache.cassandra.thrift.ThriftConversion; +import org.apache.thrift.TException; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.error.YAMLException; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOError; +import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +public class StressProfile implements Serializable +{ + private String keyspaceCql; + private String tableCql; + private String seedStr; + + public String keyspaceName; + public String tableName; + private Map columnConfigs; + private Map queries; + private Map insert; + + transient volatile TableMetadata tableMetaData; + + transient volatile GeneratorFactory generatorFactory; + + transient volatile BatchStatement.Type batchType; + transient volatile DistributionFactory partitions; + transient volatile RatioDistributionFactory pervisit; + transient volatile RatioDistributionFactory perbatch; + transient volatile PreparedStatement insertStatement; + transient volatile Integer thriftInsertId; + + transient volatile Map queryStatements; + transient volatile Map thriftQueryIds; + + private void init(StressYaml yaml) throws RequestValidationException + { + keyspaceName = yaml.keyspace; + keyspaceCql = yaml.keyspace_definition; + tableName = yaml.table; + tableCql = yaml.table_definition; + seedStr = yaml.seed; + queries = yaml.queries; + insert = yaml.insert; + + assert keyspaceName != null : "keyspace name is required in yaml file"; + assert tableName != null : "table name is required in yaml file"; + assert queries != null : "queries map is required in yaml file"; + + if (keyspaceCql != null && keyspaceCql.length() > 0) + { + String name = ((CreateKeyspaceStatement) QueryProcessor.parseStatement(keyspaceCql)).keyspace(); + assert name.equalsIgnoreCase(keyspaceName) : "Name in keyspace_definition doesn't match keyspace property: '" + name + "' != '" + keyspaceName + "'"; + } + else + { + keyspaceCql = null; + } + + if (tableCql != null && tableCql.length() > 0) + { + String name = CFMetaData.compile(tableCql, keyspaceName).cfName; + assert name.equalsIgnoreCase(tableName) : "Name in table_definition doesn't match table property: '" + name + "' != '" + tableName + "'"; + } + else + { + tableCql = null; + } + + columnConfigs = new HashMap<>(); + for (Map spec : yaml.columnspec) + { + lowerCase(spec); + String name = (String) spec.remove("name"); + DistributionFactory population = !spec.containsKey("population") ? null : OptionDistribution.get((String) spec.remove("population")); + DistributionFactory size = !spec.containsKey("size") ? null : OptionDistribution.get((String) spec.remove("size")); + DistributionFactory clustering = !spec.containsKey("cluster") ? null : OptionDistribution.get((String) spec.remove("cluster")); + + if (!spec.isEmpty()) + throw new IllegalArgumentException("Unrecognised option(s) in column spec: " + spec); + if (name == null) + throw new IllegalArgumentException("Missing name argument in column spec"); + + GeneratorConfig config = new GeneratorConfig(yaml.seed + name, clustering, size, population); + columnConfigs.put(name, config); + } + } + + public void maybeCreateSchema(StressSettings settings) + { + JavaDriverClient client = settings.getJavaDriverClient(false); + + if (keyspaceCql != null) + { + try + { + client.execute(keyspaceCql, org.apache.cassandra.db.ConsistencyLevel.ONE); + } + catch (AlreadyExistsException e) + { + } + } + + client.execute("use "+keyspaceName, org.apache.cassandra.db.ConsistencyLevel.ONE); + + if (tableCql != null) + { + try + { + client.execute(tableCql, org.apache.cassandra.db.ConsistencyLevel.ONE); + } + catch (AlreadyExistsException e) + { + } + + System.out.println(String.format("Created schema. Sleeping %ss for propagation.", settings.node.nodes.size())); + Uninterruptibles.sleepUninterruptibly(settings.node.nodes.size(), TimeUnit.SECONDS); + } + + maybeLoadSchemaInfo(settings); + } + + + private void maybeLoadSchemaInfo(StressSettings settings) + { + if (tableMetaData == null) + { + JavaDriverClient client = settings.getJavaDriverClient(); + + synchronized (client) + { + + if (tableMetaData != null) + return; + + TableMetadata metadata = client.getCluster() + .getMetadata() + .getKeyspace(keyspaceName) + .getTable(tableName); + + //Fill in missing column configs + for (ColumnMetadata col : metadata.getColumns()) + { + if (columnConfigs.containsKey(col.getName())) + continue; + + columnConfigs.put(col.getName(), new GeneratorConfig(seedStr + col.getName(), null, null, null)); + } + + tableMetaData = metadata; + } + } + } + + public SchemaQuery getQuery(String name, Timer timer, PartitionGenerator generator, StressSettings settings) + { + if (queryStatements == null) + { + synchronized (this) + { + if (queryStatements == null) + { + try + { + JavaDriverClient jclient = settings.getJavaDriverClient(); + ThriftClient tclient = settings.getThriftClient(); + Map stmts = new HashMap<>(); + Map tids = new HashMap<>(); + for (Map.Entry e : queries.entrySet()) + { + stmts.put(e.getKey().toLowerCase(), jclient.prepare(e.getValue())); + tids.put(e.getKey().toLowerCase(), tclient.prepare_cql3_query(e.getValue(), Compression.NONE)); + } + thriftQueryIds = tids; + queryStatements = stmts; + } + catch (TException e) + { + throw new RuntimeException(e); + } + } + } + } + + // TODO validation + name = name.toLowerCase(); + return new SchemaQuery(timer, generator, settings, thriftQueryIds.get(name), queryStatements.get(name), ThriftConversion.fromThrift(settings.command.consistencyLevel), ValidationType.NOT_FAIL); + } + + public SchemaInsert getInsert(Timer timer, PartitionGenerator generator, StressSettings settings) + { + if (insertStatement == null) + { + synchronized (this) + { + if (insertStatement == null) + { + maybeLoadSchemaInfo(settings); + + Set keyColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getPrimaryKey()); + + //Non PK Columns + StringBuilder sb = new StringBuilder(); + + sb.append("UPDATE \"").append(tableName).append("\" SET "); + + //PK Columns + StringBuilder pred = new StringBuilder(); + pred.append(" WHERE "); + + boolean firstCol = true; + boolean firstPred = true; + for (ColumnMetadata c : tableMetaData.getColumns()) + { + + if (keyColumns.contains(c)) + { + if (firstPred) + firstPred = false; + else + pred.append(" AND "); + + pred.append(c.getName()).append(" = ?"); + } + else + { + if (firstCol) + firstCol = false; + else + sb.append(","); + + sb.append(c.getName()).append(" = "); + + switch (c.getType().getName()) + { + case SET: + case LIST: + case COUNTER: + sb.append(c.getName()).append(" + ?"); + break; + default: + sb.append("?"); + break; + } + } + } + + //Put PK predicates at the end + sb.append(pred); + + if (insert == null) + insert = new HashMap<>(); + lowerCase(insert); + + partitions = OptionDistribution.get(!insert.containsKey("partitions") ? "fixed(1)" : insert.remove("partitions")); + pervisit = OptionRatioDistribution.get(!insert.containsKey("pervisit") ? "fixed(1)/1" : insert.remove("pervisit")); + perbatch = OptionRatioDistribution.get(!insert.containsKey("perbatch") ? "fixed(1)/1" : insert.remove("perbatch")); + batchType = !insert.containsKey("batchtype") ? BatchStatement.Type.UNLOGGED : BatchStatement.Type.valueOf(insert.remove("batchtype")); + if (!insert.isEmpty()) + throw new IllegalArgumentException("Unrecognised insert option(s): " + insert); + + if (generator.maxRowCount > 100 * 1000 * 1000) + System.err.printf("WARNING: You have defined a schema that permits very large partitions (%.0f max rows (>100M))\n", generator.maxRowCount); + if (perbatch.get().max() * pervisit.get().max() * partitions.get().maxValue() * generator.maxRowCount > 100000) + System.err.printf("WARNING: You have defined a schema that permits very large batches (%.0f max rows (>100K)). This may OOM this stress client, or the server.\n", + perbatch.get().max() * pervisit.get().max() * partitions.get().maxValue() * generator.maxRowCount); + + JavaDriverClient client = settings.getJavaDriverClient(); + String query = sb.toString(); + try + { + thriftInsertId = settings.getThriftClient().prepare_cql3_query(query, Compression.NONE); + } + catch (TException e) + { + throw new RuntimeException(e); + } + insertStatement = client.prepare(query); + } + } + } + + return new SchemaInsert(timer, generator, settings, partitions.get(), pervisit.get(), perbatch.get(), thriftInsertId, insertStatement, ThriftConversion.fromThrift(settings.command.consistencyLevel), batchType); + } + + public PartitionGenerator newGenerator(StressSettings settings) + { + if (generatorFactory == null) + { + synchronized (this) + { + maybeLoadSchemaInfo(settings); + if (generatorFactory == null) + generatorFactory = new GeneratorFactory(); + } + } + + return generatorFactory.newGenerator(); + } + + private class GeneratorFactory + { + final List partitionKeys = new ArrayList<>(); + final List clusteringColumns = new ArrayList<>(); + final List valueColumns = new ArrayList<>(); + + private GeneratorFactory() + { + Set keyColumns = com.google.common.collect.Sets.newHashSet(tableMetaData.getPrimaryKey()); + + for (ColumnMetadata metadata : tableMetaData.getPartitionKey()) + partitionKeys.add(new ColumnInfo(metadata.getName(), metadata.getType(), columnConfigs.get(metadata.getName()))); + for (ColumnMetadata metadata : tableMetaData.getClusteringColumns()) + clusteringColumns.add(new ColumnInfo(metadata.getName(), metadata.getType(), columnConfigs.get(metadata.getName()))); + for (ColumnMetadata metadata : tableMetaData.getColumns()) + if (!keyColumns.contains(metadata)) + valueColumns.add(new ColumnInfo(metadata.getName(), metadata.getType(), columnConfigs.get(metadata.getName()))); + } + + PartitionGenerator newGenerator() + { + return new PartitionGenerator(get(partitionKeys), get(clusteringColumns), get(valueColumns)); + } + + List get(List columnInfos) + { + List result = new ArrayList<>(); + for (ColumnInfo columnInfo : columnInfos) + result.add(columnInfo.getGenerator()); + return result; + } + } + + static class ColumnInfo + { + final String name; + final DataType type; + final GeneratorConfig config; + + ColumnInfo(String name, DataType type, GeneratorConfig config) + { + this.name = name; + this.type = type; + this.config = config; + } + + Generator getGenerator() + { + return getGenerator(name, type, config); + } + + static Generator getGenerator(final String name, final DataType type, GeneratorConfig config) + { + switch (type.getName()) + { + case ASCII: + case TEXT: + case VARCHAR: + return new Strings(name, config); + case BIGINT: + case COUNTER: + return new Longs(name, config); + case BLOB: + return new Bytes(name, config); + case BOOLEAN: + return new Booleans(name, config); + case DECIMAL: + case DOUBLE: + return new Doubles(name, config); + case FLOAT: + return new Floats(name, config); + case INET: + return new Inets(name, config); + case INT: + case VARINT: + return new Integers(name, config); + case TIMESTAMP: + return new Dates(name, config); + case UUID: + return new UUIDs(name, config); + case TIMEUUID: + return new TimeUUIDs(name, config); + case SET: + return new Sets(name, getGenerator(name, type.getTypeArguments().get(0), config), config); + case LIST: + return new Lists(name, getGenerator(name, type.getTypeArguments().get(0), config), config); + default: + throw new UnsupportedOperationException(); + } + } + } + + public static StressProfile load(File file) throws IOError + { + try + { + byte[] profileBytes = Files.readAllBytes(Paths.get(file.toURI())); + + Constructor constructor = new Constructor(StressYaml.class); + + Yaml yaml = new Yaml(constructor); + + StressYaml profileYaml = yaml.loadAs(new ByteArrayInputStream(profileBytes), StressYaml.class); + + StressProfile profile = new StressProfile(); + profile.init(profileYaml); + + return profile; + } + catch (YAMLException | IOException | RequestValidationException e) + { + throw new IOError(e); + } + } + + static void lowerCase(Map map) + { + List> reinsert = new ArrayList<>(); + Iterator> iter = map.entrySet().iterator(); + while (iter.hasNext()) + { + Map.Entry e = iter.next(); + if (!e.getKey().toLowerCase().equalsIgnoreCase(e.getKey())) + { + reinsert.add(e); + iter.remove(); + } + } + for (Map.Entry e : reinsert) + map.put(e.getKey().toLowerCase(), e.getValue()); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGen.java b/tools/stress/src/org/apache/cassandra/stress/StressYaml.java similarity index 66% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/DataGen.java rename to tools/stress/src/org/apache/cassandra/stress/StressYaml.java index 334506d2b7..e94fa77e2d 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGen.java +++ b/tools/stress/src/org/apache/cassandra/stress/StressYaml.java @@ -1,6 +1,5 @@ -package org.apache.cassandra.stress.generatedata; /* - * + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,32 +7,32 @@ package org.apache.cassandra.stress.generatedata; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. - * + * */ +package org.apache.cassandra.stress; - -import java.nio.ByteBuffer; import java.util.List; +import java.util.Map; -public abstract class DataGen +public class StressYaml { + public String seed; + public String keyspace; + public String keyspace_definition; + public String table; + public String table_definition; - public abstract void generate(ByteBuffer fill, long index, ByteBuffer seed); - public abstract boolean isDeterministic(); - - public void generate(List fills, long index, ByteBuffer seed) - { - for (ByteBuffer fill : fills) - generate(fill, index++, seed); - } + public List> columnspec; + public Map queries; + public Map insert; } diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/Distribution.java b/tools/stress/src/org/apache/cassandra/stress/generate/Distribution.java new file mode 100644 index 0000000000..4662454ac4 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/Distribution.java @@ -0,0 +1,57 @@ +package org.apache.cassandra.stress.generate; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + + +import java.io.Serializable; + +public abstract class Distribution implements Serializable +{ + + public abstract long next(); + public abstract double nextDouble(); + public abstract long inverseCumProb(double cumProb); + public abstract void setSeed(long seed); + + public long maxValue() + { + return inverseCumProb(1d); + } + + public long minValue() + { + return inverseCumProb(0d); + } + + // approximation of the average; slightly costly to calculate, so should not be invoked frequently + public long average() + { + double sum = 0; + int count = 0; + for (float d = 0 ; d <= 1.0d ; d += 0.02d) + { + sum += inverseCumProb(d); + count += 1; + } + return (long) (sum / count); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionBoundApache.java b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionBoundApache.java similarity index 77% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionBoundApache.java rename to tools/stress/src/org/apache/cassandra/stress/generate/DistributionBoundApache.java index 0fdd437f2e..23ce3e9c66 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionBoundApache.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionBoundApache.java @@ -1,4 +1,4 @@ -package org.apache.cassandra.stress.generatedata; +package org.apache.cassandra.stress.generate; /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -42,12 +42,22 @@ public class DistributionBoundApache extends Distribution return bound(min, max, delegate.sample()); } + public double nextDouble() + { + return boundDouble(min, max, delegate.sample()); + } + @Override public long inverseCumProb(double cumProb) { return bound(min, max, delegate.inverseCumulativeProbability(cumProb)); } + public void setSeed(long seed) + { + delegate.reseedRandomGenerator(seed); + } + private static long bound(long min, long max, double val) { long r = (long) val; @@ -60,4 +70,15 @@ public class DistributionBoundApache extends Distribution throw new IllegalStateException(); } + private static double boundDouble(long min, long max, double r) + { + if ((r >= min) & (r <= max)) + return r; + if (r < min) + return min; + if (r > max) + return max; + throw new IllegalStateException(); + } + } diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionFactory.java b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionFactory.java similarity index 94% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionFactory.java rename to tools/stress/src/org/apache/cassandra/stress/generate/DistributionFactory.java index b80f8512be..d0dfa893a4 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionFactory.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionFactory.java @@ -1,4 +1,4 @@ -package org.apache.cassandra.stress.generatedata; +package org.apache.cassandra.stress.generate; /* * * Licensed to the Apache Software Foundation (ASF) under one diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionFixed.java b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionFixed.java similarity index 87% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionFixed.java rename to tools/stress/src/org/apache/cassandra/stress/generate/DistributionFixed.java index 43372cab2d..bbfb89400b 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionFixed.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionFixed.java @@ -1,4 +1,4 @@ -package org.apache.cassandra.stress.generatedata; +package org.apache.cassandra.stress.generate; /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -37,10 +37,18 @@ public class DistributionFixed extends Distribution return key; } + public double nextDouble() + { + return key; + } + @Override public long inverseCumProb(double cumProb) { return key; } + public void setSeed(long seed) + { + } } diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/DistributionInverted.java b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionInverted.java new file mode 100644 index 0000000000..df52cb874c --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionInverted.java @@ -0,0 +1,37 @@ +package org.apache.cassandra.stress.generate; + +public class DistributionInverted extends Distribution +{ + + final Distribution wrapped; + final long min; + final long max; + + public DistributionInverted(Distribution wrapped) + { + this.wrapped = wrapped; + this.min = wrapped.minValue(); + this.max = wrapped.maxValue(); + } + + public long next() + { + return max - (wrapped.next() - min); + } + + public double nextDouble() + { + return max - (wrapped.nextDouble() - min); + } + + public long inverseCumProb(double cumProb) + { + return max - (wrapped.inverseCumProb(cumProb) - min); + } + + public void setSeed(long seed) + { + wrapped.setSeed(seed); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionOffsetApache.java b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionOffsetApache.java similarity index 79% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionOffsetApache.java rename to tools/stress/src/org/apache/cassandra/stress/generate/DistributionOffsetApache.java index cfbc3851fb..b0e41eb0d8 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionOffsetApache.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/DistributionOffsetApache.java @@ -1,4 +1,4 @@ -package org.apache.cassandra.stress.generatedata; +package org.apache.cassandra.stress.generate; /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -36,12 +36,22 @@ public class DistributionOffsetApache extends Distribution this.delta = max - min; } + public void setSeed(long seed) + { + delegate.reseedRandomGenerator(seed); + } + @Override public long next() { return offset(min, delta, delegate.sample()); } + public double nextDouble() + { + return offsetDouble(min, delta, delegate.sample()); + } + @Override public long inverseCumProb(double cumProb) { @@ -58,4 +68,13 @@ public class DistributionOffsetApache extends Distribution return min + r; } + private double offsetDouble(long min, long delta, double r) + { + if (r < 0) + r = 0; + if (r > delta) + r = delta; + return min + r; + } + } diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/Partition.java b/tools/stress/src/org/apache/cassandra/stress/generate/Partition.java new file mode 100644 index 0000000000..856d5504b1 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/Partition.java @@ -0,0 +1,343 @@ +package org.apache.cassandra.stress.generate; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Queue; +import java.util.Random; +import java.util.Set; +import java.util.UUID; + +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.stress.generate.values.Generator; + +// a partition is re-used to reduce garbage generation, as is its internal RowIterator +public class Partition +{ + + private long idseed; + private final Object[] partitionKey; + private final PartitionGenerator generator; + private final RowIterator iterator; + + public Partition(PartitionGenerator generator) + { + this.generator = generator; + this.partitionKey = new Object[generator.partitionKey.size()]; + if (generator.clusteringComponents.size() > 0) + iterator = new MultiRowIterator(); + else + iterator = new SingleRowIterator(); + } + + void setSeed(long seed) + { + long idseed = 0; + for (int i = 0 ; i < partitionKey.length ; i++) + { + Generator generator = this.generator.partitionKey.get(i); + // set the partition key seed based on the current work item we're processing + generator.setSeed(seed); + Object key = generator.generate(); + partitionKey[i] = key; + // then contribute this value to the data seed + idseed = seed(key, generator.type, idseed); + } + this.idseed = idseed; + } + + public RowIterator iterator(double useChance) + { + iterator.reset(useChance, 0); + return iterator; + } + + public RowIterator iterator(int targetCount) + { + iterator.reset(Double.NaN, targetCount); + return iterator; + } + + class SingleRowIterator extends RowIterator + { + boolean done; + + void reset(double useChance, int targetCount) + { + done = false; + } + + public Iterable batch(double ratio) + { + if (done) + return Collections.emptyList(); + for (int i = 0 ; i < row.row.length ; i++) + { + Generator gen = generator.valueComponents.get(i); + gen.setSeed(idseed); + row.row[i] = gen.generate(); + } + done = true; + return Collections.singleton(row); + } + + public boolean done() + { + return done; + } + } + + public abstract class RowIterator + { + // we reuse the row object to save garbage + final Row row = new Row(partitionKey, new Object[generator.clusteringComponents.size() + generator.valueComponents.size()]); + + public abstract Iterable batch(double ratio); + abstract void reset(double useChance, int targetCount); + + public abstract boolean done(); + + public Partition partition() + { + return Partition.this; + } + } + + // permits iterating a random subset of the procedurally generated rows in this partition; this is the only mechanism for visiting rows + // we maintain a stack of clustering components and their seeds; for each clustering component we visit, we generate all values it takes at that level, + // and then, using the average (total) number of children it takes we randomly choose whether or not we visit its children; + // if we do, we generate all possible values the children can take, and repeat the process. So at any one time we are using space proportional + // to C.N, where N is the average number of values each clustering component takes, as opposed to N^C total values in the partition. + class MultiRowIterator extends RowIterator + { + + // probability any single row will be generated in this iteration + double useChance; + double expectedRowCount; + + // the current seed in use at any given level; used to save recalculating it for each row, so we only need to recalc + // from prior row + final long[] clusteringSeeds = new long[generator.clusteringComponents.size()]; + // the components remaining to be visited for each level of the current stack + final Queue[] clusteringComponents = new ArrayDeque[generator.clusteringComponents.size()]; + + // we want our chance of selection to be applied uniformly, so we compound the roll we make at each level + // so that we know with what chance we reached there, and we adjust our roll at that level by that amount + double[] chancemodifier = new double[generator.clusteringComponents.size()]; + double[] rollmodifier = new double[generator.clusteringComponents.size()]; + + // reusable set for generating unique clustering components + final Set unique = new HashSet<>(); + final Random random = new Random(); + + MultiRowIterator() + { + for (int i = 0 ; i < clusteringComponents.length ; i++) + clusteringComponents[i] = new ArrayDeque<>(); + rollmodifier[0] = 1f; + chancemodifier[0] = generator.clusteringChildAverages[0]; + } + + void reset(double useChance, int targetCount) + { + generator.clusteringComponents.get(0).setSeed(idseed); + int firstComponentCount = (int) generator.clusteringComponents.get(0).clusteringDistribution.next(); + this.expectedRowCount = firstComponentCount * generator.clusteringChildAverages[0]; + if (Double.isNaN(useChance)) + useChance = Math.max(0d, Math.min(1d, targetCount / expectedRowCount)); + + for (Queue q : clusteringComponents) + q.clear(); + + this.useChance = useChance; + clusteringSeeds[0] = idseed; + clusteringComponents[0].add(this); + fill(clusteringComponents[0], firstComponentCount, generator.clusteringComponents.get(0)); + advance(0, 1f); + } + + void fill(int component) + { + long seed = clusteringSeeds[component - 1]; + Generator gen = generator.clusteringComponents.get(component); + gen.setSeed(seed); + clusteringSeeds[component] = seed(clusteringComponents[component - 1].peek(), generator.clusteringComponents.get(component - 1).type, seed); + fill(clusteringComponents[component], (int) gen.clusteringDistribution.next(), gen); + } + + void fill(Queue queue, int count, Generator generator) + { + if (count == 1) + { + queue.add(generator.generate()); + } + else + { + unique.clear(); + for (int i = 0 ; i < count ; i++) + { + Object next = generator.generate(); + if (unique.add(next)) + queue.add(next); + } + } + } + + private boolean advance(double continueChance) + { + // we always start at the leaf level + int depth = clusteringComponents.length - 1; + // fill the row with the position we *were* at (unless pre-start) + for (int i = clusteringSeeds.length ; i < row.row.length ; i++) + { + Generator gen = generator.valueComponents.get(i - clusteringSeeds.length); + long seed = clusteringSeeds[depth]; + seed = seed(clusteringComponents[depth].peek(), generator.clusteringComponents.get(depth).type, seed); + gen.setSeed(seed); + row.row[i] = gen.generate(); + } + clusteringComponents[depth].poll(); + + return advance(depth, continueChance); + } + + private boolean advance(int depth, double continueChance) + { + // advance the leaf component + clusteringComponents[depth].poll(); + while (true) + { + if (clusteringComponents[depth].isEmpty()) + { + if (depth == 0) + return false; + depth--; + clusteringComponents[depth].poll(); + continue; + } + + // the chance of descending is the uniform use chance, multiplied by the number of children + // we would on average generate (so if we have a 0.1 use chance, but should generate 10 children + // then we will always descend), multiplied by 1/(compound roll), where (compound roll) is the + // chance with which we reached this depth, i.e. if we already beat 50/50 odds, we double our + // chance of beating this next roll + double thischance = useChance * chancemodifier[depth]; + if (thischance > 0.999f || thischance >= random.nextDouble()) + { + row.row[depth] = clusteringComponents[depth].peek(); + depth++; + if (depth == clusteringComponents.length) + break; + rollmodifier[depth] = rollmodifier[depth - 1] / Math.min(1d, thischance); + chancemodifier[depth] = generator.clusteringChildAverages[depth] * rollmodifier[depth]; + fill(depth); + continue; + } + + clusteringComponents[depth].poll(); + } + + return continueChance >= 1.0d || continueChance >= random.nextDouble(); + } + + public Iterable batch(final double ratio) + { + final double continueChance = 1d - (Math.pow(ratio, expectedRowCount * useChance)); + return new Iterable() + { + public Iterator iterator() + { + return new Iterator() + { + boolean hasNext = true; + public boolean hasNext() + { + return hasNext; + } + + public Row next() + { + hasNext = advance(continueChance); + return row; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + }; + } + }; + } + + public boolean done() + { + return clusteringComponents[0].isEmpty(); + } + + public Partition partition() + { + return Partition.this; + } + } + + public String getKeyAsString() + { + StringBuilder sb = new StringBuilder(); + int i = 0; + for (Object key : partitionKey) + { + if (i > 0) + sb.append("|"); + AbstractType type = generator.partitionKey.get(i++).type; + sb.append(type.getString(type.decompose(key))); + } + return sb.toString(); + } + + static long seed(Object object, AbstractType type, long seed) + { + if (object instanceof ByteBuffer) + { + ByteBuffer buf = (ByteBuffer) object; + for (int i = buf.position() ; i < buf.limit() ; i++) + seed = (31 * seed) + buf.get(i); + return seed; + } + else if (object instanceof String) + { + String str = (String) object; + for (int i = 0 ; i < str.length() ; i++) + seed = (31 * seed) + str.charAt(i); + return seed; + } + else if (object instanceof Number) + { + return (seed * 31) + ((Number) object).longValue(); + } + else if (object instanceof UUID) + { + return seed * 31 + (((UUID) object).getLeastSignificantBits() ^ ((UUID) object).getMostSignificantBits()); + } + else + { + return seed(type.decompose(object), BytesType.instance, seed); + } + } + + public Object getPartitionKey(int i) + { + return partitionKey[i]; + } + + // used for thrift smart routing - if it's a multi-part key we don't try to route correctly right now + public ByteBuffer getToken() + { + return generator.partitionKey.get(0).type.decompose(partitionKey[0]); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/PartitionGenerator.java b/tools/stress/src/org/apache/cassandra/stress/generate/PartitionGenerator.java new file mode 100644 index 0000000000..78002fb4bd --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/PartitionGenerator.java @@ -0,0 +1,80 @@ +package org.apache.cassandra.stress.generate; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +import com.google.common.collect.Iterables; + +import org.apache.cassandra.stress.generate.values.Generator; + +public class PartitionGenerator +{ + + public final double maxRowCount; + final List partitionKey; + final List clusteringComponents; + final List valueComponents; + final int[] clusteringChildAverages; + + private final Map indexMap; + + final List recyclable = new ArrayList<>(); + int partitionsInUse = 0; + + public void reset() + { + partitionsInUse = 0; + } + + public PartitionGenerator(List partitionKey, List clusteringComponents, List valueComponents) + { + this.partitionKey = partitionKey; + this.clusteringComponents = clusteringComponents; + this.valueComponents = valueComponents; + this.clusteringChildAverages = new int[clusteringComponents.size()]; + for (int i = clusteringChildAverages.length - 1 ; i >= 0 ; i--) + clusteringChildAverages[i] = (int) (i < (clusteringChildAverages.length - 1) ? clusteringComponents.get(i + 1).clusteringDistribution.average() * clusteringChildAverages[i + 1] : 1); + double maxRowCount = 1d; + for (Generator component : clusteringComponents) + maxRowCount *= component.clusteringDistribution.maxValue(); + this.maxRowCount = maxRowCount; + this.indexMap = new HashMap<>(); + int i = 0; + for (Generator generator : partitionKey) + indexMap.put(generator.name, --i); + i = 0; + for (Generator generator : Iterables.concat(clusteringComponents, valueComponents)) + indexMap.put(generator.name, i++); + } + + public int indexOf(String name) + { + Integer i = indexMap.get(name); + if (i == null) + throw new NoSuchElementException(); + return i; + } + + public Partition generate(long seed) + { + if (recyclable.size() <= partitionsInUse || recyclable.get(partitionsInUse) == null) + recyclable.add(new Partition(this)); + + Partition partition = recyclable.get(partitionsInUse++); + partition.setSeed(seed); + return partition; + } + + public ByteBuffer convert(int c, Object v) + { + if (c < 0) + return partitionKey.get(-1-c).type.decompose(v); + if (c < clusteringComponents.size()) + return clusteringComponents.get(c).type.decompose(v); + return valueComponents.get(c - clusteringComponents.size()).type.decompose(v); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/RatioDistribution.java b/tools/stress/src/org/apache/cassandra/stress/generate/RatioDistribution.java new file mode 100644 index 0000000000..fb5a373d68 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/RatioDistribution.java @@ -0,0 +1,25 @@ +package org.apache.cassandra.stress.generate; + +public class RatioDistribution +{ + + final Distribution distribution; + final double divisor; + + public RatioDistribution(Distribution distribution, double divisor) + { + this.distribution = distribution; + this.divisor = divisor; + } + + // yields a value between 0 and 1 + public double next() + { + return Math.max(0f, Math.min(1f, distribution.nextDouble() / divisor)); + } + + public double max() + { + return Math.min(1d, distribution.maxValue() / divisor); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenFactory.java b/tools/stress/src/org/apache/cassandra/stress/generate/RatioDistributionFactory.java similarity index 86% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenFactory.java rename to tools/stress/src/org/apache/cassandra/stress/generate/RatioDistributionFactory.java index dadb792e21..16474d8451 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenFactory.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/RatioDistributionFactory.java @@ -1,4 +1,4 @@ -package org.apache.cassandra.stress.generatedata; +package org.apache.cassandra.stress.generate; /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -23,8 +23,9 @@ package org.apache.cassandra.stress.generatedata; import java.io.Serializable; -public interface DataGenFactory extends Serializable +public interface RatioDistributionFactory extends Serializable { - DataGen get(); -} + RatioDistribution get(); + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/Row.java b/tools/stress/src/org/apache/cassandra/stress/generate/Row.java new file mode 100644 index 0000000000..d3b9a2afc2 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/Row.java @@ -0,0 +1,22 @@ +package org.apache.cassandra.stress.generate; + +public class Row +{ + + final Object[] partitionKey; + final Object[] row; + + public Row(Object[] partitionKey, Object[] row) + { + this.partitionKey = partitionKey; + this.row = row; + } + + public Object get(int column) + { + if (column < 0) + return partitionKey[-1-column]; + return row[column]; + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/SeedGenerator.java b/tools/stress/src/org/apache/cassandra/stress/generate/SeedGenerator.java new file mode 100644 index 0000000000..6c1bc7f9c8 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/SeedGenerator.java @@ -0,0 +1,8 @@ +package org.apache.cassandra.stress.generate; + +public interface SeedGenerator +{ + + long next(long workIndex); + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/SeedRandomGenerator.java b/tools/stress/src/org/apache/cassandra/stress/generate/SeedRandomGenerator.java new file mode 100644 index 0000000000..bbaaeb8e73 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/SeedRandomGenerator.java @@ -0,0 +1,33 @@ +package org.apache.cassandra.stress.generate; + +public class SeedRandomGenerator implements SeedGenerator +{ + + final Distribution distribution; + final Distribution clustering; + + private long next; + private int count; + + public SeedRandomGenerator(Distribution distribution, Distribution clustering) + { + this.distribution = distribution; + this.clustering = clustering; + } + + public long next(long workIndex) + { + if (count == 0) + { + next = distribution.next(); + count = (int) clustering.next(); + } + long result = next; + count--; + if (next == distribution.maxValue()) + next = distribution.minValue(); + else + next++; + return result; + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/SeedSeriesGenerator.java b/tools/stress/src/org/apache/cassandra/stress/generate/SeedSeriesGenerator.java new file mode 100644 index 0000000000..27967d23aa --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/SeedSeriesGenerator.java @@ -0,0 +1,21 @@ +package org.apache.cassandra.stress.generate; + +public class SeedSeriesGenerator implements SeedGenerator +{ + + final long min; + final long count; + + public SeedSeriesGenerator(long min, long max) + { + if (min > max) + throw new IllegalStateException(); + this.min = min; + this.count = 1 + max - min; + } + + public long next(long workIndex) + { + return min + (workIndex % count); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/Booleans.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Booleans.java new file mode 100644 index 0000000000..b1d84d6926 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Booleans.java @@ -0,0 +1,37 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + +import org.apache.cassandra.db.marshal.BooleanType; + +public class Booleans extends Generator +{ + public Booleans(String name, GeneratorConfig config) + { + super(BooleanType.instance, config, name); + } + + @Override + public Boolean generate() + { + return identityDistribution.next() % 1 == 0; + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/Bytes.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Bytes.java new file mode 100644 index 0000000000..2a5bddf3c9 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Bytes.java @@ -0,0 +1,54 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + +import org.apache.cassandra.db.marshal.BytesType; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Random; + +public class Bytes extends Generator +{ + private final byte[] bytes; + private final Random rand = new Random(); + + public Bytes(String name, GeneratorConfig config) + { + super(BytesType.instance, config, name); + bytes = new byte[(int) sizeDistribution.maxValue()]; + } + + @Override + public ByteBuffer generate() + { + long seed = identityDistribution.next(); + sizeDistribution.setSeed(seed); + rand.setSeed(~seed); + int size = (int) sizeDistribution.next(); + for (int i = 0; i < size; ) + for (int v = rand.nextInt(), + n = Math.min(size - i, Integer.SIZE/Byte.SIZE); + n-- > 0; v >>= Byte.SIZE) + bytes[i++] = (byte)v; + return ByteBuffer.wrap(Arrays.copyOf(bytes, size)); + } +} \ No newline at end of file diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHexFromOpIndex.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Dates.java similarity index 57% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHexFromOpIndex.java rename to tools/stress/src/org/apache/cassandra/stress/generate/values/Dates.java index e6bda68e50..7d36be2d3d 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHexFromOpIndex.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Dates.java @@ -1,6 +1,5 @@ -package org.apache.cassandra.stress.generatedata; /* - * + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,41 +7,40 @@ package org.apache.cassandra.stress.generatedata; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. - * + * */ +package org.apache.cassandra.stress.generate.values; +import java.util.Date; -public class DataGenHexFromOpIndex extends DataGenHex +import org.apache.cassandra.db.marshal.DateType; +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.stress.settings.OptionDistribution; + +public class Dates extends Generator { - - final long minKey; - final long maxKey; - - public DataGenHexFromOpIndex(long minKey, long maxKey) + public Dates(String name, GeneratorConfig config) { - this.minKey = minKey; - this.maxKey = maxKey; + super(DateType.instance, config, name); } @Override - public boolean isDeterministic() + public Date generate() { - return true; + return new Date(identityDistribution.next()); } - @Override - long next(long operationIndex) + DistributionFactory defaultIdentityDistribution() { - long range = maxKey + 1 - minKey; - return Math.abs((operationIndex % range) + minKey); + return OptionDistribution.get("uniform(1.." + Long.toString(50L*365L*24L*60L*60L*1000L) + ")"); } } diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/Doubles.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Doubles.java new file mode 100644 index 0000000000..76e983dec8 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Doubles.java @@ -0,0 +1,37 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + +import org.apache.cassandra.db.marshal.DoubleType; + +public class Doubles extends Generator +{ + public Doubles(String name, GeneratorConfig config) + { + super(DoubleType.instance, config, name); + } + + @Override + public Double generate() + { + return identityDistribution.nextDouble(); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/Floats.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Floats.java new file mode 100644 index 0000000000..8e23c11222 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Floats.java @@ -0,0 +1,37 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + +import org.apache.cassandra.db.marshal.FloatType; + +public class Floats extends Generator +{ + public Floats(String name, GeneratorConfig config) + { + super(FloatType.instance, config, name); + } + + @Override + public Float generate() + { + return (float) identityDistribution.nextDouble(); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/Generator.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Generator.java new file mode 100644 index 0000000000..13343def35 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Generator.java @@ -0,0 +1,50 @@ +package org.apache.cassandra.stress.generate.values; + +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.stress.generate.Distribution; +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.stress.settings.OptionDistribution; + +public abstract class Generator +{ + + public final String name; + public final AbstractType type; + final long salt; + final Distribution identityDistribution; + final Distribution sizeDistribution; + public final Distribution clusteringDistribution; + + public Generator(AbstractType type, GeneratorConfig config, String name) + { + this.type = type; + this.name = name; + this.salt = config.salt; + this.identityDistribution = config.getIdentityDistribution(defaultIdentityDistribution()); + this.sizeDistribution = config.getSizeDistribution(defaultSizeDistribution()); + this.clusteringDistribution = config.getClusteringDistribution(defaultClusteringDistribution()); + } + + public void setSeed(long seed) + { + identityDistribution.setSeed(seed ^ salt); + clusteringDistribution.setSeed(seed ^ ~salt); + } + + public abstract T generate(); + + DistributionFactory defaultIdentityDistribution() + { + return OptionDistribution.get("uniform(1..100B)"); + } + + DistributionFactory defaultSizeDistribution() + { + return OptionDistribution.get("uniform(4..8)"); + } + + DistributionFactory defaultClusteringDistribution() + { + return OptionDistribution.get("fixed(1)"); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/GeneratorConfig.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/GeneratorConfig.java new file mode 100644 index 0000000000..8f7b2eaaf6 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/GeneratorConfig.java @@ -0,0 +1,68 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + +import org.apache.cassandra.stress.generate.Distribution; +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.MurmurHash; + + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Map; + +public class GeneratorConfig implements Serializable +{ + public final long salt; + + private final DistributionFactory clusteringDistributions; + private final DistributionFactory sizeDistributions; + private final DistributionFactory identityDistributions; + + public GeneratorConfig(String seedStr, DistributionFactory clusteringDistributions, DistributionFactory sizeDistributions, DistributionFactory identityDistributions) + { + this.clusteringDistributions = clusteringDistributions; + this.sizeDistributions = sizeDistributions; + this.identityDistributions = identityDistributions; + ByteBuffer buf = ByteBufferUtil.bytes(seedStr); + long[] hash = new long[2]; + MurmurHash.hash3_x64_128(buf, buf.position(), buf.remaining(), 0, hash); + salt = hash[0]; + } + + Distribution getClusteringDistribution(DistributionFactory deflt) + { + return (clusteringDistributions == null ? deflt : clusteringDistributions).get(); + } + + Distribution getIdentityDistribution(DistributionFactory deflt) + { + return (identityDistributions == null ? deflt : identityDistributions).get(); + } + + Distribution getSizeDistribution(DistributionFactory deflt) + { + return (sizeDistributions == null ? deflt : sizeDistributions).get(); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/HexBytes.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/HexBytes.java new file mode 100644 index 0000000000..db46bacf0e --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/HexBytes.java @@ -0,0 +1,56 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +import org.apache.cassandra.db.marshal.BytesType; + +public class HexBytes extends Generator +{ + private final byte[] bytes; + + public HexBytes(String name, GeneratorConfig config) + { + super(BytesType.instance, config, name); + bytes = new byte[(int) sizeDistribution.maxValue()]; + } + + @Override + public ByteBuffer generate() + { + long seed = identityDistribution.next(); + sizeDistribution.setSeed(seed); + int size = (int) sizeDistribution.next(); + for (int i = 0 ; i < size ; i +=16) + { + long value = identityDistribution.next(); + for (int j = 0 ; j < 16 && i + j < size ; j++) + { + int v = (int) (value & 15); + bytes[i + j] = (byte) ((v < 10 ? '0' : 'A') + v); + value >>>= 4; + } + } + return ByteBuffer.wrap(Arrays.copyOf(bytes, size)); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/HexStrings.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/HexStrings.java new file mode 100644 index 0000000000..ce65b8aa64 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/HexStrings.java @@ -0,0 +1,55 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + +import java.util.Random; + +import org.apache.cassandra.db.marshal.UTF8Type; + +public class HexStrings extends Generator +{ + private final char[] chars; + + public HexStrings(String name, GeneratorConfig config) + { + super(UTF8Type.instance, config, name); + chars = new char[(int) sizeDistribution.maxValue()]; + } + + @Override + public String generate() + { + long seed = identityDistribution.next(); + sizeDistribution.setSeed(seed); + int size = (int) sizeDistribution.next(); + for (int i = 0 ; i < size ; i +=16) + { + long value = identityDistribution.next(); + for (int j = 0 ; j < 16 && i + j < size ; j++) + { + int v = (int) (value & 15); + chars[i + j] = (char) ((v < 10 ? '0' : 'A') + v); + value >>>= 4; + } + } + return new String(chars, 0, size); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/Inets.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Inets.java new file mode 100644 index 0000000000..334d73c5b5 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Inets.java @@ -0,0 +1,57 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import org.apache.cassandra.db.marshal.InetAddressType; + + +public class Inets extends Generator +{ + final byte[] buf; + public Inets(String name, GeneratorConfig config) + { + super(InetAddressType.instance, config, name); + buf = new byte[4]; + } + + @Override + public InetAddress generate() + { + int val = (int) identityDistribution.next(); + + buf[0] = (byte)(val >>> 24); + buf[1] = (byte)(val >>> 16); + buf[2] = (byte)(val >>> 8); + buf[3] = (byte)val; + + try + { + return InetAddress.getByAddress(buf); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenBytesRandom.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Integers.java similarity index 66% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenBytesRandom.java rename to tools/stress/src/org/apache/cassandra/stress/generate/values/Integers.java index 1dc77b49b6..8b9b33a077 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenBytesRandom.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Integers.java @@ -1,6 +1,5 @@ -package org.apache.cassandra.stress.generatedata; /* - * + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,38 +7,32 @@ package org.apache.cassandra.stress.generatedata; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. - * + * */ +package org.apache.cassandra.stress.generate.values; +import org.apache.cassandra.db.marshal.Int32Type; -import java.nio.ByteBuffer; -import java.util.Random; - -public class DataGenBytesRandom extends DataGen +public class Integers extends Generator { - private final Random rnd = new Random(); - - @Override - public void generate(ByteBuffer fill, long index, ByteBuffer seed) + public Integers(String name, GeneratorConfig config) { - fill.clear(); - rnd.nextBytes(fill.array()); + super(Int32Type.instance, config, name); } @Override - public boolean isDeterministic() + public Integer generate() { - return false; + return (int) identityDistribution.next(); } - } diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/Lists.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Lists.java new file mode 100644 index 0000000000..d188f7ecfb --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Lists.java @@ -0,0 +1,55 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + + +import java.util.Arrays; +import java.util.List; + +import org.apache.cassandra.db.marshal.ListType; + +public class Lists extends Generator +{ + final Generator valueType; + final Object[] buffer; + + public Lists(String name, Generator valueType, GeneratorConfig config) + { + super(ListType.getInstance(valueType.type), config, name); + this.valueType = valueType; + buffer = new Object[(int) sizeDistribution.maxValue()]; + } + + public void setSeed(long seed) + { + super.setSeed(seed); + valueType.setSeed(seed * 31); + } + + @Override + public List generate() + { + int size = (int) sizeDistribution.next(); + for (int i = 0 ; i < size ; i++) + buffer[i] = valueType.generate(); + return com.google.common.collect.Lists.newArrayList(Arrays.copyOf(buffer, size)); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/Distribution.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Longs.java similarity index 70% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/Distribution.java rename to tools/stress/src/org/apache/cassandra/stress/generate/values/Longs.java index ed40290f71..0584ed1ba4 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/Distribution.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Longs.java @@ -1,6 +1,5 @@ -package org.apache.cassandra.stress.generatedata; /* - * + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,33 +7,31 @@ package org.apache.cassandra.stress.generatedata; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. - * + * */ +package org.apache.cassandra.stress.generate.values; +import org.apache.cassandra.db.marshal.LongType; -public abstract class Distribution +public class Longs extends Generator { - - public abstract long next(); - public abstract long inverseCumProb(double cumProb); - - public long maxValue() + public Longs(String name, GeneratorConfig config) { - return inverseCumProb(1d); + super(LongType.instance, config, name); } - public long minValue() + @Override + public Long generate() { - return inverseCumProb(0d); + return identityDistribution.next(); } - } diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/KeyGen.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Sets.java similarity index 53% rename from tools/stress/src/org/apache/cassandra/stress/generatedata/KeyGen.java rename to tools/stress/src/org/apache/cassandra/stress/generate/values/Sets.java index dad5918d0e..48bf293548 100644 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/KeyGen.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Sets.java @@ -1,6 +1,5 @@ -package org.apache.cassandra.stress.generatedata; /* - * + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,47 +7,48 @@ package org.apache.cassandra.stress.generatedata; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. - * + * */ +package org.apache.cassandra.stress.generate.values; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; +import java.util.HashSet; +import java.util.Set; -public class KeyGen +import org.apache.cassandra.db.marshal.SetType; + +public class Sets extends Generator { + final Generator valueType; - final DataGen dataGen; - final int keySize; - final List keyBuffers = new ArrayList<>(); - - public KeyGen(DataGen dataGen, int keySize) + public Sets(String name, Generator valueType, GeneratorConfig config) { - this.dataGen = dataGen; - this.keySize = keySize; + super(SetType.getInstance(valueType.type), config, name); + this.valueType = valueType; } - public List getKeys(int n, long index) + public void setSeed(long seed) { - while (keyBuffers.size() < n) - keyBuffers.add(ByteBuffer.wrap(new byte[keySize])); - dataGen.generate(keyBuffers, index, null); - return keyBuffers; + super.setSeed(seed); + valueType.setSeed(seed * 31); } - public boolean isDeterministic() + @Override + public Set generate() { - return dataGen.isDeterministic(); + final Set set = new HashSet(); + int size = (int) sizeDistribution.next(); + for (int i = 0 ; i < size ; i++) + set.add(valueType.generate()); + return set; } - } diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/Strings.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/Strings.java new file mode 100644 index 0000000000..e01ff20b36 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/Strings.java @@ -0,0 +1,49 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + +import java.util.Random; + +import org.apache.cassandra.db.marshal.UTF8Type; + +public class Strings extends Generator +{ + private final char[] chars; + private final Random rnd = new Random(); + + public Strings(String name, GeneratorConfig config) + { + super(UTF8Type.instance, config, name); + chars = new char[(int) sizeDistribution.maxValue()]; + } + + @Override + public String generate() + { + long seed = identityDistribution.next(); + sizeDistribution.setSeed(seed); + rnd.setSeed(~seed); + int size = (int) sizeDistribution.next(); + for (int i = 0 ; i < size ; i++) + chars[i] = (char) (32 +rnd.nextInt(128-32)); + return new String(chars, 0, size); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/generate/values/TimeUUIDs.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/TimeUUIDs.java new file mode 100644 index 0000000000..714959de54 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/TimeUUIDs.java @@ -0,0 +1,51 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.stress.generate.values; + + +import org.apache.cassandra.db.marshal.TimeUUIDType; +import org.apache.cassandra.utils.UUIDGen; + +import java.util.UUID; + +public class TimeUUIDs extends Generator +{ + final Dates dateGen; + final long clockSeqAndNode; + + public TimeUUIDs(String name, GeneratorConfig config) + { + super(TimeUUIDType.instance, config, name); + dateGen = new Dates(name, config); + clockSeqAndNode = config.salt; + } + + public void setSeed(long seed) + { + dateGen.setSeed(seed); + } + + @Override + public UUID generate() + { + return UUIDGen.getTimeUUID(dateGen.generate().getTime(), clockSeqAndNode); + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/CqlMultiGetter.java b/tools/stress/src/org/apache/cassandra/stress/generate/values/UUIDs.java similarity index 66% rename from tools/stress/src/org/apache/cassandra/stress/operations/CqlMultiGetter.java rename to tools/stress/src/org/apache/cassandra/stress/generate/values/UUIDs.java index 80a7118aaf..e8d6501c3d 100644 --- a/tools/stress/src/org/apache/cassandra/stress/operations/CqlMultiGetter.java +++ b/tools/stress/src/org/apache/cassandra/stress/generate/values/UUIDs.java @@ -1,4 +1,3 @@ -package org.apache.cassandra.stress.operations; /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -19,24 +18,22 @@ package org.apache.cassandra.stress.operations; * under the License. * */ +package org.apache.cassandra.stress.generate.values; +import java.util.UUID; -import java.io.IOException; +import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.stress.Operation; -import org.apache.cassandra.stress.util.ThriftClient; - -public class CqlMultiGetter extends Operation +public class UUIDs extends Generator { - public CqlMultiGetter(State state, long idx) + public UUIDs(String name, GeneratorConfig config) { - super(state, idx); - throw new RuntimeException("Multiget is not implemented for CQL"); + super(UUIDType.instance, config, name); } @Override - public void run(ThriftClient client) throws IOException + public UUID generate() { + return new UUID(identityDistribution.next(), identityDistribution.next()); } - } diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHex.java b/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHex.java deleted file mode 100644 index 124a07b181..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHex.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.apache.cassandra.stress.generatedata; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import java.nio.ByteBuffer; - -public abstract class DataGenHex extends DataGen -{ - - abstract long next(long operationIndex); - - @Override - public final void generate(ByteBuffer fill, long operationIndex, ByteBuffer seed) - { - fill.clear(); - fillKeyStringBytes(next(operationIndex), fill.array()); - } - - public static void fillKeyStringBytes(long key, byte[] fill) - { - int ub = fill.length - 1; - int offset = 0; - while (key != 0) - { - int digit = ((int) key) & 15; - key >>>= 4; - fill[ub - offset++] = digit(digit); - } - while (offset < fill.length) - fill[ub - offset++] = '0'; - } - - // needs to be UTF-8, but for these chars there is no difference - private static byte digit(int num) - { - if (num < 10) - return (byte)('0' + num); - return (byte)('A' + (num - 10)); - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHexFromDistribution.java b/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHexFromDistribution.java deleted file mode 100644 index 25284dc83a..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenHexFromDistribution.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.apache.cassandra.stress.generatedata; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import org.apache.commons.math3.distribution.NormalDistribution; -import org.apache.commons.math3.distribution.UniformRealDistribution; - -public class DataGenHexFromDistribution extends DataGenHex -{ - - final Distribution distribution; - - public DataGenHexFromDistribution(Distribution distribution) - { - this.distribution = distribution; - } - - @Override - public boolean isDeterministic() - { - return false; - } - - @Override - long next(long operationIndex) - { - return distribution.next(); - } - - public static DataGenHex buildGaussian(long minKey, long maxKey, double stdevsToLimit) - { - double midRange = (maxKey + minKey) / 2d; - double halfRange = (maxKey - minKey) / 2d; - return new DataGenHexFromDistribution(new DistributionBoundApache(new NormalDistribution(midRange, halfRange / stdevsToLimit), minKey, maxKey)); - } - - public static DataGenHex buildGaussian(long minKey, long maxKey, double mean, double stdev) - { - return new DataGenHexFromDistribution(new DistributionBoundApache(new NormalDistribution(mean, stdev), minKey, maxKey)); - } - - public static DataGenHex buildUniform(long minKey, long maxKey) - { - return new DataGenHexFromDistribution(new DistributionBoundApache(new UniformRealDistribution(minKey, maxKey), minKey, maxKey)); - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringDictionary.java b/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringDictionary.java deleted file mode 100644 index 73814cbe42..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringDictionary.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.apache.cassandra.stress.generatedata; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.math3.distribution.EnumeratedDistribution; -import org.apache.commons.math3.util.Pair; - -import static com.google.common.base.Charsets.UTF_8; - -public class DataGenStringDictionary extends DataGen -{ - - private final byte space = ' '; - private final EnumeratedDistribution words; - - public DataGenStringDictionary(EnumeratedDistribution wordDistribution) - { - words = wordDistribution; - } - - @Override - public void generate(ByteBuffer fill, long index, ByteBuffer seed) - { - fill(fill); - } - - @Override - public void generate(List fills, long index, ByteBuffer seed) - { - for (int i = 0 ; i < fills.size() ; i++) - fill(fills.get(0)); - } - - private void fill(ByteBuffer fill) - { - fill.clear(); - byte[] trg = fill.array(); - int i = 0; - while (i < trg.length) - { - if (i > 0) - trg[i++] = space; - byte[] src = words.sample(); - System.arraycopy(src, 0, trg, i, Math.min(src.length, trg.length - i)); - i += src.length; - } - } - - @Override - public boolean isDeterministic() - { - return false; - } - - public static DataGenFactory getFactory(File file) throws IOException - { - final List> words = new ArrayList<>(); - try (final BufferedReader reader = new BufferedReader(new FileReader(file))) - { - String line; - while ( null != (line = reader.readLine()) ) - { - String[] pair = line.split(" +"); - if (pair.length != 2) - throw new IllegalArgumentException("Invalid record in dictionary: \"" + line + "\""); - words.add(new Pair<>(pair[1].getBytes(UTF_8), Double.parseDouble(pair[0]))); - } - final EnumeratedDistribution dist = new EnumeratedDistribution(words); - return new DataGenFactory() - { - @Override - public DataGen get() - { - return new DataGenStringDictionary(dist); - } - }; - } - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringRepeats.java b/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringRepeats.java deleted file mode 100644 index ad7cb247c2..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DataGenStringRepeats.java +++ /dev/null @@ -1,90 +0,0 @@ -package org.apache.cassandra.stress.generatedata; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.cassandra.utils.FBUtilities; - -import static com.google.common.base.Charsets.UTF_8; - -public class DataGenStringRepeats extends DataGen -{ - - private static final ConcurrentHashMap> CACHE_LOOKUP = new ConcurrentHashMap<>(); - - private final ConcurrentHashMap cache; - private final int repeatFrequency; - public DataGenStringRepeats(int repeatFrequency) - { - if (!CACHE_LOOKUP.containsKey(repeatFrequency)) - CACHE_LOOKUP.putIfAbsent(repeatFrequency, new ConcurrentHashMap()); - cache = CACHE_LOOKUP.get(repeatFrequency); - this.repeatFrequency = repeatFrequency; - } - - @Override - public void generate(ByteBuffer fill, long index, ByteBuffer seed) - { - fill(fill, index, 0, seed); - } - - @Override - public void generate(List fills, long index, ByteBuffer seed) - { - for (int i = 0 ; i < fills.size() ; i++) - { - fill(fills.get(i), index, i, seed); - } - } - - private void fill(ByteBuffer fill, long index, int column, ByteBuffer seed) - { - fill.clear(); - byte[] trg = fill.array(); - byte[] src = getData(index, column, seed); - for (int j = 0 ; j < trg.length ; j += src.length) - System.arraycopy(src, 0, trg, j, Math.min(src.length, trg.length - j)); - } - - private byte[] getData(long index, int column, ByteBuffer seed) - { - final long key = ((long)column * repeatFrequency) + ((seed == null ? index : Math.abs(seed.hashCode())) % repeatFrequency); - byte[] r = cache.get(key); - if (r != null) - return r; - MessageDigest md = FBUtilities.threadLocalMD5Digest(); - r = md.digest(Long.toString(key).getBytes(UTF_8)); - cache.putIfAbsent(key, r); - return r; - } - - @Override - public boolean isDeterministic() - { - return true; - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionSeqBatch.java b/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionSeqBatch.java deleted file mode 100644 index 8e1a5d55d9..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/DistributionSeqBatch.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.apache.cassandra.stress.generatedata; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -public class DistributionSeqBatch extends DataGenHex -{ - - final Distribution delegate; - final int batchSize; - final long maxKey; - - private int batchIndex; - private long batchKey; - - // object must be published safely if passed between threadCount, due to batchIndex not being volatile. various - // hacks possible, but not ideal. don't want to use volatile as object intended for single threaded use. - public DistributionSeqBatch(int batchSize, long maxKey, Distribution delegate) - { - this.batchIndex = batchSize; - this.batchSize = batchSize; - this.maxKey = maxKey; - this.delegate = delegate; - } - - @Override - long next(long operationIndex) - { - if (batchIndex >= batchSize) - { - batchKey = delegate.next(); - batchIndex = 0; - } - long r = batchKey + batchIndex++; - if (r > maxKey) - { - batchKey = delegate.next(); - batchIndex = 1; - r = batchKey; - } - return r; - } - - @Override - public boolean isDeterministic() - { - return false; - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/RowGen.java b/tools/stress/src/org/apache/cassandra/stress/generatedata/RowGen.java deleted file mode 100644 index 9c6ca43ee7..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/RowGen.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.apache.cassandra.stress.generatedata; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import java.nio.ByteBuffer; -import java.util.List; - -/** - * Generates a row of data, by constructing one byte buffers per column according to some algorithm - * and delegating the work of populating the values of those byte buffers to the provided data generator - */ -public abstract class RowGen -{ - - final DataGen dataGen; - protected RowGen(DataGen dataGenerator) - { - this.dataGen = dataGenerator; - } - - public List generate(long operationIndex, ByteBuffer key) - { - List fill = getColumns(operationIndex); - dataGen.generate(fill, operationIndex, key); - return fill; - } - - // these byte[] may be re-used - abstract List getColumns(long operationIndex); - abstract public int count(long operationIndex); - - abstract public boolean isDeterministic(); - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/generatedata/RowGenDistributedSize.java b/tools/stress/src/org/apache/cassandra/stress/generatedata/RowGenDistributedSize.java deleted file mode 100644 index fffad2f05a..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/generatedata/RowGenDistributedSize.java +++ /dev/null @@ -1,116 +0,0 @@ -package org.apache.cassandra.stress.generatedata; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -public class RowGenDistributedSize extends RowGen -{ - - // TODO - make configurable - static final int MAX_SINGLE_CACHE_SIZE = 16 * 1024; - - final Distribution countDistribution; - final Distribution sizeDistribution; - - final TreeMap cache = new TreeMap<>(); - - // array re-used for returning columns - final ByteBuffer[] ret; - final int[] sizes; - - final boolean isDeterministic; - - public RowGenDistributedSize(DataGen dataGenerator, Distribution countDistribution, Distribution sizeDistribution) - { - super(dataGenerator); - this.countDistribution = countDistribution; - this.sizeDistribution = sizeDistribution; - ret = new ByteBuffer[(int) countDistribution.maxValue()]; - sizes = new int[ret.length]; - // TODO: should keep it deterministic in event that count distribution is not, but size and dataGen are, so that - // we simply need to generate the correct selection of columns - this.isDeterministic = dataGen.isDeterministic() && countDistribution.maxValue() == countDistribution.minValue() - && sizeDistribution.minValue() == sizeDistribution.maxValue(); - } - - ByteBuffer getBuffer(int size) - { - if (size >= MAX_SINGLE_CACHE_SIZE) - return ByteBuffer.allocate(size); - Map.Entry found = cache.ceilingEntry(size); - if (found == null) - { - // remove the next entry down, and replace it with a cache of this size - Integer del = cache.lowerKey(size); - if (del != null) - cache.remove(del); - return ByteBuffer.allocate(size); - } - ByteBuffer r = found.getValue(); - cache.remove(found.getKey()); - return r; - } - - @Override - List getColumns(long operationIndex) - { - int i = 0; - int count = (int) countDistribution.next(); - while (i < count) - { - int columnSize = (int) sizeDistribution.next(); - sizes[i] = columnSize; - ret[i] = getBuffer(columnSize); - i++; - } - while (i < ret.length && ret[i] != null) - ret[i] = null; - i = 0; - while (i < count) - { - ByteBuffer b = ret[i]; - cache.put(b.capacity(), b); - b.position(b.capacity() - sizes[i]); - ret[i] = b.slice(); - b.position(0); - i++; - } - return Arrays.asList(ret).subList(0, count); - } - - public int count(long operationIndex) - { - return (int) countDistribution.next(); - } - - @Override - public boolean isDeterministic() - { - return isDeterministic; - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/CqlIndexedRangeSlicer.java b/tools/stress/src/org/apache/cassandra/stress/operations/CqlIndexedRangeSlicer.java deleted file mode 100644 index 046381e582..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/operations/CqlIndexedRangeSlicer.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.apache.cassandra.stress.operations; -/* -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*/ - - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; - -import org.apache.cassandra.utils.FBUtilities; - -public class CqlIndexedRangeSlicer extends CqlOperation -{ - - volatile boolean acceptNoResults = false; - - public CqlIndexedRangeSlicer(State state, long idx) - { - super(state, idx); - } - - @Override - protected List getQueryParameters(byte[] key) - { - throw new UnsupportedOperationException(); - } - - @Override - protected String buildQuery() - { - StringBuilder query = new StringBuilder("SELECT"); - query.append(wrapInQuotesIfRequired("key")); - query.append(" FROM "); - query.append(wrapInQuotesIfRequired(state.type.table)); - - if (state.isCql2()) - query.append(" USING CONSISTENCY ").append(state.settings.command.consistencyLevel); - - final String columnName = (state.settings.columns.namestrs.get(1)); - query.append(" WHERE ").append(columnName).append("=?") - .append(" AND KEY > ? LIMIT ").append(state.settings.command.keysAtOnce); - return query.toString(); - } - - @Override - protected void run(CqlOperation.ClientWrapper client) throws IOException - { - acceptNoResults = false; - final List columns = generateColumnValues(getKey()); - final ByteBuffer value = columns.get(1); // only C1 column is indexed - byte[] minKey = new byte[0]; - int rowCount; - do - { - List params = Arrays.asList(value, ByteBuffer.wrap(minKey)); - CqlRunOp op = run(client, params, value, new String(value.array())); - byte[][] keys = op.result; - rowCount = keys.length; - minKey = getNextMinKey(minKey, keys); - acceptNoResults = true; - } while (rowCount > 0); - } - - private final class IndexedRangeSliceRunOp extends CqlRunOpFetchKeys - { - - protected IndexedRangeSliceRunOp(ClientWrapper client, String query, Object queryId, List params, String keyid, ByteBuffer key) - { - super(client, query, queryId, params, keyid, key); - } - - @Override - public boolean validate(byte[][] result) - { - return acceptNoResults || result.length > 0; - } - } - - @Override - protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, String keyid, ByteBuffer key) - { - return new IndexedRangeSliceRunOp(client, query, queryId, params, keyid, key); - } - - private static byte[] getNextMinKey(byte[] cur, byte[][] keys) - { - // find max - for (byte[] key : keys) - if (FBUtilities.compareUnsigned(cur, key) < 0) - cur = key; - - // increment - for (int i = 0 ; i < cur.length ; i++) - if (++cur[i] != 0) - break; - return cur; - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/CqlRangeSlicer.java b/tools/stress/src/org/apache/cassandra/stress/operations/CqlRangeSlicer.java deleted file mode 100644 index 16cdff31b6..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/operations/CqlRangeSlicer.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.apache.cassandra.stress.operations; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; - -public class CqlRangeSlicer extends CqlOperation -{ - public CqlRangeSlicer(State state, long idx) - { - super(state, idx); - } - - @Override - protected List getQueryParameters(byte[] key) - { - return Collections.singletonList(ByteBuffer.wrap(key)); - } - - @Override - protected String buildQuery() - { - StringBuilder query = new StringBuilder("SELECT FIRST ").append(state.settings.columns.maxColumnsPerKey) - .append(" ''..'' FROM ").append(wrapInQuotesIfRequired(state.type.table)); - - if (state.isCql2()) - query.append(" USING CONSISTENCY ").append(state.settings.command.consistencyLevel); - - return query.append(" WHERE KEY > ?").toString(); - } - - @Override - protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, String keyid, ByteBuffer key) - { - return new CqlRunOpTestNonEmpty(client, query, queryId, params, keyid, key); - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/FixedOpDistribution.java b/tools/stress/src/org/apache/cassandra/stress/operations/FixedOpDistribution.java new file mode 100644 index 0000000000..914d21233d --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/FixedOpDistribution.java @@ -0,0 +1,25 @@ +package org.apache.cassandra.stress.operations; + +import org.apache.cassandra.stress.Operation; + +public class FixedOpDistribution implements OpDistribution +{ + + final Operation operation; + + public FixedOpDistribution(Operation operation) + { + this.operation = operation; + } + + public Operation next() + { + return operation; + } + + public int maxBatchSize() + { + return (int) operation.partitionCount.maxValue(); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/OpDistribution.java b/tools/stress/src/org/apache/cassandra/stress/operations/OpDistribution.java new file mode 100644 index 0000000000..a744f188b6 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/OpDistribution.java @@ -0,0 +1,11 @@ +package org.apache.cassandra.stress.operations; + +import org.apache.cassandra.stress.Operation; + +public interface OpDistribution +{ + + Operation next(); + public int maxBatchSize(); + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/OpDistributionFactory.java b/tools/stress/src/org/apache/cassandra/stress/operations/OpDistributionFactory.java new file mode 100644 index 0000000000..08d5f5692e --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/OpDistributionFactory.java @@ -0,0 +1,12 @@ +package org.apache.cassandra.stress.operations; + +import org.apache.cassandra.stress.util.Timer; + +public interface OpDistributionFactory +{ + + public OpDistribution get(Timer timer); + public String desc(); + Iterable each(); + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistribution.java b/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistribution.java new file mode 100644 index 0000000000..8bd280603a --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistribution.java @@ -0,0 +1,41 @@ +package org.apache.cassandra.stress.operations; + +import org.apache.commons.math3.distribution.EnumeratedDistribution; +import org.apache.commons.math3.util.Pair; + +import org.apache.cassandra.stress.Operation; +import org.apache.cassandra.stress.generate.Distribution; + +public class SampledOpDistribution implements OpDistribution +{ + + final EnumeratedDistribution operations; + final Distribution clustering; + private Operation cur; + private long remaining; + + public SampledOpDistribution(EnumeratedDistribution operations, Distribution clustering) + { + this.operations = operations; + this.clustering = clustering; + } + + public int maxBatchSize() + { + int max = 1; + for (Pair pair : operations.getPmf()) + max = Math.max(max, (int) pair.getFirst().partitionCount.maxValue()); + return max; + } + + public Operation next() + { + while (remaining == 0) + { + remaining = clustering.next(); + cur = operations.sample(); + } + remaining--; + return cur; + } +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistributionFactory.java b/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistributionFactory.java new file mode 100644 index 0000000000..575da12872 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/SampledOpDistributionFactory.java @@ -0,0 +1,72 @@ +package org.apache.cassandra.stress.operations; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.math3.distribution.EnumeratedDistribution; +import org.apache.commons.math3.util.Pair; + +import org.apache.cassandra.stress.Operation; +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.util.Timer; + +public abstract class SampledOpDistributionFactory implements OpDistributionFactory +{ + + final List> ratios; + final DistributionFactory clustering; + protected SampledOpDistributionFactory(List> ratios, DistributionFactory clustering) + { + this.ratios = ratios; + this.clustering = clustering; + } + + protected abstract Operation get(Timer timer, PartitionGenerator generator, T key); + protected abstract PartitionGenerator newGenerator(); + + public OpDistribution get(Timer timer) + { + PartitionGenerator generator = newGenerator(); + List> operations = new ArrayList<>(); + for (Pair ratio : ratios) + operations.add(new Pair<>(get(timer, generator, ratio.getFirst()), ratio.getSecond())); + return new SampledOpDistribution(new EnumeratedDistribution<>(operations), clustering.get()); + } + + public String desc() + { + List keys = new ArrayList<>(); + for (Pair p : ratios) + keys.add(p.getFirst()); + return keys.toString(); + } + + public Iterable each() + { + List out = new ArrayList<>(); + for (final Pair ratio : ratios) + { + out.add(new OpDistributionFactory() + { + public OpDistribution get(Timer timer) + { + return new FixedOpDistribution(SampledOpDistributionFactory.this.get(timer, newGenerator(), ratio.getFirst())); + } + + public String desc() + { + return ratio.getFirst().toString(); + } + + public Iterable each() + { + return Collections.singleton(this); + } + }); + } + return out; + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftIndexedRangeSlicer.java b/tools/stress/src/org/apache/cassandra/stress/operations/ThriftIndexedRangeSlicer.java deleted file mode 100644 index 8c8ec31c18..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftIndexedRangeSlicer.java +++ /dev/null @@ -1,114 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.cassandra.stress.operations; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; - -import org.apache.cassandra.stress.Operation; -import org.apache.cassandra.stress.util.ThriftClient; -import org.apache.cassandra.thrift.*; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; - -public class ThriftIndexedRangeSlicer extends Operation -{ - public ThriftIndexedRangeSlicer(State state, long index) - { - super(state, index); - if (!state.rowGen.isDeterministic() || !state.keyGen.isDeterministic()) - throw new IllegalStateException("Only run with a isDeterministic row/key generator"); - if (state.settings.columns.useSuperColumns || state.columnParents.size() != 1) - throw new IllegalStateException("Does not support super columns"); - if (state.settings.columns.useTimeUUIDComparator) - throw new IllegalStateException("Does not support TimeUUID column names"); - } - - public void run(final ThriftClient client) throws IOException - { - - final SlicePredicate predicate = new SlicePredicate() - .setSlice_range(new SliceRange(ByteBufferUtil.EMPTY_BYTE_BUFFER, - ByteBufferUtil.EMPTY_BYTE_BUFFER, - false, state.settings.columns.maxColumnsPerKey)); - final List columns = generateColumnValues(getKey()); - final ColumnParent parent = state.columnParents.get(0); - - final ByteBuffer columnName = state.settings.columns.names.get(1); - final ByteBuffer value = columns.get(1); // only C1 column is indexed - - IndexExpression expression = new IndexExpression(columnName, IndexOperator.EQ, value); - byte[] minKey = new byte[0]; - final List[] results = new List[1]; - do - { - - final boolean first = minKey.length == 0; - final IndexClause clause = new IndexClause(Arrays.asList(expression), - ByteBuffer.wrap(minKey), - state.settings.command.keysAtOnce); - - timeWithRetry(new RunOp() - { - @Override - public boolean run() throws Exception - { - results[0] = client.get_indexed_slices(parent, clause, predicate, state.settings.command.consistencyLevel); - return !first || results[0].size() > 0; - } - - @Override - public String key() - { - return new String(value.array()); - } - - @Override - public int keyCount() - { - return results[0].size(); - } - }); - - minKey = getNextMinKey(minKey, results[0]); - - } while (results[0].size() > 0); - } - - /** - * Get maximum key from keySlice list - * @param slices list of the KeySlice objects - * @return maximum key value of the list - */ - private static byte[] getNextMinKey(byte[] cur, List slices) - { - // find max - for (KeySlice slice : slices) - if (FBUtilities.compareUnsigned(cur, slice.getKey()) < 0) - cur = slice.getKey(); - - // increment - for (int i = 0 ; i < cur.length ; i++) - if (++cur[i] != 0) - break; - return cur; - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftInserter.java b/tools/stress/src/org/apache/cassandra/stress/operations/ThriftInserter.java deleted file mode 100644 index 7077a95049..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftInserter.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.stress.operations; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.apache.cassandra.db.marshal.TimeUUIDType; -import org.apache.cassandra.stress.Operation; -import org.apache.cassandra.stress.util.ThriftClient; -import org.apache.cassandra.thrift.*; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.UUIDGen; - -public final class ThriftInserter extends Operation -{ - - public ThriftInserter(State state, long index) - { - super(state, index); - } - - public void run(final ThriftClient client) throws IOException - { - final ByteBuffer key = getKey(); - final List columns = generateColumns(key); - - Map> row; - if (!state.settings.columns.useSuperColumns) - { - List mutations = new ArrayList<>(columns.size()); - for (Column c : columns) - { - ColumnOrSuperColumn column = new ColumnOrSuperColumn().setColumn(c); - mutations.add(new Mutation().setColumn_or_supercolumn(column)); - } - row = Collections.singletonMap(state.type.table, mutations); - } - else - { - List mutations = new ArrayList<>(state.columnParents.size()); - for (ColumnParent parent : state.columnParents) - { - final SuperColumn s = new SuperColumn(parent.bufferForSuper_column(), columns); - final ColumnOrSuperColumn cosc = new ColumnOrSuperColumn().setSuper_column(s); - mutations.add(new Mutation().setColumn_or_supercolumn(cosc)); - } - row = Collections.singletonMap(state.settings.command.type.supertable, mutations); - } - - final Map>> record = Collections.singletonMap(key, row); - - timeWithRetry(new RunOp() - { - @Override - public boolean run() throws Exception - { - client.batch_mutate(record, state.settings.command.consistencyLevel); - return true; - } - - @Override - public String key() - { - return new String(key.array()); - } - - @Override - public int keyCount() - { - return 1; - } - }); - } - - protected List generateColumns(ByteBuffer key) - { - final List values = generateColumnValues(key); - final List columns = new ArrayList<>(values.size()); - - if (state.settings.columns.useTimeUUIDComparator) - for (int i = 0 ; i < values.size() ; i++) - new Column(TimeUUIDType.instance.decompose(UUIDGen.getTimeUUID())); - else - // TODO : consider randomly allocating column names in case where have fewer than max columns - // but need to think about implications for indexes / indexed range slicer / other knock on effects - for (int i = 0 ; i < values.size() ; i++) - columns.add(new Column(state.settings.columns.names.get(i))); - - for (int i = 0 ; i < values.size() ; i++) - columns.get(i) - .setValue(values.get(i)) - .setTimestamp(FBUtilities.timestampMicros()); - - return columns; - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftMultiGetter.java b/tools/stress/src/org/apache/cassandra/stress/operations/ThriftMultiGetter.java deleted file mode 100644 index d8e0117e98..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftMultiGetter.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.stress.operations; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.cassandra.stress.Operation; -import org.apache.cassandra.stress.util.ThriftClient; -import org.apache.cassandra.thrift.ColumnParent; -import org.apache.cassandra.thrift.SlicePredicate; -import org.apache.cassandra.thrift.SliceRange; -import org.apache.cassandra.utils.ByteBufferUtil; - - -public final class ThriftMultiGetter extends Operation -{ - - public ThriftMultiGetter(State state, long index) - { - super(state, index); - } - - public void run(final ThriftClient client) throws IOException - { - - final SlicePredicate predicate = new SlicePredicate().setSlice_range( - new SliceRange( - ByteBufferUtil.EMPTY_BYTE_BUFFER, - ByteBufferUtil.EMPTY_BYTE_BUFFER, - false, - state.settings.columns.maxColumnsPerKey - ) - ); - - final List keys = getKeys(state.settings.command.keysAtOnce); - - for (final ColumnParent parent : state.columnParents) - { - timeWithRetry(new RunOp() - { - int count; - @Override - public boolean run() throws Exception - { - return (count = client.multiget_slice(keys, parent, predicate, state.settings.command.consistencyLevel).size()) != 0; - } - - @Override - public String key() - { - return keys.toString(); - } - - @Override - public int keyCount() - { - return count; - } - }); - } - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftRangeSlicer.java b/tools/stress/src/org/apache/cassandra/stress/operations/ThriftRangeSlicer.java deleted file mode 100644 index 021c4e842e..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftRangeSlicer.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.stress.operations; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.apache.cassandra.stress.Operation; -import org.apache.cassandra.stress.util.ThriftClient; -import org.apache.cassandra.thrift.ColumnParent; -import org.apache.cassandra.thrift.KeyRange; -import org.apache.cassandra.thrift.SlicePredicate; -import org.apache.cassandra.thrift.SliceRange; -import org.apache.cassandra.utils.ByteBufferUtil; - -public final class ThriftRangeSlicer extends Operation -{ - - public ThriftRangeSlicer(State state, long index) - { - super(state, index); - } - - @Override - public void run(final ThriftClient client) throws IOException - { - final SlicePredicate predicate = new SlicePredicate() - .setSlice_range( - new SliceRange( - ByteBufferUtil.EMPTY_BYTE_BUFFER, - ByteBufferUtil.EMPTY_BYTE_BUFFER, - false, - state.settings.columns.maxColumnsPerKey - ) - ); - - final ByteBuffer start = getKey(); - final KeyRange range = - new KeyRange(state.settings.columns.maxColumnsPerKey) - .setStart_key(start) - .setEnd_key(ByteBufferUtil.EMPTY_BYTE_BUFFER) - .setCount(state.settings.command.keysAtOnce); - - for (final ColumnParent parent : state.columnParents) - { - timeWithRetry(new RunOp() - { - private int count = 0; - @Override - public boolean run() throws Exception - { - return (count = client.get_range_slices(parent, predicate, range, state.settings.command.consistencyLevel).size()) != 0; - } - - @Override - public String key() - { - return new String(range.bufferForStart_key().array()); - } - - @Override - public int keyCount() - { - return count; - } - }); - } - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftReader.java b/tools/stress/src/org/apache/cassandra/stress/operations/ThriftReader.java deleted file mode 100644 index dccf4696bc..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftReader.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.stress.operations; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.cassandra.stress.Operation; -import org.apache.cassandra.stress.util.ThriftClient; -import org.apache.cassandra.thrift.ColumnOrSuperColumn; -import org.apache.cassandra.thrift.ColumnParent; -import org.apache.cassandra.thrift.SlicePredicate; -import org.apache.cassandra.thrift.SuperColumn; - -public final class ThriftReader extends Operation -{ - - public ThriftReader(State state, long index) - { - super(state, index); - } - - public void run(final ThriftClient client) throws IOException - { - final SlicePredicate predicate = slicePredicate(); - final ByteBuffer key = getKey(); - final List expect = state.rowGen.isDeterministic() ? generateColumnValues(key) : null; - for (final ColumnParent parent : state.columnParents) - { - timeWithRetry(new RunOp() - { - @Override - public boolean run() throws Exception - { - List row = client.get_slice(key, parent, predicate, state.settings.command.consistencyLevel); - if (expect == null) - return !row.isEmpty(); - if (row == null) - return false; - if (!state.settings.columns.useSuperColumns) - { - if (row.size() != expect.size()) - return false; - for (int i = 0 ; i < row.size() ; i++) - if (!row.get(i).getColumn().bufferForValue().equals(expect.get(i))) - return false; - } - else - { - for (ColumnOrSuperColumn col : row) - { - SuperColumn superColumn = col.getSuper_column(); - if (superColumn.getColumns().size() != expect.size()) - return false; - for (int i = 0 ; i < expect.size() ; i++) - if (!superColumn.getColumns().get(i).bufferForValue().equals(expect.get(i))) - return false; - } - } - return true; - } - - @Override - public String key() - { - return new String(key.array()); - } - - @Override - public int keyCount() - { - return 1; - } - }); - } - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/CqlCounterAdder.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterAdder.java similarity index 61% rename from tools/stress/src/org/apache/cassandra/stress/operations/CqlCounterAdder.java rename to tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterAdder.java index 9a8c37de2f..f794e75a15 100644 --- a/tools/stress/src/org/apache/cassandra/stress/operations/CqlCounterAdder.java +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterAdder.java @@ -1,4 +1,4 @@ -package org.apache.cassandra.stress.operations; +package org.apache.cassandra.stress.operations.predefined; /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -23,32 +23,39 @@ package org.apache.cassandra.stress.operations; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.stress.generate.Distribution; +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.Command; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.util.Timer; public class CqlCounterAdder extends CqlOperation { - public CqlCounterAdder(State state, long idx) + + final Distribution counteradd; + public CqlCounterAdder(DistributionFactory counteradd, Timer timer, PartitionGenerator generator, StressSettings settings) { - super(state, idx); + super(Command.COUNTER_WRITE, timer, generator, settings); + this.counteradd = counteradd.get(); } @Override protected String buildQuery() { - String counterCF = state.isCql2() ? state.type.table : "Counter3"; + String counterCF = isCql2() ? type.table : "Counter3"; StringBuilder query = new StringBuilder("UPDATE ").append(wrapInQuotesIfRequired(counterCF)); - if (state.isCql2()) - query.append(" USING CONSISTENCY ").append(state.settings.command.consistencyLevel); + if (isCql2()) + query.append(" USING CONSISTENCY ").append(settings.command.consistencyLevel); query.append(" SET "); // TODO : increment distribution subset of columns - for (int i = 0; i < state.settings.columns.maxColumnsPerKey; i++) + for (int i = 0; i < settings.columns.maxColumnsPerKey; i++) { if (i > 0) query.append(","); @@ -63,15 +70,15 @@ public class CqlCounterAdder extends CqlOperation protected List getQueryParameters(byte[] key) { final List list = new ArrayList<>(); - for (int i = 0; i < state.settings.columns.maxColumnsPerKey; i++) - list.add(state.counteradd.next()); + for (int i = 0; i < settings.columns.maxColumnsPerKey; i++) + list.add(counteradd.next()); list.add(ByteBuffer.wrap(key)); return list; } @Override - protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, String keyid, ByteBuffer key) + protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, ByteBuffer key) { - return new CqlRunOpAlwaysSucceed(client, query, queryId, params, keyid, key, 1); + return new CqlRunOpAlwaysSucceed(client, query, queryId, params, key, 1); } } diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/CqlCounterGetter.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterGetter.java similarity index 66% rename from tools/stress/src/org/apache/cassandra/stress/operations/CqlCounterGetter.java rename to tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterGetter.java index 88d622ebbb..94c8faf097 100644 --- a/tools/stress/src/org/apache/cassandra/stress/operations/CqlCounterGetter.java +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlCounterGetter.java @@ -1,4 +1,4 @@ -package org.apache.cassandra.stress.operations; +package org.apache.cassandra.stress.operations.predefined; /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -25,12 +25,17 @@ import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.Command; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.util.Timer; + public class CqlCounterGetter extends CqlOperation { - public CqlCounterGetter(State state, long idx) + public CqlCounterGetter(Timer timer, PartitionGenerator generator, StressSettings settings) { - super(state, idx); + super(Command.COUNTER_READ, timer, generator, settings); } @Override @@ -45,25 +50,25 @@ public class CqlCounterGetter extends CqlOperation StringBuilder query = new StringBuilder("SELECT "); // TODO: obey slice/noslice option (instead of always slicing) - if (state.isCql2()) - query.append("FIRST ").append(state.settings.columns.maxColumnsPerKey).append(" ''..''"); + if (isCql2()) + query.append("FIRST ").append(settings.columns.maxColumnsPerKey).append(" ''..''"); else query.append("*"); - String counterCF = state.isCql2() ? state.type.table : "Counter3"; + String counterCF = isCql2() ? type.table : "Counter3"; query.append(" FROM ").append(wrapInQuotesIfRequired(counterCF)); - if (state.isCql2()) - query.append(" USING CONSISTENCY ").append(state.settings.command.consistencyLevel); + if (isCql2()) + query.append(" USING CONSISTENCY ").append(settings.command.consistencyLevel); return query.append(" WHERE KEY=?").toString(); } @Override - protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, String keyid, ByteBuffer key) + protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, ByteBuffer key) { - return new CqlRunOpTestNonEmpty(client, query, queryId, params, keyid, key); + return new CqlRunOpTestNonEmpty(client, query, queryId, params, key); } } diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/CqlInserter.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlInserter.java similarity index 59% rename from tools/stress/src/org/apache/cassandra/stress/operations/CqlInserter.java rename to tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlInserter.java index 71cdadfa9a..c422f2b165 100644 --- a/tools/stress/src/org/apache/cassandra/stress/operations/CqlInserter.java +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlInserter.java @@ -1,4 +1,4 @@ -package org.apache.cassandra.stress.operations; +package org.apache.cassandra.stress.operations.predefined; /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -25,45 +25,36 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.Command; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.util.Timer; import org.apache.cassandra.utils.UUIDGen; public class CqlInserter extends CqlOperation { - public CqlInserter(State state, long idx) + public CqlInserter(Timer timer, PartitionGenerator generator, StressSettings settings) { - super(state, idx); - if (state.settings.columns.useTimeUUIDComparator) - throw new IllegalStateException("Cannot use TimeUUID Comparator with CQL"); + super(Command.WRITE, timer, generator, settings); } @Override protected String buildQuery() { - StringBuilder query = new StringBuilder("UPDATE ").append(wrapInQuotesIfRequired(state.type.table)); + StringBuilder query = new StringBuilder("UPDATE ").append(wrapInQuotesIfRequired(type.table)); - if (state.isCql2()) - query.append(" USING CONSISTENCY ").append(state.settings.command.consistencyLevel); + if (isCql2()) + query.append(" USING CONSISTENCY ").append(settings.command.consistencyLevel); query.append(" SET "); - for (int i = 0 ; i < state.settings.columns.maxColumnsPerKey; i++) + for (int i = 0 ; i < settings.columns.maxColumnsPerKey ; i++) { if (i > 0) query.append(','); - if (state.settings.columns.useTimeUUIDComparator) - { - if (state.isCql3()) - throw new UnsupportedOperationException("Cannot use UUIDs in column names with CQL3"); - - query.append(wrapInQuotesIfRequired(UUIDGen.getTimeUUID().toString())) - .append(" = ?"); - } - else - { - query.append(wrapInQuotesIfRequired("C" + i)).append(" = ?"); - } + query.append(wrapInQuotesIfRequired(settings.columns.namestrs.get(i))).append(" = ?"); } query.append(" WHERE KEY=?"); @@ -74,15 +65,15 @@ public class CqlInserter extends CqlOperation protected List getQueryParameters(byte[] key) { final ArrayList queryParams = new ArrayList<>(); - final List values = generateColumnValues(ByteBuffer.wrap(key)); + List values = getColumnValues(); queryParams.addAll(values); queryParams.add(ByteBuffer.wrap(key)); return queryParams; } @Override - protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, String keyid, ByteBuffer key) + protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, ByteBuffer key) { - return new CqlRunOpAlwaysSucceed(client, query, queryId, params, keyid, key, 1); + return new CqlRunOpAlwaysSucceed(client, query, queryId, params, key, 1); } } diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/CqlOperation.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlOperation.java similarity index 88% rename from tools/stress/src/org/apache/cassandra/stress/operations/CqlOperation.java rename to tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlOperation.java index 1c59e2d1b3..0264cd1cb0 100644 --- a/tools/stress/src/org/apache/cassandra/stress/operations/CqlOperation.java +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlOperation.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.cassandra.stress.operations; +package org.apache.cassandra.stress.operations.predefined; import java.io.IOException; import java.nio.ByteBuffer; @@ -29,10 +29,15 @@ import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; import com.google.common.base.Function; import org.apache.cassandra.stress.Operation; +import org.apache.cassandra.stress.StressMetrics; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.Command; import org.apache.cassandra.stress.settings.ConnectionStyle; import org.apache.cassandra.stress.settings.CqlVersion; +import org.apache.cassandra.stress.settings.StressSettings; import org.apache.cassandra.stress.util.JavaDriverClient; import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; import org.apache.cassandra.thrift.Compression; import org.apache.cassandra.thrift.CqlResult; import org.apache.cassandra.thrift.CqlRow; @@ -42,31 +47,27 @@ import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.thrift.TException; -public abstract class CqlOperation extends Operation +public abstract class CqlOperation extends PredefinedOperation { protected abstract List getQueryParameters(byte[] key); protected abstract String buildQuery(); - protected abstract CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, String id, ByteBuffer key); + protected abstract CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, ByteBuffer key); - public CqlOperation(State state, long idx) + public CqlOperation(Command type, Timer timer, PartitionGenerator generator, StressSettings settings) { - super(state, idx); - if (state.settings.columns.useSuperColumns) - throw new IllegalStateException("Super columns are not implemented for CQL"); - if (state.settings.columns.variableColumnCount) + super(type, timer, generator, settings); + if (settings.columns.variableColumnCount) throw new IllegalStateException("Variable column counts are not implemented for CQL"); - if (state.settings.columns.useTimeUUIDComparator) - throw new IllegalStateException("Cannot use TimeUUID Comparator with CQL"); } - protected CqlRunOp run(final ClientWrapper client, final List queryParams, final ByteBuffer key, final String keyid) throws IOException + protected CqlRunOp run(final ClientWrapper client, final List queryParams, final ByteBuffer key) throws IOException { final CqlRunOp op; - if (state.settings.mode.style == ConnectionStyle.CQL_PREPARED) + if (settings.mode.style == ConnectionStyle.CQL_PREPARED) { final Object id; - Object idobj = state.getCqlCache(); + Object idobj = getCqlCache(); if (idobj == null) { try @@ -76,23 +77,23 @@ public abstract class CqlOperation extends Operation { throw new RuntimeException(e); } - state.storeCqlCache(id); + storeCqlCache(id); } else id = idobj; - op = buildRunOp(client, null, id, queryParams, keyid, key); + op = buildRunOp(client, null, id, queryParams, key); } else { final String query; - Object qobj = state.getCqlCache(); + Object qobj = getCqlCache(); if (qobj == null) - state.storeCqlCache(query = buildQuery()); + storeCqlCache(query = buildQuery()); else query = qobj.toString(); - op = buildRunOp(client, query, null, queryParams, keyid, key); + op = buildRunOp(client, query, null, queryParams, key); } timeWithRetry(op); @@ -103,7 +104,7 @@ public abstract class CqlOperation extends Operation { final byte[] key = getKey().array(); final List queryParams = getQueryParameters(key); - run(client, queryParams, ByteBuffer.wrap(key), new String(key)); + run(client, queryParams, ByteBuffer.wrap(key)); } // Classes to process Cql results @@ -114,9 +115,9 @@ public abstract class CqlOperation extends Operation final int keyCount; - protected CqlRunOpAlwaysSucceed(ClientWrapper client, String query, Object queryId, List params, String id, ByteBuffer key, int keyCount) + protected CqlRunOpAlwaysSucceed(ClientWrapper client, String query, Object queryId, List params, ByteBuffer key, int keyCount) { - super(client, query, queryId, RowCountHandler.INSTANCE, params, id, key); + super(client, query, queryId, RowCountHandler.INSTANCE, params, key); this.keyCount = keyCount; } @@ -127,7 +128,13 @@ public abstract class CqlOperation extends Operation } @Override - public int keyCount() + public int partitionCount() + { + return keyCount; + } + + @Override + public int rowCount() { return keyCount; } @@ -137,9 +144,9 @@ public abstract class CqlOperation extends Operation protected final class CqlRunOpTestNonEmpty extends CqlRunOp { - protected CqlRunOpTestNonEmpty(ClientWrapper client, String query, Object queryId, List params, String id, ByteBuffer key) + protected CqlRunOpTestNonEmpty(ClientWrapper client, String query, Object queryId, List params, ByteBuffer key) { - super(client, query, queryId, RowCountHandler.INSTANCE, params, id, key); + super(client, query, queryId, RowCountHandler.INSTANCE, params, key); } @Override @@ -149,7 +156,13 @@ public abstract class CqlOperation extends Operation } @Override - public int keyCount() + public int partitionCount() + { + return result; + } + + @Override + public int rowCount() { return result; } @@ -159,17 +172,22 @@ public abstract class CqlOperation extends Operation protected abstract class CqlRunOpFetchKeys extends CqlRunOp { - protected CqlRunOpFetchKeys(ClientWrapper client, String query, Object queryId, List params, String id, ByteBuffer key) + protected CqlRunOpFetchKeys(ClientWrapper client, String query, Object queryId, List params, ByteBuffer key) { - super(client, query, queryId, KeysHandler.INSTANCE, params, id, key); + super(client, query, queryId, KeysHandler.INSTANCE, params, key); } @Override - public int keyCount() + public int partitionCount() { return result.length; } + @Override + public int rowCount() + { + return result.length; + } } protected final class CqlRunOpMatchResults extends CqlRunOp @@ -178,14 +196,20 @@ public abstract class CqlOperation extends Operation final List> expect; // a null value for an item in expect means we just check the row is present - protected CqlRunOpMatchResults(ClientWrapper client, String query, Object queryId, List params, String id, ByteBuffer key, List> expect) + protected CqlRunOpMatchResults(ClientWrapper client, String query, Object queryId, List params, ByteBuffer key, List> expect) { - super(client, query, queryId, RowsHandler.INSTANCE, params, id, key); + super(client, query, queryId, RowsHandler.INSTANCE, params, key); this.expect = expect; } @Override - public int keyCount() + public int partitionCount() + { + return result == null ? 0 : result.length; + } + + @Override + public int rowCount() { return result == null ? 0 : result.length; } @@ -209,19 +233,17 @@ public abstract class CqlOperation extends Operation final String query; final Object queryId; final List params; - final String id; final ByteBuffer key; final ResultHandler handler; V result; - private CqlRunOp(ClientWrapper client, String query, Object queryId, ResultHandler handler, List params, String id, ByteBuffer key) + private CqlRunOp(ClientWrapper client, String query, Object queryId, ResultHandler handler, List params, ByteBuffer key) { this.client = client; this.query = query; this.queryId = queryId; this.handler = handler; this.params = params; - this.id = id; this.key = key; } @@ -233,12 +255,6 @@ public abstract class CqlOperation extends Operation : validate(result = client.execute(query, key, params, handler)); } - @Override - public String key() - { - return id; - } - public abstract boolean validate(V result); } @@ -267,7 +283,7 @@ public abstract class CqlOperation extends Operation public ClientWrapper wrap(ThriftClient client) { - return state.isCql3() + return isCql3() ? new Cql3CassandraClientWrapper(client) : new Cql2CassandraClientWrapper(client); @@ -301,8 +317,8 @@ public abstract class CqlOperation extends Operation @Override public V execute(String query, ByteBuffer key, List queryParams, ResultHandler handler) { - String formattedQuery = formatCqlQuery(query, queryParams, state.isCql3()); - return handler.javaDriverHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(state.settings.command.consistencyLevel))); + String formattedQuery = formatCqlQuery(query, queryParams, isCql3()); + return handler.javaDriverHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(settings.command.consistencyLevel))); } @Override @@ -312,7 +328,7 @@ public abstract class CqlOperation extends Operation client.executePrepared( (PreparedStatement) preparedStatementId, queryParams, - ThriftConversion.fromThrift(state.settings.command.consistencyLevel))); + ThriftConversion.fromThrift(settings.command.consistencyLevel))); } @Override @@ -333,8 +349,8 @@ public abstract class CqlOperation extends Operation @Override public V execute(String query, ByteBuffer key, List queryParams, ResultHandler handler) { - String formattedQuery = formatCqlQuery(query, queryParams, state.isCql3()); - return handler.thriftHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(state.settings.command.consistencyLevel))); + String formattedQuery = formatCqlQuery(query, queryParams, isCql3()); + return handler.thriftHandler().apply(client.execute(formattedQuery, ThriftConversion.fromThrift(settings.command.consistencyLevel))); } @Override @@ -344,7 +360,7 @@ public abstract class CqlOperation extends Operation client.executePrepared( (byte[]) preparedStatementId, toByteBufferParams(queryParams), - ThriftConversion.fromThrift(state.settings.command.consistencyLevel))); + ThriftConversion.fromThrift(settings.command.consistencyLevel))); } @Override @@ -368,7 +384,7 @@ public abstract class CqlOperation extends Operation { String formattedQuery = formatCqlQuery(query, queryParams, true); return handler.simpleNativeHandler().apply( - client.execute_cql3_query(formattedQuery, key, Compression.NONE, state.settings.command.consistencyLevel) + client.execute_cql3_query(formattedQuery, key, Compression.NONE, settings.command.consistencyLevel) ); } @@ -377,7 +393,7 @@ public abstract class CqlOperation extends Operation { Integer id = (Integer) preparedStatementId; return handler.simpleNativeHandler().apply( - client.execute_prepared_cql3_query(id, key, toByteBufferParams(queryParams), state.settings.command.consistencyLevel) + client.execute_prepared_cql3_query(id, key, toByteBufferParams(queryParams), settings.command.consistencyLevel) ); } @@ -690,7 +706,7 @@ public abstract class CqlOperation extends Operation protected String wrapInQuotesIfRequired(String string) { - return state.settings.mode.cqlVersion == CqlVersion.CQL3 + return settings.mode.cqlVersion == CqlVersion.CQL3 ? "\"" + string + "\"" : string; } diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/CqlReader.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlReader.java similarity index 58% rename from tools/stress/src/org/apache/cassandra/stress/operations/CqlReader.java rename to tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlReader.java index fb07edcea5..3a7f75ab81 100644 --- a/tools/stress/src/org/apache/cassandra/stress/operations/CqlReader.java +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/CqlReader.java @@ -1,4 +1,4 @@ -package org.apache.cassandra.stress.operations; +package org.apache.cassandra.stress.operations.predefined; /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -23,19 +23,22 @@ package org.apache.cassandra.stress.operations; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.Command; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.util.Timer; import org.apache.cassandra.utils.ByteBufferUtil; public class CqlReader extends CqlOperation { - public CqlReader(State state, long idx) + public CqlReader(Timer timer, PartitionGenerator generator, StressSettings settings) { - super(state, idx); + super(Command.READ, timer, generator, settings); } @Override @@ -43,34 +46,27 @@ public class CqlReader extends CqlOperation { StringBuilder query = new StringBuilder("SELECT "); - if (state.settings.columns.slice) + if (settings.columns.slice) { - if (state.isCql2()) - query.append("FIRST ").append(state.settings.columns.maxColumnsPerKey).append(" ''..''"); + if (isCql2()) + query.append("FIRST ").append(settings.columns.maxColumnsPerKey).append(" ''..''"); else query.append("*"); } else { - try + for (int i = 0; i < settings.columns.maxColumnsPerKey ; i++) { - for (int i = 0; i < state.settings.columns.names.size() ; i++) - { - if (i > 0) - query.append(","); - query.append(wrapInQuotesIfRequired(ByteBufferUtil.string(state.settings.columns.names.get(i)))); - } - } - catch (CharacterCodingException e) - { - throw new IllegalStateException(e); + if (i > 0) + query.append(","); + query.append(wrapInQuotesIfRequired(settings.columns.namestrs.get(i))); } } - query.append(" FROM ").append(wrapInQuotesIfRequired(state.type.table)); + query.append(" FROM ").append(wrapInQuotesIfRequired(type.table)); - if (state.isCql2()) - query.append(" USING CONSISTENCY ").append(state.settings.command.consistencyLevel); + if (isCql2()) + query.append(" USING CONSISTENCY ").append(settings.command.consistencyLevel); query.append(" WHERE KEY=?"); return query.toString(); } @@ -82,10 +78,10 @@ public class CqlReader extends CqlOperation } @Override - protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, String keyid, ByteBuffer key) + protected CqlRunOp buildRunOp(ClientWrapper client, String query, Object queryId, List params, ByteBuffer key) { - List expectRow = state.rowGen.isDeterministic() ? generateColumnValues(key) : null; - return new CqlRunOpMatchResults(client, query, queryId, params, keyid, key, Arrays.asList(expectRow)); + List expectRow = getColumnValues(); + return new CqlRunOpMatchResults(client, query, queryId, params, key, Arrays.asList(expectRow)); } } diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/PredefinedOperation.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/PredefinedOperation.java new file mode 100644 index 0000000000..7f6412b3ba --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/PredefinedOperation.java @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.stress.operations.predefined; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +import org.apache.cassandra.stress.Operation; +import org.apache.cassandra.stress.StressMetrics; +import org.apache.cassandra.stress.generate.Distribution; +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.stress.generate.DistributionFixed; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.generate.Row; +import org.apache.cassandra.stress.settings.Command; +import org.apache.cassandra.stress.settings.CqlVersion; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.util.Timer; +import org.apache.cassandra.thrift.ColumnParent; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.SliceRange; +import org.apache.cassandra.utils.ByteBufferUtil; + +public abstract class PredefinedOperation extends Operation +{ + public final Command type; + private final Distribution columnCount; + private Object cqlCache; + + public PredefinedOperation(Command type, Timer timer, PartitionGenerator generator, StressSettings settings) + { + super(timer, generator, settings, new DistributionFixed(1)); + this.type = type; + this.columnCount = settings.columns.countDistribution.get(); + } + + public boolean isCql3() + { + return settings.mode.cqlVersion == CqlVersion.CQL3; + } + public boolean isCql2() + { + return settings.mode.cqlVersion == CqlVersion.CQL2; + } + public Object getCqlCache() + { + return cqlCache; + } + public void storeCqlCache(Object val) + { + cqlCache = val; + } + + protected ByteBuffer getKey() + { + return (ByteBuffer) partitions.get(0).getPartitionKey(0); + } + + final class ColumnSelection + { + final int[] indices; + final int lb, ub; + private ColumnSelection(int[] indices, int lb, int ub) + { + this.indices = indices; + this.lb = lb; + this.ub = ub; + } + + public List select(List in) + { + List out = new ArrayList<>(); + if (indices != null) + { + for (int i : indices) + out.add(in.get(i)); + } + else + { + out.addAll(in.subList(lb, ub)); + } + return out; + } + + int count() + { + return indices != null ? indices.length : ub - lb; + } + + SlicePredicate predicate() + { + final SlicePredicate predicate = new SlicePredicate(); + if (indices == null) + { + predicate.setSlice_range(new SliceRange() + .setStart(settings.columns.names.get(lb)) + .setFinish(new byte[] {}) + .setReversed(false) + .setCount(count()) + ); + } + else + predicate.setColumn_names(select(settings.columns.names)); + return predicate; + + } + } + + public String toString() + { + return type.toString(); + } + + ColumnSelection select() + { + if (settings.columns.slice) + { + int count = (int) columnCount.next(); + int start; + if (count == settings.columns.maxColumnsPerKey) + start = 0; + else + start = 1 + ThreadLocalRandom.current().nextInt(settings.columns.maxColumnsPerKey - count); + return new ColumnSelection(null, start, start + count); + } + + int count = (int) columnCount.next(); + int totalCount = settings.columns.names.size(); + if (count == settings.columns.names.size()) + return new ColumnSelection(null, 0, count); + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + int[] indices = new int[count]; + int c = 0, o = 0; + while (c < count && count + o < totalCount) + { + int leeway = totalCount - (count + o); + int spreadover = count - c; + o += Math.round(rnd.nextDouble() * (leeway / (double) spreadover)); + indices[c] = o + c; + c++; + } + while (c < count) + { + indices[c] = o + c; + c++; + } + return new ColumnSelection(indices, 0, 0); + } + + protected List getColumnValues() + { + return getColumnValues(new ColumnSelection(null, 0, settings.columns.names.size())); + } + + protected List getColumnValues(ColumnSelection columns) + { + Row row = partitions.get(0).iterator(1).batch(1f).iterator().next(); + ByteBuffer[] r = new ByteBuffer[columns.count()]; + int c = 0; + if (columns.indices != null) + for (int i : columns.indices) + r[c++] = (ByteBuffer) row.get(i); + else + for (int i = columns.lb ; i < columns.ub ; i++) + r[c++] = (ByteBuffer) row.get(i); + return Arrays.asList(r); + } + + public static Operation operation(Command type, Timer timer, PartitionGenerator generator, StressSettings settings, DistributionFactory counteradd) + { + switch (type) + { + case READ: + switch(settings.mode.style) + { + case THRIFT: + return new ThriftReader(timer, generator, settings); + case CQL: + case CQL_PREPARED: + return new CqlReader(timer, generator, settings); + default: + throw new UnsupportedOperationException(); + } + + + case COUNTER_READ: + switch(settings.mode.style) + { + case THRIFT: + return new ThriftCounterGetter(timer, generator, settings); + case CQL: + case CQL_PREPARED: + return new CqlCounterGetter(timer, generator, settings); + default: + throw new UnsupportedOperationException(); + } + + case WRITE: + + switch(settings.mode.style) + { + case THRIFT: + return new ThriftInserter(timer, generator, settings); + case CQL: + case CQL_PREPARED: + return new CqlInserter(timer, generator, settings); + default: + throw new UnsupportedOperationException(); + } + + case COUNTER_WRITE: + switch(settings.mode.style) + { + case THRIFT: + return new ThriftCounterAdder(counteradd, timer, generator, settings); + case CQL: + case CQL_PREPARED: + return new CqlCounterAdder(counteradd, timer, generator, settings); + default: + throw new UnsupportedOperationException(); + } + + } + + throw new UnsupportedOperationException(); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftCounterAdder.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterAdder.java similarity index 50% rename from tools/stress/src/org/apache/cassandra/stress/operations/ThriftCounterAdder.java rename to tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterAdder.java index 9bfe4406a3..ee766c36d8 100644 --- a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftCounterAdder.java +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterAdder.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.stress.operations; +package org.apache.cassandra.stress.operations.predefined; import java.io.IOException; import java.nio.ByteBuffer; @@ -23,48 +23,39 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; -import org.apache.cassandra.stress.Operation; +import org.apache.cassandra.stress.generate.Distribution; +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.Command; +import org.apache.cassandra.stress.settings.StressSettings; import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; import org.apache.cassandra.thrift.*; -public class ThriftCounterAdder extends Operation +public class ThriftCounterAdder extends PredefinedOperation { - public ThriftCounterAdder(State state, long index) + + final Distribution counteradd; + public ThriftCounterAdder(DistributionFactory counteradd, Timer timer, PartitionGenerator generator, StressSettings settings) { - super(state, index); + super(Command.COUNTER_WRITE, timer, generator, settings); + this.counteradd = counteradd.get(); } public void run(final ThriftClient client) throws IOException { List columns = new ArrayList<>(); - for (ByteBuffer name : randomNames()) - columns.add(new CounterColumn(name, state.counteradd.next())); + for (ByteBuffer name : select().select(settings.columns.names)) + columns.add(new CounterColumn(name, counteradd.next())); - Map> row; - if (state.settings.columns.useSuperColumns) + List mutations = new ArrayList<>(columns.size()); + for (CounterColumn c : columns) { - List mutations = new ArrayList<>(); - for (ColumnParent parent : state.columnParents) - { - CounterSuperColumn csc = new CounterSuperColumn(ByteBuffer.wrap(parent.getSuper_column()), columns); - ColumnOrSuperColumn cosc = new ColumnOrSuperColumn().setCounter_super_column(csc); - mutations.add(new Mutation().setColumn_or_supercolumn(cosc)); - } - row = Collections.singletonMap(state.type.supertable, mutations); - } - else - { - List mutations = new ArrayList<>(columns.size()); - for (CounterColumn c : columns) - { - ColumnOrSuperColumn cosc = new ColumnOrSuperColumn().setCounter_column(c); - mutations.add(new Mutation().setColumn_or_supercolumn(cosc)); - } - row = Collections.singletonMap(state.type.table, mutations); + ColumnOrSuperColumn cosc = new ColumnOrSuperColumn().setCounter_column(c); + mutations.add(new Mutation().setColumn_or_supercolumn(cosc)); } + Map> row = Collections.singletonMap(type.table, mutations); final ByteBuffer key = getKey(); final Map>> record = Collections.singletonMap(key, row); @@ -74,18 +65,18 @@ public class ThriftCounterAdder extends Operation @Override public boolean run() throws Exception { - client.batch_mutate(record, state.settings.command.consistencyLevel); + client.batch_mutate(record, settings.command.consistencyLevel); return true; } @Override - public String key() + public int partitionCount() { - return new String(key.array()); + return 1; } @Override - public int keyCount() + public int rowCount() { return 1; } diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftCounterGetter.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterGetter.java similarity index 52% rename from tools/stress/src/org/apache/cassandra/stress/operations/ThriftCounterGetter.java rename to tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterGetter.java index 6e36a28e48..10c6aabcef 100644 --- a/tools/stress/src/org/apache/cassandra/stress/operations/ThriftCounterGetter.java +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftCounterGetter.java @@ -15,54 +15,52 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.stress.operations; +package org.apache.cassandra.stress.operations.predefined; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.cassandra.stress.Operation; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.Command; +import org.apache.cassandra.stress.settings.StressSettings; import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; import org.apache.cassandra.thrift.ColumnParent; import org.apache.cassandra.thrift.SlicePredicate; -import org.apache.cassandra.thrift.SliceRange; -public class ThriftCounterGetter extends Operation +public class ThriftCounterGetter extends PredefinedOperation { - public ThriftCounterGetter(State state, long index) + public ThriftCounterGetter(Timer timer, PartitionGenerator generator, StressSettings settings) { - super(state, index); + super(Command.COUNTER_READ, timer, generator, settings); } public void run(final ThriftClient client) throws IOException { - final SlicePredicate predicate = slicePredicate(); + final SlicePredicate predicate = select().predicate(); final ByteBuffer key = getKey(); - for (final ColumnParent parent : state.columnParents) + timeWithRetry(new RunOp() { - - timeWithRetry(new RunOp() + @Override + public boolean run() throws Exception { - @Override - public boolean run() throws Exception - { - List r = client.get_slice(key, parent, predicate, state.settings.command.consistencyLevel); - return r != null && r.size() > 0; - } + List r = client.get_slice(key, new ColumnParent(type.table), predicate, settings.command.consistencyLevel); + return r != null && r.size() > 0; + } - @Override - public String key() - { - return new String(key.array()); - } + @Override + public int partitionCount() + { + return 1; + } - @Override - public int keyCount() - { - return 1; - } - }); - } + @Override + public int rowCount() + { + return 1; + } + }); } } diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftInserter.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftInserter.java new file mode 100644 index 0000000000..5c2acfe22f --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftInserter.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.stress.operations.predefined; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.cassandra.db.marshal.TimeUUIDType; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.Command; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; +import org.apache.cassandra.thrift.*; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.UUIDGen; + +public final class ThriftInserter extends PredefinedOperation +{ + + public ThriftInserter(Timer timer, PartitionGenerator generator, StressSettings settings) + { + super(Command.WRITE, timer, generator, settings); + } + + public void run(final ThriftClient client) throws IOException + { + final ByteBuffer key = getKey(); + final List columns = getColumns(); + + List mutations = new ArrayList<>(columns.size()); + for (Column c : columns) + { + ColumnOrSuperColumn column = new ColumnOrSuperColumn().setColumn(c); + mutations.add(new Mutation().setColumn_or_supercolumn(column)); + } + Map> row = Collections.singletonMap(type.table, mutations); + + final Map>> record = Collections.singletonMap(key, row); + + timeWithRetry(new RunOp() + { + @Override + public boolean run() throws Exception + { + client.batch_mutate(record, settings.command.consistencyLevel); + return true; + } + + @Override + public int partitionCount() + { + return 1; + } + + @Override + public int rowCount() + { + return 1; + } + }); + } + + protected List getColumns() + { + final ColumnSelection selection = select(); + final List values = getColumnValues(selection); + final List columns = new ArrayList<>(values.size()); + final List names = select().select(settings.columns.names); + for (int i = 0 ; i < values.size() ; i++) + columns.add(new Column(names.get(i)) + .setValue(values.get(i)) + .setTimestamp(FBUtilities.timestampMicros())); + return columns; + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftReader.java b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftReader.java new file mode 100644 index 0000000000..276d8c5b41 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/predefined/ThriftReader.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.stress.operations.predefined; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.Command; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; +import org.apache.cassandra.thrift.ColumnOrSuperColumn; +import org.apache.cassandra.thrift.ColumnParent; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.SuperColumn; + +public final class ThriftReader extends PredefinedOperation +{ + + public ThriftReader(Timer timer, PartitionGenerator generator, StressSettings settings) + { + super(Command.READ, timer, generator, settings); + } + + public void run(final ThriftClient client) throws IOException + { + final ColumnSelection select = select(); + final ByteBuffer key = getKey(); + final List expect = getColumnValues(select); + timeWithRetry(new RunOp() + { + @Override + public boolean run() throws Exception + { + List row = client.get_slice(key, new ColumnParent(type.table), select.predicate(), settings.command.consistencyLevel); + if (expect == null) + return !row.isEmpty(); + if (row == null) + return false; + if (row.size() != expect.size()) + return false; + for (int i = 0 ; i < row.size() ; i++) + if (!row.get(i).getColumn().bufferForValue().equals(expect.get(i))) + return false; + return true; + } + + @Override + public int partitionCount() + { + return 1; + } + + @Override + public int rowCount() + { + return 1; + } + }); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaInsert.java b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaInsert.java new file mode 100644 index 0000000000..7c5efaca27 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaInsert.java @@ -0,0 +1,144 @@ +package org.apache.cassandra.stress.operations.userdefined; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.BoundStatement; +import com.datastax.driver.core.PreparedStatement; +import com.datastax.driver.core.Statement; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.stress.generate.Distribution; +import org.apache.cassandra.stress.generate.Partition; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.generate.RatioDistribution; +import org.apache.cassandra.stress.generate.Row; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.settings.ValidationType; +import org.apache.cassandra.stress.util.JavaDriverClient; +import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; + +public class SchemaInsert extends SchemaStatement +{ + + private final BatchStatement.Type batchType; + private final RatioDistribution perVisit; + private final RatioDistribution perBatch; + + public SchemaInsert(Timer timer, PartitionGenerator generator, StressSettings settings, Distribution partitionCount, RatioDistribution perVisit, RatioDistribution perBatch, Integer thriftId, PreparedStatement statement, ConsistencyLevel cl, BatchStatement.Type batchType) + { + super(timer, generator, settings, partitionCount, statement, thriftId, cl, ValidationType.NOT_FAIL); + this.batchType = batchType; + this.perVisit = perVisit; + this.perBatch = perBatch; + } + + private class JavaDriverRun extends Runner + { + final JavaDriverClient client; + + private JavaDriverRun(JavaDriverClient client) + { + this.client = client; + } + + public boolean run() throws Exception + { + Partition.RowIterator[] iterators = new Partition.RowIterator[partitions.size()]; + for (int i = 0 ; i < iterators.length ; i++) + iterators[i] = partitions.get(i).iterator(perVisit.next()); + List stmts = new ArrayList<>(); + partitionCount = partitions.size(); + + boolean done; + do + { + done = true; + stmts.clear(); + for (Partition.RowIterator iterator : iterators) + { + if (iterator.done()) + continue; + + for (Row row : iterator.batch(perBatch.next())) + stmts.add(bindRow(row)); + + done &= iterator.done(); + } + + rowCount += stmts.size(); + + Statement stmt; + if (stmts.size() == 1) + { + stmt = stmts.get(0); + } + else + { + BatchStatement batch = new BatchStatement(batchType); + batch.setConsistencyLevel(JavaDriverClient.from(cl)); + batch.addAll(stmts); + stmt = batch; + } + validate(client.getSession().execute(stmt)); + + } while (!done); + + return true; + } + } + + private class ThriftRun extends Runner + { + final ThriftClient client; + + private ThriftRun(ThriftClient client) + { + this.client = client; + } + + public boolean run() throws Exception + { + Partition.RowIterator[] iterators = new Partition.RowIterator[partitions.size()]; + for (int i = 0 ; i < iterators.length ; i++) + iterators[i] = partitions.get(i).iterator(perVisit.next()); + partitionCount = partitions.size(); + + boolean done; + do + { + done = true; + for (Partition.RowIterator iterator : iterators) + { + if (iterator.done()) + continue; + + for (Row row : iterator.batch(perBatch.next())) + { + validate(client.execute_prepared_cql3_query(thriftId, iterator.partition().getToken(), thriftRowArgs(row), settings.command.consistencyLevel)); + rowCount += 1; + } + + done &= iterator.done(); + } + } while (!done); + + return true; + } + } + + @Override + public void run(JavaDriverClient client) throws IOException + { + timeWithRetry(new JavaDriverRun(client)); + } + + @Override + public void run(ThriftClient client) throws IOException + { + timeWithRetry(new ThriftRun(client)); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaQuery.java b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaQuery.java new file mode 100644 index 0000000000..9cec39b978 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaQuery.java @@ -0,0 +1,86 @@ +package org.apache.cassandra.stress.operations.userdefined; + +import java.io.IOException; + +import com.datastax.driver.core.PreparedStatement; +import com.datastax.driver.core.ResultSet; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.settings.OptionDistribution; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.settings.ValidationType; +import org.apache.cassandra.stress.util.JavaDriverClient; +import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.thrift.ThriftConversion; + +public class SchemaQuery extends SchemaStatement +{ + + public SchemaQuery(Timer timer, PartitionGenerator generator, StressSettings settings, Integer thriftId, PreparedStatement statement, ConsistencyLevel cl, ValidationType validationType) + { + super(timer, generator, settings, OptionDistribution.get("fixed(1)").get(), statement, thriftId, cl, validationType); + } + + int execute(JavaDriverClient client) throws Exception + { + return client.getSession().execute(bindRandom(partitions.get(0))).all().size(); + } + + int execute(ThriftClient client) throws Exception + { + return client.execute_prepared_cql3_query(thriftId, partitions.get(0).getToken(), thriftRandomArgs(partitions.get(0)), ThriftConversion.toThrift(cl)).getRowsSize(); + } + + private class JavaDriverRun extends Runner + { + final JavaDriverClient client; + + private JavaDriverRun(JavaDriverClient client) + { + this.client = client; + } + + public boolean run() throws Exception + { + ResultSet rs = client.getSession().execute(bindRandom(partitions.get(0))); + validate(rs); + rowCount = rs.all().size(); + partitionCount = Math.min(1, rowCount); + return true; + } + } + + private class ThriftRun extends Runner + { + final ThriftClient client; + + private ThriftRun(ThriftClient client) + { + this.client = client; + } + + public boolean run() throws Exception + { + CqlResult rs = client.execute_prepared_cql3_query(thriftId, partitions.get(0).getToken(), thriftRandomArgs(partitions.get(0)), ThriftConversion.toThrift(cl)); + validate(rs); + rowCount = rs.getRowsSize(); + partitionCount = Math.min(1, rowCount); + return true; + } + } + + @Override + public void run(JavaDriverClient client) throws IOException + { + timeWithRetry(new JavaDriverRun(client)); + } + + @Override + public void run(ThriftClient client) throws IOException + { + timeWithRetry(new ThriftRun(client)); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaStatement.java b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaStatement.java new file mode 100644 index 0000000000..aac40c576f --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/operations/userdefined/SchemaStatement.java @@ -0,0 +1,164 @@ +package org.apache.cassandra.stress.operations.userdefined; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import com.datastax.driver.core.BoundStatement; +import com.datastax.driver.core.ColumnDefinitions; +import com.datastax.driver.core.PreparedStatement; +import com.datastax.driver.core.ResultSet; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.stress.Operation; +import org.apache.cassandra.stress.generate.Distribution; +import org.apache.cassandra.stress.generate.Partition; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.generate.Row; +import org.apache.cassandra.stress.settings.StressSettings; +import org.apache.cassandra.stress.settings.ValidationType; +import org.apache.cassandra.stress.util.JavaDriverClient; +import org.apache.cassandra.stress.util.ThriftClient; +import org.apache.cassandra.stress.util.Timer; +import org.apache.cassandra.thrift.CqlResult; +import org.apache.cassandra.transport.SimpleClient; + +public abstract class SchemaStatement extends Operation +{ + + final PartitionGenerator generator; + private final PreparedStatement statement; + final Integer thriftId; + final ConsistencyLevel cl; + final ValidationType validationType; + private final int[] argumentIndex; + private final Object[] bindBuffer; + private final Object[][] randomBuffer; + private final Random random = new Random(); + + public SchemaStatement(Timer timer, PartitionGenerator generator, StressSettings settings, Distribution partitionCount, + PreparedStatement statement, Integer thriftId, ConsistencyLevel cl, ValidationType validationType) + { + super(timer, generator, settings, partitionCount); + this.generator = generator; + this.statement = statement; + this.thriftId = thriftId; + this.cl = cl; + this.validationType = validationType; + argumentIndex = new int[statement.getVariables().size()]; + bindBuffer = new Object[argumentIndex.length]; + randomBuffer = new Object[argumentIndex.length][argumentIndex.length]; + int i = 0; + for (ColumnDefinitions.Definition definition : statement.getVariables()) + argumentIndex[i++] = generator.indexOf(definition.getName()); + } + + private int filLRandom(Partition partition) + { + int c = 0; + for (Row row : partition.iterator(randomBuffer.length).batch(1f)) + { + Object[] randomRow = randomBuffer[c++]; + for (int i = 0 ; i < argumentIndex.length ; i++) + randomRow[i] = row.get(argumentIndex[i]); + if (c >= randomBuffer.length) + break; + } + return c; + } + + BoundStatement bindRandom(Partition partition) + { + int c = filLRandom(partition); + for (int i = 0 ; i < argumentIndex.length ; i++) + { + int argIndex = argumentIndex[i]; + bindBuffer[i] = randomBuffer[argIndex < 0 ? 0 : random.nextInt(c)][i]; + } + return statement.bind(bindBuffer); + } + + BoundStatement bindRow(Row row) + { + for (int i = 0 ; i < argumentIndex.length ; i++) + bindBuffer[i] = row.get(argumentIndex[i]); + return statement.bind(bindBuffer); + } + + List thriftRowArgs(Row row) + { + List args = new ArrayList<>(); + for (int i : argumentIndex) + args.add(generator.convert(i, row.get(i))); + return args; + } + + List thriftRandomArgs(Partition partition) + { + List args = new ArrayList<>(); + int c = filLRandom(partition); + for (int i = 0 ; i < argumentIndex.length ; i++) + { + int argIndex = argumentIndex[i]; + args.add(generator.convert(argIndex, randomBuffer[argIndex < 0 ? 0 : random.nextInt(c)][i])); + } + return args; + } + + void validate(ResultSet rs) + { + switch (validationType) + { + case NOT_FAIL: + return; + case NON_ZERO: + if (rs.all().size() == 0) + throw new IllegalStateException("Expected non-zero results"); + break; + default: + throw new IllegalStateException("Unsupported validation type"); + } + } + + void validate(CqlResult rs) + { + switch (validationType) + { + case NOT_FAIL: + return; + case NON_ZERO: + if (rs.getRowsSize() == 0) + throw new IllegalStateException("Expected non-zero results"); + break; + default: + throw new IllegalStateException("Unsupported validation type"); + } + } + + @Override + public void run(SimpleClient client) throws IOException + { + throw new UnsupportedOperationException(); + } + + abstract class Runner implements RunOp + { + int partitionCount; + int rowCount; + + @Override + public int partitionCount() + { + return partitionCount; + } + + @Override + public int rowCount() + { + return rowCount; + } + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/Command.java b/tools/stress/src/org/apache/cassandra/stress/settings/Command.java index ac10014188..7138cbbaf2 100644 --- a/tools/stress/src/org/apache/cassandra/stress/settings/Command.java +++ b/tools/stress/src/org/apache/cassandra/stress/settings/Command.java @@ -41,19 +41,6 @@ public enum Command "Interleaving of any basic commands, with configurable ratio and distribution - the cluster must first be populated by a write test", CommandCategory.MIXED ), - RANGE_SLICE(false, "Standard1", "Super1", - "Range slice queries - the cluster must first be populated by a write test", - CommandCategory.MULTI - ), - INDEXED_RANGE_SLICE(false, "Standard1", "Super1", - "Range slice queries through a secondary index. The cluster must first be populated by a write test, with indexing enabled.", - CommandCategory.BASIC - ), - READ_MULTI(false, "Standard1", "Super1", - "multi_read", - "Multiple concurrent reads fetching multiple rows at once. The cluster must first be populated by a write test.", - CommandCategory.MULTI - ), COUNTER_WRITE(true, "Counter1", "SuperCounter1", "counter_add", "Multiple concurrent updates of counters.", @@ -64,6 +51,10 @@ public enum Command "Multiple concurrent reads of counters. The cluster must first be populated by a counterwrite test.", CommandCategory.BASIC ), + USER(true, null, null, + "Interleaving of user provided queries, with configurable ratio and distribution", + CommandCategory.USER + ), HELP(false, null, null, "-?", "Print help for a command or option", null), PRINT(false, null, null, "Inspect the output of a distribution definition", null), @@ -136,11 +127,12 @@ public enum Command } switch (category) { + case USER: + return SettingsCommandUser.helpPrinter(); case BASIC: - case MULTI: - return SettingsCommand.helpPrinter(this); + return SettingsCommandPreDefined.helpPrinter(this); case MIXED: - return SettingsCommandMixed.helpPrinter(); + return SettingsCommandPreDefinedMixed.helpPrinter(); } throw new AssertionError(); } diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/CommandCategory.java b/tools/stress/src/org/apache/cassandra/stress/settings/CommandCategory.java index 4372f590c9..e9dd946089 100644 --- a/tools/stress/src/org/apache/cassandra/stress/settings/CommandCategory.java +++ b/tools/stress/src/org/apache/cassandra/stress/settings/CommandCategory.java @@ -24,6 +24,6 @@ package org.apache.cassandra.stress.settings; public enum CommandCategory { BASIC, - MULTI, - MIXED + MIXED, + USER } diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/OptionAnyProbabilities.java b/tools/stress/src/org/apache/cassandra/stress/settings/OptionAnyProbabilities.java new file mode 100644 index 0000000000..a9af1bd1ca --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/settings/OptionAnyProbabilities.java @@ -0,0 +1,78 @@ +package org.apache.cassandra.stress.settings; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.math3.util.Pair; + +public final class OptionAnyProbabilities extends OptionMulti +{ + public OptionAnyProbabilities(String name, String description) + { + super(name, description, false); + } + + final CollectRatios ratios = new CollectRatios(); + + private static final class CollectRatios extends Option + { + Map options = new LinkedHashMap<>(); + + boolean accept(String param) + { + String[] args = param.split("="); + if (args.length == 2 && args[1].length() > 0 && args[0].length() > 0) + { + if (options.put(args[0], Double.parseDouble(args[1])) != null) + throw new IllegalArgumentException(args[0] + " set twice"); + return true; + } + return false; + } + + boolean happy() + { + return !options.isEmpty(); + } + + String shortDisplay() + { + return null; + } + + String longDisplay() + { + return null; + } + + List multiLineDisplay() + { + return Collections.emptyList(); + } + + boolean setByUser() + { + return !options.isEmpty(); + } + } + + + @Override + public List options() + { + return Arrays.asList(ratios); + } + + List> ratios() + { + List> ratiosOut = new ArrayList<>(); + for (Map.Entry e : ratios.options.entrySet()) + ratiosOut.add(new Pair(e.getKey(), e.getValue())); + return ratiosOut; + } +} + diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/OptionDataGen.java b/tools/stress/src/org/apache/cassandra/stress/settings/OptionDataGen.java deleted file mode 100644 index bde2b104d2..0000000000 --- a/tools/stress/src/org/apache/cassandra/stress/settings/OptionDataGen.java +++ /dev/null @@ -1,203 +0,0 @@ -package org.apache.cassandra.stress.settings; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - -import java.io.File; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.cassandra.stress.generatedata.*; - -/** - * For selecting a data generator - */ -class OptionDataGen extends Option -{ - - private static final Pattern FULL = Pattern.compile("([A-Z]+)\\(([^)]+)\\)", Pattern.CASE_INSENSITIVE); - private static final Pattern ARGS = Pattern.compile("[^,]+"); - - final String prefix; - private DataGenFactory factory; - private final DataGenFactory defaultFactory; - - public OptionDataGen(String prefix, String defaultSpec) - { - this.prefix = prefix; - this.defaultFactory = defaultSpec == null ? null : get(defaultSpec); - } - - @Override - public boolean accept(String param) - { - if (!param.toLowerCase().startsWith(prefix)) - return false; - factory = get(param.substring(prefix.length())); - return true; - } - - private static DataGenFactory get(String spec) - { - Matcher m = FULL.matcher(spec); - if (!m.matches()) - throw new IllegalArgumentException("Illegal data generator specification: " + spec); - String name = m.group(1); - Impl impl = LOOKUP.get(name.toLowerCase()); - if (impl == null) - throw new IllegalArgumentException("Illegal data generator type: " + name); - List params = new ArrayList<>(); - m = ARGS.matcher(m.group(2)); - while (m.find()) - params.add(m.group()); - return impl.getFactory(params); - } - - public DataGenFactory get() - { - return factory != null ? factory : defaultFactory; - } - - @Override - public boolean happy() - { - return factory != null || defaultFactory != null; - } - - public boolean setByUser() - { - return factory != null; - } - - @Override - public String shortDisplay() - { - return prefix + "ALG()"; - } - - public String longDisplay() - { - return shortDisplay() + ": Specify a data generator from:"; - } - - @Override - public List multiLineDisplay() - { - return Arrays.asList( - GroupedOptions.formatMultiLine("RANDOM()", "Completely random byte generation"), - GroupedOptions.formatMultiLine("REPEAT()", "An MD5 hash of (opIndex % freq) combined with the column index"), - GroupedOptions.formatMultiLine("DICT()","Random words from a dictionary; the file should be in the format \" \"") - ); - } - - private static final Map LOOKUP; - static - { - final Map lookup = new HashMap<>(); - lookup.put("random", new RandomImpl()); - lookup.put("rand", new RandomImpl()); - lookup.put("rnd", new RandomImpl()); - lookup.put("repeat", new RepeatImpl()); - lookup.put("dict", new DictionaryImpl()); - lookup.put("dictionary", new DictionaryImpl()); - LOOKUP = lookup; - } - - private static interface Impl - { - public DataGenFactory getFactory(List params); - } - - private static final class RandomImpl implements Impl - { - @Override - public DataGenFactory getFactory(List params) - { - if (params.size() != 0) - throw new IllegalArgumentException("Invalid parameter list for random generator: " + params); - return new RandomFactory(); - } - } - - private static final class RepeatImpl implements Impl - { - - @Override - public DataGenFactory getFactory(List params) - { - if (params.size() != 1) - throw new IllegalArgumentException("Invalid parameter list for repeating generator: " + params); - try - { - int repeatFrequency = Integer.parseInt(params.get(0)); - return new RepeatsFactory(repeatFrequency); - } catch (Exception _) - { - throw new IllegalArgumentException("Invalid parameter list for repeating generator: " + params); - } - } - } - - private static final class DictionaryImpl implements Impl - { - - @Override - public DataGenFactory getFactory(List params) - { - if (params.size() != 1) - throw new IllegalArgumentException("Invalid parameter list for dictionary generator: " + params); - try - { - final File file = new File(params.get(0)); - return DataGenStringDictionary.getFactory(file); - } catch (Exception e) - { - throw new IllegalArgumentException("Invalid parameter list for dictionary generator: " + params, e); - } - } - } - - private static final class RandomFactory implements DataGenFactory - { - @Override - public DataGen get() - { - return new DataGenBytesRandom(); - } - } - - private static final class RepeatsFactory implements DataGenFactory - { - final int frequency; - private RepeatsFactory(int frequency) - { - this.frequency = frequency; - } - - @Override - public DataGen get() - { - return new DataGenStringRepeats(frequency); - } - } - -} diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/OptionDistribution.java b/tools/stress/src/org/apache/cassandra/stress/settings/OptionDistribution.java index 76fa0a9599..70a85aee1e 100644 --- a/tools/stress/src/org/apache/cassandra/stress/settings/OptionDistribution.java +++ b/tools/stress/src/org/apache/cassandra/stress/settings/OptionDistribution.java @@ -25,19 +25,20 @@ import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.cassandra.stress.generatedata.*; +import org.apache.cassandra.stress.generate.*; import org.apache.commons.math3.distribution.ExponentialDistribution; import org.apache.commons.math3.distribution.NormalDistribution; import org.apache.commons.math3.distribution.UniformRealDistribution; import org.apache.commons.math3.distribution.WeibullDistribution; +import org.apache.commons.math3.random.JDKRandomGenerator; /** * For selecting a mathematical distribution */ -class OptionDistribution extends Option +public class OptionDistribution extends Option { - private static final Pattern FULL = Pattern.compile("([A-Z]+)\\((.+)\\)", Pattern.CASE_INSENSITIVE); + private static final Pattern FULL = Pattern.compile("(~?)([A-Z]+)\\((.+)\\)", Pattern.CASE_INSENSITIVE); private static final Pattern ARGS = Pattern.compile("[^,]+"); final String prefix; @@ -61,20 +62,22 @@ class OptionDistribution extends Option return true; } - private static DistributionFactory get(String spec) + public static DistributionFactory get(String spec) { Matcher m = FULL.matcher(spec); if (!m.matches()) throw new IllegalArgumentException("Illegal distribution specification: " + spec); - String name = m.group(1); + boolean inverse = m.group(1).equals("~"); + String name = m.group(2); Impl impl = LOOKUP.get(name.toLowerCase()); if (impl == null) throw new IllegalArgumentException("Illegal distribution type: " + name); List params = new ArrayList<>(); - m = ARGS.matcher(m.group(2)); + m = ARGS.matcher(m.group(3)); while (m.find()) params.add(m.group()); - return impl.getFactory(params); + DistributionFactory factory = impl.getFactory(params); + return inverse ? new InverseFactory(factory) : factory; } public DistributionFactory get() @@ -103,6 +106,7 @@ class OptionDistribution extends Option GroupedOptions.formatMultiLine("GAUSSIAN(min..max,mean,stdev)", "A gaussian/normal distribution, with explicitly defined mean and stdev"), GroupedOptions.formatMultiLine("UNIFORM(min..max)", "A uniform distribution over the range [min, max]"), GroupedOptions.formatMultiLine("FIXED(val)", "A fixed distribution, always returning the same value"), + "Preceding the name with ~ will invert the distribution, e.g. ~exp(1..10) will yield 10 most, instead of least, often", "Aliases: extr, gauss, normal, norm, weibull" ); } @@ -142,6 +146,23 @@ class OptionDistribution extends Option public DistributionFactory getFactory(List params); } + public static long parseLong(String value) + { + long multiplier = 1; + value = value.trim().toLowerCase(); + switch (value.charAt(value.length() - 1)) + { + case 'b': + multiplier *= 1000; + case 'm': + multiplier *= 1000; + case 'k': + multiplier *= 1000; + value = value.substring(0, value.length() - 1); + } + return Long.parseLong(value) * multiplier; + } + private static final class GaussianImpl implements Impl { @@ -153,8 +174,8 @@ class OptionDistribution extends Option try { String[] bounds = params.get(0).split("\\.\\.+"); - final long min = Long.parseLong(bounds[0]); - final long max = Long.parseLong(bounds[1]); + final long min = parseLong(bounds[0]); + final long max = parseLong(bounds[1]); final double mean, stdev; if (params.size() == 3) { @@ -185,8 +206,8 @@ class OptionDistribution extends Option try { String[] bounds = params.get(0).split("\\.\\.+"); - final long min = Long.parseLong(bounds[0]); - final long max = Long.parseLong(bounds[1]); + final long min = parseLong(bounds[0]); + final long max = parseLong(bounds[1]); ExponentialDistribution findBounds = new ExponentialDistribution(1d); // max probability should be roughly equal to accuracy of (max-min) to ensure all values are visitable, // over entire range, but this results in overly skewed distribution, so take sqrt @@ -209,8 +230,8 @@ class OptionDistribution extends Option try { String[] bounds = params.get(0).split("\\.\\.+"); - final long min = Long.parseLong(bounds[0]); - final long max = Long.parseLong(bounds[1]); + final long min = parseLong(bounds[0]); + final long max = parseLong(bounds[1]); final double shape = Double.parseDouble(params.get(1)); WeibullDistribution findBounds = new WeibullDistribution(shape, 1d); // max probability should be roughly equal to accuracy of (max-min) to ensure all values are visitable, @@ -235,8 +256,8 @@ class OptionDistribution extends Option try { String[] bounds = params.get(0).split("\\.\\.+"); - final long min = Long.parseLong(bounds[0]); - final long max = Long.parseLong(bounds[1]); + final long min = parseLong(bounds[0]); + final long max = parseLong(bounds[1]); return new UniformFactory(min, max); } catch (Exception _) { @@ -255,7 +276,7 @@ class OptionDistribution extends Option throw new IllegalArgumentException("Invalid parameter list for uniform distribution: " + params); try { - final long key = Long.parseLong(params.get(0)); + final long key = parseLong(params.get(0)); return new FixedFactory(key); } catch (Exception _) { @@ -264,6 +285,20 @@ class OptionDistribution extends Option } } + private static final class InverseFactory implements DistributionFactory + { + final DistributionFactory wrapped; + private InverseFactory(DistributionFactory wrapped) + { + this.wrapped = wrapped; + } + + public Distribution get() + { + return new DistributionInverted(wrapped.get()); + } + } + // factories private static final class ExpFactory implements DistributionFactory @@ -280,7 +315,7 @@ class OptionDistribution extends Option @Override public Distribution get() { - return new DistributionOffsetApache(new ExponentialDistribution(mean), min, max); + return new DistributionOffsetApache(new ExponentialDistribution(new JDKRandomGenerator(), mean, ExponentialDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY), min, max); } } @@ -299,7 +334,7 @@ class OptionDistribution extends Option @Override public Distribution get() { - return new DistributionOffsetApache(new WeibullDistribution(shape, scale), min, max); + return new DistributionOffsetApache(new WeibullDistribution(new JDKRandomGenerator(), shape, scale, WeibullDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY), min, max); } } @@ -318,7 +353,7 @@ class OptionDistribution extends Option @Override public Distribution get() { - return new DistributionBoundApache(new NormalDistribution(mean, stdev), min, max); + return new DistributionBoundApache(new NormalDistribution(new JDKRandomGenerator(), mean, stdev, NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY), min, max); } } @@ -334,7 +369,7 @@ class OptionDistribution extends Option @Override public Distribution get() { - return new DistributionBoundApache(new UniformRealDistribution(min, max + 1), min, max); + return new DistributionBoundApache(new UniformRealDistribution(new JDKRandomGenerator(), min, max + 1), min, max); } } diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/OptionEnumProbabilities.java b/tools/stress/src/org/apache/cassandra/stress/settings/OptionEnumProbabilities.java new file mode 100644 index 0000000000..88ebd34ede --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/settings/OptionEnumProbabilities.java @@ -0,0 +1,62 @@ +package org.apache.cassandra.stress.settings; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.math3.util.Pair; + +public final class OptionEnumProbabilities extends OptionMulti +{ + final List> options; + + public static class Opt + { + final T option; + final String defaultValue; + + public Opt(T option, String defaultValue) + { + this.option = option; + this.defaultValue = defaultValue; + } + } + + private static final class OptMatcher extends OptionSimple + { + final T opt; + OptMatcher(T opt, String defaultValue) + { + super(opt.toString().toLowerCase() + "=", "[0-9]+(\\.[0-9]+)?", defaultValue, "Performs this many " + opt + " operations out of total", false); + this.opt = opt; + } + } + + public OptionEnumProbabilities(List> universe, String name, String description) + { + super(name, description, false); + List> options = new ArrayList<>(); + for (Opt option : universe) + options.add(new OptMatcher(option.option, option.defaultValue)); + this.options = options; + } + + @Override + public List options() + { + return options; + } + + List> ratios() + { + List ratiosIn = setByUser() ? optionsSetByUser() : defaultOptions(); + List> ratiosOut = new ArrayList<>(); + for (Option opt : ratiosIn) + { + OptMatcher optMatcher = (OptMatcher) opt; + double d = Double.parseDouble(optMatcher.value()); + ratiosOut.add(new Pair<>(optMatcher.opt, d)); + } + return ratiosOut; + } +} + diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/OptionMulti.java b/tools/stress/src/org/apache/cassandra/stress/settings/OptionMulti.java index 60faad87f1..32bfc652cf 100644 --- a/tools/stress/src/org/apache/cassandra/stress/settings/OptionMulti.java +++ b/tools/stress/src/org/apache/cassandra/stress/settings/OptionMulti.java @@ -156,7 +156,8 @@ abstract class OptionMulti extends Option String[] args = param.split("="); if (args.length == 2 && args[1].length() > 0 && args[0].length() > 0) { - options.put(args[0], args[1]); + if (options.put(args[0], args[1]) != null) + throw new IllegalArgumentException(args[0] + " set twice"); return true; } return false; diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/OptionRatioDistribution.java b/tools/stress/src/org/apache/cassandra/stress/settings/OptionRatioDistribution.java new file mode 100644 index 0000000000..2459c204b6 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/settings/OptionRatioDistribution.java @@ -0,0 +1,166 @@ +package org.apache.cassandra.stress.settings; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.math3.distribution.ExponentialDistribution; +import org.apache.commons.math3.distribution.NormalDistribution; +import org.apache.commons.math3.distribution.UniformRealDistribution; +import org.apache.commons.math3.distribution.WeibullDistribution; +import org.apache.commons.math3.random.JDKRandomGenerator; + +import org.apache.cassandra.stress.generate.Distribution; +import org.apache.cassandra.stress.generate.DistributionBoundApache; +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.stress.generate.DistributionFixed; +import org.apache.cassandra.stress.generate.DistributionInverted; +import org.apache.cassandra.stress.generate.DistributionOffsetApache; +import org.apache.cassandra.stress.generate.RatioDistribution; +import org.apache.cassandra.stress.generate.RatioDistributionFactory; + +/** + * For selecting a mathematical distribution + */ +public class OptionRatioDistribution extends Option +{ + + private static final Pattern FULL = Pattern.compile("(.*)/([0-9]+[KMB]?)", Pattern.CASE_INSENSITIVE); + + final OptionDistribution delegate; + private double divisor; + + private static final RatioDistribution DEFAULT = new RatioDistribution(new DistributionFixed(1), 1); + + public OptionRatioDistribution(String prefix, String defaultSpec, String description) + { + delegate = new OptionDistribution(prefix, defaultSpec, description); + } + + @Override + public boolean accept(String param) + { + Matcher m = FULL.matcher(param); + if (!m.matches() || !delegate.accept(m.group(1))) + return false; + divisor = OptionDistribution.parseLong(m.group(2)); + return true; + } + + public static RatioDistributionFactory get(String spec) + { + OptionRatioDistribution opt = new OptionRatioDistribution("", "", ""); + if (!opt.accept(spec)) + throw new IllegalArgumentException(); + return opt.get(); + } + + public RatioDistributionFactory get() + { + return !delegate.setByUser() ? new DefaultFactory() : new DelegateFactory(delegate.get(), divisor); + } + + @Override + public boolean happy() + { + return delegate.happy(); + } + + public String longDisplay() + { + return delegate.longDisplay(); + } + + @Override + public List multiLineDisplay() + { + return Arrays.asList( + GroupedOptions.formatMultiLine("EXP(min..max)/divisor", "An exponential ratio distribution over the range [min..max]/divisor"), + GroupedOptions.formatMultiLine("EXTREME(min..max,shape)/divisor", "An extreme value (Weibull) ratio distribution over the range [min..max]/divisor"), + GroupedOptions.formatMultiLine("GAUSSIAN(min..max,stdvrng)/divisor", "A gaussian/normal ratio distribution, where mean=(min+max)/2, and stdev is ((mean-min)/stdvrng)/divisor"), + GroupedOptions.formatMultiLine("GAUSSIAN(min..max,mean,stdev)/divisor", "A gaussian/normal ratio distribution, with explicitly defined mean and stdev"), + GroupedOptions.formatMultiLine("UNIFORM(min..max)/divisor", "A uniform ratio distribution over the range [min, max]/divisor"), + GroupedOptions.formatMultiLine("FIXED(val)/divisor", "A fixed ratio distribution, always returning the same value"), + "Preceding the name with ~ will invert the distribution, e.g. ~exp(1..10)/10 will yield 0.1 least, instead of most, often", + "Aliases: extr, gauss, normal, norm, weibull" + ); + } + + boolean setByUser() + { + return delegate.setByUser(); + } + + @Override + public String shortDisplay() + { + return delegate.shortDisplay(); + } + + // factories + + private static final class DefaultFactory implements RatioDistributionFactory + { + @Override + public RatioDistribution get() + { + return DEFAULT; + } + } + + private static final class DelegateFactory implements RatioDistributionFactory + { + final DistributionFactory delegate; + final double divisor; + + private DelegateFactory(DistributionFactory delegate, double divisor) + { + this.delegate = delegate; + this.divisor = divisor; + } + + @Override + public RatioDistribution get() + { + return new RatioDistribution(delegate.get(), divisor); + } + } + + @Override + public int hashCode() + { + return delegate.hashCode(); + } + + @Override + public boolean equals(Object that) + { + return super.equals(that) && ((OptionRatioDistribution) that).delegate.equals(this.delegate); + } + +} diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/OptionSimple.java b/tools/stress/src/org/apache/cassandra/stress/settings/OptionSimple.java index 9365e45525..71c1ffe1aa 100644 --- a/tools/stress/src/org/apache/cassandra/stress/settings/OptionSimple.java +++ b/tools/stress/src/org/apache/cassandra/stress/settings/OptionSimple.java @@ -21,6 +21,7 @@ package org.apache.cassandra.stress.settings; */ +import java.io.Serializable; import java.util.Collections; import java.util.List; import java.util.regex.Pattern; @@ -30,7 +31,7 @@ import com.google.common.base.Function; /** * For parsing a simple (sub)option for a command/major option */ -class OptionSimple extends Option +class OptionSimple extends Option implements Serializable { final String displayPrefix; @@ -41,7 +42,7 @@ class OptionSimple extends Option private final boolean required; private String value; - private static final class ValueMatcher implements Function + private static final class ValueMatcher implements Function, Serializable { final Pattern pattern; private ValueMatcher(Pattern pattern) diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java index 7e20ec6f03..04c2a47140 100644 --- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java +++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsColumn.java @@ -23,13 +23,15 @@ package org.apache.cassandra.stress.settings; import java.io.Serializable; import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.stress.generatedata.*; +import org.apache.cassandra.stress.generate.*; import org.apache.cassandra.utils.ByteBufferUtil; /** @@ -39,17 +41,13 @@ public class SettingsColumn implements Serializable { public final int maxColumnsPerKey; - public final List names; + public transient final List names; public final List namestrs; public final String comparator; - public final boolean useTimeUUIDComparator; - public final int superColumns; - public final boolean useSuperColumns; public final boolean variableColumnCount; public final boolean slice; - private final DistributionFactory sizeDistribution; - private final DistributionFactory countDistribution; - private final DataGenFactory dataGenFactory; + public final DistributionFactory sizeDistribution; + public final DistributionFactory countDistribution; public SettingsColumn(GroupedOptions options) { @@ -62,9 +60,6 @@ public class SettingsColumn implements Serializable public SettingsColumn(Options options, NameOptions name, CountOptions count) { sizeDistribution = options.size.get(); - superColumns = Integer.parseInt(options.superColumns.value()); - dataGenFactory = options.generator.get(); - useSuperColumns = superColumns > 0; { comparator = options.comparator.value(); AbstractType parsed = null; @@ -79,8 +74,6 @@ public class SettingsColumn implements Serializable System.exit(1); } - useTimeUUIDComparator = parsed instanceof TimeUUIDType; - if (!(parsed instanceof TimeUUIDType || parsed instanceof AsciiType || parsed instanceof UTF8Type)) { System.err.println("Currently supported types are: TimeUUIDType, AsciiType, UTF8Type."); @@ -102,10 +95,13 @@ public class SettingsColumn implements Serializable final String[] names = name.name.value().split(","); this.names = new ArrayList<>(names.length); - this.namestrs = Arrays.asList(names); for (String columnName : names) this.names.add(comparator.fromString(columnName)); + Collections.sort(this.names, BytesType.instance); + this.namestrs = new ArrayList<>(); + for (ByteBuffer columnName : this.names) + this.namestrs.add(comparator.getString(columnName)); final int nameCount = this.names.size(); countDistribution = new DistributionFactory() @@ -123,23 +119,24 @@ public class SettingsColumn implements Serializable ByteBuffer[] names = new ByteBuffer[(int) countDistribution.get().maxValue()]; String[] namestrs = new String[(int) countDistribution.get().maxValue()]; for (int i = 0 ; i < names.length ; i++) - { names[i] = ByteBufferUtil.bytes("C" + i); - namestrs[i] = "C" + i; + Arrays.sort(names, BytesType.instance); + try + { + for (int i = 0 ; i < names.length ; i++) + namestrs[i] = ByteBufferUtil.string(names[i]); } + catch (CharacterCodingException e) + { + throw new RuntimeException(e); + } + this.names = Arrays.asList(names); this.namestrs = Arrays.asList(namestrs); } maxColumnsPerKey = (int) countDistribution.get().maxValue(); variableColumnCount = countDistribution.get().minValue() < maxColumnsPerKey; - // TODO: should warn that we always slice for useTimeUUIDComparator? - slice = options.slice.setByUser() || useTimeUUIDComparator; - // TODO: with useTimeUUIDCOmparator, should we still try to select a random start for reads if possible? - } - - public RowGen newRowGen() - { - return new RowGenDistributedSize(dataGenFactory.get(), countDistribution.get(), sizeDistribution.get()); + slice = options.slice.setByUser(); } // Option Declarations @@ -150,7 +147,6 @@ public class SettingsColumn implements Serializable final OptionSimple comparator = new OptionSimple("comparator=", "TimeUUIDType|AsciiType|UTF8Type", "AsciiType", "Column Comparator to use", false); final OptionSimple slice = new OptionSimple("slice", "", null, "If set, range slices will be used for reads, otherwise a names query will be", false); final OptionDistribution size = new OptionDistribution("size=", "FIXED(34)", "Cell size distribution"); - final OptionDataGen generator = new OptionDataGen("data=", "REPEAT(50)"); } private static final class NameOptions extends Options @@ -160,7 +156,7 @@ public class SettingsColumn implements Serializable @Override public List options() { - return Arrays.asList(name, slice, superColumns, comparator, size, generator); + return Arrays.asList(name, slice, superColumns, comparator, size); } } @@ -171,7 +167,7 @@ public class SettingsColumn implements Serializable @Override public List options() { - return Arrays.asList(count, slice, superColumns, comparator, size, generator); + return Arrays.asList(count, slice, superColumns, comparator, size); } } diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommand.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommand.java index b4917074b4..032f00c3b0 100644 --- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommand.java +++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommand.java @@ -26,23 +26,24 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import org.apache.cassandra.stress.generatedata.DistributionFactory; +import org.apache.cassandra.stress.operations.OpDistributionFactory; import org.apache.cassandra.thrift.ConsistencyLevel; // Generic command settings - common to read/write/etc -public class SettingsCommand implements Serializable +public abstract class SettingsCommand implements Serializable { public final Command type; public final long count; public final int tries; public final boolean ignoreErrors; + public final boolean noWarmup; public final ConsistencyLevel consistencyLevel; public final double targetUncertainty; public final int minimumUncertaintyMeasurements; public final int maximumUncertaintyMeasurements; - public final DistributionFactory add; - public final int keysAtOnce; + + public abstract OpDistributionFactory getFactory(StressSettings settings); public SettingsCommand(Command type, GroupedOptions options) { @@ -58,8 +59,7 @@ public class SettingsCommand implements Serializable this.tries = Math.max(1, Integer.parseInt(options.retries.value()) + 1); this.ignoreErrors = options.ignoreErrors.setByUser(); this.consistencyLevel = ConsistencyLevel.valueOf(options.consistencyLevel.value().toUpperCase()); - this.keysAtOnce = Integer.parseInt(options.atOnce.value()); - this.add = options.add.get(); + this.noWarmup = options.noWarmup.setByUser(); if (count != null) { this.count = Long.parseLong(count.count.value()); @@ -82,8 +82,8 @@ public class SettingsCommand implements Serializable { final OptionSimple retries = new OptionSimple("tries=", "[0-9]+", "9", "Number of tries to perform for each operation before failing", false); final OptionSimple ignoreErrors = new OptionSimple("ignore_errors", "", null, "Do not print/log errors", false); + final OptionSimple noWarmup = new OptionSimple("no_warmup", "", null, "Do not warmup the process", false); final OptionSimple consistencyLevel = new OptionSimple("cl=", "ONE|QUORUM|LOCAL_QUORUM|EACH_QUORUM|ALL|ANY", "ONE", "Consistency level to use", false); - final OptionDistribution add = new OptionDistribution("add=", "fixed(1)", "Distribution of value of counter increments"); final OptionSimple atOnce = new OptionSimple("at-once=", "[0-9]+", "1000", "Number of keys per operation for multiget", false); } @@ -93,7 +93,7 @@ public class SettingsCommand implements Serializable @Override public List options() { - return Arrays.asList(count, retries, ignoreErrors, consistencyLevel, add, atOnce); + return Arrays.asList(count, retries, noWarmup, ignoreErrors, consistencyLevel, atOnce); } } @@ -105,7 +105,7 @@ public class SettingsCommand implements Serializable @Override public List options() { - return Arrays.asList(uncertainty, minMeasurements, maxMeasurements, retries, ignoreErrors, consistencyLevel, add, atOnce); + return Arrays.asList(uncertainty, minMeasurements, maxMeasurements, retries, noWarmup, ignoreErrors, consistencyLevel, atOnce); } } @@ -127,48 +127,16 @@ public class SettingsCommand implements Serializable switch (cmd.category) { case BASIC: - case MULTI: - return build(cmd, params); + return SettingsCommandPreDefined.build(cmd, params); case MIXED: - return SettingsCommandMixed.build(params); + return SettingsCommandPreDefinedMixed.build(params); + case USER: + return SettingsCommandUser.build(params); } } } return null; } - static SettingsCommand build(Command type, String[] params) - { - GroupedOptions options = GroupedOptions.select(params, new Count(), new Uncertainty()); - if (options == null) - { - printHelp(type); - System.out.println("Invalid " + type + " options provided, see output for valid options"); - System.exit(1); - } - return new SettingsCommand(type, options); - } - - static void printHelp(Command type) - { - printHelp(type.toString().toLowerCase()); - } - - static void printHelp(String type) - { - GroupedOptions.printOptions(System.out, type.toLowerCase(), new Uncertainty(), new Count()); - } - - static Runnable helpPrinter(final Command type) - { - return new Runnable() - { - @Override - public void run() - { - printHelp(type); - } - }; - } } diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefined.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefined.java new file mode 100644 index 0000000000..ac113d17b7 --- /dev/null +++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsCommandPreDefined.java @@ -0,0 +1,145 @@ +package org.apache.cassandra.stress.settings; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.cassandra.stress.generate.DistributionFactory; +import org.apache.cassandra.stress.generate.PartitionGenerator; +import org.apache.cassandra.stress.generate.values.Bytes; +import org.apache.cassandra.stress.generate.values.Generator; +import org.apache.cassandra.stress.generate.values.GeneratorConfig; +import org.apache.cassandra.stress.generate.values.HexBytes; +import org.apache.cassandra.stress.operations.FixedOpDistribution; +import org.apache.cassandra.stress.operations.OpDistribution; +import org.apache.cassandra.stress.operations.OpDistributionFactory; +import org.apache.cassandra.stress.operations.predefined.PredefinedOperation; +import org.apache.cassandra.stress.util.Timer; + +// Settings unique to the mixed command type +public class SettingsCommandPreDefined extends SettingsCommand +{ + + public final DistributionFactory add; + + public OpDistributionFactory getFactory(final StressSettings settings) + { + return new OpDistributionFactory() + { + public OpDistribution get(Timer timer) + { + return new FixedOpDistribution(PredefinedOperation.operation(type, timer, newGenerator(settings), settings, add)); + } + + public String desc() + { + return type.toString(); + } + + public Iterable each() + { + return Collections.singleton(this); + } + }; + } + + PartitionGenerator newGenerator(StressSettings settings) + { + List names = settings.columns.namestrs; + List partitionKey = Collections.singletonList(new HexBytes("key", + new GeneratorConfig("randomstrkey", null, + OptionDistribution.get("fixed(" + settings.keys.keySize + ")"), null))); + + List columns = new ArrayList<>(); + for (int i = 0 ; i < settings.columns.maxColumnsPerKey ; i++) + columns.add(new Bytes(names.get(i), new GeneratorConfig("randomstr" + names.get(i), null, settings.columns.sizeDistribution, null))); + return new PartitionGenerator(partitionKey, Collections.emptyList(), columns); + } + + public SettingsCommandPreDefined(Command type, Options options) + { + super(type, options.parent); + add = options.add.get(); + } + + // Option Declarations + + static class Options extends GroupedOptions + { + final SettingsCommand.Options parent; + protected Options(SettingsCommand.Options parent) + { + this.parent = parent; + } + final OptionDistribution add = new OptionDistribution("add=", "fixed(1)", "Distribution of value of counter increments"); + + @Override + public List options() + { + final List