diff --git a/CHANGES.txt b/CHANGES.txt index 31d6a817e8..303b0f7ae7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -50,11 +50,6 @@ dev (CASSANDRA-1232) * extend option to lower compaction priority to hinted handoff as well (CASSANDRA-1260) - * log thread pool stats when GC is excessive (CASSANDRA-1275) - * added describe_partitioner Thrift method (CASSANDRA-1047) - * Hadoop jobs no longer require the Cassandra storage-conf.xml - (CASSANDRA-1280, CASSANDRA-1047) - * log thread pool stats when GC is excessive (CASSANDRA-1275) * log errors in gossip instead of re-throwing (CASSANDRA-1289) * avoid aborting commitlog replay prematurely if a flushed-but- not-removed commitlog segment is encountered (CASSANDRA-1297) @@ -69,6 +64,7 @@ dev * Hadoop jobs no longer require the Cassandra storage-conf.xml (CASSANDRA-1280, CASSANDRA-1047) * log thread pool stats when GC is excessive (CASSANDRA-1275) + * remove gossip message size limit (CASSANDRA-1138) 0.6.3 diff --git a/contrib/word_count/bin/word_count b/contrib/word_count/bin/word_count index 6a20222af1..d3c136ba89 100644 --- a/contrib/word_count/bin/word_count +++ b/contrib/word_count/bin/word_count @@ -31,7 +31,7 @@ if [ ! -e $cwd/../build/word_count.jar ]; then fi CLASSPATH=$CLASSPATH:$cwd/../build/word_count.jar -CLASSPATH=$CLASSPATH:.:$cwd/../../../build/classes +CLASSPATH=$CLASSPATH:$cwd/../../../build/classes for jar in $cwd/../build/lib/jars/*.jar; do CLASSPATH=$CLASSPATH:$jar done diff --git a/contrib/word_count/src/WordCount.java b/contrib/word_count/src/WordCount.java index 7f7beb71f1..5d807196e0 100644 --- a/contrib/word_count/src/WordCount.java +++ b/contrib/word_count/src/WordCount.java @@ -112,13 +112,12 @@ public class WordCount extends Configured implements Tool public int run(String[] args) throws Exception { - Configuration conf = getConf(); for (int i = 0; i < WordCountSetup.TEST_COUNT; i++) { String columnName = "text" + i; - conf.set(CONF_COLUMN_NAME, columnName); - Job job = new Job(conf, "wordcount"); + getConf().set(CONF_COLUMN_NAME, columnName); + Job job = new Job(getConf(), "wordcount"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); @@ -129,7 +128,7 @@ public class WordCount extends Configured implements Tool job.setInputFormatClass(ColumnFamilyInputFormat.class); FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH_PREFIX + i)); - ConfigHelper.setThriftContact(conf, "localhost", 9160); + ConfigHelper.setThriftContact(job.getConfiguration(), "localhost", 9160); ConfigHelper.setInputColumnFamily(job.getConfiguration(), KEYSPACE, COLUMN_FAMILY); SlicePredicate predicate = new SlicePredicate().setColumn_names(Arrays.asList(columnName.getBytes())); ConfigHelper.setInputSlicePredicate(job.getConfiguration(), predicate); diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index fba25c1e3a..22ef597f63 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -205,7 +205,7 @@ public class DatabaseDescriptor } try { - partitioner = newPartitioner(conf.partitioner); + partitioner = FBUtilities.newPartitioner(conf.partitioner); } catch (Exception e) { @@ -385,22 +385,6 @@ public class DatabaseDescriptor } } - public static IPartitioner newPartitioner(String partitionerClassName) - { - if (!partitionerClassName.contains(".")) - partitionerClassName = "org.apache.cassandra.dht." + partitionerClassName; - - try - { - Class cls = Class.forName(partitionerClassName); - return (IPartitioner) cls.getConstructor().newInstance(); - } - catch (Exception e) - { - throw new RuntimeException("Invalid partitioner class " + partitionerClassName); - } - } - private static IEndpointSnitch createEndpointSnitch(String endpointSnitchClassName) throws ConfigurationException { IEndpointSnitch snitch; @@ -667,39 +651,9 @@ public class DatabaseDescriptor Class typeClass; if (compareWith == null) - { - typeClass = BytesType.class; - } - else - { - String className = compareWith.contains(".") ? compareWith : "org.apache.cassandra.db.marshal." + compareWith; - try - { - typeClass = (Class)Class.forName(className); - } - catch (ClassNotFoundException e) - { - throw new ConfigurationException("Unable to load class " + className); - } - } + compareWith = "BytesType"; - try - { - Field field = typeClass.getDeclaredField("instance"); - return (AbstractType) field.get(null); - } - catch (NoSuchFieldException e) - { - ConfigurationException ex = new ConfigurationException("Invalid comparator: must define a public static instance field."); - ex.initCause(e); - throw ex; - } - catch (IllegalAccessException e) - { - ConfigurationException ex = new ConfigurationException("Invalid comparator: must define a public static instance field."); - ex.initCause(e); - throw ex; - } + return FBUtilities.getComparator(compareWith); } public static AbstractReconciler getReconciler(String reconcileWith) throws ConfigurationException diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordReader.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordReader.java index b494ba28ce..b070527e85 100644 --- a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordReader.java +++ b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordReader.java @@ -40,6 +40,7 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.thrift.*; import org.apache.cassandra.thrift.Column; import org.apache.cassandra.thrift.SuperColumn; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.InputSplit; @@ -130,10 +131,10 @@ public class ColumnFamilyRecordReader extends RecordReader info = client.describe_keyspace(keyspace).get(cfName); - comparator = DatabaseDescriptor.getComparator(info.get("CompareWith")); - subComparator = DatabaseDescriptor.getComparator(info.get("CompareSubcolumnsWith")); + comparator = FBUtilities.getComparator(info.get("CompareWith")); + subComparator = FBUtilities.getComparator(info.get("CompareSubcolumnsWith")); } catch (ConfigurationException e) { diff --git a/src/java/org/apache/cassandra/hadoop/ConfigHelper.java b/src/java/org/apache/cassandra/hadoop/ConfigHelper.java index 4faef3c1db..64239fb5f8 100644 --- a/src/java/org/apache/cassandra/hadoop/ConfigHelper.java +++ b/src/java/org/apache/cassandra/hadoop/ConfigHelper.java @@ -21,11 +21,7 @@ package org.apache.cassandra.hadoop; */ -import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.thrift.InvalidRequestException; import org.apache.cassandra.thrift.SlicePredicate; import org.apache.cassandra.utils.FBUtilities; import org.apache.hadoop.conf.Configuration; @@ -52,9 +48,6 @@ public class ConfigHelper private static final int DEFAULT_RANGE_BATCH_SIZE = 4096; private static final String THRIFT_PORT = "cassandra.thrift.port"; private static final String INITIAL_THRIFT_ADDRESS = "cassandra.thrift.address"; - private static final String COMPARATOR = "cassandra.input.comparator"; - private static final String SUB_COMPARATOR = "cassandra.input.subcomparator"; - private static final String PARTITIONER = "cassandra.partitioner"; /** * Set the keyspace and column family for the input of this job. diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index 855f9e5bca..eaac1ab332 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -19,6 +19,9 @@ package org.apache.cassandra.utils; import java.io.*; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.math.BigInteger; import java.net.InetAddress; import java.net.URL; @@ -43,6 +46,9 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.IClock; import org.apache.cassandra.db.IClock.ClockRelationship; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.PropertyFileSnitch; @@ -547,4 +553,52 @@ public class FBUtilities } } } + + public static IPartitioner newPartitioner(String partitionerClassName) + { + if (!partitionerClassName.contains(".")) + partitionerClassName = "org.apache.cassandra.dht." + partitionerClassName; + + try + { + Class cls = Class.forName(partitionerClassName); + return (IPartitioner) cls.getConstructor().newInstance(); + } + catch (Exception e) + { + throw new RuntimeException("Invalid partitioner class " + partitionerClassName); + } + } + + public static AbstractType getComparator(String compareWith) throws ConfigurationException + { + String className = compareWith.contains(".") ? compareWith : "org.apache.cassandra.db.marshal." + compareWith; + Class typeClass; + try + { + typeClass = (Class)Class.forName(className); + } + catch (ClassNotFoundException e) + { + throw new ConfigurationException("Unable to load class " + className); + } + + try + { + Field field = typeClass.getDeclaredField("instance"); + return (AbstractType) field.get(null); + } + catch (NoSuchFieldException e) + { + ConfigurationException ex = new ConfigurationException("Invalid comparator: must define a public static instance field."); + ex.initCause(e); + throw ex; + } + catch (IllegalAccessException e) + { + ConfigurationException ex = new ConfigurationException("Invalid comparator: must define a public static instance field."); + ex.initCause(e); + throw ex; + } + } }