natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge from 0.6 

git-svn-id: https://svn.apache.org/repos/asf/cassandra/trunk@979406 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Ellis 2010-07-26 18:44:25 +00:00
parent 3015876a86
commit 1babf7602b
7 changed files with 66 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGES.txt
View File

@ -50,11 +50,6 @@ dev
(CASSANDRA-1232)
* extend option to lower compaction priority to hinted handoff
as well (CASSANDRA-1260)
* log thread pool stats when GC is excessive (CASSANDRA-1275)
* added describe_partitioner Thrift method (CASSANDRA-1047)
* Hadoop jobs no longer require the Cassandra storage-conf.xml
(CASSANDRA-1280, CASSANDRA-1047)
* log thread pool stats when GC is excessive (CASSANDRA-1275)
* log errors in gossip instead of re-throwing (CASSANDRA-1289)
* avoid aborting commitlog replay prematurely if a flushed-but-
not-removed commitlog segment is encountered (CASSANDRA-1297)
@ -69,6 +64,7 @@ dev
* Hadoop jobs no longer require the Cassandra storage-conf.xml
(CASSANDRA-1280, CASSANDRA-1047)
* log thread pool stats when GC is excessive (CASSANDRA-1275)
* remove gossip message size limit (CASSANDRA-1138)
0.6.3

2
contrib/word_count/bin/word_count
View File

@ -31,7 +31,7 @@ if [ ! -e $cwd/../build/word_count.jar ]; then
fi
CLASSPATH=$CLASSPATH:$cwd/../build/word_count.jar
CLASSPATH=$CLASSPATH:.:$cwd/../../../build/classes
CLASSPATH=$CLASSPATH:$cwd/../../../build/classes
for jar in $cwd/../build/lib/jars/*.jar; do
CLASSPATH=$CLASSPATH:$jar
done

7
contrib/word_count/src/WordCount.java
View File

@ -112,13 +112,12 @@ public class WordCount extends Configured implements Tool
public int run(String[] args) throws Exception
{
Configuration conf = getConf();
for (int i = 0; i < WordCountSetup.TEST_COUNT; i++)
{
String columnName = "text" + i;
conf.set(CONF_COLUMN_NAME, columnName);
Job job = new Job(conf, "wordcount");
getConf().set(CONF_COLUMN_NAME, columnName);
Job job = new Job(getConf(), "wordcount");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
@ -129,7 +128,7 @@ public class WordCount extends Configured implements Tool
job.setInputFormatClass(ColumnFamilyInputFormat.class);
FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH_PREFIX + i));
ConfigHelper.setThriftContact(conf, "localhost", 9160);
ConfigHelper.setThriftContact(job.getConfiguration(), "localhost", 9160);
ConfigHelper.setInputColumnFamily(job.getConfiguration(), KEYSPACE, COLUMN_FAMILY);
SlicePredicate predicate = new SlicePredicate().setColumn_names(Arrays.asList(columnName.getBytes()));
ConfigHelper.setInputSlicePredicate(job.getConfiguration(), predicate);

52
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -205,7 +205,7 @@ public class DatabaseDescriptor
}
try
{
partitioner = newPartitioner(conf.partitioner);
partitioner = FBUtilities.newPartitioner(conf.partitioner);
}
catch (Exception e)
{
@ -385,22 +385,6 @@ public class DatabaseDescriptor
}
}
public static IPartitioner newPartitioner(String partitionerClassName)
{
if (!partitionerClassName.contains("."))
partitionerClassName = "org.apache.cassandra.dht." + partitionerClassName;
try
{
Class cls = Class.forName(partitionerClassName);
return (IPartitioner) cls.getConstructor().newInstance();
}
catch (Exception e)
{
throw new RuntimeException("Invalid partitioner class " + partitionerClassName);
}
}
private static IEndpointSnitch createEndpointSnitch(String endpointSnitchClassName) throws ConfigurationException
{
IEndpointSnitch snitch;
@ -667,39 +651,9 @@ public class DatabaseDescriptor
Class<? extends AbstractType> typeClass;
if (compareWith == null)
{
typeClass = BytesType.class;
}
else
{
String className = compareWith.contains(".") ? compareWith : "org.apache.cassandra.db.marshal." + compareWith;
try
{
typeClass = (Class<? extends AbstractType>)Class.forName(className);
}
catch (ClassNotFoundException e)
{
throw new ConfigurationException("Unable to load class " + className);
}
}
compareWith = "BytesType";
try
{
Field field = typeClass.getDeclaredField("instance");
return (AbstractType) field.get(null);
}
catch (NoSuchFieldException e)
{
ConfigurationException ex = new ConfigurationException("Invalid comparator: must define a public static instance field.");
ex.initCause(e);
throw ex;
}
catch (IllegalAccessException e)
{
ConfigurationException ex = new ConfigurationException("Invalid comparator: must define a public static instance field.");
ex.initCause(e);
throw ex;
}
return FBUtilities.getComparator(compareWith);
}
public static AbstractReconciler getReconciler(String reconcileWith) throws ConfigurationException

7
src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordReader.java
View File

@ -40,6 +40,7 @@ import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.thrift.*;
import org.apache.cassandra.thrift.Column;
import org.apache.cassandra.thrift.SuperColumn;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
@ -130,10 +131,10 @@ public class ColumnFamilyRecordReader extends RecordReader<byte[], SortedMap<byt
{
try
{
partitioner = DatabaseDescriptor.newPartitioner(client.describe_partitioner());
partitioner = FBUtilities.newPartitioner(client.describe_partitioner());
Map<String, String> info = client.describe_keyspace(keyspace).get(cfName);
comparator = DatabaseDescriptor.getComparator(info.get("CompareWith"));
subComparator = DatabaseDescriptor.getComparator(info.get("CompareSubcolumnsWith"));
comparator = FBUtilities.getComparator(info.get("CompareWith"));
subComparator = FBUtilities.getComparator(info.get("CompareSubcolumnsWith"));
}
catch (ConfigurationException e)
{

7
src/java/org/apache/cassandra/hadoop/ConfigHelper.java
View File

@ -21,11 +21,7 @@ package org.apache.cassandra.hadoop;
*/
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.thrift.SlicePredicate;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.hadoop.conf.Configuration;
@ -52,9 +48,6 @@ public class ConfigHelper
private static final int DEFAULT_RANGE_BATCH_SIZE = 4096;
private static final String THRIFT_PORT = "cassandra.thrift.port";
private static final String INITIAL_THRIFT_ADDRESS = "cassandra.thrift.address";
private static final String COMPARATOR = "cassandra.input.comparator";
private static final String SUB_COMPARATOR = "cassandra.input.subcomparator";
private static final String PARTITIONER = "cassandra.partitioner";
/**
* Set the keyspace and column family for the input of this job.

54
src/java/org/apache/cassandra/utils/FBUtilities.java
View File

@ -19,6 +19,9 @@
package org.apache.cassandra.utils;
import java.io.*;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.URL;
@ -43,6 +46,9 @@ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.IClock;
import org.apache.cassandra.db.IClock.ClockRelationship;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.PropertyFileSnitch;
@ -547,4 +553,52 @@ public class FBUtilities
}
}
}
public static IPartitioner newPartitioner(String partitionerClassName)
{
if (!partitionerClassName.contains("."))
partitionerClassName = "org.apache.cassandra.dht." + partitionerClassName;
try
{
Class cls = Class.forName(partitionerClassName);
return (IPartitioner) cls.getConstructor().newInstance();
}
catch (Exception e)
{
throw new RuntimeException("Invalid partitioner class " + partitionerClassName);
}
}
public static AbstractType getComparator(String compareWith) throws ConfigurationException
{
String className = compareWith.contains(".") ? compareWith : "org.apache.cassandra.db.marshal." + compareWith;
Class<? extends AbstractType> typeClass;
try
{
typeClass = (Class<? extends AbstractType>)Class.forName(className);
}
catch (ClassNotFoundException e)
{
throw new ConfigurationException("Unable to load class " + className);
}
try
{
Field field = typeClass.getDeclaredField("instance");
return (AbstractType) field.get(null);
}
catch (NoSuchFieldException e)
{
ConfigurationException ex = new ConfigurationException("Invalid comparator: must define a public static instance field.");
ex.initCause(e);
throw ex;
}
catch (IllegalAccessException e)
{
ConfigurationException ex = new ConfigurationException("Invalid comparator: must define a public static instance field.");
ex.initCause(e);
throw ex;
}
}
}