diff --git a/CHANGES.txt b/CHANGES.txt
index 13f2495c9d..8081dc472d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -47,6 +47,8 @@ dev
* change multiget key collection from list to set (CASSANDRA-1329)
* ability to modify keyspaces and column family definitions on a live cluster
(CASSANDRA-1285)
+ * support for Hadoop Streaming [non-jvm map/reduce via stdin/out]
+ (CASSANDRA-1368)
0.7-beta1
diff --git a/NEWS.txt b/NEWS.txt
index ac5bf56a2c..2858470898 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -15,7 +15,8 @@ Features
- Optional per-Column time-to-live field allows expiring data without
have to issue explicit remove commands
- `truncate` thrift method allows clearing an entire ColumnFamily at once
- - Hadoop OutputFormat support
+ - Hadoop OutputFormat and Streaming [non-jvm map/reduce via stdin/out]
+ support
- Up to 8x faster reads from row cache
- A new ByteOrderedPartitioner supports bytes keys with arbitrary content,
and orders keys by their byte value. This should be used in new
diff --git a/interface/cassandra.genavro b/interface/cassandra.genavro
index a5b174d464..96eda25763 100644
--- a/interface/cassandra.genavro
+++ b/interface/cassandra.genavro
@@ -169,6 +169,11 @@ protocol Cassandra {
array cf_defs;
}
+ record StreamingMutation {
+ bytes key;
+ Mutation mutation;
+ }
+
record MutationsMapEntry {
bytes key;
map> mutations;
diff --git a/ivy.xml b/ivy.xml
index dfb83e3781..1de56d3066 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -28,6 +28,7 @@
+
diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java
index 98fbf70688..4403ed5886 100644
--- a/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java
+++ b/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java
@@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory;
*
*/
public class ColumnFamilyOutputFormat extends OutputFormat>
+ implements org.apache.hadoop.mapred.OutputFormat>
{
private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyOutputFormat.class);
@@ -85,13 +86,17 @@ public class ColumnFamilyOutputFormat extends OutputFormat> getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException
+ public ColumnFamilyRecordWriter getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException
{
return new ColumnFamilyRecordWriter(context);
}
@@ -126,30 +145,29 @@ public class ColumnFamilyOutputFormat extends OutputFormat creds = new HashMap();
- creds.put(SimpleAuthenticator.USERNAME_KEY, ConfigHelper.getOutputKeyspaceUserName(context.getConfiguration()));
- creds.put(SimpleAuthenticator.PASSWORD_KEY, ConfigHelper.getOutputKeyspacePassword(context.getConfiguration()));
+ creds.put(SimpleAuthenticator.USERNAME_KEY, ConfigHelper.getOutputKeyspaceUserName(conf));
+ creds.put(SimpleAuthenticator.PASSWORD_KEY, ConfigHelper.getOutputKeyspacePassword(conf));
AuthenticationRequest authRequest = new AuthenticationRequest(creds);
client.login(authRequest);
}
return client;
-
}
/**
diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
index e3bfce6f37..0a9bcbc3fb 100644
--- a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
+++ b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
@@ -47,6 +47,8 @@ import org.apache.cassandra.thrift.SlicePredicate;
import org.apache.cassandra.thrift.SliceRange;
import org.apache.cassandra.thrift.SuperColumn;
import org.apache.cassandra.utils.FBUtilities;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -75,10 +77,10 @@ import org.apache.thrift.transport.TSocket;
* @see OutputFormat
*
*/
-final class ColumnFamilyRecordWriter extends RecordWriter>
+final class ColumnFamilyRecordWriter extends RecordWriter> implements org.apache.hadoop.mapred.RecordWriter>
{
- // The task attempt context this writer is associated with.
- private final TaskAttemptContext context;
+ // The configuration this writer is associated with.
+ private final Configuration conf;
// The batched set of mutations grouped by endpoints.
private Map>>> mutationsByEndpoint;
@@ -104,15 +106,20 @@ final class ColumnFamilyRecordWriter extends RecordWriter>>>();
- this.ringCache = new RingCache(ConfigHelper.getOutputKeyspace(context.getConfiguration()),
- ConfigHelper.getPartitioner(context.getConfiguration()),
- ConfigHelper.getInitialAddress(context.getConfiguration()),
- ConfigHelper.getRpcPort(context.getConfiguration()));
- this.batchThreshold = context.getConfiguration().getLong(ColumnFamilyOutputFormat.BATCH_THRESHOLD, Long.MAX_VALUE);
+ this(context.getConfiguration());
}
+ ColumnFamilyRecordWriter(Configuration conf) throws IOException
+ {
+ this.conf = conf;
+ this.mutationsByEndpoint = new HashMap>>>();
+ this.ringCache = new RingCache(ConfigHelper.getOutputKeyspace(conf),
+ ConfigHelper.getPartitioner(conf),
+ ConfigHelper.getInitialAddress(conf),
+ ConfigHelper.getRpcPort(conf));
+ this.batchThreshold = conf.getLong(ColumnFamilyOutputFormat.BATCH_THRESHOLD, Long.MAX_VALUE);
+ }
+
/**
* Return the endpoint responsible for the given key. The selected endpoint
* one whose token range contains the given key.
@@ -145,7 +152,7 @@ final class ColumnFamilyRecordWriter extends RecordWriter value) throws IOException, InterruptedException
+ public synchronized void write(ByteBuffer keybuff, List value) throws IOException
{
maybeFlush();
byte[] key = copy(keybuff);
@@ -164,11 +171,11 @@ final class ColumnFamilyRecordWriter extends RecordWriter mutationList = cfMutation.get(ConfigHelper.getOutputColumnFamily(context.getConfiguration()));
+ List mutationList = cfMutation.get(ConfigHelper.getOutputColumnFamily(conf));
if (mutationList == null)
{
mutationList = new ArrayList();
- cfMutation.put(ConfigHelper.getOutputColumnFamily(context.getConfiguration()), mutationList);
+ cfMutation.put(ConfigHelper.getOutputColumnFamily(conf), mutationList);
}
for (org.apache.cassandra.avro.Mutation amut : value)
@@ -254,6 +261,13 @@ final class ColumnFamilyRecordWriter extends RecordWriter> mutationFutures = new ArrayList>();
for (Map.Entry>>> entry : mutationsByEndpoint.entrySet())
{
- mutationFutures.add(executor.submit(new EndpointCallable(context, entry.getKey(), entry.getValue())));
+ mutationFutures.add(executor.submit(new EndpointCallable(conf, entry.getKey(), entry.getValue())));
}
// wait until we have all the results back
for (Future> mutationFuture : mutationFutures)
@@ -321,7 +335,7 @@ final class ColumnFamilyRecordWriter extends RecordWriter
{
// The task attempt context associated with this callable.
- private TaskAttemptContext taskContext;
+ private Configuration conf;
// The endpoint of the primary replica for the rows being mutated
private InetAddress endpoint;
// The mutations to be performed in the node referenced by {@link
@@ -332,13 +346,14 @@ final class ColumnFamilyRecordWriter extends RecordWriter>> mutations)
+ public EndpointCallable(Configuration conf, InetAddress endpoint, Map>> mutations)
{
- this.taskContext = taskContext;
+ this.conf = conf;
this.endpoint = endpoint;
this.mutations = mutations;
}
@@ -352,8 +367,8 @@ final class ColumnFamilyRecordWriter extends RecordWriter