diff --git a/CHANGES.txt b/CHANGES.txt index 13f2495c9d..8081dc472d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -47,6 +47,8 @@ dev * change multiget key collection from list to set (CASSANDRA-1329) * ability to modify keyspaces and column family definitions on a live cluster (CASSANDRA-1285) + * support for Hadoop Streaming [non-jvm map/reduce via stdin/out] + (CASSANDRA-1368) 0.7-beta1 diff --git a/NEWS.txt b/NEWS.txt index ac5bf56a2c..2858470898 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -15,7 +15,8 @@ Features - Optional per-Column time-to-live field allows expiring data without have to issue explicit remove commands - `truncate` thrift method allows clearing an entire ColumnFamily at once - - Hadoop OutputFormat support + - Hadoop OutputFormat and Streaming [non-jvm map/reduce via stdin/out] + support - Up to 8x faster reads from row cache - A new ByteOrderedPartitioner supports bytes keys with arbitrary content, and orders keys by their byte value. This should be used in new diff --git a/interface/cassandra.genavro b/interface/cassandra.genavro index a5b174d464..96eda25763 100644 --- a/interface/cassandra.genavro +++ b/interface/cassandra.genavro @@ -169,6 +169,11 @@ protocol Cassandra { array cf_defs; } + record StreamingMutation { + bytes key; + Mutation mutation; + } + record MutationsMapEntry { bytes key; map> mutations; diff --git a/ivy.xml b/ivy.xml index dfb83e3781..1de56d3066 100644 --- a/ivy.xml +++ b/ivy.xml @@ -28,6 +28,7 @@ + diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java index 98fbf70688..4403ed5886 100644 --- a/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java +++ b/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java @@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory; *

*/ public class ColumnFamilyOutputFormat extends OutputFormat> + implements org.apache.hadoop.mapred.OutputFormat> { private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyOutputFormat.class); @@ -85,13 +86,17 @@ public class ColumnFamilyOutputFormat extends OutputFormat> getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException + public ColumnFamilyRecordWriter getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException { return new ColumnFamilyRecordWriter(context); } @@ -126,30 +145,29 @@ public class ColumnFamilyOutputFormat extends OutputFormat creds = new HashMap(); - creds.put(SimpleAuthenticator.USERNAME_KEY, ConfigHelper.getOutputKeyspaceUserName(context.getConfiguration())); - creds.put(SimpleAuthenticator.PASSWORD_KEY, ConfigHelper.getOutputKeyspacePassword(context.getConfiguration())); + creds.put(SimpleAuthenticator.USERNAME_KEY, ConfigHelper.getOutputKeyspaceUserName(conf)); + creds.put(SimpleAuthenticator.PASSWORD_KEY, ConfigHelper.getOutputKeyspacePassword(conf)); AuthenticationRequest authRequest = new AuthenticationRequest(creds); client.login(authRequest); } return client; - } /** diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java index e3bfce6f37..0a9bcbc3fb 100644 --- a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java +++ b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java @@ -47,6 +47,8 @@ import org.apache.cassandra.thrift.SlicePredicate; import org.apache.cassandra.thrift.SliceRange; import org.apache.cassandra.thrift.SuperColumn; import org.apache.cassandra.utils.FBUtilities; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.OutputFormat; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -75,10 +77,10 @@ import org.apache.thrift.transport.TSocket; * @see OutputFormat * */ -final class ColumnFamilyRecordWriter extends RecordWriter> +final class ColumnFamilyRecordWriter extends RecordWriter> implements org.apache.hadoop.mapred.RecordWriter> { - // The task attempt context this writer is associated with. - private final TaskAttemptContext context; + // The configuration this writer is associated with. + private final Configuration conf; // The batched set of mutations grouped by endpoints. private Map>>> mutationsByEndpoint; @@ -104,15 +106,20 @@ final class ColumnFamilyRecordWriter extends RecordWriter>>>(); - this.ringCache = new RingCache(ConfigHelper.getOutputKeyspace(context.getConfiguration()), - ConfigHelper.getPartitioner(context.getConfiguration()), - ConfigHelper.getInitialAddress(context.getConfiguration()), - ConfigHelper.getRpcPort(context.getConfiguration())); - this.batchThreshold = context.getConfiguration().getLong(ColumnFamilyOutputFormat.BATCH_THRESHOLD, Long.MAX_VALUE); + this(context.getConfiguration()); } + ColumnFamilyRecordWriter(Configuration conf) throws IOException + { + this.conf = conf; + this.mutationsByEndpoint = new HashMap>>>(); + this.ringCache = new RingCache(ConfigHelper.getOutputKeyspace(conf), + ConfigHelper.getPartitioner(conf), + ConfigHelper.getInitialAddress(conf), + ConfigHelper.getRpcPort(conf)); + this.batchThreshold = conf.getLong(ColumnFamilyOutputFormat.BATCH_THRESHOLD, Long.MAX_VALUE); + } + /** * Return the endpoint responsible for the given key. The selected endpoint * one whose token range contains the given key. @@ -145,7 +152,7 @@ final class ColumnFamilyRecordWriter extends RecordWriter value) throws IOException, InterruptedException + public synchronized void write(ByteBuffer keybuff, List value) throws IOException { maybeFlush(); byte[] key = copy(keybuff); @@ -164,11 +171,11 @@ final class ColumnFamilyRecordWriter extends RecordWriter mutationList = cfMutation.get(ConfigHelper.getOutputColumnFamily(context.getConfiguration())); + List mutationList = cfMutation.get(ConfigHelper.getOutputColumnFamily(conf)); if (mutationList == null) { mutationList = new ArrayList(); - cfMutation.put(ConfigHelper.getOutputColumnFamily(context.getConfiguration()), mutationList); + cfMutation.put(ConfigHelper.getOutputColumnFamily(conf), mutationList); } for (org.apache.cassandra.avro.Mutation amut : value) @@ -254,6 +261,13 @@ final class ColumnFamilyRecordWriter extends RecordWriter> mutationFutures = new ArrayList>(); for (Map.Entry>>> entry : mutationsByEndpoint.entrySet()) { - mutationFutures.add(executor.submit(new EndpointCallable(context, entry.getKey(), entry.getValue()))); + mutationFutures.add(executor.submit(new EndpointCallable(conf, entry.getKey(), entry.getValue()))); } // wait until we have all the results back for (Future mutationFuture : mutationFutures) @@ -321,7 +335,7 @@ final class ColumnFamilyRecordWriter extends RecordWriter { // The task attempt context associated with this callable. - private TaskAttemptContext taskContext; + private Configuration conf; // The endpoint of the primary replica for the rows being mutated private InetAddress endpoint; // The mutations to be performed in the node referenced by {@link @@ -332,13 +346,14 @@ final class ColumnFamilyRecordWriter extends RecordWriter>> mutations) + public EndpointCallable(Configuration conf, InetAddress endpoint, Map>> mutations) { - this.taskContext = taskContext; + this.conf = conf; this.endpoint = endpoint; this.mutations = mutations; } @@ -352,8 +367,8 @@ final class ColumnFamilyRecordWriter extends RecordWriter