From 337e4a8292d29ea1a07c932d8853ca62939548ed Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Fri, 18 Jul 2014 12:31:17 -0500 Subject: [PATCH 1/3] Fix word count Patch by Ala' Alkhaldi, reviewed by brandonwilliams for CASSANDRA-7200 --- examples/hadoop_cql3_word_count/README.txt | 6 +++-- .../conf/log4j.properties | 2 +- .../src/WordCountSetup.java | 5 +++- examples/hadoop_word_count/README.txt | 6 +++-- .../hadoop_word_count/conf/log4j.properties | 2 +- .../hadoop_word_count/src/WordCountSetup.java | 25 ++++++++++++++++++- 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/examples/hadoop_cql3_word_count/README.txt b/examples/hadoop_cql3_word_count/README.txt index f984b0294f..e01632b07c 100644 --- a/examples/hadoop_cql3_word_count/README.txt +++ b/examples/hadoop_cql3_word_count/README.txt @@ -36,8 +36,10 @@ Read the code in src/ for more details. The word_count_counters example sums the counter columns for a row. The output is written to a text file in /tmp/word_count_counters. -*If you want to point wordcount at a real cluster, modify the seed -and listenaddress settings accordingly. +*It is recommended to turn off vnodes when running Cassandra with hadoop. +This is done by setting "num_tokens: 1" in cassandra.yaml. If you want to +point wordcount at a real cluster, modify the seed and listenaddress +settings accordingly. Troubleshooting diff --git a/examples/hadoop_cql3_word_count/conf/log4j.properties b/examples/hadoop_cql3_word_count/conf/log4j.properties index 70f7657591..508d60fa2d 100644 --- a/examples/hadoop_cql3_word_count/conf/log4j.properties +++ b/examples/hadoop_cql3_word_count/conf/log4j.properties @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=DEBUG,stdout,F +log4j.rootLogger=INFO,stdout,F #stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/examples/hadoop_cql3_word_count/src/WordCountSetup.java b/examples/hadoop_cql3_word_count/src/WordCountSetup.java index ebf7485d6b..cffe2724c1 100644 --- a/examples/hadoop_cql3_word_count/src/WordCountSetup.java +++ b/examples/hadoop_cql3_word_count/src/WordCountSetup.java @@ -70,7 +70,10 @@ public class WordCountSetup client.execute_cql3_query(ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE); - int magnitude = client.describe_ring(WordCount.KEYSPACE).size(); + String verifyQuery = "select count(*) from system.peers"; + CqlResult result = client.execute_cql3_query(ByteBufferUtil.bytes(verifyQuery), Compression.NONE, ConsistencyLevel.ONE); + + long magnitude = ByteBufferUtil.toLong(result.rows.get(0).columns.get(0).value); try { Thread.sleep(1000 * magnitude); diff --git a/examples/hadoop_word_count/README.txt b/examples/hadoop_word_count/README.txt index cf8a34469e..9a93908366 100644 --- a/examples/hadoop_word_count/README.txt +++ b/examples/hadoop_word_count/README.txt @@ -37,8 +37,10 @@ Read the code in src/ for more details. The word_count_counters example sums the counter columns for a row. The output is written to a text file in /tmp/word_count_counters. -*If you want to point wordcount at a real cluster, modify the seed -and listenaddress settings accordingly. +*It is recommended to turn off vnodes when running Cassandra with hadoop. +This is done by setting "num_tokens: 1" in cassandra.yaml. If you want to +point wordcount at a real cluster, modify the seed and listenaddress +settings accordingly. Troubleshooting diff --git a/examples/hadoop_word_count/conf/log4j.properties b/examples/hadoop_word_count/conf/log4j.properties index 70f7657591..508d60fa2d 100644 --- a/examples/hadoop_word_count/conf/log4j.properties +++ b/examples/hadoop_word_count/conf/log4j.properties @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=DEBUG,stdout,F +log4j.rootLogger=INFO,stdout,F #stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/examples/hadoop_word_count/src/WordCountSetup.java b/examples/hadoop_word_count/src/WordCountSetup.java index e222327283..6dd2ba7628 100644 --- a/examples/hadoop_word_count/src/WordCountSetup.java +++ b/examples/hadoop_word_count/src/WordCountSetup.java @@ -176,10 +176,33 @@ public class WordCountSetup KsDef ksDef = new KsDef(WordCount.KEYSPACE, "org.apache.cassandra.locator.SimpleStrategy", cfDefList); ksDef.putToStrategy_options("replication_factor", "1"); client.system_add_keyspace(ksDef); - int magnitude = client.describe_ring(WordCount.KEYSPACE).size(); + + int magnitude = getNumberOfHosts(client); Uninterruptibles.sleepUninterruptibly(magnitude, TimeUnit.SECONDS); } + private static int getNumberOfHosts(Cassandra.Iface client) + throws InvalidRequestException, UnavailableException, TimedOutException, TException + { + client.set_keyspace("system"); + SlicePredicate predicate = new SlicePredicate(); + SliceRange sliceRange = new SliceRange(); + sliceRange.setStart(new byte[0]); + sliceRange.setFinish(new byte[0]); + predicate.setSlice_range(sliceRange); + + KeyRange keyrRange = new KeyRange(); + keyrRange.setStart_key(new byte[0]); + keyrRange.setEnd_key(new byte[0]); + //keyrRange.setCount(100); + + ColumnParent parent = new ColumnParent("peers"); + + List ls = client.get_range_slices(parent, predicate, keyrRange, ConsistencyLevel.ONE); + + return ls.size(); + } + private static Cassandra.Iface createConnection() throws TTransportException { if (System.getProperty("cassandra.host") == null || System.getProperty("cassandra.port") == null) From 8fd8832a8723ef5e663834def078ea0afc7ff8d6 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Fri, 18 Jul 2014 12:35:57 -0500 Subject: [PATCH 3/3] Fix word count Patch by Ala' Alkhaldi, reviewed by brandonwilliams for CASSANDRA-7200 --- examples/hadoop_cql3_word_count/README.txt | 6 +++-- .../conf/log4j.properties | 2 +- .../src/WordCountSetup.java | 5 +++- examples/hadoop_word_count/README.txt | 6 +++-- .../hadoop_word_count/conf/log4j.properties | 2 +- examples/hadoop_word_count/src/WordCount.java | 5 +++- .../src/WordCountCounters.java | 2 ++ .../hadoop_word_count/src/WordCountSetup.java | 25 ++++++++++++++++++- 8 files changed, 44 insertions(+), 9 deletions(-) diff --git a/examples/hadoop_cql3_word_count/README.txt b/examples/hadoop_cql3_word_count/README.txt index f984b0294f..e01632b07c 100644 --- a/examples/hadoop_cql3_word_count/README.txt +++ b/examples/hadoop_cql3_word_count/README.txt @@ -36,8 +36,10 @@ Read the code in src/ for more details. The word_count_counters example sums the counter columns for a row. The output is written to a text file in /tmp/word_count_counters. -*If you want to point wordcount at a real cluster, modify the seed -and listenaddress settings accordingly. +*It is recommended to turn off vnodes when running Cassandra with hadoop. +This is done by setting "num_tokens: 1" in cassandra.yaml. If you want to +point wordcount at a real cluster, modify the seed and listenaddress +settings accordingly. Troubleshooting diff --git a/examples/hadoop_cql3_word_count/conf/log4j.properties b/examples/hadoop_cql3_word_count/conf/log4j.properties index 70f7657591..508d60fa2d 100644 --- a/examples/hadoop_cql3_word_count/conf/log4j.properties +++ b/examples/hadoop_cql3_word_count/conf/log4j.properties @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=DEBUG,stdout,F +log4j.rootLogger=INFO,stdout,F #stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/examples/hadoop_cql3_word_count/src/WordCountSetup.java b/examples/hadoop_cql3_word_count/src/WordCountSetup.java index ebf7485d6b..e514d631fe 100644 --- a/examples/hadoop_cql3_word_count/src/WordCountSetup.java +++ b/examples/hadoop_cql3_word_count/src/WordCountSetup.java @@ -70,7 +70,10 @@ public class WordCountSetup client.execute_cql3_query(ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE); - int magnitude = client.describe_ring(WordCount.KEYSPACE).size(); + String verifyQuery = "select count(*) from system.peers"; + CqlResult result = client.execute_cql3_query(ByteBufferUtil.bytes(verifyQuery), Compression.NONE, ConsistencyLevel.ONE); + + long magnitude = ByteBufferUtil.toLong(result.rows.get(0).columns.get(0).value); try { Thread.sleep(1000 * magnitude); diff --git a/examples/hadoop_word_count/README.txt b/examples/hadoop_word_count/README.txt index cf8a34469e..9a93908366 100644 --- a/examples/hadoop_word_count/README.txt +++ b/examples/hadoop_word_count/README.txt @@ -37,8 +37,10 @@ Read the code in src/ for more details. The word_count_counters example sums the counter columns for a row. The output is written to a text file in /tmp/word_count_counters. -*If you want to point wordcount at a real cluster, modify the seed -and listenaddress settings accordingly. +*It is recommended to turn off vnodes when running Cassandra with hadoop. +This is done by setting "num_tokens: 1" in cassandra.yaml. If you want to +point wordcount at a real cluster, modify the seed and listenaddress +settings accordingly. Troubleshooting diff --git a/examples/hadoop_word_count/conf/log4j.properties b/examples/hadoop_word_count/conf/log4j.properties index 70f7657591..508d60fa2d 100644 --- a/examples/hadoop_word_count/conf/log4j.properties +++ b/examples/hadoop_word_count/conf/log4j.properties @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=DEBUG,stdout,F +log4j.rootLogger=INFO,stdout,F #stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/examples/hadoop_word_count/src/WordCount.java b/examples/hadoop_word_count/src/WordCount.java index ce9565da80..f6bca776f2 100644 --- a/examples/hadoop_word_count/src/WordCount.java +++ b/examples/hadoop_word_count/src/WordCount.java @@ -18,8 +18,11 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.*; import org.apache.cassandra.db.Cell; +import org.apache.cassandra.thrift.*; +import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.hadoop.ColumnFamilyOutputFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +86,7 @@ public class WordCount extends Configured implements Tool { for (Cell cell : columns.values()) { - String name = ByteBufferUtil.string(cell.name()); + String name = ByteBufferUtil.string(cell.name().toByteBuffer()); String value = null; if (name.contains("int")) diff --git a/examples/hadoop_word_count/src/WordCountCounters.java b/examples/hadoop_word_count/src/WordCountCounters.java index 4ea063b583..39fb7782fd 100644 --- a/examples/hadoop_word_count/src/WordCountCounters.java +++ b/examples/hadoop_word_count/src/WordCountCounters.java @@ -18,8 +18,10 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.SortedMap; import org.apache.cassandra.db.Cell; +import org.apache.cassandra.thrift.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/examples/hadoop_word_count/src/WordCountSetup.java b/examples/hadoop_word_count/src/WordCountSetup.java index e222327283..0ef5341ed1 100644 --- a/examples/hadoop_word_count/src/WordCountSetup.java +++ b/examples/hadoop_word_count/src/WordCountSetup.java @@ -176,10 +176,33 @@ public class WordCountSetup KsDef ksDef = new KsDef(WordCount.KEYSPACE, "org.apache.cassandra.locator.SimpleStrategy", cfDefList); ksDef.putToStrategy_options("replication_factor", "1"); client.system_add_keyspace(ksDef); - int magnitude = client.describe_ring(WordCount.KEYSPACE).size(); + + int magnitude = getNumberOfHosts(client); Uninterruptibles.sleepUninterruptibly(magnitude, TimeUnit.SECONDS); } + private static int getNumberOfHosts(Cassandra.Iface client) + throws InvalidRequestException, UnavailableException, TimedOutException, TException + { + client.set_keyspace("system"); + SlicePredicate predicate = new SlicePredicate(); + SliceRange sliceRange = new SliceRange(); + sliceRange.setStart(new byte[0]); + sliceRange.setFinish(new byte[0]); + predicate.setSlice_range(sliceRange); + + KeyRange keyrRange = new KeyRange(); + keyrRange.setStart_key(new byte[0]); + keyrRange.setEnd_key(new byte[0]); + //keyrRange.setCount(100); + + ColumnParent parent = new ColumnParent("peers"); + + List ls = client.get_range_slices(parent, predicate, keyrRange, ConsistencyLevel.ONE); + + return ls.size(); + } + private static Cassandra.Iface createConnection() throws TTransportException { if (System.getProperty("cassandra.host") == null || System.getProperty("cassandra.port") == null)