From 87c4efe819b36c9f99f9f616e076fc61cbf3316e Mon Sep 17 00:00:00 2001 From: Sylvain Lebresne Date: Wed, 25 Jun 2014 18:25:30 +0200 Subject: [PATCH 1/2] Update versions for 1.2.17 release --- NEWS.txt | 8 ++++++++ build.xml | 2 +- debian/changelog | 6 ++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/NEWS.txt b/NEWS.txt index f297634115..97d32f2381 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -13,6 +13,14 @@ restore snapshots created with the previous major version using the 'sstableloader' tool. You can upgrade the file format of your snapshots using the provided 'sstableupgrade' tool. +1.2.17 +====== + +Upgrading +--------- + - Nothing specific to this release, but please see 1.2.16 if you are upgrading + from a previous version. + 1.2.16 ====== diff --git a/build.xml b/build.xml index 5db0a6af9e..1310d6432f 100644 --- a/build.xml +++ b/build.xml @@ -25,7 +25,7 @@ - + diff --git a/debian/changelog b/debian/changelog index 50318c8a6c..adb7a37046 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +cassandra (1.2.17) unstable; urgency=medium + + * New release + + -- Sylvain Lebresne Wed, 25 Jun 2014 17:54:01 +0200 + cassandra (1.2.16) unstable; urgency=low * New release From c9cef44a1a6c10036200f410c8a26942e64c8f12 Mon Sep 17 00:00:00 2001 From: Sylvain Lebresne Date: Thu, 26 Jun 2014 10:40:22 +0200 Subject: [PATCH 2/2] Support Thrift tables clustering columns on CqlPagingInputFormat patch by pauloricardomg; reviewed by alexliu68 for CASSANDRA-7445 --- CHANGES.txt | 3 + .../hadoop/cql3/CqlPagingRecordReader.java | 16 ++--- .../cassandra/pig/ThriftColumnFamilyTest.java | 61 ++++++++++++++++--- 3 files changed, 65 insertions(+), 15 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6adef97c09..9fbcd9a243 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,6 @@ +1.2.18 + * Support Thrift tables clustering columns on CqlPagingInputFormat (CASSANDRA-7445) + 1.2.17 * cqlsh: Fix CompositeType columns in DESCRIBE TABLE output (CASSANDRA-7399) * Expose global ColmunFamily metrics (CASSANDRA-7273) diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlPagingRecordReader.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlPagingRecordReader.java index b6e793c7e4..0542f7e15c 100644 --- a/src/java/org/apache/cassandra/hadoop/cql3/CqlPagingRecordReader.java +++ b/src/java/org/apache/cassandra/hadoop/cql3/CqlPagingRecordReader.java @@ -261,7 +261,7 @@ public class CqlPagingRecordReader extends RecordReader, { value.clear(); value.putAll(getCurrentValue()); - + keys.clear(); keys.putAll(getCurrentKey()); @@ -703,7 +703,7 @@ public class CqlPagingRecordReader extends RecordReader, clusterColumns.add(new BoundColumn(key)); parseKeyValidators(ByteBufferUtil.string(ByteBuffer.wrap(cqlRow.columns.get(2).getValue()))); - + Column rawComparator = cqlRow.columns.get(3); String comparator = ByteBufferUtil.string(ByteBuffer.wrap(rawComparator.getValue())); logger.debug("comparator: {}", comparator); @@ -719,8 +719,8 @@ public class CqlPagingRecordReader extends RecordReader, } } - /** - * retrieve the fake partition keys and cluster keys for classic thrift table + /** + * retrieve the fake partition keys and cluster keys for classic thrift table * use CFDefinition to get keys and columns * */ private void retrieveKeysForThriftTables() throws Exception @@ -732,8 +732,10 @@ public class CqlPagingRecordReader extends RecordReader, { CFMetaData cfMeta = CFMetaData.fromThrift(cfDef); CFDefinition cfDefinition = new CFDefinition(cfMeta); - for (ColumnIdentifier columnIdentifier : cfDefinition.keys.keySet()) - partitionBoundColumns.add(new BoundColumn(columnIdentifier.toString())); + for (ColumnIdentifier key : cfDefinition.keys.keySet()) + partitionBoundColumns.add(new BoundColumn(key.toString())); + for (ColumnIdentifier column : cfDefinition.columns.keySet()) + clusterColumns.add(new BoundColumn(column.toString())); parseKeyValidators(cfDef.key_validation_class); return; } @@ -814,7 +816,7 @@ public class CqlPagingRecordReader extends RecordReader, this.name = name; } } - + /** get string from a ByteBuffer, catch the exception and throw it as runtime exception*/ private static String stringValue(ByteBuffer value) { diff --git a/test/unit/org/apache/cassandra/pig/ThriftColumnFamilyTest.java b/test/unit/org/apache/cassandra/pig/ThriftColumnFamilyTest.java index 223cbf4897..6f6aa0bdd5 100644 --- a/test/unit/org/apache/cassandra/pig/ThriftColumnFamilyTest.java +++ b/test/unit/org/apache/cassandra/pig/ThriftColumnFamilyTest.java @@ -47,7 +47,7 @@ import org.junit.BeforeClass; import org.junit.Test; public class ThriftColumnFamilyTest extends PigTestBase -{ +{ private static String[] statements = { "create keyspace thriftKs with placement_strategy = 'org.apache.cassandra.locator.SimpleStrategy' and" + " strategy_options={replication_factor:1};", @@ -125,7 +125,7 @@ public class ThriftColumnFamilyTest extends PigTestBase "create column family U8 with " + "key_validation_class = UTF8Type and " + "comparator = UTF8Type;", - + "create column family Bytes with " + "key_validation_class = BytesType and " + "comparator = UTF8Type;", @@ -181,7 +181,22 @@ public class ThriftColumnFamilyTest extends PigTestBase "create column family CompoKeyCopy " + "with key_validation_class = 'CompositeType(UTF8Type,LongType)' " + "and default_validation_class = UTF8Type " + - "and comparator = LongType;" + "and comparator = LongType;", + + + "create column family WideCf " + + " with comparator = UTF8Type " + + " and default_validation_class = UTF8Type " + + " and key_validation_class = UTF8Type " + + " and comparator = UTF8Type;", + + "set WideCf['2014-06-06']['1'] = 'event1';", + "set WideCf['2014-06-06']['2'] = 'event2';", + + "set WideCf['2014-06-07']['3'] = 'event3';", + "set WideCf['2014-06-07']['4'] = 'event4';", + "set WideCf['2014-06-07']['5'] = 'event5';", + "set WideCf['2014-06-07']['6'] = 'event6';", }; @BeforeClass @@ -325,6 +340,36 @@ public class ThriftColumnFamilyTest extends PigTestBase Assert.assertEquals(count, 4); } + @Test + public void testCqlStorageWithThriftWideRowCf() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException, AuthenticationException, AuthorizationException + { + //regular thrift wide row column family with page size set to 1 to cause CASSANDRA-7445 + pig.registerQuery("rows = load 'cql://thriftKs/WideCf?" + defaultParameters + "&page_size=1' using CqlStorage();"); + + /** + "set WideCf['2014-06-06']['1'] = 'event1';", + "set WideCf['2014-06-06']['2'] = 'event2';", + --------------------------------------------- + "set WideCf['2014-06-07']['3'] = 'event3';", + "set WideCf['2014-06-07']['4'] = 'event4';", + "set WideCf['2014-06-07']['5'] = 'event5';", + "set WideCf['2014-06-07']['6'] = 'event6';", + */ + + Iterator it = pig.openIterator("rows"); + for (Integer i = 1; i <= 6; i++) { + Assert.assertTrue(it.hasNext()); + Tuple t = it.next(); + if (i < 3) { + Assert.assertEquals(t.get(0).toString(), "2014-06-06"); + } else { + Assert.assertEquals(t.get(0).toString(), "2014-06-07"); + } + Assert.assertEquals(t.get(1).toString(), i.toString()); + Assert.assertEquals(t.get(2).toString(), "event" + i); + } + } + @Test public void testCassandraStorageSchema() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException { @@ -707,7 +752,7 @@ public class ThriftColumnFamilyTest extends PigTestBase Iterator it = pig.openIterator("compokeys"); if (it.hasNext()) { Tuple t = it.next(); - Tuple key = (Tuple) t.get(0); + Tuple key = (Tuple) t.get(0); Assert.assertEquals(key.get(0), "clock"); Assert.assertEquals(key.get(1), 40L); DataBag columns = (DataBag) t.get(1); @@ -735,7 +780,7 @@ public class ThriftColumnFamilyTest extends PigTestBase count ++; if (count == 1) { - Tuple key = (Tuple) t.get(0); + Tuple key = (Tuple) t.get(0); Assert.assertEquals(key.get(0), "clock"); Assert.assertEquals(key.get(1), 10L); DataBag columns = (DataBag) t.get(1); @@ -749,7 +794,7 @@ public class ThriftColumnFamilyTest extends PigTestBase } else if (count == 2) { - Tuple key = (Tuple) t.get(0); + Tuple key = (Tuple) t.get(0); Assert.assertEquals(key.get(0), "clock"); Assert.assertEquals(key.get(1), 20L); DataBag columns = (DataBag) t.get(1); @@ -763,7 +808,7 @@ public class ThriftColumnFamilyTest extends PigTestBase } else if (count == 3) { - Tuple key = (Tuple) t.get(0); + Tuple key = (Tuple) t.get(0); Assert.assertEquals(key.get(0), "clock"); Assert.assertEquals(key.get(1), 30L); DataBag columns = (DataBag) t.get(1); @@ -777,7 +822,7 @@ public class ThriftColumnFamilyTest extends PigTestBase } else if (count == 4) { - Tuple key = (Tuple) t.get(0); + Tuple key = (Tuple) t.get(0); Assert.assertEquals(key.get(0), "clock"); Assert.assertEquals(key.get(1), 40L); DataBag columns = (DataBag) t.get(1);