From 3a044a9140abd8f77c935d8551ee6a62ecbdc934 Mon Sep 17 00:00:00 2001 From: Jonathan Ellis Date: Tue, 27 Jul 2010 17:46:28 +0000 Subject: [PATCH] add IntegerType for arbitrary-length integers. patch by Folke Behrens; reviewed by mdennis and jbellis for CASSANDRA-1282 git-svn-id: https://svn.apache.org/repos/asf/cassandra/trunk@979784 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + NEWS.txt | 2 + conf/cassandra.yaml | 3 +- .../cassandra/db/marshal/IntegerType.java | 121 +++++++++++ test/conf/cassandra.yaml | 3 + test/system/__init__.py | 1 + test/system/test_thrift_server.py | 35 +++- .../cassandra/db/marshal/IntegerTypeTest.java | 193 ++++++++++++++++++ 8 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 src/java/org/apache/cassandra/db/marshal/IntegerType.java create mode 100644 test/unit/org/apache/cassandra/db/marshal/IntegerTypeTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 8f18552898..d170719700 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -41,6 +41,7 @@ dev * take advantage of row cache during range queries (CASSANDRA-1302) * make GCGraceSeconds a per-ColumnFamily value (CASSANDRA-1276) * keep persistent row size and column count statistics (CASSANDRA-1155) + * add IntegerType (CASSANDRA-1282) 0.6.4 diff --git a/NEWS.txt b/NEWS.txt index e15b0c5626..ffe6baacb9 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -22,6 +22,8 @@ Features clusters - dynamic endpoint snitch mitigates the impact of impaired nodes - significantly faster reads from row cache + - introduced IntegerType that is both faster than LongType and + allows integers of both less and more bits than Long's 64 Configuraton ------------ diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 6f5f57b65e..535032dd1c 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -213,7 +213,8 @@ request_scheduler_id: keyspace # - compare_with: tells Cassandra how to sort the columns for slicing # operations. The default is BytesType, which is a straightforward # lexical comparison of the bytes in each column. Other options are -# AsciiType, UTF8Type, LexicalUUIDType, TimeUUIDType, and LongType. +# AsciiType, UTF8Type, LexicalUUIDType, TimeUUIDType, LongType, +# and IntegerType (a generic variable-length integer type). # You can also specify the fully-qualified class name to a class of # your choice extending org.apache.cassandra.db.marshal.AbstractType. # diff --git a/src/java/org/apache/cassandra/db/marshal/IntegerType.java b/src/java/org/apache/cassandra/db/marshal/IntegerType.java new file mode 100644 index 0000000000..e5c359150b --- /dev/null +++ b/src/java/org/apache/cassandra/db/marshal/IntegerType.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cassandra.db.marshal; + +public final class IntegerType extends AbstractType +{ + public static final IntegerType instance = new IntegerType(); + + private static int findMostSignificantByte(byte[] bytes) + { + int len = bytes.length - 1; + int i = 0; + for (; i < len; i++) + { + byte b0 = bytes[i]; + if (b0 != 0 && b0 != -1) + break; + byte b1 = bytes[i + 1]; + if (b0 == 0 && b1 != 0) + { + if (b1 > 0) + i++; + break; + } + if (b0 == -1 && b1 != -1) + { + if (b1 < 0) + i++; + break; + } + } + return i; + } + + IntegerType() {/* singleton */} + + public int compare(byte[] lhs, byte[] rhs) + { + int lhsLen = lhs.length; + int rhsLen = rhs.length; + + if (lhsLen == 0) + return rhsLen == 0 ? 0 : -1; + if (rhsLen == 0) + return 1; + + int lhsMsbIdx = findMostSignificantByte(lhs); + int rhsMsbIdx = findMostSignificantByte(rhs); + + //diffs contain number of "meaningful" bytes (i.e. ignore padding) + int lhsLenDiff = lhsLen - lhsMsbIdx; + int rhsLenDiff = rhsLen - rhsMsbIdx; + + byte lhsMsb = lhs[lhsMsbIdx]; + byte rhsMsb = rhs[rhsMsbIdx]; + + /* + - + * ----------- + * + | -d | 1 | + * LHS ----------- + * - | -1 | d | + * ----------- + * RHS + * + * d = difference of length in significant bytes + */ + if (lhsLenDiff != rhsLenDiff) + { + if (lhsMsb < 0) + return rhsMsb < 0 ? rhsLenDiff - lhsLenDiff : -1; + if (rhsMsb < 0) + return 1; + return lhsLenDiff - rhsLenDiff; + } + + // msb uses signed comparison + if (lhsMsb != rhsMsb) + return lhsMsb - rhsMsb; + lhsMsbIdx++; + rhsMsbIdx++; + + // remaining bytes are compared unsigned + while (lhsMsbIdx < lhsLen) + { + lhsMsb = lhs[lhsMsbIdx++]; + rhsMsb = rhs[rhsMsbIdx++]; + if (lhsMsb != rhsMsb) + return (lhsMsb & 0xFF) - (rhsMsb & 0xFF); + } + + return 0; + } + + @Override + public String getString(byte[] bytes) + { + if (bytes == null) + return "null"; + if (bytes.length == 0) + return "empty"; + + return new java.math.BigInteger(bytes).toString(10); + } +} diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml index 0356a8f138..ae6600f57f 100644 --- a/test/conf/cassandra.yaml +++ b/test/conf/cassandra.yaml @@ -49,6 +49,9 @@ keyspaces: - name: StandardLong2 compare_with: LongType + - name: StandardInteger1 + compare_with: IntegerType + - name: Super1 column_type: Super compare_subcolumns_with: LongType diff --git a/test/system/__init__.py b/test/system/__init__.py index ce55d76896..251e235060 100644 --- a/test/system/__init__.py +++ b/test/system/__init__.py @@ -148,6 +148,7 @@ class ThriftTester(BaseTester): Cassandra.CfDef('Keyspace1', 'Standard2'), Cassandra.CfDef('Keyspace1', 'StandardLong1', comparator_type='LongType'), Cassandra.CfDef('Keyspace1', 'StandardLong2', comparator_type='LongType'), + Cassandra.CfDef('Keyspace1', 'StandardInteger1', comparator_type='IntegerType'), Cassandra.CfDef('Keyspace1', 'Super1', column_type='Super', subcomparator_type='LongType', row_cache_size=1000, key_cache_size=0), Cassandra.CfDef('Keyspace1', 'Super2', column_type='Super', subcomparator_type='LongType'), Cassandra.CfDef('Keyspace1', 'Super3', column_type='Super', subcomparator_type='LongType'), diff --git a/test/system/test_thrift_server.py b/test/system/test_thrift_server.py index 99a8b201de..770c9664ba 100644 --- a/test/system/test_thrift_server.py +++ b/test/system/test_thrift_server.py @@ -278,6 +278,22 @@ class TestMutations(ThriftTester): slice = [result.column.name for result in _big_slice('key1', ColumnParent('StandardLong1'))] assert slice == L, slice + def test_integer_order(self): + _set_keyspace('Keyspace1') + def long_xrange(start, stop, step): + i = start + while i >= stop: + yield i + i -= step + L = [] + for i in long_xrange(104294967296, 0, 429496729): + name = _i64(i) + client.insert('key1', ColumnParent('StandardInteger1'), Column(name, 'v', Clock(0)), ConsistencyLevel.ONE) + L.append(name) + slice = [result.column.name for result in _big_slice('key1', ColumnParent('StandardInteger1'))] + L.sort() + assert slice == L, slice + def test_time_uuid(self): import uuid L = [] @@ -338,6 +354,23 @@ class TestMutations(ThriftTester): for result in client.get_slice('key1', column_parent, sp, ConsistencyLevel.ONE)] assert slice == [Column(_i64(i), 'value2', Clock(10 * i + 2))], (slice, i) + def test_integer_remove(self): + column_parent = ColumnParent('StandardInteger1') + sp = SlicePredicate(slice_range=SliceRange('', '', False, 1)) + _set_keyspace('Keyspace1') + for i in xrange(10): + parent = ColumnParent('StandardInteger1') + + client.insert('key1', parent, Column(_i64(i), 'value1', Clock(10 * i)), ConsistencyLevel.ONE) + client.remove('key1', ColumnPath('StandardInteger1'), Clock(10 * i + 1), ConsistencyLevel.ONE) + slice = client.get_slice('key1', column_parent, sp, ConsistencyLevel.ONE) + assert slice == [], slice + # resurrect + client.insert('key1', parent, Column(_i64(i), 'value2', Clock(10 * i + 2)), ConsistencyLevel.ONE) + slice = [result.column + for result in client.get_slice('key1', column_parent, sp, ConsistencyLevel.ONE)] + assert slice == [Column(_i64(i), 'value2', Clock(10 * i + 2))], (slice, i) + def test_batch_insert(self): _set_keyspace('Keyspace1') _insert_batch(False) @@ -1077,7 +1110,7 @@ class TestMutations(ThriftTester): kspaces = client.describe_keyspaces() assert len(kspaces) == 5, kspaces # ['system', 'Keyspace2', 'Keyspace3', 'Keyspace1', 'Keyspace4'] ks1 = client.describe_keyspace("Keyspace1") - assert set(ks1.keys()) == set(['Super1', 'Standard1', 'Standard2', 'StandardLong1', 'StandardLong2', 'Super3', 'Super2', 'Super4', 'Indexed1']) + assert set(ks1.keys()) == set(['Super1', 'Standard1', 'Standard2', 'StandardLong1', 'StandardLong2', 'StandardInteger1', 'Super3', 'Super2', 'Super4', 'Indexed1']) sysks = client.describe_keyspace("system") def test_describe(self): diff --git a/test/unit/org/apache/cassandra/db/marshal/IntegerTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/IntegerTypeTest.java new file mode 100644 index 0000000000..ce06addf8b --- /dev/null +++ b/test/unit/org/apache/cassandra/db/marshal/IntegerTypeTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cassandra.db.marshal; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Random; + +import org.junit.ComparisonFailure; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class IntegerTypeTest +{ + private static void assertSignum(String message, int expected, double value) + { + int signum = (int)Math.signum(value); + if (signum != expected) + throw new ComparisonFailure(message, Integer.toString(expected), Integer.toString(signum)); + } + + private final IntegerType comparator = IntegerType.instance; + + @Test + public void testTrimming() + { + byte[] n1, n2; + n1 = new byte[] {0}; + n2 = new byte[] {0, 0, 0, 0}; + assertEquals(0, comparator.compare(n1, n2)); + n1 = new byte[] {1, 0, 0, 1}; + n2 = new byte[] {0, 0, 0, 1, 0, 0, 1}; + assertEquals(0, comparator.compare(n1, n2)); + n1 = new byte[] {-1, 0, 0, -1 }; + n2 = new byte[] {-1, -1, -1, -1, 0, 0, -1}; + assertEquals(0, comparator.compare(n1, n2)); + n1 = new byte[] {-1, 0}; + n2 = new byte[] {0, -1, 0}; + assertSignum("", -1, comparator.compare(n1, n2)); + n1 = new byte[] {1, 0}; + n2 = new byte[] {0, -1, 0}; + assertSignum("", -1, comparator.compare(n1, n2)); + } + + @Test(expected = NullPointerException.class) + public void testNullLeft() + { + comparator.compare(null, new byte[1]); + } + + @Test(expected = NullPointerException.class) + public void testNullRight() + { + comparator.compare(new byte[1], null); + } + + @Test(expected = NullPointerException.class) + public void testNullBoth() + { + comparator.compare(null, null); + } + + @Test + public void testZeroLengthArray() + { + assertSignum("0-1", -1, comparator.compare(new byte[0], new byte[1])); + assertSignum("1-0", 1, comparator.compare(new byte[1], new byte[0])); + assertSignum("0-0", 0, comparator.compare(new byte[0], new byte[0])); + } + + @Test + public void testSanity() + { + byte[] nN = new byte[] {-1}; + byte[] nZ = new byte[] {0}; + byte[] nP = new byte[] {1}; + assertSignum("ZN", 1, comparator.compare(nZ, nN)); + assertSignum("NZ", -1, comparator.compare(nN, nZ)); + assertSignum("ZP", -1, comparator.compare(nZ, nP)); + assertSignum("PZ", 1, comparator.compare(nP, nZ)); + assertSignum("PN", 1, comparator.compare(nP, nN)); + assertSignum("NP", -1, comparator.compare(nN, nP)); + } + + @Test + public void testSameLength() + { + byte[] n1 = new byte[] {-2, 2, -4, -5}; + byte[] n2 = new byte[] {-2, 3, -5, -4}; + byte[] p1 = new byte[] {2, 3, -4, -5}; + byte[] p2 = new byte[] {2, -2, -5, -4}; + + assertSignum("n1n2", -1, comparator.compare(n1, n2)); + assertSignum("n2n1", 1, comparator.compare(n2, n1)); + + assertSignum("p1p2", -1, comparator.compare(p1, p2)); + assertSignum("p2p1", 1, comparator.compare(p2, p1)); + + assertSignum("p1n1", 1, comparator.compare(p1, n1)); + assertSignum("p1n2", 1, comparator.compare(p1, n2)); + assertSignum("n1p1", -1, comparator.compare(n1, p1)); + assertSignum("n2p1", -1, comparator.compare(n2, p1)); + } + + @Test + public void testCommonPrefix() + { + byte[][] data = { + {1, 0, 0, 1}, + {1, 0, 0, 1, 0}, + {1, 0, 0, 1}, + {1, 0, 0, 1, 0}, + {-1, 0, 0, 1}, + {-1, 0, 0, 1, 0}, + {-1, 0, 0, 1}, + {-1, 0, 0, 1, 0} + }; + + Arrays.sort(data, comparator); + assertArrayEquals(new byte[]{-1, 0, 0, 1, 0}, data[0]); + assertArrayEquals(new byte[]{-1, 0, 0, 1, 0}, data[1]); + assertArrayEquals(new byte[]{-1, 0, 0, 1}, data[2]); + assertArrayEquals(new byte[]{-1, 0, 0, 1}, data[3]); + assertArrayEquals(new byte[]{1, 0, 0, 1}, data[4]); + assertArrayEquals(new byte[]{1, 0, 0, 1}, data[5]); + assertArrayEquals(new byte[]{1, 0, 0, 1, 0}, data[6]); + assertArrayEquals(new byte[]{1, 0, 0, 1, 0}, data[7]); + } + + @Test + public void testSorting() + { + byte[][] data = { + { 1, 0, 0, 0}, + {-2, 0, 0}, + { 3, 0}, + {-4}, + { 4}, + {-3, 0}, + { 2, 0, 0}, + {-1, 0, 0, 0} + }; + + Arrays.sort(data, comparator); + assertArrayEquals("-1", new byte[] {-1, 0, 0, 0}, data[0]); + assertArrayEquals("-2", new byte[] {-2, 0, 0}, data[1]); + assertArrayEquals("-3", new byte[] {-3, 0}, data[2]); + assertArrayEquals("-4", new byte[] {-4}, data[3]); + assertArrayEquals(" 4", new byte[] { 4}, data[4]); + assertArrayEquals(" 3", new byte[] { 3, 0}, data[5]); + assertArrayEquals(" 2", new byte[] { 2, 0, 0}, data[6]); + assertArrayEquals(" 1", new byte[] { 1, 0, 0, 0}, data[7]); + } + + @Test + public void testSortingSpecialExtendedVersion() + { + Random rng = new Random(-9078270684023566599L); + + byte[][] data = new byte[10000][]; + for (int i = 0; i < data.length; i++) + { + data[i] = new byte[rng.nextInt(32) + 1]; + rng.nextBytes(data[i]); + } + + Arrays.sort(data, comparator); + + for (int i = 1; i < data.length; i++) + { + BigInteger i0 = new BigInteger(data[i - 1]); + BigInteger i1 = new BigInteger(data[i]); + assertTrue("#" + i, i0.compareTo(i1) <= 0); + } + } +}