natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-2.1.0' into cassandra-2.1 

Conflicts:
	CHANGES.txt
	src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java
This commit is contained in:
Aleksey Yeschenko 2014-08-11 01:07:32 +03:00
parent 8aed4196d2 9268820952
commit f2f2d47324
10 changed files with 247 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGES.txt
View File

@ -19,7 +19,7 @@ Merged from 2.0:
* cqlsh fails when version number parts are not int (CASSANDRA-7524)
2.1.0-final
2.1.0-rc6
* Include stress yaml example in release and deb (CASSANDRA-7717)
* workaround for netty issue causing corrupted data off the wire (CASSANDRA-7695)
* cqlsh DESC CLUSTER fails retrieving ring information (CASSANDRA-7687)
@ -27,6 +27,8 @@ Merged from 2.0:
* Fix UDT field selection with empty fields (CASSANDRA-7670)
* Bogus deserialization of static cells from sstable (CASSANDRA-7684)
Merged from 2.0:
* Fix potential AssertionError in RangeTombstoneList (CASSANDRA-7700)
* Validate arguments of blobAs* functions (CASSANDRA-7707)
* Fix potential AssertionError with 2ndary indexes (CASSANDRA-6612)
* Avoid logging CompactionInterrupted at ERROR (CASSANDRA-7694)
* Minor leak in sstable2jon (CASSANDRA-7709)

5
NEWS.txt
View File

@ -72,13 +72,12 @@ Upgrading
2.0.10
====
======
New features
------------
- CqlPaginRecordReader and CqlPagingInputFormat have both been removed.
Use CqlInputFormat instead.
- If you are using Leveled Compaction, you can now disable doing
size-tiered
- If you are using Leveled Compaction, you can now disable doing size-tiered
compaction in L0 by starting Cassandra with -Dcassandra.disable_stcs_in_l0
(see CASSANDRA-6621 for details).
- Shuffle and taketoken have been removed. For clusters that choose to

2
build.xml
View File

@ -25,7 +25,7 @@
<property name="debuglevel" value="source,lines,vars"/>
<!-- default version and SCM information -->
<property name="base.version" value="2.1.0-rc5"/>
<property name="base.version" value="2.1.0-rc6"/>
<property name="scm.connection" value="scm:git://git.apache.org/cassandra.git"/>
<property name="scm.developerConnection" value="scm:git://git.apache.org/cassandra.git"/>
<property name="scm.url" value="http://git-wip-us.apache.org/repos/asf?p=cassandra.git;a=tree"/>

6
debian/changelog vendored
View File

@ -1,3 +1,9 @@
cassandra (2.1.0~rc6) unstable; urgency=medium
* New RC release
-- Sylvain Lebresne <slebresne@apache.org> Sat, 09 Aug 2014 13:46:39 +0200
cassandra (2.1.0~rc5) unstable; urgency=medium
* New RC release

22
src/java/org/apache/cassandra/cql3/functions/BytesConversionFcts.java
View File

@ -23,6 +23,9 @@ import java.util.List;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.serializers.MarshalException;
public abstract class BytesConversionFcts
{
@ -40,14 +43,25 @@ public abstract class BytesConversionFcts
};
}
public static Function makeFromBlobFunction(AbstractType<?> toType)
public static Function makeFromBlobFunction(final AbstractType<?> toType)
{
String name = "blobas" + toType.asCQL3Type();
final String name = "blobas" + toType.asCQL3Type();
return new AbstractFunction(name, toType, BytesType.instance)
{
public ByteBuffer execute(List<ByteBuffer> parameters)
public ByteBuffer execute(List<ByteBuffer> parameters) throws InvalidRequestException
{
return parameters.get(0);
ByteBuffer val = parameters.get(0);
try
{
if (val != null)
toType.validate(val);
return val;
}
catch (MarshalException e)
{
throw new InvalidRequestException(String.format("In call to function %s, value 0x%s is not a valid binary representation for type %s",
name, ByteBufferUtil.bytesToHex(val), toType.asCQL3Type()));
}
}
};
}

23
src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
View File

@ -28,6 +28,8 @@ import org.apache.cassandra.db.marshal.ListType;
import org.apache.cassandra.db.marshal.MapType;
import org.apache.cassandra.db.marshal.SetType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.serializers.MarshalException;
public class FunctionCall extends Term.NonTerminal
{
@ -63,8 +65,24 @@ public class FunctionCall extends Term.NonTerminal
throw new InvalidRequestException(String.format("Invalid null value for argument to %s", fun));
buffers.add(val);
}
return executeInternal(fun, buffers);
}
return fun.execute(buffers);
private static ByteBuffer executeInternal(Function fun, List<ByteBuffer> params) throws InvalidRequestException
{
ByteBuffer result = fun.execute(params);
try
{
// Check the method didn't lied on it's declared return type
if (result != null)
fun.returnType().validate(result);
return result;
}
catch (MarshalException e)
{
throw new RuntimeException(String.format("Return of function %s (%s) is not a valid value for its declared return type %s",
fun, ByteBufferUtil.bytesToHex(result), fun.returnType().asCQL3Type()));
}
}
public boolean containsBindMarker()
@ -132,7 +150,8 @@ public class FunctionCall extends Term.NonTerminal
assert t instanceof Term.Terminal;
buffers.add(((Term.Terminal)t).get(QueryOptions.DEFAULT));
}
return fun.execute(buffers);
return executeInternal(fun, buffers);
}
public boolean isAssignableTo(String keyspace, ColumnSpecification receiver)

79
src/java/org/apache/cassandra/db/RangeTombstoneList.java
View File

@ -168,7 +168,7 @@ public class RangeTombstoneList implements Iterable<RangeTombstone>, IMeasurable
{
// Note: insertFrom expect i to be the insertion point in term of interval ends
int pos = Arrays.binarySearch(ends, 0, size, start, comparator);
insertFrom((pos >= 0 ? pos+1 : -pos-1), start, end, markedAt, delTime);
insertFrom((pos >= 0 ? pos : -pos-1), start, end, markedAt, delTime);
}
boundaryHeapSize += start.unsharedHeapSize() + end.unsharedHeapSize();
}
@ -216,7 +216,7 @@ public class RangeTombstoneList implements Iterable<RangeTombstone>, IMeasurable
int j = 0;
while (i < size && j < tombstones.size)
{
if (comparator.compare(tombstones.starts[j], ends[i]) < 0)
if (comparator.compare(tombstones.starts[j], ends[i]) <= 0)
{
insertFrom(i, tombstones.starts[j], tombstones.ends[j], tombstones.markedAts[j], tombstones.delTimes[j]);
j++;
@ -517,16 +517,52 @@ public class RangeTombstoneList implements Iterable<RangeTombstone>, IMeasurable
}
/*
* Inserts a new element starting at index i. This method assumes that i is the insertion point
* in term of intervals for start:
* ends[i-1] <= start < ends[i]
* Inserts a new element starting at index i. This method assumes that:
* ends[i-1] <= start <= ends[i]
*
* A RangeTombstoneList is a list of range [s_0, e_0]...[s_n, e_n] such that:
* - s_i <= e_i
* - e_i <= s_i+1
* - if s_i == e_i and e_i == s_i+1 then s_i+1 < e_i+1
* Basically, range are non overlapping except for their bound and in order. And while
* we allow ranges with the same value for the start and end, we don't allow repeating
* such range (so we can't have [0, 0][0, 0] even though it would respect the first 2
* conditions).
*
*/
private void insertFrom(int i, Composite start, Composite end, long markedAt, int delTime)
{
while (i < size)
{
assert i == 0 || comparator.compare(start, ends[i-1]) >= 0;
assert i >= size || comparator.compare(start, ends[i]) < 0;
assert i == 0 || comparator.compare(ends[i-1], start) <= 0;
int c = comparator.compare(start, ends[i]);
assert c <= 0;
if (c == 0)
{
// If start == ends[i], then we can insert from the next one (basically the new element
// really start at the next element), except for the case where starts[i] == ends[i].
// In this latter case, if we were to move to next element, we could end up with ...[x, x][x, x]...
if (comparator.compare(starts[i], ends[i]) == 0)
{
// The current element cover a single value which is equal to the start of the inserted
// element. If the inserted element overwrites the current one, just remove the current
// (it's included in what we insert) and proceed with the insert.
if (markedAt > markedAts[i])
{
removeInternal(i);
continue;
}
// Otherwise (the current singleton interval override the new one), we want to leave the
// current element and move to the next, unless start == end since that means the new element
// is in fact fully covered by the current one (so we're done)
if (comparator.compare(start, end) == 0)
return;
}
i++;
continue;
}
// Do we overwrite the current element?
if (markedAt > markedAts[i])
@ -544,11 +580,18 @@ public class RangeTombstoneList implements Iterable<RangeTombstone>, IMeasurable
// now, start <= starts[i]
// If the new element stops before the current one, insert it and
// we're done
if (comparator.compare(end, starts[i]) <= 0)
// Does the new element stops before/at the current one,
int endCmp = comparator.compare(end, starts[i]);
if (endCmp <= 0)
{
addInternal(i, start, end, markedAt, delTime);
// Here start <= starts[i] and end <= starts[i]
// This means the current element is before the current one. However, one special
// case is if end == starts[i] and starts[i] == ends[i]. In that case,
// the new element entirely overwrite the current one and we can just overwrite
if (endCmp == 0 && comparator.compare(starts[i], ends[i]) == 0)
setInternal(i, start, end, markedAt, delTime);
else
addInternal(i, start, end, markedAt, delTime);
return;
}
@ -640,6 +683,20 @@ public class RangeTombstoneList implements Iterable<RangeTombstone>, IMeasurable
size++;
}
private void removeInternal(int i)
{
assert i >= 0;
System.arraycopy(starts, i+1, starts, i, size - i - 1);
System.arraycopy(ends, i+1, ends, i, size - i - 1);
System.arraycopy(markedAts, i+1, markedAts, i, size - i - 1);
System.arraycopy(delTimes, i+1, delTimes, i, size - i - 1);
--size;
starts[size] = null;
ends[size] = null;
}
/*
* Grow the arrays, leaving index i "free" in the process.
*/

17
src/java/org/apache/cassandra/hadoop/pig/CqlNativeStorage.java
View File

@ -1,3 +1,20 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.hadoop.pig;
import java.io.IOException;

49
test/system/test_thrift_server.py
View File

@ -24,7 +24,6 @@ from . import thrift_client as client
from thrift.Thrift import TApplicationException
from ttypes import *
from constants import VERSION
def _i64(n):
@ -631,7 +630,7 @@ class TestMutations(ThriftTester):
for column in columns:
client.insert('key', ColumnParent('Standard1'), column, ConsistencyLevel.ONE)
d = Deletion(1, predicate=SlicePredicate(slice_range=SliceRange(start='c2', finish='c5')))
d = Deletion(1, predicate=SlicePredicate(slice_range=SliceRange(start='c2', finish='c4')))
client.batch_mutate({'key': {'Standard1' : [Mutation(deletion=d)]}}, ConsistencyLevel.ONE)
_assert_columnpath_exists('key', ColumnPath('Standard1', column='c1'))
@ -678,7 +677,7 @@ class TestMutations(ThriftTester):
for column in columns:
client.insert('key', ColumnParent('Super1', 'sc1'), column, ConsistencyLevel.ONE)
r = SliceRange(start=_i64(2), finish=_i64(5))
r = SliceRange(start=_i64(2), finish=_i64(4))
d = Deletion(1, super_column='sc1', predicate=SlicePredicate(slice_range=r))
client.batch_mutate({'key': {'Super1' : [Mutation(deletion=d)]}}, ConsistencyLevel.ONE)
@ -1304,7 +1303,7 @@ class TestMutations(ThriftTester):
def test_describe_keyspace(self):
kspaces = client.describe_keyspaces()
assert len(kspaces) == 5, kspaces # ['Keyspace2', 'Keyspace1', 'system', 'system_traces', 'system_auth']
assert len(kspaces) == 4, kspaces # ['Keyspace2', 'Keyspace1', 'system', 'system_traces']
sysks = client.describe_keyspace("system")
assert sysks in kspaces
@ -1318,8 +1317,6 @@ class TestMutations(ThriftTester):
assert cf0.comparator_type == "org.apache.cassandra.db.marshal.BytesType"
def test_describe(self):
server_version = client.describe_version()
assert server_version == VERSION, (server_version, VERSION)
assert client.describe_cluster_name() == 'Test Cluster'
def test_describe_ring(self):
@ -1767,13 +1764,13 @@ class TestMutations(ThriftTester):
_assert_no_columnpath('key1', ColumnPath(column_family='Counter1', column='c1'))
# insert again and this time delete the whole row, check that it is gone
client.add('key1', ColumnParent(column_family='Counter1'), CounterColumn('c1', d1), ConsistencyLevel.ONE)
client.add('key2', ColumnParent(column_family='Counter1'), CounterColumn('c1', d1), ConsistencyLevel.ONE)
time.sleep(5)
rv2 = client.get('key1', ColumnPath(column_family='Counter1', column='c1'), ConsistencyLevel.ONE)
rv2 = client.get('key2', ColumnPath(column_family='Counter1', column='c1'), ConsistencyLevel.ONE)
assert rv2.counter_column.value == d1
client.remove_counter('key1', ColumnPath(column_family='Counter1'), ConsistencyLevel.ONE)
client.remove_counter('key2', ColumnPath(column_family='Counter1'), ConsistencyLevel.ONE)
time.sleep(5)
_assert_no_columnpath('key1', ColumnPath(column_family='Counter1', column='c1'))
_assert_no_columnpath('key2', ColumnPath(column_family='Counter1', column='c1'))
def test_incr_super_remove(self):
_set_keyspace('Keyspace1')
@ -1792,13 +1789,13 @@ class TestMutations(ThriftTester):
_assert_no_columnpath('key1', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'))
# insert again and this time delete the whole row, check that it is gone
client.add('key1', ColumnParent(column_family='SuperCounter1', super_column='sc1'), CounterColumn('c1', d1), ConsistencyLevel.ONE)
client.add('key2', ColumnParent(column_family='SuperCounter1', super_column='sc1'), CounterColumn('c1', d1), ConsistencyLevel.ONE)
time.sleep(5)
rv2 = client.get('key1', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'), ConsistencyLevel.ONE)
rv2 = client.get('key2', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'), ConsistencyLevel.ONE)
assert rv2.counter_column.value == d1
client.remove_counter('key1', ColumnPath(column_family='SuperCounter1', super_column='sc1'), ConsistencyLevel.ONE)
client.remove_counter('key2', ColumnPath(column_family='SuperCounter1', super_column='sc1'), ConsistencyLevel.ONE)
time.sleep(5)
_assert_no_columnpath('key1', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'))
_assert_no_columnpath('key2', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'))
def test_incr_decr_standard_remove(self):
_set_keyspace('Keyspace1')
@ -1817,13 +1814,13 @@ class TestMutations(ThriftTester):
_assert_no_columnpath('key1', ColumnPath(column_family='Counter1', column='c1'))
# insert again and this time delete the whole row, check that it is gone
client.add('key1', ColumnParent(column_family='Counter1'), CounterColumn('c1', d1), ConsistencyLevel.ONE)
client.add('key2', ColumnParent(column_family='Counter1'), CounterColumn('c1', d1), ConsistencyLevel.ONE)
time.sleep(5)
rv2 = client.get('key1', ColumnPath(column_family='Counter1', column='c1'), ConsistencyLevel.ONE)
rv2 = client.get('key2', ColumnPath(column_family='Counter1', column='c1'), ConsistencyLevel.ONE)
assert rv2.counter_column.value == d1
client.remove_counter('key1', ColumnPath(column_family='Counter1'), ConsistencyLevel.ONE)
client.remove_counter('key2', ColumnPath(column_family='Counter1'), ConsistencyLevel.ONE)
time.sleep(5)
_assert_no_columnpath('key1', ColumnPath(column_family='Counter1', column='c1'))
_assert_no_columnpath('key2', ColumnPath(column_family='Counter1', column='c1'))
def test_incr_decr_super_remove(self):
_set_keyspace('Keyspace1')
@ -1842,13 +1839,13 @@ class TestMutations(ThriftTester):
_assert_no_columnpath('key1', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'))
# insert again and this time delete the whole row, check that it is gone
client.add('key1', ColumnParent(column_family='SuperCounter1', super_column='sc1'), CounterColumn('c1', d1), ConsistencyLevel.ONE)
client.add('key2', ColumnParent(column_family='SuperCounter1', super_column='sc1'), CounterColumn('c1', d1), ConsistencyLevel.ONE)
time.sleep(5)
rv2 = client.get('key1', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'), ConsistencyLevel.ONE)
rv2 = client.get('key2', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'), ConsistencyLevel.ONE)
assert rv2.counter_column.value == d1
client.remove_counter('key1', ColumnPath(column_family='SuperCounter1', super_column='sc1'), ConsistencyLevel.ONE)
client.remove_counter('key2', ColumnPath(column_family='SuperCounter1', super_column='sc1'), ConsistencyLevel.ONE)
time.sleep(5)
_assert_no_columnpath('key1', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'))
_assert_no_columnpath('key2', ColumnPath(column_family='SuperCounter1', super_column='sc1', column='c1'))
def test_incr_decr_standard_batch_add(self):
_set_keyspace('Keyspace1')
@ -1891,21 +1888,21 @@ class TestMutations(ThriftTester):
_assert_no_columnpath('key1', ColumnPath(column_family='Counter1', column='c1'))
# insert again and this time delete the whole row, check that it is gone
update_map = {'key1': {'Counter1': [
update_map = {'key2': {'Counter1': [
Mutation(column_or_supercolumn=ColumnOrSuperColumn(counter_column=CounterColumn('c1', d1))),
Mutation(column_or_supercolumn=ColumnOrSuperColumn(counter_column=CounterColumn('c1', d2))),
]}}
client.batch_mutate(update_map, ConsistencyLevel.ONE)
time.sleep(5)
rv2 = client.get('key1', ColumnPath(column_family='Counter1', column='c1'), ConsistencyLevel.ONE)
rv2 = client.get('key2', ColumnPath(column_family='Counter1', column='c1'), ConsistencyLevel.ONE)
assert rv2.counter_column.value == d1+d2
update_map = {'key1': {'Counter1': [
update_map = {'key2': {'Counter1': [
Mutation(deletion=Deletion()),
]}}
client.batch_mutate(update_map, ConsistencyLevel.ONE)
time.sleep(5)
_assert_no_columnpath('key1', ColumnPath(column_family='Counter1', column='c1'))
_assert_no_columnpath('key2', ColumnPath(column_family='Counter1', column='c1'))
def test_incr_decr_standard_slice(self):
_set_keyspace('Keyspace1')

88
test/unit/org/apache/cassandra/db/RangeTombstoneListTest.java
View File

@ -31,6 +31,7 @@ import org.apache.cassandra.utils.ByteBufferUtil;
public class RangeTombstoneListTest
{
private static final Comparator<Composite> cmp = new SimpleDenseCellNameType(IntegerType.instance);
private static final Random rand = new Random();
@Test
public void sortedAdditionTest()
@ -290,16 +291,103 @@ public class RangeTombstoneListTest
assertEquals(6, l.maxMarkedAt());
}
private RangeTombstoneList makeRandom(int size, int maxItSize, int maxItDistance, int maxMarkedAt)
{
RangeTombstoneList l = new RangeTombstoneList(cmp, size);
int prevStart = -1;
int prevEnd = 0;
for (int i = 0; i < size; i++)
{
int nextStart = prevEnd + rand.nextInt(maxItDistance);
int nextEnd = nextStart + rand.nextInt(maxItSize);
// We can have an interval [x, x], but not 2 consecutives ones for the same x
if (nextEnd == nextStart && prevEnd == prevStart && prevEnd == nextStart)
nextEnd += 1 + rand.nextInt(maxItDistance);
l.add(rt(nextStart, nextEnd, rand.nextInt(maxMarkedAt)));
prevStart = nextStart;
prevEnd = nextEnd;
}
return l;
}
@Test
public void addAllRandomTest() throws Throwable
{
int TEST_COUNT = 1000;
int MAX_LIST_SIZE = 50;
int MAX_IT_SIZE = 20;
int MAX_IT_DISTANCE = 10;
int MAX_MARKEDAT = 10;
for (int i = 0; i < TEST_COUNT; i++)
{
RangeTombstoneList l1 = makeRandom(rand.nextInt(MAX_LIST_SIZE) + 1, rand.nextInt(MAX_IT_SIZE) + 1, rand.nextInt(MAX_IT_DISTANCE) + 1, rand.nextInt(MAX_MARKEDAT) + 1);
RangeTombstoneList l2 = makeRandom(rand.nextInt(MAX_LIST_SIZE) + 1, rand.nextInt(MAX_IT_SIZE) + 1, rand.nextInt(MAX_IT_DISTANCE) + 1, rand.nextInt(MAX_MARKEDAT) + 1);
RangeTombstoneList l1Initial = l1.copy();
try
{
// We generate the list randomly, so "all" we check is that the resulting range tombstone list looks valid.
l1.addAll(l2);
assertValid(l1);
}
catch (Throwable e)
{
System.out.println("Error merging:");
System.out.println(" l1: " + toString(l1Initial));
System.out.println(" l2: " + toString(l2));
throw e;
}
}
}
private static void assertRT(RangeTombstone expected, RangeTombstone actual)
{
assertEquals(String.format("Expected %s but got %s", toString(expected), toString(actual)), expected, actual);
}
private static void assertValid(RangeTombstoneList l)
{
// We check that ranges are in the right order and that we never have something
// like ...[x, x][x, x] ...
int prevStart = -2;
int prevEnd = -1;
for (RangeTombstone rt : l)
{
int curStart = i(rt.min);
int curEnd = i(rt.max);
assertTrue("Invalid " + toString(l), prevEnd <= curStart);
assertTrue("Invalid " + toString(l), curStart <= curEnd);
if (curStart == curEnd && prevEnd == curStart)
assertTrue("Invalid " + toString(l), prevStart != prevEnd);
prevStart = curStart;
prevEnd = curEnd;
}
}
private static String toString(RangeTombstone rt)
{
return String.format("[%d, %d]@%d", i(rt.min), i(rt.max), rt.data.markedForDeleteAt);
}
private static String toString(RangeTombstoneList l)
{
StringBuilder sb = new StringBuilder();
sb.append("{");
for (RangeTombstone rt : l)
sb.append(" ").append(toString(rt));
return sb.append(" }").toString();
}
private static Composite b(int i)
{
return Util.cellname(i);