natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-3.1' into trunk

This commit is contained in:
Benjamin Lerer 2015-12-03 22:57:09 +01:00
parent 4ca7a8cfdc 2a33c3b192
commit c19aa154ae
6 changed files with 266 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -14,6 +14,7 @@
3.1
Merged from 3.0:
* Remove 64k limit on collection elements (CASSANDRA-10374)
* Remove unclear Indexer.indexes() method (CASSANDRA-10690)
* Fix NPE on stream read error (CASSANDRA-10771)
* Normalize cqlsh DESC output (CASSANDRA-10431)

13
src/java/org/apache/cassandra/cql3/Lists.java
View File

@ -36,7 +36,6 @@ import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.Server;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
/**
@ -211,12 +210,6 @@ public abstract class Lists
if (bytes == ByteBufferUtil.UNSET_BYTE_BUFFER)
return UNSET_VALUE;
// We don't support value > 64K because the serialization format encode the length as an unsigned short.
if (bytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
throw new InvalidRequestException(String.format("List value is too long. List values are limited to %d bytes but %d bytes value provided",
FBUtilities.MAX_UNSIGNED_SHORT,
bytes.remaining()));
buffers.add(bytes);
}
return new Value(buffers);
@ -370,12 +363,6 @@ public abstract class Lists
}
else if (value != ByteBufferUtil.UNSET_BYTE_BUFFER)
{
// We don't support value > 64K because the serialization format encode the length as an unsigned short.
if (value.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
throw new InvalidRequestException(String.format("List value is too long. List values are limited to %d bytes but %d bytes value provided",
FBUtilities.MAX_UNSIGNED_SHORT,
value.remaining()));
params.addCell(column, elementPath, value);
}
}

17
src/java/org/apache/cassandra/cql3/Maps.java
View File

@ -35,7 +35,6 @@ import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.Server;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
/**
@ -225,14 +224,11 @@ public abstract class Maps
{
// We don't support values > 64K because the serialization format encode the length as an unsigned short.
ByteBuffer keyBytes = entry.getKey().bindAndGet(options);
if (keyBytes == null)
throw new InvalidRequestException("null is not supported inside collections");
if (keyBytes == ByteBufferUtil.UNSET_BYTE_BUFFER)
throw new InvalidRequestException("unset value is not supported for map keys");
if (keyBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
throw new InvalidRequestException(String.format("Map key is too long. Map keys are limited to %d bytes but %d bytes keys provided",
FBUtilities.MAX_UNSIGNED_SHORT,
keyBytes.remaining()));
ByteBuffer valueBytes = entry.getValue().bindAndGet(options);
if (valueBytes == null)
@ -240,11 +236,6 @@ public abstract class Maps
if (valueBytes == ByteBufferUtil.UNSET_BYTE_BUFFER)
return UNSET_VALUE;
if (valueBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
throw new InvalidRequestException(String.format("Map value is too long. Map values are limited to %d bytes but %d bytes value provided",
FBUtilities.MAX_UNSIGNED_SHORT,
valueBytes.remaining()));
buffers.put(keyBytes, valueBytes);
}
return new Value(buffers);
@ -331,12 +322,6 @@ public abstract class Maps
}
else if (value != ByteBufferUtil.UNSET_BYTE_BUFFER)
{
// We don't support value > 64K because the serialization format encode the length as an unsigned short.
if (value.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
throw new InvalidRequestException(String.format("Map value is too long. Map values are limited to %d bytes but %d bytes value provided",
FBUtilities.MAX_UNSIGNED_SHORT,
value.remaining()));
params.addCell(column, path, value);
}
}

7
src/java/org/apache/cassandra/cql3/Sets.java
View File

@ -35,7 +35,6 @@ import org.apache.cassandra.serializers.CollectionSerializer;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.transport.Server;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
/**
* Static helper methods and classes for sets.
@ -213,12 +212,6 @@ public abstract class Sets
if (bytes == ByteBufferUtil.UNSET_BYTE_BUFFER)
return UNSET_VALUE;
// We don't support value > 64K because the serialization format encode the length as an unsigned short.
if (bytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
throw new InvalidRequestException(String.format("Set value is too long. Set values are limited to %d bytes but %d bytes value provided",
FBUtilities.MAX_UNSIGNED_SHORT,
bytes.remaining()));
buffers.add(bytes);
}
return new Value(buffers);

166
test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
View File

@ -20,11 +20,13 @@ package org.apache.cassandra.cql3.validation.entities;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.Arrays;
import java.util.UUID;
import org.junit.Test;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
@ -686,4 +688,168 @@ public class CollectionsTest extends CQLTester
execute("delete s_list[0] from %s where k1='a'");
assertRows(execute("select s_list from %s where k1='a'"), row(list(0)));
}
@Test
public void testListWithElementsBiggerThan64K() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, l list<text>)");
byte[] bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 1);
String largeText = new String(bytes);
bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 2);
String largeText2 = new String(bytes);
execute("INSERT INTO %s(k, l) VALUES (0, ?)", list(largeText, "v2"));
flush();
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list(largeText, "v2")));
execute("DELETE l[?] FROM %s WHERE k = 0", 0);
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list("v2")));
execute("UPDATE %s SET l[?] = ? WHERE k = 0", 0, largeText);
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list(largeText)));
// Full overwrite
execute("UPDATE %s SET l = ? WHERE k = 0", list("v1", largeText));
flush();
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list("v1", largeText)));
execute("UPDATE %s SET l = l + ? WHERE k = 0", list("v2", largeText2));
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list("v1", largeText, "v2", largeText2)));
execute("UPDATE %s SET l = l - ? WHERE k = 0", list(largeText, "v2"));
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list("v1", largeText2)));
execute("DELETE l FROM %s WHERE k = 0");
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row((Object) null));
execute("INSERT INTO %s(k, l) VALUES (0, ['" + largeText + "', 'v2'])");
flush();
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list(largeText, "v2")));
}
@Test
public void testMapsWithElementsBiggerThan64K() throws Throwable
{
byte[] bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 1);
String largeText = new String(bytes);
bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 2);
String largeText2 = new String(bytes);
createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<text, text>)");
execute("INSERT INTO %s(k, m) VALUES (0, ?)", map("k1", largeText, largeText, "v2"));
flush();
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map("k1", largeText, largeText, "v2")));
execute("UPDATE %s SET m[?] = ? WHERE k = 0", "k3", largeText);
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map("k1", largeText, largeText, "v2", "k3", largeText)));
execute("UPDATE %s SET m[?] = ? WHERE k = 0", largeText2, "v4");
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map("k1", largeText, largeText, "v2", "k3", largeText, largeText2, "v4")));
execute("DELETE m[?] FROM %s WHERE k = 0", "k1");
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map(largeText, "v2", "k3", largeText, largeText2, "v4")));
execute("DELETE m[?] FROM %s WHERE k = 0", largeText2);
flush();
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map(largeText, "v2", "k3", largeText)));
// Full overwrite
execute("UPDATE %s SET m = ? WHERE k = 0", map("k5", largeText, largeText, "v6"));
flush();
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map("k5", largeText, largeText, "v6")));
execute("UPDATE %s SET m = m + ? WHERE k = 0", map("k7", largeText));
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map("k5", largeText, largeText, "v6", "k7", largeText)));
execute("UPDATE %s SET m = m + ? WHERE k = 0", map(largeText2, "v8"));
flush();
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map("k5", largeText, largeText, "v6", "k7", largeText, largeText2, "v8")));
execute("DELETE m FROM %s WHERE k = 0");
assertRows(execute("SELECT m FROM %s WHERE k = 0"), row((Object) null));
execute("INSERT INTO %s(k, m) VALUES (0, {'" + largeText + "' : 'v1', 'k2' : '" + largeText + "'})");
flush();
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map(largeText, "v1", "k2", largeText)));
}
@Test
public void testSetsWithElementsBiggerThan64K() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, s set<text>)");
byte[] bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 1);
String largeText = new String(bytes);
bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 2);
String largeText2 = new String(bytes);
execute("INSERT INTO %s(k, s) VALUES (0, ?)", set(largeText, "v2"));
flush();
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set(largeText, "v2")));
execute("DELETE s[?] FROM %s WHERE k = 0", largeText);
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set("v2")));
// Full overwrite
execute("UPDATE %s SET s = ? WHERE k = 0", set("v1", largeText));
flush();
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set("v1", largeText)));
execute("UPDATE %s SET s = s + ? WHERE k = 0", set("v2", largeText2));
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set("v1", largeText, "v2", largeText2)));
execute("UPDATE %s SET s = s - ? WHERE k = 0", set(largeText, "v2"));
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set("v1", largeText2)));
execute("DELETE s FROM %s WHERE k = 0");
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row((Object) null));
execute("INSERT INTO %s(k, s) VALUES (0, {'" + largeText + "', 'v2'})");
flush();
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set(largeText, "v2")));
}
}

98
test/unit/org/apache/cassandra/cql3/validation/entities/FrozenCollectionsTest.java
View File

@ -33,6 +33,7 @@ import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
@ -1113,4 +1114,101 @@ public class FrozenCollectionsTest extends CQLTester
TupleType tuple = new TupleType(types);
assertEquals("TupleType(SetType(Int32Type))", clean(tuple.toString()));
}
@Test
public void testListWithElementsBiggerThan64K() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, l frozen<list<text>>)");
byte[] bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 1);
String largeText = new String(bytes);
execute("INSERT INTO %s(k, l) VALUES (0, ?)", list(largeText, "v2"));
flush();
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list(largeText, "v2")));
// Full overwrite
execute("UPDATE %s SET l = ? WHERE k = 0", list("v1", largeText));
flush();
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list("v1", largeText)));
execute("DELETE l FROM %s WHERE k = 0");
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row((Object) null));
execute("INSERT INTO %s(k, l) VALUES (0, ['" + largeText + "', 'v2'])");
flush();
assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list(largeText, "v2")));
}
@Test
public void testMapsWithElementsBiggerThan64K() throws Throwable
{
byte[] bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 1);
String largeText = new String(bytes);
bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 2);
String largeText2 = new String(bytes);
createTable("CREATE TABLE %s (k int PRIMARY KEY, m frozen<map<text, text>>)");
execute("INSERT INTO %s(k, m) VALUES (0, ?)", map(largeText, "v1", "k2", largeText));
flush();
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map(largeText, "v1", "k2", largeText)));
// Full overwrite
execute("UPDATE %s SET m = ? WHERE k = 0", map("k5", largeText, largeText2, "v6"));
flush();
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map("k5", largeText, largeText2, "v6")));
execute("DELETE m FROM %s WHERE k = 0");
assertRows(execute("SELECT m FROM %s WHERE k = 0"), row((Object) null));
execute("INSERT INTO %s(k, m) VALUES (0, {'" + largeText + "' : 'v1', 'k2' : '" + largeText + "'})");
flush();
assertRows(execute("SELECT m FROM %s WHERE k = 0"),
row(map(largeText, "v1", "k2", largeText)));
}
@Test
public void testSetsWithElementsBiggerThan64K() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, s frozen<set<text>>)");
byte[] bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
Arrays.fill(bytes, (byte) 1);
String largeText = new String(bytes);
execute("INSERT INTO %s(k, s) VALUES (0, ?)", set(largeText, "v1", "v2"));
flush();
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set(largeText, "v1", "v2")));
// Full overwrite
execute("UPDATE %s SET s = ? WHERE k = 0", set(largeText, "v3"));
flush();
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set(largeText, "v3")));
execute("DELETE s FROM %s WHERE k = 0");
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row((Object) null));
execute("INSERT INTO %s(k, s) VALUES (0, {'" + largeText + "', 'v1', 'v2'})");
flush();
assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set(largeText, "v1", "v2")));
}
}