natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch cassandra-3.0 into trunk

This commit is contained in:
Benjamin Lerer 2016-02-17 11:56:48 +01:00
parent 09fd433cb5 7a3c3abe1d
commit 0d8e853d92
14 changed files with 184 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -23,6 +23,9 @@
* Add nodetool gettimeout and settimeout commands (CASSANDRA-10953)
* Add 3.0 metadata to sstablemetadata output (CASSANDRA-10838)
Merged from 3.0:
=======
3.0.4
* Disallow drop/alter operations of UDTs used by UDAs (CASSANDRA-10721)
* Add query time validation method on Index (CASSANDRA-11043)
* Avoid potential AssertionError in mixed version cluster (CASSANDRA-11128)
* Properly handle hinted handoff after topology changes (CASSANDRA-5902)

28
src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java
View File

@ -36,7 +36,6 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
protected AlterTypeStatement(UTName name)
{
super();
this.name = name;
}
@ -50,7 +49,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
throw new InvalidRequestException("You need to be logged in a keyspace or use a fully qualified user type name");
}
protected abstract UserType makeUpdatedType(UserType toUpdate) throws InvalidRequestException;
protected abstract UserType makeUpdatedType(UserType toUpdate, KeyspaceMetadata ksm) throws InvalidRequestException;
public static AlterTypeStatement addition(UTName name, ColumnIdentifier fieldName, CQL3Type.Raw type)
{
@ -94,7 +93,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
ksm.types.get(name.getUserTypeName())
.orElseThrow(() -> new InvalidRequestException(String.format("No user type named %s exists.", name)));
UserType updated = makeUpdatedType(toUpdate);
UserType updated = makeUpdatedType(toUpdate, ksm);
// Now, we need to announce the type update to basically change it for new tables using this type,
// but we also need to find all existing user types and CF using it and change them.
@ -236,6 +235,15 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
return updatedTypes;
}
protected void checkTypeNotUsedByAggregate(KeyspaceMetadata ksm)
{
ksm.functions.udas().filter(aggregate -> aggregate.initialCondition() != null && aggregate.stateType().referencesUserType(name.getStringTypeName()))
.findAny()
.ifPresent((aggregate) -> {
throw new InvalidRequestException(String.format("Cannot alter user type %s as it is still used as an INITCOND by aggregate %s", name, aggregate));
});
}
private static class AddOrAlter extends AlterTypeStatement
{
private final boolean isAdd;
@ -260,7 +268,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
newNames.add(fieldName.bytes);
AbstractType<?> addType = type.prepare(keyspace()).getType();
if (addType.references(toUpdate))
if (addType.referencesUserType(toUpdate.getNameAsString()))
throw new InvalidRequestException(String.format("Cannot add new field %s of type %s to type %s as this would create a circular reference", fieldName, type, name));
List<AbstractType<?>> newTypes = new ArrayList<>(toUpdate.size() + 1);
@ -270,8 +278,10 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
return new UserType(toUpdate.keyspace, toUpdate.name, newNames, newTypes);
}
private UserType doAlter(UserType toUpdate) throws InvalidRequestException
private UserType doAlter(UserType toUpdate, KeyspaceMetadata ksm) throws InvalidRequestException
{
checkTypeNotUsedByAggregate(ksm);
int idx = getIdxOfField(toUpdate, fieldName);
if (idx < 0)
throw new InvalidRequestException(String.format("Unknown field %s in type %s", fieldName, name));
@ -287,9 +297,9 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
return new UserType(toUpdate.keyspace, toUpdate.name, newNames, newTypes);
}
protected UserType makeUpdatedType(UserType toUpdate) throws InvalidRequestException
protected UserType makeUpdatedType(UserType toUpdate, KeyspaceMetadata ksm) throws InvalidRequestException
{
return isAdd ? doAdd(toUpdate) : doAlter(toUpdate);
return isAdd ? doAdd(toUpdate) : doAlter(toUpdate, ksm);
}
}
@ -303,8 +313,10 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
this.renames = renames;
}
protected UserType makeUpdatedType(UserType toUpdate) throws InvalidRequestException
protected UserType makeUpdatedType(UserType toUpdate, KeyspaceMetadata ksm) throws InvalidRequestException
{
checkTypeNotUsedByAggregate(ksm);
List<ByteBuffer> newNames = new ArrayList<>(toUpdate.fieldNames());
List<AbstractType<?>> newTypes = new ArrayList<>(toUpdate.fieldTypes());

40
src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java
View File

@ -35,7 +35,6 @@ public class DropTypeStatement extends SchemaAlteringStatement
public DropTypeStatement(UTName name, boolean ifExists)
{
super();
this.name = name;
this.ifExists = ifExists;
}
@ -80,55 +79,24 @@ public class DropTypeStatement extends SchemaAlteringStatement
for (Function function : ksm.functions)
{
if (isUsedBy(function.returnType()))
if (function.returnType().referencesUserType(name.getStringTypeName()))
throw new InvalidRequestException(String.format("Cannot drop user type %s as it is still used by function %s", name, function));
for (AbstractType<?> argType : function.argTypes())
if (isUsedBy(argType))
if (argType.referencesUserType(name.getStringTypeName()))
throw new InvalidRequestException(String.format("Cannot drop user type %s as it is still used by function %s", name, function));
}
for (UserType ut : ksm.types)
if (!ut.name.equals(name.getUserTypeName()) && isUsedBy(ut))
if (!ut.name.equals(name.getUserTypeName()) && ut.referencesUserType(name.getStringTypeName()))
throw new InvalidRequestException(String.format("Cannot drop user type %s as it is still used by user type %s", name, ut.getNameAsString()));
for (CFMetaData cfm : ksm.tablesAndViews())
for (ColumnDefinition def : cfm.allColumns())
if (isUsedBy(def.type))
if (def.type.referencesUserType(name.getStringTypeName()))
throw new InvalidRequestException(String.format("Cannot drop user type %s as it is still used by table %s.%s", name, cfm.ksName, cfm.cfName));
}
private boolean isUsedBy(AbstractType<?> toCheck) throws RequestValidationException
{
if (toCheck instanceof UserType)
{
UserType ut = (UserType)toCheck;
if (name.getKeyspace().equals(ut.keyspace) && name.getUserTypeName().equals(ut.name))
return true;
for (AbstractType<?> subtype : ut.fieldTypes())
if (isUsedBy(subtype))
return true;
}
else if (toCheck instanceof CompositeType)
{
CompositeType ct = (CompositeType)toCheck;
for (AbstractType<?> subtype : ct.types)
if (isUsedBy(subtype))
return true;
}
else if (toCheck instanceof CollectionType)
{
if (toCheck instanceof ListType)
return isUsedBy(((ListType)toCheck).getElementsType());
else if (toCheck instanceof SetType)
return isUsedBy(((SetType)toCheck).getElementsType());
else
return isUsedBy(((MapType)toCheck).getKeysType()) || isUsedBy(((MapType)toCheck).getValuesType());
}
return false;
}
@Override
public String keyspace()
{

6
src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java
View File

@ -296,6 +296,12 @@ public abstract class AbstractCompositeType extends AbstractType<ByteBuffer>
return BytesSerializer.instance;
}
@Override
public boolean referencesUserType(String name)
{
return getComponents().stream().anyMatch(f -> f.referencesUserType(name));
}
/**
* @return the comparator for the given component. static CompositeType will consult
* @param i DynamicCompositeType will read the type information from @param bb

7
src/java/org/apache/cassandra/db/marshal/AbstractType.java
View File

@ -400,12 +400,9 @@ public abstract class AbstractType<T> implements Comparator<ByteBuffer>
ByteBufferUtil.skipWithVIntLength(in);
}
/**
* Checks whether this type or any of the types this type contains references the given type.
*/
public boolean references(AbstractType<?> check)
public boolean referencesUserType(String userTypeName)
{
return this.equals(check);
return false;
}
/**

11
src/java/org/apache/cassandra/db/marshal/CompositeType.java
View File

@ -120,17 +120,6 @@ public class CompositeType extends AbstractCompositeType
this.types = ImmutableList.copyOf(types);
}
@Override
public boolean references(AbstractType<?> check)
{
if (super.references(check))
return true;
for (AbstractType<?> type : types)
if (type.references(check))
return true;
return false;
}
protected AbstractType<?> getComparator(int i, ByteBuffer bb)
{
try

4
src/java/org/apache/cassandra/db/marshal/ListType.java
View File

@ -75,9 +75,9 @@ public class ListType<T> extends CollectionType<List<T>>
}
@Override
public boolean references(AbstractType<?> check)
public boolean referencesUserType(String userTypeName)
{
return super.references(check) || elements.references(check);
return getElementsType().referencesUserType(userTypeName);
}
public AbstractType<T> getElementsType()

5
src/java/org/apache/cassandra/db/marshal/MapType.java
View File

@ -75,9 +75,10 @@ public class MapType<K, V> extends CollectionType<Map<K, V>>
}
@Override
public boolean references(AbstractType<?> check)
public boolean referencesUserType(String userTypeName)
{
return super.references(check) || keys.references(check) || values.references(check);
return getKeysType().referencesUserType(userTypeName) ||
getValuesType().referencesUserType(userTypeName);
}
public AbstractType<K> getKeysType()

4
src/java/org/apache/cassandra/db/marshal/ReversedType.java
View File

@ -135,9 +135,9 @@ public class ReversedType<T> extends AbstractType<T>
return baseType.getSerializer();
}
public boolean references(AbstractType<?> check)
public boolean referencesUserType(String userTypeName)
{
return super.references(check) || baseType.references(check);
return baseType.referencesUserType(userTypeName);
}
@Override

4
src/java/org/apache/cassandra/db/marshal/SetType.java
View File

@ -69,9 +69,9 @@ public class SetType<T> extends CollectionType<Set<T>>
}
@Override
public boolean references(AbstractType<?> check)
public boolean referencesUserType(String userTypeName)
{
return super.references(check) || elements.references(check);
return getElementsType().referencesUserType(userTypeName);
}
public AbstractType<T> getElementsType()

9
src/java/org/apache/cassandra/db/marshal/TupleType.java
View File

@ -66,14 +66,9 @@ public class TupleType extends AbstractType<ByteBuffer>
}
@Override
public boolean references(AbstractType<?> check)
public boolean referencesUserType(String name)
{
if (super.references(check))
return true;
for (AbstractType<?> type : types)
if (type.references(check))
return true;
return false;
return allTypes().stream().anyMatch(f -> f.referencesUserType(name));
}
public AbstractType<?> type(int i)

9
src/java/org/apache/cassandra/db/marshal/UserType.java
View File

@ -19,7 +19,6 @@ package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.*;
@ -29,7 +28,6 @@ import org.apache.cassandra.cql3.*;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.serializers.*;
import org.apache.cassandra.transport.Server;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
@ -240,6 +238,13 @@ public class UserType extends TupleType
return CQL3Type.UserDefined.create(this);
}
@Override
public boolean referencesUserType(String userTypeName)
{
return getNameAsString().equals(userTypeName) ||
fieldTypes().stream().anyMatch(f -> f.referencesUserType(userTypeName));
}
@Override
public String toString()
{

68
test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
View File

@ -447,6 +447,74 @@ public class UserTypesTest extends CQLTester
assertInvalidMessage("would create a circular reference", "ALTER TYPE " + typeWithKs(type1) + " ADD needs_to_fail frozen<list<" + typeWithKs(type1) + ">>");
}
@Test
public void testTypeAlterUsedInFunction() throws Throwable
{
String type1 = createType("CREATE TYPE %s (foo ascii)");
assertComplexInvalidAlterDropStatements(type1, type1, "{foo: 'abc'}");
type1 = createType("CREATE TYPE %s (foo ascii)");
assertComplexInvalidAlterDropStatements(type1, "list<frozen<" + type1 + ">>", "[{foo: 'abc'}]");
type1 = createType("CREATE TYPE %s (foo ascii)");
assertComplexInvalidAlterDropStatements(type1, "set<frozen<" + type1 + ">>", "{{foo: 'abc'}}");
type1 = createType("CREATE TYPE %s (foo ascii)");
assertComplexInvalidAlterDropStatements(type1, "map<text, frozen<" + type1 + ">>", "{'key': {foo: 'abc'}}");
type1 = createType("CREATE TYPE %s (foo ascii)");
String type2 = createType("CREATE TYPE %s (foo frozen<" + type1 + ">)");
assertComplexInvalidAlterDropStatements(type1, type2, "{foo: 'abc'}");
type1 = createType("CREATE TYPE %s (foo ascii)");
type2 = createType("CREATE TYPE %s (foo frozen<" + type1 + ">)");
assertComplexInvalidAlterDropStatements(type1,
"list<frozen<" + type2 + ">>",
"[{foo: 'abc'}]");
type1 = createType("CREATE TYPE %s (foo ascii)");
type2 = createType("CREATE TYPE %s (foo frozen<set<" + type1 + ">>)");
assertComplexInvalidAlterDropStatements(type1,
"map<text, frozen<" + type2 + ">>",
"{'key': {foo: {{foo: 'abc'}}}}");
type1 = createType("CREATE TYPE %s (foo ascii)");
assertComplexInvalidAlterDropStatements(type1,
"tuple<text, frozen<" + type1 + ">>",
"('key', {foo: 'abc'})");
type1 = createType("CREATE TYPE %s (foo ascii)");
assertComplexInvalidAlterDropStatements(type1,
"tuple<text, frozen<tuple<tuple<" + type1 + ", int>, int>>>",
"('key', (({foo: 'abc'}, 0), 0))");
type1 = createType("CREATE TYPE %s (foo ascii)");
type2 = createType("CREATE TYPE %s (foo frozen<set<" + type1 + ">>)");
assertComplexInvalidAlterDropStatements(type1,
"tuple<text, frozen<" + type2 + ">>",
"('key', {foo: {{foo: 'abc'}}})");
}
private void assertComplexInvalidAlterDropStatements(String type1, String fArgType, String initcond) throws Throwable
{
String f = createFunction(KEYSPACE, type1, "CREATE FUNCTION %s(arg " + fArgType + ", col int) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS " + fArgType + ' ' +
"LANGUAGE java AS 'return arg;'");
createAggregate(KEYSPACE, "int", "CREATE AGGREGATE %s(int) " +
"SFUNC " + shortFunctionName(f) + ' ' +
"STYPE " + fArgType + ' ' +
"INITCOND " + initcond);
assertInvalidAlterDropStatements(type1);
}
private void assertInvalidAlterDropStatements(String t) throws Throwable
{
assertInvalidMessage("Cannot alter user type " + typeWithKs(t), "ALTER TYPE " + typeWithKs(t) + " RENAME foo TO bar;");
assertInvalidMessage("Cannot alter user type " + typeWithKs(t), "ALTER TYPE " + typeWithKs(t) + " ALTER foo TYPE text;");
assertInvalidMessage("Cannot drop user type " + typeWithKs(t), "DROP TYPE " + typeWithKs(t) + ';');
}
private String typeWithKs(String type1)
{
return keyspace() + '.' + type1;

63
test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java Normal file
View File

@ -0,0 +1,63 @@
package org.apache.cassandra.db.commitlog;
import java.nio.ByteBuffer;
import java.util.Random;
import javax.naming.ConfigurationException;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.Config.CommitLogSync;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.schema.KeyspaceParams;
import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.collect.ImmutableMap;
public class CommitLogSegmentManagerTest
{
private static final String KEYSPACE1 = "CommitLogTest";
private static final String STANDARD1 = "Standard1";
private static final String STANDARD2 = "Standard2";
private final static byte[] entropy = new byte[1024 * 256];
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
new Random().nextBytes(entropy);
DatabaseDescriptor.setCommitLogCompression(new ParameterizedClass("LZ4Compressor", ImmutableMap.of()));
DatabaseDescriptor.setCommitLogSegmentSize(1);
DatabaseDescriptor.setCommitLogSync(CommitLogSync.periodic);
DatabaseDescriptor.setCommitLogSyncPeriod(10 * 1000);
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE1, STANDARD1, 0, AsciiType.instance, BytesType.instance),
SchemaLoader.standardCFMD(KEYSPACE1, STANDARD2, 0, AsciiType.instance, BytesType.instance));
CompactionManager.instance.disableAutoCompaction();
}
@Test
public void testCompressedCommitLogBackpressure() throws Throwable
{
CommitLog.instance.resetUnsafe(true);
ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
.clustering("bytes")
.add("val", ByteBuffer.wrap(entropy))
.build();
for (int i = 0; i < 20000; i++)
CommitLog.instance.add(m);
}
}