diff --git a/CHANGES.txt b/CHANGES.txt index c39f1194ed..9966bfd5b1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0 + * Store UDA initcond as CQL literal in the schema table, instead of a blob (CASSANDRA-10650) * Don't use -1 for the position of partition key in schema (CASSANDRA-10491) * Fix distinct queries in mixed version cluster (CASSANDRA-10573) * Skip sstable on clustering in names query (CASSANDRA-10571) diff --git a/lib/cassandra-driver-core-3.0.0-beta1-92c4c80-SNAPSHOT-shaded.jar b/lib/cassandra-driver-core-3.0.0-beta1-bb1bce4-SNAPSHOT-shaded.jar similarity index 87% rename from lib/cassandra-driver-core-3.0.0-beta1-92c4c80-SNAPSHOT-shaded.jar rename to lib/cassandra-driver-core-3.0.0-beta1-bb1bce4-SNAPSHOT-shaded.jar index 2026c52f00..b8f002f704 100644 Binary files a/lib/cassandra-driver-core-3.0.0-beta1-92c4c80-SNAPSHOT-shaded.jar and b/lib/cassandra-driver-core-3.0.0-beta1-bb1bce4-SNAPSHOT-shaded.jar differ diff --git a/lib/licenses/cassandra-driver-2.1.3.txt b/lib/licenses/cassandra-driver-3.0.0.txt similarity index 100% rename from lib/licenses/cassandra-driver-2.1.3.txt rename to lib/licenses/cassandra-driver-3.0.0.txt diff --git a/src/java/org/apache/cassandra/config/ViewDefinition.java b/src/java/org/apache/cassandra/config/ViewDefinition.java index 02acc689c0..b29a8f9dd7 100644 --- a/src/java/org/apache/cassandra/config/ViewDefinition.java +++ b/src/java/org/apache/cassandra/config/ViewDefinition.java @@ -151,22 +151,9 @@ public class ViewDefinition private static List whereClauseToRelations(String whereClause) { - ErrorCollector errorCollector = new ErrorCollector(whereClause); - CharStream stream = new ANTLRStringStream(whereClause); - CqlLexer lexer = new CqlLexer(stream); - lexer.addErrorListener(errorCollector); - - TokenStream tokenStream = new CommonTokenStream(lexer); - CqlParser parser = new CqlParser(tokenStream); - parser.addErrorListener(errorCollector); - try { - List relations = parser.whereClause().build().relations; - - // The errorCollector has queued up any errors that the lexer and parser may have encountered - // along the way, if necessary, we turn the last error into exceptions here. - errorCollector.throwFirstSyntaxError(); + List relations = CQLFragmentParser.parseAnyUnhandled(CqlParser::whereClause, whereClause).build().relations; return relations; } diff --git a/src/java/org/apache/cassandra/cql3/CQL3Type.java b/src/java/org/apache/cassandra/cql3/CQL3Type.java index fde3fab40a..0c4ef172ca 100644 --- a/src/java/org/apache/cassandra/cql3/CQL3Type.java +++ b/src/java/org/apache/cassandra/cql3/CQL3Type.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.cql3; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -30,6 +31,9 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Types; +import org.apache.cassandra.serializers.CollectionSerializer; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.ByteBufferUtil; public interface CQL3Type { @@ -38,6 +42,23 @@ public interface CQL3Type public boolean isCollection(); public AbstractType getType(); + /** + * Generate CQL literal from this type's serialized representation using the specified protocol version. + * Convinience method for {@link #toCQLLiteral(ByteBuffer, int, StringBuilder)} that just returns a {@code String}. + */ + public default String asCQLLiteral(ByteBuffer buffer, int version) + { + StringBuilder sb = new StringBuilder(); + toCQLLiteral(buffer, version, sb); + return sb.toString(); + } + + /** + * Generate CQL literal from this type's serialized representation using the specified protocol version. + * Some work is delegated to {@link org.apache.cassandra.serializers.TypeSerializer#toCQLLiteral(ByteBuffer, StringBuilder)}. + */ + public void toCQLLiteral(ByteBuffer buffer, int version, StringBuilder target); + public enum Native implements CQL3Type { ASCII (AsciiType.instance), @@ -79,6 +100,18 @@ public interface CQL3Type return type; } + /** + * Delegate to + * {@link org.apache.cassandra.serializers.TypeSerializer#toCQLLiteral(ByteBuffer, StringBuilder)} + * for native types as most CQL literal representations work fine with the default + * {@link org.apache.cassandra.serializers.TypeSerializer#toString(Object)} + * {@link org.apache.cassandra.serializers.TypeSerializer#deserialize(ByteBuffer)} implementations. + */ + public void toCQLLiteral(ByteBuffer buffer, int version, StringBuilder target) + { + type.getSerializer().toCQLLiteral(buffer, target); + } + @Override public String toString() { @@ -110,6 +143,14 @@ public interface CQL3Type return type; } + public void toCQLLiteral(ByteBuffer buffer, int version, StringBuilder target) + { + if (buffer == null) + target.append("null"); + else + target.append(type.getString(buffer)); + } + @Override public final boolean equals(Object o) { @@ -129,7 +170,7 @@ public interface CQL3Type @Override public String toString() { - return "'" + type + "'"; + return "'" + type + '\''; } } @@ -152,6 +193,88 @@ public interface CQL3Type return true; } + public void toCQLLiteral(ByteBuffer buffer, int version, StringBuilder target) + { + // Not sure whether the !buffer.hasRemaining() check is correct here or whether an empty + // BB should be returned as "[]" resp "{}" or whether it is not valid at all. + // + // Currently, all empty collections return '[]' or '{}'. Except frozen collections with + // a null BB return 'null'. + // + if (buffer == null || !buffer.hasRemaining()) + { + if (buffer == null && type.isFrozenCollection()) + { + target.append("null"); + } + else + { + switch (type.kind) + { + case LIST: + target.append("[]"); + break; + case SET: + case MAP: + target.append("{}"); + break; + } + } + } + else + { + int size = CollectionSerializer.readCollectionSize(buffer, version); + + switch (type.kind) + { + case LIST: + CQL3Type elements = ((ListType) type).getElementsType().asCQL3Type(); + target.append('['); + generateSetOrListCQLLiteral(buffer, version, target, size, elements); + target.append(']'); + break; + case SET: + elements = ((SetType) type).getElementsType().asCQL3Type(); + target.append('{'); + generateSetOrListCQLLiteral(buffer, version, target, size, elements); + target.append('}'); + break; + case MAP: + target.append('{'); + generateMapCQLLiteral(buffer, version, target, size); + target.append('}'); + break; + } + } + } + + private void generateMapCQLLiteral(ByteBuffer buffer, int version, StringBuilder target, int size) + { + CQL3Type keys = ((MapType) type).getKeysType().asCQL3Type(); + CQL3Type values = ((MapType) type).getValuesType().asCQL3Type(); + for (int i = 0; i < size; i++) + { + if (i > 0) + target.append(", "); + ByteBuffer element = CollectionSerializer.readValue(buffer, version); + keys.toCQLLiteral(element, version, target); + target.append(": "); + element = CollectionSerializer.readValue(buffer, version); + values.toCQLLiteral(element, version, target); + } + } + + private static void generateSetOrListCQLLiteral(ByteBuffer buffer, int version, StringBuilder target, int size, CQL3Type elements) + { + for (int i = 0; i < size; i++) + { + if (i > 0) + target.append(", "); + ByteBuffer element = CollectionSerializer.readValue(buffer, version); + elements.toCQLLiteral(element, version, target); + } + } + @Override public final boolean equals(Object o) { @@ -191,9 +314,9 @@ public interface CQL3Type default: throw new AssertionError(); } - sb.append(">"); + sb.append('>'); if (isFrozen) - sb.append(">"); + sb.append('>'); return sb.toString(); } } @@ -225,6 +348,49 @@ public interface CQL3Type return type; } + public void toCQLLiteral(ByteBuffer buffer, int version, StringBuilder target) + { + if (buffer == null) + { + target.append("null"); + } + else + { + target.append('{'); + for (int i = 0; i < type.size(); i++) + { + // we allow the input to have less fields than declared so as to support field addition. + if (!buffer.hasRemaining()) + break; + + if (buffer.remaining() < 4) + throw new MarshalException(String.format("Not enough bytes to read size of %dth field %s", i, type.fieldName(i))); + + int size = buffer.getInt(); + + if (i > 0) + target.append(", "); + + target.append(ColumnIdentifier.maybeQuote(type.fieldNameAsString(i))); + target.append(": "); + + // size < 0 means null value + if (size < 0) + { + target.append("null"); + continue; + } + + if (buffer.remaining() < size) + throw new MarshalException(String.format("Not enough bytes to read %dth field %s", i, type.fieldName(i))); + + ByteBuffer field = ByteBufferUtil.readBytes(buffer, size); + type.fieldType(i).asCQL3Type().toCQLLiteral(field, version, target); + } + target.append('}'); + } + } + @Override public final boolean equals(Object o) { @@ -272,6 +438,49 @@ public interface CQL3Type return type; } + public void toCQLLiteral(ByteBuffer buffer, int version, StringBuilder target) + { + if (buffer == null) + { + target.append("null"); + } + else + { + target.append('('); + boolean first = true; + for (int i = 0; i < type.size(); i++) + { + // we allow the input to have less fields than declared so as to support field addition. + if (!buffer.hasRemaining()) + break; + + if (buffer.remaining() < 4) + throw new MarshalException(String.format("Not enough bytes to read size of %dth component", i)); + + int size = buffer.getInt(); + + if (first) + first = false; + else + target.append(", "); + + // size < 0 means null value + if (size < 0) + { + target.append("null"); + continue; + } + + if (buffer.remaining() < size) + throw new MarshalException(String.format("Not enough bytes to read %dth component", i)); + + ByteBuffer field = ByteBufferUtil.readBytes(buffer, size); + type.type(i).asCQL3Type().toCQLLiteral(field, version, target); + } + target.append(')'); + } + } + @Override public final boolean equals(Object o) { @@ -515,9 +724,9 @@ public interface CQL3Type String end = frozen ? ">" : ""; switch (kind) { - case LIST: return start + "list<" + values + ">" + end; - case SET: return start + "set<" + values + ">" + end; - case MAP: return start + "map<" + keys + ", " + values + ">" + end; + case LIST: return start + "list<" + values + '>' + end; + case SET: return start + "set<" + values + '>' + end; + case MAP: return start + "map<" + keys + ", " + values + '>' + end; } throw new AssertionError(); } @@ -650,7 +859,7 @@ public interface CQL3Type sb.append(", "); sb.append(types.get(i)); } - sb.append(">"); + sb.append('>'); return sb.toString(); } } diff --git a/src/java/org/apache/cassandra/cql3/CQLFragmentParser.java b/src/java/org/apache/cassandra/cql3/CQLFragmentParser.java new file mode 100644 index 0000000000..d6f4732bfe --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/CQLFragmentParser.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3; + +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.TokenStream; +import org.apache.cassandra.exceptions.SyntaxException; + +/** + * Helper class to encapsulate common code that calls one of the generated methods in {@code CqlParser}. + */ +public final class CQLFragmentParser +{ + + @FunctionalInterface + public interface CQLParserFunction + { + R parse(CqlParser parser) throws RecognitionException; + } + + public static R parseAny(CQLParserFunction parserFunction, String input, String meaning) + { + try + { + return parseAnyUnhandled(parserFunction, input); + } + catch (RuntimeException re) + { + throw new SyntaxException(String.format("Failed parsing %s: [%s] reason: %s %s", + meaning, + input, + re.getClass().getSimpleName(), + re.getMessage())); + } + catch (RecognitionException e) + { + throw new SyntaxException("Invalid or malformed " + meaning + ": " + e.getMessage()); + } + } + + /** + * Just call a parser method in {@link CqlParser} - does not do any error handling. + */ + public static R parseAnyUnhandled(CQLParserFunction parserFunction, String input) throws RecognitionException + { + // Lexer and parser + ErrorCollector errorCollector = new ErrorCollector(input); + CharStream stream = new ANTLRStringStream(input); + CqlLexer lexer = new CqlLexer(stream); + lexer.addErrorListener(errorCollector); + + TokenStream tokenStream = new CommonTokenStream(lexer); + CqlParser parser = new CqlParser(tokenStream); + parser.addErrorListener(errorCollector); + + // Parse the query string to a statement instance + R r = parserFunction.parse(parser); + + // The errorCollector has queue up any errors that the lexer and parser may have encountered + // along the way, if necessary, we turn the last error into exceptions here. + errorCollector.throwFirstSyntaxError(); + + return r; + } +} diff --git a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java index 745e4f05fc..93734e9be4 100644 --- a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java +++ b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java @@ -18,11 +18,10 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.List; import java.util.Locale; -import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentMap; +import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.collect.MapMaker; @@ -36,7 +35,6 @@ import org.apache.cassandra.cql3.selection.SimpleSelector; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.memory.AbstractAllocator; @@ -45,8 +43,10 @@ import org.apache.cassandra.utils.memory.AbstractAllocator; * Represents an identifer for a CQL column definition. * TODO : should support light-weight mode without text representation for when not interned */ -public class ColumnIdentifier extends org.apache.cassandra.cql3.selection.Selectable implements IMeasurableMemory, Comparable +public class ColumnIdentifier extends Selectable implements IMeasurableMemory, Comparable { + private static final Pattern PATTERN_DOUBLE_QUOTE = Pattern.compile("\"", Pattern.LITERAL); + public final ByteBuffer bytes; private final String text; /** @@ -332,6 +332,6 @@ public class ColumnIdentifier extends org.apache.cassandra.cql3.selection.Select { if (UNQUOTED_IDENTIFIER.matcher(text).matches()) return text; - return "\"" + text.replace("\"", "\"\"") + "\""; + return '"' + PATTERN_DOUBLE_QUOTE.matcher(text).replaceAll(Matcher.quoteReplacement("\"\"")) + '"'; } } diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java index 18f63c2bcd..59d41488a0 100644 --- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java @@ -508,24 +508,7 @@ public class QueryProcessor implements QueryHandler { try { - // Lexer and parser - ErrorCollector errorCollector = new ErrorCollector(queryStr); - CharStream stream = new ANTLRStringStream(queryStr); - CqlLexer lexer = new CqlLexer(stream); - lexer.addErrorListener(errorCollector); - - TokenStream tokenStream = new CommonTokenStream(lexer); - CqlParser parser = new CqlParser(tokenStream); - parser.addErrorListener(errorCollector); - - // Parse the query string to a statement instance - ParsedStatement statement = parser.query(); - - // The errorCollector has queue up any errors that the lexer and parser may have encountered - // along the way, if necessary, we turn the last error into exceptions here. - errorCollector.throwFirstSyntaxError(); - - return statement; + return CQLFragmentParser.parseAnyUnhandled(CqlParser::query, queryStr); } catch (CassandraException ce) { diff --git a/src/java/org/apache/cassandra/cql3/Terms.java b/src/java/org/apache/cassandra/cql3/Terms.java index 0b049b96e9..ae4bc133f6 100644 --- a/src/java/org/apache/cassandra/cql3/Terms.java +++ b/src/java/org/apache/cassandra/cql3/Terms.java @@ -17,11 +17,13 @@ */ package org.apache.cassandra.cql3; +import java.nio.ByteBuffer; import java.util.Collections; import com.google.common.collect.Iterables; import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.db.marshal.AbstractType; public class Terms { @@ -42,4 +44,11 @@ public class Terms return Iterables.concat(Iterables.transform(terms, TO_FUNCTION_ITERABLE)); } + + public static ByteBuffer asBytes(String keyspace, String term, AbstractType type) + { + ColumnSpecification receiver = new ColumnSpecification(keyspace, "--dummy--", new ColumnIdentifier("(dummy)", true), type); + Term.Raw rawTerm = CQLFragmentParser.parseAny(CqlParser::term, term, "CQL term"); + return rawTerm.prepare(keyspace, receiver).bindAndGet(QueryOptions.DEFAULT); + } } diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateAggregateStatement.java index 98354d5543..6fd0334fb6 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateAggregateStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateAggregateStatement.java @@ -113,8 +113,7 @@ public final class CreateAggregateStatement extends SchemaAlteringStatement if (ival != null) { - ColumnSpecification receiver = new ColumnSpecification(functionName.keyspace, "--dummy--", new ColumnIdentifier("(aggregate_initcond)", true), stateType); - initcond = ival.prepare(functionName.keyspace, receiver).bindAndGet(QueryOptions.DEFAULT); + initcond = Terms.asBytes(functionName.keyspace, ival.toString(), stateType); if (Constants.NULL_LITERAL != ival && UDHelper.isNullOrEmpty(stateType, initcond)) throw new InvalidRequestException("INITCOND must not be empty for all types except TEXT, ASCII, BLOB"); } diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractType.java b/src/java/org/apache/cassandra/db/marshal/AbstractType.java index a0d915f34c..0253ac64ff 100644 --- a/src/java/org/apache/cassandra/db/marshal/AbstractType.java +++ b/src/java/org/apache/cassandra/db/marshal/AbstractType.java @@ -119,7 +119,7 @@ public abstract class AbstractType implements Comparator return getSerializer().serialize(value); } - /** get a string representation of the bytes suitable for log messages */ + /** get a string representation of the bytes used for various identifier (NOT just for log messages) */ public String getString(ByteBuffer bytes) { if (bytes == null) diff --git a/src/java/org/apache/cassandra/schema/CQLTypeParser.java b/src/java/org/apache/cassandra/schema/CQLTypeParser.java index ed68498b04..c79de88155 100644 --- a/src/java/org/apache/cassandra/schema/CQLTypeParser.java +++ b/src/java/org/apache/cassandra/schema/CQLTypeParser.java @@ -19,11 +19,9 @@ package org.apache.cassandra.schema; import com.google.common.collect.ImmutableSet; -import org.antlr.runtime.*; import org.apache.cassandra.cql3.*; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.UserType; -import org.apache.cassandra.exceptions.SyntaxException; import static org.apache.cassandra.utils.ByteBufferUtil.bytes; @@ -57,37 +55,6 @@ public final class CQLTypeParser static CQL3Type.Raw parseRaw(String type) { - try - { - // Lexer and parser - ErrorCollector errorCollector = new ErrorCollector(type); - CharStream stream = new ANTLRStringStream(type); - CqlLexer lexer = new CqlLexer(stream); - lexer.addErrorListener(errorCollector); - - TokenStream tokenStream = new CommonTokenStream(lexer); - CqlParser parser = new CqlParser(tokenStream); - parser.addErrorListener(errorCollector); - - // Parse the query string to a statement instance - CQL3Type.Raw rawType = parser.comparatorType(); - - // The errorCollector has queue up any errors that the lexer and parser may have encountered - // along the way, if necessary, we turn the last error into exceptions here. - errorCollector.throwFirstSyntaxError(); - - return rawType; - } - catch (RuntimeException re) - { - throw new SyntaxException(String.format("Failed parsing statement: [%s] reason: %s %s", - type, - re.getClass().getSimpleName(), - re.getMessage())); - } - catch (RecognitionException e) - { - throw new SyntaxException("Invalid or malformed CQL type: " + e.getMessage()); - } + return CQLFragmentParser.parseAny(CqlParser::comparatorType, type, "CQL type"); } } diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index c715a2abe7..f1ea6cf05e 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -43,6 +43,7 @@ import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.view.View; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.transport.Server; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; @@ -223,7 +224,7 @@ public final class SchemaKeyspace + "aggregate_name text," + "argument_types frozen>," + "final_func text," - + "initcond blob," + + "initcond text," + "return_type text," + "state_func text," + "state_type text," @@ -833,11 +834,12 @@ public final class SchemaKeyspace RowUpdateBuilder adder = new RowUpdateBuilder(Aggregates, timestamp, mutation) .clustering(aggregate.name().name, functionArgumentsList(aggregate)); + CQL3Type stateCqlType = aggregate.stateType().asCQL3Type(); adder.add("return_type", aggregate.returnType().asCQL3Type().toString()) .add("state_func", aggregate.stateFunction().name().name) - .add("state_type", aggregate.stateType() != null ? aggregate.stateType().asCQL3Type().toString() : null) + .add("state_type", aggregate.stateType() != null ? stateCqlType.toString() : null) .add("final_func", aggregate.finalFunction() != null ? aggregate.finalFunction().name().name : null) - .add("initcond", aggregate.initialCondition()) + .add("initcond", aggregate.initialCondition() != null ? stateCqlType.asCQLLiteral(aggregate.initialCondition(), Server.CURRENT_VERSION) : null) .build(); } @@ -1219,7 +1221,7 @@ public final class SchemaKeyspace FunctionName stateFunc = new FunctionName(ksName, (row.getString("state_func"))); FunctionName finalFunc = row.has("final_func") ? new FunctionName(ksName, row.getString("final_func")) : null; AbstractType stateType = row.has("state_type") ? parse(ksName, row.getString("state_type"), types) : null; - ByteBuffer initcond = row.has("initcond") ? row.getBytes("initcond") : null; + ByteBuffer initcond = row.has("initcond") ? Terms.asBytes(ksName, row.getString("initcond"), stateType) : null; try { diff --git a/src/java/org/apache/cassandra/serializers/AbstractTextSerializer.java b/src/java/org/apache/cassandra/serializers/AbstractTextSerializer.java index f1de6a460d..4d2694e086 100644 --- a/src/java/org/apache/cassandra/serializers/AbstractTextSerializer.java +++ b/src/java/org/apache/cassandra/serializers/AbstractTextSerializer.java @@ -58,4 +58,32 @@ public abstract class AbstractTextSerializer implements TypeSerializer { return String.class; } + + /** + * Generates CQL literal for TEXT/VARCHAR/ASCII types. + * Caveat: it does only generate literals with single quotes and not pg-style literals. + */ + @Override + public void toCQLLiteral(ByteBuffer buffer, StringBuilder target) + { + if (buffer == null) + { + target.append("null"); + } + else + { + String s = deserialize(buffer); + + target.append('\''); + for (int i=0; i { return ByteBuffer.class; } + + @Override + public void toCQLLiteral(ByteBuffer buffer, StringBuilder target) + { + if (buffer == null) + { + target.append("null"); + } + else + { + target.append("0x"); + target.append(toString(deserialize(buffer))); + } + } } diff --git a/src/java/org/apache/cassandra/serializers/TimestampSerializer.java b/src/java/org/apache/cassandra/serializers/TimestampSerializer.java index ab81fcc740..01a85e0786 100644 --- a/src/java/org/apache/cassandra/serializers/TimestampSerializer.java +++ b/src/java/org/apache/cassandra/serializers/TimestampSerializer.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import java.text.SimpleDateFormat; import java.text.ParseException; import java.util.Date; +import java.util.TimeZone; import java.util.regex.Pattern; import org.apache.commons.lang3.time.DateUtils; @@ -73,7 +74,7 @@ public class TimestampSerializer implements TypeSerializer "yyyy-MM-dd'T'HH:mm:ss.SSS z", "yyyy-MM-dd'T'HH:mm:ss.SSS zz", "yyyy-MM-dd'T'HH:mm:ss.SSS zzz", - "yyyy-MM-dd'T'HH:mm:ss.SSSX", + "yyyy-MM-dd'T'HH:mm:ss.SSSX", // UTC_FORMAT "yyyy-MM-dd'T'HH:mm:ss.SSSXX", "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", "yyyy-MM-dd", @@ -96,6 +97,17 @@ public class TimestampSerializer implements TypeSerializer } }; + private static final String UTC_FORMAT = dateStringPatterns[40]; + private static final ThreadLocal FORMATTER_UTC = new ThreadLocal() + { + protected SimpleDateFormat initialValue() + { + SimpleDateFormat sdf = new SimpleDateFormat(UTC_FORMAT); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + return sdf; + } + }; + public static final SimpleDateFormat TO_JSON_FORMAT = new SimpleDateFormat(dateStringPatterns[15]); public static final TimestampSerializer instance = new TimestampSerializer(); @@ -150,8 +162,26 @@ public class TimestampSerializer implements TypeSerializer return value == null ? "" : FORMATTER.get().format(value); } + public String toStringUTC(Date value) + { + return value == null ? "" : FORMATTER_UTC.get().format(value); + } + public Class getType() { return Date.class; } + + /** + * Builds CQL literal for a timestamp using time zone UTC and fixed date format. + * @see #FORMATTER_UTC + */ + @Override + public void toCQLLiteral(ByteBuffer buffer, StringBuilder target) + { + if (buffer == null || !buffer.hasRemaining()) + target.append("null"); + else + target.append(FORMATTER_UTC.get().format(deserialize(buffer))); + } } diff --git a/src/java/org/apache/cassandra/serializers/TypeSerializer.java b/src/java/org/apache/cassandra/serializers/TypeSerializer.java index cddef08379..e7bb8303cb 100644 --- a/src/java/org/apache/cassandra/serializers/TypeSerializer.java +++ b/src/java/org/apache/cassandra/serializers/TypeSerializer.java @@ -34,5 +34,13 @@ public interface TypeSerializer public String toString(T value); public Class getType(); + + public default void toCQLLiteral(ByteBuffer buffer, StringBuilder target) + { + if (buffer == null || !buffer.hasRemaining()) + target.append("null"); + else + target.append(toString(deserialize(buffer))); + } } diff --git a/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java b/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java new file mode 100644 index 0000000000..f6cd8df748 --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java @@ -0,0 +1,786 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.regex.Pattern; + +import org.junit.Test; + +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.serializers.*; +import org.apache.cassandra.transport.Server; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.UUIDGen; + +import static org.junit.Assert.assertEquals; + +/** + * Test functionality to re-create a CQL literal from its serialized representation. + * This test uses some randomness to generate the values and nested structures (collections,tuples,UDTs). + */ +public class CQL3TypeLiteralTest +{ + private static final Pattern QUOTE = Pattern.compile("'"); + + /** + * Container holding the expected CQL literal for a type and serialized value. + * The CQL literal is generated independently from the code in {@link CQL3Type}. + */ + static class Value + { + final String expected; + final CQL3Type cql3Type; + final ByteBuffer value; + + Value(String expected, CQL3Type cql3Type, ByteBuffer value) + { + this.expected = expected; + this.cql3Type = cql3Type; + this.value = value; + } + } + + static final Map> nativeTypeValues = new EnumMap<>(CQL3Type.Native.class); + + static void addNativeValue(String expected, CQL3Type.Native cql3Type, ByteBuffer value) + { + List l = nativeTypeValues.get(cql3Type); + if (l == null) + nativeTypeValues.put(cql3Type, l = new ArrayList<>()); + l.add(new Value(expected, cql3Type, value)); + } + + static + { + // Add some (random) values for each native type. + // Also adds null values and empty values, if the type allows this. + + for (int i = 0; i < 20; i++) + { + String v = randString(true); + addNativeValue(quote(v), CQL3Type.Native.ASCII, AsciiSerializer.instance.serialize(v)); + } + addNativeValue("''", CQL3Type.Native.ASCII, AsciiSerializer.instance.serialize("")); + addNativeValue("''", CQL3Type.Native.ASCII, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.ASCII, null); + + for (int i = 0; i < 20; i++) + { + String v = randString(false); + addNativeValue(quote(v), CQL3Type.Native.TEXT, UTF8Serializer.instance.serialize(v)); + } + addNativeValue("''", CQL3Type.Native.TEXT, UTF8Serializer.instance.serialize("")); + addNativeValue("''", CQL3Type.Native.TEXT, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.TEXT, null); + + for (int i = 0; i < 20; i++) + { + String v = randString(false); + addNativeValue(quote(v), CQL3Type.Native.VARCHAR, UTF8Serializer.instance.serialize(v)); + } + addNativeValue("''", CQL3Type.Native.VARCHAR, UTF8Serializer.instance.serialize("")); + addNativeValue("''", CQL3Type.Native.VARCHAR, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.VARCHAR, null); + + addNativeValue("0", CQL3Type.Native.BIGINT, LongType.instance.decompose(0L)); + for (int i = 0; i < 20; i++) + { + long v = randLong(); + addNativeValue(Long.toString(v), CQL3Type.Native.BIGINT, LongType.instance.decompose(v)); + } + addNativeValue("null", CQL3Type.Native.BIGINT, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.BIGINT, null); + + addNativeValue("0", CQL3Type.Native.COUNTER, LongType.instance.decompose(0L)); + for (int i = 0; i < 20; i++) + { + long v = randLong(); + addNativeValue(Long.toString(v), CQL3Type.Native.COUNTER, LongType.instance.decompose(v)); + } + addNativeValue("null", CQL3Type.Native.COUNTER, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.COUNTER, null); + + addNativeValue("0", CQL3Type.Native.INT, Int32Type.instance.decompose(0)); + for (int i = 0; i < 20; i++) + { + int v = randInt(); + addNativeValue(Integer.toString(v), CQL3Type.Native.INT, Int32Type.instance.decompose(v)); + } + addNativeValue("null", CQL3Type.Native.INT, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.INT, null); + + addNativeValue("0", CQL3Type.Native.SMALLINT, ShortType.instance.decompose((short) 0)); + for (int i = 0; i < 20; i++) + { + short v = randShort(); + addNativeValue(Short.toString(v), CQL3Type.Native.SMALLINT, ShortType.instance.decompose(v)); + } + addNativeValue("null", CQL3Type.Native.SMALLINT, null); + + addNativeValue("0", CQL3Type.Native.TINYINT, ByteType.instance.decompose((byte) 0)); + for (int i = 0; i < 20; i++) + { + byte v = randByte(); + addNativeValue(Short.toString(v), CQL3Type.Native.TINYINT, ByteType.instance.decompose(v)); + } + addNativeValue("null", CQL3Type.Native.TINYINT, null); + + addNativeValue("0.0", CQL3Type.Native.FLOAT, FloatType.instance.decompose((float) 0)); + for (int i = 0; i < 20; i++) + { + float v = randFloat(); + addNativeValue(Float.toString(v), CQL3Type.Native.FLOAT, FloatType.instance.decompose(v)); + } + addNativeValue("NaN", CQL3Type.Native.FLOAT, FloatType.instance.decompose(Float.NaN)); + addNativeValue("null", CQL3Type.Native.FLOAT, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.FLOAT, null); + + addNativeValue("0.0", CQL3Type.Native.DOUBLE, DoubleType.instance.decompose((double) 0)); + for (int i = 0; i < 20; i++) + { + double v = randDouble(); + addNativeValue(Double.toString(v), CQL3Type.Native.DOUBLE, DoubleType.instance.decompose(v)); + } + addNativeValue("NaN", CQL3Type.Native.DOUBLE, DoubleType.instance.decompose(Double.NaN)); + addNativeValue("null", CQL3Type.Native.DOUBLE, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.DOUBLE, null); + + addNativeValue("0", CQL3Type.Native.DECIMAL, DecimalType.instance.decompose(BigDecimal.ZERO)); + for (int i = 0; i < 20; i++) + { + BigDecimal v = BigDecimal.valueOf(randDouble()); + addNativeValue(v.toString(), CQL3Type.Native.DECIMAL, DecimalType.instance.decompose(v)); + } + addNativeValue("null", CQL3Type.Native.DECIMAL, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.DECIMAL, null); + + addNativeValue("0", CQL3Type.Native.VARINT, IntegerType.instance.decompose(BigInteger.ZERO)); + for (int i = 0; i < 20; i++) + { + BigInteger v = BigInteger.valueOf(randLong()); + addNativeValue(v.toString(), CQL3Type.Native.VARINT, IntegerType.instance.decompose(v)); + } + addNativeValue("null", CQL3Type.Native.VARINT, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.VARINT, null); + + // boolean doesn't have that many possible values... + addNativeValue("false", CQL3Type.Native.BOOLEAN, BooleanType.instance.decompose(false)); + addNativeValue("true", CQL3Type.Native.BOOLEAN, BooleanType.instance.decompose(true)); + addNativeValue("null", CQL3Type.Native.BOOLEAN, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.BOOLEAN, null); + + // (mostly generates date values with surreal values like in year 14273) + for (int i = 0; i < 20; i++) + { + int v = randInt(); + addNativeValue(SimpleDateSerializer.instance.toString(v), CQL3Type.Native.DATE, SimpleDateSerializer.instance.serialize(v)); + } + addNativeValue("null", CQL3Type.Native.DATE, null); + + for (int i = 0; i < 100; i++) + { + long v = randLong(24L * 60 * 60 * 1000 * 1000 * 1000); + addNativeValue(TimeSerializer.instance.toString(v), CQL3Type.Native.TIME, TimeSerializer.instance.serialize(v)); + } + addNativeValue("null", CQL3Type.Native.TIME, null); + + // (mostly generates timestamp values with surreal values like in year 14273) + for (int i = 0; i < 20; i++) + { + long v = randLong(); + addNativeValue(TimestampSerializer.instance.toStringUTC(new Date(v)), CQL3Type.Native.TIMESTAMP, TimestampType.instance.fromString(Long.toString(v))); + } + addNativeValue("null", CQL3Type.Native.TIMESTAMP, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.TIMESTAMP, null); + + for (int i = 0; i < 20; i++) + { + UUID v = UUIDGen.getTimeUUID(randLong(System.currentTimeMillis())); + addNativeValue(v.toString(), CQL3Type.Native.TIMEUUID, TimeUUIDType.instance.decompose(v)); + } + addNativeValue("null", CQL3Type.Native.TIMEUUID, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.TIMEUUID, null); + + for (int i = 0; i < 20; i++) + { + UUID v = UUID.randomUUID(); + addNativeValue(v.toString(), CQL3Type.Native.UUID, UUIDType.instance.decompose(v)); + } + addNativeValue("null", CQL3Type.Native.UUID, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.UUID, null); + + for (int i = 0; i < 20; i++) + { + ByteBuffer v = randBytes(); + addNativeValue("0x" + BytesSerializer.instance.toString(v), CQL3Type.Native.BLOB, BytesType.instance.decompose(v)); + } + addNativeValue("0x", CQL3Type.Native.BLOB, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.BLOB, null); + + for (int i = 0; i < 20; i++) + { + InetAddress v; + try + { + v = InetAddress.getByAddress(new byte[]{ randByte(), randByte(), randByte(), randByte() }); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + addNativeValue(v.getHostAddress(), CQL3Type.Native.INET, InetAddressSerializer.instance.serialize(v)); + } + addNativeValue("null", CQL3Type.Native.INET, ByteBufferUtil.EMPTY_BYTE_BUFFER); + addNativeValue("null", CQL3Type.Native.INET, null); + } + + @Test + public void testNative() + { + // test each native type against each supported protocol version (although it doesn't make sense to + // iterate through all protocol versions as of C* 3.0). + + for (int version = Server.MIN_SUPPORTED_VERSION; version <= Server.CURRENT_VERSION; version++) + { + for (Map.Entry> entry : nativeTypeValues.entrySet()) + { + for (Value value : entry.getValue()) + { + compareCqlLiteral(version, value); + } + } + } + } + + @Test + public void testCollectionWithNatives() + { + // test 100 collections with varying element/key/value types against each supported protocol version, + // type of collection is randomly chosen + + for (int version = Server.MIN_SUPPORTED_VERSION; version <= Server.CURRENT_VERSION; version++) + { + for (int n = 0; n < 100; n++) + { + Value value = generateCollectionValue(version, randomCollectionType(0), true); + compareCqlLiteral(version, value); + } + } + } + + @Test + public void testCollectionNullAndEmpty() + { + for (int version = Server.MIN_SUPPORTED_VERSION; version <= Server.CURRENT_VERSION; version++) + { + // empty, frozen collections + Value value = new Value("[]", ListType.getInstance(UTF8Type.instance, false).asCQL3Type(), ByteBufferUtil.EMPTY_BYTE_BUFFER); + compareCqlLiteral(version, value); + value = new Value("{}", SetType.getInstance(UTF8Type.instance, false).asCQL3Type(), ByteBufferUtil.EMPTY_BYTE_BUFFER); + compareCqlLiteral(version, value); + value = new Value("{}", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false).asCQL3Type(), ByteBufferUtil.EMPTY_BYTE_BUFFER); + compareCqlLiteral(version, value); + + // empty, non-frozen collections + value = new Value("[]", ListType.getInstance(UTF8Type.instance, true).asCQL3Type(), ByteBufferUtil.EMPTY_BYTE_BUFFER); + compareCqlLiteral(version, value); + value = new Value("{}", SetType.getInstance(UTF8Type.instance, true).asCQL3Type(), ByteBufferUtil.EMPTY_BYTE_BUFFER); + compareCqlLiteral(version, value); + value = new Value("{}", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, true).asCQL3Type(), ByteBufferUtil.EMPTY_BYTE_BUFFER); + compareCqlLiteral(version, value); + + // null, frozen collections + value = new Value("null", ListType.getInstance(UTF8Type.instance, false).asCQL3Type(), null); + compareCqlLiteral(version, value); + value = new Value("null", SetType.getInstance(UTF8Type.instance, false).asCQL3Type(), null); + compareCqlLiteral(version, value); + value = new Value("null", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false).asCQL3Type(), null); + compareCqlLiteral(version, value); + + // null, non-frozen collections + value = new Value("[]", ListType.getInstance(UTF8Type.instance, true).asCQL3Type(), null); + compareCqlLiteral(version, value); + value = new Value("{}", SetType.getInstance(UTF8Type.instance, true).asCQL3Type(), null); + compareCqlLiteral(version, value); + value = new Value("{}", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, true).asCQL3Type(), null); + compareCqlLiteral(version, value); + } + } + + @Test + public void testTupleWithNatives() + { + // test 100 tuples with varying element/key/value types against each supported protocol version + + for (int version = Server.MIN_SUPPORTED_VERSION; version <= Server.CURRENT_VERSION; version++) + { + for (int n = 0; n < 100; n++) + { + Value value = generateTupleValue(version, randomTupleType(0), true); + compareCqlLiteral(version, value); + } + } + } + + @Test + public void testUserDefinedWithNatives() + { + // test 100 UDTs with varying element/key/value types against each supported protocol version + + for (int version = Server.MIN_SUPPORTED_VERSION; version <= Server.CURRENT_VERSION; version++) + { + for (int n = 0; n < 100; n++) + { + Value value = generateUserDefinedValue(version, randomUserType(0), true); + compareCqlLiteral(version, value); + } + } + } + + @Test + public void testNested() + { + // This is the "nice" part of this unit test - it tests (probably) nested type structures + // like 'tuple, tuple, user>' or 'map, set>' with + // random types against each supported protocol version. + + for (int version = Server.MIN_SUPPORTED_VERSION; version <= Server.CURRENT_VERSION; version++) + { + for (int n = 0; n < 100; n++) + { + Value value = randomNested(version); + compareCqlLiteral(version, value); + } + } + } + + static void compareCqlLiteral(int version, Value value) + { + ByteBuffer buffer = value.value != null ? value.value.duplicate() : null; + String msg = "Failed to get expected value for type " + value.cql3Type + " / " + value.cql3Type.getType() + " with protocol-version " + version + " expected:\"" + value.expected + '"'; + try + { + assertEquals(msg, + value.expected, + value.cql3Type.asCQLLiteral(buffer, version)); + } + catch (RuntimeException e) + { + throw new RuntimeException(msg, e); + } + } + + static Value randomNested(int version) + { + AbstractType type = randomNestedType(2); + + return generateAnyValue(version, type.asCQL3Type()); + } + + /** + * Generates type of randomly nested type structures. + */ + static AbstractType randomNestedType(int level) + { + if (level == 0) + return randomNativeType(); + switch (randInt(level == 2 ? 3 : 4)) + { + case 0: + return randomCollectionType(level - 1); + case 1: + return randomTupleType(level - 1); + case 2: + return randomUserType(level - 1); + case 3: + return randomNativeType(); + } + throw new AssertionError(); + } + + static Value generateCollectionValue(int version, CollectionType collectionType, boolean allowNull) + { + StringBuilder expected = new StringBuilder(); + ByteBuffer buffer; + + if (allowNull && randBool(0.05d)) + { + // generate 'null' collection + if (collectionType.isMultiCell()) + { + switch (collectionType.kind) + { + case LIST: + expected.append("[]"); + break; + case SET: + case MAP: + expected.append("{}"); + break; + } + } + else + { + expected.append("null"); + } + buffer = null; + } + else + { + int size = randInt(20); + + CQL3Type elements; + CQL3Type values = null; + char bracketOpen; + char bracketClose; + switch (collectionType.kind) + { + case LIST: + elements = ((ListType) collectionType).getElementsType().asCQL3Type(); + bracketOpen = '['; + bracketClose = ']'; + break; + case SET: + elements = ((SetType) collectionType).getElementsType().asCQL3Type(); + bracketOpen = '{'; + bracketClose = '}'; + break; + case MAP: + elements = ((MapType) collectionType).getKeysType().asCQL3Type(); + values = ((MapType) collectionType).getValuesType().asCQL3Type(); + bracketOpen = '{'; + bracketClose = '}'; + break; + default: + throw new AssertionError(); + } + + expected.append(bracketOpen); + Collection buffers = new ArrayList<>(); + Set added = new HashSet<>(); + for (int i = 0; i < size; i++) + { + Value el = generateAnyValue(version, elements); + if (!added.add(el.value)) + continue; + + buffers.add(el.value.duplicate()); + if (expected.length() > 1) + expected.append(", "); + el.cql3Type.toCQLLiteral(el.value, version, expected); + + if (collectionType.kind == CollectionType.Kind.MAP) + { + // add map value + el = generateAnyValue(version, values); + buffers.add(el.value.duplicate()); + expected.append(": "); + el.cql3Type.toCQLLiteral(el.value, version, expected); + } + } + expected.append(bracketClose); + buffer = CollectionSerializer.pack(buffers, added.size(), version); + } + + return new Value(expected.toString(), collectionType.asCQL3Type(), buffer); + } + + /** + * Generates a value for any type or type structure. + */ + static Value generateAnyValue(int version, CQL3Type type) + { + if (type instanceof CQL3Type.Native) + return generateNativeValue(type, false); + if (type instanceof CQL3Type.Tuple) + return generateTupleValue(version, (TupleType) type.getType(), false); + if (type instanceof CQL3Type.UserDefined) + return generateUserDefinedValue(version, (UserType) type.getType(), false); + if (type instanceof CQL3Type.Collection) + return generateCollectionValue(version, (CollectionType) type.getType(), false); + throw new AssertionError(); + } + + static Value generateTupleValue(int version, TupleType tupleType, boolean allowNull) + { + StringBuilder expected = new StringBuilder(); + ByteBuffer buffer; + + if (allowNull && randBool(0.05d)) + { + // generate 'null' collection + expected.append("null"); + buffer = null; + } + else + { + expected.append('('); + + // # of fields in this value + int fields = tupleType.size(); + if (randBool(0.2d)) + fields = randInt(fields); + + ByteBuffer[] buffers = new ByteBuffer[fields]; + for (int i = 0; i < fields; i++) + { + AbstractType fieldType = tupleType.type(i); + + if (i > 0) + expected.append(", "); + + if (allowNull && randBool(.1)) + { + expected.append("null"); + continue; + } + + Value value = generateAnyValue(version, fieldType.asCQL3Type()); + expected.append(value.expected); + buffers[i] = value.value.duplicate(); + } + expected.append(')'); + buffer = TupleType.buildValue(buffers); + } + + return new Value(expected.toString(), tupleType.asCQL3Type(), buffer); + } + + static Value generateUserDefinedValue(int version, UserType userType, boolean allowNull) + { + StringBuilder expected = new StringBuilder(); + ByteBuffer buffer; + + if (allowNull && randBool(0.05d)) + { + // generate 'null' collection + expected.append("null"); + buffer = null; + } + else + { + expected.append('{'); + + // # of fields in this value + int fields = userType.size(); + if (randBool(0.2d)) + fields = randInt(fields); + + ByteBuffer[] buffers = new ByteBuffer[fields]; + for (int i = 0; i < fields; i++) + { + AbstractType fieldType = userType.type(i); + + if (i > 0) + expected.append(", "); + + expected.append(ColumnIdentifier.maybeQuote(userType.fieldNameAsString(i))); + expected.append(": "); + + if (randBool(.1)) + { + expected.append("null"); + continue; + } + + Value value = generateAnyValue(version, fieldType.asCQL3Type()); + expected.append(value.expected); + buffers[i] = value.value.duplicate(); + } + expected.append('}'); + buffer = TupleType.buildValue(buffers); + } + + return new Value(expected.toString(), userType.asCQL3Type(), buffer); + } + + static Value generateNativeValue(CQL3Type type, boolean allowNull) + { + List values = nativeTypeValues.get(type); + assert values != null : type.toString() + " needs to be defined"; + while (true) + { + Value v = values.get(randInt(values.size())); + if (allowNull || v.value != null) + return v; + } + } + + static CollectionType randomCollectionType(int level) + { + CollectionType.Kind kind = CollectionType.Kind.values()[randInt(CollectionType.Kind.values().length)]; + switch (kind) + { + case LIST: + case SET: + return ListType.getInstance(randomNestedType(level), randBool()); + case MAP: + return MapType.getInstance(randomNestedType(level), randomNestedType(level), randBool()); + } + throw new AssertionError(); + } + + static TupleType randomTupleType(int level) + { + int typeCount = 2 + randInt(5); + List> types = new ArrayList<>(); + for (int i = 0; i < typeCount; i++) + types.add(randomNestedType(level)); + return new TupleType(types); + } + + static UserType randomUserType(int level) + { + int typeCount = 2 + randInt(5); + List names = new ArrayList<>(); + List> types = new ArrayList<>(); + for (int i = 0; i < typeCount; i++) + { + names.add(UTF8Type.instance.fromString('f' + randLetters(i))); + types.add(randomNestedType(level)); + } + return new UserType("ks", UTF8Type.instance.fromString("u" + randInt(1000000)), names, types); + } + + // + // Following methods are just helper methods. Mostly to generate many kinds of random values. + // + + private static String randLetters(int len) + { + StringBuilder sb = new StringBuilder(len); + while (len-- > 0) + { + int i = randInt(52); + if (i < 26) + sb.append((char) ('A' + i)); + else + sb.append((char) ('a' + i - 26)); + } + return sb.toString(); + } + + static AbstractType randomNativeType() + { + while (true) + { + CQL3Type.Native t = CQL3Type.Native.values()[randInt(CQL3Type.Native.values().length)]; + if (t != CQL3Type.Native.EMPTY) + return t.getType(); + } + } + + static boolean randBool() + { + return randBool(0.5d); + } + + static boolean randBool(double probability) + { + return ThreadLocalRandom.current().nextDouble() < probability; + } + + static long randLong() + { + return ThreadLocalRandom.current().nextLong(); + } + + static long randLong(long max) + { + return ThreadLocalRandom.current().nextLong(max); + } + + static int randInt() + { + return ThreadLocalRandom.current().nextInt(); + } + + static int randInt(int max) + { + return ThreadLocalRandom.current().nextInt(max); + } + + static short randShort() + { + return (short) ThreadLocalRandom.current().nextInt(); + } + + static byte randByte() + { + return (byte) ThreadLocalRandom.current().nextInt(); + } + + static double randDouble() + { + return ThreadLocalRandom.current().nextDouble(); + } + + static float randFloat() + { + return ThreadLocalRandom.current().nextFloat(); + } + + static String randString(boolean ascii) + { + int l = randInt(20); + StringBuilder sb = new StringBuilder(l); + for (int i = 0; i < l; i++) + { + if (randBool(.05)) + sb.append('\''); + else + { + char c = (char) (ascii ? randInt(128) : randShort()); + sb.append(c); + } + } + return UTF8Serializer.instance.deserialize(UTF8Serializer.instance.serialize(sb.toString())); + } + + static ByteBuffer randBytes() + { + int l = randInt(20); + byte[] v = new byte[l]; + for (int i = 0; i < l; i++) + { + v[i] = randByte(); + } + return ByteBuffer.wrap(v); + } + + private static String quote(String v) + { + return '\'' + QUOTE.matcher(v).replaceAll("''") + '\''; + } +} diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java index 4341258d19..1a532ac895 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java @@ -809,6 +809,9 @@ public class AggregationTest extends CQLTester "FINALFUNC " + shortFunctionName(fFinal) + " " + "INITCOND 42"); + assertRows(execute("SELECT initcond FROM system_schema.aggregates WHERE keyspace_name=? AND aggregate_name=?", KEYSPACE, shortFunctionName(a)), + row("42")); + // 42 + 1 + 2 + 3 = 48 assertRows(execute("SELECT " + a + "(b) FROM %s"), row("48")); @@ -896,6 +899,9 @@ public class AggregationTest extends CQLTester "FINALFUNC " + shortFunctionName(fFinal) + " " + "INITCOND (0, 0)"); + assertRows(execute("SELECT initcond FROM system_schema.aggregates WHERE keyspace_name=? AND aggregate_name=?", KEYSPACE, shortFunctionName(a)), + row("(0, 0)")); + // 1 + 2 + 3 = 6 / 3 = 2 assertRows(execute("SELECT " + a + "(b) FROM %s"), row(2d)); @@ -1364,6 +1370,9 @@ public class AggregationTest extends CQLTester "FINALFUNC " + parseFunctionName(fFinal).name + ' ' + "INITCOND null"); + assertRows(execute("SELECT initcond FROM system_schema.aggregates WHERE keyspace_name=? AND aggregate_name=?", KEYSPACE, shortFunctionName(aggregation)), + row(null)); + assertRows(execute("SELECT " + aggregation + "(b) FROM %s"), row(set(7, 8, 9))); @@ -1600,6 +1609,9 @@ public class AggregationTest extends CQLTester "FINALFUNC " + shortFunctionName(fCONf) + ' ' + "INITCOND ''"); + assertRows(execute("SELECT initcond FROM system_schema.aggregates WHERE keyspace_name=? AND aggregate_name=?", KEYSPACE, shortFunctionName(aCON)), + row("''")); + String fRNON = createFunction(KEYSPACE, "text, text", "CREATE FUNCTION %s(a text, b text) " + @@ -1655,6 +1667,9 @@ public class AggregationTest extends CQLTester "STYPE list " + "INITCOND [ ]"); + assertRows(execute("SELECT initcond FROM system_schema.aggregates WHERE keyspace_name=? AND aggregate_name=?", KEYSPACE, shortFunctionName(a)), + row("[]")); + createTable("CREATE TABLE %s (a int primary key, b int)"); execute("INSERT INTO %s (a, b) VALUES (1, 1)"); execute("INSERT INTO %s (a, b) VALUES (2, null)"); diff --git a/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java b/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java index ad75fd043d..7d0be4d420 100644 --- a/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java +++ b/test/unit/org/apache/cassandra/schema/LegacySchemaMigratorTest.java @@ -511,13 +511,17 @@ public class LegacySchemaMigratorTest null ); + ByteBuffer twoNullEntries = ByteBuffer.allocate(8); + twoNullEntries.putInt(-1); + twoNullEntries.putInt(-1); + twoNullEntries.flip(); UDAggregate uda2 = UDAggregate.create(udfs, new FunctionName(keyspace, "uda2"), ImmutableList.of(udf2.argTypes().get(1)), udf3.returnType(), udf2.name(), udf3.name(), udf2.argTypes().get(0), - LongType.instance.decompose(0L) + twoNullEntries ); return KeyspaceMetadata.create(keyspace,