diff --git a/CHANGES.txt b/CHANGES.txt index 35f9d355cb..b562e0dc6e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0 + * Added support for type VECTOR (CASSANDRA-18504) * Expose bootstrap and decommission state to nodetool info (CASSANDRA-18555) * Fix SSTabledump errors when dumping data from index (CASSANDRA-17698) * Avoid unnecessary deserialization of terminal arguments when executing CQL functions (CASSANDRA-18566) diff --git a/NEWS.txt b/NEWS.txt index 7027b6e489..11eafa231b 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -71,6 +71,9 @@ using the provided 'sstableupgrade' tool. New features ------------ + - New `VectorType` (cql `vector`) which adds new fixed-length element arrays. See CASSANDRA-18504 + - Removed UDT type migration logic for 3.6+ clusters upgrading to 4.0. If migration has been disabled, it must be + enabled before upgrading to 5.0 if the cluster used UDTs. See CASSANDRA-18504 - Entended max expiration time from 2038-01-19T03:14:06+00:00 to 2106-02-07T06:28:13+00:00 - Added new Mathematical CQL functions: abs, exp, log, log10 and round. - Added a trie-based memtable implementation, which improves memory use, garbage collection efficiency and lookup diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile index c1ce85ffa4..0073ff7ed3 100644 --- a/doc/cql3/CQL.textile +++ b/doc/cql3/CQL.textile @@ -131,7 +131,9 @@ bc(syntax).. ::= '{' ( ( ',' )* )? '}' ::= '[' ( ( ',' )* )? ']' - ::= + ::= '[' ( ( ',' )* )? ']' + + ::= ::= (AND )* ::= '=' ( | | ) @@ -1730,6 +1732,7 @@ bc(syntax).. ::= | | + | | // Used for custom types. The fully-qualified name of a JAVA class ::= ascii @@ -1757,7 +1760,11 @@ bc(syntax).. ::= list '<' '>' | set '<' '>' | map '<' ',' '>' + ::= tuple '<' (',' )* '>' + + ::= vector '<' ',' '>' + p. Note that the native types are keywords and as such are case-insensitive. They are however not reserved ones. p. The following table gives additional informations on the native data types, and on which kind of "constants":#constants each type supports: @@ -1787,6 +1794,8 @@ p. The following table gives additional informations on the native data types, a For more information on how to use the collection types, see the "Working with collections":#collections section below. +For more information on how to use the vector type, see the "Working with vectors":#vectors section below. + h3(#usingtimestamps). Working with timestamps Values of the @timestamp@ type are encoded as 64-bit signed integers representing a number of milliseconds since the standard base time known as "the epoch": January 1 1970 at 00:00:00 GMT. @@ -2006,6 +2015,34 @@ UPDATE plays SET scores = scores - [ 12, 21 ] WHERE id = '123-afde'; // removes As with "maps":#map, TTLs if used only apply to the newly inserted/updated _values_. +h3(#vectors). Working with vectors + +Vectors are fixed-size sequences of non-null values of a certain data type. They use the same literals as lists and it isn't possible to select individual elements of a vector. + +To create a column of type @vector@, use the @vector@ keyword suffixed with the value type and the number of dimensions enclosed in angle brackets. For example: + +bc(sample). +CREATE TABLE plays ( + id text PRIMARY KEY, + game text, + players int, + scores vector +) + +That would create a @vector@ of 3 @int@ values. + +To write a record using @INSERT@, specify the entire vector as a JSON array: + +bc(sample). +INSERT INTO plays (id, game, players, scores) VALUES ('123-afde', 'quake', 3, [17, 4, 2]); + +To update a vector using @UPDATE@, specify the entire vector as a JSON array: + +bc(sample). +UPDATE plays SET players = 5, scores = [ 14, 4, 3 ] WHERE id = '123-afde'; + +It isn't possible to change the individual values of a vector. + h2(#arithmeticOperators). Arithmetic Operators h3(#numberArithmetic). Number Arithmetic diff --git a/doc/modules/cassandra/examples/BNF/term.bnf b/doc/modules/cassandra/examples/BNF/term.bnf index 504c4c40d8..af99b93c08 100644 --- a/doc/modules/cassandra/examples/BNF/term.bnf +++ b/doc/modules/cassandra/examples/BNF/term.bnf @@ -1,5 +1,5 @@ term::= constant | literal | function_call | arithmetic_operation | type_hint | bind_marker -literal::= collection_literal | udt_literal | tuple_literal +literal::= collection_literal | vector_literal | udt_literal | tuple_literal function_call::= identifier '(' [ term (',' term)* ] ')' arithmetic_operation::= '-' term | term ('+' | '-' | '*' | '/' | '%') term type_hint::= '(' cql_type ')' term diff --git a/doc/modules/cassandra/examples/BNF/vector_literal.bnf b/doc/modules/cassandra/examples/BNF/vector_literal.bnf new file mode 100644 index 0000000000..890cf4b389 --- /dev/null +++ b/doc/modules/cassandra/examples/BNF/vector_literal.bnf @@ -0,0 +1 @@ +vector_literal::= '[' [ term (',' term)* ] ']' diff --git a/doc/modules/cassandra/examples/CQL/vector.cql b/doc/modules/cassandra/examples/CQL/vector.cql new file mode 100644 index 0000000000..19366c9348 --- /dev/null +++ b/doc/modules/cassandra/examples/CQL/vector.cql @@ -0,0 +1,12 @@ +CREATE TABLE plays ( + id text PRIMARY KEY, + game text, + players int, + scores vector // A vector of 3 integers +) + +INSERT INTO plays (id, game, players, scores) + VALUES ('123-afde', 'quake', 3, [17, 4, 2]); + +// Replace the existing vector entirely +UPDATE plays SET scores = [ 3, 9, 4] WHERE id = '123-afde'; diff --git a/doc/modules/cassandra/pages/developing/cql/changes.adoc b/doc/modules/cassandra/pages/developing/cql/changes.adoc index 452a182230..9dc3af00cb 100644 --- a/doc/modules/cassandra/pages/developing/cql/changes.adoc +++ b/doc/modules/cassandra/pages/developing/cql/changes.adoc @@ -10,6 +10,7 @@ The following describes the changes in each version of CQL. * Add SELECT_MASKED permission (`18070`) * Add support for using UDFs as masking functions (`18071`) * Adopt snake_case function names, deprecating all previous camelCase or alltogetherwithoutspaces function names (`18037`) +* Add new `vector` data type (`18504`) == 3.4.6 diff --git a/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc index 702ac3a2b0..b6a0966cbf 100644 --- a/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc +++ b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc @@ -115,6 +115,31 @@ where `hex` is an hexadecimal character, e.g. `[0-9a-fA-F]`. For how these constants are typed, see the link:#types[data types section]. +== Terms + +CQL has the notion of a _term_, which denotes the kind of values that +CQL support. Terms are defined by: + +[source, bnf] +---- +include::example$BNF/term.bnf[] +---- + +A term is thus one of: + +* A xref:cql/defintions.adoc#constants[constant] +* A literal for either a xref:cql/types.adoc#collections[collection], a xref:cql/types.adoc#vectors[vector], +a xref:cql/types.adoc#udts[user-defined type] or a xref:cql/types.adoc#tuples[tuple] +* A xref:cql/functions.adoc#cql-functions[function] call, either a xref:cql/functions.adoc#scalar-native-functions[native function] +or a xref:cql/functions.adoc#user-defined-scalar-functions[user-defined function] +* An xref:cql/operators.adoc#arithmetic_operators[arithmetic operation] between terms +* A type hint +* A bind marker, which denotes a variable to be bound at execution time. +See the section on `prepared-statements` for details. A bind marker can +be either anonymous (`?`) or named (`:some_name`). The latter form +provides a more convenient way to refer to the variable for binding it +and should generally be preferred. + ==== Comments A comment in CQL is a line beginning by either double dashes (`--`) or @@ -2955,6 +2980,21 @@ removes all occurrences of 12 and 21 from scores As with link:#map[maps], TTLs if used only apply to the newly inserted/updated _values_. +[[vectors]] +==== Working with vectors + +Vectors are fixed-size sequences of non-null values of a certain data type. They use the same literals as lists. + +You can define, insert and update a vector with: + +[source,cql] +---- +include::example$CQL/vector.cql[] +---- + +Note that it isn't possible to change the individual values of a vector, and it isn't possible to select individual +elements of a vector. + === Functions CQL3 distinguishes between built-in functions (so called `native diff --git a/doc/modules/cassandra/pages/developing/cql/definitions.adoc b/doc/modules/cassandra/pages/developing/cql/definitions.adoc index 14e60487f3..c28179d6cc 100644 --- a/doc/modules/cassandra/pages/developing/cql/definitions.adoc +++ b/doc/modules/cassandra/pages/developing/cql/definitions.adoc @@ -121,7 +121,7 @@ include::example$BNF/term.bnf[] A term is thus one of: * A xref:cql/defintions.adoc#constants[constant] -* A literal for either a xref:cql/types.adoc#collections[collection], +* A literal for either a xref:cql/types.adoc#collections[collection], a xref:cql/types.adoc#vectors[vector], a xref:cql/types.adoc#udts[user-defined type] or a xref:cql/types.adoc#tuples[tuple] * A xref:cql/functions.adoc#cql-functions[function] call, either a xref:cql/functions.adoc#scalar-native-functions[native function] or a xref:cql/functions.adoc#user-defined-scalar-functions[user-defined function] diff --git a/doc/modules/cassandra/pages/developing/cql/types.adoc b/doc/modules/cassandra/pages/developing/cql/types.adoc index c17dc38674..14a6c56d8c 100644 --- a/doc/modules/cassandra/pages/developing/cql/types.adoc +++ b/doc/modules/cassandra/pages/developing/cql/types.adoc @@ -380,6 +380,21 @@ exclusion of conditional write that have their own cost). Lastly, for xref:cql/types.adoc#lists[lists], TTLs only apply to newly inserted values. +[[vectors]] +==== Working with vectors + +Vectors are fixed-size sequences of non-null values of a certain data type. They use the same literals as lists. + +You can define, insert and update a vector with: + +[source,cql] +---- +include::example$CQL/vector.cql[] +---- + +Note that it isn't possible to change the individual values of a vector, and it isn't possible to select individual +elements of a vector. + [[udts]] == User-Defined Types (UDTs) diff --git a/doc/native_protocol_v5.spec b/doc/native_protocol_v5.spec index 17f7368b15..cbc7f9fe9c 100644 --- a/doc/native_protocol_v5.spec +++ b/doc/native_protocol_v5.spec @@ -1207,6 +1207,13 @@ Table of Contents value. Implementors should pad positive values that have a MSB >= 0x80 with a leading 0x00 byte. +5.25 vector + + For a vector of n dimensions of a fixed-length type, a sequence of those n elements. + For a vector with variable-length elements, the size of the elements will preced + each element. Each element is the [bytes] representing the serialized value. The + number of dimensions is not encoded, since it's part of the type definition. + 6. User Defined Types diff --git a/src/antlr/Lexer.g b/src/antlr/Lexer.g index 512a82dc28..a4f8ea715f 100644 --- a/src/antlr/Lexer.g +++ b/src/antlr/Lexer.g @@ -222,6 +222,8 @@ K_MASKED: M A S K E D; K_UNMASK: U N M A S K; K_SELECT_MASKED: S E L E C T '_' M A S K E D; +K_VECTOR: V E C T O R; + // Case-insensitive alpha characters fragment A: ('a'|'A'); fragment B: ('b'|'B'); diff --git a/src/antlr/Parser.g b/src/antlr/Parser.g index 029d7fcde7..65ed92a397 100644 --- a/src/antlr/Parser.g +++ b/src/antlr/Parser.g @@ -376,7 +376,7 @@ selectionTypeHint returns [Selectable.Raw s] selectionList returns [Selectable.Raw s] @init { List l = new ArrayList<>(); } - @after { $s = new Selectable.WithList.Raw(l); } + @after { $s = new Selectable.WithArrayLiteral.Raw(l); } : '[' ( t1=unaliasedSelector { l.add(t1); } ( ',' tn=unaliasedSelector { l.add(tn); } )* )? ']' ; @@ -1525,8 +1525,8 @@ collectionLiteral returns [Term.Raw value] listLiteral returns [Term.Raw value] @init {List l = new ArrayList();} - @after {$value = new Lists.Literal(l);} - : '[' ( t1=term { l.add(t1); } ( ',' tn=term { l.add(tn); } )* )? ']' { $value = new Lists.Literal(l); } + @after {$value = new ArrayLiteral(l);} + : '[' ( t1=term { l.add(t1); } ( ',' tn=term { l.add(tn); } )* )? ']' { $value = new ArrayLiteral(l); } ; usertypeLiteral returns [UserTypes.Literal ut] @@ -1802,6 +1802,7 @@ comparatorType returns [CQL3Type.Raw t] : n=native_type { $t = CQL3Type.Raw.from(n); } | c=collection_type { $t = c; } | tt=tuple_type { $t = tt; } + | vc=vector_type { $t = vc; } | id=userTypeName { $t = CQL3Type.Raw.userType(id); } | K_FROZEN '<' f=comparatorType '>' { @@ -1866,6 +1867,11 @@ tuple_type returns [CQL3Type.Raw t] : K_TUPLE '<' t1=comparatorType { types.add(t1); } (',' tn=comparatorType { types.add(tn); })* '>' ; +vector_type returns [CQL3Type.Raw vt] + : K_VECTOR '<' t1=comparatorType ',' d=INTEGER '>' + { $vt = CQL3Type.Raw.vector(t1, Integer.parseInt($d.text)); } + ; + username : IDENT | STRING_LITERAL @@ -1959,5 +1965,6 @@ basic_unreserved_keyword returns [String str] | K_MASKED | K_UNMASK | K_SELECT_MASKED + | K_VECTOR ) { $str = $k.text; } ; diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 843511f3b0..5e4e60939f 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -217,7 +217,7 @@ public class DatabaseDescriptor private static StartupChecksOptions startupChecksOptions; private static ImmutableMap> sstableFormats; - private static SSTableFormat selectedSSTableFormat; + private static volatile SSTableFormat selectedSSTableFormat; private static Function commitLogSegmentMgrProvider = c -> DatabaseDescriptor.isCDCEnabled() ? new CommitLogSegmentManagerCDC(c, DatabaseDescriptor.getCommitLogLocation()) @@ -4758,6 +4758,12 @@ public class DatabaseDescriptor return Objects.requireNonNull(selectedSSTableFormat, "Forgot to initialize DatabaseDescriptor?"); } + @VisibleForTesting + public static void setSelectedSSTableFormat(SSTableFormat format) + { + selectedSSTableFormat = Objects.requireNonNull(format); + } + public static boolean getDynamicDataMaskingEnabled() { return conf.dynamic_data_masking_enabled; diff --git a/src/java/org/apache/cassandra/cql3/ArrayLiteral.java b/src/java/org/apache/cassandra/cql3/ArrayLiteral.java new file mode 100644 index 0000000000..76392fdc01 --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/ArrayLiteral.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3; + +import java.util.List; + +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.VectorType; +import org.apache.cassandra.exceptions.InvalidRequestException; + +/** + * Represents {@code [a, b, c, d]} in CQL. Mutliple {@link org.apache.cassandra.db.marshal.AbstractType} use array literals, + * so this class is meant to act as a proxy once the real type is known. + */ +public class ArrayLiteral extends Term.Raw +{ + private final List elements; + + public ArrayLiteral(List elements) + { + this.elements = elements; + } + + private Term.Raw forReceiver(ColumnSpecification receiver) + { + AbstractType type = receiver.type.unwrap(); + if (type instanceof VectorType) + return new Vectors.Literal(elements); + if (type instanceof ListType) + return new Lists.Literal(elements); + throw new InvalidRequestException(String.format("Unexpected receiver type '%s'; only list and vector are expected", type.asCQL3Type())); + } + + @Override + public Term prepare(String keyspace, ColumnSpecification receiver) throws InvalidRequestException + { + return forReceiver(receiver).prepare(keyspace, receiver); + } + + @Override + public AssignmentTestable.TestResult testAssignment(String keyspace, ColumnSpecification receiver) + { + return forReceiver(receiver).testAssignment(keyspace, receiver); + } + + @Override + public AbstractType getExactTypeIfKnown(String keyspace) + { + // without knowing the receiver, can't say if this is a ListType or a VectorType + return null; + } + + @Override + public String getText() + { + return Lists.listToString(elements, Term.Raw::getText); + } +} diff --git a/src/java/org/apache/cassandra/cql3/CQL3Type.java b/src/java/org/apache/cassandra/cql3/CQL3Type.java index 1dd641a35d..57474103bb 100644 --- a/src/java/org/apache/cassandra/cql3/CQL3Type.java +++ b/src/java/org/apache/cassandra/cql3/CQL3Type.java @@ -31,10 +31,10 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.Types; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; import static java.util.stream.Collectors.toList; @@ -59,9 +59,24 @@ public interface CQL3Type * Generates CQL literal from a binary value of this type. * @param bytes the value to convert to a CQL literal. This value must be * serialized with {@code version} of the native protocol. - * @param version the native protocol version in which {@code buffer} is encoded. */ - String toCQLLiteral(ByteBuffer bytes, ProtocolVersion version); + String toCQLLiteral(ByteBuffer bytes); + + /** + * Generates a binary value for the CQL literal of this type + */ + default ByteBuffer fromCQLLiteral(String literal) + { + return fromCQLLiteral(SchemaConstants.DUMMY_KEYSPACE_OR_TABLE_NAME, literal); + } + + /** + * Generates a binary value for the CQL literal of this type + */ + default ByteBuffer fromCQLLiteral(String keyspaceName, String literal) + { + return Terms.asBytes(keyspaceName, literal, getType()); + } public enum Native implements CQL3Type { @@ -107,7 +122,7 @@ public interface CQL3Type * {@link org.apache.cassandra.serializers.TypeSerializer#toString(Object)} * {@link org.apache.cassandra.serializers.TypeSerializer#deserialize(ByteBuffer)} implementations. */ - public String toCQLLiteral(ByteBuffer buffer, ProtocolVersion version) + public String toCQLLiteral(ByteBuffer buffer) { return type.getSerializer().toCQLLiteral(buffer); } @@ -139,10 +154,10 @@ public interface CQL3Type } @Override - public String toCQLLiteral(ByteBuffer buffer, ProtocolVersion version) + public String toCQLLiteral(ByteBuffer buffer) { // *always* use the 'blob' syntax to express custom types in CQL - return Native.BLOB.toCQLLiteral(buffer, version); + return Native.BLOB.toCQLLiteral(buffer); } @Override @@ -188,7 +203,7 @@ public interface CQL3Type } @Override - public String toCQLLiteral(ByteBuffer buffer, ProtocolVersion version) + public String toCQLLiteral(ByteBuffer buffer) { if (buffer == null) return "null"; @@ -203,25 +218,25 @@ public interface CQL3Type case LIST: CQL3Type elements = ((ListType) type).getElementsType().asCQL3Type(); target.append('['); - generateSetOrListCQLLiteral(buffer, version, target, size, elements); + generateSetOrListCQLLiteral(buffer, target, size, elements); target.append(']'); break; case SET: elements = ((SetType) type).getElementsType().asCQL3Type(); target.append('{'); - generateSetOrListCQLLiteral(buffer, version, target, size, elements); + generateSetOrListCQLLiteral(buffer, target, size, elements); target.append('}'); break; case MAP: target.append('{'); - generateMapCQLLiteral(buffer, version, target, size); + generateMapCQLLiteral(buffer, target, size); target.append('}'); break; } return target.toString(); } - private void generateMapCQLLiteral(ByteBuffer buffer, ProtocolVersion version, StringBuilder target, int size) + private void generateMapCQLLiteral(ByteBuffer buffer, StringBuilder target, int size) { CQL3Type keys = ((MapType) type).getKeysType().asCQL3Type(); CQL3Type values = ((MapType) type).getValuesType().asCQL3Type(); @@ -232,15 +247,15 @@ public interface CQL3Type target.append(", "); ByteBuffer element = CollectionSerializer.readValue(buffer, ByteBufferAccessor.instance, offset); offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance); - target.append(keys.toCQLLiteral(element, version)); + target.append(keys.toCQLLiteral(element)); target.append(": "); element = CollectionSerializer.readValue(buffer, ByteBufferAccessor.instance, offset); offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance); - target.append(values.toCQLLiteral(element, version)); + target.append(values.toCQLLiteral(element)); } } - private static void generateSetOrListCQLLiteral(ByteBuffer buffer, ProtocolVersion version, StringBuilder target, int size, CQL3Type elements) + private static void generateSetOrListCQLLiteral(ByteBuffer buffer, StringBuilder target, int size, CQL3Type elements) { int offset = 0; for (int i = 0; i < size; i++) @@ -249,7 +264,7 @@ public interface CQL3Type target.append(", "); ByteBuffer element = CollectionSerializer.readValue(buffer, ByteBufferAccessor.instance, offset); offset += CollectionSerializer.sizeOfValue(element, ByteBufferAccessor.instance); - target.append(elements.toCQLLiteral(element, version)); + target.append(elements.toCQLLiteral(element)); } } @@ -327,7 +342,7 @@ public interface CQL3Type } @Override - public String toCQLLiteral(ByteBuffer buffer, ProtocolVersion version) + public String toCQLLiteral(ByteBuffer buffer) { if (buffer == null) return "null"; @@ -364,7 +379,7 @@ public interface CQL3Type throw new MarshalException(String.format("Not enough bytes to read %dth field %s", i, type.fieldName(i))); ByteBuffer field = ByteBufferUtil.readBytes(buffer, size); - target.append(type.fieldType(i).asCQL3Type().toCQLLiteral(field, version)); + target.append(type.fieldType(i).asCQL3Type().toCQLLiteral(field)); } target.append('}'); return target.toString(); @@ -415,7 +430,7 @@ public interface CQL3Type return type; } - public String toCQLLiteral(ByteBuffer buffer, ProtocolVersion version) + public String toCQLLiteral(ByteBuffer buffer) { if (buffer == null) return "null"; @@ -451,7 +466,7 @@ public interface CQL3Type throw new MarshalException(String.format("Not enough bytes to read %dth component", i)); ByteBuffer field = ByteBufferUtil.readBytes(buffer, size); - target.append(type.type(i).asCQL3Type().toCQLLiteral(field, version)); + target.append(type.type(i).asCQL3Type().toCQLLiteral(field)); } target.append(')'); return target.toString(); @@ -499,6 +514,55 @@ public interface CQL3Type } } + public static class Vector implements CQL3Type + { + private final VectorType type; + + public Vector(VectorType type) + { + this.type = type; + } + + public Vector(AbstractType elementType, int dimensions) + { + this.type = VectorType.getInstance(elementType, dimensions); + } + + @Override + public VectorType getType() + { + return type; + } + + @Override + public String toCQLLiteral(ByteBuffer buffer) + { + if (type.isNull(buffer)) + return "null"; + buffer = buffer.duplicate(); + CQL3Type elementType = type.elementType.asCQL3Type(); + List values = getType().split(buffer); + StringBuilder sb = new StringBuilder(); + sb.append('['); + for (int i = 0; i < values.size(); i++) + { + if (i > 0) + sb.append(", "); + sb.append(elementType.toCQLLiteral(values.get(i))); + } + sb.append(']'); + return sb.toString(); + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append("vector<").append(type.elementType.asCQL3Type()).append(", ").append(type.dimension).append('>'); + return sb.toString(); + } + } + // For UserTypes, we need to know the current keyspace to resolve the // actual type used, so Raw is a "not yet prepared" CQL3Type. public abstract class Raw @@ -537,6 +601,16 @@ public interface CQL3Type return false; } + public boolean isImplicitlyFrozen() + { + return isTuple() || isVector(); + } + + public boolean isVector() + { + return false; + } + public String keyspace() { return null; @@ -598,6 +672,11 @@ public interface CQL3Type return new RawTuple(ts); } + public static Raw vector(CQL3Type.Raw t, int dimension) + { + return new RawVector(t, dimension); + } + private static class RawType extends Raw { private final CQL3Type type; @@ -689,8 +768,7 @@ public interface CQL3Type { assert values != null : "Got null values type for a collection"; - // skip if innerType is tuple, since tuple is implicitly forzen - if (!frozen && values.supportsFreezing() && !values.frozen && !values.isTuple()) + if (!frozen && values.supportsFreezing() && !values.frozen) throwNestedNonFrozenError(values); // we represent supercolumns as maps, internally, and we do allow counters in supercolumns. Thus, @@ -753,6 +831,44 @@ public interface CQL3Type } } + private static class RawVector extends Raw + { + private final CQL3Type.Raw element; + private final int dimention; + + private RawVector(Raw element, int dimention) + { + super(true); + this.element = element; + this.dimention = dimention; + } + + @Override + public boolean isVector() + { + return true; + } + + @Override + public boolean supportsFreezing() + { + return true; + } + + @Override + public Raw freeze() + { + return this; + } + + @Override + public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException + { + CQL3Type type = element.prepare(keyspace, udts); + return new Vector(type.getType(), dimention); + } + } + private static class RawUT extends Raw { private final UTName name; diff --git a/src/java/org/apache/cassandra/cql3/Terms.java b/src/java/org/apache/cassandra/cql3/Terms.java index 31b3fd0bd4..c6e110afe0 100644 --- a/src/java/org/apache/cassandra/cql3/Terms.java +++ b/src/java/org/apache/cassandra/cql3/Terms.java @@ -24,6 +24,7 @@ import org.apache.cassandra.cql3.Term.MultiItemTerminal; import org.apache.cassandra.cql3.Term.Terminal; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.transport.ProtocolVersion; /** @@ -263,7 +264,7 @@ public interface Terms public static ByteBuffer asBytes(String keyspace, String term, AbstractType type) { - ColumnSpecification receiver = new ColumnSpecification(keyspace, "--dummy--", new ColumnIdentifier("(dummy)", true), type); + ColumnSpecification receiver = new ColumnSpecification(keyspace, SchemaConstants.DUMMY_KEYSPACE_OR_TABLE_NAME, new ColumnIdentifier("(dummy)", true), type); Term.Raw rawTerm = CQLFragmentParser.parseAny(CqlParser::term, term, "CQL term"); return rawTerm.prepare(keyspace, receiver).bindAndGet(QueryOptions.DEFAULT); } diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java index b73207ed93..1bac792941 100644 --- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java +++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java @@ -484,6 +484,12 @@ public abstract class UntypedResultSet implements Iterable return getFrozenMap(column, UTF8Type.instance, UTF8Type.instance); } + public List getVector(String column, AbstractType elementType, int dimension) + { + ByteBuffer raw = data.get(column); + return raw == null ? null : VectorType.getInstance(elementType, dimension).compose(raw); + } + public List getColumns() { return columns; diff --git a/src/java/org/apache/cassandra/cql3/Vectors.java b/src/java/org/apache/cassandra/cql3/Vectors.java new file mode 100644 index 0000000000..2c53b9e716 --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/Vectors.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.VectorType; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.ByteBufferUtil; + +public class Vectors +{ + private Vectors() {} + + private static AbstractType elementsType(AbstractType type) + { + return ((VectorType) type.unwrap()).getElementsType(); + } + + private static ColumnSpecification valueSpecOf(ColumnSpecification column) + { + return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ')', true), elementsType(column.type)); + } + + /** + * Tests that the vector with the specified elements can be assigned to the specified column. + * + * @param receiver the receiving column + * @param elements the vector elements + */ + public static AssignmentTestable.TestResult testVectorAssignment(ColumnSpecification receiver, + List elements) + { + if (!receiver.type.isVector()) + return AssignmentTestable.TestResult.NOT_ASSIGNABLE; + + // If there is no elements, we can't say it's an exact match (an empty vector if fundamentally polymorphic). + if (elements.isEmpty()) + return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE; + + ColumnSpecification valueSpec = valueSpecOf(receiver); + return AssignmentTestable.TestResult.testAll(receiver.ksName, valueSpec, elements); + } + + /** + * Returns the exact VectorType from the items if it can be known. + * + * @param items the items mapped to the vector elements + * @param mapper the mapper used to retrieve the element types from the items + * @return the exact VectorType from the items if it can be known or null + */ + public static VectorType getExactVectorTypeIfKnown(List items, + java.util.function.Function> mapper) + { + // TODO - this doesn't feel right... if you are dealing with a literal then the value is `null`, so we will ignore + // if there are multiple times, we randomly select the first? This logic matches Lists.getExactListTypeIfKnown but feels flawed + Optional> type = items.stream().map(mapper).filter(Objects::nonNull).findFirst(); + return type.isPresent() ? VectorType.getInstance(type.get(), items.size()) : null; + } + + public static VectorType getPreferredCompatibleType(List items, + java.util.function.Function> mapper) + { + Set> types = items.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toSet()); + AbstractType type = AssignmentTestable.getCompatibleTypeIfKnown(types); + return type == null ? null : VectorType.getInstance(type, items.size()); + } + + public static class Literal extends Term.Raw + { + private final List elements; + + public Literal(List elements) + { + this.elements = elements; + } + + @Override + public TestResult testAssignment(String keyspace, ColumnSpecification receiver) + { + if (!receiver.type.isVector()) + return AssignmentTestable.TestResult.NOT_ASSIGNABLE; + VectorType type = (VectorType) receiver.type; + if (elements.size() != type.dimension) + return AssignmentTestable.TestResult.NOT_ASSIGNABLE; + ColumnSpecification valueSpec = valueSpecOf(receiver); + return AssignmentTestable.TestResult.testAll(receiver.ksName, valueSpec, elements); + } + + @Override + public Term prepare(String keyspace, ColumnSpecification receiver) throws InvalidRequestException + { + if (!receiver.type.isVector()) + throw new InvalidRequestException(String.format("Invalid vector literal for %s of type %s", receiver.name, receiver.type.asCQL3Type())); + VectorType type = (VectorType) receiver.type; + if (elements.size() != type.dimension) + throw new InvalidRequestException(String.format("Invalid vector literal for %s of type %s; expected %d elements, but given %d", receiver.name, receiver.type.asCQL3Type(), type.dimension, elements.size())); + + ColumnSpecification valueSpec = valueSpecOf(receiver); + List values = new ArrayList<>(elements.size()); + boolean allTerminal = true; + for (Term.Raw rt : elements) + { + if (!rt.testAssignment(keyspace, valueSpec).isAssignable()) + throw new InvalidRequestException(String.format("Invalid vector literal for %s: value %s is not of type %s", receiver.name, rt, valueSpec.type.asCQL3Type())); + + Term t = rt.prepare(keyspace, valueSpec); + + if (t instanceof Term.NonTerminal) + allTerminal = false; + + values.add(t); + } + DelayedValue value = new DelayedValue<>(type, values); + return allTerminal ? value.bind(QueryOptions.DEFAULT) : value; + } + + @Override + public String getText() + { + return Lists.listToString(elements, Term.Raw::getText); + } + + @Override + public AbstractType getExactTypeIfKnown(String keyspace) + { + return getExactVectorTypeIfKnown(elements, e -> e.getExactTypeIfKnown(keyspace)); + } + + @Override + public AbstractType getCompatibleTypeIfKnown(String keyspace) + { + return getPreferredCompatibleType(elements, e -> e.getCompatibleTypeIfKnown(keyspace)); + } + } + + public static class Value extends Term.MultiItemTerminal + { + public final VectorType type; + public final List elements; + + public Value(VectorType type, List elements) + { + this.type = type; + this.elements = elements; + } + + public ByteBuffer get(ProtocolVersion version) + { + return type.decomposeRaw(elements); + } + + public List getElements() + { + return elements; + } + } + + /** + * Basically similar to a Value, but with some non-pure function (that need + * to be evaluated at execution time) in it. + */ + public static class DelayedValue extends Term.NonTerminal + { + private final VectorType type; + private final List elements; + + public DelayedValue(VectorType type, List elements) + { + this.type = type; + this.elements = elements; + } + + public boolean containsBindMarker() + { + return elements.stream().anyMatch(Term::containsBindMarker); + } + + public void collectMarkerSpecification(VariableSpecifications boundNames) + { + elements.forEach(t -> t.collectMarkerSpecification(boundNames)); + } + + public Terminal bind(QueryOptions options) throws InvalidRequestException + { + List buffers = new ArrayList<>(elements.size()); + for (Term t : elements) + { + ByteBuffer bytes = t.bindAndGet(options); + + if (bytes == null || bytes == ByteBufferUtil.UNSET_BYTE_BUFFER || type.elementType.isNull(bytes)) + throw new InvalidRequestException("null is not supported inside vectors"); + + buffers.add(bytes); + } + return new Value<>(type, buffers); + } + + public void addFunctionsTo(List functions) + { + Terms.addFunctions(elements, functions); + } + } +} diff --git a/src/java/org/apache/cassandra/cql3/functions/ArgumentDeserializer.java b/src/java/org/apache/cassandra/cql3/functions/ArgumentDeserializer.java index 19c2093996..de72b9e329 100644 --- a/src/java/org/apache/cassandra/cql3/functions/ArgumentDeserializer.java +++ b/src/java/org/apache/cassandra/cql3/functions/ArgumentDeserializer.java @@ -25,6 +25,7 @@ import org.apache.cassandra.transport.ProtocolVersion; /** * Utility used to deserialize function arguments. */ +@FunctionalInterface public interface ArgumentDeserializer { /** diff --git a/src/java/org/apache/cassandra/cql3/functions/CastFcts.java b/src/java/org/apache/cassandra/cql3/functions/CastFcts.java index 0246a0bac7..156164a26b 100644 --- a/src/java/org/apache/cassandra/cql3/functions/CastFcts.java +++ b/src/java/org/apache/cassandra/cql3/functions/CastFcts.java @@ -390,7 +390,7 @@ public final class CastFcts if (buffer == null || (!buffer.hasRemaining() && argType.isEmptyValueMeaningless())) return null; - return argType.getSerializer().toCQLLiteral(buffer); + return argType.getSerializer().toCQLLiteralNoQuote(buffer); }); } diff --git a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java index 99f8966b4d..2b15c8d935 100644 --- a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java +++ b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java @@ -370,7 +370,7 @@ public class UDAggregate extends UserFunction implements AggregateFunction if (initialCondition() != null) builder.newLine() .append("INITCOND ") - .append(stateType().asCQL3Type().toCQLLiteral(initialCondition(), ProtocolVersion.CURRENT)); + .append(stateType().asCQL3Type().toCQLLiteral(initialCondition())); return builder.append(";") .toString(); diff --git a/src/java/org/apache/cassandra/cql3/functions/masking/ColumnMask.java b/src/java/org/apache/cassandra/cql3/functions/masking/ColumnMask.java index 77f88c0ee9..e8b7931718 100644 --- a/src/java/org/apache/cassandra/cql3/functions/masking/ColumnMask.java +++ b/src/java/org/apache/cassandra/cql3/functions/masking/ColumnMask.java @@ -181,7 +181,7 @@ public class ColumnMask { CQL3Type type = types.get(i).asCQL3Type(); ByteBuffer value = partialArgumentValues[i]; - arguments.add(type.toCQLLiteral(value, ProtocolVersion.CURRENT)); + arguments.add(type.toCQLLiteral(value)); } return format("%s(%s)", function.name(), StringUtils.join(arguments, ", ")); } diff --git a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java index a6f227a91d..0d3d7e12dd 100644 --- a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java @@ -61,7 +61,12 @@ final class PartitionKeySingleRestrictionSet extends RestrictionSetWrapper imple { List l = new ArrayList<>(clusterings.size()); for (ClusteringPrefix clustering : clusterings) + { + // Can not use QueryProcessor.validateKey here to validate each column as that validates that empty are not allowed + // but composite partition keys actually allow empty! + clustering.validate(); l.add(clustering.serializeAsPartitionKey()); + } return l; } diff --git a/src/java/org/apache/cassandra/cql3/selection/ListSelector.java b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java index 5d095d51c5..a777bc5feb 100644 --- a/src/java/org/apache/cassandra/cql3/selection/ListSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java @@ -68,7 +68,7 @@ final class ListSelector extends Selector public static Factory newFactory(final AbstractType type, final SelectorFactories factories) { - return new CollectionFactory(type, factories) + return new MultiElementFactory(type, factories) { protected String getColumnName() { diff --git a/src/java/org/apache/cassandra/cql3/selection/CollectionFactory.java b/src/java/org/apache/cassandra/cql3/selection/MultiElementFactory.java similarity index 90% rename from src/java/org/apache/cassandra/cql3/selection/CollectionFactory.java rename to src/java/org/apache/cassandra/cql3/selection/MultiElementFactory.java index 816980d686..ef1d0d1b49 100644 --- a/src/java/org/apache/cassandra/cql3/selection/CollectionFactory.java +++ b/src/java/org/apache/cassandra/cql3/selection/MultiElementFactory.java @@ -27,21 +27,21 @@ import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.schema.ColumnMetadata; /** - * A base Selector.Factory for collections or tuples. + * A base Selector.Factory for list/set, tuples, and vectors. */ -abstract class CollectionFactory extends Factory +abstract class MultiElementFactory extends Factory { /** - * The collection or tuple type. + * The list/set, tuples, and vectors type. */ private final AbstractType type; /** - * The collection or tuple element factories. + * The list/set, tuples, and vectors element factories. */ private final SelectorFactories factories; - public CollectionFactory(AbstractType type, SelectorFactories factories) + public MultiElementFactory(AbstractType type, SelectorFactories factories) { this.type = type; this.factories = factories; diff --git a/src/java/org/apache/cassandra/cql3/selection/Selectable.java b/src/java/org/apache/cassandra/cql3/selection/Selectable.java index 9476deaf03..dbde53fc4d 100644 --- a/src/java/org/apache/cassandra/cql3/selection/Selectable.java +++ b/src/java/org/apache/cassandra/cql3/selection/Selectable.java @@ -721,6 +721,89 @@ public interface Selectable extends AssignmentTestable /** * Selectable for literal Lists. */ + public static class WithArrayLiteral implements Selectable + { + /** + * The list elements + */ + private final List selectables; + + public WithArrayLiteral(List selectables) + { + this.selectables = selectables; + } + + private Selectable target(AbstractType target) + { + // when the target isn't known, fallback to list; cases like "SELECT [1, 2]" can't be known, but used to be list type! + // If a vector is actually desired, then can use type cast/hints: "SELECT (vector) [k, v1]" + if (target == null || target instanceof ListType) + return new WithList(selectables); + else if (target.isVector()) + return new WithVector(selectables); + throw new IllegalArgumentException("Unsupported target type: " + target.asCQL3Type()); + } + + @Override + public TestResult testAssignment(String keyspace, ColumnSpecification receiver) + { + return target(receiver == null ? null : receiver.type).testAssignment(keyspace, receiver); + } + + @Override + public Factory newSelectorFactory(TableMetadata cfm, + AbstractType expectedType, + List defs, + VariableSpecifications boundNames) + { + return target(expectedType).newSelectorFactory(cfm, expectedType, defs, boundNames); + } + + @Override + public AbstractType getExactTypeIfKnown(String keyspace) + { + // TODO try to pass in a target to this API + // default to list when type is being infered + return new WithList(selectables).getExactTypeIfKnown(keyspace); + } + + @Override + public AbstractType getCompatibleTypeIfKnown(String keyspace) + { + // TODO try to pass in a target to this API + // default to list when type is being infered + return new WithList(selectables).getCompatibleTypeIfKnown(keyspace); + } + + @Override + public boolean selectColumns(Predicate predicate) + { + return Selectable.selectColumns(selectables, predicate); + } + + @Override + public String toString() + { + return Lists.listToString(selectables); + } + + public static class Raw implements Selectable.Raw + { + private final List raws; + + public Raw(List raws) + { + this.raws = raws; + } + + @Override + public Selectable prepare(TableMetadata cfm) + { + return new WithArrayLiteral(raws.stream().map(p -> p.prepare(cfm)).collect(Collectors.toList())); + } + } + } + public static class WithList implements Selectable { /** @@ -791,21 +874,78 @@ public interface Selectable extends AssignmentTestable { return Lists.listToString(selectables); } + } - public static class Raw implements Selectable.Raw + public static class WithVector implements Selectable + { + /** + * The vector elements + */ + private final List selectables; + + public WithVector(List selectables) { - private final List raws; + this.selectables = selectables; + } - public Raw(List raws) + @Override + public TestResult testAssignment(String keyspace, ColumnSpecification receiver) + { + return Vectors.testVectorAssignment(receiver, selectables); + } + + @Override + public Factory newSelectorFactory(TableMetadata cfm, + AbstractType expectedType, + List defs, + VariableSpecifications boundNames) + { + AbstractType type = getExactTypeIfKnown(cfm.keyspace); + if (type == null) { - this.raws = raws; + type = expectedType; + if (type == null) + throw invalidRequest("Cannot infer type for term %s in selection clause (try using a cast to force a type)", + this); } - @Override - public Selectable prepare(TableMetadata cfm) - { - return new WithList(raws.stream().map(p -> p.prepare(cfm)).collect(Collectors.toList())); - } + VectorType vectorType = (VectorType) type; + assert vectorType.dimension == selectables.size() : String.format("Unable to create a vector selector of type %s from %d elements", vectorType.asCQL3Type(), selectables.size()); + + List> expectedTypes = new ArrayList<>(selectables.size()); + for (int i = 0, m = selectables.size(); i < m; i++) + expectedTypes.add(vectorType.getElementsType()); + + SelectorFactories factories = createFactoriesAndCollectColumnDefinitions(selectables, + expectedTypes, + cfm, + defs, + boundNames); + return VectorSelector.newFactory(type, factories); + } + + @Override + public AbstractType getExactTypeIfKnown(String keyspace) + { + return Vectors.getExactVectorTypeIfKnown(selectables, p -> p.getExactTypeIfKnown(keyspace)); + } + + @Override + public AbstractType getCompatibleTypeIfKnown(String keyspace) + { + return Vectors.getPreferredCompatibleType(selectables, p -> p.getCompatibleTypeIfKnown(keyspace)); + } + + @Override + public boolean selectColumns(Predicate predicate) + { + return Selectable.selectColumns(selectables, predicate); + } + + @Override + public String toString() + { + return Lists.listToString(selectables); } } diff --git a/src/java/org/apache/cassandra/cql3/selection/Selector.java b/src/java/org/apache/cassandra/cql3/selection/Selector.java index 12245ac1a4..fce2ef0634 100644 --- a/src/java/org/apache/cassandra/cql3/selection/Selector.java +++ b/src/java/org/apache/cassandra/cql3/selection/Selector.java @@ -91,7 +91,8 @@ public abstract class Selector SCALAR_FUNCTION_SELECTOR(ScalarFunctionSelector.deserializer), AGGREGATE_FUNCTION_SELECTOR(AggregateFunctionSelector.deserializer), ELEMENT_SELECTOR(ElementsSelector.ElementSelector.deserializer), - SLICE_SELECTOR(ElementsSelector.SliceSelector.deserializer); + SLICE_SELECTOR(ElementsSelector.SliceSelector.deserializer), + VECTOR_SELECTOR(VectorSelector.deserializer); private final SelectorDeserializer deserializer; diff --git a/src/java/org/apache/cassandra/cql3/selection/SetSelector.java b/src/java/org/apache/cassandra/cql3/selection/SetSelector.java index c6aa225314..de97249af5 100644 --- a/src/java/org/apache/cassandra/cql3/selection/SetSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/SetSelector.java @@ -70,7 +70,7 @@ final class SetSelector extends Selector public static Factory newFactory(final AbstractType type, final SelectorFactories factories) { - return new CollectionFactory(type, factories) + return new MultiElementFactory(type, factories) { protected String getColumnName() { diff --git a/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java b/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java index 111d63c744..c06d911651 100644 --- a/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java @@ -68,7 +68,7 @@ final class TupleSelector extends Selector public static Factory newFactory(final AbstractType type, final SelectorFactories factories) { - return new CollectionFactory(type, factories) + return new MultiElementFactory(type, factories) { protected String getColumnName() { diff --git a/src/java/org/apache/cassandra/cql3/selection/VectorSelector.java b/src/java/org/apache/cassandra/cql3/selection/VectorSelector.java new file mode 100644 index 0000000000..8eeacea940 --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/selection/VectorSelector.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3.selection; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.google.common.base.Preconditions; + +import org.apache.cassandra.cql3.Lists; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.VectorType; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.transport.ProtocolVersion; + +public class VectorSelector extends Selector +{ + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + @Override + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + VectorType type = (VectorType) readType(metadata, in); + List elements = new ArrayList<>(type.dimension); + for (int i = 0; i < type.dimension; i++) + elements.add(serializer.deserialize(in, version, metadata)); + + return new VectorSelector(type, elements); + } + }; + + /** + * The vector type. + */ + private final VectorType type; + + /** + * The list elements + */ + private final List elements; + + private VectorSelector(VectorType type, List elements) + { + super(Kind.VECTOR_SELECTOR); + Preconditions.checkArgument(elements.size() == type.dimension, + "Unable to create a vector select of type %s from %s elements", + type.asCQL3Type(), + elements.size()); + this.type = type; + this.elements = elements; + } + + public static Factory newFactory(final AbstractType type, final SelectorFactories factories) + { + assert type.isVector() : String.format("Unable to create vector selector from typs %s", type.asCQL3Type()); + VectorType vt = (VectorType) type; + return new MultiElementFactory(type, factories) + { + protected String getColumnName() + { + return Lists.listToString(factories, Factory::getColumnName); + } + + public Selector newInstance(final QueryOptions options) + { + return new VectorSelector(vt, factories.newInstances(options)); + } + }; + } + + @Override + public void addFetchedColumns(ColumnFilter.Builder builder) + { + for (int i = 0, m = elements.size(); i < m; i++) + elements.get(i).addFetchedColumns(builder); + } + + @Override + public void addInput(InputRow input) + { + for (int i = 0, m = elements.size(); i < m; i++) + elements.get(i).addInput(input); + } + + @Override + public AbstractType getType() + { + return type; + } + + @Override + public void reset() + { + for (int i = 0, m = elements.size(); i < m; i++) + elements.get(i).reset(); + } + + @Override + public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException + { + List buffers = new ArrayList<>(elements.size()); + for (int i = 0, m = elements.size(); i < m; i++) + buffers.add(elements.get(i).getOutput(protocolVersion)); + + return type.decomposeRaw(buffers); + } + + @Override + protected int serializedSize(int version) + { + int size = sizeOf(type); + for (int i = 0, m = elements.size(); i < m; i++) + size += serializer.serializedSize(elements.get(i), version); + + return size; + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + writeType(out, type); + for (int i = 0, m = elements.size(); i < m; i++) + serializer.serialize(elements.get(i), out, version); + } + + @Override + public boolean isTerminal() + { + for (int i = 0, m = elements.size(); i < m; i++) + { + if (!elements.get(i).isTerminal()) + return false; + } + return true; + } + + @Override + public String toString() + { + return Lists.listToString(elements); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VectorSelector that = (VectorSelector) o; + return type.equals(that.type) && elements.equals(that.elements); + } + + @Override + public int hashCode() + { + return Objects.hash(type, elements); + } +} diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java index 6baf95259b..0252732e96 100644 --- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java @@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.auth.Permission; import org.apache.cassandra.db.guardrails.Guardrails; -import org.apache.cassandra.db.marshal.ValueAccessor; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaLayout; @@ -813,7 +812,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa { for (Clustering clustering : clusterings) { - validateClustering(clustering); + clustering.validate(); addUpdateForKey(updateBuilder, clustering, params); } } @@ -821,18 +820,6 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa } } - private void validateClustering(Clustering clustering) - { - ValueAccessor accessor = clustering.accessor(); - for (V v : clustering.getRawValues()) - { - if (v != null && accessor.size(v) > FBUtilities.MAX_UNSIGNED_SHORT) - throw new InvalidRequestException(String.format("Key length of %d is longer than maximum of %d", - clustering.dataSize(), - FBUtilities.MAX_UNSIGNED_SHORT)); - } - } - public Slices createSlices(QueryOptions options) { SortedSet> startBounds = restrictions.getClusteringColumnsBounds(Bound.START, options); diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java index 1940bbc945..1487aa30ed 100644 --- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java @@ -949,6 +949,8 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement public static ByteBuffer[] getComponents(TableMetadata metadata, DecoratedKey dk) { ByteBuffer key = dk.getKey(); + if (metadata.partitionKeyColumns().size() == 1) + return new ByteBuffer[]{ key }; if (metadata.partitionKeyType instanceof CompositeType) { return ((CompositeType)metadata.partitionKeyType).split(key); diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java index 7fc1185ed2..eb9f33a949 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateAggregateStatement.java @@ -47,7 +47,6 @@ import org.apache.cassandra.service.ClientState; import org.apache.cassandra.transport.Event.SchemaChange; import org.apache.cassandra.transport.Event.SchemaChange.Change; import org.apache.cassandra.transport.Event.SchemaChange.Target; -import org.apache.cassandra.transport.ProtocolVersion; import static java.lang.String.format; import static java.lang.String.join; @@ -99,11 +98,11 @@ public final class CreateAggregateStatement extends AlterSchemaStatement throw ire("Aggregate name '%s' is invalid", aggregateName); rawArgumentTypes.stream() - .filter(raw -> !raw.isTuple() && raw.isFrozen()) + .filter(raw -> !raw.isImplicitlyFrozen() && raw.isFrozen()) .findFirst() .ifPresent(t -> { throw ire("Argument '%s' cannot be frozen; remove frozen<> modifier from '%s'", t, t); }); - if (!rawStateType.isTuple() && rawStateType.isFrozen()) + if (!rawStateType.isImplicitlyFrozen() && rawStateType.isFrozen()) throw ire("State type '%s' cannot be frozen; remove frozen<> modifier from '%s'", rawStateType, rawStateType); KeyspaceMetadata keyspace = schema.getNullable(keyspaceName); @@ -177,8 +176,9 @@ public final class CreateAggregateStatement extends AlterSchemaStatement } // Converts initcond to a CQL literal and parse it back to avoid another CASSANDRA-11064 - String initialValueString = stateType.asCQL3Type().toCQLLiteral(initialValue, ProtocolVersion.CURRENT); - assert Objects.equal(initialValue, Terms.asBytes(keyspaceName, initialValueString, stateType)); + String initialValueString = stateType.asCQL3Type().toCQLLiteral(initialValue); + if (!Objects.equal(initialValue, stateType.asCQL3Type().fromCQLLiteral(keyspaceName, initialValueString))) + throw new AssertionError(String.format("CQL literal '%s' (from type %s) parsed with a different value", initialValueString, stateType.asCQL3Type())); if (Constants.NULL_LITERAL != rawInitialValue && isNullOrEmpty(stateType, initialValue)) throw ire("INITCOND must not be empty for all types except TEXT, ASCII, BLOB"); diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java index 8e3c79ca70..f04ae37cd5 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateFunctionStatement.java @@ -96,11 +96,11 @@ public final class CreateFunctionStatement extends AlterSchemaStatement throw ire("Duplicate argument names for given function %s with argument names %s", functionName, argumentNames); rawArgumentTypes.stream() - .filter(raw -> !raw.isTuple() && raw.isFrozen()) + .filter(raw -> !raw.isImplicitlyFrozen() && raw.isFrozen()) .findFirst() .ifPresent(t -> { throw ire("Argument '%s' cannot be frozen; remove frozen<> modifier from '%s'", t, t); }); - if (!rawReturnType.isTuple() && rawReturnType.isFrozen()) + if (!rawReturnType.isImplicitlyFrozen() && rawReturnType.isFrozen()) throw ire("Return type '%s' cannot be frozen; remove frozen<> modifier from '%s'", rawReturnType, rawReturnType); KeyspaceMetadata keyspace = schema.getNullable(keyspaceName); diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java index 186891e2f1..d83fbbf97f 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropAggregateStatement.java @@ -91,7 +91,7 @@ public final class DropAggregateStatement extends AlterSchemaStatement } arguments.stream() - .filter(raw -> !raw.isTuple() && raw.isFrozen()) + .filter(raw -> !raw.isImplicitlyFrozen() && raw.isFrozen()) .findFirst() .ifPresent(t -> { throw ire("Argument '%s' cannot be frozen; remove frozen<> modifier from '%s'", t, t); }); diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java index e743a1a579..af82206322 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/DropFunctionStatement.java @@ -92,7 +92,7 @@ public final class DropFunctionStatement extends AlterSchemaStatement } arguments.stream() - .filter(raw -> !raw.isTuple() && raw.isFrozen()) + .filter(raw -> !raw.isImplicitlyFrozen() && raw.isFrozen()) .findFirst() .ifPresent(t -> { throw ire("Argument '%s' cannot be frozen; remove frozen<> modifier from '%s'", t, t); }); diff --git a/src/java/org/apache/cassandra/db/ClusteringPrefix.java b/src/java/org/apache/cassandra/db/ClusteringPrefix.java index be81e5d57f..02f9330b43 100644 --- a/src/java/org/apache/cassandra/db/ClusteringPrefix.java +++ b/src/java/org/apache/cassandra/db/ClusteringPrefix.java @@ -30,10 +30,12 @@ import org.apache.cassandra.db.marshal.CompositeType; import org.apache.cassandra.db.marshal.ValueAccessor; import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteArrayUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version; import org.apache.cassandra.utils.bytecomparable.ByteSource; @@ -267,6 +269,24 @@ public interface ClusteringPrefix extends IMeasurableMemory, Clusterable return comparator.subtype(i).getString(get(i), accessor()); } + default void validate() + { + ValueAccessor accessor = accessor(); + int sum = 0; + for (V v : getRawValues()) + { + if (v != null && accessor.size(v) > FBUtilities.MAX_UNSIGNED_SHORT) + throw new InvalidRequestException(String.format("Key length of %d is longer than maximum of %d", + dataSize(), + FBUtilities.MAX_UNSIGNED_SHORT)); + sum += v == null ? 0 : accessor.size(v); + } + if (sum > FBUtilities.MAX_UNSIGNED_SHORT) + throw new InvalidRequestException(String.format("Key length of %d is longer than maximum of %d", + sum, + FBUtilities.MAX_UNSIGNED_SHORT)); + } + default void validate(int i, ClusteringComparator comparator) { comparator.subtype(i).validate(get(i), accessor()); diff --git a/src/java/org/apache/cassandra/db/NativeClustering.java b/src/java/org/apache/cassandra/db/NativeClustering.java index 1b6761d3d4..f3a8380470 100644 --- a/src/java/org/apache/cassandra/db/NativeClustering.java +++ b/src/java/org/apache/cassandra/db/NativeClustering.java @@ -23,6 +23,7 @@ import java.nio.ByteOrder; import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.db.marshal.ValueAccessor; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.concurrent.OpOrder; import org.apache.cassandra.utils.memory.HeapCloner; @@ -45,7 +46,7 @@ public class NativeClustering implements Clustering int bitmapSize = ((count + 7) >>> 3); assert count < 64 << 10; - assert dataSize < 64 << 10; + assert dataSize <= FBUtilities.MAX_UNSIGNED_SHORT : String.format("Data size %d >= %d", dataSize, FBUtilities.MAX_UNSIGNED_SHORT + 1); peer = allocator.allocate(metadataSize + dataSize + bitmapSize, writeOp); long bitmapStart = peer + metadataSize; diff --git a/src/java/org/apache/cassandra/db/SerializationHeader.java b/src/java/org/apache/cassandra/db/SerializationHeader.java index 5b64e31e77..841f7b3051 100644 --- a/src/java/org/apache/cassandra/db/SerializationHeader.java +++ b/src/java/org/apache/cassandra/db/SerializationHeader.java @@ -289,18 +289,6 @@ public class SerializationHeader this.stats = stats; } - /** - * Only exposed for {@link org.apache.cassandra.io.sstable.SSTableHeaderFix}. - */ - public static Component buildComponentForTools(AbstractType keyType, - List> clusteringTypes, - Map> staticColumns, - Map> regularColumns, - EncodingStats stats) - { - return new Component(keyType, clusteringTypes, staticColumns, regularColumns, stats); - } - public MetadataType getType() { return MetadataType.HEADER; diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java b/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java index f120349785..350c124fa1 100644 --- a/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java +++ b/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java @@ -24,8 +24,6 @@ import java.util.List; import java.util.regex.Pattern; import org.apache.cassandra.cql3.Term; -import org.apache.cassandra.serializers.TypeSerializer; -import org.apache.cassandra.serializers.BytesSerializer; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; @@ -232,6 +230,7 @@ public abstract class AbstractCompositeType extends AbstractType part = p.getRemainingPart(); ByteBuffer component = type.fromString(unescape(part)); + type.validate(component); totalLength += p.getComparatorSerializedSize() + 2 + component.remaining() + 1; components.add(component); comparators.add(p); @@ -290,7 +289,7 @@ public abstract class AbstractCompositeType extends AbstractType if (accessor.sizeFromOffset(input, offset) < 2) throw new MarshalException("Not enough bytes to read value size of component " + i); - int length = accessor.getShort(input, offset); + int length = accessor.getUnsignedShort(input, offset); offset += 2; if (accessor.sizeFromOffset(input, offset) < length) @@ -313,11 +312,6 @@ public abstract class AbstractCompositeType extends AbstractType public abstract ByteBuffer decompose(Object... objects); - public TypeSerializer getSerializer() - { - return BytesSerializer.instance; - } - abstract protected int getComparatorSize(int i, V value, ValueAccessor accessor, int offset); /** * @return the comparator for the given component. static CompositeType will consult diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractType.java b/src/java/org/apache/cassandra/db/marshal/AbstractType.java index c8a3904851..3f4124011b 100644 --- a/src/java/org/apache/cassandra/db/marshal/AbstractType.java +++ b/src/java/org/apache/cassandra/db/marshal/AbstractType.java @@ -162,7 +162,7 @@ public abstract class AbstractType implements Comparator, Assignm public String toCQLString(ByteBuffer bytes) { - return asCQL3Type().toCQLLiteral(bytes, ProtocolVersion.CURRENT); + return asCQL3Type().toCQLLiteral(bytes); } /** get a byte representation of the given string. */ @@ -303,6 +303,11 @@ public abstract class AbstractType implements Comparator, Assignm return false; } + public AbstractType unwrap() + { + return isReversed() ? ((ReversedType) this).baseType.unwrap() : this; + } + public static AbstractType parseDefaultParameters(AbstractType baseType, TypeParser parser) throws SyntaxException { Map parameters = parser.getKeyValueParameters(); @@ -404,6 +409,11 @@ public abstract class AbstractType implements Comparator, Assignm return false; } + public boolean isVector() + { + return false; + } + public boolean isMultiCell() { return false; @@ -419,6 +429,11 @@ public abstract class AbstractType implements Comparator, Assignm return this; } + public AbstractType unfreeze() + { + return this; + } + public List> subTypes() { return Collections.emptyList(); @@ -487,6 +502,29 @@ public abstract class AbstractType implements Comparator, Assignm return valueLengthIfFixed() != VARIABLE_LENGTH; } + /** + * Defines if the type allows an empty set of bytes ({@code new byte[0]}) as valid input. The {@link #validate(Object, ValueAccessor)} + * and {@link #compose(Object, ValueAccessor)} methods must allow empty bytes when this returns true, and must reject empty bytes + * when this is false. + *

+ * As of this writing, the main user of this API is for testing to know what types allow empty values and what types don't, + * so that the data that gets generated understands when {@link ByteBufferUtil#EMPTY_BYTE_BUFFER} is allowed as valid data. + */ + public boolean allowsEmpty() + { + return true; + } + + public boolean isNull(ByteBuffer bb) + { + return isNull(bb, ByteBufferAccessor.instance); + } + + public boolean isNull(V buffer, ValueAccessor accessor) + { + return getSerializer().isNull(buffer, accessor); + } + // This assumes that no empty values are passed public void writeValue(ByteBuffer value, DataOutputPlus out) throws IOException { @@ -496,7 +534,7 @@ public abstract class AbstractType implements Comparator, Assignm // This assumes that no empty values are passed public void writeValue(V value, ValueAccessor accessor, DataOutputPlus out) throws IOException { - assert !accessor.isEmpty(value) : "bytes should not be empty for type " + this; + assert !isNull(value, accessor) : "bytes should not be null for type " + this; int expectedValueLength = valueLengthIfFixed(); if (expectedValueLength >= 0) { diff --git a/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java b/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java index d7108992da..affe450317 100644 --- a/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java +++ b/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java @@ -206,6 +206,18 @@ public class ByteArrayAccessor implements ValueAccessor return ByteArrayUtil.getInt(value, offset); } + @Override + public float getFloat(byte[] value, int offset) + { + return ByteArrayUtil.getFloat(value, offset); + } + + @Override + public double getDouble(byte[] value, int offset) + { + return ByteArrayUtil.getDouble(value, offset); + } + @Override public long toLong(byte[] value) { @@ -276,6 +288,13 @@ public class ByteArrayAccessor implements ValueAccessor return TypeSizes.LONG_SIZE; } + @Override + public int putFloat(byte[] dst, int offset, float value) + { + ByteArrayUtil.putFloat(dst, offset, value); + return TypeSizes.FLOAT_SIZE; + } + @Override public byte[] empty() { diff --git a/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java b/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java index 0712930c3a..1baf7b052c 100644 --- a/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java +++ b/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java @@ -104,6 +104,9 @@ public class ByteBufferAccessor implements ValueAccessor @Override public ByteBuffer slice(ByteBuffer input, int offset, int length) { + int size = sizeFromOffset(input, offset); + if (size < length) + throw new IndexOutOfBoundsException(String.format("Attempted to read %d, but the size is %d", length, size)); ByteBuffer copy = input.duplicate(); copy.position(copy.position() + offset); copy.limit(copy.position() + length); @@ -210,6 +213,18 @@ public class ByteBufferAccessor implements ValueAccessor return value.getInt(value.position() + offset); } + @Override + public float getFloat(ByteBuffer value, int offset) + { + return value.getFloat(offset); + } + + @Override + public double getDouble(ByteBuffer value, int offset) + { + return value.getDouble(offset); + } + @Override public long toLong(ByteBuffer value) { @@ -280,6 +295,13 @@ public class ByteBufferAccessor implements ValueAccessor return TypeSizes.LONG_SIZE; } + @Override + public int putFloat(ByteBuffer dst, int offset, float value) + { + dst.putFloat(dst.position() + offset, value); + return TypeSizes.FLOAT_SIZE; + } + @Override public ByteBuffer empty() { diff --git a/src/java/org/apache/cassandra/db/marshal/ByteType.java b/src/java/org/apache/cassandra/db/marshal/ByteType.java index 614bdf9052..720168bcb3 100644 --- a/src/java/org/apache/cassandra/db/marshal/ByteType.java +++ b/src/java/org/apache/cassandra/db/marshal/ByteType.java @@ -46,6 +46,12 @@ public class ByteType extends NumberType super(ComparisonType.CUSTOM); } // singleton + @Override + public boolean allowsEmpty() + { + return false; + } + public int compareCustom(VL left, ValueAccessor accessorL, VR right, ValueAccessor accessorR) { return accessorL.getByte(left, 0) - accessorR.getByte(right, 0); diff --git a/src/java/org/apache/cassandra/db/marshal/CollectionType.java b/src/java/org/apache/cassandra/db/marshal/CollectionType.java index e4346d5fab..5e546cbc96 100644 --- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java +++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Iterator; +import java.util.Objects; import java.util.function.Consumer; import org.apache.cassandra.cql3.CQL3Type; @@ -92,6 +93,12 @@ public abstract class CollectionType extends AbstractType protected abstract List serializedValues(Iterator> cells); + @Override + public boolean allowsEmpty() + { + return false; + } + @Override public abstract CollectionSerializer getSerializer(); @@ -122,6 +129,14 @@ public abstract class CollectionType extends AbstractType return true; } + @Override + public void validate(V value, ValueAccessor accessor) throws MarshalException + { + if (accessor.isEmpty(value)) + throw new MarshalException("Not enough bytes to read a " + kind.name().toLowerCase()); + super.validate(value, accessor); + } + @Override public void validateCellValue(V cellValue, ValueAccessor accessor) throws MarshalException { @@ -245,6 +260,12 @@ public abstract class CollectionType extends AbstractType return nameComparator().equals(other.nameComparator()) && valueComparator().equals(other.valueComparator()); } + @Override + public int hashCode() + { + return Objects.hash(kind, isMultiCell(), nameComparator(), valueComparator()); + } + @Override public String toString() { diff --git a/src/java/org/apache/cassandra/db/marshal/CompositeType.java b/src/java/org/apache/cassandra/db/marshal/CompositeType.java index 00cbeb5898..df7ee99070 100644 --- a/src/java/org/apache/cassandra/db/marshal/CompositeType.java +++ b/src/java/org/apache/cassandra/db/marshal/CompositeType.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -30,7 +31,9 @@ import com.google.common.collect.Lists; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; +import org.apache.cassandra.serializers.BytesSerializer; import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version; import org.apache.cassandra.utils.bytecomparable.ByteSource; @@ -66,9 +69,37 @@ import static com.google.common.collect.Iterables.transform; */ public class CompositeType extends AbstractCompositeType { + public static class Serializer extends BytesSerializer + { + // types are held to make sure the serializer is unique for each collection of types, this is to make sure it's + // safe to cache in all cases + public final List> types; + + public Serializer(List> types) + { + this.types = types; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Serializer that = (Serializer) o; + return types.equals(that.types); + } + + @Override + public int hashCode() + { + return Objects.hash(types); + } + } + private static final int STATIC_MARKER = 0xFFFF; public final List> types; + private final Serializer serializer; // interning instances private static final ConcurrentMap>, CompositeType> instances = new ConcurrentHashMap<>(); @@ -140,6 +171,19 @@ public class CompositeType extends AbstractCompositeType protected CompositeType(List> types) { this.types = ImmutableList.copyOf(types); + this.serializer = new Serializer(this.types); + } + + @Override + public List> subTypes() + { + return types; + } + + @Override + public TypeSerializer getSerializer() + { + return serializer; } protected AbstractType getComparator(int i, V value, ValueAccessor accessor, int offset) @@ -268,7 +312,7 @@ public class CompositeType extends AbstractCompositeType public ByteBuffer decompose(Object... objects) { - assert objects.length == types.size(); + assert objects.length == types.size() : String.format("Expected length %d but given %d", types.size(), objects.length); ByteBuffer[] serialized = new ByteBuffer[objects.length]; for (int i = 0; i < objects.length; i++) @@ -440,6 +484,21 @@ public class CompositeType extends AbstractCompositeType public void serializeComparator(ByteBuffer bb) {} } + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CompositeType that = (CompositeType) o; + return types.equals(that.types); + } + + @Override + public int hashCode() + { + return Objects.hash(types); + } + @Override public String toString() { diff --git a/src/java/org/apache/cassandra/db/marshal/DurationType.java b/src/java/org/apache/cassandra/db/marshal/DurationType.java index 0c46617554..9353b2f6b7 100644 --- a/src/java/org/apache/cassandra/db/marshal/DurationType.java +++ b/src/java/org/apache/cassandra/db/marshal/DurationType.java @@ -46,6 +46,12 @@ public class DurationType extends AbstractType super(ComparisonType.BYTE_ORDER); } // singleton + @Override + public boolean allowsEmpty() + { + return false; + } + public ByteBuffer fromString(String source) throws MarshalException { // Return an empty ByteBuffer for an empty string. diff --git a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java index e7a2360fa9..658a147bc7 100644 --- a/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java +++ b/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java @@ -25,11 +25,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +40,7 @@ import org.apache.cassandra.cql3.Term; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; +import org.apache.cassandra.serializers.BytesSerializer; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.transport.ProtocolVersion; @@ -68,13 +72,41 @@ import static com.google.common.collect.Iterables.any; */ public class DynamicCompositeType extends AbstractCompositeType { + public static class Serializer extends BytesSerializer + { + // aliases are held to make sure the serializer is unique for each collection of types, this is to make sure it's + // safe to cache in all cases + private final Map> aliases; + + public Serializer(Map> aliases) + { + this.aliases = aliases; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Serializer that = (Serializer) o; + return aliases.equals(that.aliases); + } + + @Override + public int hashCode() + { + return Objects.hash(aliases); + } + } private static final Logger logger = LoggerFactory.getLogger(DynamicCompositeType.class); private static final ByteSource[] EMPTY_BYTE_SOURCE_ARRAY = new ByteSource[0]; private static final String REVERSED_TYPE = ReversedType.class.getSimpleName(); - private final Map> aliases; + @VisibleForTesting + public final Map> aliases; private final Map, Byte> inverseMapping; + private final Serializer serializer; // interning instances private static final ConcurrentHashMap>, DynamicCompositeType> instances = new ConcurrentHashMap<>(); @@ -94,12 +126,30 @@ public class DynamicCompositeType extends AbstractCompositeType private DynamicCompositeType(Map> aliases) { - this.aliases = aliases; + this.aliases = ImmutableMap.copyOf(aliases); + this.serializer = new Serializer(this.aliases); this.inverseMapping = new HashMap<>(); for (Map.Entry> en : aliases.entrySet()) this.inverseMapping.put(en.getValue(), en.getKey()); } + public int size() + { + return aliases.size(); + } + + @Override + public List> subTypes() + { + return new ArrayList<>(aliases.values()); + } + + @Override + public TypeSerializer getSerializer() + { + return serializer; + } + protected boolean readIsStatic(V value, ValueAccessor accessor) { // We don't have the static nothing for DCT @@ -329,6 +379,23 @@ public class DynamicCompositeType extends AbstractCompositeType return build(accessor, types, inverseMapping, values, lastEoc); } + public ByteBuffer build(Map valuesMap) + { + Sets.SetView unknownAliases = Sets.difference(valuesMap.keySet(), aliases.keySet()); + if (!unknownAliases.isEmpty()) + throw new IllegalArgumentException(String.format("Aliases %s used; only valid values are %s", unknownAliases, aliases.keySet())); + List> types = new ArrayList<>(valuesMap.size()); + List values = new ArrayList<>(valuesMap.size()); + for (Map.Entry e : valuesMap.entrySet()) + { + @SuppressWarnings("rawtype") + AbstractType type = aliases.get(e.getKey()); + types.add(type); + values.add(type.decompose(e.getValue())); + } + return build(ByteBufferAccessor.instance, types, inverseMapping, values, (byte) 0); + } + public static ByteBuffer build(List types, List values) { return build(ByteBufferAccessor.instance, @@ -392,6 +459,8 @@ public class DynamicCompositeType extends AbstractCompositeType // Write the type payload data (2-byte length header + the payload). V value = values.get(i); int bytesToCopy = accessor.size(value); + if ((short) bytesToCopy != bytesToCopy) + throw new IllegalArgumentException(String.format("Value of type %s is of length %d; does not fit in a short", type.asCQL3Type(), bytesToCopy)); accessor.putShort(result, offset, (short) bytesToCopy); offset += 2; accessor.copyTo(value, 0, result, accessor, offset, bytesToCopy); @@ -517,11 +586,24 @@ public class DynamicCompositeType extends AbstractCompositeType DynamicParsedComparator(String part) { String[] splits = part.split("@"); - if (splits.length != 2) - throw new IllegalArgumentException("Invalid component representation: " + part); - - comparatorName = splits[0]; - remainingPart = splits[1]; + switch (splits.length) + { + default: + throw new IllegalArgumentException("Invalid component representation: " + part); + case 1: + { + // empty is allowed for some types, so leave this to the higher level to validate empty makes sense for the type + comparatorName = splits[0]; + remainingPart = ""; + } + break; + case 2: + { + comparatorName = splits[0]; + remainingPart = splits[1]; + } + break; + } try { @@ -575,6 +657,21 @@ public class DynamicCompositeType extends AbstractCompositeType } } + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DynamicCompositeType that = (DynamicCompositeType) o; + return aliases.equals(that.aliases); + } + + @Override + public int hashCode() + { + return Objects.hash(aliases); + } + @Override public String toString() { @@ -585,7 +682,8 @@ public class DynamicCompositeType extends AbstractCompositeType * A comparator that always sorts it's first argument before the second * one. */ - private static class FixedValueComparator extends AbstractType + @VisibleForTesting + public static class FixedValueComparator extends AbstractType { public static final FixedValueComparator alwaysLesserThan = new FixedValueComparator(-1); public static final FixedValueComparator alwaysGreaterThan = new FixedValueComparator(1); diff --git a/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java b/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java index 1dfc997d97..498ca49584 100644 --- a/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java +++ b/src/java/org/apache/cassandra/db/marshal/LexicalUUIDType.java @@ -33,6 +33,9 @@ import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; public class LexicalUUIDType extends AbstractType { + public static class Serializer extends UUIDSerializer {} + private static final Serializer SERIALIZER = new Serializer(); + public static final LexicalUUIDType instance = new LexicalUUIDType(); private static final ArgumentDeserializer ARGUMENT_DESERIALIZER = new DefaultArgumentDeserializer(instance); @@ -129,7 +132,7 @@ public class LexicalUUIDType extends AbstractType @Override public TypeSerializer getSerializer() { - return UUIDSerializer.instance; + return SERIALIZER; } @Override diff --git a/src/java/org/apache/cassandra/db/marshal/ListType.java b/src/java/org/apache/cassandra/db/marshal/ListType.java index 159680dc9b..71f400dbb1 100644 --- a/src/java/org/apache/cassandra/db/marshal/ListType.java +++ b/src/java/org/apache/cassandra/db/marshal/ListType.java @@ -54,7 +54,7 @@ public class ListType extends CollectionType> if (l.size() != 1) throw new ConfigurationException("ListType takes exactly 1 type parameter"); - return getInstance(l.get(0), true); + return getInstance(l.get(0).freeze(), true); } public static ListType getInstance(AbstractType elements, boolean isMultiCell) @@ -126,10 +126,14 @@ public class ListType extends CollectionType> @Override public AbstractType freeze() { - if (isMultiCell) - return getInstance(this.elements, false); - else - return this; + // freeze elements to match org.apache.cassandra.cql3.CQL3Type.Raw.RawCollection.freeze + return isMultiCell ? getInstance(this.elements.freeze(), false) : this; + } + + @Override + public AbstractType unfreeze() + { + return isMultiCell ? this : getInstance(this.elements, true); } @Override diff --git a/src/java/org/apache/cassandra/db/marshal/MapType.java b/src/java/org/apache/cassandra/db/marshal/MapType.java index 40cd8bf096..16d533ec5c 100644 --- a/src/java/org/apache/cassandra/db/marshal/MapType.java +++ b/src/java/org/apache/cassandra/db/marshal/MapType.java @@ -61,7 +61,7 @@ public class MapType extends CollectionType> if (l.size() != 2) throw new ConfigurationException("MapType takes exactly 2 type parameters"); - return getInstance(l.get(0), l.get(1), true); + return getInstance(l.get(0).freeze(), l.get(1).freeze(), true); } public static MapType getInstance(AbstractType keys, AbstractType values, boolean isMultiCell) @@ -150,10 +150,14 @@ public class MapType extends CollectionType> @Override public AbstractType freeze() { - if (isMultiCell) - return getInstance(this.keys, this.values, false); - else - return this; + // freeze key/value to match org.apache.cassandra.cql3.CQL3Type.Raw.RawCollection.freeze + return isMultiCell ? getInstance(this.keys.freeze(), this.values.freeze(), false) : this; + } + + @Override + public AbstractType unfreeze() + { + return isMultiCell ? this : getInstance(this.keys, this.values, true); } @Override diff --git a/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java b/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java index 57786edbb8..3186e6e08b 100644 --- a/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java +++ b/src/java/org/apache/cassandra/db/marshal/PartitionerDefinedOrder.java @@ -140,6 +140,12 @@ public class PartitionerDefinedOrder extends AbstractType throw new IllegalStateException("You shouldn't be validating this."); } + @Override + public boolean isNull(V buffer, ValueAccessor accessor) + { + return buffer == null || accessor.isEmpty(buffer); + } + @Override public TypeSerializer getSerializer() { diff --git a/src/java/org/apache/cassandra/db/marshal/SetType.java b/src/java/org/apache/cassandra/db/marshal/SetType.java index 1bb1ed50ae..0a12bfa2fc 100644 --- a/src/java/org/apache/cassandra/db/marshal/SetType.java +++ b/src/java/org/apache/cassandra/db/marshal/SetType.java @@ -50,7 +50,7 @@ public class SetType extends CollectionType> if (l.size() != 1) throw new ConfigurationException("SetType takes exactly 1 type parameter"); - return getInstance(l.get(0), true); + return getInstance(l.get(0).freeze(), true); } public static SetType getInstance(AbstractType elements, boolean isMultiCell) @@ -117,10 +117,14 @@ public class SetType extends CollectionType> @Override public AbstractType freeze() { - if (isMultiCell) - return getInstance(this.elements, false); - else - return this; + // freeze elements to match org.apache.cassandra.cql3.CQL3Type.Raw.RawCollection.freeze + return isMultiCell ? getInstance(this.elements.freeze(), false) : this; + } + + @Override + public AbstractType unfreeze() + { + return isMultiCell ? this : getInstance(this.elements, true); } @Override diff --git a/src/java/org/apache/cassandra/db/marshal/ShortType.java b/src/java/org/apache/cassandra/db/marshal/ShortType.java index 96047daf73..bd7c2fd653 100644 --- a/src/java/org/apache/cassandra/db/marshal/ShortType.java +++ b/src/java/org/apache/cassandra/db/marshal/ShortType.java @@ -46,6 +46,12 @@ public class ShortType extends NumberType super(ComparisonType.CUSTOM); } // singleton + @Override + public boolean allowsEmpty() + { + return false; + } + public int compareCustom(VL left, ValueAccessor accessorL, VR right, ValueAccessor accessorR) { int diff = accessorL.getByte(left, 0) - accessorR.getByte(right, 0); diff --git a/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java b/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java index a474d39a81..54eabafc0c 100644 --- a/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java +++ b/src/java/org/apache/cassandra/db/marshal/SimpleDateType.java @@ -43,6 +43,12 @@ public class SimpleDateType extends TemporalType SimpleDateType() {super(ComparisonType.BYTE_ORDER);} // singleton + @Override + public boolean allowsEmpty() + { + return false; + } + @Override public ByteSource asComparableBytes(ValueAccessor accessor, V data, Version version) { diff --git a/src/java/org/apache/cassandra/db/marshal/TimeType.java b/src/java/org/apache/cassandra/db/marshal/TimeType.java index 67cf7dbceb..2a87808f1b 100644 --- a/src/java/org/apache/cassandra/db/marshal/TimeType.java +++ b/src/java/org/apache/cassandra/db/marshal/TimeType.java @@ -47,6 +47,12 @@ public class TimeType extends TemporalType private TimeType() {super(ComparisonType.BYTE_ORDER);} // singleton + @Override + public boolean allowsEmpty() + { + return false; + } + public ByteBuffer fromString(String source) throws MarshalException { return decompose(TimeSerializer.timeStringToLong(source)); diff --git a/src/java/org/apache/cassandra/db/marshal/TypeParser.java b/src/java/org/apache/cassandra/db/marshal/TypeParser.java index db8d3c43cd..a12804224e 100644 --- a/src/java/org/apache/cassandra/db/marshal/TypeParser.java +++ b/src/java/org/apache/cassandra/db/marshal/TypeParser.java @@ -188,6 +188,17 @@ public class TypeParser throw new SyntaxException("Syntax error parsing '" + str + ": for msg unexpected character '" + str.charAt(idx) + "'"); } + public static String stringifyTKeyValueParameters(Map map) + { + StringBuilder sb = new StringBuilder(); + sb.append('('); + for (Map.Entry e : map.entrySet()) + sb.append(e.getKey()).append(" = ").append(e.getValue()).append(", "); + if (!map.isEmpty()) + sb.setLength(sb.length() - 2); + return sb.append(')').toString(); + } + public Map getKeyValueParameters() throws SyntaxException { if (isEOS()) @@ -225,6 +236,32 @@ public class TypeParser throw new SyntaxException(String.format("Syntax error parsing '%s' at char %d: unexpected end of string", str, idx)); } + public static String stringifyVectorParameters(AbstractType type, boolean ignoreFreezing, int dimension) + { + return "(" + type.toString(ignoreFreezing) + " , " + dimension + ")"; + } + + public Vector getVectorParameters() + { + if (isEOS()) + return null; + if (str.charAt(idx) != '(') + throw new IllegalStateException(); + + ++idx; // skipping '(' + AbstractType type = parse(); + if (!skipBlankAndComma()) + throw new IllegalStateException(); + String s = readNextIdentifier(); + if (s.isEmpty()) + throw new IllegalStateException(); + int dimension = Integer.parseInt(s); + if (str.charAt(idx) != ')') + throw new IllegalStateException(); + ++idx; // skipping ')' + return new Vector(type, dimension); + } + public List> getTypeParameters() throws SyntaxException, ConfigurationException { List> list = new ArrayList<>(); @@ -555,6 +592,12 @@ public class TypeParser return str.substring(i, idx); } + @Override + public String toString() + { + return "TypeParser[" + str.substring(idx) + "]"; + } + /** * Helper function to ease the writing of AbstractType.toString() methods. */ @@ -633,4 +676,16 @@ public class TypeParser sb.append(')'); return sb.toString(); } + + public static class Vector + { + public final int dimension; + public final AbstractType type; + + public Vector(AbstractType type, int dimension) + { + this.dimension = dimension; + this.type = type; + } + } } diff --git a/src/java/org/apache/cassandra/db/marshal/UserType.java b/src/java/org/apache/cassandra/db/marshal/UserType.java index 28f654e47d..c2308e7b15 100644 --- a/src/java/org/apache/cassandra/db/marshal/UserType.java +++ b/src/java/org/apache/cassandra/db/marshal/UserType.java @@ -287,10 +287,13 @@ public class UserType extends TupleType implements SchemaElement @Override public UserType freeze() { - if (isMultiCell) - return new UserType(keyspace, name, fieldNames, fieldTypes(), false); - else - return this; + return isMultiCell ? new UserType(keyspace, name, fieldNames, fieldTypes(), false) : this; + } + + @Override + public UserType unfreeze() + { + return isMultiCell ? this : new UserType(keyspace, name, fieldNames, fieldTypes(), true); } @Override diff --git a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java index c9bcdb76d4..d3726b3a04 100644 --- a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java +++ b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java @@ -39,6 +39,7 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.vint.VIntCoding; import static org.apache.cassandra.db.ClusteringPrefix.Kind.*; @@ -125,6 +126,13 @@ public interface ValueAccessor return 2 + size(value); } + default int remaining(V value, int offset) + { + int size = size(value); + int rem = size - offset; + return rem > 0 ? rem : 0; + } + /** * @return true if the size of the given value is zero, false otherwise */ @@ -311,6 +319,29 @@ public interface ValueAccessor int toInt(V value); /** returns an int from offset {@param offset} */ int getInt(V value, int offset); + + default long getUnsignedVInt(V value, int offset) + { + return VIntCoding.getUnsignedVInt(value, this, offset); + } + + default int getUnsignedVInt32(V value, int offset) + { + return VIntCoding.getUnsignedVInt32(value, this, offset); + } + + default long getVInt(V value, int offset) + { + return VIntCoding.getVInt(value, this, offset); + } + + default int getVInt32(V value, int offset) + { + return VIntCoding.getVInt32(value, this, offset); + } + + float getFloat(V value, int offset); + double getDouble(V value, int offset); /** returns a long from offset 0 */ long toLong(V value); /** returns a long from offset {@param offset} */ @@ -354,6 +385,42 @@ public interface ValueAccessor */ int putLong(V dst, int offset, long value); + /** + * writes the float value {@param value} to {@param dst} at offset {@param offset} + * @return the number of bytes written to {@param value} + */ + int putFloat(V dst, int offset, float value); + + default int putBytes(V dst, int offset, byte[] src, int srcOffset, int length) + { + return ByteArrayAccessor.instance.copyTo(src, srcOffset, dst, this, offset, length); + } + + default int putBytes(V dst, int offset, byte[] src) + { + return putBytes(dst, offset, src, 0, src.length); + } + + default int putUnsignedVInt(V dst, int offset, long value) + { + return VIntCoding.writeUnsignedVInt(value, dst, offset, this); + } + + default int putUnsignedVInt32(V dst, int offset, int value) + { + return VIntCoding.writeUnsignedVInt32(value, dst, offset, this); + } + + default int putVInt(V dst, int offset, long value) + { + return VIntCoding.writeVInt(value, dst, offset, this); + } + + default int putVInt32(V dst, int offset, int value) + { + return VIntCoding.writeVInt32(value, dst, offset, this); + } + /** return a value with a length of 0 */ V empty(); diff --git a/src/java/org/apache/cassandra/db/marshal/VectorType.java b/src/java/org/apache/cassandra/db/marshal/VectorType.java new file mode 100644 index 0000000000..a184be6bb6 --- /dev/null +++ b/src/java/org/apache/cassandra/db/marshal/VectorType.java @@ -0,0 +1,638 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.marshal; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.cassandra.cql3.CQL3Type; +import org.apache.cassandra.cql3.Term; +import org.apache.cassandra.cql3.Vectors; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.serializers.TypeSerializer; +import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.JsonUtils; +import org.apache.cassandra.utils.bytecomparable.ByteComparable; +import org.apache.cassandra.utils.bytecomparable.ByteSource; + +public final class VectorType extends AbstractType> +{ + private static class Key + { + private final AbstractType type; + private final int dimension; + + private Key(AbstractType type, int dimension) + { + this.type = type; + this.dimension = dimension; + } + + private VectorType create() + { + return new VectorType<>(type, dimension); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Key key = (Key) o; + return dimension == key.dimension && Objects.equals(type, key.type); + } + + @Override + public int hashCode() + { + return Objects.hash(type, dimension); + } + } + @SuppressWarnings("rawtypes") + private static final ConcurrentHashMap instances = new ConcurrentHashMap<>(); + + public final AbstractType elementType; + public final int dimension; + private final TypeSerializer elementSerializer; + private final int valueLengthIfFixed; + private final VectorSerializer serializer; + + private VectorType(AbstractType elementType, int dimension) + { + super(ComparisonType.CUSTOM); + if (dimension <= 0) + throw new InvalidRequestException(String.format("vectors may only have positive dimentions; given %d", dimension)); + this.elementType = elementType; + this.dimension = dimension; + this.elementSerializer = elementType.getSerializer(); + this.valueLengthIfFixed = elementType.isValueLengthFixed() ? + elementType.valueLengthIfFixed() * dimension : + super.valueLengthIfFixed(); + this.serializer = elementType.isValueLengthFixed() ? + new FixedLengthSerializer() : + new VariableLengthSerializer(); + } + + @SuppressWarnings("unchecked") + public static VectorType getInstance(AbstractType elements, int dimension) + { + Key key = new Key(elements, dimension); + return instances.computeIfAbsent(key, Key::create); + } + + public static VectorType getInstance(TypeParser parser) + { + TypeParser.Vector v = parser.getVectorParameters(); + return getInstance(v.type.freeze(), v.dimension); + } + + @Override + public boolean isVector() + { + return true; + } + + @Override + public int compareCustom(VL left, ValueAccessor accessorL, VR right, ValueAccessor accessorR) + { + return getSerializer().compareCustom(left, accessorL, right, accessorR); + } + + @Override + public int valueLengthIfFixed() + { + return valueLengthIfFixed; + } + + @Override + public VectorSerializer getSerializer() + { + return serializer; + } + + public List split(ByteBuffer buffer) + { + return split(buffer, ByteBufferAccessor.instance); + } + + public List split(V buffer, ValueAccessor accessor) + { + return getSerializer().split(buffer, accessor); + } + + public float[] composeAsFloat(ByteBuffer input) + { + return composeAsFloat(input, ByteBufferAccessor.instance); + } + + public float[] composeAsFloat(V input, ValueAccessor accessor) + { + if (!(elementType instanceof FloatType)) + throw new IllegalStateException("Attempted to read as float, but element type is " + elementType.asCQL3Type()); + + if (isNull(input, accessor)) + return null; + float[] array = new float[dimension]; + int offset = 0; + for (int i = 0; i < dimension; i++) + { + array[i] = accessor.getFloat(input, offset); + offset += Float.BYTES; + } + return array; + } + + public ByteBuffer decomposeAsFloat(float[] value) + { + return decomposeAsFloat(ByteBufferAccessor.instance, value); + } + + public V decomposeAsFloat(ValueAccessor accessor, float[] value) + { + if (!(elementType instanceof FloatType)) + throw new IllegalStateException("Attempted to read as float, but element type is " + elementType.asCQL3Type()); + if (value == null) + return null; + if (value.length != dimension) + throw new IllegalArgumentException(String.format("Attempted to add float vector of dimension %d to %s", value.length, asCQL3Type())); + // TODO : should we use TypeSizes to be consistent with other code? Its the same value at the end of the day... + V buffer = accessor.allocate(Float.BYTES * dimension); + int offset = 0; + for (int i = 0; i < dimension; i++) + { + accessor.putFloat(buffer, offset, value[i]); + offset+= Float.BYTES; + } + return buffer; + } + + public ByteBuffer decomposeRaw(List elements) + { + return decomposeRaw(elements, ByteBufferAccessor.instance); + } + + public V decomposeRaw(List elements, ValueAccessor accessor) + { + return getSerializer().serializeRaw(elements, accessor); + } + + @Override + public ByteSource asComparableBytes(ValueAccessor accessor, V value, ByteComparable.Version version) + { + if (isNull(value, accessor)) + return null; + ByteSource[] srcs = new ByteSource[dimension]; + List split = split(value, accessor); + for (int i = 0; i < dimension; i++) + srcs[i] = elementType.asComparableBytes(accessor, split.get(i), version); + return ByteSource.withTerminatorMaybeLegacy(version, 0x00, srcs); + } + + @Override + public V fromComparableBytes(ValueAccessor accessor, ByteSource.Peekable comparableBytes, ByteComparable.Version version) + { + if (comparableBytes == null) + return accessor.empty(); + assert version != ByteComparable.Version.LEGACY; // legacy translation is not reversible + + List buffers = new ArrayList<>(); + int separator = comparableBytes.next(); + while (separator != ByteSource.TERMINATOR) + { + buffers.add(elementType.fromComparableBytes(accessor, comparableBytes, version)); + separator = comparableBytes.next(); + } + return decomposeRaw(buffers, accessor); + } + + @Override + public CQL3Type asCQL3Type() + { + return new CQL3Type.Vector(this); + } + + public AbstractType getElementsType() + { + return elementType; + } + + // vector of nested types is hard to parse, so fall back to bytes string matching ListType + @Override + public String getString(V value, ValueAccessor accessor) + { + return BytesType.instance.getString(value, accessor); + } + + @Override + public ByteBuffer fromString(String source) throws MarshalException + { + try + { + return ByteBufferUtil.hexToBytes(source); + } + catch (NumberFormatException e) + { + throw new MarshalException(String.format("cannot parse '%s' as hex bytes", source), e); + } + } + + @Override + public List> subTypes() + { + return Collections.singletonList(elementType); + } + + @Override + public String toJSONString(ByteBuffer buffer, ProtocolVersion protocolVersion) + { + return toJSONString(buffer, ByteBufferAccessor.instance, protocolVersion); + } + + @Override + public String toJSONString(V value, ValueAccessor accessor, ProtocolVersion protocolVersion) + { + StringBuilder sb = new StringBuilder(); + sb.append('['); + List split = split(value, accessor); + for (int i = 0; i < dimension; i++) + { + if (i > 0) + sb.append(", "); + sb.append(elementType.toJSONString(split.get(i), accessor, protocolVersion)); + } + sb.append(']'); + return sb.toString(); + } + + @Override + public Term fromJSONObject(Object parsed) throws MarshalException + { + if (parsed instanceof String) + parsed = JsonUtils.decodeJson((String) parsed); + + if (!(parsed instanceof List)) + throw new MarshalException(String.format( + "Expected a list, but got a %s: %s", parsed.getClass().getSimpleName(), parsed)); + + List list = (List) parsed; + if (list.size() != dimension) + throw new MarshalException(String.format("List had incorrect size: expected %d but given %d; %s", dimension, list.size(), list)); + List terms = new ArrayList<>(list.size()); + for (Object element : list) + { + if (element == null) + throw new MarshalException("Invalid null element in list"); + terms.add(elementType.fromJSONObject(element)); + } + + return new Vectors.DelayedValue<>(this, terms); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VectorType that = (VectorType) o; + return dimension == that.dimension && Objects.equals(elementType, that.elementType); + } + + @Override + public int hashCode() + { + return Objects.hash(elementType, dimension); + } + + @Override + public String toString() + { + return toString(false); + } + + @Override + public String toString(boolean ignoreFreezing) + { + return getClass().getName() + TypeParser.stringifyVectorParameters(elementType, ignoreFreezing, dimension); + } + + private void check(List values) + { + if (values.size() != dimension) + throw new MarshalException(String.format("Required %d elements, but saw %d", dimension, values.size())); + + // This code base always works with a list that is RandomAccess, so can use .get to avoid allocation + for (int i = 0; i < dimension; i++) + { + Object value = values.get(i); + if (value == null || (value instanceof ByteBuffer && elementSerializer.isNull((ByteBuffer) value))) + throw new MarshalException(String.format("Element at index %d is null (expected type %s); given %s", i, elementType.asCQL3Type(), values)); + } + } + + private static void checkConsumedFully(V buffer, ValueAccessor accessor, int offset) + { + if (!accessor.isEmptyFromOffset(buffer, offset)) + throw new MarshalException("Unexpected extraneous bytes after vector value"); + } + + public abstract class VectorSerializer extends TypeSerializer> + { + public abstract int compareCustom(VL left, ValueAccessor accessorL, VR right, ValueAccessor accessorR); + + public abstract List split(V buffer, ValueAccessor accessor); + public abstract V serializeRaw(List elements, ValueAccessor accessor); + + @Override + public String toString(List value) + { + StringBuilder sb = new StringBuilder(); + boolean isFirst = true; + sb.append('['); + for (T element : value) + { + if (isFirst) + isFirst = false; + else + sb.append(", "); + sb.append(elementSerializer.toString(element)); + } + sb.append(']'); + return sb.toString(); + } + + @Override + @SuppressWarnings({ "rawtypes", "unchecked" }) + public Class> getType() + { + return (Class) List.class; + } + } + + private class FixedLengthSerializer extends VectorSerializer + { + private FixedLengthSerializer() + { + } + + @Override + public int compareCustom(VL left, ValueAccessor accessorL, + VR right, ValueAccessor accessorR) + { + if (elementType.isByteOrderComparable) + return ValueAccessor.compare(left, accessorL, right, accessorR); + if (accessorL.isEmpty(left) || accessorR.isEmpty(right)) + return Boolean.compare(accessorR.isEmpty(right), accessorL.isEmpty(left)); + int offset = 0; + int elementLength = elementType.valueLengthIfFixed(); + for (int i = 0; i < dimension; i++) + { + VL leftBytes = accessorL.slice(left, offset, elementLength); + VR rightBytes = accessorR.slice(right, offset, elementLength); + int rc = elementType.compare(leftBytes, accessorL, rightBytes, accessorR); + if (rc != 0) + return rc; + + offset += elementLength; + } + return 0; + } + + @Override + public List split(V buffer, ValueAccessor accessor) + { + List result = new ArrayList<>(dimension); + int offset = 0; + int elementLength = elementType.valueLengthIfFixed(); + for (int i = 0; i < dimension; i++) + { + V bb = accessor.slice(buffer, offset, elementLength); + offset += elementLength; + elementSerializer.validate(bb, accessor); + result.add(bb); + } + checkConsumedFully(buffer, accessor, offset); + + return result; + } + + @Override + public V serializeRaw(List value, ValueAccessor accessor) + { + if (value == null) + return accessor.empty(); + check(value); + + int size = elementType.valueLengthIfFixed(); + V bb = accessor.allocate(size * dimension); + int position = 0; + for (V v : value) + position += accessor.copyTo(v, 0, bb, accessor, position, size); + return bb; + } + + @Override + public ByteBuffer serialize(List value) + { + if (value == null) + return ByteBufferUtil.EMPTY_BYTE_BUFFER; + check(value); + + ByteBuffer bb = ByteBuffer.allocate(elementType.valueLengthIfFixed() * dimension); + for (T v : value) + bb.put(elementSerializer.serialize(v).duplicate()); + bb.flip(); + return bb; + } + + @Override + public List deserialize(V input, ValueAccessor accessor) + { + if (isNull(input, accessor)) + return null; + List result = new ArrayList<>(dimension); + int offset = 0; + int elementLength = elementType.valueLengthIfFixed(); + for (int i = 0; i < dimension; i++) + { + V bb = accessor.slice(input, offset, elementLength); + offset += elementLength; + elementSerializer.validate(bb, accessor); + result.add(elementSerializer.deserialize(bb, accessor)); + } + checkConsumedFully(input, accessor, offset); + + return result; + } + + @Override + public void validate(V input, ValueAccessor accessor) throws MarshalException + { + if (accessor.isEmpty(input)) + return; + int offset = 0; + int elementSize = elementType.valueLengthIfFixed(); + for (int i = 0; i < dimension; i++) + { + V bb = accessor.slice(input, offset, elementSize); + offset += elementSize; + elementSerializer.validate(bb, accessor); + } + checkConsumedFully(input, accessor, offset); + } + } + + private class VariableLengthSerializer extends VectorSerializer + { + private VariableLengthSerializer() + { + } + + @Override + public int compareCustom(VL left, ValueAccessor accessorL, + VR right, ValueAccessor accessorR) + { + if (accessorL.isEmpty(left) || accessorR.isEmpty(right)) + return Boolean.compare(accessorR.isEmpty(right), accessorL.isEmpty(left)); + + int leftOffset = 0; + int rightOffset = 0; + for (int i = 0; i < dimension; i++) + { + VL leftBytes = readValue(left, accessorL, leftOffset); + leftOffset += sizeOf(leftBytes, accessorL); + + VR rightBytes = readValue(right, accessorR, rightOffset); + rightOffset += sizeOf(rightBytes, accessorR); + + int rc = elementType.compare(leftBytes, accessorL, rightBytes, accessorR); + if (rc != 0) + return rc; + } + return 0; + } + + private V readValue(V input, ValueAccessor accessor, int offset) + { + int size = accessor.getUnsignedVInt32(input, offset); + if (size < 0) + throw new AssertionError("Invalidate data at offset " + offset + "; saw size of " + size + " but only >= 0 is expected"); + + return accessor.slice(input, offset + TypeSizes.sizeofUnsignedVInt(size), size); + } + + private int writeValue(V src, V dst, ValueAccessor accessor, int offset) + { + int size = accessor.size(src); + int written = 0; + written += accessor.putUnsignedVInt32(dst, offset + written, size); + written += accessor.copyTo(src, 0, dst, accessor, offset + written, size); + return written; + } + + private int sizeOf(V bb, ValueAccessor accessor) + { + return accessor.sizeWithVIntLength(bb); + } + + @Override + public List split(V buffer, ValueAccessor accessor) + { + List result = new ArrayList<>(dimension); + int offset = 0; + for (int i = 0; i < dimension; i++) + { + V bb = readValue(buffer, accessor, offset); + offset += sizeOf(bb, accessor); + elementSerializer.validate(bb, accessor); + result.add(bb); + } + checkConsumedFully(buffer, accessor, offset); + + return result; + } + + @Override + public V serializeRaw(List value, ValueAccessor accessor) + { + if (value == null) + return accessor.empty(); + check(value); + + V bb = accessor.allocate(value.stream().mapToInt(v -> sizeOf(v, accessor)).sum()); + int offset = 0; + for (V b : value) + offset += writeValue(b, bb, accessor, offset); + return bb; + } + + @Override + public ByteBuffer serialize(List value) + { + if (value == null) + return ByteBufferUtil.EMPTY_BYTE_BUFFER; + check(value); + + List bbs = new ArrayList<>(dimension); + for (int i = 0; i < dimension; i++) + bbs.add(elementSerializer.serialize(value.get(i))); + return serializeRaw(bbs, ByteBufferAccessor.instance); + } + + @Override + public List deserialize(V input, ValueAccessor accessor) + { + if (isNull(input, accessor)) + return null; + List result = new ArrayList<>(dimension); + int offset = 0; + for (int i = 0; i < dimension; i++) + { + V bb = readValue(input, accessor, offset); + offset += sizeOf(bb, accessor); + elementSerializer.validate(bb, accessor); + result.add(elementSerializer.deserialize(bb, accessor)); + } + checkConsumedFully(input, accessor, offset); + + return result; + } + + @Override + public void validate(V input, ValueAccessor accessor) throws MarshalException + { + if (accessor.isEmpty(input)) + return; + int offset = 0; + for (int i = 0; i < dimension; i++) + { + V bb = readValue(input, accessor, offset); + offset += sizeOf(bb, accessor); + elementSerializer.validate(bb, accessor); + } + checkConsumedFully(input, accessor, offset); + } + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java b/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java deleted file mode 100644 index e785196c74..0000000000 --- a/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java +++ /dev/null @@ -1,930 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.cql3.CQL3Type; -import org.apache.cassandra.cql3.statements.schema.IndexTarget; -import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.db.marshal.AbstractCompositeType; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.CollectionType; -import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.db.marshal.DynamicCompositeType; -import org.apache.cassandra.db.marshal.ListType; -import org.apache.cassandra.db.marshal.MapType; -import org.apache.cassandra.db.marshal.SetType; -import org.apache.cassandra.db.marshal.TupleType; -import org.apache.cassandra.db.marshal.UserType; -import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; -import org.apache.cassandra.io.sstable.format.SSTableFormat.Components.Types; -import org.apache.cassandra.io.sstable.metadata.MetadataComponent; -import org.apache.cassandra.io.sstable.metadata.MetadataType; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.IndexMetadata; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.CassandraVersion; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Pair; - -import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_SKIP_AUTOMATIC_UDT_FIX; - -/** - * Validates and fixes type issues in the serialization-header of sstables. - */ -public abstract class SSTableHeaderFix -{ - // C* 3.0 upgrade code - private static final boolean SKIP_AUTOMATIC_FIX_ON_UPGRADE = CASSANDRA_SKIP_AUTOMATIC_UDT_FIX.getBoolean(); - - public static void fixNonFrozenUDTIfUpgradeFrom30() - { - String previousVersionString = FBUtilities.getPreviousReleaseVersionString(); - if (previousVersionString == null) - return; - CassandraVersion previousVersion = new CassandraVersion(previousVersionString); - if (previousVersion.major != 3 || previousVersion.minor > 0) - { - // Not an upgrade from 3.0 to 3.x, nothing to do here - return; - } - - if (SKIP_AUTOMATIC_FIX_ON_UPGRADE) - { - logger.warn("Detected upgrade from {} to {}, but -D{}=true, NOT fixing UDT type references in " + - "sstable metadata serialization-headers", - previousVersionString, - FBUtilities.getReleaseVersionString(), - CASSANDRA_SKIP_AUTOMATIC_UDT_FIX.getKey()); - return; - } - - logger.info("Detected upgrade from {} to {}, fixing UDT type references in sstable metadata serialization-headers", - previousVersionString, - FBUtilities.getReleaseVersionString()); - - SSTableHeaderFix instance = SSTableHeaderFix.builder() - .schemaCallback(() -> Schema.instance::getTableMetadata) - .build(); - instance.execute(); - } - - // "regular" SSTableHeaderFix code, also used by StandaloneScrubber. - - private static final Logger logger = LoggerFactory.getLogger(SSTableHeaderFix.class); - - protected final Consumer info; - protected final Consumer warn; - protected final Consumer error; - protected final boolean dryRun; - protected final Function schemaCallback; - - private final List descriptors; - - private final List>> updates = new ArrayList<>(); - private boolean hasErrors; - - SSTableHeaderFix(Builder builder) - { - this.info = builder.info; - this.warn = builder.warn; - this.error = builder.error; - this.dryRun = builder.dryRun; - this.schemaCallback = builder.schemaCallback.get(); - this.descriptors = new ArrayList<>(builder.descriptors); - Objects.requireNonNull(this.info, "info is null"); - Objects.requireNonNull(this.warn, "warn is null"); - Objects.requireNonNull(this.error, "error is null"); - Objects.requireNonNull(this.schemaCallback, "schemaCallback is null"); - } - - public static Builder builder() - { - return new Builder(); - } - - /** - * Builder to configure and construct an instance of {@link SSTableHeaderFix}. - * Default settings: - *

    - *
  • log via the slf4j logger of {@link SSTableHeaderFix}
    • - *
  • no dry-run (i.e. validate and fix, if no serious errors are detected)
    • - *
  • no schema callback
    • - *
- * If neither {@link #withDescriptor(Descriptor)} nor {@link #withPath(Path)} are used, - * all "live" sstables in all data directories will be scanned. - */ - public static class Builder - { - private final List paths = new ArrayList<>(); - private final List descriptors = new ArrayList<>(); - private Consumer info = (ln) -> logger.info("{}", ln); - private Consumer warn = (ln) -> logger.warn("{}", ln); - private Consumer error = (ln) -> logger.error("{}", ln); - private boolean dryRun; - private Supplier> schemaCallback = () -> null; - - private Builder() - {} - - /** - * Only validate and prepare fix, but do not write updated (fixed) sstable serialization-headers. - */ - public Builder dryRun() - { - dryRun = true; - return this; - } - - public Builder info(Consumer output) - { - this.info = output; - return this; - } - - public Builder warn(Consumer warn) - { - this.warn = warn; - return this; - } - - public Builder error(Consumer error) - { - this.error = error; - return this; - } - - /** - * Manually provide an individual sstable or directory containing sstables. - * - * Implementation note: procesing "live" sstables in their data directories as well as sstables - * in snapshots and backups in the data directories works. - * - * But processing sstables that reside somewhere else (i.e. verifying sstables before import) - * requires the use of {@link #withDescriptor(Descriptor)}. - */ - public Builder withPath(Path path) - { - this.paths.add(path); - return this; - } - - public Builder withDescriptor(Descriptor descriptor) - { - this.descriptors.add(descriptor); - return this; - } - - /** - * Schema callback to retrieve the schema of a table. Production code always delegates to the - * live schema ({@code Schema.instance}). Unit tests use this method to feed a custom schema. - */ - public Builder schemaCallback(Supplier> schemaCallback) - { - this.schemaCallback = schemaCallback; - return this; - } - - public SSTableHeaderFix build() - { - if (paths.isEmpty() && descriptors.isEmpty()) - return new AutomaticHeaderFix(this); - - return new ManualHeaderFix(this); - } - - public Builder logToList(List output) - { - return info(ln -> output.add("INFO " + ln)) - .warn(ln -> output.add("WARN " + ln)) - .error(ln -> output.add("ERROR " + ln)); - } - } - - public final void execute() - { - prepare(); - - logger.debug("Processing {} sstables:{}", - descriptors.size(), - descriptors.stream().map(Descriptor::toString).collect(Collectors.joining("\n ", "\n ", ""))); - - descriptors.forEach(this::processSSTable); - - if (updates.isEmpty()) - return; - - if (hasErrors) - { - info.accept("Stopping due to previous errors. Either fix the errors or specify the ignore-errors option."); - return; - } - - if (dryRun) - { - info.accept("Not fixing identified and fixable serialization-header issues."); - return; - } - - info.accept("Writing new metadata files"); - updates.forEach(descAndMeta -> writeNewMetadata(descAndMeta.left, descAndMeta.right)); - info.accept("Finished writing new metadata files"); - } - - /** - * Whether {@link #execute()} encountered an error. - */ - public boolean hasError() - { - return hasErrors; - } - - /** - * Whether {@link #execute()} found mismatches. - */ - public boolean hasChanges() - { - return !updates.isEmpty(); - } - - abstract void prepare(); - - private void error(String format, Object... args) - { - hasErrors = true; - error.accept(String.format(format, args)); - } - - void processFileOrDirectory(Path path) - { - Stream.of(path) - .flatMap(SSTableHeaderFix::maybeExpandDirectory) - .filter(p -> { - try - { - return Descriptor.fromFileWithComponent(new File(p)).right.type == Types.DATA; - } - catch (IllegalArgumentException t) - { - logger.info("Couldn't parse filename {}, ignoring", p); - return false; - } - }) - .map(File::new) - .map(file -> Descriptor.fromFileWithComponent(file, false).left) - .forEach(descriptors::add); - } - - private static Stream maybeExpandDirectory(Path path) - { - if (Files.isRegularFile(path)) - return Stream.of(path); - return LifecycleTransaction.getFiles(path, (file, fileType) -> fileType == Directories.FileType.FINAL, Directories.OnTxnErr.IGNORE) - .stream() - .map(File::toPath); - } - - private void processSSTable(Descriptor desc) - { - if (desc.cfname.indexOf('.') != -1) - { - // secondary index not checked - - // partition-key is the indexed column type - // clustering-key is org.apache.cassandra.db.marshal.PartitionerDefinedOrder - // no static columns, no regular columns - return; - } - - TableMetadata tableMetadata = schemaCallback.apply(desc); - if (tableMetadata == null) - { - error("Table %s.%s not found in the schema - NOT checking sstable %s", desc.ksname, desc.cfname, desc); - return; - } - - Set components = desc.discoverComponents(); - if (components.stream().noneMatch(c -> c.type == Types.STATS)) - { - error("sstable %s has no -Statistics.db component.", desc); - return; - } - - Map metadata = readSSTableMetadata(desc); - if (metadata == null) - return; - - MetadataComponent component = metadata.get(MetadataType.HEADER); - if (!(component instanceof SerializationHeader.Component)) - { - error("sstable %s: Expected %s, but got %s from metadata.get(MetadataType.HEADER)", - desc, - SerializationHeader.Component.class.getName(), - component != null ? component.getClass().getName() : "'null'"); - return; - } - SerializationHeader.Component header = (SerializationHeader.Component) component; - - // check partition key type - AbstractType keyType = validatePartitionKey(desc, tableMetadata, header); - - // check clustering columns - List> clusteringTypes = validateClusteringColumns(desc, tableMetadata, header); - - // check static and regular columns - Map> staticColumns = validateColumns(desc, tableMetadata, header.getStaticColumns(), ColumnMetadata.Kind.STATIC); - Map> regularColumns = validateColumns(desc, tableMetadata, header.getRegularColumns(), ColumnMetadata.Kind.REGULAR); - - SerializationHeader.Component newHeader = SerializationHeader.Component.buildComponentForTools(keyType, - clusteringTypes, - staticColumns, - regularColumns, - header.getEncodingStats()); - - // SerializationHeader.Component has no equals(), but a "good" toString() - if (header.toString().equals(newHeader.toString())) - return; - - Map newMetadata = new LinkedHashMap<>(metadata); - newMetadata.put(MetadataType.HEADER, newHeader); - - updates.add(Pair.create(desc, newMetadata)); - } - - private AbstractType validatePartitionKey(Descriptor desc, TableMetadata tableMetadata, SerializationHeader.Component header) - { - boolean keyMismatch = false; - AbstractType headerKeyType = header.getKeyType(); - AbstractType schemaKeyType = tableMetadata.partitionKeyType; - boolean headerKeyComposite = headerKeyType instanceof CompositeType; - boolean schemaKeyComposite = schemaKeyType instanceof CompositeType; - if (headerKeyComposite != schemaKeyComposite) - { - // one is a composite partition key, the other is not - very suspicious - keyMismatch = true; - } - else if (headerKeyComposite) // && schemaKeyComposite - { - // Note, the logic is similar as just calling 'fixType()' using the composite partition key, - // but the log messages should use the composite partition key column names. - List> headerKeyComponents = ((CompositeType) headerKeyType).types; - List> schemaKeyComponents = ((CompositeType) schemaKeyType).types; - if (headerKeyComponents.size() != schemaKeyComponents.size()) - { - // different number of components in composite partition keys - very suspicious - keyMismatch = true; - // Just use the original type from the header. Since the number of partition key components - // don't match, there's nothing to meaningfully validate against. - } - else - { - // fix components in composite partition key, if necessary - List> newComponents = new ArrayList<>(schemaKeyComponents.size()); - for (int i = 0; i < schemaKeyComponents.size(); i++) - { - AbstractType headerKeyComponent = headerKeyComponents.get(i); - AbstractType schemaKeyComponent = schemaKeyComponents.get(i); - AbstractType fixedType = fixType(desc, - tableMetadata.partitionKeyColumns().get(i).name.bytes, - headerKeyComponent, - schemaKeyComponent, - false); - if (fixedType == null) - keyMismatch = true; - else - headerKeyComponent = fixedType; - newComponents.add(fixType(desc, - tableMetadata.partitionKeyColumns().get(i).name.bytes, - headerKeyComponent, - schemaKeyComponent, - false)); - } - headerKeyType = CompositeType.getInstance(newComponents); - } - } - else - { - // fix non-composite partition key, if necessary - AbstractType fixedType = fixType(desc, tableMetadata.partitionKeyColumns().get(0).name.bytes, headerKeyType, schemaKeyType, false); - if (fixedType == null) - // non-composite partition key doesn't match and cannot be fixed - keyMismatch = true; - else - headerKeyType = fixedType; - } - if (keyMismatch) - error("sstable %s: Mismatch in partition key type between sstable serialization-header and schema (%s vs %s)", - desc, - headerKeyType.asCQL3Type(), - schemaKeyType.asCQL3Type()); - return headerKeyType; - } - - private List> validateClusteringColumns(Descriptor desc, TableMetadata tableMetadata, SerializationHeader.Component header) - { - List> headerClusteringTypes = header.getClusteringTypes(); - List> clusteringTypes = new ArrayList<>(); - boolean clusteringMismatch = false; - List schemaClustering = tableMetadata.clusteringColumns(); - if (schemaClustering.size() != headerClusteringTypes.size()) - { - clusteringMismatch = true; - // Just use the original types. Since the number of clustering columns don't match, there's nothing to - // meaningfully validate against. - clusteringTypes.addAll(headerClusteringTypes); - } - else - { - for (int i = 0; i < headerClusteringTypes.size(); i++) - { - AbstractType headerType = headerClusteringTypes.get(i); - ColumnMetadata column = schemaClustering.get(i); - AbstractType schemaType = column.type; - AbstractType fixedType = fixType(desc, column.name.bytes, headerType, schemaType, false); - if (fixedType == null) - clusteringMismatch = true; - else - headerType = fixedType; - clusteringTypes.add(headerType); - } - } - if (clusteringMismatch) - error("sstable %s: mismatch in clustering columns between sstable serialization-header and schema (%s vs %s)", - desc, - headerClusteringTypes.stream().map(AbstractType::asCQL3Type).map(CQL3Type::toString).collect(Collectors.joining(",")), - schemaClustering.stream().map(cd -> cd.type.asCQL3Type().toString()).collect(Collectors.joining(","))); - return clusteringTypes; - } - - private Map> validateColumns(Descriptor desc, TableMetadata tableMetadata, Map> columns, ColumnMetadata.Kind kind) - { - Map> target = new LinkedHashMap<>(); - for (Map.Entry> nameAndType : columns.entrySet()) - { - ByteBuffer name = nameAndType.getKey(); - AbstractType type = nameAndType.getValue(); - - AbstractType fixedType = validateColumn(desc, tableMetadata, kind, name, type); - if (fixedType == null) - { - error("sstable %s: contains column '%s' of type '%s', which could not be validated", - desc, - type, - logColumnName(name)); - // don't use a "null" type instance - fixedType = type; - } - - target.put(name, fixedType); - } - return target; - } - - private AbstractType validateColumn(Descriptor desc, TableMetadata tableMetadata, ColumnMetadata.Kind kind, ByteBuffer name, AbstractType type) - { - ColumnMetadata cd = tableMetadata.getColumn(name); - if (cd == null) - { - // In case the column was dropped, there is not much that we can actually validate. - // The column could have been recreated using the same or a different kind or the same or - // a different type. Lottery... - - cd = tableMetadata.getDroppedColumn(name, kind == ColumnMetadata.Kind.STATIC); - if (cd == null) - { - for (IndexMetadata indexMetadata : tableMetadata.indexes) - { - String target = indexMetadata.options.get(IndexTarget.TARGET_OPTION_NAME); - if (target != null && ByteBufferUtil.bytes(target).equals(name)) - { - warn.accept(String.format("sstable %s: contains column '%s', which is not a column in the table '%s.%s', but a target for that table's index '%s'", - desc, - logColumnName(name), - tableMetadata.keyspace, - tableMetadata.name, - indexMetadata.name)); - return type; - } - } - - warn.accept(String.format("sstable %s: contains column '%s', which is not present in the schema", - desc, - logColumnName(name))); - } - else - { - // This is a best-effort approach to handle the case of a UDT column created *AND* dropped in - // C* 3.0. - if (type instanceof UserType && cd.type instanceof TupleType) - { - // At this point, we know that the type belongs to a dropped column, recorded with the - // dropped column type "TupleType" and using "UserType" in the sstable. So it is very - // likely, that this belongs to a dropped UDT. Fix that information to tuple-type. - return fixType(desc, name, type, cd.type, true); - } - } - - return type; - } - - // At this point, the column name is known to be a "non-dropped" column in the table. - if (cd.kind != kind) - error("sstable %s: contains column '%s' as a %s column, but is of kind %s in the schema", - desc, - logColumnName(name), - kind.name().toLowerCase(), - cd.kind.name().toLowerCase()); - else - type = fixType(desc, name, type, cd.type, false); - return type; - } - - private AbstractType fixType(Descriptor desc, ByteBuffer name, AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) - { - AbstractType fixedType = fixTypeInner(typeInHeader, typeInSchema, droppedColumnMode); - if (fixedType != null) - { - if (fixedType != typeInHeader) - info.accept(String.format("sstable %s: Column '%s' needs to be updated from type '%s' to '%s'", - desc, - logColumnName(name), - typeInHeader.asCQL3Type(), - fixedType.asCQL3Type())); - return fixedType; - } - - error("sstable %s: contains column '%s' as type '%s', but schema mentions '%s'", - desc, - logColumnName(name), - typeInHeader.asCQL3Type(), - typeInSchema.asCQL3Type()); - - return typeInHeader; - } - - private AbstractType fixTypeInner(AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) - { - if (typeEquals(typeInHeader, typeInSchema)) - return typeInHeader; - - if (typeInHeader instanceof CollectionType) - return fixTypeInnerCollection(typeInHeader, typeInSchema, droppedColumnMode); - - if (typeInHeader instanceof AbstractCompositeType) - return fixTypeInnerAbstractComposite(typeInHeader, typeInSchema, droppedColumnMode); - - if (typeInHeader instanceof TupleType) - return fixTypeInnerAbstractTuple(typeInHeader, typeInSchema, droppedColumnMode); - - // all types, beside CollectionType + AbstractCompositeType + TupleType, should be ok (no nested types) - just check for compatibility - if (typeInHeader.isCompatibleWith(typeInSchema)) - return typeInHeader; - - return null; - } - - private AbstractType fixTypeInnerAbstractTuple(AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) - { - // This first 'if' handles the case when a UDT has been dropped, as a dropped UDT is recorded as a tuple - // in dropped_columns. If a UDT is to be replaced with a tuple, then also do that for the inner UDTs. - if (droppedColumnMode && typeInHeader.getClass() == UserType.class && typeInSchema instanceof TupleType) - return fixTypeInnerUserTypeDropped((UserType) typeInHeader, (TupleType) typeInSchema); - - if (typeInHeader.getClass() != typeInSchema.getClass()) - return null; - - if (typeInHeader.getClass() == UserType.class) - return fixTypeInnerUserType((UserType) typeInHeader, (UserType) typeInSchema); - - if (typeInHeader.getClass() == TupleType.class) - return fixTypeInnerTuple((TupleType) typeInHeader, (TupleType) typeInSchema, droppedColumnMode); - - throw new IllegalArgumentException("Unknown tuple type class " + typeInHeader.getClass().getName()); - } - - private AbstractType fixTypeInnerCollection(AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) - { - if (typeInHeader.getClass() != typeInSchema.getClass()) - return null; - - if (typeInHeader.getClass() == ListType.class) - return fixTypeInnerList((ListType) typeInHeader, (ListType) typeInSchema, droppedColumnMode); - - if (typeInHeader.getClass() == SetType.class) - return fixTypeInnerSet((SetType) typeInHeader, (SetType) typeInSchema, droppedColumnMode); - - if (typeInHeader.getClass() == MapType.class) - return fixTypeInnerMap((MapType) typeInHeader, (MapType) typeInSchema, droppedColumnMode); - - throw new IllegalArgumentException("Unknown collection type class " + typeInHeader.getClass().getName()); - } - - private AbstractType fixTypeInnerAbstractComposite(AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) - { - if (typeInHeader.getClass() != typeInSchema.getClass()) - return null; - - if (typeInHeader.getClass() == CompositeType.class) - return fixTypeInnerComposite((CompositeType) typeInHeader, (CompositeType) typeInSchema, droppedColumnMode); - - if (typeInHeader.getClass() == DynamicCompositeType.class) - { - // Not sure if we should care about UDTs in DynamicCompositeType at all... - if (!typeInHeader.isCompatibleWith(typeInSchema)) - return null; - - return typeInHeader; - } - - throw new IllegalArgumentException("Unknown composite type class " + typeInHeader.getClass().getName()); - } - - private AbstractType fixTypeInnerUserType(UserType cHeader, UserType cSchema) - { - if (!cHeader.keyspace.equals(cSchema.keyspace) || !cHeader.name.equals(cSchema.name)) - // different UDT - bummer... - return null; - - if (cHeader.isMultiCell() != cSchema.isMultiCell()) - { - if (cHeader.isMultiCell() && !cSchema.isMultiCell()) - { - // C* 3.0 writes broken SerializationHeader.Component instances - i.e. broken UDT type - // definitions into the sstable -Stats.db file, because 3.0 does not enclose frozen UDTs - // (and all UDTs in 3.0 were frozen) with an '' bracket. Since CASSANDRA-7423 (support - // for non-frozen UDTs, committed to C* 3.6), that frozen-bracket is quite important. - // Non-frozen (= multi-cell) UDTs are serialized in a fundamentally different way than - // frozen UDTs in sstables - most importantly, the order of serialized columns depends on - // the type: fixed-width types first, then variable length types (like frozen types), - // multi-cell types last. If C* >= 3.6 reads an sstable with a UDT that's written by - // C* < 3.6, a variety of CorruptSSTableExceptions get logged and clients will encounter - // read errors. - // At this point, we know that the type belongs to a "live" (non-dropped) column, so it - // is safe to correct the information from the header. - return cSchema; - } - - // In all other cases, there's not much we can do. - return null; - } - - return cHeader; - } - - private AbstractType fixTypeInnerUserTypeDropped(UserType cHeader, TupleType cSchema) - { - // Do not mess around with the UserType in the serialization header, if the column has been dropped. - // Only fix the multi-cell status when the header contains it as a multicell (non-frozen) UserType, - // but the schema says "frozen". - if (cHeader.isMultiCell() && !cSchema.isMultiCell()) - { - return new UserType(cHeader.keyspace, cHeader.name, cHeader.fieldNames(), cHeader.fieldTypes(), cSchema.isMultiCell()); - } - - return cHeader; - } - - private AbstractType fixTypeInnerTuple(TupleType cHeader, TupleType cSchema, boolean droppedColumnMode) - { - if (cHeader.size() != cSchema.size()) - // different number of components - bummer... - return null; - List> cHeaderFixed = new ArrayList<>(cHeader.size()); - boolean anyChanged = false; - for (int i = 0; i < cHeader.size(); i++) - { - AbstractType cHeaderComp = cHeader.type(i); - AbstractType cHeaderCompFixed = fixTypeInner(cHeaderComp, cSchema.type(i), droppedColumnMode); - if (cHeaderCompFixed == null) - // incompatible, bummer... - return null; - cHeaderFixed.add(cHeaderCompFixed); - anyChanged |= cHeaderComp != cHeaderCompFixed; - } - if (anyChanged || cSchema.isMultiCell() != cHeader.isMultiCell()) - // TODO this should create a non-frozen tuple type for the sake of handling a dropped, non-frozen UDT - return new TupleType(cHeaderFixed); - return cHeader; - } - - private AbstractType fixTypeInnerComposite(CompositeType cHeader, CompositeType cSchema, boolean droppedColumnMode) - { - if (cHeader.types.size() != cSchema.types.size()) - // different number of components - bummer... - return null; - List> cHeaderFixed = new ArrayList<>(cHeader.types.size()); - boolean anyChanged = false; - for (int i = 0; i < cHeader.types.size(); i++) - { - AbstractType cHeaderComp = cHeader.types.get(i); - AbstractType cHeaderCompFixed = fixTypeInner(cHeaderComp, cSchema.types.get(i), droppedColumnMode); - if (cHeaderCompFixed == null) - // incompatible, bummer... - return null; - cHeaderFixed.add(cHeaderCompFixed); - anyChanged |= cHeaderComp != cHeaderCompFixed; - } - if (anyChanged) - return CompositeType.getInstance(cHeaderFixed); - return cHeader; - } - - private AbstractType fixTypeInnerList(ListType cHeader, ListType cSchema, boolean droppedColumnMode) - { - AbstractType cHeaderElem = cHeader.getElementsType(); - AbstractType cHeaderElemFixed = fixTypeInner(cHeaderElem, cSchema.getElementsType(), droppedColumnMode); - if (cHeaderElemFixed == null) - // bummer... - return null; - if (cHeaderElem != cHeaderElemFixed) - // element type changed - return ListType.getInstance(cHeaderElemFixed, cHeader.isMultiCell()); - return cHeader; - } - - private AbstractType fixTypeInnerSet(SetType cHeader, SetType cSchema, boolean droppedColumnMode) - { - AbstractType cHeaderElem = cHeader.getElementsType(); - AbstractType cHeaderElemFixed = fixTypeInner(cHeaderElem, cSchema.getElementsType(), droppedColumnMode); - if (cHeaderElemFixed == null) - // bummer... - return null; - if (cHeaderElem != cHeaderElemFixed) - // element type changed - return SetType.getInstance(cHeaderElemFixed, cHeader.isMultiCell()); - return cHeader; - } - - private AbstractType fixTypeInnerMap(MapType cHeader, MapType cSchema, boolean droppedColumnMode) - { - AbstractType cHeaderKey = cHeader.getKeysType(); - AbstractType cHeaderVal = cHeader.getValuesType(); - AbstractType cHeaderKeyFixed = fixTypeInner(cHeaderKey, cSchema.getKeysType(), droppedColumnMode); - AbstractType cHeaderValFixed = fixTypeInner(cHeaderVal, cSchema.getValuesType(), droppedColumnMode); - if (cHeaderKeyFixed == null || cHeaderValFixed == null) - // bummer... - return null; - if (cHeaderKey != cHeaderKeyFixed || cHeaderVal != cHeaderValFixed) - // element type changed - return MapType.getInstance(cHeaderKeyFixed, cHeaderValFixed, cHeader.isMultiCell()); - return cHeader; - } - - private boolean typeEquals(AbstractType typeInHeader, AbstractType typeInSchema) - { - // Quite annoying, but the implementations of equals() on some implementation of AbstractType seems to be - // wrong, but toString() seems to work in such cases. - return typeInHeader.equals(typeInSchema) || typeInHeader.toString().equals(typeInSchema.toString()); - } - - private static String logColumnName(ByteBuffer columnName) - { - try - { - return ByteBufferUtil.string(columnName); - } - catch (CharacterCodingException e) - { - return "?? " + e; - } - } - - private Map readSSTableMetadata(Descriptor desc) - { - Map metadata; - try - { - metadata = desc.getMetadataSerializer().deserialize(desc, EnumSet.allOf(MetadataType.class)); - } - catch (IOException e) - { - error("Failed to deserialize metadata for sstable %s: %s", desc, e.toString()); - return null; - } - return metadata; - } - - private void writeNewMetadata(Descriptor desc, Map newMetadata) - { - File file = desc.fileFor(Components.STATS); - info.accept(String.format(" Writing new metadata file %s", file)); - try - { - desc.getMetadataSerializer().rewriteSSTableMetadata(desc, newMetadata); - } - catch (IOException e) - { - error("Failed to write metadata component for %s: %s", file, e.toString()); - throw new RuntimeException(e); - } - } - - /** - * Fix individually provided sstables or directories containing sstables. - */ - static class ManualHeaderFix extends SSTableHeaderFix - { - private final List paths; - - ManualHeaderFix(Builder builder) - { - super(builder); - this.paths = builder.paths; - } - - public void prepare() - { - paths.forEach(this::processFileOrDirectory); - } - } - - /** - * Fix all sstables in the configured data-directories. - */ - static class AutomaticHeaderFix extends SSTableHeaderFix - { - AutomaticHeaderFix(Builder builder) - { - super(builder); - } - - public void prepare() - { - info.accept("Scanning all data directories..."); - for (Directories.DataDirectory dataDirectory : Directories.dataDirectories) - scanDataDirectory(dataDirectory); - info.accept("Finished scanning all data directories..."); - } - - private void scanDataDirectory(Directories.DataDirectory dataDirectory) - { - info.accept(String.format("Scanning data directory %s", dataDirectory.location)); - File[] ksDirs = dataDirectory.location.tryList(); - if (ksDirs == null) - return; - for (File ksDir : ksDirs) - { - if (!ksDir.isDirectory() || !ksDir.isReadable()) - continue; - - String name = ksDir.name(); - - // silently ignore all system keyspaces - if (SchemaConstants.isLocalSystemKeyspace(name) || SchemaConstants.isReplicatedSystemKeyspace(name)) - continue; - - File[] tabDirs = ksDir.tryList(); - if (tabDirs == null) - continue; - for (File tabDir : tabDirs) - { - if (!tabDir.isDirectory() || !tabDir.isReadable()) - continue; - - processFileOrDirectory(tabDir.toPath()); - } - } - } - } -} diff --git a/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java index 7fc1985ea9..05073ea12f 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java @@ -52,4 +52,10 @@ public abstract class AbstractSSTableFormat CONFIGURATIONS = new HashMap<>(); public static final MemtableParams DEFAULT = get(null); + public static Set knownDefinitions() + { + return CONFIGURATION_DEFINITIONS.keySet(); + } + public static MemtableParams get(String key) { if (key == null) diff --git a/src/java/org/apache/cassandra/schema/SchemaConstants.java b/src/java/org/apache/cassandra/schema/SchemaConstants.java index 888ea4a31f..d9fc4935fa 100644 --- a/src/java/org/apache/cassandra/schema/SchemaConstants.java +++ b/src/java/org/apache/cassandra/schema/SchemaConstants.java @@ -47,6 +47,8 @@ public final class SchemaConstants public static final String VIRTUAL_VIEWS = "system_views"; + public static final String DUMMY_KEYSPACE_OR_TABLE_NAME = "--dummy--"; + /* system keyspace names (the ones with LocalStrategy replication strategy) */ public static final Set LOCAL_SYSTEM_KEYSPACE_NAMES = ImmutableSet.of(SYSTEM_KEYSPACE_NAME, SCHEMA_KEYSPACE_NAME); diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 241ec8dca0..f7044b55f7 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -47,7 +47,6 @@ import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder; import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; -import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Simulate; @@ -913,7 +912,7 @@ public final class SchemaKeyspace .add("final_func", aggregate.finalFunction() != null ? aggregate.finalFunction().name().name : null) .add("initcond", aggregate.initialCondition() != null // must use the frozen state type here, as 'null' for unfrozen collections may mean 'empty' - ? aggregate.stateType().freeze().asCQL3Type().toCQLLiteral(aggregate.initialCondition(), ProtocolVersion.CURRENT) + ? aggregate.stateType().freeze().asCQL3Type().toCQLLiteral(aggregate.initialCondition()) : null); } diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 53a22abcad..ed2deacd8c 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -65,7 +65,6 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; -import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.AbstractIterator; import org.github.jamm.Unmetered; @@ -1530,7 +1529,7 @@ public class TableMetadata implements SchemaElement private static String asCQLLiteral(AbstractType type, ByteBuffer value) { - return type.asCQL3Type().toCQLLiteral(value, ProtocolVersion.CURRENT); + return type.asCQL3Type().toCQLLiteral(value); } public static class CompactTableMetadata extends TableMetadata diff --git a/src/java/org/apache/cassandra/serializers/AbstractTextSerializer.java b/src/java/org/apache/cassandra/serializers/AbstractTextSerializer.java index a1362536b0..a2b0f2435b 100644 --- a/src/java/org/apache/cassandra/serializers/AbstractTextSerializer.java +++ b/src/java/org/apache/cassandra/serializers/AbstractTextSerializer.java @@ -63,15 +63,22 @@ public abstract class AbstractTextSerializer extends TypeSerializer return String.class; } - /** - * Generates CQL literal for TEXT/VARCHAR/ASCII types. - * Caveat: it does only generate literals with single quotes and not pg-style literals. - */ @Override - public String toCQLLiteral(ByteBuffer buffer) + public boolean shouldQuoteCQLLiterals() { - return buffer == null - ? "null" - : '\'' + StringUtils.replace(deserialize(buffer), "'", "''") + '\''; + return true; + } + + @Override + public boolean isNull(V buffer, ValueAccessor accessor) + { + // !buffer.hasRemaining() is not "null" for string types, it is the empty string + return buffer == null; + } + + @Override + protected String toCQLLiteralNonNull(ByteBuffer buffer) + { + return StringUtils.replace(deserialize(buffer), "'", "''"); } } diff --git a/src/java/org/apache/cassandra/serializers/BytesSerializer.java b/src/java/org/apache/cassandra/serializers/BytesSerializer.java index b9fe4b393f..1c60733ceb 100644 --- a/src/java/org/apache/cassandra/serializers/BytesSerializer.java +++ b/src/java/org/apache/cassandra/serializers/BytesSerializer.java @@ -55,10 +55,15 @@ public class BytesSerializer extends TypeSerializer } @Override - public String toCQLLiteral(ByteBuffer buffer) + public boolean isNull(V buffer, ValueAccessor accessor) { - return buffer == null - ? "null" - : "0x" + toString(deserialize(buffer)); + // !buffer.hasRemaining() is not "null" for bytes types, it is byte[0] + return buffer == null; + } + + @Override + protected String toCQLLiteralNonNull(ByteBuffer buffer) + { + return "0x" + toString(deserialize(buffer)); } } diff --git a/src/java/org/apache/cassandra/serializers/CounterSerializer.java b/src/java/org/apache/cassandra/serializers/CounterSerializer.java index b08ec1e9ad..1737c793dd 100644 --- a/src/java/org/apache/cassandra/serializers/CounterSerializer.java +++ b/src/java/org/apache/cassandra/serializers/CounterSerializer.java @@ -19,4 +19,5 @@ package org.apache.cassandra.serializers; public class CounterSerializer extends LongSerializer { + public static final CounterSerializer instance = new CounterSerializer(); } diff --git a/src/java/org/apache/cassandra/serializers/InetAddressSerializer.java b/src/java/org/apache/cassandra/serializers/InetAddressSerializer.java index 992de6251d..ba9e63f79f 100644 --- a/src/java/org/apache/cassandra/serializers/InetAddressSerializer.java +++ b/src/java/org/apache/cassandra/serializers/InetAddressSerializer.java @@ -73,4 +73,10 @@ public class InetAddressSerializer extends TypeSerializer { return InetAddress.class; } + + @Override + public boolean shouldQuoteCQLLiterals() + { + return true; + } } diff --git a/src/java/org/apache/cassandra/serializers/ListSerializer.java b/src/java/org/apache/cassandra/serializers/ListSerializer.java index f10e8f7e75..c443c84a91 100644 --- a/src/java/org/apache/cassandra/serializers/ListSerializer.java +++ b/src/java/org/apache/cassandra/serializers/ListSerializer.java @@ -72,6 +72,8 @@ public class ListSerializer extends CollectionSerializer> @Override public void validate(V input, ValueAccessor accessor) { + if (accessor.isEmpty(input)) + throw new MarshalException("Not enough bytes to read a list"); try { int n = readCollectionSize(input, accessor); diff --git a/src/java/org/apache/cassandra/serializers/MapSerializer.java b/src/java/org/apache/cassandra/serializers/MapSerializer.java index 9eb064273b..d19849d861 100644 --- a/src/java/org/apache/cassandra/serializers/MapSerializer.java +++ b/src/java/org/apache/cassandra/serializers/MapSerializer.java @@ -87,6 +87,8 @@ public class MapSerializer extends AbstractMapSerializer> @Override public void validate(T input, ValueAccessor accessor) { + if (accessor.isEmpty(input)) + throw new MarshalException("Not enough bytes to read a map"); try { // Empty values are still valid. diff --git a/src/java/org/apache/cassandra/serializers/SetSerializer.java b/src/java/org/apache/cassandra/serializers/SetSerializer.java index 599c2a822d..f03f586798 100644 --- a/src/java/org/apache/cassandra/serializers/SetSerializer.java +++ b/src/java/org/apache/cassandra/serializers/SetSerializer.java @@ -76,6 +76,8 @@ public class SetSerializer extends AbstractMapSerializer> @Override public void validate(V input, ValueAccessor accessor) { + if (accessor.isEmpty(input)) + throw new MarshalException("Not enough bytes to read a set"); try { // Empty values are still valid. diff --git a/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java b/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java index c367705fc7..764565c384 100644 --- a/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java +++ b/src/java/org/apache/cassandra/serializers/SimpleDateSerializer.java @@ -133,4 +133,10 @@ public class SimpleDateSerializer extends TypeSerializer { return Integer.class; } + + @Override + public boolean shouldQuoteCQLLiterals() + { + return true; + } } diff --git a/src/java/org/apache/cassandra/serializers/TimeSerializer.java b/src/java/org/apache/cassandra/serializers/TimeSerializer.java index 922c16c215..a9937b15b2 100644 --- a/src/java/org/apache/cassandra/serializers/TimeSerializer.java +++ b/src/java/org/apache/cassandra/serializers/TimeSerializer.java @@ -74,6 +74,12 @@ public class TimeSerializer extends TypeSerializer throw new MarshalException(String.format("Expected 8 byte long for time (%d)", accessor.size(value))); } + @Override + public boolean shouldQuoteCQLLiterals() + { + return true; + } + public String toString(Long value) { if (value == null) diff --git a/src/java/org/apache/cassandra/serializers/TimestampSerializer.java b/src/java/org/apache/cassandra/serializers/TimestampSerializer.java index 71782e5434..111abe9af1 100644 --- a/src/java/org/apache/cassandra/serializers/TimestampSerializer.java +++ b/src/java/org/apache/cassandra/serializers/TimestampSerializer.java @@ -197,15 +197,15 @@ public class TimestampSerializer extends TypeSerializer return Date.class; } - /** - * Builds CQL literal for a timestamp using time zone UTC and fixed date format. - * @see #FORMATTER_UTC - */ @Override - public String toCQLLiteral(ByteBuffer buffer) + public boolean shouldQuoteCQLLiterals() { - return buffer == null || !buffer.hasRemaining() - ? "null" - : FORMATTER_UTC.get().format(deserialize(buffer)); + return true; + } + + @Override + protected String toCQLLiteralNonNull(ByteBuffer buffer) + { + return FORMATTER_UTC.get().format(deserialize(buffer)); } } diff --git a/src/java/org/apache/cassandra/serializers/TypeSerializer.java b/src/java/org/apache/cassandra/serializers/TypeSerializer.java index 595c360b68..ac241d0fb4 100644 --- a/src/java/org/apache/cassandra/serializers/TypeSerializer.java +++ b/src/java/org/apache/cassandra/serializers/TypeSerializer.java @@ -19,12 +19,20 @@ package org.apache.cassandra.serializers; import java.nio.ByteBuffer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.db.marshal.ValueAccessor; public abstract class TypeSerializer { + private static final Pattern PATTERN_SINGLE_QUOTE = Pattern.compile("'", Pattern.LITERAL); + private static final String ESCAPED_SINGLE_QUOTE = Matcher.quoteReplacement("\'"); + public abstract ByteBuffer serialize(T value); public abstract T deserialize(V value, ValueAccessor accessor); @@ -55,11 +63,45 @@ public abstract class TypeSerializer public abstract Class getType(); - public String toCQLLiteral(ByteBuffer buffer) + public final boolean isNull(@Nullable ByteBuffer buffer) { - return buffer == null || !buffer.hasRemaining() + return isNull(buffer, ByteBufferAccessor.instance); + } + + public boolean isNull(@Nullable V buffer, ValueAccessor accessor) + { + return buffer == null || accessor.isEmpty(buffer); + } + + protected String toCQLLiteralNonNull(@Nonnull ByteBuffer buffer) + { + return toString(deserialize(buffer)); + } + + public final @Nonnull String toCQLLiteral(@Nullable ByteBuffer buffer) + { + return isNull(buffer) ? "null" - : toString(deserialize(buffer)); + : maybeQuote(toCQLLiteralNonNull(buffer)); + } + + public final @Nonnull String toCQLLiteralNoQuote(@Nullable ByteBuffer buffer) + { + return isNull(buffer) + ? "null" + : toCQLLiteralNonNull(buffer); + } + + public boolean shouldQuoteCQLLiterals() + { + return false; + } + + private String maybeQuote(String value) + { + if (shouldQuoteCQLLiterals()) + return "'" + PATTERN_SINGLE_QUOTE.matcher(value).replaceAll(ESCAPED_SINGLE_QUOTE) + "'"; + return value; } } diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 84fc5b1e89..fe91e81d81 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -66,7 +66,6 @@ import org.apache.cassandra.db.virtual.VirtualSchemaKeyspace; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.StartupException; import org.apache.cassandra.gms.Gossiper; -import org.apache.cassandra.io.sstable.SSTableHeaderFix; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.InetAddressAndPort; @@ -295,8 +294,6 @@ public class CassandraDaemon setupVirtualKeyspaces(); - SSTableHeaderFix.fixNonFrozenUDTIfUpgradeFrom30(); - try { scrubDataDirectories(); diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java index 26cc363263..36fda102e9 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java +++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java @@ -45,7 +45,6 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.IScrubber; -import org.apache.cassandra.io.sstable.SSTableHeaderFix; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -75,6 +74,9 @@ public class StandaloneScrubber private static final String SKIP_CORRUPTED_OPTION = "skip-corrupted"; private static final String NO_VALIDATE_OPTION = "no-validate"; private static final String REINSERT_OVERFLOWED_TTL_OPTION = "reinsert-overflowed-ttl"; + /** + * This option was logically removed from the code, but to avoid breaking backwards compatability the option remains + */ private static final String HEADERFIX_OPTION = "header-fix"; public static void main(String args[]) @@ -133,66 +135,6 @@ public class StandaloneScrubber } System.out.println(String.format("Pre-scrub sstables snapshotted into snapshot %s", snapshotName)); - if (options.headerFixMode != Options.HeaderFixMode.OFF) - { - // Run the frozen-UDT checks _before_ the sstables are opened - - List logOutput = new ArrayList<>(); - - SSTableHeaderFix.Builder headerFixBuilder = SSTableHeaderFix.builder() - .logToList(logOutput) - .schemaCallback(() -> Schema.instance::getTableMetadata); - if (options.headerFixMode == Options.HeaderFixMode.VALIDATE) - headerFixBuilder = headerFixBuilder.dryRun(); - - for (Pair> p : listResult) - headerFixBuilder.withPath(p.left.fileFor(Components.DATA).toPath()); - - SSTableHeaderFix headerFix = headerFixBuilder.build(); - try - { - headerFix.execute(); - } - catch (Exception e) - { - JVMStabilityInspector.inspectThrowable(e); - if (options.debug) - e.printStackTrace(System.err); - } - - if (headerFix.hasChanges() || headerFix.hasError()) - logOutput.forEach(System.out::println); - - if (headerFix.hasError()) - { - System.err.println("Errors in serialization-header detected, aborting."); - System.exit(1); - } - - switch (options.headerFixMode) - { - case VALIDATE_ONLY: - case FIX_ONLY: - System.out.printf("Not continuing with scrub, since '--%s %s' was specified.%n", - HEADERFIX_OPTION, - options.headerFixMode.asCommandLineOption()); - System.exit(0); - case VALIDATE: - if (headerFix.hasChanges()) - { - System.err.printf("Unfixed, but fixable errors in serialization-header detected, aborting. " + - "Use a non-validating mode ('-e %s' or '-e %s') for --%s%n", - Options.HeaderFixMode.FIX.asCommandLineOption(), - Options.HeaderFixMode.FIX_ONLY.asCommandLineOption(), - HEADERFIX_OPTION); - System.exit(2); - } - break; - case FIX: - break; - } - } - List sstables = new ArrayList<>(); // Open sstables @@ -350,17 +292,7 @@ public class StandaloneScrubber opts.checkData(!cmd.hasOption(NO_VALIDATE_OPTION)); opts.reinsertOverflowedTTLRows(cmd.hasOption(REINSERT_OVERFLOWED_TTL_OPTION)); if (cmd.hasOption(HEADERFIX_OPTION)) - { - try - { - opts.headerFixMode = HeaderFixMode.fromCommandLine(cmd.getOptionValue(HEADERFIX_OPTION)); - } - catch (Exception e) - { - errorMsg(String.format("Invalid argument value '%s' for --%s", cmd.getOptionValue(HEADERFIX_OPTION), HEADERFIX_OPTION), options); - return null; - } - } + System.err.println(String.format("Option %s is deprecated and no longer functional", HEADERFIX_OPTION)); return opts; } catch (ParseException e) diff --git a/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java b/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java index 692d18383a..dabb21e97b 100644 --- a/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java +++ b/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java @@ -224,7 +224,7 @@ public class ExecuteMessage extends Message.Request { ColumnSpecification cs = prepared.statement.getBindVariables().get(i); String boundName = cs.name.toString(); - String boundValue = cs.type.asCQL3Type().toCQLLiteral(options.getValues().get(i), options.getProtocolVersion()); + String boundValue = cs.type.asCQL3Type().toCQLLiteral(options.getValues().get(i)); if (boundValue.length() > 1000) boundValue = boundValue.substring(0, 1000) + "...'"; diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java index 790ff2ae02..e87cea9425 100644 --- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java +++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java @@ -734,6 +734,8 @@ public class ByteBufferUtil // changes bb position public static void writeShortLength(ByteBuffer bb, int length) { + if (length > FBUtilities.MAX_UNSIGNED_SHORT) + throw new IllegalArgumentException(String.format("Length %d > max length %d", length, FBUtilities.MAX_UNSIGNED_SHORT)); bb.put((byte) ((length >> 8) & 0xFF)); bb.put((byte) (length & 0xFF)); } diff --git a/src/java/org/apache/cassandra/utils/FastByteOperations.java b/src/java/org/apache/cassandra/utils/FastByteOperations.java index ffaee1f084..2a86712951 100644 --- a/src/java/org/apache/cassandra/utils/FastByteOperations.java +++ b/src/java/org/apache/cassandra/utils/FastByteOperations.java @@ -70,6 +70,11 @@ public class FastByteOperations return BestHolder.BEST.compare(b1, b2); } + public static int compareUnsigned(byte[] b1, byte[] b2) + { + return compareUnsigned(b1, 0, b1.length, b2, 0, b2.length); + } + public static void copy(byte[] src, int srcPosition, byte[] trg, int trgPosition, int length) { BestHolder.BEST.copy(src, srcPosition, trg, trgPosition, length); diff --git a/src/java/org/apache/cassandra/utils/JavaDriverUtils.java b/src/java/org/apache/cassandra/utils/JavaDriverUtils.java index 6c0567d03f..d58a26c359 100644 --- a/src/java/org/apache/cassandra/utils/JavaDriverUtils.java +++ b/src/java/org/apache/cassandra/utils/JavaDriverUtils.java @@ -54,9 +54,21 @@ public final class JavaDriverUtils public static TypeCodec codecFor(AbstractType abstractType) { + if (containsVector(abstractType.unwrap())) + throw new IllegalArgumentException("Vectors are not supported on UDFs; given " + abstractType.asCQL3Type()); return codecFor(driverType(abstractType)); } + private static boolean containsVector(AbstractType type) + { + if (type.isVector()) return true; + for (AbstractType child : type.subTypes()) + { + if (containsVector(child)) return true; + } + return false; + } + public static TypeCodec codecFor(DataType dataType) { return codecRegistry.codecFor(dataType); diff --git a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java index c2455bbce3..32455b81be 100644 --- a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java +++ b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java @@ -52,6 +52,7 @@ import java.nio.ByteBuffer; import io.netty.util.concurrent.FastThreadLocal; import net.nicoulaj.compilecommand.annotations.Inline; +import org.apache.cassandra.db.marshal.ValueAccessor; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -134,6 +135,11 @@ public class VIntCoding return checkedCast(decodeZigZag64(getUnsignedVInt(input, readerIndex))); } + public static long getVInt(ByteBuffer input, int readerIndex) + { + return decodeZigZag64(getUnsignedVInt(input, readerIndex)); + } + public static long getUnsignedVInt(ByteBuffer input, int readerIndex) { return getUnsignedVInt(input, readerIndex, input.limit()); @@ -167,6 +173,55 @@ public class VIntCoding return retval; } + public static int getUnsignedVInt32(V input, ValueAccessor accessor, int readerIndex) + { + return checkedCast(getUnsignedVInt(input, accessor, readerIndex)); + } + + public static int getVInt32(V input, ValueAccessor accessor, int readerIndex) + { + return checkedCast(decodeZigZag64(getUnsignedVInt(input, accessor, readerIndex))); + } + + public static long getVInt(V input, ValueAccessor accessor, int readerIndex) + { + return decodeZigZag64(getUnsignedVInt(input, accessor, readerIndex)); + } + + public static long getUnsignedVInt(V input, ValueAccessor accessor, int readerIndex) + { + return getUnsignedVInt(input, accessor, readerIndex, accessor.size(input)); + } + + public static long getUnsignedVInt(V input, ValueAccessor accessor, int readerIndex, int readerLimit) + { + if (readerIndex < 0) + throw new IllegalArgumentException("Reader index should be non-negative, but was " + readerIndex); + + if (readerIndex >= readerLimit) + return -1; + + int firstByte = accessor.getByte(input, readerIndex++); + + //Bail out early if this is one byte, necessary or it fails later + if (firstByte >= 0) + return firstByte; + + int size = numberOfExtraBytesToRead(firstByte); + if (readerIndex + size > readerLimit) + return -1; + + long retval = firstByte & firstByteValueMask(size); + for (int ii = 0; ii < size; ii++) + { + byte b = accessor.getByte(input, readerIndex++); + retval <<= 8; + retval |= b & 0xff; + } + + return retval; + } + /** * Computes size of an unsigned vint that starts at readerIndex of the provided ByteBuf. * @@ -318,6 +373,61 @@ public class VIntCoding } } + @Inline + public static int writeVInt(long value, V output, int offset, ValueAccessor accessor) + { + return writeUnsignedVInt(encodeZigZag64(value), output, offset, accessor); + } + + @Inline + public static int writeVInt32(int value, V output, int offset, ValueAccessor accessor) + { + return writeVInt(value, output, offset, accessor); + } + + @Inline + public static int writeUnsignedVInt32(int value, V output, int offset, ValueAccessor accessor) + { + return writeUnsignedVInt(value, output, offset, accessor); + } + + @Inline + public static int writeUnsignedVInt(long value, V output, int offset, ValueAccessor accessor) + { + int size = VIntCoding.computeUnsignedVIntSize(value); + int written = 0; + if (size == 1) + { + written += accessor.putByte(output, offset, (byte) (value)); + } + else if (size < 9) + { + if (accessor.remaining(output, offset) >= 8) + { + int shift = (8 - size) << 3; + int extraBytes = size - 1; + long mask = (long)VIntCoding.encodeExtraBytesToRead(extraBytes) << 56; + long register = (value << shift) | mask; + accessor.putLong(output, offset, register); + written += size; + } + else + { + written += accessor.putBytes(output, offset, VIntCoding.encodeUnsignedVInt(value, size), 0, size); + } + } + else if (size == 9) + { + written += accessor.putByte(output, offset, (byte) 0xFF); + written += accessor.putLong(output, offset + written, value); + } + else + { + throw new AssertionError(); + } + return written; + } + @Inline public static void writeUnsignedVInt32(int value, ByteBuffer output) { diff --git a/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java b/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java deleted file mode 100644 index 0d73731d60..0000000000 --- a/test/unit/org/apache/cassandra/cql3/CQL3TypeLiteralTest.java +++ /dev/null @@ -1,760 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.cql3; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.util.*; -import java.util.regex.Pattern; - -import org.junit.Test; - -import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.serializers.*; -import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.transport.ProtocolVersion; -import org.apache.cassandra.utils.ByteBufferUtil; - -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; -import static org.junit.Assert.assertEquals; - -/** - * Test functionality to re-create a CQL literal from its serialized representation. - * This test uses some randomness to generate the values and nested structures (collections,tuples,UDTs). - */ -public class CQL3TypeLiteralTest -{ - private static final Pattern QUOTE = Pattern.compile("'"); - - private static final Random r = new Random(); - /** - * Container holding the expected CQL literal for a type and serialized value. - * The CQL literal is generated independently from the code in {@link CQL3Type}. - */ - static class Value - { - final String expected; - final CQL3Type cql3Type; - final ByteBuffer value; - - Value(String expected, CQL3Type cql3Type, ByteBuffer value) - { - this.expected = expected; - this.cql3Type = cql3Type; - this.value = value; - } - } - - static final Map> nativeTypeValues = new EnumMap<>(CQL3Type.Native.class); - - static void addNativeValue(String expected, CQL3Type.Native cql3Type, ByteBuffer value) - { - List l = nativeTypeValues.get(cql3Type); - if (l == null) - nativeTypeValues.put(cql3Type, l = new ArrayList<>()); - l.add(new Value(expected, cql3Type, value)); - } - - static - { - // Add some (random) values for each native type. - // Also adds null values and empty values, if the type allows this. - - for (int i = 0; i < 20; i++) - { - String v = randString(true); - addNativeValue(quote(v), CQL3Type.Native.ASCII, AsciiSerializer.instance.serialize(v)); - } - addNativeValue("''", CQL3Type.Native.ASCII, AsciiSerializer.instance.serialize("")); - addNativeValue("''", CQL3Type.Native.ASCII, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.ASCII, null); - - for (int i = 0; i < 20; i++) - { - String v = randString(false); - addNativeValue(quote(v), CQL3Type.Native.TEXT, UTF8Serializer.instance.serialize(v)); - } - addNativeValue("''", CQL3Type.Native.TEXT, UTF8Serializer.instance.serialize("")); - addNativeValue("''", CQL3Type.Native.TEXT, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.TEXT, null); - - for (int i = 0; i < 20; i++) - { - String v = randString(false); - addNativeValue(quote(v), CQL3Type.Native.VARCHAR, UTF8Serializer.instance.serialize(v)); - } - addNativeValue("''", CQL3Type.Native.VARCHAR, UTF8Serializer.instance.serialize("")); - addNativeValue("''", CQL3Type.Native.VARCHAR, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.VARCHAR, null); - - addNativeValue("0", CQL3Type.Native.BIGINT, LongType.instance.decompose(0L)); - for (int i = 0; i < 20; i++) - { - long v = randLong(); - addNativeValue(Long.toString(v), CQL3Type.Native.BIGINT, LongType.instance.decompose(v)); - } - addNativeValue("null", CQL3Type.Native.BIGINT, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.BIGINT, null); - - addNativeValue("0", CQL3Type.Native.COUNTER, LongType.instance.decompose(0L)); - for (int i = 0; i < 20; i++) - { - long v = randLong(); - addNativeValue(Long.toString(v), CQL3Type.Native.COUNTER, LongType.instance.decompose(v)); - } - addNativeValue("null", CQL3Type.Native.COUNTER, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.COUNTER, null); - - addNativeValue("0", CQL3Type.Native.INT, Int32Type.instance.decompose(0)); - for (int i = 0; i < 20; i++) - { - int v = randInt(); - addNativeValue(Integer.toString(v), CQL3Type.Native.INT, Int32Type.instance.decompose(v)); - } - addNativeValue("null", CQL3Type.Native.INT, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.INT, null); - - addNativeValue("0", CQL3Type.Native.SMALLINT, ShortType.instance.decompose((short) 0)); - for (int i = 0; i < 20; i++) - { - short v = randShort(); - addNativeValue(Short.toString(v), CQL3Type.Native.SMALLINT, ShortType.instance.decompose(v)); - } - addNativeValue("null", CQL3Type.Native.SMALLINT, null); - - addNativeValue("0", CQL3Type.Native.TINYINT, ByteType.instance.decompose((byte) 0)); - for (int i = 0; i < 20; i++) - { - byte v = randByte(); - addNativeValue(Short.toString(v), CQL3Type.Native.TINYINT, ByteType.instance.decompose(v)); - } - addNativeValue("null", CQL3Type.Native.TINYINT, null); - - addNativeValue("0.0", CQL3Type.Native.FLOAT, FloatType.instance.decompose((float) 0)); - for (int i = 0; i < 20; i++) - { - float v = randFloat(); - addNativeValue(Float.toString(v), CQL3Type.Native.FLOAT, FloatType.instance.decompose(v)); - } - addNativeValue("NaN", CQL3Type.Native.FLOAT, FloatType.instance.decompose(Float.NaN)); - addNativeValue("null", CQL3Type.Native.FLOAT, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.FLOAT, null); - - addNativeValue("0.0", CQL3Type.Native.DOUBLE, DoubleType.instance.decompose((double) 0)); - for (int i = 0; i < 20; i++) - { - double v = randDouble(); - addNativeValue(Double.toString(v), CQL3Type.Native.DOUBLE, DoubleType.instance.decompose(v)); - } - addNativeValue("NaN", CQL3Type.Native.DOUBLE, DoubleType.instance.decompose(Double.NaN)); - addNativeValue("null", CQL3Type.Native.DOUBLE, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.DOUBLE, null); - - addNativeValue("0", CQL3Type.Native.DECIMAL, DecimalType.instance.decompose(BigDecimal.ZERO)); - for (int i = 0; i < 20; i++) - { - BigDecimal v = BigDecimal.valueOf(randDouble()); - addNativeValue(v.toString(), CQL3Type.Native.DECIMAL, DecimalType.instance.decompose(v)); - } - addNativeValue("null", CQL3Type.Native.DECIMAL, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.DECIMAL, null); - - addNativeValue("0", CQL3Type.Native.VARINT, IntegerType.instance.decompose(BigInteger.ZERO)); - for (int i = 0; i < 20; i++) - { - BigInteger v = BigInteger.valueOf(randLong()); - addNativeValue(v.toString(), CQL3Type.Native.VARINT, IntegerType.instance.decompose(v)); - } - addNativeValue("null", CQL3Type.Native.VARINT, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.VARINT, null); - - // boolean doesn't have that many possible values... - addNativeValue("false", CQL3Type.Native.BOOLEAN, BooleanType.instance.decompose(false)); - addNativeValue("true", CQL3Type.Native.BOOLEAN, BooleanType.instance.decompose(true)); - addNativeValue("null", CQL3Type.Native.BOOLEAN, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.BOOLEAN, null); - - // (mostly generates date values with surreal values like in year 14273) - for (int i = 0; i < 20; i++) - { - int v = randInt(); - addNativeValue(SimpleDateSerializer.instance.toString(v), CQL3Type.Native.DATE, SimpleDateSerializer.instance.serialize(v)); - } - addNativeValue("null", CQL3Type.Native.DATE, null); - - for (int i = 0; i < 100; i++) - { - long v = randLong(24L * 60 * 60 * 1000 * 1000 * 1000); - addNativeValue(TimeSerializer.instance.toString(v), CQL3Type.Native.TIME, TimeSerializer.instance.serialize(v)); - } - addNativeValue("null", CQL3Type.Native.TIME, null); - - for (int i = 0; i < 100; i++) - { - Duration duration = Duration.newInstance(Math.abs(randInt()), Math.abs(randInt()), Math.abs(randLong())); - addNativeValue(DurationSerializer.instance.toString(duration), CQL3Type.Native.DURATION, DurationSerializer.instance.serialize(duration)); - } - addNativeValue("null", CQL3Type.Native.DURATION, null); - - // (mostly generates timestamp values with surreal values like in year 14273) - for (int i = 0; i < 20; i++) - { - long v = randLong(); - addNativeValue(TimestampSerializer.instance.toStringUTC(new Date(v)), CQL3Type.Native.TIMESTAMP, TimestampType.instance.fromString(Long.toString(v))); - } - addNativeValue("null", CQL3Type.Native.TIMESTAMP, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.TIMESTAMP, null); - - for (int i = 0; i < 20; i++) - { - TimeUUID v = TimeUUID.Generator.atUnixMillis(randLong(currentTimeMillis())); - addNativeValue(v.toString(), CQL3Type.Native.TIMEUUID, v.toBytes()); - } - addNativeValue("null", CQL3Type.Native.TIMEUUID, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.TIMEUUID, null); - - for (int i = 0; i < 20; i++) - { - UUID v = UUID.randomUUID(); - addNativeValue(v.toString(), CQL3Type.Native.UUID, UUIDType.instance.decompose(v)); - } - addNativeValue("null", CQL3Type.Native.UUID, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.UUID, null); - - for (int i = 0; i < 20; i++) - { - ByteBuffer v = randBytes(); - addNativeValue("0x" + BytesSerializer.instance.toString(v), CQL3Type.Native.BLOB, BytesType.instance.decompose(v)); - } - addNativeValue("0x", CQL3Type.Native.BLOB, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.BLOB, null); - - for (int i = 0; i < 20; i++) - { - InetAddress v; - try - { - v = InetAddress.getByAddress(new byte[]{ randByte(), randByte(), randByte(), randByte() }); - } - catch (UnknownHostException e) - { - throw new RuntimeException(e); - } - addNativeValue(v.getHostAddress(), CQL3Type.Native.INET, InetAddressSerializer.instance.serialize(v)); - } - addNativeValue("null", CQL3Type.Native.INET, ByteBufferUtil.EMPTY_BYTE_BUFFER); - addNativeValue("null", CQL3Type.Native.INET, null); - } - - @Test - public void testNative() - { - // test each native type against each supported protocol version (although it doesn't make sense to - // iterate through all protocol versions as of C* 3.0). - - for (ProtocolVersion version : ProtocolVersion.SUPPORTED) - { - for (Map.Entry> entry : nativeTypeValues.entrySet()) - { - for (Value value : entry.getValue()) - { - compareCqlLiteral(version, value); - } - } - } - } - - @Test - public void testCollectionWithNatives() - { - // test 100 collections with varying element/key/value types against each supported protocol version, - // type of collection is randomly chosen - - for (ProtocolVersion version : ProtocolVersion.SUPPORTED) - { - for (int n = 0; n < 100; n++) - { - Value value = generateCollectionValue(version, randomCollectionType(0), true); - compareCqlLiteral(version, value); - } - } - } - - @Test - public void testCollectionNullAndEmpty() - { - // An empty collection is one with a size of 0 (note that rely on the fact that protocol version < 3 are not - // supported anymore and so the size of a collection is always on 4 bytes). - ByteBuffer emptyCollection = ByteBufferUtil.bytes(0); - - for (ProtocolVersion version : ProtocolVersion.SUPPORTED) - { - for (boolean frozen : Arrays.asList(true, false)) - { - // empty - Value value = new Value("[]", ListType.getInstance(UTF8Type.instance, frozen).asCQL3Type(), emptyCollection); - compareCqlLiteral(version, value); - value = new Value("{}", SetType.getInstance(UTF8Type.instance, frozen).asCQL3Type(), emptyCollection); - compareCqlLiteral(version, value); - value = new Value("{}", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, frozen).asCQL3Type(), emptyCollection); - compareCqlLiteral(version, value); - - // null - value = new Value("null", ListType.getInstance(UTF8Type.instance, frozen).asCQL3Type(), null); - compareCqlLiteral(version, value); - value = new Value("null", SetType.getInstance(UTF8Type.instance, frozen).asCQL3Type(), null); - compareCqlLiteral(version, value); - value = new Value("null", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, frozen).asCQL3Type(), null); - compareCqlLiteral(version, value); - } - } - } - - @Test - public void testTupleWithNatives() - { - // test 100 tuples with varying element/key/value types against each supported protocol version - - for (ProtocolVersion version : ProtocolVersion.SUPPORTED) - { - for (int n = 0; n < 100; n++) - { - Value value = generateTupleValue(version, randomTupleType(0), true); - compareCqlLiteral(version, value); - } - } - } - - @Test - public void testUserDefinedWithNatives() - { - // test 100 UDTs with varying element/key/value types against each supported protocol version - - for (ProtocolVersion version : ProtocolVersion.SUPPORTED) - { - for (int n = 0; n < 100; n++) - { - Value value = generateUserDefinedValue(version, randomUserType(0), true); - compareCqlLiteral(version, value); - } - } - } - - @Test - public void testNested() - { - // This is the "nice" part of this unit test - it tests (probably) nested type structures - // like 'tuple, tuple, user>' or 'map, set>' with - // random types against each supported protocol version. - - for (ProtocolVersion version : ProtocolVersion.SUPPORTED) - { - for (int n = 0; n < 100; n++) - { - Value value = randomNested(version); - compareCqlLiteral(version, value); - } - } - } - - static void compareCqlLiteral(ProtocolVersion version, Value value) - { - ByteBuffer buffer = value.value != null ? value.value.duplicate() : null; - String msg = "Failed to get expected value for type " + value.cql3Type + " / " + value.cql3Type.getType() + " with protocol-version " + version + " expected:\"" + value.expected + '"'; - try - { - assertEquals(msg, - value.expected, - value.cql3Type.toCQLLiteral(buffer, version)); - } - catch (RuntimeException e) - { - throw new RuntimeException(msg, e); - } - } - - static Value randomNested(ProtocolVersion version) - { - AbstractType type = randomNestedType(2); - - return generateAnyValue(version, type.asCQL3Type()); - } - - /** - * Generates type of randomly nested type structures. - */ - static AbstractType randomNestedType(int level) - { - if (level == 0) - return randomNativeType(); - switch (randInt(level == 2 ? 3 : 4)) - { - case 0: - return randomCollectionType(level - 1); - case 1: - return randomTupleType(level - 1); - case 2: - return randomUserType(level - 1); - case 3: - return randomNativeType(); - } - throw new AssertionError(); - } - - static Value generateCollectionValue(ProtocolVersion version, CollectionType collectionType, boolean allowNull) - { - StringBuilder expected = new StringBuilder(); - ByteBuffer buffer; - - if (allowNull && randBool(0.05d)) - { - expected.append("null"); - buffer = null; - } - else - { - int size = randInt(20); - - CQL3Type elements; - CQL3Type values = null; - char bracketOpen; - char bracketClose; - switch (collectionType.kind) - { - case LIST: - elements = ((ListType) collectionType).getElementsType().asCQL3Type(); - bracketOpen = '['; - bracketClose = ']'; - break; - case SET: - elements = ((SetType) collectionType).getElementsType().asCQL3Type(); - bracketOpen = '{'; - bracketClose = '}'; - break; - case MAP: - elements = ((MapType) collectionType).getKeysType().asCQL3Type(); - values = ((MapType) collectionType).getValuesType().asCQL3Type(); - bracketOpen = '{'; - bracketClose = '}'; - break; - default: - throw new AssertionError(); - } - - expected.append(bracketOpen); - Collection buffers = new ArrayList<>(); - Set added = new HashSet<>(); - for (int i = 0; i < size; i++) - { - Value el = generateAnyValue(version, elements); - if (!added.add(el.value)) - continue; - - buffers.add(el.value.duplicate()); - if (expected.length() > 1) - expected.append(", "); - expected.append(el.cql3Type.toCQLLiteral(el.value, version)); - - if (collectionType.kind == CollectionType.Kind.MAP) - { - // add map value - el = generateAnyValue(version, values); - buffers.add(el.value.duplicate()); - expected.append(": "); - expected.append(el.cql3Type.toCQLLiteral(el.value, version)); - } - } - expected.append(bracketClose); - buffer = CollectionSerializer.pack(buffers, added.size()); - } - - return new Value(expected.toString(), collectionType.asCQL3Type(), buffer); - } - - /** - * Generates a value for any type or type structure. - */ - static Value generateAnyValue(ProtocolVersion version, CQL3Type type) - { - if (type instanceof CQL3Type.Native) - return generateNativeValue(type, false); - if (type instanceof CQL3Type.Tuple) - return generateTupleValue(version, (TupleType) type.getType(), false); - if (type instanceof CQL3Type.UserDefined) - return generateUserDefinedValue(version, (UserType) type.getType(), false); - if (type instanceof CQL3Type.Collection) - return generateCollectionValue(version, (CollectionType) type.getType(), false); - throw new AssertionError(); - } - - static Value generateTupleValue(ProtocolVersion version, TupleType tupleType, boolean allowNull) - { - StringBuilder expected = new StringBuilder(); - ByteBuffer buffer; - - if (allowNull && randBool(0.05d)) - { - // generate 'null' collection - expected.append("null"); - buffer = null; - } - else - { - expected.append('('); - - // # of fields in this value - int fields = tupleType.size(); - if (randBool(0.2d)) - fields = randInt(fields); - - ByteBuffer[] buffers = new ByteBuffer[fields]; - for (int i = 0; i < fields; i++) - { - AbstractType fieldType = tupleType.type(i); - - if (i > 0) - expected.append(", "); - - if (allowNull && randBool(.1)) - { - expected.append("null"); - continue; - } - - Value value = generateAnyValue(version, fieldType.asCQL3Type()); - expected.append(value.expected); - buffers[i] = value.value.duplicate(); - } - expected.append(')'); - buffer = TupleType.buildValue(buffers); - } - - return new Value(expected.toString(), tupleType.asCQL3Type(), buffer); - } - - static Value generateUserDefinedValue(ProtocolVersion version, UserType userType, boolean allowNull) - { - StringBuilder expected = new StringBuilder(); - ByteBuffer buffer; - - if (allowNull && randBool(0.05d)) - { - // generate 'null' collection - expected.append("null"); - buffer = null; - } - else - { - expected.append('{'); - - // # of fields in this value - int fields = userType.size(); - if (randBool(0.2d)) - fields = randInt(fields); - - ByteBuffer[] buffers = new ByteBuffer[fields]; - for (int i = 0; i < fields; i++) - { - AbstractType fieldType = userType.type(i); - - if (i > 0) - expected.append(", "); - - expected.append(ColumnIdentifier.maybeQuote(userType.fieldNameAsString(i))); - expected.append(": "); - - if (randBool(.1)) - { - expected.append("null"); - continue; - } - - Value value = generateAnyValue(version, fieldType.asCQL3Type()); - expected.append(value.expected); - buffers[i] = value.value.duplicate(); - } - expected.append('}'); - buffer = TupleType.buildValue(buffers); - } - - return new Value(expected.toString(), userType.asCQL3Type(), buffer); - } - - static Value generateNativeValue(CQL3Type type, boolean allowNull) - { - List values = nativeTypeValues.get(type); - assert values != null : type.toString() + " needs to be defined"; - while (true) - { - Value v = values.get(randInt(values.size())); - if (allowNull || v.value != null) - return v; - } - } - - static CollectionType randomCollectionType(int level) - { - CollectionType.Kind kind = CollectionType.Kind.values()[randInt(CollectionType.Kind.values().length)]; - switch (kind) - { - case LIST: - case SET: - return ListType.getInstance(randomNestedType(level), randBool()); - case MAP: - return MapType.getInstance(randomNestedType(level), randomNestedType(level), randBool()); - } - throw new AssertionError(); - } - - static TupleType randomTupleType(int level) - { - int typeCount = 2 + randInt(5); - List> types = new ArrayList<>(); - for (int i = 0; i < typeCount; i++) - types.add(randomNestedType(level)); - return new TupleType(types); - } - - static UserType randomUserType(int level) - { - int typeCount = 2 + randInt(5); - List names = new ArrayList<>(); - List> types = new ArrayList<>(); - for (int i = 0; i < typeCount; i++) - { - names.add(FieldIdentifier.forQuoted('f' + randLetters(i))); - types.add(randomNestedType(level)); - } - return new UserType("ks", UTF8Type.instance.fromString("u" + randInt(1000000)), names, types, true); - } - - // - // Following methods are just helper methods. Mostly to generate many kinds of random values. - // - - private static String randLetters(int len) - { - StringBuilder sb = new StringBuilder(len); - while (len-- > 0) - { - int i = randInt(52); - if (i < 26) - sb.append((char) ('A' + i)); - else - sb.append((char) ('a' + i - 26)); - } - return sb.toString(); - } - - static AbstractType randomNativeType() - { - while (true) - { - CQL3Type.Native t = CQL3Type.Native.values()[randInt(CQL3Type.Native.values().length)]; - if (t != CQL3Type.Native.EMPTY) - return t.getType(); - } - } - - static boolean randBool() - { - return randBool(0.5d); - } - - static boolean randBool(double probability) - { - return r.nextDouble() < probability; - } - - static long randLong() - { - return r.nextLong(); - } - - static long randLong(long max) - { - return r.nextLong() % max; - } - - static int randInt() - { - return r.nextInt(); - } - - static int randInt(int max) - { - return r.nextInt(max); - } - - static short randShort() - { - return (short) r.nextInt(); - } - - static byte randByte() - { - return (byte) r.nextInt(); - } - - static double randDouble() - { - return r.nextDouble(); - } - - static float randFloat() - { - return r.nextFloat(); - } - - static String randString(boolean ascii) - { - int l = randInt(20); - StringBuilder sb = new StringBuilder(l); - for (int i = 0; i < l; i++) - { - if (randBool(.05)) - sb.append('\''); - else - { - char c = (char) (ascii ? randInt(128) : randShort()); - sb.append(c); - } - } - return UTF8Serializer.instance.deserialize(UTF8Serializer.instance.serialize(sb.toString())); - } - - static ByteBuffer randBytes() - { - int l = randInt(20); - byte[] v = new byte[l]; - for (int i = 0; i < l; i++) - { - v[i] = randByte(); - } - return ByteBuffer.wrap(v); - } - - private static String quote(String v) - { - return '\'' + QUOTE.matcher(v).replaceAll("''") + '\''; - } -} diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index 41b86f1c6e..c2e40d6f32 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -26,6 +26,7 @@ import java.net.MalformedURLException; import java.net.ServerSocket; import java.nio.ByteBuffer; import java.rmi.server.RMISocketFactory; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -45,9 +46,12 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; + import javax.management.MBeanServerConnection; import javax.management.remote.JMXConnector; import javax.management.remote.JMXConnectorFactory; @@ -59,6 +63,7 @@ import com.google.common.base.Objects; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import org.apache.commons.lang3.ArrayUtils; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -79,6 +84,8 @@ import com.datastax.driver.core.Session; import com.datastax.driver.core.SimpleStatement; import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.Statement; +import com.datastax.driver.core.UDTValue; +import com.datastax.driver.core.UserType; import com.datastax.driver.core.exceptions.UnauthorizedException; import com.datastax.shaded.netty.channel.EventLoopGroup; import org.apache.cassandra.SchemaLoader; @@ -102,6 +109,7 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.BooleanType; +import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.db.marshal.ByteType; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.CollectionType; @@ -122,6 +130,7 @@ import org.apache.cassandra.db.marshal.TimestampType; import org.apache.cassandra.db.marshal.TupleType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.db.marshal.VectorType; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.db.virtual.VirtualSchemaKeyspace; import org.apache.cassandra.dht.Murmur3Partitioner; @@ -141,6 +150,7 @@ import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SchemaKeyspace; import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.serializers.TypeSerializer; @@ -495,6 +505,14 @@ public abstract class CQLTester }); } + protected void resetSchema() throws Throwable + { + for (TableMetadata table : SchemaKeyspace.metadata().tables) + execute(String.format("TRUNCATE %s", table)); + Schema.instance.loadFromDisk(); + beforeTest(); + } + public static List buildNodetoolArgs(List args) { int port = jmxPort == 0 ? CASSANDRA_JMX_LOCAL_PORT.getInt(7199) : jmxPort; @@ -710,6 +728,21 @@ public abstract class CQLTester Util.flush(store); } + public void flush(String keyspace, String table1, String... tables) + { + tables = ArrayUtils.add(tables, table1); + for (ColumnFamilyStore store : getTables(keyspace, tables)) + Util.flush(store); + } + + private List getTables(String keyspace, String[] tables) + { + List stores = new ArrayList<>(tables.length); + for (String name : tables) + stores.add(getColumnFamilyStore(keyspace, name)); + return stores; + } + public void disableCompaction(String keyspace) { ColumnFamilyStore store = getCurrentColumnFamilyStore(keyspace); @@ -724,6 +757,13 @@ public abstract class CQLTester store.forceMajorCompaction(); } + public void compact(String keyspace, String table1, String... tables) + { + tables = ArrayUtils.add(tables, table1); + for (ColumnFamilyStore store : getTables(keyspace, tables)) + store.forceMajorCompaction(); + } + public void disableCompaction() { disableCompaction(KEYSPACE); @@ -1306,12 +1346,12 @@ public abstract class CQLTester return Schema.instance.getTableMetadata(KEYSPACE, currentTable()); } - protected com.datastax.driver.core.ResultSet executeNet(ProtocolVersion protocolVersion, String query, Object... values) throws Throwable + protected com.datastax.driver.core.ResultSet executeNet(ProtocolVersion protocolVersion, String query, Object... values) { return sessionNet(protocolVersion).execute(formatQuery(query), values); } - protected com.datastax.driver.core.ResultSet executeNet(String query, Object... values) throws Throwable + protected com.datastax.driver.core.ResultSet executeNet(String query, Object... values) { return sessionNet().execute(formatQuery(query), values); } @@ -1461,6 +1501,7 @@ public abstract class CQLTester protected void assertRowsNet(ProtocolVersion protocolVersion, ResultSet result, Object[]... rows) { + com.datastax.driver.core.ProtocolVersion version = com.datastax.driver.core.ProtocolVersion.fromInt(protocolVersion.asInt()); // necessary as we need cluster objects to supply CodecRegistry. // It's reasonably certain that the network setup has already been done // by the time we arrive at this point, but adding this check doesn't hurt @@ -1487,24 +1528,32 @@ public abstract class CQLTester for (int j = 0; j < meta.size(); j++) { + String name = meta.getName(j); DataType type = meta.getType(j); com.datastax.driver.core.TypeCodec codec = getCluster(protocolVersion).getConfiguration() .getCodecRegistry() .codecFor(type); - ByteBuffer expectedByteValue = codec.serialize(expected[j], com.datastax.driver.core.ProtocolVersion.fromInt(protocolVersion.asInt())); - int expectedBytes = expectedByteValue == null ? -1 : expectedByteValue.remaining(); - ByteBuffer actualValue = actual.getBytesUnsafe(meta.getName(j)); - int actualBytes = actualValue == null ? -1 : actualValue.remaining(); + ByteBuffer expectedByteValue = expected[j] instanceof ByteBuffer ? (ByteBuffer) expected[j] : codec.serialize(expected[j], version); + // Do not use the by-name lookup as the client calls toLowerCase, so may have cases where "J" and "j" are the same! + // See https://datastax-oss.atlassian.net/browse/JAVA-3067 +// ByteBuffer actualValue = actual.getBytesUnsafe(name); + ByteBuffer actualValue = actual.getBytesUnsafe(j); if (!Objects.equal(expectedByteValue, actualValue)) + { + if (isEmptyContainerNull(type, codec, version, expectedByteValue, actualValue)) + continue; + int expectedBytes = expectedByteValue == null ? -1 : expectedByteValue.remaining(); + int actualBytes = actualValue == null ? -1 : actualValue.remaining(); Assert.fail(String.format("Invalid value for row %d column %d (%s of type %s), " + "expected <%s> (%d bytes) but got <%s> (%d bytes) " + "(using protocol version %s)", - i, j, meta.getName(j), type, - codec.format(expected[j]), + i, j, name, type, + codec.format(expected[j] instanceof ByteBuffer ? codec.deserialize((ByteBuffer) expected[j], version) : expected[j]), expectedBytes, - codec.format(codec.deserialize(actualValue, com.datastax.driver.core.ProtocolVersion.fromInt(protocolVersion.asInt()))), + safeToString(() -> codec.format(codec.deserialize(actualValue, version))), actualBytes, protocolVersion)); + } } i++; } @@ -1524,6 +1573,57 @@ public abstract class CQLTester rows.length>i ? "less" : "more", rows.length, i, protocolVersion), i == rows.length); } + private static String safeToString(Supplier fn) + { + try + { + return fn.get(); + } + catch (Throwable t) + { + return "Unexpected error: " + t.getMessage(); + } + } + + private static boolean isEmptyContainerNull(AbstractType type, + ByteBuffer expectedByteValue, ByteBuffer actualValue) + { + // MAINTANCE : this MUST be in-sync with the DataType version + + // TODO confirm this isn't a bug... + // There is an edge case, UDTs... its always UDTs that cause problems.... :shakes-fist: + // If the user writes a null for each column, then the whole tuple is null + if (type.isUDT() && actualValue == null) + { + ByteBuffer[] cells = ((TupleType) type).split(ByteBufferAccessor.instance, expectedByteValue); + return Stream.of(cells).allMatch(b -> b == null); + } + return false; + } + + private static boolean isEmptyContainerNull(DataType type, + com.datastax.driver.core.TypeCodec codec, + com.datastax.driver.core.ProtocolVersion version, + ByteBuffer expectedByteValue, ByteBuffer actualValue) + { + // MAINTANCE : this MUST be in-sync with the AbstractType version + + // TODO confirm this isn't a bug... + // There is an edge case, UDTs... its always UDTs that cause problems.... :shakes-fist: + // If the user writes a null for each column, then the whole tuple is null + if (type instanceof UserType && actualValue == null) + { + UDTValue value = (UDTValue) codec.deserialize(expectedByteValue, version); + for (int c = 0; c < value.getType().size(); c++) + { + if (!value.isNull(c)) + return false; + } + return true; + } + return false; + } + protected void assertRowCountNet(ResultSet r1, int expectedCount) { Assert.assertFalse("Received a null resultset when expected count was > 0", expectedCount > 0 && r1 == null); @@ -1563,6 +1663,9 @@ public abstract class CQLTester { Object actualValueDecoded = actualValue == null ? null : column.type.getSerializer().deserialize(actualValue); if (!Objects.equal(expected != null ? expected[j] : null, actualValueDecoded)) + { + if (isEmptyContainerNull(column.type, expectedByteValue, actualValue)) + continue; error.append(String.format("Invalid value for row %d column %d (%s of type %s), expected <%s> but got <%s>", i, j, @@ -1570,6 +1673,7 @@ public abstract class CQLTester column.type.asCQL3Type(), formatValue(expectedByteValue != null ? expectedByteValue.duplicate() : null, column.type), formatValue(actualValue, column.type))).append("\n"); + } } } if (error.length() > 0) @@ -2217,7 +2321,14 @@ public abstract class CQLTester return ser.toString(ser.deserialize(bb)); } - return type.getString(bb); + try + { + return type.getString(bb); + } + catch (Exception | Error e) + { + return "getString failed for type " + type.asCQL3Type() + ": " + e.getMessage(); + } } protected TupleValue tuple(Object...values) @@ -2247,6 +2358,12 @@ public abstract class CQLTester return Arrays.asList(values); } + @SafeVarargs + protected final Vector vector(T... values) + { + return new Vector<>(values); + } + protected Set set(Object...values) { return ImmutableSet.copyOf(values); @@ -2295,6 +2412,28 @@ public abstract class CQLTester return metrics.get(metricName); } + public static class Vector extends AbstractList + { + private final T[] values; + + public Vector(T[] values) + { + this.values = values; + } + + @Override + public T get(int index) + { + return values[index]; + } + + @Override + public int size() + { + return values.length; + } + } + // Attempt to find an AbstracType from a value (for serialization/printing sake). // Will work as long as we use types we know of, which is good enough for testing private static AbstractType typeFor(Object value) @@ -2347,6 +2486,13 @@ public abstract class CQLTester if (value instanceof TimeUUID) return TimeUUIDType.instance; + // vector impl list, so have to check first + if (value instanceof Vector) + { + Vector v = (Vector) value; + return VectorType.getInstance(typeFor(v.values[0]), v.values.length); + } + if (value instanceof List) { List l = (List)value; diff --git a/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java b/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java new file mode 100644 index 0000000000..35297400bc --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableMap; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableList; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.marshal.UserType; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.AbstractTypeGenerators; +import org.apache.cassandra.utils.AbstractTypeGenerators.TypeGenBuilder; +import org.apache.cassandra.utils.CassandraGenerators; +import org.apache.cassandra.utils.CassandraGenerators.TableMetadataBuilder; +import org.apache.cassandra.utils.FailingConsumer; +import org.apache.cassandra.utils.Generators; +import org.quicktheories.core.Gen; +import org.quicktheories.core.RandomnessSource; +import org.quicktheories.generators.SourceDSL; +import org.quicktheories.impl.JavaRandom; + +import static org.apache.cassandra.utils.Generators.IDENTIFIER_GEN; + +public class RandomSchemaTest extends CQLTester.InMemory +{ + private static final Logger logger = LoggerFactory.getLogger(RandomSchemaTest.class); + + static + { + // make sure blob is always the same + CassandraRelevantProperties.TEST_BLOB_SHARED_SEED.setInt(42); + + requireNetwork(); + } + + @Test + public void test() + { + // in accord branch there is a much cleaner api for this pattern... + Gen domainGen = SourceDSL.integers().between(1, 100).map(i -> i < 2 ? AbstractTypeGenerators.ValueDomain.NULL : i < 4 ? AbstractTypeGenerators.ValueDomain.EMPTY_BYTES : AbstractTypeGenerators.ValueDomain.NORMAL); + // make sure ordering is determanstic, else repeatability breaks + NavigableMap> formats = new TreeMap<>(DatabaseDescriptor.getSSTableFormats()); + Gen> ssTableFormatGen = SourceDSL.arbitrary().pick(new ArrayList<>(formats.values())); + qt().checkAssert(random -> { + resetSchema(); + + // TODO : when table level override of sstable format is allowed, migrate to that + SSTableFormat sstableFormat = ssTableFormatGen.generate(random); + DatabaseDescriptor.setSelectedSSTableFormat(sstableFormat); + + Gen udtName = Generators.unique(IDENTIFIER_GEN); + + TypeGenBuilder withoutUnsafeEquality = AbstractTypeGenerators.withoutUnsafeEquality() + .withUserTypeKeyspace(KEYSPACE) + .withUDTNames(udtName); + TableMetadata metadata = new TableMetadataBuilder() + .withKeyspaceName(KEYSPACE) + .withTableKinds(TableMetadata.Kind.REGULAR) + .withKnownMemtables() + .withDefaultTypeGen(AbstractTypeGenerators.builder() + .withoutEmpty() + .withUserTypeKeyspace(KEYSPACE) + .withMaxDepth(2) + .withDefaultSetKey(withoutUnsafeEquality) + .withoutTypeKinds(AbstractTypeGenerators.TypeKind.COUNTER) + .withUDTNames(udtName) + .build()) + .withPartitionColumnsCount(1) + .withPrimaryColumnTypeGen(new TypeGenBuilder(withoutUnsafeEquality) + // map of vector of map crossed the size cut-off for one of the tests, so changed max depth from 2 to 1, so we can't have the second map + .withMaxDepth(1) + .build()) + .withClusteringColumnsBetween(1, 2) + .withRegularColumnsBetween(1, 5) + .withStaticColumnsBetween(0, 2) + .build(random); + maybeCreateUDTs(metadata); + String createTable = metadata.toCqlString(false, false); + // just to make the CREATE TABLE stmt easier to read for CUSTOM types + createTable = createTable.replaceAll("org.apache.cassandra.db.marshal.", ""); + createTable(KEYSPACE, createTable); + + Gen dataGen = CassandraGenerators.data(metadata, domainGen); + String insertStmt = insertStmt(metadata); + int partitionColumnCount = metadata.partitionKeyColumns().size(); + int primaryColumnCount = primaryColumnCount(metadata); + String selectStmt = selectStmt(metadata); + String tokenStmt = tokenStmt(metadata); + + for (int i = 0; i < 100; i++) + { + ByteBuffer[] expected = dataGen.generate(random); + try + { + ByteBuffer[] partitionKeys = Arrays.copyOf(expected, partitionColumnCount); + ByteBuffer[] rowKey = Arrays.copyOf(expected, primaryColumnCount); + execute(insertStmt, expected); + // check memtable + assertRows(execute(selectStmt, rowKey), expected); + assertRows(execute(tokenStmt, partitionKeys), partitionKeys); + assertRowsNet(executeNet(selectStmt, rowKey), expected); + + // check sstable + flush(KEYSPACE, metadata.name); + compact(KEYSPACE, metadata.name); + assertRows(execute(selectStmt, rowKey), expected); + assertRows(execute(tokenStmt, partitionKeys), partitionKeys); + assertRowsNet(executeNet(selectStmt, rowKey), expected); + + execute("TRUNCATE " + metadata); + } + catch (Throwable t) + { + Iterator it = metadata.allColumnsInSelectOrder(); + List literals = new ArrayList<>(expected.length); + for (int idx = 0; idx < expected.length; idx++) + { + assert it.hasNext(); + ColumnMetadata meta = it.next(); + ByteBuffer expct = expected[idx]; + literals.add(expct == null ? "null" : !expct.hasRemaining() ? "empty" : meta.type.asCQL3Type().toCQLLiteral(expct)); + } + throw new AssertionError(String.format("Failure at attempt %d with schema\n%s\nAnd SSTable Format %s\nfor values %s", i, createTable, DatabaseDescriptor.getSelectedSSTableFormat(), literals), t); + } + } + }); + } + + private void maybeCreateUDTs(TableMetadata metadata) + { + Set udts = CassandraGenerators.extractUDTs(metadata); + if (!udts.isEmpty()) + { + Deque pending = new ArrayDeque<>(udts); + Set created = new HashSet<>(); + while (!pending.isEmpty()) + { + UserType next = pending.poll(); + Set subTypes = AbstractTypeGenerators.extractUDTs(next); + subTypes.remove(next); // it includes self + if (subTypes.isEmpty() || subTypes.stream().allMatch(t -> created.contains(t.name))) + { + String cql = next.toCqlString(false, false); + logger.warn("Creating UDT {}", cql); + schemaChange(cql); + created.add(next.name); + } + else + { + logger.warn("Unable to create UDT {}; following sub-types still not created: {}", + next.getCqlTypeName(), + subTypes.stream().filter(t -> !created.contains(t.name)).collect(Collectors.toSet())); + pending.add(next); + } + } + } + } + + private static int primaryColumnCount(TableMetadata metadata) + { + return metadata.partitionKeyColumns().size() + metadata.clusteringColumns().size(); + } + + private static String tokenStmt(TableMetadata metadata) + { + StringBuilder sb = new StringBuilder(); + List columns = metadata.partitionKeyColumns().stream().map(column -> column.name.toCQLString()).collect(Collectors.toList()); + List binds = columns.stream().map(ignore -> "?").collect(Collectors.toList()); + sb.append("SELECT ").append(String.join(", ", columns)); + sb.append(" FROM ").append(metadata); + sb.append(" WHERE token(").append(String.join(", ", columns)).append(") = token(").append(String.join(", ", binds)).append(")"); + return sb.toString(); + } + + private String selectStmt(TableMetadata metadata) + { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ").append(metadata).append(" WHERE "); + for (ColumnMetadata column : ImmutableList.builder() + .addAll(metadata.partitionKeyColumns()) + .addAll(metadata.clusteringColumns()) + .build()) + { + sb.append(column.name.toCQLString()).append(" = ? AND "); + } + sb.setLength(sb.length() - " AND ".length()); + return sb.toString(); + } + + private String insertStmt(TableMetadata metadata) + { + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO ").append(metadata.toString()).append(" ("); + Iterator cols = metadata.allColumnsInSelectOrder(); + while (cols.hasNext()) + sb.append(cols.next().name.toCQLString()).append(", "); + sb.setLength(sb.length() - 2); // remove last ", " + sb.append(") VALUES ("); + for (int i = 0; i < metadata.columns().size(); i++) + { + if (i > 0) + sb.append(", "); + sb.append('?'); + } + sb.append(")"); + return sb.toString(); + } + + private static Builder qt() + { + return new Builder(); + } + + public static class Builder + { + private long seed = System.currentTimeMillis(); + private int examples = 10; + + /** + * Used to reproduce the test with a fixed seed; main use case is to rerun the same test that failed in CI. + * + * This function is dead code, it exists to help authors debug when a failing test is found. + */ + @SuppressWarnings("unused") + public Builder withFixedSeed(long seed) + { + this.seed = seed; + return this; + } + + /** + * Used to override how many examples to run with. This is dead code, but exists to allow authors to run the test + * many times to make sure its stable before committing. + */ + @SuppressWarnings("unused") + public Builder withExamples(int examples) + { + this.examples = examples; + return this; + } + + // copied from java.util.Random + private static final long multiplier = 0x5DEECE66DL; + private static final long addend = 0xBL; + private static final long mask = (1L << 48) - 1; + + public void checkAssert(FailingConsumer test) + { + JavaRandom random = new JavaRandom(seed); + for (int i = 0; i < examples; i++) + { + if (i > 0) + seed = (seed * multiplier + addend) & mask; + random.setSeed(seed); + try + { + test.doAccept(random); + } + catch (Throwable e) + { + throw new PropertyError(seed, e); + } + } + } + } + + public static class PropertyError extends AssertionError + { + public PropertyError(long seed, Throwable cause) + { + super(message(seed), cause); + } + + private static String message(long seed) + { + return "Failure for seed " + seed; + } + } +} diff --git a/test/unit/org/apache/cassandra/cql3/ViewAbstractParameterizedTest.java b/test/unit/org/apache/cassandra/cql3/ViewAbstractParameterizedTest.java index 629d40700d..a8c73ff84c 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewAbstractParameterizedTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewAbstractParameterizedTest.java @@ -54,7 +54,7 @@ public abstract class ViewAbstractParameterizedTest extends ViewAbstractTest } @Override - protected com.datastax.driver.core.ResultSet executeNet(String query, Object... values) throws Throwable + protected com.datastax.driver.core.ResultSet executeNet(String query, Object... values) { return executeNet(version, query, values); } diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java index 8e83fd2fb8..59f84de5f7 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java @@ -33,15 +33,18 @@ import org.junit.Test; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.ByteBufferAccessor; +import org.apache.cassandra.db.marshal.DecimalType; +import org.apache.cassandra.db.marshal.DurationType; import org.apache.cassandra.db.marshal.TupleType; +import org.apache.cassandra.utils.AbstractTypeGenerators; import org.apache.cassandra.utils.AbstractTypeGenerators.TypeSupport; import org.quicktheories.core.Gen; import org.quicktheories.generators.SourceDSL; import static org.apache.cassandra.db.SchemaCQLHelper.toCqlType; import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport; -import static org.apache.cassandra.utils.AbstractTypeGenerators.primitiveTypeGen; import static org.apache.cassandra.utils.AbstractTypeGenerators.tupleTypeGen; import static org.apache.cassandra.utils.FailingConsumer.orFail; import static org.apache.cassandra.utils.Generators.filter; @@ -333,7 +336,14 @@ public class TupleTypeTest extends CQLTester private static Gen typesAndRowsGen(int numRows) { - Gen typeGen = tupleTypeGen(primitiveTypeGen(), SourceDSL.integers().between(1, 10)); + Gen> subTypeGen = AbstractTypeGenerators.builder() + .withTypeKinds(AbstractTypeGenerators.TypeKind.PRIMITIVE) + // ordering doesn't make sense for duration + .withoutPrimitive(DurationType.instance) + // data is "normalized" causing equality matches to fail + .withoutPrimitive(DecimalType.instance) + .build(); + Gen typeGen = tupleTypeGen(subTypeGen, SourceDSL.integers().between(1, 10)); Set distinctRows = new HashSet<>(numRows); // reuse the memory Gen gen = rnd -> { TypeAndRows c = new TypeAndRows(); diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java index fb37f3f5c8..8202e8031a 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AggregationTest.java @@ -1981,7 +1981,7 @@ public class AggregationTest extends CQLTester AbstractType compositeType = TypeParser.parse(type); ByteBuffer compositeTypeValue = compositeType.fromString("s@foo:i@32"); - String compositeTypeString = compositeType.asCQL3Type().toCQLLiteral(compositeTypeValue, ProtocolVersion.CURRENT); + String compositeTypeString = compositeType.asCQL3Type().toCQLLiteral(compositeTypeValue); // ensure that the composite type is serialized using the 'blob syntax' assertTrue(compositeTypeString.startsWith("0x")); diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java new file mode 100644 index 0000000000..c1025659ca --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CQLVectorTest.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3.validation.operations; + +import java.nio.ByteBuffer; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.functions.Arguments; +import org.apache.cassandra.cql3.functions.FunctionArguments; +import org.apache.cassandra.cql3.functions.NativeFunctions; +import org.apache.cassandra.cql3.functions.NativeScalarFunction; +import org.apache.cassandra.db.marshal.FloatType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.VectorType; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.transport.ProtocolVersion; +import org.assertj.core.api.Assertions; + +public class CQLVectorTest extends CQLTester.InMemory +{ + @Test + public void select() + { + createTable(KEYSPACE, "CREATE TABLE %s (pk vector primary key)"); + + execute("INSERT INTO %s (pk) VALUES ([1, 2])"); + + Vector vector = vector(1, 2); + Object[] row = row(vector); + + assertRows(execute("SELECT * FROM %s WHERE pk = [1, 2]"), row); + assertRows(execute("SELECT * FROM %s WHERE pk = ?", vector), row); + assertRows(execute("SELECT * FROM %s WHERE pk = [1, 1 + 1]"), row); + assertRows(execute("SELECT * FROM %s WHERE pk = [1, ?]", 2), row); + assertRows(execute("SELECT * FROM %s WHERE pk = [1, (int) ?]", 2), row); + assertRows(execute("SELECT * FROM %s WHERE pk = [1, 1 + (int) ?]", 1), row); + + assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, 2])"), row); + assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, 2], [1, 2])"), row); + assertRows(execute("SELECT * FROM %s WHERE pk IN (?)", vector), row); + assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, 1 + 1])"), row); + assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, ?])", 2), row); + assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, (int) ?])", 2), row); + assertRows(execute("SELECT * FROM %s WHERE pk IN ([1, 1 + (int) ?])", 1), row); + + assertRows(execute("SELECT * FROM %s WHERE pk > [0, 0] AND pk < [1, 3] ALLOW FILTERING"), row); + assertRows(execute("SELECT * FROM %s WHERE token(pk) = token([1, 2])"), row); + + assertRows(execute("SELECT * FROM %s"), row); + Assertions.assertThat(execute("SELECT * FROM %s").one().getVector("pk", Int32Type.instance, 2)) + .isEqualTo(vector); + } + + @Test + public void selectNonPk() + { + createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector)"); + + execute("INSERT INTO %s (pk, value) VALUES (0, [1, 2])"); + assertRows(execute("SELECT * FROM %s WHERE value=[1, 2] ALLOW FILTERING"), row(0, list(1, 2))); + } + + @Test + public void insert() + { + Runnable test = () -> { + assertRows(execute("SELECT * FROM %s"), row(list(1, 2))); + execute("TRUNCATE %s"); + assertRows(execute("SELECT * FROM %s")); + }; + + createTable(KEYSPACE, "CREATE TABLE %s (pk vector primary key)"); + + execute("INSERT INTO %s (pk) VALUES ([1, 2])"); + test.run(); + + execute("INSERT INTO %s (pk) VALUES (?)", vector(1, 2)); + test.run(); + + execute("INSERT INTO %s (pk) VALUES ([1, 1 + 1])"); + test.run(); + + execute("INSERT INTO %s (pk) VALUES ([1, ?])", 2); + test.run(); + + execute("INSERT INTO %s (pk) VALUES ([1, (int) ?])", 2); + test.run(); + + execute("INSERT INTO %s (pk) VALUES ([1, 1 + (int) ?])", 1); + test.run(); + } + + @Test + public void insertNonPK() + { + Runnable test = () -> { + assertRows(execute("SELECT * FROM %s"), row(0, list(1, 2))); + execute("TRUNCATE %s"); + assertRows(execute("SELECT * FROM %s")); + }; + + createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector)"); + + execute("INSERT INTO %s (pk, value) VALUES (0, [1, 2])"); + test.run(); + + execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector(1, 2)); + test.run(); + + execute("INSERT INTO %s (pk, value) VALUES (0, [1, 1 + 1])"); + test.run(); + + execute("INSERT INTO %s (pk, value) VALUES (0, [1, ?])", 2); + test.run(); + + execute("INSERT INTO %s (pk, value) VALUES (0, [1, (int) ?])", 2); + test.run(); + + execute("INSERT INTO %s (pk, value) VALUES (0, [1, 1 + (int) ?])", 1); + test.run(); + } + + @Test + public void update() + { + Runnable test = () -> { + assertRows(execute("SELECT * FROM %s"), row(0, list(1, 2))); + execute("TRUNCATE %s"); + assertRows(execute("SELECT * FROM %s")); + }; + + createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector)"); + + execute("UPDATE %s set VALUE = [1, 2] WHERE pk = 0"); + test.run(); + + execute("UPDATE %s set VALUE = ? WHERE pk = 0", vector(1, 2)); + test.run(); + + execute("UPDATE %s set VALUE = [1, 1 + 1] WHERE pk = 0"); + test.run(); + + execute("UPDATE %s set VALUE = [1, ?] WHERE pk = 0", 2); + test.run(); + + execute("UPDATE %s set VALUE = [1, (int) ?] WHERE pk = 0", 2); + test.run(); + + execute("UPDATE %s set VALUE = [1, 1 + (int) ?] WHERE pk = 0", 1); + test.run(); + } + + @Test + public void functions() + { + VectorType type = VectorType.getInstance(Int32Type.instance, 2); + Vector vector = vector(1, 2); + + NativeFunctions.instance.add(new NativeScalarFunction("f", type, type) + { + @Override + public ByteBuffer execute(Arguments arguments) throws InvalidRequestException + { + return arguments.get(0); + } + + @Override + public Arguments newArguments(ProtocolVersion version) + { + return FunctionArguments.newNoopInstance(version, 1); + } + }); + + createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector)"); + execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector); + + assertRows(execute("SELECT f(value) FROM %s WHERE pk=0"), row(vector)); + assertRows(execute("SELECT f([1, 2]) FROM %s WHERE pk=0"), row(vector)); + } + + @Test + public void specializedFunctions() + { + VectorType type = VectorType.getInstance(FloatType.instance, 2); + Vector vector = vector(1.0f, 2.0f); + + NativeFunctions.instance.add(new NativeScalarFunction("f", type, type, type) + { + @Override + public ByteBuffer execute(Arguments arguments) throws InvalidRequestException + { + float[] left = arguments.get(0); + float[] right = arguments.get(1); + int size = Math.min(left.length, right.length); + float[] sum = new float[size]; + for (int i = 0; i < size; i++) + sum[i] = left[i] + right[i]; + return type.decomposeAsFloat(sum); + } + + @Override + public Arguments newArguments(ProtocolVersion version) + { + return new FunctionArguments(version, + (v, b) -> type.composeAsFloat(b), + (v, b) -> type.composeAsFloat(b)); + } + }); + + createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector)"); + execute("INSERT INTO %s (pk, value) VALUES (0, ?)", vector); + execute("INSERT INTO %s (pk, value) VALUES (1, ?)", vector); + + Object[][] expected = { row(vector(2f, 4f)), row(vector(2f, 4f)) }; + assertRows(execute("SELECT f(value, [1.0, 2.0]) FROM %s"), expected); + assertRows(execute("SELECT f([1.0, 2.0], value) FROM %s"), expected); + } + + @Test + public void token() + { + createTable(KEYSPACE, "CREATE TABLE %s (pk vector primary key)"); + execute("INSERT INTO %s (pk) VALUES (?)", vector(1, 2)); + long tokenColumn = execute("SELECT token(pk) as t FROM %s").one().getLong("t"); + long tokenTerminal = execute("SELECT token([1, 2]) as t FROM %s").one().getLong("t"); + Assert.assertEquals(tokenColumn, tokenTerminal); + } + + @Test + public void udf() throws Throwable + { + // For future authors, if this test starts to fail as vectors become supported in UDFs, please update this test + // to test the integration and remove the requirement that we reject UDFs all together + createTable(KEYSPACE, "CREATE TABLE %s (pk int primary key, value vector)"); + Assertions.assertThatThrownBy(() -> createFunction(KEYSPACE, + "", + "CREATE FUNCTION %s (x vector) " + + "CALLED ON NULL INPUT " + + "RETURNS vector " + + "LANGUAGE java " + + "AS 'return x;'")) + .hasRootCauseMessage("Vectors are not supported on UDFs; given vector"); + + Assertions.assertThatThrownBy(() -> createFunction(KEYSPACE, + "", + "CREATE FUNCTION %s (x list>) " + + "CALLED ON NULL INPUT " + + "RETURNS list> " + + "LANGUAGE java " + + "AS 'return x;'")) + .hasRootCauseMessage("Vectors are not supported on UDFs; given list>"); + } +} diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertInvalidateSizedRecordsTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertInvalidateSizedRecordsTest.java new file mode 100644 index 0000000000..cf9709c315 --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertInvalidateSizedRecordsTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3.validation.operations; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.UUID; + +import com.google.common.base.StandardSystemProperty; +import org.junit.Test; + +import com.datastax.driver.core.exceptions.InvalidQueryException; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.tools.ToolRunner; +import org.apache.cassandra.utils.FBUtilities; +import org.assertj.core.api.Assertions; + +import static org.apache.cassandra.utils.ByteBufferUtil.EMPTY_BYTE_BUFFER; + +public class InsertInvalidateSizedRecordsTest extends CQLTester +{ + private static final ByteBuffer LARGE_BLOB = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT + 1); + private static final ByteBuffer MEDIUM_BLOB = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT / 2 + 10); + + { + requireNetwork(); + } + + @Test + public void singleValuePk() + { + createTable(KEYSPACE, "CREATE TABLE %s (a blob PRIMARY KEY)"); + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a) VALUES (?)", LARGE_BLOB)) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Key length of " + LARGE_BLOB.remaining() + " is longer than maximum of 65535"); + + // null / empty checks + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a) VALUES (?)", new Object[] {null})) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Invalid null value in condition for column a"); + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a) VALUES (?)", EMPTY_BYTE_BUFFER)) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Key may not be empty"); + } + + @Test + public void compositeValuePk() + { + createTable(KEYSPACE, "CREATE TABLE %s (a blob, b blob, PRIMARY KEY ((a, b)))"); + // sum of columns is too large + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", MEDIUM_BLOB, MEDIUM_BLOB)) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Key length of " + (MEDIUM_BLOB.remaining() * 2) + " is longer than maximum of 65535"); + + // single column is too large + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", MEDIUM_BLOB, LARGE_BLOB)) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Key length of " + (MEDIUM_BLOB.remaining() + LARGE_BLOB.remaining()) + " is longer than maximum of 65535"); + + // null / empty checks + // this is an inconsistent behavior... null is blocked by org.apache.cassandra.db.MultiCBuilder.OneClusteringBuilder.addElementToAll + // but this does not count empty as null, and doesn't check for this case... We have a requirement in cqlsh that empty is allowed when + // user opts-in to allow it (NULL='-'), so we will find that null is blocked, but empty is allowed! + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", new Object[] {null, null})) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Invalid null value in condition for column a"); + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", new Object[] {MEDIUM_BLOB, null})) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Invalid null value in condition for column b"); + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", new Object[] {null, MEDIUM_BLOB})) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Invalid null value in condition for column a"); + + // empty is allowed when composite partition columns... + executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", EMPTY_BYTE_BUFFER, EMPTY_BYTE_BUFFER); + execute("TRUNCATE %s"); + + executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", MEDIUM_BLOB, EMPTY_BYTE_BUFFER); + execute("TRUNCATE %s"); + + executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", EMPTY_BYTE_BUFFER, MEDIUM_BLOB); + } + + @Test + public void singleValueClustering() + { + createTable(KEYSPACE, "CREATE TABLE %s (a blob, b blob, PRIMARY KEY (a, b))"); + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", MEDIUM_BLOB, LARGE_BLOB)) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Key length of " + LARGE_BLOB.remaining() + " is longer than maximum of 65535"); + + // null / empty checks + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", MEDIUM_BLOB, null)) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Invalid null value in condition for column b"); + + // org.apache.cassandra.db.MultiCBuilder.OneClusteringBuilder.addElementToAll defines "null" differently than most of the code; + // most of the code defines null as: + // value == null || accessor.isEmpty(value) + // but the code defines null as + // value == null + // In CASSANDRA-18504 a new isNull method was added to the type, as blob and text both "should" allow empty, but this scattered null logic doesn't allow... + // For backwards compatability reasons, need to keep empty support + executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", MEDIUM_BLOB, EMPTY_BYTE_BUFFER); + } + + @Test + public void compositeValueClustering() + { + createTable(KEYSPACE, "CREATE TABLE %s (a blob, b blob, c blob, PRIMARY KEY (a, b, c))"); + // sum of columns is too large + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", MEDIUM_BLOB, MEDIUM_BLOB, MEDIUM_BLOB)) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Key length of " + (MEDIUM_BLOB.remaining() * 2) + " is longer than maximum of 65535"); + + // single column is too large + // the logic prints the total clustering size and not the single column's size that was too large + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", MEDIUM_BLOB, MEDIUM_BLOB, LARGE_BLOB)) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Key length of " + (MEDIUM_BLOB.remaining() + LARGE_BLOB.remaining()) + " is longer than maximum of 65535"); + } + + @Test + public void singleValueIndex() + { + createTable(KEYSPACE, "CREATE TABLE %s (a blob, b blob, PRIMARY KEY (a))"); + String table = KEYSPACE + "." + currentTable(); + execute("CREATE INDEX single_value_index ON %s (b)"); + Assertions.assertThatThrownBy(() -> executeNet("INSERT INTO %s (a, b) VALUES (?, ?)", MEDIUM_BLOB, LARGE_BLOB)) + .hasRootCauseInstanceOf(InvalidQueryException.class) + .hasRootCauseMessage("Cannot index value of size " + LARGE_BLOB.remaining() + " for index single_value_index on " + table + "(b) (maximum allowed size=65535)"); + } + + /** + * Partial port of cqlsh_tests.test_cqlsh_copy.TestCqlshCopy#test_reading_empty_strings_for_different_types + */ + @Test + public void readingEmptyStringsForDifferentTypes() throws IOException + { + createTable(KEYSPACE, "CREATE TABLE %s (\n" + + " a text,\n" + + " b text,\n" + + " c text,\n" + + " d text,\n" + + " o uuid,\n" + + " i1 bigint,\n" + + " i2 bigint,\n" + + " t text,\n" + + " i3 bigint,\n" + + " PRIMARY KEY ((a, b, c, d), o)\n" + + " )"); + + File csv = new File(Files.createTempFile(new File(StandardSystemProperty.JAVA_IO_TMPDIR.value()).toPath(), "test_reading_empty_strings_for_different_types", ".csv")); + try (Writer out = new OutputStreamWriter(new BufferedOutputStream(csv.newOutputStream(File.WriteMode.OVERWRITE)), StandardCharsets.UTF_8)) + { + out.write(",,,a1,645e7d3c-aef7-4e3c-b834-24b792cf2e55,,,,r1\n"); + } + String table = KEYSPACE + "." + currentTable(); + String template = "COPY %s FROM '%s' WITH NULL='-' AND PREPAREDSTATEMENTS = %s"; + // This is different from cqlsh_tests.test_cqlsh_copy.TestCqlshCopy#test_reading_empty_strings_for_different_types as "false" actually is broken! + // If you do false, then the parsing actually fails... and the test didn't actually check the return code from cqlsh so doesn't detect that it failed! + // What is worse is the test didn't truncate either, which means that the "false" case is reading the "true" case data! + // Rather than try to fix the test and cqlsh... this test only keeps the "true" logic as it is what is needed to make sure CASSANDRA-18504 + // didn't break things. + ToolRunner.invokeCqlsh(String.format(template, table, csv.absolutePath(), true)).assertOnCleanExit(); + assertRowsNet(executeNet("SELECT * FROM %s"), row("", "", "", "a1", UUID.fromString("645e7d3c-aef7-4e3c-b834-24b792cf2e55"), null, null, null, "r1")); + } +} diff --git a/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java new file mode 100644 index 0000000000..c38e4dd2ff --- /dev/null +++ b/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java @@ -0,0 +1,729 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.marshal; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.security.CodeSource; +import java.security.ProtectionDomain; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import org.junit.Test; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.cql3.AssignmentTestable; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.ColumnSpecification; +import org.apache.cassandra.cql3.Constants; +import org.apache.cassandra.cql3.Json; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.Term; +import org.apache.cassandra.cql3.VariableSpecifications; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.CQLTypeParser; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.Types; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.serializers.TypeSerializer; +import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.AbstractTypeGenerators; +import org.apache.cassandra.utils.AbstractTypeGenerators.Releaser; +import org.apache.cassandra.utils.AbstractTypeGenerators.TypeGenBuilder; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FastByteOperations; +import org.apache.cassandra.utils.Generators; +import org.apache.cassandra.utils.bytecomparable.ByteComparable; +import org.apache.cassandra.utils.bytecomparable.ByteSource; +import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; +import org.quicktheories.core.Gen; +import org.quicktheories.generators.SourceDSL; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ConfigurationBuilder; + +import static org.apache.cassandra.utils.AbstractTypeGenerators.TypeKind.*; +import static org.apache.cassandra.utils.AbstractTypeGenerators.TypeSupport.of; +import static org.apache.cassandra.utils.AbstractTypeGenerators.extractUDTs; +import static org.apache.cassandra.utils.AbstractTypeGenerators.overridePrimitiveTypeSupport; +import static org.apache.cassandra.utils.AbstractTypeGenerators.stringComparator; +import static org.apache.cassandra.utils.AbstractTypeGenerators.typeTree; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.quicktheories.QuickTheory.qt; +import static org.quicktheories.generators.SourceDSL.doubles; +import static org.quicktheories.generators.SourceDSL.floats; + +public class AbstractTypeTest +{ + static + { + // make sure blob is always the same + CassandraRelevantProperties.TEST_BLOB_SHARED_SEED.setInt(42); + } + + private static final Reflections reflections = new Reflections(new ConfigurationBuilder() + .forPackage("org.apache.cassandra") + .setScanners(Scanners.SubTypes) + .setExpandSuperTypes(true) + .setParallel(true)); + + // TODO + // isCompatibleWith/isValueCompatibleWith/isSerializationCompatibleWith, + // withUpdatedUserType/expandUserTypes/referencesDuration - types that recursive check types + // getMaskedValue + + @Test + public void empty() + { + qt().forAll(genBuilder().build()).checkAssert(type -> { + if (type.allowsEmpty()) + { + type.validate(ByteBufferUtil.EMPTY_BYTE_BUFFER); + // empty container or null is valid; only checks that this method doesn't fail + type.compose(ByteBufferUtil.EMPTY_BYTE_BUFFER); + } + else + { + assertThatThrownBy(() -> type.validate(ByteBufferUtil.EMPTY_BYTE_BUFFER)).isInstanceOf(MarshalException.class); + assertThatThrownBy(() -> type.getSerializer().validate(ByteBufferUtil.EMPTY_BYTE_BUFFER)).isInstanceOf(MarshalException.class); + // ByteSerializer returns null +// assertThatThrownBy(() -> type.compose(ByteBufferUtil.EMPTY_BYTE_BUFFER)).isInstanceOf(MarshalException.class); + } + }); + } + + @Test + @SuppressWarnings("rawtypes") + public void allTypesCovered() + { + // this test just makes sure that all types are covered and no new type is left out + Set> subTypes = reflections.getSubTypesOf(AbstractType.class); + Set> coverage = AbstractTypeGenerators.knownTypes(); + StringBuilder sb = new StringBuilder(); + for (Class klass : Sets.difference(subTypes, coverage)) + { + if (Modifier.isAbstract(klass.getModifiers())) + continue; + if (isTestType(klass)) + continue; + String name = klass.getCanonicalName(); + if (name == null) + name = klass.getName(); + sb.append(name).append('\n'); + } + if (sb.length() > 0) + throw new AssertionError("Uncovered types:\n" + sb); + } + + @SuppressWarnings("rawtypes") + private boolean isTestType(Class klass) + { + String name = klass.getCanonicalName(); + if (name == null) + name = klass.getName(); + if (name == null) + name = klass.toString(); + if (name.contains("Test")) + return true; + ProtectionDomain domain = klass.getProtectionDomain(); + if (domain == null) return false; + CodeSource src = domain.getCodeSource(); + if (src == null) return false; + return "test".equals(new File(src.getLocation().getPath()).name()); + } + + @Test + public void unsafeSharedSerializer() + { + // For all types, make sure the serializer returned is unique to that type, + // this is required as some places, such as SetSerializer, cache at this level! + Map, AbstractType> lookup = new HashMap<>(); + qt().forAll(genBuilder().withMaxDepth(0).build()).checkAssert(t -> { + AbstractType old = lookup.put(t.getSerializer(), t); + // for frozen types, ignore the fact that the mapping breaks... The reason this test exists is that + // org.apache.cassandra.db.marshal.AbstractType.comparatorSet needs to match the serializer, but when serialziers + // break this mapping they may cause the wrong comparator (happened in cases like uuid and lexecal uuid; which have different orderings!). + // Frozen types (as of this writing) do not change the sort ordering, so this simplification is fine... + if (old != null && !old.unfreeze().equals(t.unfreeze())) + throw new AssertionError(String.format("Different types detected that shared the same serializer: %s != %s", old.asCQL3Type(), t.asCQL3Type())); + }); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + public void eqHashSafe() + { + StringBuilder sb = new StringBuilder(); + outter: for (Class type : reflections.getSubTypesOf(AbstractType.class)) + { + if (Modifier.isAbstract(type.getModifiers()) || isTestType(type) || AbstractTypeGenerators.UNSUPPORTED.containsKey(type)) + continue; + boolean hasEq = false; + boolean hasHashCode = false; + for (Class t = type; !t.equals(AbstractType.class); t = (Class) t.getSuperclass()) + { + try + { + t.getDeclaredMethod("getInstance"); + continue outter; + } + catch (NoSuchMethodException e) + { + // ignore + } + try + { + t.getDeclaredField("instance"); + continue outter; + } + catch (NoSuchFieldException e) + { + // ignore + } + try + { + t.getDeclaredMethod("equals", Object.class); + hasEq = true; + } + catch (NoSuchMethodException e) + { + // ignore + } + try + { + t.getDeclaredMethod("hashCode"); + hasHashCode = true; + } + catch (NoSuchMethodException e) + { + // ignore + } + if (hasEq && hasHashCode) + continue outter; + } + sb.append("AbstractType must be safe for map keys, so must either be a singleton or define "); + if (!hasEq) + sb.append("equals"); + if (!hasHashCode) + { + if (!hasEq) + sb.append('/'); + sb.append("hashCode"); + } + sb.append("; ").append(type).append('\n'); + } + if (sb.length() != 0) + { + sb.setLength(sb.length() - 1); + throw new AssertionError(sb.toString()); + } + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + public void comparableBytes() + { + // decimal "normalizes" the data to compare, so primary columns "may" mutate the data, causing missmatches + // see CASSANDRA-18530 + TypeGenBuilder baseline = genBuilder().withoutPrimitive(DecimalType.instance) + .withoutTypeKinds(COUNTER); + // composite requires all elements fit into Short.MAX_VALUE bytes + // so try to limit the possible expansion of types + Gen> gen = baseline.withCompositeElementGen(new TypeGenBuilder(baseline).withDefaultSizeGen(1).withMaxDepth(1).build()) + .build(); + qt().withShrinkCycles(0).forAll(examples(1, gen)).checkAssert(example -> { + AbstractType type = example.type; + for (Object value : example.samples) + { + ByteBuffer bb = type.decompose(value); + for (ByteComparable.Version bcv : ByteComparable.Version.values()) + { + // LEGACY, // Encoding used in legacy sstable format; forward (value to byte-comparable) translation only + // Legacy is a one-way conversion, so for this test ignore + if (bcv == ByteComparable.Version.LEGACY) + continue; + // Test normal type APIs + ByteSource.Peekable comparable = ByteSource.peekable(type.asComparableBytes(bb, bcv)); + if (comparable == null) + throw new NullPointerException(); + ByteBuffer read; + try + { + read = type.fromComparableBytes(comparable, bcv); + } + catch (Exception | Error e) + { + throw new AssertionError(String.format("Unable to parse comparable bytes for type %s and version %s; value %s", type.asCQL3Type(), bcv, type.toCQLString(bb)), e); + } + assertBytesEquals(read, bb, "fromComparableBytes(asComparableBytes(bb)) != bb; version %s", bcv); + + // test byte[] api + byte[] bytes = ByteSourceInverse.readBytes(type.asComparableBytes(bb, bcv)); + assertBytesEquals(type.fromComparableBytes(ByteSource.peekable(ByteSource.fixedLength(bytes)), bcv), bb, "fromOrderedBytes(toOrderedBytes(bb)) != bb"); + } + } + }); + } + + @Test + public void knowThySelf() + { + qt().withShrinkCycles(0).forAll(AbstractTypeGenerators.typeGen()).checkAssert(type -> { + assertThat(type.testAssignment(null, new ColumnSpecification(null, null, null, type))).isEqualTo(AssignmentTestable.TestResult.EXACT_MATCH); + assertThat(type.testAssignment(type)).isEqualTo(AssignmentTestable.TestResult.EXACT_MATCH); + }); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + public void json() + { + // Double type is special as NaN and Infinite are treated differently than other code paths as they are convered to null! + // This is fine in most cases, but when found in a collection, this is not allowed and can cause flakeyness + try (Releaser i1 = overridePrimitiveTypeSupport(DoubleType.instance, + of(DoubleType.instance, doubles().between(Double.MIN_VALUE, Double.MAX_VALUE))); + Releaser i2 = overridePrimitiveTypeSupport(FloatType.instance, + of(FloatType.instance, floats().between(Float.MIN_VALUE, Float.MAX_VALUE)))) + { + Gen> typeGen = genBuilder() + .withDefaultSetKey(AbstractTypeGenerators.withoutUnsafeEquality().withoutTypeKinds(COMPOSITE, DYNAMIC_COMPOSITE, COUNTER)) + // toCQLLiteral is lossy, which causes deserialization to produce different bytes + .withoutPrimitive(DecimalType.instance) + // does not support toJSONString + .withoutTypeKinds(COMPOSITE, DYNAMIC_COMPOSITE, COUNTER) + .build(); + qt().withShrinkCycles(0).forAll(examples(1, typeGen)).checkAssert(es -> { + AbstractType type = es.type; + for (Object example : es.samples) + { + ByteBuffer bb = type.decompose(example); + String json = type.toJSONString(bb, ProtocolVersion.CURRENT); + ColumnMetadata column = fake(type); + String cqlJson = "{\"" + column.name + "\": " + json + "}"; + try + { + Json.Prepared prepared = new Json.Literal(cqlJson).prepareAndCollectMarkers(null, Collections.singletonList(column), VariableSpecifications.empty()); + Term.Raw literal = prepared.getRawTermForColumn(column, false); + assertThat(literal).isNotEqualTo(Constants.NULL_LITERAL); + Term term = literal.prepare(column.ksName, column); + ByteBuffer read = term.bindAndGet(QueryOptions.DEFAULT); + assertBytesEquals(read, bb, "fromJSONString(toJSONString(bb)) != bb"); + } + catch (Exception e) + { + throw new AssertionError("Unable to parse JSON for " + json + "; type " + type.asCQL3Type(), e); + } + } + }); + } + } + + @Test + @SuppressWarnings("rawtypes") + public void nested() + { + Map, Function, Integer>> complexTypes = ImmutableMap.of(MapType.class, ignore -> 2, + TupleType.class, t -> ((TupleType) t).size(), + UserType.class, t -> ((UserType) t).size(), + CompositeType.class, t -> ((CompositeType) t).types.size(), + DynamicCompositeType.class, t -> ((DynamicCompositeType) t).size()); + qt().withShrinkCycles(0).forAll(AbstractTypeGenerators.builder().withoutTypeKinds(PRIMITIVE, COUNTER).build()).checkAssert(type -> { + int expectedSize = complexTypes.containsKey(type.getClass()) ? complexTypes.get(type.getClass()).apply(type) : 1; + assertThat(type.subTypes()).hasSize(expectedSize); + }); + } + + /** + * @see 
CASSANDRA-18526: TupleType getString and fromString are not safe with string types
+ */ + private static boolean containsUnsafeGetString(AbstractType type) + { + type = type.unwrap(); + if (type instanceof TupleType) + { + TupleType tt = (TupleType) type; + for (AbstractType e : tt.subTypes()) + { + AbstractType unwrap = e.unwrap(); + if (unwrap instanceof StringType || unwrap instanceof TupleType) + return true; + } + } + else if (type instanceof DynamicCompositeType || type instanceof CompositeType) + { + for (AbstractType e : type.subTypes()) + { + AbstractType unwrap = e.unwrap(); + if (unwrap instanceof StringType) + return true; + } + } + for (AbstractType e : type.subTypes()) + { + if (containsUnsafeGetString(e)) + return true; + } + return false; + } + + private boolean containsUnsafeToLiteral(AbstractType type) + { + type = type.unwrap(); + if (type instanceof DecimalType) + // toCQLLiteral is loss + return true; + for (AbstractType e : type.subTypes()) + { + if (containsUnsafeToLiteral(e)) + return true; + } + return false; + } + + @Test + public void typeParser() + { + Gen> gen = genBuilder() + .withMaxDepth(1) + // UDTs produce bad type strings, which is required by org.apache.cassandra.io.sstable.SSTableHeaderFix + // fixing this may have bad side effects between 3.6 upgrading to 5.0... + .withoutTypeKinds(UDT) + .build(); + qt().withShrinkCycles(0).forAll(gen).checkAssert(type -> { + AbstractType parsed = TypeParser.parse(type.toString()); + assertThat(parsed).describedAs("TypeParser mismatch:\nExpected: %s\nActual: %s", typeTree(type), typeTree(parsed)).isEqualTo(type); + }); + } + + @Test + public void toStringIsCQLYo() + { + cqlTypeSerde(type -> "'" + type.toString() + "'"); + } + + @Test + public void cqlTypeSerde() + { + cqlTypeSerde(type -> type.asCQL3Type().toString()); + } + + private static void cqlTypeSerde(Function, String> cqlFunc) + { + // TODO : add UDT back + // exclude UDT from CQLTypeParser as the different toString methods do not produce a consistent types, unlike TypeParser + Gen> gen = genBuilder() + .withDefaultSetKey(AbstractTypeGenerators.withoutUnsafeEquality().withoutTypeKinds(UDT)) + .withoutTypeKinds(UDT) + .build(); + qt().withShrinkCycles(0).forAll(gen).checkAssert(type -> { + // to -> from cql + String cqlType = cqlFunc.apply(type); + // just easier to read this way... + cqlType = cqlType.replaceAll("org.apache.cassandra.db.marshal.", ""); + AbstractType fromCQLTypeParser = CQLTypeParser.parse(null, cqlType, toTypes(extractUDTs(type))); + assertThat(fromCQLTypeParser) + .describedAs("CQL type %s parse did not match the expected type:\nExpected: %s\nActual: %s", cqlType, typeTree(type), typeTree(fromCQLTypeParser)) + .isEqualTo(type); + }); + } + + @Test + public void serdeFromString() + { + // avoid empty bytes as fromString can't figure out what to do in cases such as tuple(bytes); the tuple getString = "" so was the column not defined or was it empty? + try (Releaser i1 = overridePrimitiveTypeSupport(BytesType.instance, of(BytesType.instance, Generators.bytes(1, 1024), FastByteOperations::compareUnsigned)); + Releaser i2 = overridePrimitiveTypeSupport(AsciiType.instance, of(AsciiType.instance, SourceDSL.strings().ascii().ofLengthBetween(1, 1024), stringComparator(AsciiType.instance))); + Releaser i3 = overridePrimitiveTypeSupport(UTF8Type.instance, of(UTF8Type.instance, Generators.utf8(1, 1024), stringComparator(UTF8Type.instance)))) + { + Gen> typeGen = genBuilder() + // a type maybe safe, but for some container types, specific element types are unsafe + .withTypeFilter(type -> !containsUnsafeGetString(type)) + // fromString(getString(bb)) does not work + .withoutPrimitive(DurationType.instance) + .withDefaultSetKey(AbstractTypeGenerators.withoutUnsafeEquality().withTypeFilter(type -> !containsUnsafeGetString(type))) + // composite requires all elements fit into Short.MAX_VALUE bytes + // so try to limit the possible expansion of types + .withCompositeElementGen(genBuilder().withoutPrimitive(DurationType.instance).withDefaultSizeGen(1).withMaxDepth(1).withTypeFilter(type -> !containsUnsafeGetString(type)).build()) + .build(); + qt().withShrinkCycles(0).forAll(examples(1, typeGen)).checkAssert(example -> { + AbstractType type = example.type; + + for (Object expected : example.samples) + { + ByteBuffer bb = type.decompose(expected); + type.validate(bb); + String str = type.getString(bb); + assertBytesEquals(type.fromString(str), bb, "fromString(getString(bb)) != bb; %s", str); + } + }); + } + } + + @Test + public void serdeFromCQLLiteral() + { + Gen> typeGen = genBuilder() + // parseLiteralType(toCQLLiteral(bb)) does not work + .withoutPrimitive(DurationType.instance) + .withDefaultSetKey(AbstractTypeGenerators.withoutUnsafeEquality()) + // composite requires all elements fit into Short.MAX_VALUE bytes + // so try to limit the possible expansion of types + .withCompositeElementGen(genBuilder().withoutPrimitive(DurationType.instance).withDefaultSizeGen(1).withMaxDepth(1).build()) + .build(); + qt().withShrinkCycles(0).forAll(examples(1, typeGen)).checkAssert(example -> { + AbstractType type = example.type; + + for (Object expected : example.samples) + { + ByteBuffer bb = type.decompose(expected); + type.validate(bb); + + String literal = type.asCQL3Type().toCQLLiteral(bb); + ByteBuffer cqlBB = parseLiteralType(type, literal); + assertBytesEquals(cqlBB, bb, "Deserializing literal %s did not match expected bytes", literal); + }}); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + public void serde() + { + Gen> typeGen = genBuilder() + .withDefaultSetKey(AbstractTypeGenerators.withoutUnsafeEquality()) + // composite requires all elements fit into Short.MAX_VALUE bytes + // so try to limit the possible expansion of types + .withCompositeElementGen(genBuilder().withDefaultSizeGen(1).withMaxDepth(1).build()) + .build(); + qt().withShrinkCycles(0).forAll(examples(1, typeGen)).checkAssert(example -> { + AbstractType type = example.type; + + for (Object expected : example.samples) + { + ByteBuffer bb = type.decompose(expected); + int position = bb.position(); + type.validate(bb); + Object read = type.compose(bb); + assertThat(bb.position()).describedAs("ByteBuffer was mutated by %s", type).isEqualTo(position); + assertThat(read).isEqualTo(expected); + + try (DataOutputBuffer out = DataOutputBuffer.scratchBuffer.get()) + { + type.writeValue(bb, out); + ByteBuffer written = out.unsafeGetBufferAndFlip(); + DataInputPlus in = new DataInputBuffer(written, true); + assertBytesEquals(type.readBuffer(in), bb, "readBuffer(writeValue(bb)) != bb"); + in = new DataInputBuffer(written, false); + type.skipValue(in); + assertThat(written.remaining()).isEqualTo(0); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + }); + } + + private static void assertBytesEquals(ByteBuffer actual, ByteBuffer expected, String msg, Object... args) + { + assertThat(ByteBufferUtil.bytesToHex(actual)).describedAs(msg, args).isEqualTo(ByteBufferUtil.bytesToHex(expected)); + } + + private static ColumnMetadata fake(AbstractType type) + { + return new ColumnMetadata(null, null, new ColumnIdentifier("", true), type, 0, ColumnMetadata.Kind.PARTITION_KEY, null); + } + + private static ByteBuffer parseLiteralType(AbstractType type, String literal) + { + try + { + return type.asCQL3Type().fromCQLLiteral(literal); + } + catch (Exception e) + { + throw new AssertionError(String.format("Unable to parse CQL literal %s from type %s", literal, type.asCQL3Type()), e); + } + } + + private static Types toTypes(Set udts) + { + if (udts.isEmpty()) + return Types.none(); + Types.Builder builder = Types.builder(); + for (UserType udt : udts) + builder.add(udt.unfreeze()); + return builder.build(); + } + + private static ByteComparable fromBytes(AbstractType type, ByteBuffer bb) + { + return version -> type.asComparableBytes(bb, version); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + public void ordering() + { + TypeGenBuilder baseline = genBuilder() + .withoutPrimitive(DurationType.instance) // this uses byte ordering and vint, which makes the ordering effectivlly random from a user's point of view + .withoutTypeKinds(COUNTER); // counters don't allow ordering + // composite requires all elements fit into Short.MAX_VALUE bytes + // so try to limit the possible expansion of types + Gen> types = baseline.withCompositeElementGen(new TypeGenBuilder(baseline).withDefaultSizeGen(1).withMaxDepth(1).build()) + .build(); + qt().withShrinkCycles(0).forAll(examples(10, types)).checkAssert(example -> { + AbstractType type = example.type; + List actual = decompose(type, example.samples); + actual.sort(type); + List[] byteOrdered = new List[ByteComparable.Version.values().length]; + List[] rawByteOrdered = new List[ByteComparable.Version.values().length]; + for (int i = 0; i < byteOrdered.length; i++) + { + byteOrdered[i] = new ArrayList<>(actual); + ByteComparable.Version version = ByteComparable.Version.values()[i]; + byteOrdered[i].sort((a, b) -> ByteComparable.compare(fromBytes(type, a), fromBytes(type, b), version)); + + rawByteOrdered[i] = actual.stream() + .map(bb -> new OrderedBytes(ByteSourceInverse.readBytes(fromBytes(type, bb).asComparableBytes(version)), bb)) + .collect(Collectors.toList()); + rawByteOrdered[i].sort(Comparator.naturalOrder()); + } + + example.samples.sort(comparator(type)); + List real = new ArrayList<>(actual.size()); + for (ByteBuffer bb : actual) + real.add(type.compose(bb)); + assertThat(real).isEqualTo(example.samples); + List[] realBytesOrder = new List[byteOrdered.length]; + for (int i = 0; i < realBytesOrder.length; i++) + { + ByteComparable.Version version = ByteComparable.Version.values()[i]; + assertThat(compose(type, byteOrdered[i])).describedAs("Bad ordering for type %s", version).isEqualTo(real); + assertThat(compose(type, rawByteOrdered[i].stream().map(ob -> ob.src).collect(Collectors.toList()))).describedAs("Bad ordering for type %s", version).isEqualTo(real); + } + }); + } + + /** + * For {@link AbstractType#asComparableBytes(ByteBuffer, ByteComparable.Version)} not all versions can be inverted, + * but all versions must be comparable... so this class stores the ordered bytes and the original src. + */ + private static class OrderedBytes implements Comparable + { + private final byte[] orderedBytes; + private final ByteBuffer src; + + private OrderedBytes(byte[] orderedBytes, ByteBuffer src) + { + this.orderedBytes = orderedBytes; + this.src = src; + } + + @Override + public int compareTo(OrderedBytes o) + { + return FastByteOperations.compareUnsigned(orderedBytes, o.orderedBytes); + } + } + + private static List compose(AbstractType type, List bbs) + { + List os = new ArrayList<>(bbs.size()); + for (ByteBuffer bb : bbs) + os.add(type.compose(bb)); + return os; + } + + @SuppressWarnings("unchecked") + private static Comparator comparator(AbstractType type) + { + return (Comparator) AbstractTypeGenerators.comparator(type); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private List decompose(AbstractType type, List value) + { + List expected = new ArrayList<>(value.size()); + for (int i = 0; i < value.size(); i++) + expected.add(type.decompose(value.get(i))); + return expected; + } + + private static TypeGenBuilder genBuilder() + { + return AbstractTypeGenerators.builder() + // empty is a legacy from 2.x and is only allowed in special cases and not allowed in all... as this class tests all cases, need to limit this type out + .withoutEmpty(); + } + + private static Gen examples(int samples, Gen> typeGen) + { + Gen gen = rnd -> { + AbstractType type = typeGen.generate(rnd); + AbstractTypeGenerators.TypeSupport support = AbstractTypeGenerators.getTypeSupport(type); + List list = new ArrayList<>(samples); + for (int i = 0; i < samples; i++) + list.add(support.valueGen.generate(rnd)); + return new Example(type, list); + }; + return gen.describedAs(e -> { + StringBuilder sb = new StringBuilder(); + sb.append("Type:\n").append(typeTree(e.type)); + sb.append("\nValues: ").append(e.samples); + return sb.toString(); + }); + } + + private static class Example + { + private final AbstractType type; + private final List samples; + + private Example(AbstractType type, List samples) + { + this.type = type; + this.samples = samples; + } + + @Override + public String toString() + { + return "{" + + "type=" + type + + ", value=" + samples + + '}'; + } + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java index ecdf0d70fd..0f3714870a 100644 --- a/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/CompositeTypeTest.java @@ -30,6 +30,7 @@ import org.junit.Test; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; import static org.junit.Assert.fail; import static org.junit.Assert.assertEquals; +import static org.quicktheories.QuickTheory.qt; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; @@ -44,6 +45,8 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.serializers.UUIDSerializer; import org.apache.cassandra.utils.*; +import org.assertj.core.api.Assertions; +import org.quicktheories.generators.SourceDSL; public class CompositeTypeTest { @@ -351,4 +354,19 @@ public class CompositeTypeTest for (ValueAccessor accessor : ValueAccessors.ACCESSORS) testToFromString(serialized, accessor, type); } + + @Test + public void testSizeHeader() + { + ByteBuffer bb = ByteBuffer.allocate(2); + qt().forAll(SourceDSL.integers().between(0, FBUtilities.MAX_UNSIGNED_SHORT)).checkAssert(size -> { + bb.clear(); + + ByteBufferUtil.writeShortLength(bb, size); + bb.flip(); + Assertions.assertThat(ByteBufferAccessor.instance.getUnsignedShort(bb, 0)) + .isEqualTo(ByteBufferUtil.readShortLength(bb)) + .isEqualTo(size); + }); + } } diff --git a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java index 69c1eb5cd8..c7952e6d84 100644 --- a/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/DynamicCompositeTypeTest.java @@ -25,6 +25,7 @@ import java.util.Iterator; import java.util.Map; import java.util.UUID; +import com.google.common.collect.ImmutableMap; import org.junit.BeforeClass; import org.junit.Test; @@ -42,6 +43,7 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.utils.*; +import org.assertj.core.api.Assertions; public class DynamicCompositeTypeTest { @@ -372,4 +374,18 @@ public class DynamicCompositeTypeTest bb.rewind(); return bb; } + + @Test + public void testEmptyValue() + { + DynamicCompositeType type = DynamicCompositeType.getInstance(ImmutableMap.of((byte) 'V', BytesType.instance)); + + String cqlLiteral = "0x8056000000"; + ByteBuffer bb = type.asCQL3Type().fromCQLLiteral(cqlLiteral); + type.validate(bb); + + String str = type.getString(bb); + ByteBuffer read = type.fromString(str); + Assertions.assertThat(read).isEqualTo(bb); + } } diff --git a/test/unit/org/apache/cassandra/db/marshal/ValueAccessorTest.java b/test/unit/org/apache/cassandra/db/marshal/ValueAccessorTest.java index 5a9760e706..cd6681705f 100644 --- a/test/unit/org/apache/cassandra/db/marshal/ValueAccessorTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/ValueAccessorTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.vint.VIntCoding; import org.assertj.core.api.Assertions; import org.quicktheories.core.Gen; @@ -130,6 +131,38 @@ public class ValueAccessorTest extends ValueAccessorTester Assert.assertEquals(l, accessor.toLong(value)); } + private static void testUnsignedVIntConversion(long l, ValueAccessor accessor, int padding) + { + V value = accessor.allocate(VIntCoding.computeUnsignedVIntSize(l)); + accessor.putUnsignedVInt(value, 0, l); + value = leftPad(value, padding); + Assert.assertEquals(l, accessor.getUnsignedVInt(value, 0)); + } + + private static void testVIntConversion(long l, ValueAccessor accessor, int padding) + { + V value = accessor.allocate(VIntCoding.computeVIntSize(l)); + accessor.putVInt(value, 0, l); + value = leftPad(value, padding); + Assert.assertEquals(l, accessor.getVInt(value, 0)); + } + + private static void testUnsignedVInt32Conversion(int l, ValueAccessor accessor, int padding) + { + V value = accessor.allocate(VIntCoding.computeUnsignedVIntSize(l)); + accessor.putUnsignedVInt32(value, 0, l); + value = leftPad(value, padding); + Assert.assertEquals(l, accessor.getUnsignedVInt32(value, 0)); + } + + private static void testVInt32Conversion(int l, ValueAccessor accessor, int padding) + { + V value = accessor.allocate(VIntCoding.computeVIntSize(l)); + accessor.putVInt32(value, 0, l); + value = leftPad(value, padding); + Assert.assertEquals(l, accessor.getVInt32(value, 0)); + } + private static void testFloatConversion(float f, ValueAccessor accessor, int padding) { V value = leftPad(accessor.valueOf(f), padding); @@ -165,6 +198,22 @@ public class ValueAccessorTest extends ValueAccessorTester accessors(), bbPadding()).checkAssert(ValueAccessorTest::testLongConversion); + qt().forAll(longs().between(0, Long.MAX_VALUE), + accessors(), + bbPadding()).checkAssert(ValueAccessorTest::testUnsignedVIntConversion); + + qt().forAll(longs().all(), + accessors(), + bbPadding()).checkAssert(ValueAccessorTest::testVIntConversion); + + qt().forAll(integers().between(0, Integer.MAX_VALUE), + accessors(), + bbPadding()).checkAssert(ValueAccessorTest::testUnsignedVInt32Conversion); + + qt().forAll(integers().all(), + accessors(), + bbPadding()).checkAssert(ValueAccessorTest::testVInt32Conversion); + qt().forAll(floats().any(), accessors(), bbPadding()).checkAssert(ValueAccessorTest::testFloatConversion); diff --git a/test/unit/org/apache/cassandra/db/marshal/VectorTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/VectorTypeTest.java new file mode 100644 index 0000000000..34540cf148 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/marshal/VectorTypeTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.marshal; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import org.quicktheories.core.Gen; +import org.quicktheories.generators.SourceDSL; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.quicktheories.QuickTheory.qt; + +public class VectorTypeTest +{ + @Test + public void composeAsFloat() + { + // dim + // data + Gen dimGen = SourceDSL.integers().between(1, 100); + Gen floatGen = SourceDSL.floats().any(); + Gen caseGen = rnd -> { + int dim = dimGen.generate(rnd); + float[] array = new float[dim]; + for (int i = 0; i < dim; i++) + array[i] = floatGen.generate(rnd); + return new Case(dim, array); + }; + qt().forAll(caseGen).checkAssert(c -> { + VectorType type = VectorType.getInstance(FloatType.instance, c.dim); + ByteBuffer bb = type.decompose(c.box()); + assertThat(type.composeAsFloat(bb)).isEqualTo(c.values); + assertThat(c.unbox(type.compose(bb))).isEqualTo(c.values); + assertThat(type.decomposeAsFloat(c.values)).isEqualTo(bb); + }); + } + + private static class Case + { + final int dim; + final float[] values; + + private Case(int dim, float[] values) + { + this.dim = dim; + this.values = values; + } + + List box() + { + List list = new ArrayList<>(dim); + for (int i = 0; i < dim; i++) + list.add(values[i]); + return list; + } + + float[] unbox(List list) + { + assertThat(list).hasSize(dim); + float[] array = new float[dim]; + for (int i = 0; i < dim; i++) + array[i] = list.get(i); + return array; + } + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java deleted file mode 100644 index 5273daf45b..0000000000 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java +++ /dev/null @@ -1,1004 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import com.google.common.collect.Sets; -import org.junit.After; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.cql3.FieldIdentifier; -import org.apache.cassandra.cql3.statements.schema.IndexTarget; -import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.db.marshal.AbstractCompositeType; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.CollectionType; -import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.db.marshal.FloatType; -import org.apache.cassandra.db.marshal.FrozenType; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.db.marshal.ListType; -import org.apache.cassandra.db.marshal.MapType; -import org.apache.cassandra.db.marshal.SetType; -import org.apache.cassandra.db.marshal.TupleType; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.db.marshal.UserType; -import org.apache.cassandra.db.rows.EncodingStats; -import org.apache.cassandra.io.sstable.format.Version; -import org.apache.cassandra.io.sstable.format.big.BigFormat; -import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; -import org.apache.cassandra.io.sstable.metadata.MetadataType; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.SequentialWriter; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.IndexMetadata; -import org.apache.cassandra.schema.MockSchema; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test the functionality of {@link SSTableHeaderFix}. - * It writes an 'big-m' version sstable(s) and executes against these. - */ -@RunWith(Parameterized.class) -public class SSTableHeaderFixTest -{ - static - { - DatabaseDescriptor.toolInitialization(); - } - - private File temporaryFolder; - - @Parameterized.Parameter - public Supplier sstableIdGen; - - @Parameterized.Parameters - public static Collection parameters() - { - return MockSchema.sstableIdGenerators(); - } - - @BeforeClass - public static void beforeClass() - { - Assume.assumeTrue(BigFormat.isSelected()); - } - - @Before - public void setup() - { - MockSchema.sstableIds.clear(); - MockSchema.sstableIdGenerator = sstableIdGen; - File f = FileUtils.createTempFile("SSTableUDTFixTest", ""); - f.tryDelete(); - f.tryCreateDirectories(); - temporaryFolder = f; - } - - @After - public void teardown() - { - FileUtils.deleteRecursive(temporaryFolder); - } - - private static final AbstractType udtPK = makeUDT("udt_pk"); - private static final AbstractType udtCK = makeUDT("udt_ck"); - private static final AbstractType udtStatic = makeUDT("udt_static"); - private static final AbstractType udtRegular = makeUDT("udt_regular"); - private static final AbstractType udtInner = makeUDT("udt_inner"); - private static final AbstractType udtNested = new UserType("ks", - ByteBufferUtil.bytes("udt_nested"), - Arrays.asList(new FieldIdentifier(ByteBufferUtil.bytes("a_field")), - new FieldIdentifier(ByteBufferUtil.bytes("a_udt"))), - Arrays.asList(UTF8Type.instance, - udtInner), - true); - private static final AbstractType tupleInTuple = makeTuple(makeTuple()); - private static final AbstractType udtInTuple = makeTuple(udtInner); - private static final AbstractType tupleInComposite = CompositeType.getInstance(UTF8Type.instance, makeTuple()); - private static final AbstractType udtInComposite = CompositeType.getInstance(UTF8Type.instance, udtInner); - private static final AbstractType udtInList = ListType.getInstance(udtInner, true); - private static final AbstractType udtInSet = SetType.getInstance(udtInner, true); - private static final AbstractType udtInMap = MapType.getInstance(UTF8Type.instance, udtInner, true); - private static final AbstractType udtInFrozenList = ListType.getInstance(udtInner, false); - private static final AbstractType udtInFrozenSet = SetType.getInstance(udtInner, false); - private static final AbstractType udtInFrozenMap = MapType.getInstance(UTF8Type.instance, udtInner, false); - - private static AbstractType makeUDT2(String udtName, boolean multiCell) - { - return new UserType("ks", - ByteBufferUtil.bytes(udtName), - Arrays.asList(new FieldIdentifier(ByteBufferUtil.bytes("a_field")), - new FieldIdentifier(ByteBufferUtil.bytes("a_udt"))), - Arrays.asList(UTF8Type.instance, - udtInner), - multiCell); - } - - private static AbstractType makeUDT(String udtName) - { - return new UserType("ks", - ByteBufferUtil.bytes(udtName), - Collections.singletonList(new FieldIdentifier(ByteBufferUtil.bytes("a_field"))), - Collections.singletonList(UTF8Type.instance), - true); - } - - private static TupleType makeTuple() - { - return makeTuple(Int32Type.instance); - } - - private static TupleType makeTuple(AbstractType second) - { - return new TupleType(Arrays.asList(UTF8Type.instance, - second)); - } - - private static TupleType makeTupleSimple() - { - // TODO this should create a non-frozen tuple type for the sake of handling a dropped, non-frozen UDT - return new TupleType(Collections.singletonList(UTF8Type.instance)); - } - - private static final Version version = BigFormat.getInstance().getVersion("mc"); - - private TableMetadata tableMetadata; - private final Set updatedColumns = new HashSet<>(); - - private ColumnMetadata getColDef(String n) - { - return tableMetadata.getColumn(ByteBufferUtil.bytes(n)); - } - - /** - * Very basic test whether {@link SSTableHeaderFix} detect a type mismatch (regular_c 'int' vs 'float'). - */ - @Test - public void verifyTypeMismatchTest() throws Exception - { - File dir = temporaryFolder; - File sstable = generateFakeSSTable(dir, 1); - - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - - ColumnMetadata cd = getColDef("regular_c"); - tableMetadata = tableMetadata.unbuild() - .removeRegularOrStaticColumn(cd.name) - .addRegularColumn("regular_c", FloatType.instance) - .build(); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertTrue(headerFix.hasError()); - assertTrue(headerFix.hasChanges()); - - // must not have re-written the stats-component - header = readHeader(sstable); - assertFrozenUdt(header, false, true); - } - - @Test - public void verifyTypeMatchTest() throws Exception - { - File dir = temporaryFolder; - - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk", udtPK) - .addClusteringColumn("ck", udtCK); - commonColumns(cols); - File sstable = buildFakeSSTable(dir, 1, cols, false); - - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertTrue(updatedColumns.isEmpty()); - assertFalse(headerFix.hasError()); - assertFalse(headerFix.hasChanges()); - - // must not have re-written the stats-component - header = readHeader(sstable); - assertFrozenUdt(header, false, true); - } - - /** - * Simulates the case when an sstable contains a column not present in the schema, which can just be ignored. - */ - @Test - public void verifyWithUnknownColumnTest() throws Exception - { - File dir = temporaryFolder; - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk", udtPK) - .addClusteringColumn("ck", udtCK); - commonColumns(cols); - cols.addRegularColumn("solr_query", UTF8Type.instance); - File sstable = buildFakeSSTable(dir, 1, cols, true); - - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - - ColumnMetadata cd = getColDef("solr_query"); - tableMetadata = tableMetadata.unbuild() - .removeRegularOrStaticColumn(cd.name) - .build(); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertFalse(headerFix.hasError()); - assertTrue(headerFix.hasChanges()); - - // must not have re-written the stats-component - header = readHeader(sstable); - assertFrozenUdt(header, true, true); - } - - /** - * Simulates the case when an sstable contains a column not present in the table but as a target for an index. - * It can just be ignored. - */ - @Test - public void verifyWithIndexedUnknownColumnTest() throws Exception - { - File dir = temporaryFolder; - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk", udtPK) - .addClusteringColumn("ck", udtCK); - commonColumns(cols); - cols.addRegularColumn("solr_query", UTF8Type.instance); - File sstable = buildFakeSSTable(dir, 1, cols, true); - - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - - ColumnMetadata cd = getColDef("solr_query"); - tableMetadata = tableMetadata.unbuild() - .indexes(tableMetadata.indexes.with(IndexMetadata.fromSchemaMetadata("some search index", IndexMetadata.Kind.CUSTOM, Collections.singletonMap(IndexTarget.TARGET_OPTION_NAME, "solr_query")))) - .removeRegularOrStaticColumn(cd.name) - .build(); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertFalse(headerFix.hasError()); - assertTrue(headerFix.hasChanges()); - - // must not have re-written the stats-component - header = readHeader(sstable); - assertFrozenUdt(header, true, true); - } - - @Test - public void complexTypeMatchTest() throws Exception - { - File dir = temporaryFolder; - - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk", udtPK) - .addClusteringColumn("ck", udtCK); - commonColumns(cols); - cols.addRegularColumn("tuple_in_tuple", tupleInTuple) - .addRegularColumn("udt_nested", udtNested) - .addRegularColumn("udt_in_tuple", udtInTuple) - .addRegularColumn("tuple_in_composite", tupleInComposite) - .addRegularColumn("udt_in_composite", udtInComposite) - .addRegularColumn("udt_in_list", udtInList) - .addRegularColumn("udt_in_set", udtInSet) - .addRegularColumn("udt_in_map", udtInMap) - .addRegularColumn("udt_in_frozen_list", udtInFrozenList) - .addRegularColumn("udt_in_frozen_set", udtInFrozenSet) - .addRegularColumn("udt_in_frozen_map", udtInFrozenMap); - File sstable = buildFakeSSTable(dir, 1, cols, true); - - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertFalse(headerFix.hasError()); - assertTrue(headerFix.hasChanges()); - assertEquals(Sets.newHashSet("pk", "ck", "regular_b", "static_b", - "udt_nested", "udt_in_composite", "udt_in_list", "udt_in_set", "udt_in_map"), updatedColumns); - - // must not have re-written the stats-component - header = readHeader(sstable); - assertFrozenUdt(header, true, true); - } - - @Test - public void complexTypeDroppedColumnsMatchTest() throws Exception - { - File dir = temporaryFolder; - - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk", udtPK) - .addClusteringColumn("ck", udtCK); - commonColumns(cols); - cols.addRegularColumn("tuple_in_tuple", tupleInTuple) - .addRegularColumn("udt_nested", udtNested) - .addRegularColumn("udt_in_tuple", udtInTuple) - .addRegularColumn("tuple_in_composite", tupleInComposite) - .addRegularColumn("udt_in_composite", udtInComposite) - .addRegularColumn("udt_in_list", udtInList) - .addRegularColumn("udt_in_set", udtInSet) - .addRegularColumn("udt_in_map", udtInMap) - .addRegularColumn("udt_in_frozen_list", udtInFrozenList) - .addRegularColumn("udt_in_frozen_set", udtInFrozenSet) - .addRegularColumn("udt_in_frozen_map", udtInFrozenMap); - File sstable = buildFakeSSTable(dir, 1, cols, true); - - cols = tableMetadata.unbuild(); - for (String col : new String[]{"tuple_in_tuple", "udt_nested", "udt_in_tuple", - "tuple_in_composite", "udt_in_composite", - "udt_in_list", "udt_in_set", "udt_in_map", - "udt_in_frozen_list", "udt_in_frozen_set", "udt_in_frozen_map"}) - { - ColumnIdentifier ci = new ColumnIdentifier(col, true); - ColumnMetadata cd = getColDef(col); - AbstractType dropType = cd.type.expandUserTypes(); - cols.removeRegularOrStaticColumn(ci) - .recordColumnDrop(new ColumnMetadata(cd.ksName, cd.cfName, cd.name, dropType, cd.position(), cd.kind, cd.getMask()), FBUtilities.timestampMicros()); - } - tableMetadata = cols.build(); - - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertFalse(headerFix.hasError()); - assertTrue(headerFix.hasChanges()); - assertEquals(Sets.newHashSet("pk", "ck", "regular_b", "static_b", "udt_nested"), updatedColumns); - - // must not have re-written the stats-component - header = readHeader(sstable); - // do not check the inner types, as the inner types were not fixed in the serialization-header (test thing) - assertFrozenUdt(header, true, false); - } - - @Test - public void variousDroppedUserTypes() throws Exception - { - File dir = temporaryFolder; - - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk", udtPK) - .addClusteringColumn("ck", udtCK); - - ColSpec[] colSpecs = new ColSpec[] - { - // 'frozen' / live - new ColSpec("frozen_udt_as_frozen_udt_live", - makeUDT2("frozen_udt_as_frozen_udt_live", false), - makeUDT2("frozen_udt_as_frozen_udt_live", false), - false, - false), - // 'frozen' / live / as 'udt' - new ColSpec("frozen_udt_as_unfrozen_udt_live", - makeUDT2("frozen_udt_as_unfrozen_udt_live", false), - makeUDT2("frozen_udt_as_unfrozen_udt_live", true), - false, - true), - // 'frozen' / dropped - new ColSpec("frozen_udt_as_frozen_udt_dropped", - makeUDT2("frozen_udt_as_frozen_udt_dropped", true).freezeNestedMulticellTypes().freeze().expandUserTypes(), - makeUDT2("frozen_udt_as_frozen_udt_dropped", false), - makeUDT2("frozen_udt_as_frozen_udt_dropped", false), - true, - false), - // 'frozen' / dropped / as 'udt' - new ColSpec("frozen_udt_as_unfrozen_udt_dropped", - makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes().freeze().expandUserTypes(), - makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true), - makeUDT2("frozen_udt_as_unfrozen_udt_dropped", false), - true, - true), - // 'udt' / live - new ColSpec("unfrozen_udt_as_unfrozen_udt_live", - makeUDT2("unfrozen_udt_as_unfrozen_udt_live", true), - makeUDT2("unfrozen_udt_as_unfrozen_udt_live", true), - false, - false), - // 'udt' / dropped -// TODO unable to test dropping a non-frozen UDT, as that requires an unfrozen tuple as well -// new ColSpec("unfrozen_udt_as_unfrozen_udt_dropped", -// makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes().expandUserTypes(), -// makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true), -// makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true), -// true, -// false), - // 'frozen' as 'TupleType(multiCell=false' (there is nothing like 'FrozenType(TupleType(') - new ColSpec("frozen_tuple_as_frozen_tuple_live", - makeTupleSimple(), - makeTupleSimple(), - false, - false), - // 'frozen' as 'TupleType(multiCell=false' (there is nothing like 'FrozenType(TupleType(') - new ColSpec("frozen_tuple_as_frozen_tuple_dropped", - makeTupleSimple(), - makeTupleSimple(), - true, - false) - }; - - Arrays.stream(colSpecs).forEach(c -> cols.addRegularColumn(c.name, - // use the initial column type for the serialization header header. - c.preFix)); - - Map colSpecMap = Arrays.stream(colSpecs).collect(Collectors.toMap(c -> c.name, c -> c)); - File sstable = buildFakeSSTable(dir, 1, cols, c -> { - ColSpec cs = colSpecMap.get(c.name.toString()); - if (cs == null) - return c; - // update the column type in the schema to the "correct" one. - return c.withNewType(cs.schema); - }); - - Arrays.stream(colSpecs) - .filter(c -> c.dropped) - .forEach(c -> { - ColumnMetadata cd = getColDef(c.name); - tableMetadata = tableMetadata.unbuild() - .removeRegularOrStaticColumn(cd.name) - .recordColumnDrop(cd, FBUtilities.timestampMicros()) - .build(); - }); - - SerializationHeader.Component header = readHeader(sstable); - for (ColSpec colSpec : colSpecs) - { - AbstractType hdrType = header.getRegularColumns().get(ByteBufferUtil.bytes(colSpec.name)); - assertEquals(colSpec.name, colSpec.preFix, hdrType); - assertEquals(colSpec.name, colSpec.preFix.isMultiCell(), hdrType.isMultiCell()); - } - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertFalse(headerFix.hasError()); - assertTrue(headerFix.hasChanges()); - // Verify that all columns to fix are in the updatedColumns set (paranoid, yet) - Arrays.stream(colSpecs) - .filter(c -> c.mustFix) - .forEach(c -> assertTrue("expect " + c.name + " to be updated, but was not (" + updatedColumns + ")", updatedColumns.contains(c.name))); - // Verify that the number of updated columns maches the expected number of columns to fix - assertEquals(Arrays.stream(colSpecs).filter(c -> c.mustFix).count(), updatedColumns.size()); - - header = readHeader(sstable); - for (ColSpec colSpec : colSpecs) - { - AbstractType hdrType = header.getRegularColumns().get(ByteBufferUtil.bytes(colSpec.name)); - assertEquals(colSpec.name, colSpec.expect, hdrType); - assertEquals(colSpec.name, colSpec.expect.isMultiCell(), hdrType.isMultiCell()); - } - } - - static class ColSpec - { - final String name; - final AbstractType schema; - final AbstractType preFix; - final AbstractType expect; - final boolean dropped; - final boolean mustFix; - - ColSpec(String name, AbstractType schema, AbstractType preFix, boolean dropped, boolean mustFix) - { - this(name, schema, preFix, schema, dropped, mustFix); - } - - ColSpec(String name, AbstractType schema, AbstractType preFix, AbstractType expect, boolean dropped, boolean mustFix) - { - this.name = name; - this.schema = schema; - this.preFix = preFix; - this.expect = expect; - this.dropped = dropped; - this.mustFix = mustFix; - } - } - - @Test - public void verifyTypeMatchCompositeKeyTest() throws Exception - { - File dir = temporaryFolder; - - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk1", UTF8Type.instance) - .addPartitionKeyColumn("pk2", udtPK) - .addClusteringColumn("ck", udtCK); - commonColumns(cols); - File sstable = buildFakeSSTable(dir, 1, cols, false); - - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertFalse(headerFix.hasError()); - assertFalse(headerFix.hasChanges()); - assertTrue(updatedColumns.isEmpty()); - - // must not have re-written the stats-component - header = readHeader(sstable); - assertFrozenUdt(header, false, true); - } - - @Test - public void compositePartitionKey() throws Exception - { - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk1", UTF8Type.instance) - .addPartitionKeyColumn("pk2", udtPK) - .addClusteringColumn("ck", udtCK); - commonColumns(cols); - - File dir = temporaryFolder; - File sstable = buildFakeSSTable(dir, 1, cols, true); - - SerializationHeader.Component header = readHeader(sstable); - assertTrue(header.getKeyType() instanceof CompositeType); - CompositeType keyType = (CompositeType) header.getKeyType(); - assertEquals(Arrays.asList(UTF8Type.instance, udtPK), keyType.getComponents()); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertFalse(headerFix.hasError()); - assertTrue(headerFix.hasChanges()); - assertEquals(Sets.newHashSet("pk2", "ck", "regular_b", "static_b"), updatedColumns); - - header = readHeader(sstable); - assertTrue(header.getKeyType() instanceof CompositeType); - keyType = (CompositeType) header.getKeyType(); - assertEquals(Arrays.asList(UTF8Type.instance, udtPK.freeze()), keyType.getComponents()); - } - - @Test - public void compositeClusteringKey() throws Exception - { - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk", udtPK) - .addClusteringColumn("ck1", Int32Type.instance) - .addClusteringColumn("ck2", udtCK); - commonColumns(cols); - - File dir = temporaryFolder; - File sstable = buildFakeSSTable(dir, 1, cols, true); - - SerializationHeader.Component header = readHeader(sstable); - assertEquals(Arrays.asList(Int32Type.instance, udtCK), header.getClusteringTypes()); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertFalse(headerFix.hasError()); - assertTrue(headerFix.hasChanges()); - assertEquals(Sets.newHashSet("pk", "ck2", "regular_b", "static_b"), updatedColumns); - - header = readHeader(sstable); - assertEquals(Arrays.asList(Int32Type.instance, udtCK.freeze()), header.getClusteringTypes()); - } - - /** - * Check whether {@link SSTableHeaderFix} can operate on a single file. - */ - @Test - public void singleFileUDTFixTest() throws Exception - { - File dir = temporaryFolder; - File sstable = generateFakeSSTable(dir, 1); - - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - - SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) - .build(); - headerFix.execute(); - - assertTrue(headerFix.hasChanges()); - assertFalse(headerFix.hasError()); - - header = readHeader(sstable); - assertFrozenUdt(header, true, true); - } - - /** - * Check whether {@link SSTableHeaderFix} can operate on a file in a directory. - */ - @Test - public void singleDirectoryUDTFixTest() throws Exception - { - File dir = temporaryFolder; - List sstables = IntStream.range(1, 11) - .mapToObj(g -> generateFakeSSTable(dir, g)) - .collect(Collectors.toList()); - - for (File sstable : sstables) - { - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - } - - SSTableHeaderFix headerFix = builder().withPath(dir.toPath()) - .build(); - headerFix.execute(); - - assertTrue(headerFix.hasChanges()); - assertFalse(headerFix.hasError()); - - for (File sstable : sstables) - { - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, true, true); - } - } - - /** - * Check whether {@link SSTableHeaderFix} can operate multiple, single files. - */ - @Test - public void multipleFilesUDTFixTest() throws Exception - { - File dir = temporaryFolder; - List sstables = IntStream.range(1, 11) - .mapToObj(g -> generateFakeSSTable(dir, g)) - .collect(Collectors.toList()); - - for (File sstable : sstables) - { - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - } - - SSTableHeaderFix.Builder builder = builder(); - sstables.stream().map(File::toPath).forEach(builder::withPath); - SSTableHeaderFix headerFix = builder.build(); - headerFix.execute(); - - assertTrue(headerFix.hasChanges()); - assertFalse(headerFix.hasError()); - - for (File sstable : sstables) - { - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, true, true); - } - } - - /** - * Check whether {@link SSTableHeaderFix} can operate multiple files in a directory. - */ - @Test - public void multipleFilesInDirectoryUDTFixTest() throws Exception - { - File dir = temporaryFolder; - List sstables = IntStream.range(1, 11) - .mapToObj(g -> generateFakeSSTable(dir, g)) - .collect(Collectors.toList()); - - for (File sstable : sstables) - { - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, false, true); - } - - SSTableHeaderFix headerFix = builder().withPath(dir.toPath()) - .build(); - headerFix.execute(); - - assertTrue(headerFix.hasChanges()); - assertFalse(headerFix.hasError()); - - for (File sstable : sstables) - { - SerializationHeader.Component header = readHeader(sstable); - assertFrozenUdt(header, true, true); - } - } - - @Test - public void ignoresStaleFilesTest() throws Exception - { - File dir = temporaryFolder; - IntStream.range(1, 2).forEach(g -> generateFakeSSTable(dir, g)); - - File newFile = new File(dir.toAbsolute(), "something_something-something.something"); - Assert.assertTrue(newFile.createFileIfNotExists()); - - SSTableHeaderFix headerFix = builder().withPath(dir.toPath()) - .build(); - headerFix.execute(); - } - - private static final Pattern p = Pattern.compile(".* Column '([^']+)' needs to be updated from type .*"); - - private SSTableHeaderFix.Builder builder() - { - updatedColumns.clear(); - return SSTableHeaderFix.builder() - .schemaCallback(() -> (desc) -> tableMetadata) - .info(ln -> { - System.out.println("INFO: " + ln); - Matcher m = p.matcher(ln); - if (m.matches()) - updatedColumns.add(m.group(1)); - }) - .warn(ln -> System.out.println("WARN: " + ln)) - .error(ln -> System.out.println("ERROR: " + ln)); - } - - private File generateFakeSSTable(File dir, int generation) - { - TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") - .addPartitionKeyColumn("pk", udtPK) - .addClusteringColumn("ck", udtCK); - commonColumns(cols); - return buildFakeSSTable(dir, generation, cols, true); - } - - private void commonColumns(TableMetadata.Builder cols) - { - cols.addRegularColumn("regular_a", UTF8Type.instance) - .addRegularColumn("regular_b", udtRegular) - .addRegularColumn("regular_c", Int32Type.instance) - .addStaticColumn("static_a", UTF8Type.instance) - .addStaticColumn("static_b", udtStatic) - .addStaticColumn("static_c", Int32Type.instance); - } - - private File buildFakeSSTable(File dir, int generation, TableMetadata.Builder cols, boolean freezeInSchema) - { - return buildFakeSSTable(dir, generation, cols, freezeInSchema - ? c -> c.withNewType(freezeUdt(c.type)) - : c -> c); - } - - private File buildFakeSSTable(File dir, int generation, TableMetadata.Builder cols, Function freezer) - { - TableMetadata headerMetadata = cols.build(); - - TableMetadata.Builder schemaCols = TableMetadata.builder("ks", "cf"); - for (ColumnMetadata cm : cols.columns()) - schemaCols.addColumn(freezer.apply(cm)); - tableMetadata = schemaCols.build(); - - try - { - - Descriptor desc = new Descriptor(version, dir, "ks", "cf", MockSchema.sstableId(generation)); - - // Just create the component files - we don't really need those. - for (Component component : requiredComponents) - assertTrue(desc.fileFor(component).createFileIfNotExists()); - - AbstractType partitionKey = headerMetadata.partitionKeyType; - List> clusteringKey = headerMetadata.clusteringColumns() - .stream() - .map(cd -> cd.type) - .collect(Collectors.toList()); - Map> staticColumns = headerMetadata.columns() - .stream() - .filter(cd -> cd.kind == ColumnMetadata.Kind.STATIC) - .collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a)); - Map> regularColumns = headerMetadata.columns() - .stream() - .filter(cd -> cd.kind == ColumnMetadata.Kind.REGULAR) - .collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a)); - - File statsFile = desc.fileFor(Components.STATS); - SerializationHeader.Component header = SerializationHeader.Component.buildComponentForTools(partitionKey, - clusteringKey, - staticColumns, - regularColumns, - EncodingStats.NO_STATS); - - try (SequentialWriter out = new SequentialWriter(statsFile)) - { - desc.getMetadataSerializer().serialize(Collections.singletonMap(MetadataType.HEADER, header), out, version); - out.finish(); - } - - return desc.fileFor(Components.DATA); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - private AbstractType freezeUdt(AbstractType type) - { - if (type instanceof CollectionType) - { - if (type.getClass() == ListType.class) - { - ListType cHeader = (ListType) type; - return ListType.getInstance(freezeUdt(cHeader.getElementsType()), cHeader.isMultiCell()); - } - else if (type.getClass() == SetType.class) - { - SetType cHeader = (SetType) type; - return SetType.getInstance(freezeUdt(cHeader.getElementsType()), cHeader.isMultiCell()); - } - else if (type.getClass() == MapType.class) - { - MapType cHeader = (MapType) type; - return MapType.getInstance(freezeUdt(cHeader.getKeysType()), freezeUdt(cHeader.getValuesType()), cHeader.isMultiCell()); - } - } - else if (type instanceof AbstractCompositeType) - { - if (type.getClass() == CompositeType.class) - { - CompositeType cHeader = (CompositeType) type; - return CompositeType.getInstance(cHeader.types.stream().map(this::freezeUdt).collect(Collectors.toList())); - } - } - else if (type instanceof TupleType) - { - if (type.getClass() == UserType.class) - { - UserType cHeader = (UserType) type; - cHeader = cHeader.freeze(); - return new UserType(cHeader.keyspace, cHeader.name, cHeader.fieldNames(), - cHeader.allTypes().stream().map(this::freezeUdt).collect(Collectors.toList()), - cHeader.isMultiCell()); - } - } - return type; - } - - private void assertFrozenUdt(SerializationHeader.Component header, boolean frozen, boolean checkInner) - { - AbstractType keyType = header.getKeyType(); - if (keyType instanceof CompositeType) - { - for (AbstractType component : ((CompositeType) keyType).types) - assertFrozenUdt("partition-key-component", component, frozen, checkInner); - } - assertFrozenUdt("partition-key", keyType, frozen, checkInner); - - for (AbstractType type : header.getClusteringTypes()) - assertFrozenUdt("clustering-part", type, frozen, checkInner); - for (Map.Entry> col : header.getStaticColumns().entrySet()) - assertFrozenUdt(UTF8Type.instance.compose(col.getKey()), col.getValue(), frozen, checkInner); - for (Map.Entry> col : header.getRegularColumns().entrySet()) - assertFrozenUdt(UTF8Type.instance.compose(col.getKey()), col.getValue(), frozen, checkInner); - } - - private void assertFrozenUdt(String name, AbstractType type, boolean frozen, boolean checkInner) - { - if (type instanceof CompositeType) - { - if (checkInner) - for (AbstractType component : ((CompositeType) type).types) - assertFrozenUdt(name, component, frozen, true); - } - else if (type instanceof CollectionType) - { - if (checkInner) - { - if (type instanceof MapType) - { - MapType map = (MapType) type; - // only descend for non-frozen types (checking frozen in frozen is just stupid) - if (map.isMultiCell()) - { - assertFrozenUdt(name + "", map.getKeysType(), frozen, true); - assertFrozenUdt(name + "", map.getValuesType(), frozen, true); - } - } - else if (type instanceof SetType) - { - SetType set = (SetType) type; - // only descend for non-frozen types (checking frozen in frozen is just stupid) - if (set.isMultiCell()) - assertFrozenUdt(name + "", set.getElementsType(), frozen, true); - } - else if (type instanceof ListType) - { - ListType list = (ListType) type; - // only descend for non-frozen types (checking frozen in frozen is just stupid) - if (list.isMultiCell()) - assertFrozenUdt(name + "", list.getElementsType(), frozen, true); - } - } - } - else if (type instanceof TupleType) - { - if (checkInner) - { - TupleType tuple = (TupleType) type; - // only descend for non-frozen types (checking frozen in frozen is just stupid) - if (tuple.isMultiCell()) - for (AbstractType component : tuple.allTypes()) - assertFrozenUdt(name + "", component, frozen, true); - } - } - - if (type instanceof UserType) - { - String typeString = type.toString(); - assertEquals(name + ": " + typeString, frozen, !type.isMultiCell()); - if (typeString.startsWith(UserType.class.getName() + '(')) - if (frozen) - fail(name + ": " + typeString); - if (typeString.startsWith(FrozenType.class.getName() + '(' + UserType.class.getName() + '(')) - if (!frozen) - fail(name + ": " + typeString); - } - } - - private SerializationHeader.Component readHeader(File sstable) throws Exception - { - Descriptor desc = Descriptor.fromFileWithComponent(sstable, false).left; - return (SerializationHeader.Component) desc.getMetadataSerializer().deserialize(desc, MetadataType.HEADER); - } - - private static final Component[] requiredComponents = new Component[]{ Components.DATA, Components.FILTER, Components.PRIMARY_INDEX, Components.TOC }; -} diff --git a/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java b/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java index 4391d241c2..b30390d4a1 100644 --- a/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java +++ b/test/unit/org/apache/cassandra/net/MessageSerializationPropertyTest.java @@ -69,7 +69,7 @@ public class MessageSerializationPropertyTest implements Serializable { try (DataOutputBuffer out = new DataOutputBuffer(1024)) { - qt().forAll(MESSAGE_GEN).checkAssert(orFail(message -> { + qt().withShrinkCycles(0).forAll(MESSAGE_GEN).checkAssert(orFail(message -> { for (MessagingService.Version version : MessagingService.Version.values()) { out.clear(); @@ -97,7 +97,7 @@ public class MessageSerializationPropertyTest implements Serializable try (DataOutputBuffer first = new DataOutputBuffer(1024); DataOutputBuffer second = new DataOutputBuffer(1024)) { - qt().forAll(MESSAGE_GEN).checkAssert(orFail(message -> { + qt().withShrinkCycles(0).forAll(MESSAGE_GEN).checkAssert(orFail(message -> { withTable(schema, message, orFail(ignore -> { for (MessagingService.Version version : MessagingService.Version.values()) { diff --git a/test/unit/org/apache/cassandra/tools/StandaloneScrubberTest.java b/test/unit/org/apache/cassandra/tools/StandaloneScrubberTest.java index 8376b55b65..062f7fa992 100644 --- a/test/unit/org/apache/cassandra/tools/StandaloneScrubberTest.java +++ b/test/unit/org/apache/cassandra/tools/StandaloneScrubberTest.java @@ -20,8 +20,6 @@ package org.apache.cassandra.tools; import java.util.Arrays; -import org.apache.commons.lang3.tuple.Pair; - import org.junit.Test; import org.apache.cassandra.tools.ToolRunner.ToolResult; @@ -30,7 +28,6 @@ import org.hamcrest.CoreMatchers; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; /* * We are testing cmd line params here. @@ -134,41 +131,14 @@ public class StandaloneScrubberTest extends OfflineToolUtils @Test public void testHeaderFixArg() { - Arrays.asList(Pair.of("-e", ""), - Pair.of("-e", "wrong"), - Pair.of("--header-fix", ""), - Pair.of("--header-fix", "wrong")) - .forEach(arg -> { - ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, - arg.getLeft(), - arg.getRight(), - "system_schema", - "tables"); - assertThat("Arg: [" + arg + "]", tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:")); - assertTrue("Arg: [" + arg + "]\n" + tool.getCleanedStderr(), tool.getCleanedStderr().contains("Invalid argument value")); - assertEquals(1, tool.getExitCode()); - }); - - Arrays.asList(Pair.of("-e", "validate-only"), - Pair.of("-e", "validate"), - Pair.of("-e", "fix-only"), - Pair.of("-e", "fix"), - Pair.of("-e", "off"), - Pair.of("--header-fix", "validate-only"), - Pair.of("--header-fix", "validate"), - Pair.of("--header-fix", "fix-only"), - Pair.of("--header-fix", "fix"), - Pair.of("--header-fix", "off")) - .forEach(arg -> { - ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, - arg.getLeft(), - arg.getRight(), - "system_schema", - "tables"); - assertThat("Arg: [" + arg + "]", tool.getStdout(), CoreMatchers.containsStringIgnoringCase("Pre-scrub sstables snapshotted into snapshot")); - Assertions.assertThat(tool.getCleanedStderr()).as("Arg: [%s]", arg).isEmpty(); - tool.assertOnExitCode(); - assertCorrectEnvPostTest(); - }); + Arrays.asList("-e", "--header-fix").forEach(arg -> { + ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, + arg, + "doesntmatter", + "system_schema", + "tables"); + Assertions.assertThat(tool.getCleanedStderr().trim()).isEqualTo("Option header-fix is deprecated and no longer functional"); + Assertions.assertThat(tool.getExitCode()).isEqualTo(0); + }); } } diff --git a/test/unit/org/apache/cassandra/tools/ToolRunner.java b/test/unit/org/apache/cassandra/tools/ToolRunner.java index d3b9f10c43..7b69b39a1c 100644 --- a/test/unit/org/apache/cassandra/tools/ToolRunner.java +++ b/test/unit/org/apache/cassandra/tools/ToolRunner.java @@ -56,8 +56,8 @@ public class ToolRunner { protected static final Logger logger = LoggerFactory.getLogger(ToolRunner.class); - private static final ImmutableList DEFAULT_CLEANERS = ImmutableList.of("(?im)^picked up.*\\R", - "(?im)^.*`USE ` with prepared statements is.*\\R"); + public static final ImmutableList DEFAULT_CLEANERS = ImmutableList.of("(?im)^picked up.*\\R", + "(?im)^.*`USE ` with prepared statements is.*\\R"); public static int runClassAsTool(String clazz, String... args) { @@ -389,19 +389,28 @@ public class ToolRunner { return e; } + + /** + * Checks if the stdErr is empty after removing any potential JVM env info output and other noise + * + * Some JVM configs may output env info on stdErr. We need to remove those to see what was the tool's actual + * stdErr + */ + public void assertCleanStdErr() + { + assertCleanStdErr(DEFAULT_CLEANERS); + } /** * Checks if the stdErr is empty after removing any potential JVM env info output and other noise * * Some JVM configs may output env info on stdErr. We need to remove those to see what was the tool's actual * stdErr - * - * @return The ToolRunner instance */ - public void assertCleanStdErr() + public void assertCleanStdErr(List regExpCleaners) { String raw = getStderr(); - String cleaned = getCleanedStderr(); + String cleaned = getCleanedStderr(regExpCleaners); assertTrue("Failed to clean stderr completely.\nRaw (length=" + raw.length() + "):\n" + raw + "\nCleaned (length=" + cleaned.length() + "):\n" + cleaned, cleaned.trim().isEmpty()); @@ -454,11 +463,16 @@ public class ToolRunner { return getCleanedStderr(DEFAULT_CLEANERS); } - + public void assertOnCleanExit() + { + assertOnCleanExit(DEFAULT_CLEANERS); + } + + public void assertOnCleanExit(List regExpCleaners) { assertOnExitCode(); - assertCleanStdErr(); + assertCleanStdErr(regExpCleaners); } public AssertHelp asserts() diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java index a007ce711c..ddc59b81a1 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java @@ -22,6 +22,7 @@ import java.io.IOError; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -66,6 +67,10 @@ import static org.junit.Assert.fail; public class ScrubToolTest { + private static final ImmutableList CLEANERS = ImmutableList.builder() + .addAll(ToolRunner.DEFAULT_CLEANERS) + .add("(?im)^.*header-fix is deprecated and no longer functional.*\\R") + .build(); private static final String CF = "scrub_tool_test"; private static final AtomicInteger seq = new AtomicInteger(); @@ -181,20 +186,6 @@ public class ScrubToolTest assertOrderedAll(cfs, 10); } - @Test - public void testHeaderFixValidateOnlyWithTool() - { - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); - - fillCF(cfs, 1); - assertOrderedAll(cfs, 1); - - ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "validate_only", ksName, CF); - Assertions.assertThat(tool.getStdout()).contains("Not continuing with scrub, since '--header-fix validate-only' was specified."); - tool.assertOnCleanExit(); - assertOrderedAll(cfs, 1); - } - @Test public void testHeaderFixValidateWithTool() { @@ -206,21 +197,8 @@ public class ScrubToolTest ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "validate", ksName, CF); Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into"); Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty"); - tool.assertOnCleanExit(); - assertOrderedAll(cfs, 1); - } - - @Test - public void testHeaderFixFixOnlyWithTool() - { - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); - - fillCF(cfs, 1); - assertOrderedAll(cfs, 1); - - ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "fix-only", ksName, CF); - Assertions.assertThat(tool.getStdout()).contains("Not continuing with scrub, since '--header-fix fix-only' was specified."); - tool.assertOnCleanExit(); + // TODO cleaner that ignores + tool.assertOnCleanExit(CLEANERS); assertOrderedAll(cfs, 1); } @@ -235,7 +213,7 @@ public class ScrubToolTest ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "fix", ksName, CF); Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into"); Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty"); - tool.assertOnCleanExit(); + tool.assertOnCleanExit(CLEANERS); assertOrderedAll(cfs, 1); } @@ -250,7 +228,7 @@ public class ScrubToolTest ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-e", "off", ksName, CF); Assertions.assertThat(tool.getStdout()).contains("Pre-scrub sstables snapshotted into"); Assertions.assertThat(tool.getStdout()).contains("1 partitions in new sstable and 0 empty"); - tool.assertOnCleanExit(); + tool.assertOnCleanExit(CLEANERS); assertOrderedAll(cfs, 1); } } \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java b/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java index 6b685dcd1e..d9e3f80508 100644 --- a/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java +++ b/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java @@ -20,50 +20,113 @@ package org.apache.cassandra.utils; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; +import javax.annotation.Nullable; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.commons.lang3.ArrayUtils; + +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.Duration; import org.apache.cassandra.cql3.FieldIdentifier; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.db.marshal.BooleanType; +import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.db.marshal.ByteType; import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.marshal.CounterColumnType; +import org.apache.cassandra.db.marshal.DateType; +import org.apache.cassandra.db.marshal.DecimalType; import org.apache.cassandra.db.marshal.DoubleType; +import org.apache.cassandra.db.marshal.DurationType; +import org.apache.cassandra.db.marshal.DynamicCompositeType; import org.apache.cassandra.db.marshal.EmptyType; import org.apache.cassandra.db.marshal.FloatType; +import org.apache.cassandra.db.marshal.FrozenType; import org.apache.cassandra.db.marshal.InetAddressType; import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.marshal.LegacyTimeUUIDType; +import org.apache.cassandra.db.marshal.LexicalUUIDType; import org.apache.cassandra.db.marshal.ListType; import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.marshal.PartitionerDefinedOrder; import org.apache.cassandra.db.marshal.ReversedType; import org.apache.cassandra.db.marshal.SetType; import org.apache.cassandra.db.marshal.ShortType; +import org.apache.cassandra.db.marshal.SimpleDateType; +import org.apache.cassandra.db.marshal.StringType; +import org.apache.cassandra.db.marshal.TimeType; +import org.apache.cassandra.db.marshal.TimeUUIDType; import org.apache.cassandra.db.marshal.TimestampType; import org.apache.cassandra.db.marshal.TupleType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.db.marshal.UserType; +import org.apache.cassandra.db.marshal.VectorType; import org.quicktheories.core.Gen; import org.quicktheories.core.RandomnessSource; import org.quicktheories.generators.SourceDSL; +import org.quicktheories.impl.JavaRandom; import static org.apache.cassandra.utils.Generators.IDENTIFIER_GEN; +import static org.apache.cassandra.utils.Generators.filter; +@SuppressWarnings({"unchecked", "rawtypes"}) public final class AbstractTypeGenerators { private static final Gen VERY_SMALL_POSITIVE_SIZE_GEN = SourceDSL.integers().between(1, 3); private static final Gen BOOLEAN_GEN = SourceDSL.booleans().all(); + public static final Map>, String> UNSUPPORTED = ImmutableMap.>, String>builder() + .put(DateType.class, "Says its CQL type is timestamp, but that maps to TimestampType; is this actually dead code at this point?") + .put(LegacyTimeUUIDType.class, "Says its CQL timeuuid type, but that maps to TimeUUIDType; is this actually dead code at this point?") + .put(PartitionerDefinedOrder.class, "This is a fake type used for ordering partitions using a Partitioner") + // ignore intellij saying that "(Class)" isn't needed; jdk8 fails to compile without it! + .put((Class>) (Class) ReversedType.class, "Implementation detail for cluster ordering... its expected the caller will unwrap the clustering type to always get access to the real type") + .put(DynamicCompositeType.FixedValueComparator.class, "Hack type used for special ordering case, not a real/valid type") + .put(FrozenType.class, "Fake class only used during parsing... the parsing creates this and the real type under it, then this gets swapped for the real type") + .build(); + + /** + * Java does a char by char compare, but Cassandra does a byte ordered compare. This mostly overlaps but some cases + * where chars are mixed between 1 and 2 bytes, you can get a different ordering than java's. One argument in favor + * of this is that byte order is far faster than char order, which only violates a few cases but not the general case: + * {@code "David" > "david"}. Without more research, it also isn't clear if this at all violates any UTF-8 spec (wikipedia + * mentions "sorting the corresponding byte sequences"). + * + * @see Slack + */ + public static Comparator stringComparator(StringType st) + { + return (String a, String b) -> FastByteOperations.compareUnsigned(st.decompose(a), st.decompose(b)); + } + + private static final Map, TypeSupport> PRIMITIVE_TYPE_DATA_GENS = Stream.of(TypeSupport.of(BooleanType.instance, BOOLEAN_GEN), TypeSupport.of(ByteType.instance, SourceDSL.integers().between(0, Byte.MAX_VALUE * 2 + 1).map(Integer::byteValue)), @@ -72,26 +135,45 @@ public final class AbstractTypeGenerators TypeSupport.of(LongType.instance, SourceDSL.longs().all()), TypeSupport.of(FloatType.instance, SourceDSL.floats().any()), TypeSupport.of(DoubleType.instance, SourceDSL.doubles().any()), - TypeSupport.of(BytesType.instance, Generators.bytes(1, 1024)), + TypeSupport.of(BytesType.instance, Generators.bytes(0, 1024), FastByteOperations::compareUnsigned), // use the faster version... TypeSupport.of(UUIDType.instance, Generators.UUID_RANDOM_GEN), - TypeSupport.of(InetAddressType.instance, Generators.INET_ADDRESS_UNRESOLVED_GEN), // serialization strips the hostname, only keeps the address - TypeSupport.of(AsciiType.instance, SourceDSL.strings().ascii().ofLengthBetween(1, 1024)), - TypeSupport.of(UTF8Type.instance, Generators.utf8(1, 1024)), + TypeSupport.of(TimeUUIDType.instance, Generators.UUID_TIME_GEN.map(TimeUUID::fromUuid)), + TypeSupport.of(LexicalUUIDType.instance, Generators.UUID_RANDOM_GEN.mix(Generators.UUID_TIME_GEN)), + TypeSupport.of(InetAddressType.instance, Generators.INET_ADDRESS_UNRESOLVED_GEN, (a, b) -> FastByteOperations.compareUnsigned(a.getAddress(), b.getAddress())), // serialization strips the hostname, only keeps the address + TypeSupport.of(AsciiType.instance, SourceDSL.strings().ascii().ofLengthBetween(0, 1024), stringComparator(AsciiType.instance)), + TypeSupport.of(UTF8Type.instance, Generators.utf8(0, 1024), stringComparator(UTF8Type.instance)), TypeSupport.of(TimestampType.instance, Generators.DATE_GEN), + TypeSupport.of(SimpleDateType.instance, SourceDSL.integers().between(0, Integer.MAX_VALUE)), // can't use time gen as this is an int, and in Milliseconds... so overflows... + TypeSupport.of(TimeType.instance, SourceDSL.longs().between(0, 24L * 60L * 60L * 1_000_000_000L - 1L)), // null is desired here as #decompose will call org.apache.cassandra.serializers.EmptySerializer.serialize which ignores the input and returns empty bytes - TypeSupport.of(EmptyType.instance, rnd -> null) - //TODO add the following - // IntegerType.instance, - // DecimalType.instance, - // TimeUUIDType.instance, - // LexicalUUIDType.instance, - // SimpleDateType.instance, - // TimeType.instance, - // DurationType.instance, + TypeSupport.of(EmptyType.instance, rnd -> null, (a, b) -> 0), + TypeSupport.of(DurationType.instance, CassandraGenerators.duration(), Comparator.comparingInt(Duration::getMonths) + .thenComparingInt(Duration::getDays) + .thenComparingLong(Duration::getNanoseconds)), + TypeSupport.of(IntegerType.instance, Generators.bigInt()), + TypeSupport.of(DecimalType.instance, Generators.bigDecimal()) ).collect(Collectors.toMap(t -> t.type, t -> t)); // NOTE not supporting reversed as CQL doesn't allow nested reversed types // when generating part of the clustering key, it would be good to allow reversed types as the top level - private static final Gen> PRIMITIVE_TYPE_GEN = SourceDSL.arbitrary().pick(new ArrayList<>(PRIMITIVE_TYPE_DATA_GENS.keySet())); + private static final Gen> PRIMITIVE_TYPE_GEN; + static + { + ArrayList> types = new ArrayList<>(PRIMITIVE_TYPE_DATA_GENS.keySet()); + types.sort(Comparator.comparing(a -> a.getClass().getName())); + PRIMITIVE_TYPE_GEN = SourceDSL.arbitrary().pick(types); + } + + private static final Set> NON_PRIMITIVE_TYPES = ImmutableSet.>builder() + .add(SetType.class) + .add(ListType.class) + .add(MapType.class) + .add(TupleType.class) + .add(UserType.class) + .add(VectorType.class) + .add(CompositeType.class) + .add(DynamicCompositeType.class) + .add(CounterColumnType.class) + .build(); private AbstractTypeGenerators() { @@ -99,15 +181,319 @@ public final class AbstractTypeGenerators } public enum TypeKind - {PRIMITIVE, SET, LIST, MAP, TUPLE, UDT} + { + PRIMITIVE, + SET, LIST, MAP, + TUPLE, UDT, + VECTOR, + COMPOSITE, DYNAMIC_COMPOSITE, + COUNTER + } private static final Gen TYPE_KIND_GEN = SourceDSL.arbitrary().enumValuesWithNoOrder(TypeKind.class); + public static Set> knownTypes() + { + Set> types = PRIMITIVE_TYPE_DATA_GENS.keySet().stream().map(a -> a.getClass()).collect(Collectors.toSet()); + types.addAll(NON_PRIMITIVE_TYPES); + types.addAll(UNSUPPORTED.keySet()); + return types; + } + public static Gen> primitiveTypeGen() { return PRIMITIVE_TYPE_GEN; } + public static Set>> UNSAFE_EQUALITY = ImmutableSet.of(EmptyType.class, + DurationType.class, + DecimalType.class, + CounterColumnType.class); + + public static Releaser overridePrimitiveTypeSupport(AbstractType type, TypeSupport support) + { + if (!PRIMITIVE_TYPE_DATA_GENS.containsKey(type)) + throw new IllegalArgumentException("Type " + type.asCQL3Type() + " is not a primitive"); + TypeSupport original = PRIMITIVE_TYPE_DATA_GENS.get(type); + PRIMITIVE_TYPE_DATA_GENS.put(type, support); + return () -> PRIMITIVE_TYPE_DATA_GENS.put(type, original); + } + + public static TypeGenBuilder withoutUnsafeEquality() + { + // make sure to keep UNSAFE_EQUALITY in-sync + return AbstractTypeGenerators.builder() + .withoutEmpty() + .withoutPrimitive(DurationType.instance) + // decimal "normalizes" the data to compare, so primary columns "may" mutate the data, causing missmatches + // see CASSANDRA-18530 + .withoutPrimitive(DecimalType.instance) + // counters are only for top level + .withoutTypeKinds(TypeKind.COUNTER); + } + + public interface Releaser extends AutoCloseable + { + @Override + void close(); + } + + public static class TypeGenBuilder + { + private int maxDepth = 3; + private EnumSet kinds; + private Gen typeKindGen; + private Gen defaultSizeGen = VERY_SMALL_POSITIVE_SIZE_GEN; + private Gen vectorSizeGen, tupleSizeGen, udtSizeGen, compositeSizeGen; + private Gen> primitiveGen = PRIMITIVE_TYPE_GEN, compositeElementGen; + private Gen userTypeKeyspaceGen = IDENTIFIER_GEN; + private Function>> defaultSetKeyFunc; + private Predicate> typeFilter = null; + private Gen udtName = null; + + public TypeGenBuilder() + { + } + + public TypeGenBuilder(TypeGenBuilder other) + { + maxDepth = other.maxDepth; + kinds = other.kinds == null ? null : EnumSet.copyOf(other.kinds); + typeKindGen = other.typeKindGen; + defaultSizeGen = other.defaultSizeGen; + vectorSizeGen = other.vectorSizeGen; + tupleSizeGen = other.tupleSizeGen; + udtName = other.udtName; + udtSizeGen = other.udtSizeGen; + primitiveGen = other.primitiveGen; + userTypeKeyspaceGen = other.userTypeKeyspaceGen; + defaultSetKeyFunc = other.defaultSetKeyFunc; + compositeElementGen = other.compositeElementGen; + compositeSizeGen = other.compositeSizeGen; + typeFilter = other.typeFilter; + } + + public TypeGenBuilder withTypeFilter(Predicate> fn) + { + typeFilter = fn; + return this; + } + + public TypeGenBuilder withCompositeElementGen(Gen> gen) + { + this.compositeElementGen = gen; + return this; + } + + @SuppressWarnings("unused") + public TypeGenBuilder withDefaultSetKey(Function>> mapKeyFunc) + { + this.defaultSetKeyFunc = mapKeyFunc; + return this; + } + + public TypeGenBuilder withDefaultSetKey(TypeGenBuilder builder) + { + this.defaultSetKeyFunc = builder::buildRecursive; + return this; + } + + public TypeGenBuilder withUserTypeKeyspace(String keyspace) + { + userTypeKeyspaceGen = SourceDSL.arbitrary().constant(keyspace); + return this; + } + + public TypeGenBuilder withDefaultSizeGen(int size) + { + return withDefaultSizeGen(SourceDSL.arbitrary().constant(size)); + } + + public TypeGenBuilder withDefaultSizeGen(Gen sizeGen) + { + this.defaultSizeGen = sizeGen; + return this; + } + + @SuppressWarnings("unused") + public TypeGenBuilder withVectorSizeGen(Gen sizeGen) + { + this.vectorSizeGen = sizeGen; + return this; + } + + @SuppressWarnings("unused") + public TypeGenBuilder withTupleSizeGen(Gen sizeGen) + { + this.tupleSizeGen = sizeGen; + return this; + } + + @SuppressWarnings("unused") + public TypeGenBuilder withUDTSizeGen(Gen sizeGen) + { + this.udtSizeGen = sizeGen; + return this; + } + + @SuppressWarnings("unused") + public TypeGenBuilder withCompositeSizeGen(Gen sizeGen) + { + this.compositeSizeGen = sizeGen; + return this; + } + + public TypeGenBuilder withoutEmpty() + { + return withoutPrimitive(EmptyType.instance); + } + + public TypeGenBuilder withoutPrimitive(AbstractType instance) + { + if (!PRIMITIVE_TYPE_DATA_GENS.containsKey(instance)) + throw new IllegalArgumentException("Type " + instance + " is not a primitive type, or PRIMITIVE_TYPE_DATA_GENS needs to add support"); + primitiveGen = filter(primitiveGen, t -> t != instance); + return this; + } + + @SuppressWarnings("unused") + public TypeGenBuilder withPrimitives(AbstractType first, AbstractType... remaining) + { + // any previous filters will be ignored... + primitiveGen = SourceDSL.arbitrary().pick(ArrayUtils.add(remaining, first)); + return this; + } + + public TypeGenBuilder withMaxDepth(int value) + { + this.maxDepth = value; + return this; + } + + public TypeGenBuilder withoutTypeKinds(TypeKind... values) + { + checkTypeKindValues(); + for (TypeKind kind : values) + kinds.remove(kind); + return this; + } + + public TypeGenBuilder withTypeKinds(TypeKind... values) + { + checkTypeKindValues(); + kinds.clear(); + Collections.addAll(kinds, values); + return this; + } + + private void checkTypeKindValues() + { + if (typeKindGen != null) + throw new IllegalArgumentException("Mixed both generator and individaul values for type kind"); + if (kinds == null) + kinds = EnumSet.allOf(TypeKind.class); + } + + public TypeGenBuilder withTypeKinds(Gen typeKindGen) + { + if (kinds != null) + throw new IllegalArgumentException("Mixed both generator and individaul values for type kind"); + this.typeKindGen = Objects.requireNonNull(typeKindGen); + return this; + } + + public TypeGenBuilder withUDTNames(Gen udtName) + { + this.udtName = udtName; + return this; + } + + public Gen> build() + { + if (udtName == null) + udtName = Generators.unique(IDENTIFIER_GEN); + // strip out the package to make it easier to read + // type parser assumes this package when one isn't provided, so this does not corrupt the type conversion + return buildRecursive(maxDepth).describedAs(t -> t.asCQL3Type().toString().replaceAll("org.apache.cassandra.db.marshal.", "")); + } + + private Gen> buildRecursive(int maxDepth) + { + if (udtName == null) + udtName = Generators.unique(IDENTIFIER_GEN); + Gen kindGen; + if (typeKindGen != null) + kindGen = typeKindGen; + else if (kinds != null) + { + ArrayList ts = new ArrayList<>(kinds); + Collections.sort(ts); + kindGen = SourceDSL.arbitrary().pick(ts); + } + else + kindGen = SourceDSL.arbitrary().enumValues(TypeKind.class); + return buildRecursive(maxDepth, maxDepth, kindGen, BOOLEAN_GEN); + } + + private Gen> buildRecursive(int maxDepth, int level, Gen typeKindGen, Gen multiCellGen) + { + if (level == -1) + return primitiveGen; + assert level >= 0 : "max depth must be positive or zero; given " + level; + boolean atBottom = level == 0; + boolean atTop = maxDepth == level; + Gen> gen = rnd -> { + Supplier>> next = () -> atBottom ? primitiveGen : buildRecursive(maxDepth, level - 1, typeKindGen, multiCellGen); + + // figure out type to get + TypeKind kind = typeKindGen.generate(rnd); + // counters are only allowed at the top level + while (!atTop && kind == TypeKind.COUNTER) + kind = typeKindGen.generate(rnd); + switch (kind) + { + case PRIMITIVE: + return primitiveGen.generate(rnd); + case SET: + if (defaultSetKeyFunc != null) + return setTypeGen(defaultSetKeyFunc.apply(level - 1), multiCellGen).generate(rnd); + return setTypeGen(next.get(), multiCellGen).generate(rnd); + case LIST: + return listTypeGen(next.get(), multiCellGen).generate(rnd); + case MAP: + if (defaultSetKeyFunc != null) + return mapTypeGen(defaultSetKeyFunc.apply(level - 1), next.get(), multiCellGen).generate(rnd); + return mapTypeGen(next.get(), next.get(), multiCellGen).generate(rnd); + case TUPLE: + return tupleTypeGen(atBottom ? primitiveGen : buildRecursive(maxDepth, level - 1, typeKindGen, SourceDSL.arbitrary().constant(false)), tupleSizeGen != null ? tupleSizeGen : defaultSizeGen).generate(rnd); + case UDT: + return userTypeGen(next.get(), udtSizeGen != null ? udtSizeGen : defaultSizeGen, userTypeKeyspaceGen, udtName, multiCellGen).generate(rnd); + case VECTOR: + { + Gen sizeGen = vectorSizeGen != null ? vectorSizeGen : defaultSizeGen; + return vectorTypeGen(next.get().map(AbstractType::freeze), sizeGen).generate(rnd); + } + case COMPOSITE: + return compositeTypeGen(compositeElementGen != null ? compositeElementGen : next.get(), compositeSizeGen != null ? compositeSizeGen : defaultSizeGen).generate(rnd); + case DYNAMIC_COMPOSITE: + Gen aliasGen = Generators.letterOrDigit().map(c -> (byte) c.charValue()); + // stores alias names by class and not what is actually valid by cql... so only primitive types match! + return dynamicCompositeGen(primitiveGen, aliasGen, defaultSizeGen).generate(rnd); + case COUNTER: + return CounterColumnType.instance; + default: + throw new IllegalArgumentException("Unknown kind: " + kind); + } + }; + return typeFilter == null ? gen : filter(gen, typeFilter); + } + } + + public static TypeGenBuilder builder() + { + return new TypeGenBuilder(); + } + public static Gen> typeGen() { return typeGen(3); @@ -120,28 +506,72 @@ public final class AbstractTypeGenerators public static Gen> typeGen(int maxDepth, Gen typeKindGen, Gen sizeGen) { - assert maxDepth >= 0 : "max depth must be positive or zero; given " + maxDepth; - boolean atBottom = maxDepth == 0; + return builder().withMaxDepth(maxDepth).withTypeKinds(typeKindGen).withDefaultSizeGen(sizeGen).build(); + } + + @SuppressWarnings("unused") + public static Gen> vectorTypeGen() + { + return vectorTypeGen(typeGen(2)); // lower the default depth since this is already a nested type + } + + public static Gen> vectorTypeGen(Gen> typeGen) + { + return vectorTypeGen(typeGen, SourceDSL.integers().between(1, 100)); + } + + public static Gen> vectorTypeGen(Gen> typeGen, Gen dimentionGen) + { return rnd -> { - // figure out type to get - TypeKind kind = typeKindGen.generate(rnd); - switch (kind) + int dimention = dimentionGen.generate(rnd); + AbstractType element = typeGen.generate(rnd); + // empty type not supported + while (element == EmptyType.instance) + element = typeGen.generate(rnd); + return VectorType.getInstance(element.freeze(), dimention); + }; + } + + @SuppressWarnings("unused") + public static Gen compositeTypeGen() + { + return compositeTypeGen(typeGen(2)); + } + + public static Gen compositeTypeGen(Gen> typeGen) + { + return compositeTypeGen(typeGen, VERY_SMALL_POSITIVE_SIZE_GEN); + } + + public static Gen compositeTypeGen(Gen> typeGen, Gen sizeGen) + { + return rnd -> { + int size = sizeGen.generate(rnd); + List> types = new ArrayList<>(size); + for (int i = 0; i < size; i++) { - case PRIMITIVE: - return PRIMITIVE_TYPE_GEN.generate(rnd); - case SET: - return setTypeGen(atBottom ? PRIMITIVE_TYPE_GEN : typeGen(maxDepth - 1, typeKindGen, sizeGen)).generate(rnd); - case LIST: - return listTypeGen(atBottom ? PRIMITIVE_TYPE_GEN : typeGen(maxDepth - 1, typeKindGen, sizeGen)).generate(rnd); - case MAP: - return mapTypeGen(atBottom ? PRIMITIVE_TYPE_GEN : typeGen(maxDepth - 1, typeKindGen, sizeGen)).generate(rnd); - case TUPLE: - return tupleTypeGen(atBottom ? PRIMITIVE_TYPE_GEN : typeGen(maxDepth - 1, typeKindGen, sizeGen), sizeGen).generate(rnd); - case UDT: - return userTypeGen(atBottom ? PRIMITIVE_TYPE_GEN : typeGen(maxDepth - 1, typeKindGen, sizeGen), sizeGen).generate(rnd); - default: - throw new IllegalArgumentException("Unknown kind: " + kind); + AbstractType type = typeGen.generate(rnd); + while (type == BytesType.instance || type == UUIDType.instance) + type = typeGen.generate(rnd); + types.add(type); } + return CompositeType.getInstance(types); + }; + } + + public static Gen dynamicCompositeGen(Gen> typeGen, Gen aliasGen, Gen sizeGen) + { + return rnd -> { + int size = sizeGen.generate(rnd); + Map> aliases = Maps.newHashMapWithExpectedSize(size); + for (int i = 0; i < size; i++) + { + byte alias = aliasGen.generate(rnd); + while (aliases.containsKey(alias)) + alias = aliasGen.generate(rnd); + aliases.put(alias, typeGen.generate(rnd).unfreeze()); + } + return DynamicCompositeType.getInstance(aliases); }; } @@ -153,7 +583,16 @@ public final class AbstractTypeGenerators public static Gen> setTypeGen(Gen> typeGen) { - return rnd -> SetType.getInstance(typeGen.generate(rnd), BOOLEAN_GEN.generate(rnd)); + return setTypeGen(typeGen, BOOLEAN_GEN); + } + + public static Gen> setTypeGen(Gen> typeGen, Gen multiCell) + { + return rnd -> { + boolean isMultiCell = multiCell.generate(rnd); + AbstractType element = typeGen.generate(rnd); + return SetType.getInstance(element.freeze(), isMultiCell); + }; } @SuppressWarnings("unused") @@ -164,7 +603,16 @@ public final class AbstractTypeGenerators public static Gen> listTypeGen(Gen> typeGen) { - return rnd -> ListType.getInstance(typeGen.generate(rnd), BOOLEAN_GEN.generate(rnd)); + return listTypeGen(typeGen, BOOLEAN_GEN); + } + + public static Gen> listTypeGen(Gen> typeGen, Gen multiCell) + { + return rnd -> { + boolean isMultiCell = multiCell.generate(rnd); + AbstractType element = typeGen.generate(rnd); + return ListType.getInstance(element.freeze(), isMultiCell); + }; } @SuppressWarnings("unused") @@ -180,7 +628,17 @@ public final class AbstractTypeGenerators public static Gen> mapTypeGen(Gen> keyGen, Gen> valueGen) { - return rnd -> MapType.getInstance(keyGen.generate(rnd), valueGen.generate(rnd), BOOLEAN_GEN.generate(rnd)); + return mapTypeGen(keyGen, valueGen, BOOLEAN_GEN); + } + + public static Gen> mapTypeGen(Gen> keyGen, Gen> valueGen, Gen multiCell) + { + return rnd -> { + boolean isMultiCell = multiCell.generate(rnd); + AbstractType key = keyGen.generate(rnd); + AbstractType value = valueGen.generate(rnd); + return MapType.getInstance(key.freeze(), value.freeze(), isMultiCell); + }; } public static Gen tupleTypeGen() @@ -215,24 +673,47 @@ public final class AbstractTypeGenerators } public static Gen userTypeGen(Gen> elementGen, Gen sizeGen) + { + return userTypeGen(elementGen, sizeGen, IDENTIFIER_GEN); + } + + public static Gen userTypeGen(Gen> elementGen, Gen sizeGen, Gen ksGen) + { + return userTypeGen(elementGen, sizeGen, ksGen, IDENTIFIER_GEN); + } + + public static Gen userTypeGen(Gen> elementGen, Gen sizeGen, Gen ksGen, Gen nameGen) + { + return userTypeGen(elementGen, sizeGen, ksGen, nameGen, BOOLEAN_GEN); + } + + public static Gen userTypeGen(Gen> elementGen, Gen sizeGen, Gen ksGen, Gen nameGen, Gen multiCellGen) { Gen fieldNameGen = IDENTIFIER_GEN.map(FieldIdentifier::forQuoted); return rnd -> { - boolean multiCell = BOOLEAN_GEN.generate(rnd); + boolean multiCell = multiCellGen.generate(rnd); int numElements = sizeGen.generate(rnd); List> fieldTypes = new ArrayList<>(numElements); LinkedHashSet fieldNames = new LinkedHashSet<>(numElements); - String ks = IDENTIFIER_GEN.generate(rnd); - ByteBuffer name = AsciiType.instance.decompose(IDENTIFIER_GEN.generate(rnd)); + String ks = ksGen.generate(rnd); + String name = nameGen.generate(rnd); + ByteBuffer nameBB = AsciiType.instance.decompose(name); - Gen distinctNameGen = Generators.filter(fieldNameGen, 30, e -> !fieldNames.contains(e)); + Gen distinctNameGen = filter(fieldNameGen, 30, e -> !fieldNames.contains(e)); // UDTs don't allow duplicate names, so make sure all names are unique for (int i = 0; i < numElements; i++) { - fieldTypes.add(elementGen.generate(rnd)); - fieldNames.add(distinctNameGen.generate(rnd)); + FieldIdentifier fieldName = distinctNameGen.generate(rnd); + fieldNames.add(fieldName); + + AbstractType element = elementGen.generate(rnd); + element = multiCell ? element.freeze() : element.unfreeze(); + // a UDT cannot contain a non-frozen UDT; as defined by CreateType + if (element.isUDT()) + element = element.freeze(); + fieldTypes.add(element); } - return new UserType(ks, name, new ArrayList<>(fieldNames), fieldTypes, multiCell); + return new UserType(ks, nameBB, new ArrayList<>(fieldNames), fieldTypes, multiCell); }; } @@ -249,76 +730,471 @@ public final class AbstractTypeGenerators return getTypeSupport(type, VERY_SMALL_POSITIVE_SIZE_GEN); } + public enum ValueDomain { NULL, EMPTY_BYTES, NORMAL } + + public static TypeSupport getTypeSupportWithNulls(AbstractType type, @Nullable Gen valueDomainGen) + { + return getTypeSupport(type, VERY_SMALL_POSITIVE_SIZE_GEN, valueDomainGen); + } + + public static TypeSupport getTypeSupport(AbstractType type, Gen sizeGen) + { + return getTypeSupport(type, sizeGen, null); + } + /** * For a type, create generators for data that matches that type */ - public static TypeSupport getTypeSupport(AbstractType type, Gen sizeGen) + public static TypeSupport getTypeSupport(AbstractType type, Gen sizeGen, @Nullable Gen valueDomainGen) { + Objects.requireNonNull(sizeGen, "sizeGen"); // this doesn't affect the data, only sort order, so drop it - if (type.isReversed()) - type = ((ReversedType) type).baseType; + type = type.unwrap(); // cast is safe since type is a constant and was type cast while inserting into the map @SuppressWarnings("unchecked") TypeSupport gen = (TypeSupport) PRIMITIVE_TYPE_DATA_GENS.get(type); + TypeSupport support; if (gen != null) - return gen; + { + support = gen; + } // might be... complex... - if (type instanceof SetType) + else if (type instanceof SetType) { // T = Set so can not use T here SetType setType = (SetType) type; - TypeSupport elementSupport = getTypeSupport(setType.getElementsType(), sizeGen); - @SuppressWarnings("unchecked") - TypeSupport support = (TypeSupport) TypeSupport.of(setType, rnd -> { + TypeSupport elementSupport = getTypeSupport(setType.getElementsType(), sizeGen, valueDomainGen); + Comparator elComparator = elementSupport.valueComparator; + Comparator> setComparator = listComparator(elComparator); + Comparator> comparator = (Set a, Set b) -> { + List as = new ArrayList<>(a); + as.sort(elComparator); + List bs = new ArrayList<>(b); + bs.sort(elComparator); + return setComparator.compare(as, bs); + }; + support = (TypeSupport) TypeSupport.of(setType, rnd -> { int size = sizeGen.generate(rnd); + size = normalizeSizeFromType(elementSupport, size); HashSet set = Sets.newHashSetWithExpectedSize(size); for (int i = 0; i < size; i++) - set.add(elementSupport.valueGen.generate(rnd)); + { + Object generate = elementSupport.valueGen.generate(rnd); + for (int attempts = 0; set.contains(generate); attempts++) + { + if (attempts == 42) + throw new AssertionError(String.format("Unable to get unique element for type %s with the size %d", typeTree(elementSupport.type), size)); + rnd = JavaRandom.wrap(rnd); + generate = elementSupport.valueGen.generate(rnd); + } + + set.add(generate); + } return set; - }); - return support; + }, comparator); } else if (type instanceof ListType) { // T = List so can not use T here ListType listType = (ListType) type; - TypeSupport elementSupport = getTypeSupport(listType.getElementsType(), sizeGen); - @SuppressWarnings("unchecked") - TypeSupport support = (TypeSupport) TypeSupport.of(listType, rnd -> { + TypeSupport elementSupport = getTypeSupport(listType.getElementsType(), sizeGen, valueDomainGen); + support = (TypeSupport) TypeSupport.of(listType, rnd -> { int size = sizeGen.generate(rnd); List list = new ArrayList<>(size); for (int i = 0; i < size; i++) list.add(elementSupport.valueGen.generate(rnd)); return list; - }); - return support; + }, listComparator(elementSupport.valueComparator)); } else if (type instanceof MapType) { // T = Map so can not use T here MapType mapType = (MapType) type; - TypeSupport keySupport = getTypeSupport(mapType.getKeysType(), sizeGen); - TypeSupport valueSupport = getTypeSupport(mapType.getValuesType(), sizeGen); - @SuppressWarnings("unchecked") - TypeSupport support = (TypeSupport) TypeSupport.of(mapType, rnd -> { + // do not use valueDomainGen as map doesn't allow null/empty + TypeSupport keySupport = getTypeSupport(mapType.getKeysType(), sizeGen, null); + Comparator keyType = keySupport.valueComparator; + TypeSupport valueSupport = getTypeSupport(mapType.getValuesType(), sizeGen, null); + Comparator valueType = valueSupport.valueComparator; + Comparator> comparator = (Map a, Map b) -> { + List ak = new ArrayList<>(a.keySet()); + ak.sort(keyType); + List bk = new ArrayList<>(b.keySet()); + bk.sort(keyType); + for (int i = 0, size = Math.min(ak.size(), bk.size()); i < size; i++) + { + int rc = keyType.compare(ak.get(i), bk.get(i)); + if (rc != 0) + return rc; + // why can't we use the same key? DecimalType uses BigDecimal.compareTo, which doesn't account for scale differences + // so equality won't match, but comparator says they do! + rc = valueType.compare(a.get(ak.get(i)), b.get(bk.get(i))); + if (rc != 0) + return rc; + } + return Integer.compare(a.size(), b.size()); + }; + support = (TypeSupport) TypeSupport.of(mapType, rnd -> { int size = sizeGen.generate(rnd); + size = normalizeSizeFromType(keySupport, size); Map map = Maps.newHashMapWithExpectedSize(size); // if there is conflict thats fine for (int i = 0; i < size; i++) - map.put(keySupport.valueGen.generate(rnd), valueSupport.valueGen.generate(rnd)); + { + Object key = keySupport.valueGen.generate(rnd); + for (int attempts = 0; map.containsKey(key); attempts++) + { + if (attempts == 42) + throw new AssertionError(String.format("Unable to get unique element for type %s with the size %d", typeTree(keySupport.type), size)); + rnd = JavaRandom.wrap(rnd); + key = keySupport.valueGen.generate(rnd); + } + map.put(key, valueSupport.valueGen.generate(rnd)); + } return map; - }); - return support; + }, comparator); } else if (type instanceof TupleType) // includes UserType { // T is ByteBuffer TupleType tupleType = (TupleType) type; - @SuppressWarnings("unchecked") - TypeSupport support = (TypeSupport) TypeSupport.of(tupleType, new TupleGen(tupleType, sizeGen)); - return support; + List> columns = (List>) (List) tupleType.allTypes().stream().map(AbstractTypeGenerators::comparator).collect(Collectors.toList()); + Comparator> listCompar = listComparator((i, a, b) -> columns.get(i).compare(a, b)); + Comparator comparator = (ByteBuffer a, ByteBuffer b) -> { + ByteBuffer[] abb = tupleType.split(ByteBufferAccessor.instance, a); + List av = IntStream.range(0, abb.length).mapToObj(i -> tupleType.type(i).compose(abb[i])).collect(Collectors.toList()); + + ByteBuffer[] bbb = tupleType.split(ByteBufferAccessor.instance, b); + List bv = IntStream.range(0, bbb.length).mapToObj(i -> tupleType.type(i).compose(bbb[i])).collect(Collectors.toList()); + return listCompar.compare(av, bv); + }; + support = (TypeSupport) TypeSupport.of(tupleType, new TupleGen(tupleType, sizeGen, valueDomainGen), comparator); } - throw new UnsupportedOperationException("Unsupported type: " + type); + else if (type instanceof VectorType) + { + VectorType vectorType = (VectorType) type; + TypeSupport elementSupport = getTypeSupport(vectorType.elementType, sizeGen, valueDomainGen); + support = (TypeSupport) TypeSupport.of(vectorType, rnd -> { + List list = new ArrayList<>(vectorType.dimension); + for (int i = 0; i < vectorType.dimension; i++) + { + Object generate = elementSupport.valueGen.generate(rnd); + if (generate == null) + throw new AssertionError(String.format("TypeSupport(%s) generated a null value", vectorType.elementType.asCQL3Type())); + list.add(generate); + } + return list; + }, listComparator(elementSupport.valueComparator)); + } + else if (type instanceof CompositeType) + { + CompositeType ct = (CompositeType) type; + List> elementSupport = (List>) (List) ct.types.stream().map(AbstractTypeGenerators::getTypeSupport).collect(Collectors.toList()); + Serde> serde = new Serde>() + { + @Override + public ByteBuffer from(List objects) + { + return ct.decompose(objects.toArray()); + } + + @Override + public List to(ByteBuffer buffer) + { + ByteBuffer[] bbs = ct.split(buffer); + List values = new ArrayList<>(bbs.length); + for (int i = 0; i < bbs.length; i++) + values.add(bbs[i] == null ? null : elementSupport.get(i).type.compose(bbs[i])); + return values; + } + }; + support = (TypeSupport) TypeSupport.of(ct, serde, rnd -> { + List values = new ArrayList<>(ct.types.size()); + for (int i = 0, size = ct.types.size(); i < size; i++) + values.add(elementSupport.get(i).valueGen.generate(rnd)); + return values; + }, listComparator((index, a, b) -> elementSupport.get(index).valueComparator.compare(a, b))); + } + else if (type instanceof DynamicCompositeType) + { + // data generation limits to what is valid for the type, and sadly the type + DynamicCompositeType dct = (DynamicCompositeType) type; + Map> supports = new TreeMap<>(); + for (Map.Entry> e : dct.aliases.entrySet()) + supports.put(e.getKey(), getTypeSupport(e.getValue())); + List orderedAliases = new ArrayList<>(supports.keySet()); + Serde> serde = new Serde>() + { + @Override + public ByteBuffer from(Map byteObjectMap) + { + return dct.build(byteObjectMap); + } + + @Override + public Map to(ByteBuffer buffer) + { + ByteBuffer[] parts = dct.split(buffer); + Map values = Maps.newHashMapWithExpectedSize(parts.length); + for (int i = 0; i < parts.length; i++) + { + Byte alias = orderedAliases.get(i); + AbstractType type = dct.aliases.get(alias); + ByteBuffer bytes = parts[i]; + type.validate(bytes); + values.put(alias, type.compose(bytes)); + } + return values; + } + }; + support = (TypeSupport) TypeSupport.of(dct, serde, rnd -> { + Map byteObjectMap = new TreeMap<>(); + for (Byte alias : orderedAliases) + byteObjectMap.put(alias, supports.get(alias).valueGen.generate(rnd)); + return byteObjectMap; + }, (a, b) -> { + for (Byte alias : orderedAliases) + { + @SuppressWarnings("rawtype") + TypeSupport ts = supports.get(alias); + Object as = a.get(alias); + Object bs = b.get(alias); + int rc = ts.valueComparator.compare(as, bs); + if (rc != 0) + return rc; + } + return 0; + }); + } + else if (type instanceof CounterColumnType) + { + support = (TypeSupport) TypeSupport.of(CounterColumnType.instance, SourceDSL.longs().all()); + } + else + { + throw new UnsupportedOperationException("Unsupported type: " + type); + } + return support.withValueDomain(valueDomainGen); + } + + public static Comparator comparator(AbstractType type) + { + return getTypeSupport(type).valueComparator; + } + + private static Comparator> listComparator(Comparator elements) + { + return listComparator((ignore, a, b) -> elements.compare(a, b)); + } + + private interface IndexComparator + { + int compare(int index, T a, T b); + } + private static Comparator> listComparator(IndexComparator ordering) + { + return (a, b) -> { + for (int i = 0, size = Math.min(a.size(), b.size()); i < size; i++) + { + int rc = ordering.compare(i, a.get(i), b.get(i)); + if (rc != 0) + return rc; + } + return Integer.compare(a.size(), b.size()); + }; + } + + private static int uniqueElementsForDomain(AbstractType type) + { + type = type.unwrap(); + if (type instanceof BooleanType) + return 2; + if (type instanceof EmptyType) + return 1; + if (type instanceof SetType) + return uniqueElementsForDomain(((SetType) type).getElementsType()); + if (type instanceof MapType) + return uniqueElementsForDomain(((MapType) type).getKeysType()); + if (type instanceof VectorType) + { + VectorType vector = (VectorType) type; + int uniq = uniqueElementsForDomain(vector.elementType); + if (uniq != -1) + return uniq == 1 ? 1 : uniq * vector.dimension; + } + if (type instanceof TupleType || type instanceof CompositeType || type instanceof DynamicCompositeType) + { + int product = 1; + for (AbstractType f : type.subTypes()) + { + int uniq = uniqueElementsForDomain(f); + if (uniq == -1) + return -1; + product *= uniq; + } + return product; + } + return -1; + } + + private static int normalizeSizeFromType(TypeSupport keySupport, int size) + { + int uniq = uniqueElementsForDomain(keySupport.type); + if (uniq == -1) + return size; + return Math.min(size, uniq); + } + + public static Set extractUDTs(AbstractType type) + { + Set matches = new HashSet<>(); + extractUDTs(type, matches); + return matches; + } + + public static void extractUDTs(AbstractType type, Set matches) + { + if (type instanceof ReversedType) + type = ((ReversedType) type).baseType; + if (type instanceof UserType) + matches.add((UserType) type); + for (AbstractType t : type.subTypes()) + extractUDTs(t, matches); + } + + public static String typeTree(AbstractType type) + { + StringBuilder sb = new StringBuilder(); + typeTree(sb, type, 0); + return sb.toString().trim(); + } + + private static void typeTree(StringBuilder sb, AbstractType type, int indent) + { + if (type.isUDT()) + { + if (indent != 0) + { + indent += 2; + newline(sb, indent); + } + UserType ut = (UserType) type; + if (!type.isMultiCell()) sb.append("frozen "); + sb.append("udt[").append(ColumnIdentifier.maybeQuote(ut.elementName())).append("]:"); + int elementIndent = indent + 2; + for (int i = 0; i < ut.size(); i++) + { + newline(sb, elementIndent); + FieldIdentifier fieldName = ut.fieldName(i); + AbstractType fieldType = ut.fieldType(i); + sb.append(ColumnIdentifier.maybeQuote(fieldName.toString())).append(": "); + typeTree(sb, fieldType, elementIndent); + } + newline(sb, elementIndent); + } + else if (type.isTuple()) + { + if (indent != 0) + { + indent += 2; + newline(sb, indent); + } + TupleType tt = (TupleType) type; + sb.append("tuple:"); + int elementIndent = indent + 2; + for (int i = 0; i < tt.size(); i++) + { + newline(sb, elementIndent); + AbstractType fieldType = tt.type(i); + sb.append(i).append(": "); + typeTree(sb, fieldType, elementIndent); + } + } + else if (type.isVector()) + { + if (indent != 0) + { + indent += 2; + newline(sb, indent); + } + VectorType vt = (VectorType) type; + sb.append("vector[").append(vt.dimension).append("]: "); + indent += 2; + typeTree(sb, vt.elementType, indent); + } + else if (type.isCollection()) + { + CollectionType ct = (CollectionType) type; + if (indent != 0) + { + indent += 2; + newline(sb, indent); + } + if (!type.isMultiCell()) sb.append("frozen "); + switch (ct.kind) + { + case MAP: + { + MapType mt = (MapType) type; + sb.append("map:"); + indent += 2; + newline(sb, indent); + sb.append("key: "); + int subTypeIndent = indent + 2; + typeTree(sb, mt.getKeysType(), subTypeIndent); + newline(sb, indent); + sb.append("value: "); + typeTree(sb, mt.getValuesType(), subTypeIndent); + } + break; + case LIST: + { + ListType lt = (ListType) type; + sb.append("list: "); + indent += 2; + typeTree(sb, lt.getElementsType(), indent); + } + break; + case SET: + { + SetType st = (SetType) type; + sb.append("set: "); + indent += 2; + typeTree(sb, st.getElementsType(), indent); + } + break; + default: + throw new UnsupportedOperationException("Unknown kind: " + ct.kind); + } + } + else if (type instanceof CompositeType) + { + CompositeType ct = (CompositeType) type; + if (indent != 0) + { + indent += 2; + newline(sb, indent); + } + sb.append("CompositeType:"); + indent += 2; + int idx = 0; + for (AbstractType subtype : ct.subTypes()) + { + newline(sb, indent); + sb.append(idx++).append(": "); + typeTree(sb, subtype, indent); + } + } + else + { + sb.append(type.asCQL3Type().toString().replaceAll("org.apache.cassandra.db.marshal.", "")); + } + } + + private static void newline(StringBuilder sb, int indent) + { + sb.append('\n'); + for (int i = 0; i < indent; i++) + sb.append(' '); } private static final class TupleGen implements Gen @@ -326,9 +1202,9 @@ public final class AbstractTypeGenerators private final List> elementsSupport; @SuppressWarnings("unchecked") - private TupleGen(TupleType tupleType, Gen sizeGen) + private TupleGen(TupleType tupleType, Gen sizeGen, @Nullable Gen valueDomainGen) { - this.elementsSupport = tupleType.allTypes().stream().map(t -> getTypeSupport((AbstractType) t, sizeGen)).collect(Collectors.toList()); + this.elementsSupport = tupleType.allTypes().stream().map(t -> getTypeSupport((AbstractType) t, sizeGen, valueDomainGen)).collect(Collectors.toList()); } public ByteBuffer generate(RandomnessSource rnd) @@ -344,6 +1220,12 @@ public final class AbstractTypeGenerators } } + public interface Serde + { + A from(B b); + B to(A a); + } + /** * Pair of {@link AbstractType} and a Generator of values that are handled by that type. */ @@ -351,16 +1233,38 @@ public final class AbstractTypeGenerators { public final AbstractType type; public final Gen valueGen; + private final Gen bytesGen; + public final Comparator valueComparator; - private TypeSupport(AbstractType type, Gen valueGen) + private TypeSupport(AbstractType type, Gen valueGen, Comparator valueComparator) { this.type = Objects.requireNonNull(type); this.valueGen = Objects.requireNonNull(valueGen); + this.bytesGen = rnd -> type.decompose(valueGen.generate(rnd)); + this.valueComparator = Objects.requireNonNull(valueComparator); } - public static TypeSupport of(AbstractType type, Gen valueGen) + private TypeSupport(AbstractType type, Gen valueGen, Gen bytesGen, Comparator valueComparator) { - return new TypeSupport<>(type, valueGen); + this.type = type; + this.valueGen = valueGen; + this.bytesGen = bytesGen; + this.valueComparator = valueComparator; + } + + public static > TypeSupport of(AbstractType type, Gen valueGen) + { + return new TypeSupport<>(type, valueGen, Comparator.naturalOrder()); + } + + public static TypeSupport of(AbstractType type, Gen valueGen, Comparator valueComparator) + { + return new TypeSupport<>(type, valueGen, valueComparator); + } + + public static TypeSupport of(AbstractType type, Serde serde, Gen valueGen, Comparator valueComparator) + { + return of(type, valueGen.map(serde::from), (a, b) -> valueComparator.compare(serde.to(a), serde.to(b))); } /** @@ -368,7 +1272,45 @@ public final class AbstractTypeGenerators */ public Gen bytesGen() { - return rnd -> type.decompose(valueGen.generate(rnd)); + return bytesGen; + } + + public TypeSupport mapBytes(Function fn) + { + return new TypeSupport<>(type, valueGen, bytesGen.map(fn), valueComparator); + } + + public TypeSupport withoutEmptyData() + { + if (!type.allowsEmpty()) + return this; + return new TypeSupport<>(type, valueGen, filter(bytesGen, b -> !ByteBufferAccessor.instance.isEmpty(b)), valueComparator); + } + + public TypeSupport withValueDomain(@Nullable Gen valueDomainGen) + { + if (valueDomainGen == null || !type.allowsEmpty()) + return this; + Gen gen = rnd -> { + ValueDomain domain = valueDomainGen.generate(rnd); + ByteBuffer value; + switch (domain) + { + case NULL: + value = null; + break; + case EMPTY_BYTES: + value = ByteBufferUtil.EMPTY_BYTE_BUFFER; + break; + case NORMAL: + value = bytesGen.generate(rnd); + break; + default: + throw new IllegalArgumentException("Unknown domain: " + domain); + } + return value; + }; + return new TypeSupport<>(type, valueGen, gen, valueComparator); } public String toString() diff --git a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java index 210e5cb1f5..49701859b9 100644 --- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java +++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java @@ -24,14 +24,18 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.Nullable; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.builder.MultilineRecursiveToStringStyle; @@ -39,6 +43,7 @@ import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.Duration; import org.apache.cassandra.cql3.FieldIdentifier; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.SchemaCQLHelper; @@ -50,6 +55,7 @@ import org.apache.cassandra.db.marshal.CompositeType; import org.apache.cassandra.db.marshal.EmptyType; import org.apache.cassandra.db.marshal.TimeUUIDType; import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.marshal.UserType; import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.LocalPartitioner; @@ -68,9 +74,11 @@ import org.apache.cassandra.net.NoPayload; import org.apache.cassandra.net.PingRequest; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.MemtableParams; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableParams; +import org.apache.cassandra.utils.AbstractTypeGenerators.ValueDomain; import org.quicktheories.core.Gen; import org.quicktheories.core.RandomnessSource; import org.quicktheories.generators.Generate; @@ -157,41 +165,192 @@ public final class CassandraGenerators } - private static TableMetadata createTableMetadata(String ks, RandomnessSource rnd) + public static Set extractUDTs(TableMetadata metadata) { - String tableName = IDENTIFIER_GEN.generate(rnd); - TableMetadata.Builder builder = TableMetadata.builder(ks, tableName, TableId.fromUUID(Generators.UUID_RANDOM_GEN.generate(rnd))) - .partitioner(PARTITIONER_GEN.generate(rnd)) - .kind(TABLE_KIND_GEN.generate(rnd)) - .isCounter(BOOLEAN_GEN.generate(rnd)) - .params(TableParams.builder().build()); + Set matches = new HashSet<>(); + for (ColumnMetadata col : metadata.columns()) + AbstractTypeGenerators.extractUDTs(col.type, matches); + return matches; + } - // generate columns - // must have a non-zero amount of partition columns, but may have 0 for the rest; SMALL_POSSITIVE_SIZE_GEN won't return 0 - int numPartitionColumns = SMALL_POSITIVE_SIZE_GEN.generate(rnd); - int numClusteringColumns = SMALL_POSITIVE_SIZE_GEN.generate(rnd) - 1; - int numRegularColumns = SMALL_POSITIVE_SIZE_GEN.generate(rnd) - 1; - int numStaticColumns = SMALL_POSITIVE_SIZE_GEN.generate(rnd) - 1; + public static TableMetadata createTableMetadata(String ks, RandomnessSource rnd) + { + return new TableMetadataBuilder().withKeyspaceName(ks).build(rnd); + } - Set createdColumnNames = new HashSet<>(); - for (int i = 0; i < numPartitionColumns; i++) - builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.PARTITION_KEY, createdColumnNames, rnd)); - for (int i = 0; i < numClusteringColumns; i++) - builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.CLUSTERING, createdColumnNames, rnd)); - for (int i = 0; i < numStaticColumns; i++) - builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.STATIC, createdColumnNames, rnd)); - for (int i = 0; i < numRegularColumns; i++) - builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.REGULAR, createdColumnNames, rnd)); + public static class TableMetadataBuilder + { + private Gen ksNameGen = IDENTIFIER_GEN; + private Gen> defaultTypeGen = AbstractTypeGenerators.builder() + .withDefaultSetKey(AbstractTypeGenerators.withoutUnsafeEquality()) + .withMaxDepth(1) + .build(); + private Gen> partitionColTypeGen, clusteringColTypeGen, staticColTypeGen, regularColTypeGen; + private Gen tableKindGen = SourceDSL.arbitrary().constant(TableMetadata.Kind.REGULAR); + private Gen numPartitionColumnsGen = SourceDSL.integers().between(1, 2); + private Gen numClusteringColumnsGen = SourceDSL.integers().between(1, 2); + private Gen numRegularColumnsGen = SourceDSL.integers().between(1, 5); + private Gen numStaticColumnsGen = SourceDSL.integers().between(0, 2); + private Gen memtableKeyGen = null; - return builder.build(); + public TableMetadataBuilder withKnownMemtables() + { + Set known = MemtableParams.knownDefinitions(); + // for testing reason, some invalid types are added; filter out + List valid = known.stream().filter(name -> !name.startsWith("test_")).collect(Collectors.toList()); + memtableKeyGen = SourceDSL.arbitrary().pick(valid); + return this; + } + + public TableMetadataBuilder withKeyspaceName(Gen ksNameGen) + { + this.ksNameGen = ksNameGen; + return this; + } + + public TableMetadataBuilder withKeyspaceName(String name) + { + this.ksNameGen = SourceDSL.arbitrary().constant(name); + return this; + } + + public TableMetadataBuilder withPartitionColumnsCount(int num) + { + this.numPartitionColumnsGen = SourceDSL.arbitrary().constant(num); + return this; + } + + public TableMetadataBuilder withPartitionColumnsBetween(int min, int max) + { + this.numPartitionColumnsGen = SourceDSL.integers().between(min, max); + return this; + } + + public TableMetadataBuilder withClusteringColumnsCount(int num) + { + this.numClusteringColumnsGen = SourceDSL.arbitrary().constant(num); + return this; + } + + public TableMetadataBuilder withClusteringColumnsBetween(int min, int max) + { + this.numClusteringColumnsGen = SourceDSL.integers().between(min, max); + return this; + } + + public TableMetadataBuilder withRegularColumnsCount(int num) + { + this.numRegularColumnsGen = SourceDSL.arbitrary().constant(num); + return this; + } + + public TableMetadataBuilder withRegularColumnsBetween(int min, int max) + { + this.numRegularColumnsGen = SourceDSL.integers().between(min, max); + return this; + } + + public TableMetadataBuilder withStaticColumnsCount(int num) + { + this.numStaticColumnsGen = SourceDSL.arbitrary().constant(num); + return this; + } + + public TableMetadataBuilder withStaticColumnsBetween(int min, int max) + { + this.numStaticColumnsGen = SourceDSL.integers().between(min, max); + return this; + } + + public TableMetadataBuilder withDefaultTypeGen(Gen> typeGen) + { + this.defaultTypeGen = typeGen; + return this; + } + + public TableMetadataBuilder withPrimaryColumnTypeGen(Gen> typeGen) + { + withPartitionColumnTypeGen(typeGen); + withClusteringColumnTypeGen(typeGen); + return this; + } + + public TableMetadataBuilder withPartitionColumnTypeGen(Gen> typeGen) + { + this.partitionColTypeGen = typeGen; + return this; + } + + public TableMetadataBuilder withClusteringColumnTypeGen(Gen> typeGen) + { + this.clusteringColTypeGen = typeGen; + return this; + } + + public TableMetadataBuilder withStaticColumnTypeGen(Gen> typeGen) + { + this.staticColTypeGen = typeGen; + return this; + } + + public TableMetadataBuilder withRegularColumnTypeGen(Gen> typeGen) + { + this.regularColTypeGen = typeGen; + return this; + } + + public TableMetadataBuilder withTableKinds(TableMetadata.Kind... kinds) + { + tableKindGen = SourceDSL.arbitrary().pick(kinds); + return this; + } + + public Gen build() + { + return rnd -> build(rnd); + } + + public TableMetadata build(RandomnessSource rnd) + { + if (partitionColTypeGen == null && clusteringColTypeGen == null) + withPrimaryColumnTypeGen(Generators.filter(defaultTypeGen, t -> !AbstractTypeGenerators.UNSAFE_EQUALITY.contains(t.getClass()))); + + String ks = ksNameGen.generate(rnd); + String tableName = IDENTIFIER_GEN.generate(rnd); + TableParams.Builder params = TableParams.builder(); + if (memtableKeyGen != null) + params.memtable(MemtableParams.get(memtableKeyGen.generate(rnd))); + TableMetadata.Builder builder = TableMetadata.builder(ks, tableName, TableId.fromUUID(Generators.UUID_RANDOM_GEN.generate(rnd))) + .partitioner(PARTITIONER_GEN.generate(rnd)) + .kind(tableKindGen.generate(rnd)) + .isCounter(BOOLEAN_GEN.generate(rnd)) + .params(params.build()); + + int numPartitionColumns = numPartitionColumnsGen.generate(rnd); + int numClusteringColumns = numClusteringColumnsGen.generate(rnd); + int numRegularColumns = numRegularColumnsGen.generate(rnd); + int numStaticColumns = numStaticColumnsGen.generate(rnd); + + Set createdColumnNames = new HashSet<>(); + for (int i = 0; i < numPartitionColumns; i++) + builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.PARTITION_KEY, createdColumnNames, partitionColTypeGen == null ? defaultTypeGen : partitionColTypeGen, rnd)); + for (int i = 0; i < numClusteringColumns; i++) + builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.CLUSTERING, createdColumnNames, clusteringColTypeGen == null ? defaultTypeGen : clusteringColTypeGen, rnd)); + for (int i = 0; i < numStaticColumns; i++) + builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.STATIC, createdColumnNames, staticColTypeGen == null ? defaultTypeGen : staticColTypeGen, rnd)); + for (int i = 0; i < numRegularColumns; i++) + builder.addColumn(createColumnDefinition(ks, tableName, ColumnMetadata.Kind.REGULAR, createdColumnNames, regularColTypeGen == null ? defaultTypeGen : regularColTypeGen, rnd)); + + return builder.build(); + } } private static ColumnMetadata createColumnDefinition(String ks, String table, ColumnMetadata.Kind kind, Set createdColumnNames, /* This is mutated to check for collisions, so has a side effect outside of normal random generation */ + Gen> typeGen, RandomnessSource rnd) { - Gen> typeGen = AbstractTypeGenerators.typeGen(); switch (kind) { // partition and clustering keys require frozen types, so make sure all types generated will be frozen @@ -221,7 +380,7 @@ public final class CassandraGenerators ImmutableList columns = metadata.partitionKeyColumns(); assert !columns.isEmpty() : "Unable to find partition key columns"; if (columns.size() == 1) - return getTypeSupport(columns.get(0).type).bytesGen(); + return getTypeSupport(columns.get(0).type).withoutEmptyData().bytesGen(); List> columnGens = new ArrayList<>(columns.size()); for (ColumnMetadata cm : columns) columnGens.add(getTypeSupport(cm.type).bytesGen()); @@ -233,6 +392,31 @@ public final class CassandraGenerators }; } + public static Gen data(TableMetadata metadata, @Nullable Gen valueDomainGen) + { + AbstractTypeGenerators.TypeSupport[] types = new AbstractTypeGenerators.TypeSupport[metadata.columns().size()]; + Iterator it = metadata.allColumnsInSelectOrder(); + int partitionColumns = metadata.partitionKeyColumns().size(); + int clusteringColumns = metadata.clusteringColumns().size(); + int primaryKeyColumns = partitionColumns + clusteringColumns; + for (int i = 0; it.hasNext(); i++) + { + ColumnMetadata col = it.next(); + types[i] = AbstractTypeGenerators.getTypeSupportWithNulls(col.type, i < partitionColumns ? null : valueDomainGen); + if (i < partitionColumns) + types[i] = types[i].withoutEmptyData(); + if (i >= partitionColumns && i < primaryKeyColumns) + // clustering doesn't allow null... + types[i] = types[i].mapBytes(b -> b == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : b); + } + return rnd -> { + ByteBuffer[] row = new ByteBuffer[types.length]; + for (int i = 0; i < row.length; i++) + row[i] = types[i].bytesGen().generate(rnd); + return row; + }; + } + /** * Hacky workaround to make sure different generic MessageOut types can be used for {@link #MESSAGE_GEN}. */ @@ -489,4 +673,23 @@ public final class CassandraGenerators return state; }; } + + public static Gen duration() + { + Constraint ints = Constraint.between(0, Integer.MAX_VALUE); + Constraint longs = Constraint.between(0, Long.MAX_VALUE); + Gen neg = SourceDSL.booleans().all(); + return rnd -> { + int months = (int) rnd.next(ints); + int days = (int) rnd.next(ints); + long nanoseconds = rnd.next(longs); + if (neg.generate(rnd)) + { + months = -1 * months; + days = -1 * days; + nanoseconds = -1 * nanoseconds; + } + return Duration.newInstance(months, days, nanoseconds); + }; + } } diff --git a/test/unit/org/apache/cassandra/utils/Generators.java b/test/unit/org/apache/cassandra/utils/Generators.java index 1baa16e32c..00d84a24ed 100644 --- a/test/unit/org/apache/cassandra/utils/Generators.java +++ b/test/unit/org/apache/cassandra/utils/Generators.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.utils; +import java.math.BigDecimal; +import java.math.BigInteger; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; @@ -25,7 +27,9 @@ import java.sql.Timestamp; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Date; +import java.util.HashSet; import java.util.Random; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; @@ -63,6 +67,11 @@ public final class Generators public static final Gen IDENTIFIER_GEN = Generators.regexWord(SourceDSL.integers().between(1, 50)); + public static Gen letterOrDigit() + { + return SourceDSL.integers().between(0, LETTER_OR_DIGIT_DOMAIN.length - 1).map(idx -> LETTER_OR_DIGIT_DOMAIN[idx]); + } + public static final Gen UUID_RANDOM_GEN = rnd -> { long most = rnd.next(Constraint.none()); most &= 0x0f << 8; /* clear version */ @@ -73,6 +82,16 @@ public final class Generators return new UUID(most, least); }; + public static final Gen UUID_TIME_GEN = rnd -> { + long most = rnd.next(Constraint.none()); + most &= 0x0f << 8; /* clear version */ + most += 0x10 << 8; /* set to version 1 */ + long least = rnd.next(Constraint.none()); + least &= 0x3fl << 56; /* clear variant */ + least |= 0x80l << 56; /* set to IETF variant */ + return new UUID(most, least); + }; + public static final Gen DNS_DOMAIN_NAME = rnd -> { // how many parts to generate int numParts = (int) rnd.next(DNS_DOMAIN_PARTS_CONSTRAINT); @@ -363,6 +382,70 @@ public final class Generators .map(s -> new String(s.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8)); } + public static Gen bigInt() + { + return bigInt(SourceDSL.integers().between(1, 32)); + } + + public static Gen bigInt(Gen numBitsGen) + { + Gen signumGen = SourceDSL.arbitrary().pick(-1, 0, 1); + return rnd -> { + int signum = signumGen.generate(rnd); + if (signum == 0) + return BigInteger.ZERO; + int numBits = numBitsGen.generate(rnd); + if (numBits < 0) + throw new IllegalArgumentException("numBits must be non-negative"); + int numBytes = (int)(((long)numBits+7)/8); // avoid overflow + + // Generate random bytes and mask out any excess bits + byte[] randomBits = new byte[0]; + if (numBytes > 0) { + randomBits = bytes(numBytes, numBytes).map(bb -> ByteBufferUtil.getArray(bb)).generate(rnd); + int excessBits = 8*numBytes - numBits; + randomBits[0] &= (1 << (8-excessBits)) - 1; + } + return new BigInteger(signum, randomBits); + }; + } + + public static Gen bigDecimal() + { + return bigDecimal(SourceDSL.integers().between(1, 100), bigInt()); + } + + public static Gen bigDecimal(Gen scaleGen, Gen bigIntegerGen) + { + return rnd -> { + int scale = scaleGen.generate(rnd); + BigInteger bigInt = bigIntegerGen.generate(rnd); + return new BigDecimal(bigInt, scale); + }; + } + + public static Gen unique(Gen gen) + { + Set dedup = new HashSet<>(); + return filter(gen, dedup::add); + } + + public static Gen cached(Gen gen) + { + Object cacheMissed = new Object(); + return new Gen() + { + private Object value = cacheMissed; + @Override + public T generate(RandomnessSource randomnessSource) + { + if (value == cacheMissed) + value = gen.generate(randomnessSource); + return (T) value; + } + }; + } + private static boolean isDash(char c) { switch (c) diff --git a/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java b/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java index 7894fe437c..2212a1cfba 100644 --- a/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java +++ b/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java @@ -23,16 +23,23 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; +import java.util.Arrays; import org.junit.Test; +import org.apache.cassandra.db.marshal.ByteArrayAccessor; +import org.apache.cassandra.db.marshal.ByteBufferAccessor; +import org.apache.cassandra.db.marshal.ValueAccessor; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus; import org.apache.cassandra.utils.CassandraUInt; +import org.assertj.core.api.Assertions; +import org.quicktheories.generators.SourceDSL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import static org.quicktheories.QuickTheory.qt; public class VIntCodingTest { @@ -40,6 +47,70 @@ public class VIntCodingTest 168435455L, 33251130335L, 3281283447775L, 417672546086779L, 52057592037927932L, 72057594037927937L}; + @Test + public void serdeSigned() + { + qt().forAll(SourceDSL.longs().all()).checkAssert(l -> { + for (ValueAccessor accessor : Arrays.asList(ByteBufferAccessor.instance, ByteArrayAccessor.instance)) + serdeSigned(l, accessor); + }); + } + + private static void serdeSigned(long value, ValueAccessor accessor) + { + T buffer = accessor.allocate(Long.BYTES + 1); + VIntCoding.writeVInt(value, buffer, 0, accessor); + Assertions.assertThat(VIntCoding.getVInt(buffer, accessor, 0)).isEqualTo(value); + } + + @Test + public void serdeUnsigned() + { + qt().forAll(SourceDSL.longs().between(0, Long.MAX_VALUE)).checkAssert(l -> { + for (ValueAccessor accessor : Arrays.asList(ByteBufferAccessor.instance, ByteArrayAccessor.instance)) + serdeUnsigned(l, accessor); + }); + } + + private static void serdeUnsigned(long value, ValueAccessor accessor) + { + T buffer = accessor.allocate(Long.BYTES + 1); + VIntCoding.writeUnsignedVInt(value, buffer, 0, accessor); + Assertions.assertThat(VIntCoding.getUnsignedVInt(buffer, accessor, 0)).isEqualTo(value); + } + + @Test + public void serdeSigned32() + { + qt().forAll(SourceDSL.integers().all()).checkAssert(l -> { + for (ValueAccessor accessor : Arrays.asList(ByteBufferAccessor.instance, ByteArrayAccessor.instance)) + serdeSigned32(l, accessor); + }); + } + + private static void serdeSigned32(int value, ValueAccessor accessor) + { + T buffer = accessor.allocate(Integer.BYTES + 1); + VIntCoding.writeVInt32(value, buffer, 0, accessor); + Assertions.assertThat(VIntCoding.getVInt32(buffer, accessor, 0)).isEqualTo(value); + } + + @Test + public void serdeUnsignedInt32() + { + qt().forAll(SourceDSL.integers().between(0, Integer.MAX_VALUE)).checkAssert(l -> { + for (ValueAccessor accessor : Arrays.asList(ByteBufferAccessor.instance, ByteArrayAccessor.instance)) + serdeUnsignedInt32(l, accessor); + }); + } + + private static void serdeUnsignedInt32(int value, ValueAccessor accessor) + { + T buffer = accessor.allocate(Integer.BYTES + 1); + VIntCoding.writeUnsignedVInt32(value, buffer, 0, accessor); + Assertions.assertThat(VIntCoding.getUnsignedVInt32(buffer, accessor, 0)).isEqualTo(value); + } + @Test public void testComputeSize() throws Exception { diff --git a/test/unit/org/quicktheories/impl/JavaRandom.java b/test/unit/org/quicktheories/impl/JavaRandom.java new file mode 100644 index 0000000000..b766033996 --- /dev/null +++ b/test/unit/org/quicktheories/impl/JavaRandom.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.quicktheories.impl; + +import java.util.Random; + +import org.quicktheories.core.DetatchedRandomnessSource; +import org.quicktheories.core.RandomnessSource; + +/** + * The {@link Constraint} class does not expose {@link Constraint#min()} or {@link Constraint#max()} outside the package, so + * the only way to build a {@link RandomnessSource} is to define it in the package "org.quicktheories.impl" + */ +public class JavaRandom implements RandomnessSource, DetatchedRandomnessSource +{ + private final Random random; + + public JavaRandom(long seed) + { + this.random = new Random(seed); + } + + public void setSeed(long seed) + { + this.random.setSeed(seed); + } + + public static JavaRandom wrap(RandomnessSource rnd) + { + if (rnd instanceof JavaRandom) + return (JavaRandom) rnd; + return new JavaRandom(rnd.next(Constraint.none().withNoShrinkPoint())); + } + + @Override + public long next(Constraint constraint) + { + long max = constraint.max(); + return nextLong(constraint.min(), max == Long.MAX_VALUE ? max : max + 1); + } + + private long nextLong(long minInclusive, long maxExclusive) + { + // pulled from accord.utils.RandomSource#nextLong(long, long)... + // long term it will be great to unify the two and drop QuickTheories classes all together + // this is diff behavior than ThreadLocalRandom, which returns nextLong + if (minInclusive >= maxExclusive) + throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive)); + + long result = random.nextLong(); + long delta = maxExclusive - minInclusive; + long mask = delta - 1; + if ((delta & mask) == 0L) // power of two + result = (result & mask) + minInclusive; + else if (delta > 0L) + { + // reject over-represented candidates + for (long u = result >>> 1; // ensure nonnegative + u + mask - (result = u % delta) < 0L; // rejection check + u = random.nextLong() >>> 1) // retry + ; + result += minInclusive; + } + else + { + // range not representable as long + while (result < minInclusive || result >= maxExclusive) + result = random.nextLong(); + } + return result; + } + + @Override + public DetatchedRandomnessSource detach() + { + return this; + } + + @Override + public void registerFailedAssumption() + { + + } + + @Override + public void commit() + { + // no-op + } +}