diff --git a/NEWS.txt b/NEWS.txt index 7afac5e105..a891eb3a9a 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -56,6 +56,8 @@ using the provided 'sstableupgrade' tool. New features ------------ + - Add support for the use of pure monotonic functions on the last attribute of the GROUP BY clause. + - Add floor functions that can be use to group by time range. - Support for native transport rate limiting via native_transport_rate_limiting_enabled and native_transport_max_requests_per_second in cassandra.yaml. - Support for pre hashing passwords on CQL DCL commands diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile index c60800ed06..8bedf19a78 100644 --- a/doc/cql3/CQL.textile +++ b/doc/cql3/CQL.textile @@ -1100,7 +1100,7 @@ bc(syntax).. | TOKEN '(' ( ',' )* ')' ::= '=' | '<' | '>' | '<=' | '>=' | CONTAINS | CONTAINS KEY - ::= (',' )* + ::= (,)* ( | ) ::= ( ',' )* ::= ( ASC | DESC )? ::= '(' (',' )* ')' @@ -1201,7 +1201,7 @@ h4(#selectGroupBy). @@ The @GROUP BY@ option allows to condense into a single row all selected rows that share the same values for a set of columns. -Using the @GROUP BY@ option, it is only possible to group rows at the partition key level or at a clustering column level. By consequence, the @GROUP BY@ option only accept as arguments primary key column names in the primary key order. If a primary key column is restricted by an equality restriction it is not required to be present in the @GROUP BY@ clause. +Using the @GROUP BY@ option, it is only possible to group rows at the partition key level or at a clustering column level. By consequence, the @GROUP BY@ option only accept as arguments primary key column names in the primary key order. If a primary key column is restricted by an equality restriction it is not required to be present in the @GROUP BY@ clause. The last argument can be a monotonic function on the primary key column. Aggregate functions will produce a separate value for each group. If no @GROUP BY@ clause is specified, aggregates functions will produce a single value for all the rows. @@ -2458,9 +2458,6 @@ h3. 3.4.3 h3. 3.4.2 * Support for selecting elements and slices of a collection ("CASSANDRA-7396":https://issues.apache.org/jira/browse/CASSANDRA-7396). - -h3. 3.4.2 - * "@INSERT/UPDATE options@":#updateOptions for tables having a default_time_to_live specifying a TTL of 0 will remove the TTL from the inserted or updated values * "@ALTER TABLE@":#alterTableStmt @ADD@ and @DROP@ now allow mutiple columns to be added/removed * New "@PER PARTITION LIMIT@":#selectLimit option (see "CASSANDRA-7017":https://issues.apache.org/jira/browse/CASSANDRA-7017). diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py index 0e50efbf23..7de95cf24c 100644 --- a/pylib/cqlshlib/cql3handling.py +++ b/pylib/cqlshlib/cql3handling.py @@ -742,7 +742,13 @@ syntax_rules += r''' ::= [ordercol]= ( "ASC" | "DESC" )? ; ::= [groupcol]= + | ; + ::= "(" ( ( "," )* )? ")" + ; + ::= [groupcol]= + | + ; ''' diff --git a/src/antlr/Parser.g b/src/antlr/Parser.g index dbb2682617..d061ee4df3 100644 --- a/src/antlr/Parser.g +++ b/src/antlr/Parser.g @@ -267,7 +267,7 @@ selectStatement returns [SelectStatement.RawStatement expr] Term.Raw limit = null; Term.Raw perPartitionLimit = null; Map orderings = new LinkedHashMap<>(); - List groups = new ArrayList<>(); + List groups = new ArrayList<>(); boolean allowFiltering = false; boolean isJson = false; } @@ -463,8 +463,8 @@ orderByClause[Map orderings] : c=cident (K_ASC | K_DESC { reversed = true; })? { orderings.put(c, reversed); } ; -groupByClause[List groups] - : c=cident { groups.add(c); } +groupByClause[List groups] + : s=unaliasedSelector { groups.add(s); } ; /** diff --git a/src/java/org/apache/cassandra/cql3/Duration.java b/src/java/org/apache/cassandra/cql3/Duration.java index 520d19531f..1d6b9f7b98 100644 --- a/src/java/org/apache/cassandra/cql3/Duration.java +++ b/src/java/org/apache/cassandra/cql3/Duration.java @@ -27,10 +27,13 @@ import com.google.common.base.Objects; import org.apache.cassandra.serializers.MarshalException; +import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse; import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue; import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; import static org.apache.commons.lang3.time.DateUtils.MILLIS_PER_DAY; +import io.netty.util.concurrent.FastThreadLocal; + /** * Represents a duration. A durations store separately months, days, and seconds due to the fact that * the number of days in a month varies, and a day can have 23 or 25 hours if a daylight saving is involved. @@ -45,6 +48,17 @@ public final class Duration public static final int DAYS_PER_WEEK = 7; public static final int MONTHS_PER_YEAR = 12; + // For some operations, like floor, a Calendar is needed if months or years are involved. Unfortunatly, creating a + // Calendar is a costly operation so instead of creating one with every call we reuse them. + private static final FastThreadLocal CALENDAR_PROVIDER = new FastThreadLocal() + { + @Override + public Calendar initialValue() + { + return Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.US); + } + }; + /** * The Regexp used to parse the duration provided as String. */ @@ -339,7 +353,7 @@ public final class Duration { StringBuilder builder = new StringBuilder(); - if (months < 0 || days < 0 || nanoseconds < 0) + if (isNegative()) builder.append('-'); long remainder = append(builder, Math.abs(months), MONTHS_PER_YEAR, "y"); @@ -395,6 +409,124 @@ public final class Duration return dividend % divisor; } + /** + * Rounds a timestamp down to the closest multiple of a duration. + * + * @param timeInMillis the time to round in millisecond + * @param duration the duration + * @param startingTimeInMillis the time offset in milliseconds + * @return the timestamp rounded down to the closest multiple of the duration + */ + public static long floorTimestamp(long timeInMillis, Duration duration, long startingTimeInMillis) + { + checkFalse(startingTimeInMillis > timeInMillis, "The floor function starting time is greater than the provided time"); + checkFalse(duration.isNegative(), "Negative durations are not supported by the floor function"); + + // If the duration does not contain any months we can ignore daylight saving, + // as time zones are not supported, and simply look at the milliseconds + if (duration.months == 0) + { + long durationInMillis = getDurationMilliseconds(duration); + + // If the duration is smaller than millisecond + if (durationInMillis == 0) + return timeInMillis; + + long delta = (timeInMillis - startingTimeInMillis) % durationInMillis; + return timeInMillis - delta; + } + + /* + * Otherwise, we resort to Calendar for the computation. + * What we're trying to compute is the largest integer 'multiplier' value such that + * startingTimeMillis + (multiplier * duration) <= timeInMillis + * at which point we want to return 'startingTimeMillis + (multiplier * duration)'. + * + * One option would be to add 'duration' to 'statingTimeMillis' in a loop until we + * cross 'timeInMillis' and return how many iterator we did. But this might be slow if there is very many + * steps. + * + * So instead we first estimate 'multiplier' using the number of months between 'startingTimeMillis' + * and 'timeInMillis' ('durationInMonths' below) and the duration months. As the real computation + * should also take the 'days' and 'nanoseconds' parts of the duration, this multiplier may overshoot, + * so we detect it and work back from that, decreasing the multiplier until we find the proper one. + */ + + Calendar calendar = CALENDAR_PROVIDER.get(); + + calendar.setTimeInMillis(timeInMillis); + int year = calendar.get(Calendar.YEAR); + int month = calendar.get(Calendar.MONTH); + + calendar.setTimeInMillis(startingTimeInMillis); + int startingYear = calendar.get(Calendar.YEAR); + int startingMonth = calendar.get(Calendar.MONTH); + + int durationInMonths = (year - startingYear) * MONTHS_PER_YEAR + (month - startingMonth); + int multiplier = durationInMonths / duration.months; + + calendar.add(Calendar.MONTH, multiplier * duration.months); + + // If the duration was only containing months, we are done. + if (duration.days == 0 && duration.nanoseconds == 0) + return calendar.getTimeInMillis(); + + long durationInMillis = getDurationMilliseconds(duration); + long floor = calendar.getTimeInMillis() + (multiplier * durationInMillis); + + // Once the milliseconds have been added we might have gone too far. If it is the case we will reduce the + // multiplier until the floor value is smaller than time in millis. + while (floor > timeInMillis) + { + multiplier--; + calendar.add(Calendar.MONTH, -duration.months); + floor = calendar.getTimeInMillis() + (multiplier * durationInMillis); + } + + return Math.max(startingTimeInMillis, floor); + } + + /** + * Returns the milliseconds part of the duration ignoring the month part + * + * @param duration the duration + * @return the milliseconds corresponding to the duration days and nanoseconds + */ + private static long getDurationMilliseconds(Duration duration) + { + // We can ignore daylight saving as time zones are not supported + return (duration.days * MILLIS_PER_DAY) + (duration.nanoseconds / NANOS_PER_MILLI); + } + + /** + * Rounds a time down to the closest multiple of a duration. + * + * @param timeInNanos the time of day in nanoseconds + * @param duration the duration + * @return the time rounded down to the closest multiple of the duration + */ + public static long floorTime(long timeInNanos, Duration duration) + { + checkFalse(duration.isNegative(), "Negative durations are not supported by the floor function"); + checkFalse(duration.getMonths() != 0 || duration.getDays() != 0 || duration.getNanoseconds() > (NANOS_PER_HOUR * 24), + "For time values, the floor can only be computed for durations smaller that a day"); + + if (duration.nanoseconds == 0) + return timeInNanos; + + long delta = timeInNanos % duration.nanoseconds; + return timeInNanos - delta; + } + + /** + * Checks if the duration is negative. + * @return {@code true} if the duration is negative, {@code false} otherwise + */ + public boolean isNegative() + { + return nanoseconds < 0 || days < 0 || months < 0; + } + private static class Builder { /** diff --git a/src/java/org/apache/cassandra/cql3/functions/Function.java b/src/java/org/apache/cassandra/cql3/functions/Function.java index e13e906c2d..0a8d331e2c 100644 --- a/src/java/org/apache/cassandra/cql3/functions/Function.java +++ b/src/java/org/apache/cassandra/cql3/functions/Function.java @@ -29,6 +29,12 @@ import org.github.jamm.Unmetered; @Unmetered public interface Function extends AssignmentTestable { + /** + * A marker buffer used to represent function parameters that cannot be resolved at some stage of CQL processing. + * This is used for partial function application in particular. + */ + public static final ByteBuffer UNRESOLVED = ByteBuffer.allocate(0); + public FunctionName name(); public List> argTypes(); public AbstractType returnType(); @@ -36,14 +42,21 @@ public interface Function extends AssignmentTestable /** * Checks whether the function is a native/hard coded one or not. * - * @return true if the function is a native/hard coded one, false otherwise. + * @return {@code true} if the function is a native/hard coded one, {@code false} otherwise. */ public boolean isNative(); + /** + * Checks whether the function is a pure function (as in doesn't depend on, nor produces side effects) or not. + * + * @return {@code true} if the function is a pure function, {@code false} otherwise. + */ + public boolean isPure(); + /** * Checks whether the function is an aggregate function or not. * - * @return true if the function is an aggregate function, false otherwise. + * @return {@code true} if the function is an aggregate function, {@code false} otherwise. */ public boolean isAggregate(); diff --git a/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java b/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java index df66ea07e1..cafeca1af6 100644 --- a/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java +++ b/src/java/org/apache/cassandra/cql3/functions/NativeFunction.java @@ -31,8 +31,16 @@ public abstract class NativeFunction extends AbstractFunction super(FunctionName.nativeFunction(name), Arrays.asList(argTypes), returnType); } + @Override public boolean isNative() { return true; } + + @Override + public boolean isPure() + { + // Most of our functions are pure, the other ones should override this + return true; + } } diff --git a/src/java/org/apache/cassandra/cql3/functions/NativeScalarFunction.java b/src/java/org/apache/cassandra/cql3/functions/NativeScalarFunction.java index 3ae0607f7a..e492f758f1 100644 --- a/src/java/org/apache/cassandra/cql3/functions/NativeScalarFunction.java +++ b/src/java/org/apache/cassandra/cql3/functions/NativeScalarFunction.java @@ -17,6 +17,9 @@ */ package org.apache.cassandra.cql3.functions; +import java.nio.ByteBuffer; +import java.util.List; + import org.apache.cassandra.db.marshal.AbstractType; /** @@ -38,4 +41,17 @@ public abstract class NativeScalarFunction extends NativeFunction implements Sca { return false; } + + /** + * Checks if a partial application of the function is monotonic. + * + *

A function is monotonic if it is either entirely nonincreasing or nondecreasing.

+ * + * @param partialParameters the input parameters used to create the partial application of the function + * @return {@code true} if the partial application of the function is monotonic {@code false} otherwise. + */ + protected boolean isPartialApplicationMonotonic(List partialParameters) + { + return isMonotonic(); + } } diff --git a/src/java/org/apache/cassandra/cql3/functions/PartialScalarFunction.java b/src/java/org/apache/cassandra/cql3/functions/PartialScalarFunction.java new file mode 100644 index 0000000000..6970768f58 --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/functions/PartialScalarFunction.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.cql3.functions; + +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.cassandra.transport.ProtocolVersion; + +/** + * A partial application of a function. + * + * @see ScalarFunction#partialApplication(ProtocolVersion, List) + */ +public interface PartialScalarFunction extends ScalarFunction +{ + /** + * Returns the original function. + * + * @return the original function + */ + public Function getFunction(); + + /** + * Returns the list of input parameters for the function where some parameters can be {@link #UNRESOLVED}. + * + * @return the list of input parameters for the function + */ + public List getPartialParameters(); +} diff --git a/src/java/org/apache/cassandra/cql3/functions/PartiallyAppliedScalarFunction.java b/src/java/org/apache/cassandra/cql3/functions/PartiallyAppliedScalarFunction.java new file mode 100644 index 0000000000..301160b29d --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/functions/PartiallyAppliedScalarFunction.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.cql3.functions; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.cassandra.cql3.CqlBuilder; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.transport.ProtocolVersion; + +/** + * An internal function used to hold the partial application of another function to only some of its parameters. + * + * @see ScalarFunction#partialApplication(ProtocolVersion, List) + */ +final class PartiallyAppliedScalarFunction extends NativeScalarFunction implements PartialScalarFunction +{ + private final ScalarFunction function; + private final List partialParameters; + + PartiallyAppliedScalarFunction(ScalarFunction function, List partialParameters, int unresolvedCount) + { + // Note that we never register those function, there are just used internally, so the name doesn't matter much + super("__partial_application__", function.returnType(), computeArgTypes(function, partialParameters, unresolvedCount)); + this.function = function; + this.partialParameters = partialParameters; + } + + @Override + public boolean isMonotonic() + { + return function.isNative() ? ((NativeScalarFunction) function).isPartialApplicationMonotonic(partialParameters) + : function.isMonotonic(); + } + + @Override + public boolean isPure() + { + return function.isPure(); + } + + @Override + public Function getFunction() + { + return function; + } + + @Override + public List getPartialParameters() + { + return partialParameters; + } + + private static AbstractType[] computeArgTypes(ScalarFunction function, List partialParameters, int unresolvedCount) + { + AbstractType[] argTypes = new AbstractType[unresolvedCount]; + int arg = 0; + for (int i = 0; i < partialParameters.size(); i++) + { + if (partialParameters.get(i) == UNRESOLVED) + argTypes[arg++] = function.argTypes().get(i); + } + return argTypes; + } + + public ByteBuffer execute(ProtocolVersion protocolVersion, List parameters) throws InvalidRequestException + { + List fullParameters = new ArrayList<>(partialParameters); + int arg = 0; + for (int i = 0; i < fullParameters.size(); i++) + { + if (fullParameters.get(i) == UNRESOLVED) + fullParameters.set(i, parameters.get(arg++)); + } + return function.execute(protocolVersion, fullParameters); + } + + @Override + public String toString() + { + CqlBuilder b = new CqlBuilder().append(function.name()).append(" : ("); + + List> types = function.argTypes(); + for (int i = 0, m = types.size(); i < m; i++) + { + if (i > 0) + b.append(", "); + b.append(toCqlString(types.get(i))); + if (partialParameters.get(i) != Function.UNRESOLVED) + b.append("(constant)"); + } + b.append(") -> ").append(returnType); + return b.toString(); + } +} diff --git a/src/java/org/apache/cassandra/cql3/functions/PreComputedScalarFunction.java b/src/java/org/apache/cassandra/cql3/functions/PreComputedScalarFunction.java new file mode 100644 index 0000000000..9670132386 --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/functions/PreComputedScalarFunction.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.cql3.functions; + +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.transport.ProtocolVersion; + +/** + * Function used internally to hold the pre-computed result of another function. + *

+ * See {@link ScalarFunction#partialApplication(ProtocolVersion, List)} for why this is used. + *

+ * Note : the function is cautious in keeping the protocol version used for the pre-computed value and to + * fallback to recomputation if the version we get when {@link #execute} is called. I don't think it's truly necessary + * though as I don't think we actually depend on the protocol version for values anymore (it's remnant of previous + * transitions). It's not a lot of code to be on safe side though until this is cleaned (assuming we do clean it). + */ +class PreComputedScalarFunction extends NativeScalarFunction implements PartialScalarFunction +{ + private final ByteBuffer value; + private final ProtocolVersion valueVersion; + + private final ScalarFunction function; + private final List parameters; + + PreComputedScalarFunction(AbstractType returnType, + ByteBuffer value, + ProtocolVersion valueVersion, + ScalarFunction function, + List parameters) + { + // Note that we never register those function, there are just used internally, so the name doesn't matter much + super("__constant__", returnType); + this.value = value; + this.valueVersion = valueVersion; + this.function = function; + this.parameters = parameters; + } + + @Override + public Function getFunction() + { + return function; + } + + @Override + public List getPartialParameters() + { + return parameters; + } + + public ByteBuffer execute(ProtocolVersion protocolVersion, List nothing) throws InvalidRequestException + { + if (protocolVersion == valueVersion) + return value; + + return function.execute(protocolVersion, parameters); + } + + public ScalarFunction partialApplication(ProtocolVersion protocolVersion, List nothing) throws InvalidRequestException + { + return this; + } +} diff --git a/src/java/org/apache/cassandra/cql3/functions/ScalarFunction.java b/src/java/org/apache/cassandra/cql3/functions/ScalarFunction.java index 1f98372f01..fc18aae6a2 100644 --- a/src/java/org/apache/cassandra/cql3/functions/ScalarFunction.java +++ b/src/java/org/apache/cassandra/cql3/functions/ScalarFunction.java @@ -24,12 +24,24 @@ import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.transport.ProtocolVersion; /** - * Determines a single output value based on a single input value. + * Determines a single output value based on any number of input values. */ public interface ScalarFunction extends Function { public boolean isCalledOnNullInput(); + /** + * Checks if the function is monotonic. + * + *

A function is monotonic if it is either entirely nonincreasing or nondecreasing given an ordered set of inputs.

+ * + * @return {@code true} if the function is monotonic {@code false} otherwise. + */ + public default boolean isMonotonic() + { + return false; + } + /** * Applies this function to the specified parameter. * @@ -39,4 +51,49 @@ public interface ScalarFunction extends Function * @throws InvalidRequestException if this function cannot not be applied to the parameter */ public ByteBuffer execute(ProtocolVersion protocolVersion, List parameters) throws InvalidRequestException; + + /** + * Does a partial application of the function. That is, given only some of the parameters of the function provided, + * return a new function that only expect the parameters not provided. + *

+ * To take an example, if you consider the function + * 

+     *     text foo(int a, text b, text c, int d)
+     *
+ * then {@code foo.partialApplication([3, , 'bar', ])} will return a function {@code bar} of signature: + * 
+     *     text bar(text b, int d)
+     *
+ * and such that for any value of {@code b} and {@code d}, {@code bar(b, d) == foo(3, b, 'bar', d)}. + * + * @param protocolVersion protocol version used for parameters + * @param partialParameters a list of input parameters for the function where some parameters can be {@link #UNRESOLVED}. + * The input must be of size {@code this.argsType().size()}. For convenience, it is + * allowed both to pass a list with all parameters being {@link #UNRESOLVED} (the function is + * then returned directly) and with none of them unresolved (in which case, if the function is pure, + * it is computed and a dummy no-arg function returning the result is returned). + * @return a function corresponding to the partial application of this function to the parameters of + * {@code partialParameters} that are not {@link #UNRESOLVED}. + */ + public default ScalarFunction partialApplication(ProtocolVersion protocolVersion, List partialParameters) + { + int unresolvedCount = 0; + for (ByteBuffer parameter : partialParameters) + { + if (parameter == UNRESOLVED) + ++unresolvedCount; + } + + if (unresolvedCount == argTypes().size()) + return this; + + if (isPure() && unresolvedCount == 0) + return new PreComputedScalarFunction(returnType(), + execute(protocolVersion, partialParameters), + protocolVersion, + this, + partialParameters); + + return new PartiallyAppliedScalarFunction(this, partialParameters, unresolvedCount); + } } diff --git a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java index 1bf03ff3fc..27592102b7 100644 --- a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java +++ b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java @@ -25,13 +25,17 @@ import com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.cql3.Duration; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.UUIDGen; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; + public abstract class TimeFcts { public static Logger logger = LoggerFactory.getLogger(TimeFcts.class); @@ -53,7 +57,14 @@ public abstract class TimeFcts toUnixTimestamp(TimestampType.instance), toDate(TimestampType.instance), toUnixTimestamp(SimpleDateType.instance), - toTimestamp(SimpleDateType.instance)); + toTimestamp(SimpleDateType.instance), + FloorTimestampFunction.newInstance(), + FloorTimestampFunction.newInstanceWithStartTimeArgument(), + FloorTimeUuidFunction.newInstance(), + FloorTimeUuidFunction.newInstanceWithStartTimeArgument(), + FloorDateFunction.newInstance(), + FloorDateFunction.newInstanceWithStartTimeArgument(), + floorTime); } public static final Function now(final String name, final TemporalType type) @@ -65,6 +76,12 @@ public abstract class TimeFcts { return type.now(); } + + @Override + public boolean isPure() + { + return false; // as it returns non-identical results for identical arguments + } }; }; @@ -161,6 +178,12 @@ public abstract class TimeFcts long millis = type.toTimeInMillis(bb); return SimpleDateType.instance.fromTimeInMillis(millis); } + + @Override + public boolean isMonotonic() + { + return true; + } }; } @@ -182,6 +205,12 @@ public abstract class TimeFcts long millis = type.toTimeInMillis(bb); return TimestampType.instance.fromTimeInMillis(millis); } + + @Override + public boolean isMonotonic() + { + return true; + } }; } @@ -202,7 +231,282 @@ public abstract class TimeFcts return ByteBufferUtil.bytes(type.toTimeInMillis(bb)); } + + @Override + public boolean isMonotonic() + { + return true; + } }; } -} + + /** + * Function that rounds a timestamp down to the closest multiple of a duration. + */ + private static abstract class FloorFunction extends NativeScalarFunction + { + private static final Long ZERO = Long.valueOf(0); + + protected FloorFunction(AbstractType returnType, + AbstractType... argsType) + { + super("floor", returnType, argsType); + // The function can accept either 2 parameters (time and duration) or 3 parameters (time, duration and startTime)r + assert argsType.length == 2 || argsType.length == 3; + } + + @Override + protected boolean isPartialApplicationMonotonic(List partialParameters) + { + return partialParameters.get(0) == UNRESOLVED + && partialParameters.get(1) != UNRESOLVED + && (partialParameters.size() == 2 || partialParameters.get(2) != UNRESOLVED); + } + + public final ByteBuffer execute(ProtocolVersion protocolVersion, List parameters) + { + ByteBuffer timeBuffer = parameters.get(0); + ByteBuffer durationBuffer = parameters.get(1); + Long startingTime = getStartingTime(parameters); + + if (timeBuffer == null || durationBuffer == null || startingTime == null) + return null; + + Long time = toTimeInMillis(timeBuffer); + Duration duration = DurationType.instance.compose(durationBuffer); + + if (time == null || duration == null) + return null; + + + validateDuration(duration); + + + long floor = Duration.floorTimestamp(time, duration, startingTime); + + return fromTimeInMillis(floor); + } + + /** + * Returns the time to use as the starting time. + * + * @param parameters the function parameters + * @return the time to use as the starting time + */ + private Long getStartingTime(List parameters) + { + if (parameters.size() == 3) + { + ByteBuffer startingTimeBuffer = parameters.get(2); + + if (startingTimeBuffer == null) + return null; + + return toStartingTimeInMillis(startingTimeBuffer); + } + + return ZERO; + } + + /** + * Validates that the duration has the correct precision. + * @param duration the duration to validate. + */ + protected void validateDuration(Duration duration) + { + if (!duration.hasMillisecondPrecision()) + throw invalidRequest("The floor cannot be computed for the %s duration as precision is below 1 millisecond", duration); + } + + /** + * Serializes the specified time. + * + * @param timeInMillis the time in milliseconds + * @return the serialized time + */ + protected abstract ByteBuffer fromTimeInMillis(long timeInMillis); + + /** + * Deserializes the specified input time. + * + * @param bytes the serialized time + * @return the time in milliseconds + */ + protected abstract Long toTimeInMillis(ByteBuffer bytes); + + /** + * Deserializes the specified starting time. + * + * @param bytes the serialized starting time + * @return the starting time in milliseconds + */ + protected abstract Long toStartingTimeInMillis(ByteBuffer bytes); + } + + /** + * Function that rounds a timestamp down to the closest multiple of a duration. + */ + public static final class FloorTimestampFunction extends FloorFunction + { + public static FloorTimestampFunction newInstance() + { + return new FloorTimestampFunction(TimestampType.instance, + TimestampType.instance, + DurationType.instance); + } + + public static FloorTimestampFunction newInstanceWithStartTimeArgument() + { + return new FloorTimestampFunction(TimestampType.instance, + TimestampType.instance, + DurationType.instance, + TimestampType.instance); + } + + private FloorTimestampFunction(AbstractType returnType, + AbstractType... argTypes) + { + super(returnType, argTypes); + } + + protected ByteBuffer fromTimeInMillis(long timeInMillis) + { + return TimestampType.instance.fromTimeInMillis(timeInMillis); + } + + protected Long toStartingTimeInMillis(ByteBuffer bytes) + { + return TimestampType.instance.toTimeInMillis(bytes); + } + + protected Long toTimeInMillis(ByteBuffer bytes) + { + return TimestampType.instance.toTimeInMillis(bytes); + } + } + + /** + * Function that rounds a timeUUID down to the closest multiple of a duration. + */ + public static final class FloorTimeUuidFunction extends FloorFunction + { + public static FloorTimeUuidFunction newInstance() + { + return new FloorTimeUuidFunction(TimestampType.instance, + TimeUUIDType.instance, + DurationType.instance); + } + + public static FloorTimeUuidFunction newInstanceWithStartTimeArgument() + { + return new FloorTimeUuidFunction(TimestampType.instance, + TimeUUIDType.instance, + DurationType.instance, + TimestampType.instance); + } + + private FloorTimeUuidFunction(AbstractType returnType, + AbstractType... argTypes) + { + super(returnType, argTypes); + } + + protected ByteBuffer fromTimeInMillis(long timeInMillis) + { + return TimestampType.instance.fromTimeInMillis(timeInMillis); + } + + protected Long toStartingTimeInMillis(ByteBuffer bytes) + { + return TimestampType.instance.toTimeInMillis(bytes); + } + + protected Long toTimeInMillis(ByteBuffer bytes) + { + return UUIDGen.getAdjustedTimestamp(UUIDGen.getUUID(bytes)); + } + } + + /** + * Function that rounds a date down to the closest multiple of a duration. + */ + public static final class FloorDateFunction extends FloorFunction + { + public static FloorDateFunction newInstance() + { + return new FloorDateFunction(SimpleDateType.instance, + SimpleDateType.instance, + DurationType.instance); + } + + public static FloorDateFunction newInstanceWithStartTimeArgument() + { + return new FloorDateFunction(SimpleDateType.instance, + SimpleDateType.instance, + DurationType.instance, + SimpleDateType.instance); + } + + private FloorDateFunction(AbstractType returnType, + AbstractType... argTypes) + { + super(returnType, argTypes); + } + + protected ByteBuffer fromTimeInMillis(long timeInMillis) + { + return SimpleDateType.instance.fromTimeInMillis(timeInMillis); + } + + protected Long toStartingTimeInMillis(ByteBuffer bytes) + { + return SimpleDateType.instance.toTimeInMillis(bytes); + } + + protected Long toTimeInMillis(ByteBuffer bytes) + { + return SimpleDateType.instance.toTimeInMillis(bytes); + } + + @Override + protected void validateDuration(Duration duration) + { + // Checks that the duration has no data below days. + if (duration.getNanoseconds() != 0) + throw invalidRequest("The floor on %s values cannot be computed for the %s duration as precision is below 1 day", + SimpleDateType.instance.asCQL3Type(), duration); + } + } + + /** + * Function that rounds a time down to the closest multiple of a duration. + */ + public static final NativeScalarFunction floorTime = new NativeScalarFunction("floor", TimeType.instance, TimeType.instance, DurationType.instance) + { + @Override + protected boolean isPartialApplicationMonotonic(List partialParameters) + { + return partialParameters.get(0) == UNRESOLVED && partialParameters.get(1) != UNRESOLVED; + } + + public ByteBuffer execute(ProtocolVersion protocolVersion, List parameters) + { + ByteBuffer timeBuffer = parameters.get(0); + ByteBuffer durationBuffer = parameters.get(1); + + if (timeBuffer == null || durationBuffer == null) + return null; + + Long time = TimeType.instance.compose(timeBuffer); + Duration duration = DurationType.instance.compose(durationBuffer); + + if (time == null || duration == null) + return null; + + long floor = Duration.floorTime(time, duration); + + return TimeType.instance.decompose(floor); + } + }; + } diff --git a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java index 92185d95e0..ad1609ae89 100644 --- a/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java +++ b/src/java/org/apache/cassandra/cql3/functions/UDAggregate.java @@ -100,6 +100,12 @@ public class UDAggregate extends AbstractFunction implements AggregateFunction, .orElseThrow(() -> new ConfigurationException(String.format("Unable to find function %s referenced by UDA %s", name, udaName))); } + public boolean isPure() + { + // Right now, we have no way to check if an UDA is pure. Due to that we consider them as non pure to avoid any risk. + return false; + } + public boolean hasReferenceTo(Function function) { return stateFunction == function || finalFunction == function; diff --git a/src/java/org/apache/cassandra/cql3/functions/UDFunction.java b/src/java/org/apache/cassandra/cql3/functions/UDFunction.java index ff281f12f3..5ef065b032 100644 --- a/src/java/org/apache/cassandra/cql3/functions/UDFunction.java +++ b/src/java/org/apache/cassandra/cql3/functions/UDFunction.java @@ -368,6 +368,12 @@ public abstract class UDFunction extends AbstractFunction implements ScalarFunct return builder.toString(); } + public boolean isPure() + { + // Right now, we have no way to check if an UDF is pure. Due to that we consider them as non pure to avoid any risk. + return false; + } + public final ByteBuffer execute(ProtocolVersion protocolVersion, List parameters) { assertUdfsEnabled(language); diff --git a/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java index d420857d7d..bf2564edd2 100644 --- a/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/AbstractFunctionSelector.java @@ -17,24 +17,93 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; +import com.google.common.base.Objects; import com.google.common.collect.Iterables; import org.apache.commons.lang3.text.StrBuilder; import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.cql3.functions.FunctionName; +import org.apache.cassandra.cql3.functions.PartialScalarFunction; +import org.apache.cassandra.cql3.functions.ScalarFunction; import org.apache.cassandra.cql3.statements.RequestValidations; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +import static java.util.stream.Collectors.joining; abstract class AbstractFunctionSelector extends Selector { + protected static abstract class AbstractFunctionSelectorDeserializer extends SelectorDeserializer + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + FunctionName name = new FunctionName(in.readUTF(), in.readUTF()); + + int numberOfArguments = (int) in.readUnsignedVInt(); + List> argTypes = new ArrayList<>(numberOfArguments); + for (int i = 0; i < numberOfArguments; i++) + { + argTypes.add(readType(metadata, in)); + } + + Optional optional = Schema.instance.findFunction(name, argTypes); + + if (!optional.isPresent()) + throw new IOException(String.format("Unknown serialized function %s(%s)", + name, + argTypes.stream() + .map(p -> p.asCQL3Type().toString()) + .collect(joining(", ")))); + + Function function = optional.get(); + + boolean isPartial = in.readBoolean(); + if (isPartial) + { + int bitset = (int) in.readUnsignedVInt(); + List partialParameters = new ArrayList<>(numberOfArguments); + for (int i = 0; i < numberOfArguments; i++) + { + ByteBuffer parameter = ((bitset & 1) == 1) ? ByteBufferUtil.readWithVIntLength(in) + : Function.UNRESOLVED; + partialParameters.add(parameter); + bitset >>= 1; + } + + function = ((ScalarFunction) function).partialApplication(ProtocolVersion.CURRENT, partialParameters); + } + + int numberOfRemainingArguments = (int) in.readUnsignedVInt(); + List argSelectors = new ArrayList<>(numberOfRemainingArguments); + for (int i = 0; i < numberOfRemainingArguments; i++) + { + argSelectors.add(Selector.serializer.deserialize(in, version, metadata)); + } + + return newFunctionSelector(function, argSelectors); + } + + protected abstract Selector newFunctionSelector(Function function, List argSelectors); + }; + protected final T fun; /** @@ -88,7 +157,45 @@ abstract class AbstractFunctionSelector extends Selector public Selector newInstance(QueryOptions options) throws InvalidRequestException { return fun.isAggregate() ? new AggregateFunctionSelector(fun, factories.newInstances(options)) - : new ScalarFunctionSelector(fun, factories.newInstances(options)); + : createScalarSelector(options, (ScalarFunction) fun, factories.newInstances(options)); + } + + private Selector createScalarSelector(QueryOptions options, ScalarFunction function, List argSelectors) + { + ProtocolVersion version = options.getProtocolVersion(); + int terminalCount = 0; + List terminalArgs = new ArrayList<>(argSelectors.size()); + for (Selector selector : argSelectors) + { + if (selector.isTerminal()) + { + ++terminalCount; + ByteBuffer output = selector.getOutput(version); + RequestValidations.checkBindValueSet(output, "Invalid unset value for argument in call to function %s", fun.name().name); + terminalArgs.add(output); + } + else + { + terminalArgs.add(Function.UNRESOLVED); + } + } + + if (terminalCount == 0) + return new ScalarFunctionSelector(fun, argSelectors); + + // All terminal, reduce to a simple value if the function is pure + if (terminalCount == argSelectors.size() && function.isPure()) + return new TermSelector(function.execute(version, terminalArgs), function.returnType()); + + // We have some terminal arguments but not all, do a partial application + ScalarFunction partialFunction = function.partialApplication(version, terminalArgs); + List remainingSelectors = new ArrayList<>(argSelectors.size() - terminalCount); + for (Selector selector : argSelectors) + { + if (!selector.isTerminal()) + remainingSelectors.add(selector); + } + return new ScalarFunctionSelector(partialFunction, remainingSelectors); } public boolean isWritetimeSelectorFactory() @@ -121,8 +228,9 @@ abstract class AbstractFunctionSelector extends Selector }; } - protected AbstractFunctionSelector(T fun, List argSelectors) + protected AbstractFunctionSelector(Kind kind, T fun, List argSelectors) { + super(kind); this.fun = fun; this.argSelectors = argSelectors; this.args = Arrays.asList(new ByteBuffer[argSelectors.size()]); @@ -153,6 +261,28 @@ abstract class AbstractFunctionSelector extends Selector return fun.returnType(); } + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof AbstractFunctionSelector)) + return false; + + AbstractFunctionSelector s = (AbstractFunctionSelector) o; + + return Objects.equal(fun.name(), s.fun.name()) + && Objects.equal(fun.argTypes(), s.fun.argTypes()) + && Objects.equal(argSelectors, s.argSelectors); + } + + @Override + public int hashCode() + { + return Objects.hashCode(fun.name(), fun.argTypes(), argSelectors); + } + @Override public String toString() { @@ -162,4 +292,97 @@ abstract class AbstractFunctionSelector extends Selector .append(")") .toString(); } + + @Override + protected int serializedSize(int version) + { + boolean isPartial = fun instanceof PartialScalarFunction; + Function function = isPartial ? ((PartialScalarFunction) fun).getFunction() : fun; + + FunctionName name = function.name(); + int size = TypeSizes.sizeof(name.keyspace) + TypeSizes.sizeof(name.name); + + List> argTypes = function.argTypes(); + size += TypeSizes.sizeofUnsignedVInt(argTypes.size()); + for (int i = 0, m = argTypes.size(); i < m; i++) + { + size += sizeOf(argTypes.get(i)); + } + + size += TypeSizes.sizeof(isPartial); + + if (isPartial) + { + List partialParameters = ((PartialScalarFunction) fun).getPartialParameters(); + + // We use a bitset to track the position of the unresolved arguments + size += TypeSizes.sizeofUnsignedVInt(computeBitSet(partialParameters)); + + for (int i = 0, m = partialParameters.size(); i < m; i++) + { + ByteBuffer buffer = partialParameters.get(i); + if (buffer != Function.UNRESOLVED) + size += ByteBufferUtil.serializedSizeWithVIntLength(buffer); + } + } + + int numberOfRemainingArguments = argSelectors.size(); + size += TypeSizes.sizeofUnsignedVInt(numberOfRemainingArguments); + for (int i = 0; i < numberOfRemainingArguments; i++) + size += serializer.serializedSize(argSelectors.get(i), version); + + return size; + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + boolean isPartial = fun instanceof PartialScalarFunction; + Function function = isPartial ? ((PartialScalarFunction) fun).getFunction() : fun; + + FunctionName name = function.name(); + out.writeUTF(name.keyspace); + out.writeUTF(name.name); + + List> argTypes = function.argTypes(); + int numberOfArguments = argTypes.size(); + out.writeUnsignedVInt(numberOfArguments); + + for (int i = 0; i < numberOfArguments; i++) + writeType(out, argTypes.get(i)); + + out.writeBoolean(isPartial); + + if (isPartial) + { + List partialParameters = ((PartialScalarFunction) fun).getPartialParameters(); + + // We use a bitset to track the position of the unresolved arguments + out.writeUnsignedVInt(computeBitSet(partialParameters)); + + for (int i = 0, m = partialParameters.size(); i < m; i++) + { + ByteBuffer buffer = partialParameters.get(i); + if (buffer != Function.UNRESOLVED) + ByteBufferUtil.writeWithVIntLength(buffer, out); + } + } + + int numberOfRemainingArguments = argSelectors.size(); + out.writeUnsignedVInt(numberOfRemainingArguments); + for (int i = 0; i < numberOfRemainingArguments; i++) + serializer.serialize(argSelectors.get(i), out, version); + } + + private int computeBitSet(List partialParameters) + { + assert partialParameters.size() <= 32 : "cannot serialize partial function with more than 32 parameters"; + int bitset = 0; + for (int i = 0, m = partialParameters.size(); i < m; i++) + { + if (partialParameters.get(i) != Function.UNRESOLVED) + bitset |= 1 << i; + } + return bitset; + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java index a9df220aa0..8c4f74567b 100644 --- a/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/AggregateFunctionSelector.java @@ -27,6 +27,15 @@ import org.apache.cassandra.transport.ProtocolVersion; final class AggregateFunctionSelector extends AbstractFunctionSelector { + protected static final SelectorDeserializer deserializer = new AbstractFunctionSelectorDeserializer() + { + @Override + protected Selector newFunctionSelector(Function function, List argSelectors) + { + return new AggregateFunctionSelector(function, argSelectors); + } + }; + private final AggregateFunction.Aggregate aggregate; public boolean isAggregate() @@ -34,13 +43,13 @@ final class AggregateFunctionSelector extends AbstractFunctionSelector argSelectors) throws InvalidRequestException { - super((AggregateFunction) fun, argSelectors); + super(Kind.AGGREGATE_FUNCTION_SELECTOR, (AggregateFunction) fun, argSelectors); this.aggregate = this.fun.newAggregate(); } diff --git a/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java b/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java index 9427c51d59..851d78574c 100644 --- a/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/ElementsSelector.java @@ -17,17 +17,24 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; +import com.google.common.base.Objects; + import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.Term; import org.apache.cassandra.cql3.selection.SimpleSelector.SimpleSelectorFactory; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.db.rows.CellPath; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; @@ -38,8 +45,9 @@ abstract class ElementsSelector extends Selector { protected final Selector selected; - protected ElementsSelector(Selector selected) + protected ElementsSelector(Kind kind,Selector selected) { + super(kind); this.selected = selected; } @@ -210,7 +218,7 @@ abstract class ElementsSelector extends Selector }; } - public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException + public ByteBuffer getOutput(ProtocolVersion protocolVersion) { ByteBuffer value = selected.getOutput(protocolVersion); return value == null ? null : extractSelection(value); @@ -218,9 +226,9 @@ abstract class ElementsSelector extends Selector protected abstract ByteBuffer extractSelection(ByteBuffer collection); - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInput(ProtocolVersion protocolVersion, InputRow input) { - selected.addInput(protocolVersion, rs); + selected.addInput(protocolVersion, input); } public void reset() @@ -228,14 +236,31 @@ abstract class ElementsSelector extends Selector selected.reset(); } - private static class ElementSelector extends ElementsSelector + @Override + public boolean isTerminal() { + return selected.isTerminal(); + } + + static class ElementSelector extends ElementsSelector + { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + Selector selected = Selector.serializer.deserialize(in, version, metadata); + ByteBuffer key = ByteBufferUtil.readWithVIntLength(in); + + return new ElementSelector(selected, key); + } + }; + private final CollectionType type; private final ByteBuffer key; private ElementSelector(Selector selected, ByteBuffer key) { - super(selected); + super(Kind.ELEMENT_SELECTOR, selected); assert selected.getType() instanceof MapType || selected.getType() instanceof SetType : "this shouldn't have passed validation in Selectable"; this.type = (CollectionType) selected.getType(); this.key = key; @@ -269,10 +294,60 @@ abstract class ElementsSelector extends Selector { return String.format("%s[%s]", selected, keyType(type).getString(key)); } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof ElementSelector)) + return false; + + ElementSelector s = (ElementSelector) o; + + return Objects.equal(selected, s.selected) + && Objects.equal(key, s.key); + } + + @Override + public int hashCode() + { + return Objects.hashCode(selected, key); + } + + @Override + protected int serializedSize(int version) + { + return TypeSizes.sizeofWithVIntLength(key) + serializer.serializedSize(selected, version); + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + serializer.serialize(selected, out, version); + ByteBufferUtil.serializedSizeWithVIntLength(key); + } } - private static class SliceSelector extends ElementsSelector + static class SliceSelector extends ElementsSelector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + Selector selected = Selector.serializer.deserialize(in, version, metadata); + + boolean isFromUnset = in.readBoolean(); + ByteBuffer from = isFromUnset ? ByteBufferUtil.UNSET_BYTE_BUFFER : ByteBufferUtil.readWithVIntLength(in); + + boolean isToUnset = in.readBoolean(); + ByteBuffer to = isToUnset ? ByteBufferUtil.UNSET_BYTE_BUFFER : ByteBufferUtil.readWithVIntLength(in); + + return new SliceSelector(selected, from, to); + } + }; + private final CollectionType type; // Note that neither from nor to can be null, but they can both be ByteBufferUtil.UNSET_BYTE_BUFFER to represent no particular bound @@ -281,7 +356,7 @@ abstract class ElementsSelector extends Selector private SliceSelector(Selector selected, ByteBuffer from, ByteBuffer to) { - super(selected); + super(Kind.SLICE_SELECTOR, selected); assert selected.getType() instanceof MapType || selected.getType() instanceof SetType : "this shouldn't have passed validation in Selectable"; assert from != null && to != null : "We can have unset buffers, but not nulls"; this.type = (CollectionType) selected.getType(); @@ -321,5 +396,57 @@ abstract class ElementsSelector extends Selector ? selected.toString() : String.format("%s[%s..%s]", selected, fromUnset ? "" : keyType(type).getString(from), toUnset ? "" : keyType(type).getString(to)); } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof SliceSelector)) + return false; + + SliceSelector s = (SliceSelector) o; + + return Objects.equal(selected, s.selected) + && Objects.equal(from, s.from) + && Objects.equal(to, s.to); + } + + @Override + public int hashCode() + { + return Objects.hashCode(selected, from, to); + } + + @Override + protected int serializedSize(int version) + { + int size = serializer.serializedSize(selected, version) + 2; + + if (!isUnset(from)) + size += TypeSizes.sizeofWithVIntLength(from); + + if (!isUnset(to)) + size += TypeSizes.sizeofWithVIntLength(to); + + return size; + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + serializer.serialize(selected, out, version); + + boolean isFromUnset = isUnset(from); + out.writeBoolean(isFromUnset); + if (!isFromUnset) + ByteBufferUtil.serializedSizeWithVIntLength(from); + + boolean isToUnset = isUnset(to); + out.writeBoolean(isToUnset); + if (!isToUnset) + ByteBufferUtil.serializedSizeWithVIntLength(to); + } } } diff --git a/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java b/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java index c67fc0330e..0c62397681 100644 --- a/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/FieldSelector.java @@ -17,18 +17,37 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; +import com.google.common.base.Objects; + import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.UserType; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.transport.ProtocolVersion; final class FieldSelector extends Selector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + UserType type = (UserType) readType(metadata, in); + int field = (int) in.readUnsignedVInt(); + Selector selected = Selector.serializer.deserialize(in, version, metadata); + + return new FieldSelector(type, field, selected); + } + }; + private final UserType type; private final int field; private final Selector selected; @@ -79,12 +98,12 @@ final class FieldSelector extends Selector selected.addFetchedColumns(builder); } - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInput(ProtocolVersion protocolVersion, InputRow input) { - selected.addInput(protocolVersion, rs); + selected.addInput(protocolVersion, input); } - public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException + public ByteBuffer getOutput(ProtocolVersion protocolVersion) { ByteBuffer value = selected.getOutput(protocolVersion); if (value == null) @@ -103,6 +122,12 @@ final class FieldSelector extends Selector selected.reset(); } + @Override + public boolean isTerminal() + { + return selected.isTerminal(); + } + @Override public String toString() { @@ -111,8 +136,45 @@ final class FieldSelector extends Selector private FieldSelector(UserType type, int field, Selector selected) { + super(Kind.FIELD_SELECTOR); this.type = type; this.field = field; this.selected = selected; } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof FieldSelector)) + return false; + + FieldSelector s = (FieldSelector) o; + + return Objects.equal(type, s.type) + && Objects.equal(field, s.field) + && Objects.equal(selected, s.selected); + } + + @Override + public int hashCode() + { + return Objects.hashCode(type, field, selected); + } + + @Override + protected int serializedSize(int version) + { + return sizeOf(type) + TypeSizes.sizeofUnsignedVInt(field) + serializer.serializedSize(selected, version); + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + writeType(out, type); + out.writeUnsignedVInt(field); + serializer.serialize(selected, out, version); + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/ListSelector.java b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java index a8c5d5cb02..9136ab2c5b 100644 --- a/src/java/org/apache/cassandra/cql3/selection/ListSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/ListSelector.java @@ -17,15 +17,22 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import com.google.common.base.Objects; + import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.Lists; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter.Builder; import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.transport.ProtocolVersion; @@ -35,6 +42,20 @@ import org.apache.cassandra.transport.ProtocolVersion; */ final class ListSelector extends Selector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + ListType type = (ListType) readType(metadata, in); + int size = (int) in.readUnsignedVInt(); + List elements = new ArrayList<>(size); + for (int i = 0; i < size; i++) + elements.add(serializer.deserialize(in, version, metadata)); + + return new ListSelector(type, elements); + } + }; + /** * The list type. */ @@ -68,13 +89,13 @@ final class ListSelector extends Selector elements.get(i).addFetchedColumns(builder); } - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInput(ProtocolVersion protocolVersion, InputRow input) { for (int i = 0, m = elements.size(); i < m; i++) - elements.get(i).addInput(protocolVersion, rs); + elements.get(i).addInput(protocolVersion, input); } - public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException + public ByteBuffer getOutput(ProtocolVersion protocolVersion) { List buffers = new ArrayList<>(elements.size()); for (int i = 0, m = elements.size(); i < m; i++) @@ -90,6 +111,17 @@ final class ListSelector extends Selector elements.get(i).reset(); } + @Override + public boolean isTerminal() + { + for (int i = 0, m = elements.size(); i < m; i++) + { + if (!elements.get(i).isTerminal()) + return false; + } + return true; + } + public AbstractType getType() { return type; @@ -103,7 +135,48 @@ final class ListSelector extends Selector private ListSelector(AbstractType type, List elements) { + super(Kind.LIST_SELECTOR); this.type = type; this.elements = elements; } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof ListSelector)) + return false; + + ListSelector s = (ListSelector) o; + + return Objects.equal(type, s.type) + && Objects.equal(elements, s.elements); + } + + @Override + public int hashCode() + { + return Objects.hashCode(type, elements); + } + + @Override + protected int serializedSize(int version) + { + int size = sizeOf(type) + TypeSizes.sizeofUnsignedVInt(elements.size()); + for (int i = 0, m = elements.size(); i < m; i++) + size += serializer.serializedSize(elements.get(i), version); + + return size; + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + writeType(out, type); + out.writeUnsignedVInt(elements.size()); + for (int i = 0, m = elements.size(); i < m; i++) + serializer.serialize(elements.get(i), out, version); + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/MapSelector.java b/src/java/org/apache/cassandra/cql3/selection/MapSelector.java index 09424bda8c..a43d7bfccb 100644 --- a/src/java/org/apache/cassandra/cql3/selection/MapSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/MapSelector.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -24,15 +25,21 @@ import java.util.Map; import java.util.TreeMap; import java.util.stream.Collectors; +import com.google.common.base.Objects; + import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.Maps; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter.Builder; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.MapType; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.Pair; @@ -43,6 +50,23 @@ import org.apache.cassandra.utils.Pair; */ final class MapSelector extends Selector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + MapType type = (MapType) readType(metadata, in); + int size = (int) in.readUnsignedVInt(); + List> entries = new ArrayList<>(size); + for (int i = 0; i < size; i++) + { + Pair entry = Pair.create(serializer.deserialize(in, version, metadata), + serializer.deserialize(in, version, metadata)); + entries.add(entry); + } + return new MapSelector(type, entries); + } + }; + /** * The map type. */ @@ -170,17 +194,17 @@ final class MapSelector extends Selector } } - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInput(ProtocolVersion protocolVersion, InputRow input) { for (int i = 0, m = elements.size(); i < m; i++) { Pair pair = elements.get(i); - pair.left.addInput(protocolVersion, rs); - pair.right.addInput(protocolVersion, rs); + pair.left.addInput(protocolVersion, input); + pair.right.addInput(protocolVersion, input); } } - public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException + public ByteBuffer getOutput(ProtocolVersion protocolVersion) { Map map = new TreeMap<>(type.getKeysType()); for (int i = 0, m = elements.size(); i < m; i++) @@ -208,6 +232,18 @@ final class MapSelector extends Selector } } + @Override + public boolean isTerminal() + { + for (int i = 0, m = elements.size(); i < m; i++) + { + Pair pair = elements.get(i); + if (!pair.left.isTerminal() || !pair.right.isTerminal()) + return false; + } + return true; + } + public AbstractType getType() { return type; @@ -221,7 +257,58 @@ final class MapSelector extends Selector private MapSelector(AbstractType type, List> elements) { + super(Kind.MAP_SELECTOR); this.type = (MapType) type; this.elements = elements; } + + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof MapSelector)) + return false; + + MapSelector s = (MapSelector) o; + + return Objects.equal(type, s.type) + && Objects.equal(elements, s.elements); + } + + @Override + public int hashCode() + { + return Objects.hashCode(type, elements); + } + + @Override + protected int serializedSize(int version) + { + int size = sizeOf(type) + TypeSizes.sizeofUnsignedVInt(elements.size()); + + for (int i = 0, m = elements.size(); i < m; i++) + { + Pair entry = elements.get(i); + size += serializer.serializedSize(entry.left, version) + serializer.serializedSize(entry.right, version); + } + + return size; + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + writeType(out, type); + out.writeUnsignedVInt(elements.size()); + + for (int i = 0, m = elements.size(); i < m; i++) + { + Pair entry = elements.get(i); + serializer.serialize(entry.left, out, version); + serializer.serialize(entry.right, out, version); + } + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java b/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java index cb8c57681a..b42357835a 100644 --- a/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java +++ b/src/java/org/apache/cassandra/cql3/selection/ResultSetBuilder.java @@ -19,7 +19,6 @@ package org.apache.cassandra.cql3.selection; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import org.apache.cassandra.config.Config; @@ -29,9 +28,7 @@ import org.apache.cassandra.cql3.selection.Selection.Selectors; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.aggregation.GroupMaker; -import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.db.rows.Cell; -import org.apache.cassandra.utils.ByteBufferUtil; public final class ResultSetBuilder { @@ -50,15 +47,8 @@ public final class ResultSetBuilder /* * We'll build CQL3 row one by one. - * The currentRow is the values for the (CQL3) columns we've fetched. - * We also collect timestamps and ttls for the case where the writetime and - * ttl functions are used. Note that we might collect timestamp and/or ttls - * we don't care about, but since the array below are allocated just once, - * it doesn't matter performance wise. */ - List current; - final long[] timestamps; - final int[] ttls; + private Selector.InputRow inputRow; private long size = 0; private boolean sizeWarningEmitted = false; @@ -73,14 +63,6 @@ public final class ResultSetBuilder this.resultSet = new ResultSet(metadata.copy(), new ArrayList>()); this.selectors = selectors; this.groupMaker = groupMaker; - this.timestamps = selectors.collectTimestamps() ? new long[selectors.numberOfFetchedColumns()] : null; - this.ttls = selectors.collectTTLs() ? new int[selectors.numberOfFetchedColumns()] : null; - - // We use MIN_VALUE to indicate no timestamp and -1 for no ttl - if (timestamps != null) - Arrays.fill(timestamps, Long.MIN_VALUE); - if (ttls != null) - Arrays.fill(ttls, -1); } private void addSize(List row) @@ -114,40 +96,12 @@ public final class ResultSetBuilder public void add(ByteBuffer v) { - current.add(v); + inputRow.add(v); } public void add(Cell c, int nowInSec) { - if (c == null) - { - current.add(null); - return; - } - - current.add(value(c)); - - if (timestamps != null) - timestamps[current.size() - 1] = c.timestamp(); - - if (ttls != null) - ttls[current.size() - 1] = remainingTTL(c, nowInSec); - } - - private int remainingTTL(Cell c, int nowInSec) - { - if (!c.isExpiring()) - return -1; - - int remaining = c.localDeletionTime() - nowInSec; - return remaining >= 0 ? remaining : -1; - } - - private ByteBuffer value(Cell c) - { - return c.isCounterCell() - ? ByteBufferUtil.bytes(CounterContext.instance().total(c.value(), c.accessor())) - : c.buffer(); + inputRow.add(c, nowInSec); } /** @@ -160,22 +114,24 @@ public final class ResultSetBuilder { // The groupMaker needs to be called for each row boolean isNewAggregate = groupMaker == null || groupMaker.isNewGroup(partitionKey, clustering); - if (current != null) + if (inputRow != null) { - selectors.addInputRow(this); + selectors.addInputRow(inputRow); if (isNewAggregate) { resultSet.addRow(getOutputRow()); + inputRow.reset(!selectors.hasProcessing()); selectors.reset(); } + else + { + inputRow.reset(!selectors.hasProcessing()); + } + } + else + { + inputRow = new Selector.InputRow(selectors.numberOfFetchedColumns(), selectors.collectTimestamps(), selectors.collectTTLs()); } - current = new ArrayList<>(selectors.numberOfFetchedColumns()); - - // Timestamps and TTLs are arrays per row, we must null them out between rows - if (timestamps != null) - Arrays.fill(timestamps, Long.MIN_VALUE); - if (ttls != null) - Arrays.fill(ttls, -1); } /** @@ -183,12 +139,12 @@ public final class ResultSetBuilder */ public ResultSet build() { - if (current != null) + if (inputRow != null) { - selectors.addInputRow(this); + selectors.addInputRow(inputRow); resultSet.addRow(getOutputRow()); + inputRow.reset(!selectors.hasProcessing()); selectors.reset(); - current = null; } // For aggregates we need to return a row even it no records have been found diff --git a/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java b/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java index de74678385..ed2a1406be 100644 --- a/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/ScalarFunctionSelector.java @@ -22,17 +22,27 @@ import java.util.List; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.functions.ScalarFunction; -import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.transport.ProtocolVersion; +import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue; + final class ScalarFunctionSelector extends AbstractFunctionSelector { - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + protected static final SelectorDeserializer deserializer = new AbstractFunctionSelectorDeserializer() + { + @Override + protected Selector newFunctionSelector(Function function, List argSelectors) + { + return new ScalarFunctionSelector(function, argSelectors); + } + }; + + public void addInput(ProtocolVersion protocolVersion, InputRow input) { for (int i = 0, m = argSelectors.size(); i < m; i++) { Selector s = argSelectors.get(i); - s.addInput(protocolVersion, rs); + s.addInput(protocolVersion, input); } } @@ -40,7 +50,7 @@ final class ScalarFunctionSelector extends AbstractFunctionSelector argSelectors) { - super((ScalarFunction) fun, argSelectors); + super(Kind.SCALAR_FUNCTION_SELECTOR, (ScalarFunction) fun, argSelectors); } } diff --git a/src/java/org/apache/cassandra/cql3/selection/Selection.java b/src/java/org/apache/cassandra/cql3/selection/Selection.java index 77e4cc8a6f..f07184a599 100644 --- a/src/java/org/apache/cassandra/cql3/selection/Selection.java +++ b/src/java/org/apache/cassandra/cql3/selection/Selection.java @@ -28,6 +28,7 @@ import com.google.common.collect.Lists; import org.apache.cassandra.cql3.*; import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.cql3.selection.Selector.InputRow; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -345,6 +346,12 @@ public abstract class Selection */ public ColumnFilter getColumnFilter(); + /** + * Checks if this Selectors perform some processing + * @return {@code true} if this Selectors perform some processing, {@code false} otherwise. + */ + public boolean hasProcessing(); + /** * Checks if one of the selectors perform some aggregations. * @return {@code true} if one of the selectors perform some aggregations, {@code false} otherwise. @@ -372,10 +379,9 @@ public abstract class Selection /** * Adds the current row of the specified ResultSetBuilder. * - * @param rs the ResultSetBuilder - * @throws InvalidRequestException + * @param input the input row */ - public void addInputRow(ResultSetBuilder rs); + public void addInputRow(InputRow input); public List getOutputRow(); @@ -467,9 +473,9 @@ public abstract class Selection return current; } - public void addInputRow(ResultSetBuilder rs) throws InvalidRequestException + public void addInputRow(InputRow input) { - current = rs.current; + current = input.getValues(); } public boolean isAggregate() @@ -477,6 +483,11 @@ public abstract class Selection return false; } + public boolean hasProcessing() + { + return false; + } + @Override public int numberOfFetchedColumns() { @@ -572,6 +583,11 @@ public abstract class Selection return factories.doesAggregation(); } + public boolean hasProcessing() + { + return true; + } + public List getOutputRow() { List outputRow = new ArrayList<>(selectors.size()); @@ -582,10 +598,10 @@ public abstract class Selection return isJson ? rowToJson(outputRow, options.getProtocolVersion(), metadata, orderingColumns) : outputRow; } - public void addInputRow(ResultSetBuilder rs) throws InvalidRequestException + public void addInputRow(InputRow input) { for (Selector selector : selectors) - selector.addInput(options.getProtocolVersion(), rs); + selector.addInput(options.getProtocolVersion(), input); } @Override diff --git a/src/java/org/apache/cassandra/cql3/selection/Selector.java b/src/java/org/apache/cassandra/cql3/selection/Selector.java index 3262b9c9e1..463382d584 100644 --- a/src/java/org/apache/cassandra/cql3/selection/Selector.java +++ b/src/java/org/apache/cassandra/cql3/selection/Selector.java @@ -17,18 +17,31 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.List; +import org.apache.cassandra.schema.CQLTypeParser; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; /** * A Selector is used to convert the data returned by the storage engine into the data requested by the @@ -38,6 +51,50 @@ import org.apache.cassandra.transport.ProtocolVersion; */ public abstract class Selector { + protected static abstract class SelectorDeserializer + { + protected abstract Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException; + + protected final AbstractType readType(TableMetadata metadata, DataInputPlus in) throws IOException + { + KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(metadata.keyspace); + return readType(keyspace, in); + } + + protected final AbstractType readType(KeyspaceMetadata keyspace, DataInputPlus in) throws IOException + { + String cqlType = in.readUTF(); + return CQLTypeParser.parse(keyspace.name, cqlType, keyspace.types); + } + } + + /** + * The Selector kinds. + */ + public static enum Kind + { + SIMPLE_SELECTOR(SimpleSelector.deserializer), + TERM_SELECTOR(TermSelector.deserializer), + WRITETIME_OR_TTL_SELECTOR(WritetimeOrTTLSelector.deserializer), + LIST_SELECTOR(ListSelector.deserializer), + SET_SELECTOR(SetSelector.deserializer), + MAP_SELECTOR(MapSelector.deserializer), + TUPLE_SELECTOR(TupleSelector.deserializer), + USER_TYPE_SELECTOR(UserTypeSelector.deserializer), + FIELD_SELECTOR(FieldSelector.deserializer), + SCALAR_FUNCTION_SELECTOR(ScalarFunctionSelector.deserializer), + AGGREGATE_FUNCTION_SELECTOR(AggregateFunctionSelector.deserializer), + ELEMENT_SELECTOR(ElementsSelector.ElementSelector.deserializer), + SLICE_SELECTOR(ElementsSelector.SliceSelector.deserializer); + + private final SelectorDeserializer deserializer; + + Kind(SelectorDeserializer deserializer) + { + this.deserializer = deserializer; + } + } + /** * A factory for Selector instances. */ @@ -70,7 +127,7 @@ public abstract class Selector * depends on the bound values in particular). * @return a new Selector instance */ - public abstract Selector newInstance(QueryOptions options) throws InvalidRequestException; + public abstract Selector newInstance(QueryOptions options); /** * Checks if this factory creates selectors instances that creates aggregates. @@ -174,6 +231,50 @@ public abstract class Selector abstract void addFetchedColumns(ColumnFilter.Builder builder); } + public static class Serializer + { + public void serialize(Selector selector, DataOutputPlus out, int version) throws IOException + { + out.writeByte(selector.kind().ordinal()); + selector.serialize(out, version); + } + + public Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + Kind kind = Kind.values()[in.readUnsignedByte()]; + return kind.deserializer.deserialize(in, version, metadata); + } + + public int serializedSize(Selector selector, int version) + { + return TypeSizes.sizeof((byte) selector.kind().ordinal()) + selector.serializedSize(version); + } + } + + /** + * The {@code Selector} serializer. + */ + public static final Serializer serializer = new Serializer(); + + /** + * The {@code Selector} kind. + */ + private final Kind kind; + + /** + * Returns the {@code Selector} kind. + * @return the {@code Selector} kind + */ + public final Kind kind() + { + return kind; + } + + protected Selector(Kind kind) + { + this.kind = kind; + } + /** * Add to the provided builder the column (and potential subselections) to fetch for this * selection. @@ -182,14 +283,158 @@ public abstract class Selector */ public abstract void addFetchedColumns(ColumnFilter.Builder builder); + /** + * A row of data that need to be processed by a {@code Selector} + */ + public static final class InputRow + { + private ByteBuffer[] values; + private final long[] timestamps; + private final int[] ttls; + private int index; + + public InputRow(int size, boolean collectTimestamps, boolean collectTTLs) + { + this.values = new ByteBuffer[size]; + + if (collectTimestamps) + { + this.timestamps = new long[size]; + // We use MIN_VALUE to indicate no timestamp + Arrays.fill(timestamps, Long.MIN_VALUE); + } + else + { + timestamps = null; + } + + if (collectTTLs) + { + this.ttls = new int[size]; + // We use -1 to indicate no ttl + Arrays.fill(ttls, -1); + } + else + { + ttls = null; + } + } + + public void add(ByteBuffer v) + { + values[index] = v; + + if (timestamps != null) + timestamps[index] = Long.MIN_VALUE; + + if (ttls != null) + ttls[index] = -1; + + index++; + } + + public void add(Cell c, int nowInSec) + { + if (c == null) + { + add(null); + return; + } + + values[index] = value(c); + + if (timestamps != null) + timestamps[index] = c.timestamp(); + + if (ttls != null) + ttls[index] = remainingTTL(c, nowInSec); + + index++; + } + + private int remainingTTL(Cell c, int nowInSec) + { + if (!c.isExpiring()) + return -1; + + int remaining = c.localDeletionTime() - nowInSec; + return remaining >= 0 ? remaining : -1; + } + + private ByteBuffer value(Cell c) + { + return c.isCounterCell() + ? ByteBufferUtil.bytes(CounterContext.instance().total(c.value(), c.accessor())) + : c.buffer(); + } + + /** + * Return the value of the column with the specified index. + * + * @param index the column index + * @return the value of the column with the specified index + */ + public ByteBuffer getValue(int index) + { + return values[index]; + } + + /** + * Reset the row internal state. + *

If the reset is not a deep one only the index will be reset. If the reset is a deep one a new + * array will be created to store the column values. This allow to reduce object creation when it is not + * necessary.

+ * + * @param deep {@code true} if the reset must be a deep one. + */ + public void reset(boolean deep) + { + index = 0; + if (deep) + values = new ByteBuffer[values.length]; + } + + /** + * Return the timestamp of the column with the specified index. + * + * @param index the column index + * @return the timestamp of the column with the specified index + */ + public long getTimestamp(int index) + { + return timestamps[index]; + } + + /** + * Return the ttl of the column with the specified index. + * + * @param index the column index + * @return the ttl of the column with the specified index + */ + public int getTtl(int index) + { + return ttls[index]; + } + + /** + * Returns the column values as list. + *

This content of the list will be shared with the {@code InputRow} unless a deep reset has been done.

+ * @return the column values as list. + */ + public List getValues() + { + return Arrays.asList(values); + } + } + /** * Add the current value from the specified ResultSetBuilder. * * @param protocolVersion protocol version used for serialization - * @param rs the ResultSetBuilder - * @throws InvalidRequestException if a problem occurs while add the input value + * @param input the input row + * @throws InvalidRequestException if a problem occurs while adding the input row */ - public abstract void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException; + public abstract void addInput(ProtocolVersion protocolVersion, InputRow input); /** * Returns the selector output. @@ -211,4 +456,36 @@ public abstract class Selector * Reset the internal state of this Selector. */ public abstract void reset(); + + /** + * A selector is terminal if it doesn't require any input for it's output to be computed, i.e. if {@link #getOutput} + * result doesn't depend of {@link #addInput}. This is typically the case of a constant value or functions on constant + * values. + */ + public boolean isTerminal() + { + return false; + } + + /** + * Checks that this selector is valid for GROUP BY clause. + */ + public void validateForGroupBy() + { + throw invalidRequest("Only column names and monotonic scalar functions are supported in the GROUP BY clause."); + } + + protected abstract int serializedSize(int version); + + protected abstract void serialize(DataOutputPlus out, int version) throws IOException; + + protected static void writeType(DataOutputPlus out, AbstractType type) throws IOException + { + out.writeUTF(type.asCQL3Type().toString()); + } + + protected static int sizeOf(AbstractType type) + { + return TypeSizes.sizeof(type.asCQL3Type().toString()); + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/SetSelector.java b/src/java/org/apache/cassandra/cql3/selection/SetSelector.java index 6693121fb3..b54b2d4c4a 100644 --- a/src/java/org/apache/cassandra/cql3/selection/SetSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/SetSelector.java @@ -17,17 +17,24 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.TreeSet; +import com.google.common.base.Objects; + import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.Sets; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter.Builder; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.SetType; -import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.transport.ProtocolVersion; @@ -37,6 +44,20 @@ import org.apache.cassandra.transport.ProtocolVersion; */ final class SetSelector extends Selector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + SetType type = (SetType) readType(metadata, in); + int size = (int) in.readUnsignedVInt(); + List elements = new ArrayList<>(size); + for (int i = 0; i < size; i++) + elements.add(serializer.deserialize(in, version, metadata)); + + return new SetSelector(type, elements); + } + }; + /** * The set type. */ @@ -70,13 +91,13 @@ final class SetSelector extends Selector elements.get(i).addFetchedColumns(builder); } - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInput(ProtocolVersion protocolVersion, InputRow input) { for (int i = 0, m = elements.size(); i < m; i++) - elements.get(i).addInput(protocolVersion, rs); + elements.get(i).addInput(protocolVersion, input); } - public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException + public ByteBuffer getOutput(ProtocolVersion protocolVersion) { Set buffers = new TreeSet<>(type.getElementsType()); for (int i = 0, m = elements.size(); i < m; i++) @@ -92,6 +113,17 @@ final class SetSelector extends Selector elements.get(i).reset(); } + @Override + public boolean isTerminal() + { + for (int i = 0, m = elements.size(); i < m; i++) + { + if (!elements.get(i).isTerminal()) + return false; + } + return true; + } + public AbstractType getType() { return type; @@ -105,7 +137,49 @@ final class SetSelector extends Selector private SetSelector(AbstractType type, List elements) { + super(Kind.SET_SELECTOR); this.type = (SetType) type; this.elements = elements; } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof SetSelector)) + return false; + + SetSelector s = (SetSelector) o; + + return Objects.equal(type, s.type) + && Objects.equal(elements, s.elements); + } + + @Override + public int hashCode() + { + return Objects.hashCode(type, elements); + } + + @Override + protected int serializedSize(int version) + { + int size = sizeOf(type) + TypeSizes.sizeofUnsignedVInt(elements.size()); + + for (int i = 0, m = elements.size(); i < m; i++) + size += serializer.serializedSize(elements.get(i), version); + + return size; + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + writeType(out, type); + out.writeUnsignedVInt(elements.size()); + for (int i = 0, m = elements.size(); i < m; i++) + serializer.serialize(elements.get(i), out, version); + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java b/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java index 31b1911a14..a6ae446048 100644 --- a/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/SimpleSelector.java @@ -17,19 +17,38 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; +import com.google.common.base.Objects; + import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.filter.ColumnFilter.Builder; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.ByteBufferUtil; public final class SimpleSelector extends Selector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + ByteBuffer columnName = ByteBufferUtil.readWithVIntLength(in); + ColumnMetadata column = metadata.getColumn(columnName); + int idx = in.readInt(); + return new SimpleSelector(column, idx); + } + }; + /** * The Factory for {@code SimpleSelector}. */ @@ -113,17 +132,17 @@ public final class SimpleSelector extends Selector } @Override - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInput(ProtocolVersion protocolVersion, InputRow input) throws InvalidRequestException { if (!isSet) { isSet = true; - current = rs.current.get(idx); + current = input.getValue(idx); } } @Override - public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException + public ByteBuffer getOutput(ProtocolVersion protocolVersion) { return current; } @@ -149,7 +168,48 @@ public final class SimpleSelector extends Selector private SimpleSelector(ColumnMetadata column, int idx) { + super(Kind.SIMPLE_SELECTOR); this.column = column; this.idx = idx; } + + @Override + public void validateForGroupBy() + { + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof SimpleSelector)) + return false; + + SimpleSelector s = (SimpleSelector) o; + + return Objects.equal(column, s.column) + && Objects.equal(idx, s.idx); + } + + @Override + public int hashCode() + { + return Objects.hashCode(column, idx); + } + + @Override + protected int serializedSize(int version) + { + return ByteBufferUtil.serializedSizeWithVIntLength(column.name.bytes) + + TypeSizes.sizeof(idx); + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + ByteBufferUtil.writeWithVIntLength(column.name.bytes, out); + out.writeInt(idx);; + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/TermSelector.java b/src/java/org/apache/cassandra/cql3/selection/TermSelector.java index 321cd27632..6f0c844dd4 100644 --- a/src/java/org/apache/cassandra/cql3/selection/TermSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/TermSelector.java @@ -17,16 +17,22 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; +import com.google.common.base.Objects; + import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.Term; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.ByteBufferUtil; /** * Selector representing a simple term (literals or bound variables). @@ -36,6 +42,16 @@ import org.apache.cassandra.transport.ProtocolVersion; */ public class TermSelector extends Selector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + AbstractType type = readType(metadata, in); + ByteBuffer value = ByteBufferUtil.readWithVIntLength(in); + return new TermSelector(value, type); + } + }; + private final ByteBuffer value; private final AbstractType type; @@ -74,8 +90,9 @@ public class TermSelector extends Selector }; } - private TermSelector(ByteBuffer value, AbstractType type) + TermSelector(ByteBuffer value, AbstractType type) { + super(Kind.TERM_SELECTOR); this.value = value; this.type = type; } @@ -84,11 +101,11 @@ public class TermSelector extends Selector { } - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInput(ProtocolVersion protocolVersion, InputRow input) { } - public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException + public ByteBuffer getOutput(ProtocolVersion protocolVersion) { return value; } @@ -101,4 +118,44 @@ public class TermSelector extends Selector public void reset() { } + + @Override + public boolean isTerminal() + { + return true; + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof TermSelector)) + return false; + + TermSelector s = (TermSelector) o; + + return Objects.equal(value, s.value) + && Objects.equal(type, s.type); + } + + @Override + public int hashCode() + { + return Objects.hashCode(value, type); + } + + @Override + protected int serializedSize(int version) + { + return sizeOf(type) + ByteBufferUtil.serializedSizeWithVIntLength(value); + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + writeType(out, type); + ByteBufferUtil.writeWithVIntLength(value, out); + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java b/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java index 898085bc82..0c06bc2e2f 100644 --- a/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/TupleSelector.java @@ -17,15 +17,23 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; +import com.google.common.base.Objects; + import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.Tuples; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter.Builder; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.TupleType; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.transport.ProtocolVersion; /** @@ -34,6 +42,20 @@ import org.apache.cassandra.transport.ProtocolVersion; */ final class TupleSelector extends Selector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + AbstractType type = readType(metadata, in); + int size = (int) in.readUnsignedVInt(); + List elements = new ArrayList<>(size); + for (int i = 0; i < size; i++) + elements.add(serializer.deserialize(in, version, metadata)); + + return new TupleSelector(type, elements); + } + }; + /** * The tuple type. */ @@ -67,10 +89,10 @@ final class TupleSelector extends Selector elements.get(i).addFetchedColumns(builder); } - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInput(ProtocolVersion protocolVersion, InputRow input) { for (int i = 0, m = elements.size(); i < m; i++) - elements.get(i).addInput(protocolVersion, rs); + elements.get(i).addInput(protocolVersion, input); } public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException @@ -89,6 +111,17 @@ final class TupleSelector extends Selector elements.get(i).reset(); } + @Override + public boolean isTerminal() + { + for (int i = 0, m = elements.size(); i < m; i++) + { + if (!elements.get(i).isTerminal()) + return false; + } + return true; + } + public AbstractType getType() { return type; @@ -102,7 +135,50 @@ final class TupleSelector extends Selector private TupleSelector(AbstractType type, List elements) { + super(Kind.TUPLE_SELECTOR); this.type = type; this.elements = elements; } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof TupleSelector)) + return false; + + TupleSelector s = (TupleSelector) o; + + return Objects.equal(type, s.type) + && Objects.equal(elements, s.elements); + } + + @Override + public int hashCode() + { + return Objects.hashCode(type, elements); + } + + @Override + protected int serializedSize(int version) + { + int size = sizeOf(type) + TypeSizes.sizeofUnsignedVInt(elements.size()); + + for (int i = 0, m = elements.size(); i < m; i++) + size += serializer.serializedSize(elements.get(i), version); + + return size; + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + writeType(out, type); + out.writeUnsignedVInt(elements.size()); + + for (int i = 0, m = elements.size(); i < m; i++) + serializer.serialize(elements.get(i), out, version); + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java b/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java index 61faf8d041..8007467ec3 100644 --- a/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/UserTypeSelector.java @@ -17,25 +17,32 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import com.google.common.base.Objects; + import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.FieldIdentifier; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.UserTypes; import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.filter.ColumnFilter.Builder; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.TupleType; import org.apache.cassandra.db.marshal.UserType; -import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.ByteBufferUtil; /** * Selector for literal map (e.g. {'min' : min(value), 'max' : max(value), 'count' : count(value)}). @@ -43,6 +50,23 @@ import org.apache.cassandra.transport.ProtocolVersion; */ final class UserTypeSelector extends Selector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + UserType type = (UserType) readType(metadata, in); + int size = (int) in.readUnsignedVInt(); + Map fields = new HashMap<>(size); + for (int i = 0; i < size; i++) + { + FieldIdentifier identifier = new FieldIdentifier(ByteBufferUtil.readWithVIntLength(in)); + Selector selector = serializer.deserialize(in, version, metadata); + fields.put(identifier, selector); + } + return new UserTypeSelector(type, fields); + } + }; + /** * The map type. */ @@ -158,13 +182,13 @@ final class UserTypeSelector extends Selector field.addFetchedColumns(builder); } - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInput(ProtocolVersion protocolVersion, InputRow input) { for (Selector field : fields.values()) - field.addInput(protocolVersion, rs); + field.addInput(protocolVersion, input); } - public ByteBuffer getOutput(ProtocolVersion protocolVersion) throws InvalidRequestException + public ByteBuffer getOutput(ProtocolVersion protocolVersion) { UserType userType = (UserType) type; ByteBuffer[] buffers = new ByteBuffer[userType.size()]; @@ -183,6 +207,17 @@ final class UserTypeSelector extends Selector field.reset(); } + @Override + public boolean isTerminal() + { + for (Selector field : fields.values()) + { + if(!field.isTerminal()) + return false; + } + return true; + } + public AbstractType getType() { return type; @@ -196,7 +231,53 @@ final class UserTypeSelector extends Selector private UserTypeSelector(AbstractType type, Map fields) { + super(Kind.USER_TYPE_SELECTOR); this.type = type; this.fields = fields; } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof UserTypeSelector)) + return false; + + UserTypeSelector s = (UserTypeSelector) o; + + return Objects.equal(type, s.type) + && Objects.equal(fields, s.fields); + } + + @Override + public int hashCode() + { + return Objects.hashCode(type, fields); + } + + @Override + protected int serializedSize(int version) + { + int size = sizeOf(type) + TypeSizes.sizeofUnsignedVInt(fields.size()); + + for (Map.Entry field : fields.entrySet()) + size += ByteBufferUtil.serializedSizeWithVIntLength(field.getKey().bytes) + serializer.serializedSize(field.getValue(), version); + + return size; + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + writeType(out, type); + out.writeUnsignedVInt(fields.size()); + + for (Map.Entry field : fields.entrySet()) + { + ByteBufferUtil.writeWithVIntLength(field.getKey().bytes, out); + serializer.serialize(field.getValue(), out, version); + } + } } diff --git a/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java b/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java index 95586f2130..2c56f5ced2 100644 --- a/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java +++ b/src/java/org/apache/cassandra/cql3/selection/WritetimeOrTTLSelector.java @@ -17,20 +17,39 @@ */ package org.apache.cassandra.cql3.selection; +import java.io.IOException; import java.nio.ByteBuffer; +import com.google.common.base.Objects; + import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.ColumnSpecification; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; final class WritetimeOrTTLSelector extends Selector { + protected static final SelectorDeserializer deserializer = new SelectorDeserializer() + { + protected Selector deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException + { + ByteBuffer columnName = ByteBufferUtil.readWithVIntLength(in); + ColumnMetadata column = metadata.getColumn(columnName); + int idx = in.readInt(); + boolean isWritetime = in.readBoolean(); + return new WritetimeOrTTLSelector(column, idx, isWritetime); + } + }; + private final ColumnMetadata column; private final int idx; private final boolean isWritetime; @@ -88,7 +107,7 @@ final class WritetimeOrTTLSelector extends Selector builder.add(column); } - public void addInput(ProtocolVersion protocolVersion, ResultSetBuilder rs) + public void addInput(ProtocolVersion protocolVersion, InputRow input) { if (isSet) return; @@ -97,12 +116,12 @@ final class WritetimeOrTTLSelector extends Selector if (isWritetime) { - long ts = rs.timestamps[idx]; + long ts = input.getTimestamp(idx); current = ts != Long.MIN_VALUE ? ByteBufferUtil.bytes(ts) : null; } else { - int ttl = rs.ttls[idx]; + int ttl = input.getTtl(idx); current = ttl > 0 ? ByteBufferUtil.bytes(ttl) : null; } } @@ -131,8 +150,47 @@ final class WritetimeOrTTLSelector extends Selector private WritetimeOrTTLSelector(ColumnMetadata column, int idx, boolean isWritetime) { + super(Kind.WRITETIME_OR_TTL_SELECTOR); this.column = column; this.idx = idx; this.isWritetime = isWritetime; } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof WritetimeOrTTLSelector)) + return false; + + WritetimeOrTTLSelector s = (WritetimeOrTTLSelector) o; + + return Objects.equal(column, s.column) + && Objects.equal(idx, s.idx) + && Objects.equal(isWritetime, s.isWritetime); + } + + @Override + public int hashCode() + { + return Objects.hashCode(column, idx, isWritetime); + } + + @Override + protected int serializedSize(int version) + { + return ByteBufferUtil.serializedSizeWithVIntLength(column.name.bytes) + + TypeSizes.sizeof(idx) + + TypeSizes.sizeof(isWritetime); + } + + @Override + protected void serialize(DataOutputPlus out, int version) throws IOException + { + ByteBufferUtil.writeWithVIntLength(column.name.bytes, out); + out.writeInt(idx); + out.writeBoolean(isWritetime); + } } diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java index 7bf226b27f..b27266ce7e 100644 --- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java @@ -19,6 +19,7 @@ package org.apache.cassandra.cql3.statements; import java.nio.ByteBuffer; import java.util.*; +import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; @@ -42,8 +43,10 @@ import org.apache.cassandra.cql3.restrictions.StatementRestrictions; import org.apache.cassandra.cql3.selection.RawSelector; import org.apache.cassandra.cql3.selection.ResultSetBuilder; import org.apache.cassandra.cql3.selection.Selectable; +import org.apache.cassandra.cql3.selection.Selectable.WithFunction; import org.apache.cassandra.cql3.selection.Selection; import org.apache.cassandra.cql3.selection.Selection.Selectors; +import org.apache.cassandra.cql3.selection.Selector; import org.apache.cassandra.db.*; import org.apache.cassandra.db.aggregation.AggregationSpecification; import org.apache.cassandra.db.aggregation.GroupMaker; @@ -80,6 +83,7 @@ import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull; import static org.apache.cassandra.cql3.statements.RequestValidations.checkNull; import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue; +import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; import static org.apache.cassandra.utils.ByteBufferUtil.UNSET_BYTE_BUFFER; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -110,9 +114,9 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement private final boolean isReversed; /** - * The AggregationSpecification used to make the aggregates. + * The {@code Factory} used to create the {@code AggregationSpecification}. */ - private final AggregationSpecification aggregationSpec; + private final AggregationSpecification.Factory aggregationSpecFactory; /** * The comparator used to orders results when multiple keys are selected (using IN). @@ -132,7 +136,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement Selection selection, StatementRestrictions restrictions, boolean isReversed, - AggregationSpecification aggregationSpec, + AggregationSpecification.Factory aggregationSpecFactory, Comparator> orderingComparator, Term limit, Term perPartitionLimit) @@ -142,7 +146,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement this.selection = selection; this.restrictions = restrictions; this.isReversed = isReversed; - this.aggregationSpec = aggregationSpec; + this.aggregationSpecFactory = aggregationSpecFactory; this.orderingComparator = orderingComparator; this.parameters = parameters; this.limit = limit; @@ -174,6 +178,9 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement selection.addFunctionsTo(functions); restrictions.addFunctionsTo(functions); + if (aggregationSpecFactory != null) + aggregationSpecFactory.addFunctionsTo(functions); + if (limit != null) limit.addFunctionsTo(functions); @@ -248,14 +255,15 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement int pageSize = options.getPageSize(); Selectors selectors = selection.newSelectors(options); + AggregationSpecification aggregationSpec = getAggregationSpec(options); ReadQuery query = getQuery(options, state.getClientState(), selectors.getColumnFilter(), - nowInSec, userLimit, userPerPartitionLimit, pageSize); + nowInSec, userLimit, userPerPartitionLimit, pageSize, aggregationSpec); if (options.isReadThresholdsEnabled()) query.trackWarnings(); if (aggregationSpec == null && (pageSize <= 0 || (query.limits().count() <= pageSize))) - return execute(query, options, state.getClientState(), selectors, nowInSec, userLimit, queryStartNanoTime); + return execute(query, options, state.getClientState(), selectors, nowInSec, userLimit, null, queryStartNanoTime); QueryPager pager = getPager(query, options); @@ -266,9 +274,16 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement pageSize, nowInSec, userLimit, + aggregationSpec, queryStartNanoTime); } + + public AggregationSpecification getAggregationSpec(QueryOptions options) + { + return aggregationSpecFactory == null ? null : aggregationSpecFactory.newInstance(options); + } + public ReadQuery getQuery(QueryOptions options, int nowInSec) throws RequestValidationException { Selectors selectors = selection.newSelectors(options); @@ -278,7 +293,8 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement nowInSec, getLimit(options), getPerPartitionLimit(options), - options.getPageSize()); + options.getPageSize(), + getAggregationSpec(options)); } public ReadQuery getQuery(QueryOptions options, @@ -287,11 +303,12 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement int nowInSec, int userLimit, int perPartitionLimit, - int pageSize) + int pageSize, + AggregationSpecification aggregationSpec) { boolean isPartitionRangeQuery = restrictions.isKeyRange() || restrictions.usesSecondaryIndexing(); - DataLimits limit = getDataLimits(userLimit, perPartitionLimit, pageSize); + DataLimits limit = getDataLimits(userLimit, perPartitionLimit, pageSize, aggregationSpec); if (isPartitionRangeQuery) return getRangeCommand(options, state, columnFilter, limit, nowInSec); @@ -304,11 +321,13 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement ClientState state, Selectors selectors, int nowInSec, - int userLimit, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException + int userLimit, + AggregationSpecification aggregationSpec, + long queryStartNanoTime) { try (PartitionIterator data = query.execute(options.getConsistency(), state, queryStartNanoTime)) { - return processResults(data, options, selectors, nowInSec, userLimit); + return processResults(data, options, selectors, nowInSec, userLimit, aggregationSpec); } } @@ -392,11 +411,12 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement int pageSize, int nowInSec, int userLimit, - long queryStartNanoTime) throws RequestValidationException, RequestExecutionException + AggregationSpecification aggregationSpec, + long queryStartNanoTime) { Guardrails.pageSize.guard(pageSize, table(), false, state.getClientState()); - if (aggregationSpec != null) + if (aggregationSpecFactory != null) { if (!restrictions.hasPartitionKeyRestrictions()) { @@ -417,7 +437,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement ResultMessage.Rows msg; try (PartitionIterator page = pager.fetchPage(pageSize, queryStartNanoTime)) { - msg = processResults(page, options, selectors, nowInSec, userLimit); + msg = processResults(page, options, selectors, nowInSec, userLimit, aggregationSpec); } // Please note that the isExhausted state of the pager only gets updated when we've closed the page, so this @@ -438,9 +458,10 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement QueryOptions options, Selectors selectors, int nowInSec, - int userLimit) throws RequestValidationException + int userLimit, + AggregationSpecification aggregationSpec) throws RequestValidationException { - ResultSet rset = process(partitions, options, selectors, nowInSec, userLimit); + ResultSet rset = process(partitions, options, selectors, nowInSec, userLimit, aggregationSpec); return new ResultMessage.Rows(rset); } @@ -449,15 +470,25 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement return executeInternal(state, options, options.getNowInSeconds(state), nanoTime()); } - public ResultMessage.Rows executeInternal(QueryState state, QueryOptions options, int nowInSec, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException + public ResultMessage.Rows executeInternal(QueryState state, + QueryOptions options, + int nowInSec, + long queryStartNanoTime) { int userLimit = getLimit(options); int userPerPartitionLimit = getPerPartitionLimit(options); int pageSize = options.getPageSize(); Selectors selectors = selection.newSelectors(options); - ReadQuery query = getQuery(options, state.getClientState(), selectors.getColumnFilter(), nowInSec, userLimit, - userPerPartitionLimit, pageSize); + AggregationSpecification aggregationSpec = getAggregationSpec(options); + ReadQuery query = getQuery(options, + state.getClientState(), + selectors.getColumnFilter(), + nowInSec, + userLimit, + userPerPartitionLimit, + pageSize, + aggregationSpec); try (ReadExecutionController executionController = query.executionController()) { @@ -465,7 +496,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement { try (PartitionIterator data = query.executeInternal(executionController)) { - return processResults(data, options, selectors, nowInSec, userLimit); + return processResults(data, options, selectors, nowInSec, userLimit, null); } } @@ -478,6 +509,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement pageSize, nowInSec, userLimit, + aggregationSpec, queryStartNanoTime); } } @@ -486,7 +518,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement { QueryPager pager = query.getPager(options.getPagingState(), options.getProtocolVersion()); - if (aggregationSpec == null || query.isEmpty()) + if (aggregationSpecFactory == null || query.isEmpty()) return pager; return new AggregationQueryPager(pager, query.limits()); @@ -498,11 +530,11 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement int userPerPartitionLimit = getPerPartitionLimit(options); if (options.getPageSize() > 0) throw new IllegalStateException(); - if (aggregationSpec != null) + if (aggregationSpecFactory != null) throw new IllegalStateException(); Selectors selectors = selection.newSelectors(options); - ReadQuery query = getQuery(options, state, selectors.getColumnFilter(), nowInSec, userLimit, userPerPartitionLimit, Integer.MAX_VALUE); + ReadQuery query = getQuery(options, state, selectors.getColumnFilter(), nowInSec, userLimit, userPerPartitionLimit, Integer.MAX_VALUE, null); Map> result = Collections.emptyMap(); try (ReadExecutionController executionController = query.executionController()) @@ -540,7 +572,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement { QueryOptions options = QueryOptions.DEFAULT; Selectors selectors = selection.newSelectors(options); - return process(partitions, options, selectors, nowInSec, getLimit(options)); + return process(partitions, options, selectors, nowInSec, getLimit(options), getAggregationSpec(options)); } @Override @@ -735,7 +767,10 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement return builder.build(); } - private DataLimits getDataLimits(int userLimit, int perPartitionLimit, int pageSize) + private DataLimits getDataLimits(int userLimit, + int perPartitionLimit, + int pageSize, + AggregationSpecification aggregationSpec) { int cqlRowLimit = DataLimits.NO_LIMIT; int cqlPerPartitionLimit = DataLimits.NO_LIMIT; @@ -842,7 +877,8 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement QueryOptions options, Selectors selectors, int nowInSec, - int userLimit) throws InvalidRequestException + int userLimit, + AggregationSpecification aggregationSpec) throws InvalidRequestException { GroupMaker groupMaker = aggregationSpec == null ? null : aggregationSpec.newGroupMaker(); ResultSetBuilder result = new ResultSetBuilder(getResultMetadata(), selectors, groupMaker); @@ -1089,12 +1125,14 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement validateDistinctSelection(table, selection, restrictions); } - AggregationSpecification aggregationSpec = getAggregationSpecification(table, - selection, - restrictions, - parameters.isDistinct); + AggregationSpecification.Factory aggregationSpecFactory = getAggregationSpecFactory(table, + bindVariables, + selection, + restrictions, + parameters.isDistinct); - checkFalse(aggregationSpec == AggregationSpecification.AGGREGATE_EVERYTHING && perPartitionLimit != null, + checkFalse(aggregationSpecFactory == AggregationSpecification.AGGREGATE_EVERYTHING_FACTORY + && perPartitionLimit != null, "PER PARTITION LIMIT is not allowed with aggregate queries."); Comparator> orderingComparator = null; @@ -1118,7 +1156,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement selection, restrictions, isReversed, - aggregationSpec, + aggregationSpecFactory, orderingComparator, prepareLimit(bindVariables, limit, keyspace(), limitReceiver()), prepareLimit(bindVariables, perPartitionLimit, keyspace(), perPartitionLimitReceiver())); @@ -1255,32 +1293,56 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement } /** - * Creates the AggregationSpecifications used to make the aggregates. + * Creates the {@code AggregationSpecification.Factory} used to make the aggregates. * * @param metadata the table metadata * @param selection the selection * @param restrictions the restrictions - * @param isDistinct true if the query is a DISTINCT one. - * @return the AggregationSpecifications used to make the aggregates + * @param isDistinct true if the query is a DISTINCT one. + * @return the {@code AggregationSpecification.Factory} used to make the aggregates */ - private AggregationSpecification getAggregationSpecification(TableMetadata metadata, - Selection selection, - StatementRestrictions restrictions, - boolean isDistinct) + private AggregationSpecification.Factory getAggregationSpecFactory(TableMetadata metadata, + VariableSpecifications boundNames, + Selection selection, + StatementRestrictions restrictions, + boolean isDistinct) { if (parameters.groups.isEmpty()) - return selection.isAggregate() ? AggregationSpecification.AGGREGATE_EVERYTHING + return selection.isAggregate() ? AggregationSpecification.AGGREGATE_EVERYTHING_FACTORY : null; int clusteringPrefixSize = 0; Iterator pkColumns = metadata.primaryKeyColumns().iterator(); - for (ColumnIdentifier id : parameters.groups) + Selector.Factory selectorFactory = null; + for (Selectable.Raw raw : parameters.groups) { - ColumnMetadata def = metadata.getExistingColumn(id); + Selectable selectable = raw.prepare(metadata); + ColumnMetadata def = null; - checkTrue(def.isPartitionKey() || def.isClusteringColumn(), - "Group by is currently only supported on the columns of the PRIMARY KEY, got %s", def.name); + // For GROUP BY we only allow column names or functions at the higher level. + if (selectable instanceof WithFunction) + { + WithFunction withFunction = (WithFunction) selectable; + validateGroupByFunction(withFunction); + List columns = new ArrayList(); + selectorFactory = selectable.newSelectorFactory(metadata, null, columns, boundNames); + checkFalse(columns.isEmpty(), "GROUP BY functions must have one clustering column name as parameter"); + if (columns.size() > 1) + throw invalidRequest("GROUP BY functions accept only one clustering column as parameter, got: %s", + columns.stream().map(c -> c.name.toCQLString()).collect(Collectors.joining(","))); + + def = columns.get(0); + checkTrue(def.isClusteringColumn(), + "Group by functions are only supported on clustering columns, got %s", def.name); + } + else + { + def = (ColumnMetadata) selectable; + checkTrue(def.isPartitionKey() || def.isClusteringColumn(), + "Group by is currently only supported on the columns of the PRIMARY KEY, got %s", def.name); + checkNull(selectorFactory, "Functions are only supported on the last element of the GROUP BY clause"); + } while (true) { @@ -1308,7 +1370,22 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement checkFalse(clusteringPrefixSize > 0 && isDistinct, "Grouping on clustering columns is not allowed for SELECT DISTINCT queries"); - return AggregationSpecification.aggregatePkPrefix(metadata.comparator, clusteringPrefixSize); + return selectorFactory == null ? AggregationSpecification.aggregatePkPrefixFactory(metadata.comparator, clusteringPrefixSize) + : AggregationSpecification.aggregatePkPrefixFactoryWithSelector(metadata.comparator, + clusteringPrefixSize, + selectorFactory); + } + + /** + * Checks that the function used is a valid one for the GROUP BY clause. + * + * @param withFunction the {@code Selectable} from which the function must be retrieved. + * @return the monotonic scalar function that must be used for determining the groups. + */ + private void validateGroupByFunction(WithFunction withFunction) + { + Function f = withFunction.function; + checkFalse(f.isAggregate(), "Aggregate functions are not supported within the GROUP BY clause, got: %s", f.name()); } private Comparator> getOrderingComparator(Selection selection, @@ -1410,13 +1487,13 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement { // Public because CASSANDRA-9858 public final Map orderings; - public final List groups; + public final List groups; public final boolean isDistinct; public final boolean allowFiltering; public final boolean isJson; public Parameters(Map orderings, - List groups, + List groups, boolean isDistinct, boolean allowFiltering, boolean isJson) @@ -1618,7 +1695,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement sb.append(" AND ").append(filterString); } - DataLimits limits = getDataLimits(getLimit(options), getPerPartitionLimit(options), options.getPageSize()); + DataLimits limits = getDataLimits(getLimit(options), getPerPartitionLimit(options), options.getPageSize(), getAggregationSpec(options)); if (limits != DataLimits.NONE) sb.append(' ').append(limits); return sb.toString(); diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index 721aa81080..1efb086b40 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -1054,7 +1054,7 @@ public abstract class ReadCommand extends AbstractReadQuery int nowInSec = in.readInt(); ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, metadata); RowFilter rowFilter = RowFilter.serializer.deserialize(in, version, metadata); - DataLimits limits = DataLimits.serializer.deserialize(in, version, metadata.comparator); + DataLimits limits = DataLimits.serializer.deserialize(in, version, metadata); IndexMetadata index = hasIndex ? deserializeIndexMetadata(in, version, metadata) : null; return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index); diff --git a/src/java/org/apache/cassandra/db/aggregation/AggregationSpecification.java b/src/java/org/apache/cassandra/db/aggregation/AggregationSpecification.java index 7324dfdb66..0d6c0eef69 100644 --- a/src/java/org/apache/cassandra/db/aggregation/AggregationSpecification.java +++ b/src/java/org/apache/cassandra/db/aggregation/AggregationSpecification.java @@ -18,11 +18,16 @@ package org.apache.cassandra.db.aggregation; import java.io.IOException; +import java.util.List; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.cql3.selection.Selector; import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadata; /** * Defines how rows should be grouped for creating aggregates. @@ -43,6 +48,11 @@ public abstract class AggregationSpecification } }; + /** + * Factory for AggregationSpecification that group all the row together. + */ + public static final AggregationSpecification.Factory AGGREGATE_EVERYTHING_FACTORY = options -> AGGREGATE_EVERYTHING; + /** * The AggregationSpecification kind. */ @@ -53,7 +63,7 @@ public abstract class AggregationSpecification */ public static enum Kind { - AGGREGATE_EVERYTHING, AGGREGATE_BY_PK_PREFIX + AGGREGATE_EVERYTHING, AGGREGATE_BY_PK_PREFIX, AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR } /** @@ -89,37 +99,86 @@ public abstract class AggregationSpecification public abstract GroupMaker newGroupMaker(GroupingState state); /** - * Creates a new AggregationSpecification instance that will build aggregates based on primary key - * columns. + * Creates a new {@code Factory} instance to create {@code AggregationSpecification} that will build aggregates + * based on primary key columns. * * @param comparator the comparator used to compare the clustering prefixes * @param clusteringPrefixSize the number of clustering columns used to create the aggregates - * @return a new AggregationSpecification instance that will build aggregates based on primary key - * columns + * @return a new {@code Factory} instance to create {@code AggregationSpecification} that will build aggregates + * based on primary key columns. */ - public static AggregationSpecification aggregatePkPrefix(ClusteringComparator comparator, int clusteringPrefixSize) + public static AggregationSpecification.Factory aggregatePkPrefixFactory(ClusteringComparator comparator, + int clusteringPrefixSize) { - return new AggregateByPkPrefix(comparator, clusteringPrefixSize); + return options -> new AggregateByPkPrefix(comparator, clusteringPrefixSize); + } + + public static AggregationSpecification.Factory aggregatePkPrefixFactoryWithSelector(final ClusteringComparator comparator, + final int clusteringPrefixSize, + final Selector.Factory factory) + { + return new Factory() + { + @Override + public void addFunctionsTo(List functions) + { + factory.addFunctionsTo(functions); + } + + @Override + public AggregationSpecification newInstance(QueryOptions options) + { + Selector selector = factory.newInstance(options); + selector.validateForGroupBy(); + return new AggregateByPkPrefixWithSelector(comparator, + clusteringPrefixSize, + selector); + } + }; + } + + /** + * Factory for {@code AggregationSpecification}. + * + */ + public static interface Factory + { + /** + * Creates a new {@code AggregationSpecification} instance after having binded the parameters. + * + * @param options the query options + * @return a new {@code AggregationSpecification} instance. + */ + public AggregationSpecification newInstance(QueryOptions options); + + public default void addFunctionsTo(List functions) + { + } } /** * AggregationSpecification that build aggregates based on primary key columns */ - private static final class AggregateByPkPrefix extends AggregationSpecification + private static class AggregateByPkPrefix extends AggregationSpecification { /** * The number of clustering component to compare. */ - private final int clusteringPrefixSize; + protected final int clusteringPrefixSize; /** * The comparator used to compare the clustering prefixes. */ - private final ClusteringComparator comparator; + protected final ClusteringComparator comparator; public AggregateByPkPrefix(ClusteringComparator comparator, int clusteringPrefixSize) { - super(Kind.AGGREGATE_BY_PK_PREFIX); + this(Kind.AGGREGATE_BY_PK_PREFIX, comparator, clusteringPrefixSize); + } + + protected AggregateByPkPrefix(Kind kind, ClusteringComparator comparator, int clusteringPrefixSize) + { + super(kind); this.comparator = comparator; this.clusteringPrefixSize = clusteringPrefixSize; } @@ -127,7 +186,32 @@ public abstract class AggregationSpecification @Override public GroupMaker newGroupMaker(GroupingState state) { - return GroupMaker.newInstance(comparator, clusteringPrefixSize, state); + return GroupMaker.newPkPrefixGroupMaker(comparator, clusteringPrefixSize, state); + } + } + + /** + * AggregationSpecification that build aggregates based on primary key columns using a selector. + */ + private static final class AggregateByPkPrefixWithSelector extends AggregateByPkPrefix + { + /** + * The selector. + */ + private final Selector selector; + + public AggregateByPkPrefixWithSelector(ClusteringComparator comparator, + int clusteringPrefixSize, + Selector selector) + { + super(Kind.AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR, comparator, clusteringPrefixSize); + this.selector = selector; + } + + @Override + public GroupMaker newGroupMaker(GroupingState state) + { + return GroupMaker.newSelectorGroupMaker(comparator, clusteringPrefixSize, selector, state); } } @@ -143,12 +227,17 @@ public abstract class AggregationSpecification case AGGREGATE_BY_PK_PREFIX: out.writeUnsignedVInt(((AggregateByPkPrefix) aggregationSpec).clusteringPrefixSize); break; + case AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR: + AggregateByPkPrefixWithSelector spec = (AggregateByPkPrefixWithSelector) aggregationSpec; + out.writeUnsignedVInt(spec.clusteringPrefixSize); + Selector.serializer.serialize(spec.selector, out, version); + break; default: - throw new AssertionError(); + throw new AssertionError("Unknow aggregation kind: " + aggregationSpec.kind()); } } - public AggregationSpecification deserialize(DataInputPlus in, int version, ClusteringComparator comparator) throws IOException + public AggregationSpecification deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException { Kind kind = Kind.values()[in.readUnsignedByte()]; switch (kind) @@ -156,10 +245,15 @@ public abstract class AggregationSpecification case AGGREGATE_EVERYTHING: return AggregationSpecification.AGGREGATE_EVERYTHING; case AGGREGATE_BY_PK_PREFIX: + return new AggregateByPkPrefix(metadata.comparator, (int) in.readUnsignedVInt()); + case AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR: int clusteringPrefixSize = (int) in.readUnsignedVInt(); - return AggregationSpecification.aggregatePkPrefix(comparator, clusteringPrefixSize); + Selector selector = Selector.serializer.deserialize(in, version, metadata); + return new AggregateByPkPrefixWithSelector(metadata.comparator, + clusteringPrefixSize, + selector); default: - throw new AssertionError(); + throw new AssertionError("Unknow aggregation kind: " + kind); } } @@ -173,8 +267,15 @@ public abstract class AggregationSpecification case AGGREGATE_BY_PK_PREFIX: size += TypeSizes.sizeofUnsignedVInt(((AggregateByPkPrefix) aggregationSpec).clusteringPrefixSize); break; + case AGGREGATE_BY_PK_PREFIX_WITH_SELECTOR: + AggregateByPkPrefixWithSelector spec = (AggregateByPkPrefixWithSelector) aggregationSpec; + size += TypeSizes.sizeofUnsignedVInt(spec.clusteringPrefixSize); + size += Selector.serializer.serializedSize(spec.selector, version + + ); + break; default: - throw new AssertionError(); + throw new AssertionError("Unknow aggregation kind: " + aggregationSpec.kind()); } return size; } diff --git a/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java b/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java index fc62f211b3..968219f5ce 100644 --- a/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java +++ b/src/java/org/apache/cassandra/db/aggregation/GroupMaker.java @@ -19,9 +19,11 @@ package org.apache.cassandra.db.aggregation; import java.nio.ByteBuffer; +import org.apache.cassandra.cql3.selection.Selector; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.transport.ProtocolVersion; /** * A GroupMaker can be used to determine if some sorted rows belongs to the same group or not. @@ -44,16 +46,33 @@ public abstract class GroupMaker } }; - public static GroupMaker newInstance(ClusteringComparator comparator, int clusteringPrefixSize, GroupingState state) + public static GroupMaker newPkPrefixGroupMaker(ClusteringComparator comparator, + int clusteringPrefixSize, + GroupingState state) { return new PkPrefixGroupMaker(comparator, clusteringPrefixSize, state); } - public static GroupMaker newInstance(ClusteringComparator comparator, int clusteringPrefixSize) + public static GroupMaker newPkPrefixGroupMaker(ClusteringComparator comparator, int clusteringPrefixSize) { return new PkPrefixGroupMaker(comparator, clusteringPrefixSize); } + public static GroupMaker newSelectorGroupMaker(ClusteringComparator comparator, + int clusteringPrefixSize, + Selector selector, + GroupingState state) + { + return new SelectorGroupMaker(comparator, clusteringPrefixSize, selector, state); + } + + public static GroupMaker newSelectorGroupMaker(ClusteringComparator comparator, + int clusteringPrefixSize, + Selector selector) + { + return new SelectorGroupMaker(comparator, clusteringPrefixSize, selector); + } + /** * Checks if a given row belongs to the same group that the previous row or not. * @@ -75,27 +94,27 @@ public abstract class GroupMaker return false; } - private static final class PkPrefixGroupMaker extends GroupMaker + private static class PkPrefixGroupMaker extends GroupMaker { /** * The size of the clustering prefix used to make the groups */ - private final int clusteringPrefixSize; + protected final int clusteringPrefixSize; /** * The comparator used to compare the clustering prefixes. */ - private final ClusteringComparator comparator; + protected final ClusteringComparator comparator; /** * The last partition key seen */ - private ByteBuffer lastPartitionKey; + protected ByteBuffer lastPartitionKey; /** * The last clustering seen */ - private Clustering lastClustering; + protected Clustering lastClustering; public PkPrefixGroupMaker(ClusteringComparator comparator, int clusteringPrefixSize, GroupingState state) { @@ -113,28 +132,97 @@ public abstract class GroupMaker @Override public boolean isNewGroup(DecoratedKey partitionKey, Clustering clustering) { - boolean isNew = false; + ByteBuffer key = partitionKey.getKey(); + // We are entering a new group if: + // - the partition key is a new one + // - the last clustering was not null and does not have the same prefix as the new clustering one + boolean isNew = !key.equals(lastPartitionKey) + || lastClustering == null + || comparator.compare(lastClustering, clustering, clusteringPrefixSize) != 0; + + lastPartitionKey = key; + lastClustering = Clustering.STATIC_CLUSTERING == clustering ? null : clustering; + return isNew; + } + } + + private static class SelectorGroupMaker extends PkPrefixGroupMaker + { + /** + * The selector used to build the groups. + */ + private final Selector selector; + + /** + * The output of the selector call on the last clustering + */ + private ByteBuffer lastOutput; + + private final Selector.InputRow input = new Selector.InputRow(1, false, false); + + public SelectorGroupMaker(ClusteringComparator comparator, + int clusteringPrefixSize, + Selector selector, + GroupingState state) + { + super(comparator, clusteringPrefixSize, state); + this.selector = selector; + this.lastOutput = lastClustering == null ? null : + executeSelector(lastClustering.bufferAt(clusteringPrefixSize - 1)); + } + + public SelectorGroupMaker(ClusteringComparator comparator, + int clusteringPrefixSize, + Selector selector) + { + super(comparator, clusteringPrefixSize); + this.selector = selector; + } + + @Override + public boolean isNewGroup(DecoratedKey partitionKey, Clustering clustering) + { + ByteBuffer output = + Clustering.STATIC_CLUSTERING == clustering ? null + : executeSelector(clustering.bufferAt(clusteringPrefixSize - 1)); + + ByteBuffer key = partitionKey.getKey(); // We are entering a new group if: // - the partition key is a new one // - the last clustering was not null and does not have the same prefix as the new clustering one - if (!partitionKey.getKey().equals(lastPartitionKey)) - { - lastPartitionKey = partitionKey.getKey(); - isNew = true; - if (Clustering.STATIC_CLUSTERING == clustering) - { - lastClustering = null; - return true; - } - } - else if (lastClustering != null && comparator.compare(lastClustering, clustering, clusteringPrefixSize) != 0) - { - isNew = true; - } + boolean isNew = !key.equals(lastPartitionKey) + || lastClustering == null + || comparator.compare(lastClustering, clustering, clusteringPrefixSize - 1) != 0 + || compareOutput(output) != 0; - lastClustering = clustering; + lastPartitionKey = key; + lastClustering = Clustering.STATIC_CLUSTERING == clustering ? null : clustering; + lastOutput = output; return isNew; } + + private int compareOutput(ByteBuffer output) + { + if (output == null) + return lastOutput == null ? 0 : -1; + if (lastOutput == null) + return 1; + + return selector.getType().compare(output, lastOutput); + } + + private ByteBuffer executeSelector(ByteBuffer argument) + { + input.add(argument); + + // For computing groups we do not need to use the client protocol version. + selector.addInput(ProtocolVersion.CURRENT, input); + ByteBuffer output = selector.getOutput(ProtocolVersion.CURRENT); + selector.reset(); + input.reset(false); + + return output; + } } } diff --git a/src/java/org/apache/cassandra/db/filter/DataLimits.java b/src/java/org/apache/cassandra/db/filter/DataLimits.java index 845cffd445..f988cb3801 100644 --- a/src/java/org/apache/cassandra/db/filter/DataLimits.java +++ b/src/java/org/apache/cassandra/db/filter/DataLimits.java @@ -32,6 +32,7 @@ import org.apache.cassandra.db.transform.StoppingTransformation; import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; /** @@ -1175,7 +1176,7 @@ public abstract class DataLimits } } - public DataLimits deserialize(DataInputPlus in, int version, ClusteringComparator comparator) throws IOException + public DataLimits deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException { Kind kind = Kind.values()[in.readUnsignedByte()]; switch (kind) @@ -1199,9 +1200,9 @@ public abstract class DataLimits int groupPerPartitionLimit = (int) in.readUnsignedVInt(); int rowLimit = (int) in.readUnsignedVInt(); - AggregationSpecification groupBySpec = AggregationSpecification.serializer.deserialize(in, version, comparator); + AggregationSpecification groupBySpec = AggregationSpecification.serializer.deserialize(in, version, metadata); - GroupingState state = GroupingState.serializer.deserialize(in, version, comparator); + GroupingState state = GroupingState.serializer.deserialize(in, version, metadata.comparator); if (kind == Kind.CQL_GROUP_BY_LIMIT) return new CQLGroupByLimits(groupLimit, diff --git a/test/distributed/org/apache/cassandra/distributed/test/GroupByTest.java b/test/distributed/org/apache/cassandra/distributed/test/GroupByTest.java index d75b6bac2b..789321de3d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/GroupByTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/GroupByTest.java @@ -18,6 +18,7 @@ package org.apache.cassandra.distributed.test; +import java.util.Date; import java.util.Iterator; import com.google.common.collect.Iterators; @@ -31,7 +32,11 @@ import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.serializers.SimpleDateSerializer; +import org.apache.cassandra.serializers.TimeSerializer; +import org.apache.cassandra.serializers.TimestampSerializer; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL; import static org.apache.cassandra.distributed.api.Feature.NETWORK; import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; @@ -149,6 +154,183 @@ public class GroupByTest extends TestBaseImpl } } + @Test + public void testGroupByTimeRangesWithTimestampType() throws Throwable + { + try (Cluster cluster = init(builder().withNodes(3).start())) + { + cluster.schemaChange(withKeyspace("CREATE TABLE %s.testWithTimestamp (pk int, time timestamp, v int, primary key (pk, time))")); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:10:00 UTC', 1)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:12:00 UTC', 2)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:14:00 UTC', 3)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:15:00 UTC', 4)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:21:00 UTC', 5)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:22:00 UTC', 6)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:26:00 UTC', 7)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:26:20 UTC', 8)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (2, '2016-09-27 16:26:20 UTC', 10)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (2, '2016-09-27 16:30:00 UTC', 11)"), ConsistencyLevel.QUORUM); + + for (int pageSize : new int[] {2, 3, 4, 5, 7, 10}) + { + for (String startingTime : new String[] {"", ", '2016-09-27 UTC'"} ) + { + String stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp GROUP BY pk, floor(time, 5m" + startingTime + ")"; + Iterator pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L), + row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L)); + + stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp GROUP BY pk, floor(time, 5m" + startingTime + ") LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L)); + + stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp GROUP BY pk, floor(time, 5m" + startingTime + ") PER PARTITION LIMIT 1"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L)); + + stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L)); + + stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L)); + } + } + } + } + + @Test + public void testGroupByTimeRangesWithDateType() throws Throwable + { + try (Cluster cluster = init(builder().withNodes(3).start())) + { + + cluster.schemaChange(withKeyspace("CREATE TABLE %s.testWithDate (pk int, time date, v int, primary key (pk, time))")); + + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-27', 1)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-28', 2)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-29', 3)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-30', 4)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-10-01', 5)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-10-04', 6)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-10-20', 7)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-11-27', 8)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (2, '2016-11-01', 10)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (2, '2016-11-02', 11)"), ConsistencyLevel.QUORUM); + + for (int pageSize : new int[] {2, 3, 4, 5, 7, 10}) + { + for (String startingTime : new String[] {"", ", '2016-06-01'"} ) + { + + String stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate GROUP BY pk, floor(time, 1mo" + startingTime + ")"; + Iterator pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-01"), 1, 4, 4L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L), + row(1, toLocalDate("2016-11-01"), 8, 8, 1L), + row(2, toLocalDate("2016-11-01"), 10, 11, 2L)); + + stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate GROUP BY pk, floor(time, 1mo" + startingTime + ") LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-01"), 1, 4, 4L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L)); + + stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate GROUP BY pk, floor(time, 1mo" + startingTime + ") PER PARTITION LIMIT 1"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-01"), 1, 4, 4L), + row(2, toLocalDate("2016-11-01"), 10, 11, 2L)); + + stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-11-01"), 8, 8, 1L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L), + row(1, toLocalDate("2016-09-01"), 1, 4, 4L)); + + stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-11-01"), 8, 8, 1L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L)); + } + } + } + } + + @Test + public void testGroupByTimeRangesWithTimeType() throws Throwable + { + try (Cluster cluster = init(builder().withNodes(3).start())) + { + + cluster.schemaChange(withKeyspace("CREATE TABLE %s.testWithTime (pk int, date date, time time, v int, primary key (pk, date, time))")); + + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:10:00', 1)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:12:00', 2)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:14:00', 3)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:15:00', 4)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:21:00', 5)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:22:00', 6)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:00', 7)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:20', 8)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:20', 9)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:30', 10)"), ConsistencyLevel.QUORUM); + + for (int pageSize : new int[] {2, 3, 4, 5, 7, 10}) + { + + String stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime GROUP BY pk, date, floor(time, 5m)"; + Iterator pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), + row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), + row(1, toLocalDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), + row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L)); + + stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime GROUP BY pk, date, floor(time, 5m) LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), + row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L)); + + stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), + row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L)); + + stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L)); + } + } + } + private static void initFunctions(Cluster cluster) { cluster.schemaChange(withKeyspace("CREATE FUNCTION %s.concat_strings_fn(a text, b text) " + @@ -162,4 +344,20 @@ public class GroupByTest extends TestBaseImpl " STYPE text" + " INITCOND '_'")); } + + private static Date toTimestamp(String timestampAsString) + { + return new Date(TimestampSerializer.dateStringToTimestamp(timestampAsString)); + } + + private static int toLocalDate(String dateAsString) + { + return SimpleDateSerializer.dateStringToDays(dateAsString) ; + } + + private static long toTime(String timeAsString) + { + return TimeSerializer.timeStringToLong(timeAsString) ; + } + } \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/cql3/DurationTest.java b/test/unit/org/apache/cassandra/cql3/DurationTest.java index bfaa78004f..9b63242e56 100644 --- a/test/unit/org/apache/cassandra/cql3/DurationTest.java +++ b/test/unit/org/apache/cassandra/cql3/DurationTest.java @@ -18,20 +18,19 @@ */ package org.apache.cassandra.cql3; -import java.text.ParsePosition; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.TimeZone; - -import org.apache.commons.lang3.time.DateUtils; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.assertEquals; - import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.serializers.TimeSerializer; + +import static org.junit.Assert.assertEquals; import static org.apache.cassandra.cql3.Duration.*; @@ -109,72 +108,287 @@ public class DurationTest @Test public void testAddTo() { - assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("0m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("10us").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T00:10:00"), Duration.from("10m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T01:30:00"), Duration.from("90m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T02:10:00"), Duration.from("2h10m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-23T00:10:00"), Duration.from("2d10m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-24T01:00:00"), Duration.from("2d25h").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-10-21T00:00:00"), Duration.from("1mo").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2017-11-21T00:00:00"), Duration.from("14mo").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2017-02-28T00:00:00"), Duration.from("12mo").addTo(toMillis("2016-02-29T00:00:00"))); + assertTimeEquals("2016-09-21T00:00:00.000", Duration.from("0m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T00:00:00.000", Duration.from("10us").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T00:10:00.000", Duration.from("10m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T01:30:00.000", Duration.from("90m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T02:10:00.000", Duration.from("2h10m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-23T00:10:00.000", Duration.from("2d10m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-24T01:00:00.000", Duration.from("2d25h").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-10-21T00:00:00.000", Duration.from("1mo").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2017-11-21T00:00:00.000", Duration.from("14mo").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2017-02-28T00:00:00.000", Duration.from("12mo").addTo(toMillis("2016-02-29T00:00:00"))); } @Test public void testAddToWithNegativeDurations() { - assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("-0m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("-10us").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-20T23:50:00"), Duration.from("-10m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-20T22:30:00"), Duration.from("-90m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-20T21:50:00"), Duration.from("-2h10m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-18T23:50:00"), Duration.from("-2d10m").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-17T23:00:00"), Duration.from("-2d25h").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-08-21T00:00:00"), Duration.from("-1mo").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2015-07-21T00:00:00"), Duration.from("-14mo").addTo(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2015-02-28T00:00:00"), Duration.from("-12mo").addTo(toMillis("2016-02-29T00:00:00"))); + assertTimeEquals("2016-09-21T00:00:00.000", Duration.from("-0m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T00:00:00.000", Duration.from("-10us").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-20T23:50:00.000", Duration.from("-10m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-20T22:30:00.000", Duration.from("-90m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-20T21:50:00.000", Duration.from("-2h10m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-18T23:50:00.000", Duration.from("-2d10m").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-17T23:00:00.000", Duration.from("-2d25h").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-08-21T00:00:00.000", Duration.from("-1mo").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2015-07-21T00:00:00.000", Duration.from("-14mo").addTo(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2015-02-28T00:00:00.000", Duration.from("-12mo").addTo(toMillis("2016-02-29T00:00:00"))); } @Test public void testSubstractFrom() { - assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("0m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("10us").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-20T23:50:00"), Duration.from("10m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-20T22:30:00"), Duration.from("90m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-20T21:50:00"), Duration.from("2h10m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-18T23:50:00"), Duration.from("2d10m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-17T23:00:00"), Duration.from("2d25h").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-08-21T00:00:00"), Duration.from("1mo").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2015-07-21T00:00:00"), Duration.from("14mo").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2015-02-28T00:00:00"), Duration.from("12mo").substractFrom(toMillis("2016-02-29T00:00:00"))); + assertTimeEquals("2016-09-21T00:00:00.000", Duration.from("0m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T00:00:00.000", Duration.from("10us").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-20T23:50:00.000", Duration.from("10m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-20T22:30:00.000", Duration.from("90m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-20T21:50:00.000", Duration.from("2h10m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-18T23:50:00.000", Duration.from("2d10m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-17T23:00:00.000", Duration.from("2d25h").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-08-21T00:00:00.000", Duration.from("1mo").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2015-07-21T00:00:00.000", Duration.from("14mo").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2015-02-28T00:00:00.000", Duration.from("12mo").substractFrom(toMillis("2016-02-29T00:00:00"))); } @Test public void testSubstractWithNegativeDurations() { - assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("-0m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T00:00:00"), Duration.from("-10us").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T00:10:00"), Duration.from("-10m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T01:30:00"), Duration.from("-90m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-21T02:10:00"), Duration.from("-2h10m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-23T00:10:00"), Duration.from("-2d10m").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-09-24T01:00:00"), Duration.from("-2d25h").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2016-10-21T00:00:00"), Duration.from("-1mo").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2017-11-21T00:00:00"), Duration.from("-14mo").substractFrom(toMillis("2016-09-21T00:00:00"))); - assertEquals(toMillis("2017-02-28T00:00:00"), Duration.from("-12mo").substractFrom(toMillis("2016-02-29T00:00:00"))); + assertTimeEquals("2016-09-21T00:00:00.000", Duration.from("-0m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T00:00:00.000", Duration.from("-10us").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T00:10:00.000", Duration.from("-10m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T01:30:00.000", Duration.from("-90m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-21T02:10:00.000", Duration.from("-2h10m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-23T00:10:00.000", Duration.from("-2d10m").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-09-24T01:00:00.000", Duration.from("-2d25h").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2016-10-21T00:00:00.000", Duration.from("-1mo").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2017-11-21T00:00:00.000", Duration.from("-14mo").substractFrom(toMillis("2016-09-21T00:00:00"))); + assertTimeEquals("2017-02-28T00:00:00.000", Duration.from("-12mo").substractFrom(toMillis("2016-02-29T00:00:00"))); } - private long toMillis(String timeAsString) + @Test + public void testFloorTime() { - SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - parser.setTimeZone(TimeZone.getTimeZone("UTC")); - Date date = parser.parse(timeAsString, new ParsePosition(0)); - return DateUtils.truncate(date, Calendar.SECOND).getTime(); + long time = floorTime("16:12:00", "2h"); + Duration result = Duration.newInstance(0, 0, time); + Duration expected = Duration.from("16h"); + assertEquals(expected, result); } - public void assertInvalidDuration(String duration, String expectedErrorMessage) + @Test + public void testInvalidFloorTimestamp() + { + try + { + floorTimestamp("2016-09-27T16:12:00", "2h", "2017-09-01T00:00:00"); + Assert.fail(); + } + catch (InvalidRequestException e) + { + assertEquals("The floor function starting time is greater than the provided time", e.getMessage()); + } + + try + { + floorTimestamp("2016-09-27T16:12:00", "-2h", "2016-09-27T00:00:00"); + Assert.fail(); + } + catch (InvalidRequestException e) + { + assertEquals("Negative durations are not supported by the floor function", e.getMessage()); + } + } + + @Test + public void testFloorTimestampWithDurationInHours() + { + // Test floor with a timestamp equals to the start time + long result = floorTimestamp("2016-09-27T16:12:00", "2h", "2016-09-27T16:12:00"); + assertTimeEquals("2016-09-27T16:12:00.000", result); + + // Test floor with a duration equals to zero + result = floorTimestamp("2016-09-27T18:12:00", "0h", "2016-09-27T16:12:00"); + assertTimeEquals("2016-09-27T18:12:00.000", result); + + // Test floor with a timestamp exactly equals to the start time + (1 x duration) + result = floorTimestamp("2016-09-27T18:12:00", "2h", "2016-09-27T16:12:00"); + assertTimeEquals("2016-09-27T18:12:00.000", result); + + // Test floor with a timestamp in the first bucket + result = floorTimestamp("2016-09-27T16:13:00", "2h", "2016-09-27T16:12:00"); + assertTimeEquals("2016-09-27T16:12:00.000", result); + + // Test floor with a timestamp in another bucket + result = floorTimestamp("2016-09-27T16:12:00", "2h", "2016-09-27T00:00:00"); + assertTimeEquals("2016-09-27T16:00:00.000", result); + } + + @Test + public void testFloorTimestampWithDurationInDays() + { + // Test floor with a start time at the beginning of the month + long result = floorTimestamp("2016-09-27T16:12:00", "2d", "2016-09-01T00:00:00"); + assertTimeEquals("2016-09-27T00:00:00.000", result); + + // Test floor with a start time in the previous month + result = floorTimestamp("2016-09-27T16:12:00", "2d", "2016-08-01T00:00:00"); + assertTimeEquals("2016-09-26T00:00:00.000", result); + } + + @Test + public void testFloorTimestampWithDurationInDaysAndHours() + { + long result = floorTimestamp("2016-09-27T16:12:00", "2d12h", "2016-09-01T00:00:00"); + assertTimeEquals("2016-09-26T00:00:00.000", result); + } + + @Test + public void testFloorTimestampWithDurationInMonths() + { + // Test floor with a timestamp equals to the start time + long result = floorTimestamp("2016-09-01T00:00:00", "2mo", "2016-09-01T00:00:00"); + assertTimeEquals("2016-09-01T00:00:00.000", result); + + // Test floor with a timestamp in the first bucket + result = floorTimestamp("2016-09-27T16:12:00", "2mo", "2016-09-01T00:00:00"); + assertTimeEquals("2016-09-01T00:00:00.000", result); + + // Test floor with a start time at the beginning of the year (LEAP YEAR) + result = floorTimestamp("2016-09-27T16:12:00", "1mo", "2016-01-01T00:00:00"); + assertTimeEquals("2016-09-01T00:00:00.000", result); + + // Test floor with a start time at the beginning of the previous year + result = floorTimestamp("2016-09-27T16:12:00", "2mo", "2015-01-01T00:00:00"); + assertTimeEquals("2016-09-01T00:00:00.000", result); + + // Test floor with a start time in the previous year + result = floorTimestamp("2016-09-27T16:12:00", "2mo", "2015-02-02T00:00:00"); + assertTimeEquals("2016-08-02T00:00:00.000", result); + + } + + @Test + public void testFloorTimestampWithDurationInMonthsAndDays() + { + long result = floorTimestamp("2016-09-27T16:12:00", "2mo2d", "2016-01-01T00:00:00"); + assertTimeEquals("2016-09-09T00:00:00.000", result); + + result = floorTimestamp("2016-09-27T16:12:00", "2mo5d", "2016-01-01T00:00:00"); + assertTimeEquals("2016-09-21T00:00:00.000", result); + + // Test floor with a timestamp in the first bucket + result = floorTimestamp("2016-09-04T16:12:00", "2mo5d", "2016-07-01T00:00:00"); + assertTimeEquals("2016-07-01T00:00:00.000", result); + + // Test floor with a timestamp in a bucket starting on the last day of the month + result = floorTimestamp("2016-09-27T16:12:00", "2mo10d", "2016-01-01T00:00:00"); + assertTimeEquals("2016-07-31T00:00:00.000", result); + + // Test floor with a timestamp in a bucket starting on the first day of the month + result = floorTimestamp("2016-09-27T16:12:00", "2mo12d", "2016-01-01T00:00:00"); + assertTimeEquals("2016-08-06T00:00:00.000", result); + + // Test leap years + result = floorTimestamp("2016-04-27T16:12:00", "1mo30d", "2016-01-01T00:00:00"); + assertTimeEquals("2016-03-02T00:00:00.000", result); + + result = floorTimestamp("2015-04-27T16:12:00", "1mo30d", "2015-01-01T00:00:00"); + assertTimeEquals("2015-03-03T00:00:00.000", result); + } + + @Test + public void testFloorTimestampWithDurationSmallerThanPrecision() + { + long result = floorTimestamp("2016-09-27T18:14:00", "5us", "2016-09-27T16:12:00"); + assertTimeEquals("2016-09-27T18:14:00.000", result); + + result = floorTimestamp("2016-09-27T18:14:00", "1h5us", "2016-09-27T16:12:00"); + assertTimeEquals("2016-09-27T18:12:00.000", result); + } + + @Test + public void testFloorTimestampWithLeapSecond() + { + long result = floorTimestamp("2016-07-02T00:00:00", "2m", "2016-06-30T23:58:00"); + assertTimeEquals("2016-07-02T00:00:00.000", result); + } + + @Test + public void testFloorTimestampWithComplexDuration() + { + long result = floorTimestamp("2016-07-02T00:00:00", "2mo2d8h", "2016-01-01T00:00:00"); + assertTimeEquals("2016-05-05T16:00:00.000", result); + } + + @Test + public void testInvalidFloorTime() + { + try + { + floorTime("16:12:00", "2d"); + Assert.fail(); + } + catch (InvalidRequestException e) + { + assertEquals("For time values, the floor can only be computed for durations smaller that a day", e.getMessage()); + } + + try + { + floorTime("16:12:00", "25h"); + Assert.fail(); + } + catch (InvalidRequestException e) + { + assertEquals("For time values, the floor can only be computed for durations smaller that a day", e.getMessage()); + } + + try + { + floorTime("16:12:00", "-2h"); + Assert.fail(); + } + catch (InvalidRequestException e) + { + assertEquals("Negative durations are not supported by the floor function", e.getMessage()); + } + } + + private static long toMillis(String timeAsString) + { + OffsetDateTime dateTime = LocalDateTime.parse(timeAsString, DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")) + .atOffset(ZoneOffset.UTC); + + return Instant.from(dateTime).toEpochMilli(); + } + + private static String fromMillis(long timeInMillis) + { + return Instant.ofEpochMilli(timeInMillis) + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS")); + } + + private static void assertTimeEquals(String expected, long actualTimeInMillis) + { + assertEquals(expected, fromMillis(actualTimeInMillis)); + } + + private static long floorTimestamp(String time, String duration, String startingTime) + { + return Duration.floorTimestamp(toMillis(time), Duration.from(duration), toMillis(startingTime)); + } + + private static long floorTime(String time, String duration) + { + return Duration.floorTime(timeInNanos(time), Duration.from(duration)); + } + + private static long timeInNanos(String timeAsString) + { + return TimeSerializer.timeStringToLong(timeAsString); + } + + private static void assertInvalidDuration(String duration, String expectedErrorMessage) { try { diff --git a/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java b/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java new file mode 100644 index 0000000000..ee4dd356e1 --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/selection/SelectorSerializationTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cassandra.cql3.selection; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import org.apache.cassandra.cql3.*; +import org.apache.cassandra.cql3.Constants.Literal; +import org.apache.cassandra.cql3.functions.AggregateFcts; +import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.cql3.functions.TimeFcts; +import org.apache.cassandra.cql3.selection.Selectable.RawIdentifier; +import org.apache.cassandra.cql3.selection.Selector.Serializer; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Pair; + +import static java.util.Arrays.asList; + +import static org.junit.Assert.assertEquals; + +public class SelectorSerializationTest extends CQLTester +{ + @Test + public void testSerDes() throws IOException + { + createTable("CREATE TABLE %s (pk int, c1 int, c2 timestamp, v int, PRIMARY KEY(pk, c1, c2))"); + + KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(KEYSPACE); + TableMetadata table = keyspace.getTableOrViewNullable(currentTable()); + + // Test SimpleSelector serialization + checkSerialization(table.getColumn(new ColumnIdentifier("c1", false)), table); + + // Test WritetimeOrTTLSelector serialization + checkSerialization(new Selectable.WritetimeOrTTL(table.getColumn(new ColumnIdentifier("v", false)), true), table); + checkSerialization(new Selectable.WritetimeOrTTL(table.getColumn(new ColumnIdentifier("v", false)), false), table); + + // Test ListSelector serialization + checkSerialization(new Selectable.WithList(asList(table.getColumn(new ColumnIdentifier("v", false)), + table.getColumn(new ColumnIdentifier("c1", false)))), table); + + // Test SetSelector serialization + checkSerialization(new Selectable.WithSet(asList(table.getColumn(new ColumnIdentifier("v", false)), + table.getColumn(new ColumnIdentifier("c1", false)))), table); + + // Test MapSelector serialization + Pair pair = Pair.create(RawIdentifier.forUnquoted("v"), + RawIdentifier.forUnquoted("c1")); + checkSerialization(new Selectable.WithMapOrUdt(table, asList(pair)), table, MapType.getInstance(Int32Type.instance, Int32Type.instance, false)); + + // Test TupleSelector serialization + checkSerialization(new Selectable.BetweenParenthesesOrWithTuple(asList(table.getColumn(new ColumnIdentifier("c2", false)), + table.getColumn(new ColumnIdentifier("c1", false)))), table); + // Test TermSelector serialization + checkSerialization(new Selectable.WithTerm(Literal.duration("5m")), table, DurationType.instance); + + // Test UserTypeSelector serialization + String typeName = createType("CREATE TYPE %s (f1 int, f2 int)"); + + UserType type = new UserType(KEYSPACE, ByteBufferUtil.bytes(typeName), + asList(FieldIdentifier.forUnquoted("f1"), + FieldIdentifier.forUnquoted("f2")), + asList(Int32Type.instance, + Int32Type.instance), + false); + + List> list = asList(Pair.create(RawIdentifier.forUnquoted("f1"), + RawIdentifier.forUnquoted("c1")), + Pair.create(RawIdentifier.forUnquoted("f2"), + RawIdentifier.forUnquoted("pk"))); + + checkSerialization(new Selectable.WithMapOrUdt(table, list), table, type); + + // Test FieldSelector serialization + checkSerialization(new Selectable.WithFieldSelection(new Selectable.WithTypeHint(typeName, type, new Selectable.WithMapOrUdt(table, list)), FieldIdentifier.forUnquoted("f1")), table, type); + + // Test AggregateFunctionSelector serialization + Function max = AggregateFcts.makeMaxFunction(Int32Type.instance); + checkSerialization(new Selectable.WithFunction(max, asList(table.getColumn(new ColumnIdentifier("v", false)))), table); + + // Test SCalarFunctionSelector serialization + Function toDate = TimeFcts.toDate(TimestampType.instance); + checkSerialization(new Selectable.WithFunction(toDate, asList(table.getColumn(new ColumnIdentifier("c2", false)))), table); + + Function floor = TimeFcts.FloorTimestampFunction.newInstanceWithStartTimeArgument(); + checkSerialization(new Selectable.WithFunction(floor, asList(table.getColumn(new ColumnIdentifier("c2", false)), + new Selectable.WithTerm(Literal.duration("5m")), + new Selectable.WithTerm(Literal.string("2016-09-27 16:00:00 UTC")))), table); + } + + private static void checkSerialization(Selectable selectable, TableMetadata table) throws IOException + { + checkSerialization(selectable, table, null); + } + + private static void checkSerialization(Selectable selectable, TableMetadata table, AbstractType expectedType) throws IOException + { + int version = MessagingService.current_version; + + Serializer serializer = Selector.serializer; + Selector.Factory factory = selectable.newSelectorFactory(table, expectedType, new ArrayList<>(), VariableSpecifications.empty()); + Selector selector = factory.newInstance(QueryOptions.DEFAULT); + int size = serializer.serializedSize(selector, version); + DataOutputBuffer out = new DataOutputBuffer(size); + serializer.serialize(selector, out, version); + ByteBuffer buffer = out.asNewBuffer(); + DataInputBuffer in = new DataInputBuffer(buffer, false); + Selector deserialized = serializer.deserialize(in, version, table); + + assertEquals(selector, deserialized); + } +} diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java index 17d06ac20f..f46783b14b 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectGroupByTest.java @@ -19,7 +19,19 @@ package org.apache.cassandra.cql3.validation.operations; import org.junit.Test; +import com.datastax.driver.core.LocalDate; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.UUID; + import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.serializers.SimpleDateSerializer; +import org.apache.cassandra.serializers.TimeSerializer; +import org.apache.cassandra.serializers.TimestampSerializer; +import org.apache.cassandra.utils.TimeUUID; public class SelectGroupByTest extends CQLTester { @@ -2132,4 +2144,493 @@ public class SelectGroupByTest extends CQLTester row(1, 1, 4L, 3L)); } } + + @Test + public void testGroupByTimeRangesWithTimestamTypeAndWithoutPaging() throws Throwable + { + for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" }) + { + createTable("CREATE TABLE %s (pk int, time timestamp, v int, primary key (pk, time))" + compactOption); + + assertInvalidMessage("Group by currently only support groups of columns following their declared order in the PRIMARY KEY", + "SELECT pk, floor(time, 2h, '2016-09-01'), v FROM %s GROUP BY floor(time, 2h, '2016-09-01')"); + + assertInvalidMessage("Only monotonic functions are supported in the GROUP BY clause. Got: system.floor : (timestamp, duration(constant), timestamp) -> timestamp", + "SELECT pk, floor(time, 2h, time), v FROM %s GROUP BY pk, floor(time, 2h, time)"); + + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:10:00 UTC', 1)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:12:00 UTC', 2)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:14:00 UTC', 3)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:15:00 UTC', 4)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:21:00 UTC', 5)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:22:00 UTC', 6)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:26:00 UTC', 7)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:26:20 UTC', 8)"); + execute("INSERT INTO %s (pk, time, v) VALUES (2, '2016-09-27 16:26:20 UTC', 10)"); + execute("INSERT INTO %s (pk, time, v) VALUES (2, '2016-09-27 16:30:00 UTC', 11)"); + + // Test prepared statement + assertRows(execute("SELECT pk, floor(time, 5m, ?), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m, ?)", + toTimestamp("2016-09-27 00:00:00 UTC"), + toTimestamp("2016-09-27 00:00:00 UTC")), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L), + row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L)); + + for (String startingTime : new String[]{"", ", '2016-09-27 UTC'"}) + { + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ")"), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L), + row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L)); + + // Checks with duration lower than precisions + assertInvalidMessage("The floor cannot be computed for the 10us duration as precision is below 1 millisecond", + "SELECT pk, floor(time, 10us" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 10us" + startingTime + ")"); + + // Checks with a negative duration + assertInvalidMessage("Negative durations are not supported by the floor function", + "SELECT pk, floor(time, 10us" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, -5m" + startingTime + ")"); + + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ") LIMIT 2"), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L)); + + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ") PER PARTITION LIMIT 1"), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L)); + + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC"), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L)); + + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC LIMIT 2"), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L)); + } + + // Checks with start time is greater than the timestamp + assertInvalidMessage("The floor function starting time is greater than the provided time", + "SELECT pk, floor(time, 5m, '2016-10-27'), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m, '2016-10-27')"); + } + } + + @Test + public void testGroupByTimeRangesWithTimeUUIDAndWithoutPaging() throws Throwable + { + for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" }) + { + createTable("CREATE TABLE %s (pk int, time timeuuid, v int, primary key (pk, time))" + compactOption); + + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:10:00"), 1); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:12:00"), 2); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:14:00"), 3); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:15:00"), 4); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:21:00"), 5); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:22:00"), 6); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:26:00"), 7); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:26:20"), 8); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 2, toTimeUUID("2016-09-27 16:26:00"), 10); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 2, toTimeUUID("2016-09-27 16:30:00"), 11); + + for (String startingTime : new String[]{"", ", '2016-09-27 UTC'"}) + { + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ")"), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L), + row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L)); + + assertRows(execute("SELECT pk, floor(toTimestamp(time), 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(toTimestamp(time), 5m" + startingTime + ")"), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L), + row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L)); + + // Checks with duration lower than precisions + assertInvalidMessage("The floor cannot be computed for the 10us duration as precision is below 1 millisecond", + "SELECT pk, floor(time, 10us" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 10us" + startingTime + ")"); + + // Checks with a negative duration + assertInvalidMessage("Negative durations are not supported by the floor function", + "SELECT pk, floor(time, 10us" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, -5m" + startingTime + ")"); + + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ") LIMIT 2"), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L)); + + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ") PER PARTITION LIMIT 1"), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L)); + + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC"), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L)); + + assertRows(execute("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC LIMIT 2"), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L)); + } + + // Checks with start time is greater than the timestamp + assertInvalidMessage("The floor function starting time is greater than the provided time", + "SELECT pk, floor(time, 5m, '2016-10-27'), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m, '2016-10-27')"); + } + } + + @Test + public void testGroupByTimeRangesWithDateTypeAndWithoutPaging() throws Throwable + { + for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" }) + { + createTable("CREATE TABLE %s (pk int, time date, v int, primary key (pk, time))" + compactOption); + + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27', 1)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-28', 2)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-29', 3)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-30', 4)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-10-01', 5)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-10-04', 6)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-10-20', 7)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-11-27', 8)"); + execute("INSERT INTO %s (pk, time, v) VALUES (2, '2016-11-01', 10)"); + execute("INSERT INTO %s (pk, time, v) VALUES (2, '2016-11-02', 11)"); + + for (String startingTime : new String[]{"", ", '2016-06-01'"}) + { + assertRows(execute("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 1mo" + startingTime + ")"), + row(1, toDate("2016-09-01"), 1, 4, 4L), + row(1, toDate("2016-10-01"), 5, 7, 3L), + row(1, toDate("2016-11-01"), 8, 8, 1L), + row(2, toDate("2016-11-01"), 10, 11, 2L)); + + // Checks with duration lower than precisions + assertInvalidMessage("The floor on date values cannot be computed for the 1h duration as precision is below 1 day", + "SELECT pk, floor(time, 1h" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 1h" + startingTime + ")"); + + // Checks with a negative duration + assertInvalidMessage("Negative durations are not supported by the floor function", + "SELECT pk, floor(time, -1mo" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, -1mo" + startingTime + ")"); + + assertRows(execute("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 1mo" + startingTime + ") LIMIT 2"), + row(1, toDate("2016-09-01"), 1, 4, 4L), + row(1, toDate("2016-10-01"), 5, 7, 3L)); + + assertRows(execute("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 1mo" + startingTime + ") PER PARTITION LIMIT 1"), + row(1, toDate("2016-09-01"), 1, 4, 4L), + row(2, toDate("2016-11-01"), 10, 11, 2L)); + + assertRows(execute("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC"), + row(1, toDate("2016-11-01"), 8, 8, 1L), + row(1, toDate("2016-10-01"), 5, 7, 3L), + row(1, toDate("2016-09-01"), 1, 4, 4L)); + + assertRows(execute("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC LIMIT 2"), + row(1, toDate("2016-11-01"), 8, 8, 1L), + row(1, toDate("2016-10-01"), 5, 7, 3L)); + } + + // Checks with start time is greater than the timestamp + assertInvalidMessage("The floor function starting time is greater than the provided time", + "SELECT pk, floor(time, 1mo, '2017-01-01'), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 1mo, '2017-01-01')"); + } + } + + @Test + public void testGroupByTimeRangesWithTimeTypeAndWithoutPaging() throws Throwable + { + for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" }) + { + createTable("CREATE TABLE %s (pk int, date date, time time, v int, primary key (pk, date, time))" + compactOption); + + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:10:00', 1)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:12:00', 2)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:14:00', 3)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:15:00', 4)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:21:00', 5)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:22:00', 6)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:00', 7)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:20', 8)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:20', 9)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:30', 10)"); + + assertInvalidMessage("Functions are only supported on the last element of the GROUP BY clause", + "SELECT pk, floor(date, 1w), time, min(v), max(v), count(v) FROM %s GROUP BY pk, floor(date, 1w), time"); + + assertRows(execute("SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s GROUP BY pk, date, floor(time, 5m)"), + row(1, toDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), + row(1, toDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), + row(1, toDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), + row(1, toDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), + row(1, toDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L)); + + // Checks with duration greater than dayprecisions + assertInvalidMessage("For time values, the floor can only be computed for durations smaller that a day", + "SELECT pk, date, floor(time, 10d), min(v), max(v), count(v) FROM %s GROUP BY pk, date, floor(time, 10d)"); + + // Checks with a negative duration + assertInvalidMessage("Negative durations are not supported by the floor function", + "SELECT pk, date, floor(time, -10m), min(v), max(v), count(v) FROM %s GROUP BY pk, date, floor(time, -10m)"); + + assertRows(execute("SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s GROUP BY pk, date, floor(time, 5m) LIMIT 2"), + row(1, toDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), + row(1, toDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L)); + + assertRows(execute("SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC"), + row(1, toDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), + row(1, toDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), + row(1, toDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), + row(1, toDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), + row(1, toDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L)); + + assertRows(execute("SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC LIMIT 2"), + row(1, toDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), + row(1, toDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L)); + } + } + + @Test + public void testGroupByTimeRangesWithTimestampTypeAndPaging() throws Throwable + { + for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" }) + { + createTable("CREATE TABLE %s (pk int, time timestamp, v int, primary key (pk, time))" + + compactOption); + + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:10:00 UTC', 1)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:12:00 UTC', 2)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:14:00 UTC', 3)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:15:00 UTC', 4)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:21:00 UTC', 5)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:22:00 UTC', 6)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:26:00 UTC', 7)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27 16:26:20 UTC', 8)"); + execute("INSERT INTO %s (pk, time, v) VALUES (2, '2016-09-27 16:26:20 UTC', 10)"); + execute("INSERT INTO %s (pk, time, v) VALUES (2, '2016-09-27 16:30:00 UTC', 11)"); + + for (int pageSize = 1; pageSize < 10; pageSize++) + { + for (String startingTime : new String[]{"", ", '2016-09-27 UTC'"}) + { + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ")", pageSize), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L), + row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ") LIMIT 2", pageSize), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ") PER PARTITION LIMIT 1", pageSize), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC", pageSize), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC LIMIT 2", pageSize), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L)); + } + } + } + } + + @Test + public void testGroupByTimeRangesWithTimeUUIDAndPaging() throws Throwable + { + for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" }) + { + createTable("CREATE TABLE %s (pk int, time timeuuid, v int, primary key (pk, time))" + compactOption); + + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:10:00"), 1); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:12:00"), 2); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:14:00"), 3); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:15:00"), 4); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:21:00"), 5); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:22:00"), 6); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:26:00"), 7); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 1, toTimeUUID("2016-09-27 16:26:20"), 8); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 2, toTimeUUID("2016-09-27 16:26:00"), 10); + execute("INSERT INTO %s (pk, time, v) VALUES (?, ?, ?)", 2, toTimeUUID("2016-09-27 16:30:00"), 11); + + for (int pageSize = 1; pageSize < 10; pageSize++) + { + for (String startingTime : new String[]{"", ", '2016-09-27 UTC'"}) + { + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ")", pageSize), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L), + row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ") LIMIT 2", pageSize), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 5m" + startingTime + ") PER PARTITION LIMIT 1", pageSize), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC", pageSize), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC LIMIT 2", pageSize), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L)); + } + } + } + } + + @Test + public void testGroupByTimeRangesWithDateTypeAndPaging() throws Throwable + { + for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" }) + { + createTable("CREATE TABLE %s (pk int, time date, v int, primary key (pk, time))" + compactOption); + + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-27', 1)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-28', 2)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-29', 3)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-09-30', 4)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-10-01', 5)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-10-04', 6)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-10-20', 7)"); + execute("INSERT INTO %s (pk, time, v) VALUES (1, '2016-11-27', 8)"); + execute("INSERT INTO %s (pk, time, v) VALUES (2, '2016-11-01', 10)"); + execute("INSERT INTO %s (pk, time, v) VALUES (2, '2016-11-02', 11)"); + + for (int pageSize = 1; pageSize < 10; pageSize++) + { + for (String startingTime : new String[]{"", ", '2016-06-01'"}) + { + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 1mo" + startingTime + ")", pageSize), + row(1, toLocalDate("2016-09-01"), 1, 4, 4L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L), + row(1, toLocalDate("2016-11-01"), 8, 8, 1L), + row(2, toLocalDate("2016-11-01"), 10, 11, 2L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 1mo" + startingTime + ") LIMIT 2", pageSize), + row(1, toLocalDate("2016-09-01"), 1, 4, 4L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s GROUP BY pk, floor(time, 1mo" + startingTime + ") PER PARTITION LIMIT 1", pageSize), + row(1, toLocalDate("2016-09-01"), 1, 4, 4L), + row(2, toLocalDate("2016-11-01"), 10, 11, 2L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC", pageSize), + row(1, toLocalDate("2016-11-01"), 8, 8, 1L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L), + row(1, toLocalDate("2016-09-01"), 1, 4, 4L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC LIMIT 2", pageSize), + row(1, toLocalDate("2016-11-01"), 8, 8, 1L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L)); + } + } + } + } + + @Test + public void testGroupByTimeRangesWithTimeTypeAndPaging() throws Throwable + { + for (String compactOption : new String[] { "", " WITH COMPACT STORAGE" }) + { + createTable("CREATE TABLE %s (pk int, date date, time time, v int, primary key (pk, date, time))" + compactOption); + + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:10:00', 1)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:12:00', 2)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:14:00', 3)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:15:00', 4)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:21:00', 5)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:22:00', 6)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:00', 7)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:20', 8)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:20', 9)"); + execute("INSERT INTO %s (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:30', 10)"); + + for (int pageSize = 1; pageSize < 10; pageSize++) + { + assertRowsNet(executeNetWithPaging("SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s GROUP BY pk, date, floor(time, 5m)", pageSize), + row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), + row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), + row(1, toLocalDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), + row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s GROUP BY pk, date, floor(time, 5m) LIMIT 2", pageSize), + row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), + row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC", pageSize), + row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), + row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L)); + + assertRowsNet(executeNetWithPaging("SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC LIMIT 2", pageSize), + row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L)); + } + } + } + + private static UUID toTimeUUID(String string) + { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + LocalDateTime dateTime = LocalDateTime.parse(string, formatter); + long timeInMillis = dateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); + return TimeUUID.Generator.atUnixMillis(timeInMillis).asUUID(); + } + + private static Date toTimestamp(String timestampAsString) + { + return new Date(TimestampSerializer.dateStringToTimestamp(timestampAsString)); + } + + private static int toDate(String dateAsString) + { + return SimpleDateSerializer.dateStringToDays(dateAsString); + } + + private static LocalDate toLocalDate(String dateAsString) + { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + LocalDateTime dateTime = java.time.LocalDate.parse(dateAsString, formatter).atStartOfDay(); + long timeInMillis = dateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); + + return LocalDate.fromMillisSinceEpoch(timeInMillis); + } + + private static long toTime(String timeAsString) + { + return TimeSerializer.timeStringToLong(timeAsString); + } } diff --git a/test/unit/org/apache/cassandra/db/aggregation/GroupMakerTest.java b/test/unit/org/apache/cassandra/db/aggregation/GroupMakerTest.java index 8ff262d5e0..13fb0df230 100644 --- a/test/unit/org/apache/cassandra/db/aggregation/GroupMakerTest.java +++ b/test/unit/org/apache/cassandra/db/aggregation/GroupMakerTest.java @@ -18,17 +18,30 @@ package org.apache.cassandra.db.aggregation; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.Constants.Literal; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.VariableSpecifications; +import org.apache.cassandra.cql3.functions.ScalarFunction; +import org.apache.cassandra.cql3.functions.TimeFcts; +import org.apache.cassandra.cql3.selection.Selectable; +import org.apache.cassandra.cql3.selection.Selector; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.ReversedType; +import org.apache.cassandra.db.marshal.TimestampType; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -45,7 +58,7 @@ public class GroupMakerTest public void testIsNewGroupWithClusteringColumns() { ClusteringComparator comparator = newComparator(false, false, false); - GroupMaker groupMaker = GroupMaker.newInstance(comparator, 2); + GroupMaker groupMaker = GroupMaker.newPkPrefixGroupMaker(comparator, 2); assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(1, 1, 1))); assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering(1, 1, 2))); @@ -63,7 +76,7 @@ public class GroupMakerTest public void testIsNewGroupWithOneClusteringColumnsPrefix() { ClusteringComparator comparator = newComparator(false, false, false); - GroupMaker groupMaker = GroupMaker.newInstance(comparator, 1); + GroupMaker groupMaker = GroupMaker.newPkPrefixGroupMaker(comparator, 1); assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(1, 1, 1))); assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering(1, 1, 2))); @@ -82,7 +95,7 @@ public class GroupMakerTest { ClusteringComparator comparator = newComparator(true, true, true); - GroupMaker groupMaker = GroupMaker.newInstance(comparator, 2); + GroupMaker groupMaker = GroupMaker.newPkPrefixGroupMaker(comparator, 2); assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(1, 3, 2))); assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering(1, 3, 1))); @@ -102,7 +115,7 @@ public class GroupMakerTest { ClusteringComparator comparator = newComparator(true, false, false); - GroupMaker groupMaker = GroupMaker.newInstance(comparator, 2); + GroupMaker groupMaker = GroupMaker.newPkPrefixGroupMaker(comparator, 2); assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(1, 3, 1))); assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering(1, 3, 2))); @@ -121,7 +134,7 @@ public class GroupMakerTest public void testIsNewGroupWithStaticClusteringColumns() { ClusteringComparator comparator = newComparator(false, false, false); - GroupMaker groupMaker = GroupMaker.newInstance(comparator, 2); + GroupMaker groupMaker = GroupMaker.newPkPrefixGroupMaker(comparator, 2); assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(1, 1, 1))); assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering(1, 1, 2))); @@ -135,7 +148,7 @@ public class GroupMakerTest public void testIsNewGroupWithOnlyPartitionKeyComponents() { ClusteringComparator comparator = newComparator(false, false, false); - GroupMaker goupMaker = GroupMaker.newInstance(comparator, 2); + GroupMaker goupMaker = GroupMaker.newPkPrefixGroupMaker(comparator, 2); assertTrue(goupMaker.isNewGroup(partitionKey(1, 1), clustering(1, 1, 1))); assertFalse(goupMaker.isNewGroup(partitionKey(1, 1), clustering(1, 1, 2))); @@ -146,6 +159,100 @@ public class GroupMakerTest assertTrue(goupMaker.isNewGroup(partitionKey(2, 2), clustering(1, 1, 2))); } + @Test + public void testIsNewGroupWithFunction() + { + GroupMaker groupMaker = newSelectorGroupMaker(false); + + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:10:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:12:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:14:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:15:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:21:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:22:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:26:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:26:20 UTC"))); + + assertTrue(groupMaker.isNewGroup(partitionKey(2), clustering("2016-09-27 16:26:20 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(2), clustering("2016-09-27 16:30:00 UTC"))); + } + + @Test + public void testIsNewGroupWithFunctionAndReversedOrder() + { + GroupMaker groupMaker = newSelectorGroupMaker(true); + + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:26:20 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:26:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:22:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:21:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:15:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:14:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:12:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:10:00 UTC"))); + + assertTrue(groupMaker.isNewGroup(partitionKey(2), clustering("2016-09-27 16:30:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(2), clustering("2016-09-27 16:26:20 UTC"))); + } + + @Test + public void testIsNewGroupWithFunctionWithStaticColumn() + { + GroupMaker groupMaker = newSelectorGroupMaker(false); + + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:10:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:12:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:14:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:15:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:21:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:22:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:26:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:26:20 UTC"))); + + assertTrue(groupMaker.isNewGroup(partitionKey(2), Clustering.STATIC_CLUSTERING)); + assertTrue(groupMaker.isNewGroup(partitionKey(3), Clustering.STATIC_CLUSTERING)); + assertTrue(groupMaker.isNewGroup(partitionKey(4), clustering("2016-09-27 16:26:20 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(4), clustering("2016-09-27 16:30:00 UTC"))); + } + + @Test + public void testIsNewGroupWithFunctionAndReversedOrderWithStaticColumns() + { + GroupMaker groupMaker = newSelectorGroupMaker(true); + + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:26:20 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:26:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:22:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:21:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:15:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:14:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:12:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering("2016-09-27 16:10:00 UTC"))); + + assertTrue(groupMaker.isNewGroup(partitionKey(2), Clustering.STATIC_CLUSTERING)); + assertTrue(groupMaker.isNewGroup(partitionKey(3), Clustering.STATIC_CLUSTERING)); + assertTrue(groupMaker.isNewGroup(partitionKey(4), clustering("2016-09-27 16:30:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(4), clustering("2016-09-27 16:26:20 UTC"))); + } + + @Test + public void testIsNewGroupWithPrefixAndFunction() + { + GroupMaker groupMaker = newSelectorGroupMaker(false, false); + + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(1, "2016-09-27 16:10:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering(1, "2016-09-27 16:12:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering(1, "2016-09-27 16:14:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(1, "2016-09-27 16:15:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(2, "2016-09-27 16:16:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(2, "2016-09-27 16:22:00 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(1), clustering(2, "2016-09-27 16:26:00 UTC"))); + assertFalse(groupMaker.isNewGroup(partitionKey(1), clustering(2, "2016-09-27 16:26:20 UTC"))); + + assertTrue(groupMaker.isNewGroup(partitionKey(2), clustering(1, "2016-09-27 16:26:20 UTC"))); + assertTrue(groupMaker.isNewGroup(partitionKey(2), clustering(1, "2016-09-27 16:30:00 UTC"))); + } + private static DecoratedKey partitionKey(int... components) { ByteBuffer buffer = ByteBuffer.allocate(components.length * 4); @@ -162,6 +269,19 @@ public class GroupMakerTest return Clustering.make(toByteBufferArray(components)); } + private static Clustering clustering(String timeComponent) + { + ByteBuffer buffer = TimestampType.instance.fromString(timeComponent); + return Clustering.make(buffer); + } + + private static Clustering clustering(int component, String timeComponent) + { + ByteBuffer first = Int32Type.instance.decompose(component); + ByteBuffer second = TimestampType.instance.fromString(timeComponent); + return Clustering.make(first, second); + } + private static ByteBuffer[] toByteBufferArray(int[] values) { ByteBuffer[] buffers = new ByteBuffer[values.length]; @@ -182,4 +302,31 @@ public class GroupMakerTest return new ClusteringComparator(types); } + + private GroupMaker newSelectorGroupMaker(boolean... reversed) + { + TableMetadata.Builder builder = TableMetadata.builder("keyspace", "test") + .addPartitionKeyColumn("partition_key", Int32Type.instance); + + int last = reversed.length - 1; + for (int i = 0; i < reversed.length; i++) + { + AbstractType type = i == last ? TimestampType.instance : Int32Type.instance; + builder.addClusteringColumn("clustering" + i, reversed[i] ? ReversedType.getInstance(type) : type); + } + + TableMetadata table = builder.build(); + + ColumnMetadata column = table.getColumn(new ColumnIdentifier("clustering" + last, false)); + + Selectable.WithTerm duration = new Selectable.WithTerm(Literal.duration("5m")); + Selectable.WithTerm startTime = new Selectable.WithTerm(Literal.string("2016-09-27 16:00:00 UTC")); + ScalarFunction function = TimeFcts.FloorTimestampFunction.newInstanceWithStartTimeArgument(); + + Selectable.WithFunction selectable = new Selectable.WithFunction(function, Arrays.asList(column, duration, startTime)); + Selector.Factory factory = selectable.newSelectorFactory(table, null, new ArrayList<>(), VariableSpecifications.empty()); + Selector selector = factory.newInstance(QueryOptions.DEFAULT); + + return GroupMaker.newSelectorGroupMaker(table.comparator, reversed.length, selector); + } }