diff --git a/CHANGES.txt b/CHANGES.txt index 914523113c..a3c113321d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.1 + * Support octet_length and length functions (CASSANDRA-20102) * Make JsonUtils serialize Instant always with the same format (CASSANDRA-20209) * Port Harry v2 to trunk (CASSANDRA-20200) * Enable filtering of snapshots on keyspace, table and snapshot name in nodetool listsnapshots (CASSANDRA-20151) diff --git a/NEWS.txt b/NEWS.txt index a4cec79660..530acc0bd6 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -109,6 +109,10 @@ New features Enabling both ways of configuring JMX will result in a node failing to start. - CEP-43 - it is possible to create a table by "copying" as `CREATE TABLE ks.tb_copy LIKE ks.tb;`. A newly created table will have no data. + - New functions `octet_length` and `length` were introduced. Previously, users would either have to read + the data and check the length themselves or enable UDF and register UDFs to check the length of columns. + CASSANDRA-20102 adds a subset of the SQL99 (binary) string functions: "octet_length" defined on all types + and "length" defined on UTF8 strings. See CASSANDRA-20102 for more information. Upgrading --------- diff --git a/doc/modules/cassandra/pages/developing/cql/functions.adoc b/doc/modules/cassandra/pages/developing/cql/functions.adoc index 75786de271..2344cdb728 100644 --- a/doc/modules/cassandra/pages/developing/cql/functions.adoc +++ b/doc/modules/cassandra/pages/developing/cql/functions.adoc @@ -231,6 +231,54 @@ For every xref:cassandra:developing/cql/types.adoc#native-types[type] supported Conversely, the function `blob_as_type` takes a 64-bit `blob` argument and converts it to a `bigint` value. For example, `bigint_as_blob(3)` returns `0x0000000000000003` and `blob_as_bigint(0x0000000000000003)` returns `3`. +==== Length Functions +CQL supports two functions to retrieve the length of data without returning the data itself. Note that while +network bandwidth is reduced and memory is freed earlier, these functions still require data to be read from disk. + +[cols=",,",options="header",] +|=== +|Function name |Input type |Description + +| `octet_length` | All xref:cassandra:developing/cql/types.adoc#native-types[types] | Returns the length of the data in bytes. + +| `length` | `text` | Returns the string's length - the count of UTF-8 code units. +|=== + +The `octet_length` function is defined for every xref:cassandra:developing/cql/types.adoc#native-types[type] +supported by CQL, and represents the number of bytes of the underlying `ByteBuffer` representation, not accounting for +metadata overhead. Equivalent to Java's +https://docs.oracle.com/en/java/javase/22/docs/api/java.base/java/nio/Buffer.html#remaining()[`ByteBuffer#remaining()`] +when data has just been read, which is also the `length` of the type when encoded to `byte[]`. Some examples +for common types: + +[cols=",,",options="header",] +|=== +|CQL Type | `octet_length` |Description + +| `tinyint` | `1` | 8-bit signed integer + +| `smallint` | `2` | 16-bit signed integer + +| `int` | `4` | 32-bit signed integer + +| `bigint` | `8` | 64-bit signed integer + +| `float` | `4` | 32-bit IEEE-754 floating point + +| `double` | `8` | 64-bit IEEE-754 floating point + +| `blob` | Number of bytes in blob | Variable length byte string + +| `text` | Number of bytes in `text_as_blob` representation | Variable length character string +|=== + +The `length` function is only defined for UTF-8 strings (`text`) and returns the length of that string, meaning the +number of UTF-8 code units. Equivalent to Java's +https://docs.oracle.com/en/java/javase/22/docs/api/java.base/java/lang/String.html#length()[`String#length()`] or +Python 3's `len` function when applied to a string. + +These length functions return `null` when the input is `null`. + ==== Math Functions Cql provides the following math functions: `abs`, `exp`, `log`, `log10`, and `round`. diff --git a/src/java/org/apache/cassandra/cql3/functions/LengthFcts.java b/src/java/org/apache/cassandra/cql3/functions/LengthFcts.java new file mode 100644 index 0000000000..64cb940606 --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/functions/LengthFcts.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3.functions; + +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Set; + +import org.apache.cassandra.cql3.CQL3Type; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * Lengths of stored data without returning the data. + */ +public class LengthFcts +{ + public static void addFunctionsTo(NativeFunctions functions) + { + // As all types ultimately end up as bytebuffers they should all be compatible with + // octet_length + Set> types = new HashSet<>(); + for (CQL3Type type : CQL3Type.Native.values()) + { + AbstractType udfType = type.getType().udfType(); + if (!types.add(udfType)) + continue; + functions.add(makeOctetLengthFunction(type.getType().udfType())); + } + + // Special handling for string length which is number of UTF-8 code units + functions.add(length); + } + + + public static NativeFunction makeOctetLengthFunction(AbstractType fromType) + { + // Matches SQL99 OCTET_LENGTH functions defined on bytestring and char strings + return new NativeScalarFunction("octet_length", Int32Type.instance, fromType) + { + // Do not deserialize + @Override + public Arguments newArguments(ProtocolVersion version) + { + return FunctionArguments.newNoopInstance(version, 1); + } + + @Override + public ByteBuffer execute(Arguments arguments) throws InvalidRequestException + { + if (arguments.get(0) == null) + return null; + + final ByteBuffer buffer = arguments.get(0); + return ByteBufferUtil.bytes(buffer.remaining()); + } + }; + } + + // Matches PostgreSQL length function defined as returning the number of UTF-8 code units in the text string + public static final NativeFunction length = new NativeScalarFunction("length", Int32Type.instance, UTF8Type.instance) + { + @Override + public ByteBuffer execute(Arguments arguments) throws InvalidRequestException + { + if (arguments.get(0) == null) + return null; + + final String value = arguments.get(0); + return ByteBufferUtil.bytes(value.length()); + } + }; +} diff --git a/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java b/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java index a9deba5cb1..e521c5e807 100644 --- a/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java +++ b/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java @@ -44,6 +44,7 @@ public class NativeFunctions AggregateFcts.addFunctionsTo(this); CollectionFcts.addFunctionsTo(this); BytesConversionFcts.addFunctionsTo(this); + LengthFcts.addFunctionsTo(this); MathFcts.addFunctionsTo(this); MaskingFcts.addFunctionsTo(this); VectorFcts.addFunctionsTo(this); diff --git a/test/unit/org/apache/cassandra/cql3/functions/LengthFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/LengthFctsTest.java new file mode 100644 index 0000000000..b1f1354bc2 --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/functions/LengthFctsTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3.functions; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.quicktheories.WithQuickTheories; + +public class LengthFctsTest extends CQLTester implements WithQuickTheories +{ + @Test + public void testOctetLengthNonStrings() + { + createTable("CREATE TABLE %s (a tinyint primary key," + + " b smallint," + + " c int," + + " d bigint," + + " e float," + + " f double," + + " g decimal," + + " h varint," + + " i int)"); + + execute("INSERT INTO %s (a, b, c, d, e, f, g, h) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (byte) 1, (short) 2, 3, 4L, 5.2F, 6.3, BigDecimal.valueOf(6.3), BigInteger.valueOf(4)); + + assertRows(execute("SELECT OCTET_LENGTH(a), " + + "OCTET_LENGTH(b), " + + "OCTET_LENGTH(c), " + + "OCTET_LENGTH(d), " + + "OCTET_LENGTH(e), " + + "OCTET_LENGTH(f), " + + "OCTET_LENGTH(g), " + + "OCTET_LENGTH(h), " + + "OCTET_LENGTH(i) FROM %s"), + row(1, 2, 4, 8, 4, 8, 5, 1, null)); + } + + @Test + public void testStringLengthUTF8() throws Throwable + { + createTable("CREATE TABLE %s (key text primary key, value blob)"); + // UTF-8 7 codepoint, 21 byte encoded string + String key = "こんにちは世界"; + execute("INSERT INTO %s (key) VALUES (?)", key); + + assertRows(execute("SELECT LENGTH(key), OCTET_LENGTH(key), OCTET_LENGTH(value) FROM %s where key = ?", key), + row(7, 21, null)); + + // Quickly check that multiple arguments leads to an exception as expected + assertInvalidMessage("Invalid number of arguments in call to function system.length", + "SELECT LENGTH(key, value) FROM %s where key = 'こんにちは世界'"); + assertInvalidMessage("Invalid call to function octet_length, none of its type signatures match", + "SELECT OCTET_LENGTH(key, value) FROM %s where key = 'こんにちは世界'"); + } + + @Test + public void testOctetLengthStringFuzz() + { + createTable("CREATE TABLE %s (key text primary key, value blob)"); + + qt().withExamples(1024).forAll(strings().allPossible().ofLengthBetween(32, 100)).checkAssert( + (randString) -> { + int sLen = randString.length(); + byte[] randBytes = randString.getBytes(StandardCharsets.UTF_8); + + // UTF-8 length (code unit count) and byte length are often + // different. Spot checked a few of these, and they are different + // most of the time in this test - but testing that reproducibly + // requires seeding that would decrease the test power... + execute("INSERT INTO %s (key, value) VALUES (?, ?)", randString, ByteBuffer.wrap(randBytes)); + assertRows(execute("SELECT LENGTH(key), OCTET_LENGTH(key), OCTET_LENGTH(value) FROM %s where key = ?", randString), + row(sLen, randBytes.length, randBytes.length)); + }); + } +}