From 0d39ea4917716e9996c908b3661a8db2b53ac40e Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Thu, 21 Nov 2024 08:12:04 -0800 Subject: [PATCH] Support octet_length and length functions Previously users would either have to read the data and check the length themselves or enable UDF and register UDFs to check the length of columns. This patch adds a subset of the SQL99 (binary) string functions: "octet_length" defined on all types and "length" defined on UTF8 strings. patch by Joey Lynch; reviewed by Chris Lohfink and Jordan West for CASSANDRA-20102 --- CHANGES.txt | 1 + NEWS.txt | 4 + .../pages/developing/cql/functions.adoc | 48 +++++++++ .../cassandra/cql3/functions/LengthFcts.java | 93 ++++++++++++++++++ .../cql3/functions/NativeFunctions.java | 1 + .../cql3/functions/LengthFctsTest.java | 98 +++++++++++++++++++ 6 files changed, 245 insertions(+) create mode 100644 src/java/org/apache/cassandra/cql3/functions/LengthFcts.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/LengthFctsTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 914523113c..a3c113321d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.1 + * Support octet_length and length functions (CASSANDRA-20102) * Make JsonUtils serialize Instant always with the same format (CASSANDRA-20209) * Port Harry v2 to trunk (CASSANDRA-20200) * Enable filtering of snapshots on keyspace, table and snapshot name in nodetool listsnapshots (CASSANDRA-20151) diff --git a/NEWS.txt b/NEWS.txt index a4cec79660..530acc0bd6 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -109,6 +109,10 @@ New features Enabling both ways of configuring JMX will result in a node failing to start. - CEP-43 - it is possible to create a table by "copying" as `CREATE TABLE ks.tb_copy LIKE ks.tb;`. A newly created table will have no data. + - New functions `octet_length` and `length` were introduced. Previously, users would either have to read + the data and check the length themselves or enable UDF and register UDFs to check the length of columns. + CASSANDRA-20102 adds a subset of the SQL99 (binary) string functions: "octet_length" defined on all types + and "length" defined on UTF8 strings. See CASSANDRA-20102 for more information. Upgrading --------- diff --git a/doc/modules/cassandra/pages/developing/cql/functions.adoc b/doc/modules/cassandra/pages/developing/cql/functions.adoc index 75786de271..2344cdb728 100644 --- a/doc/modules/cassandra/pages/developing/cql/functions.adoc +++ b/doc/modules/cassandra/pages/developing/cql/functions.adoc @@ -231,6 +231,54 @@ For every xref:cassandra:developing/cql/types.adoc#native-types[type] supported Conversely, the function `blob_as_type` takes a 64-bit `blob` argument and converts it to a `bigint` value. For example, `bigint_as_blob(3)` returns `0x0000000000000003` and `blob_as_bigint(0x0000000000000003)` returns `3`. +==== Length Functions +CQL supports two functions to retrieve the length of data without returning the data itself. Note that while +network bandwidth is reduced and memory is freed earlier, these functions still require data to be read from disk. + +[cols=",,",options="header",] +|=== +|Function name |Input type |Description + +| `octet_length` | All xref:cassandra:developing/cql/types.adoc#native-types[types] | Returns the length of the data in bytes. + +| `length` | `text` | Returns the string's length - the count of UTF-8 code units. +|=== + +The `octet_length` function is defined for every xref:cassandra:developing/cql/types.adoc#native-types[type] +supported by CQL, and represents the number of bytes of the underlying `ByteBuffer` representation, not accounting for +metadata overhead. Equivalent to Java's +https://docs.oracle.com/en/java/javase/22/docs/api/java.base/java/nio/Buffer.html#remaining()[`ByteBuffer#remaining()`] +when data has just been read, which is also the `length` of the type when encoded to `byte[]`. Some examples +for common types: + +[cols=",,",options="header",] +|=== +|CQL Type | `octet_length` |Description + +| `tinyint` | `1` | 8-bit signed integer + +| `smallint` | `2` | 16-bit signed integer + +| `int` | `4` | 32-bit signed integer + +| `bigint` | `8` | 64-bit signed integer + +| `float` | `4` | 32-bit IEEE-754 floating point + +| `double` | `8` | 64-bit IEEE-754 floating point + +| `blob` | Number of bytes in blob | Variable length byte string + +| `text` | Number of bytes in `text_as_blob` representation | Variable length character string +|=== + +The `length` function is only defined for UTF-8 strings (`text`) and returns the length of that string, meaning the +number of UTF-8 code units. Equivalent to Java's +https://docs.oracle.com/en/java/javase/22/docs/api/java.base/java/lang/String.html#length()[`String#length()`] or +Python 3's `len` function when applied to a string. + +These length functions return `null` when the input is `null`. + ==== Math Functions Cql provides the following math functions: `abs`, `exp`, `log`, `log10`, and `round`. diff --git a/src/java/org/apache/cassandra/cql3/functions/LengthFcts.java b/src/java/org/apache/cassandra/cql3/functions/LengthFcts.java new file mode 100644 index 0000000000..64cb940606 --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/functions/LengthFcts.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3.functions; + +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Set; + +import org.apache.cassandra.cql3.CQL3Type; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * Lengths of stored data without returning the data. + */ +public class LengthFcts +{ + public static void addFunctionsTo(NativeFunctions functions) + { + // As all types ultimately end up as bytebuffers they should all be compatible with + // octet_length + Set> types = new HashSet<>(); + for (CQL3Type type : CQL3Type.Native.values()) + { + AbstractType udfType = type.getType().udfType(); + if (!types.add(udfType)) + continue; + functions.add(makeOctetLengthFunction(type.getType().udfType())); + } + + // Special handling for string length which is number of UTF-8 code units + functions.add(length); + } + + + public static NativeFunction makeOctetLengthFunction(AbstractType fromType) + { + // Matches SQL99 OCTET_LENGTH functions defined on bytestring and char strings + return new NativeScalarFunction("octet_length", Int32Type.instance, fromType) + { + // Do not deserialize + @Override + public Arguments newArguments(ProtocolVersion version) + { + return FunctionArguments.newNoopInstance(version, 1); + } + + @Override + public ByteBuffer execute(Arguments arguments) throws InvalidRequestException + { + if (arguments.get(0) == null) + return null; + + final ByteBuffer buffer = arguments.get(0); + return ByteBufferUtil.bytes(buffer.remaining()); + } + }; + } + + // Matches PostgreSQL length function defined as returning the number of UTF-8 code units in the text string + public static final NativeFunction length = new NativeScalarFunction("length", Int32Type.instance, UTF8Type.instance) + { + @Override + public ByteBuffer execute(Arguments arguments) throws InvalidRequestException + { + if (arguments.get(0) == null) + return null; + + final String value = arguments.get(0); + return ByteBufferUtil.bytes(value.length()); + } + }; +} diff --git a/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java b/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java index a9deba5cb1..e521c5e807 100644 --- a/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java +++ b/src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java @@ -44,6 +44,7 @@ public class NativeFunctions AggregateFcts.addFunctionsTo(this); CollectionFcts.addFunctionsTo(this); BytesConversionFcts.addFunctionsTo(this); + LengthFcts.addFunctionsTo(this); MathFcts.addFunctionsTo(this); MaskingFcts.addFunctionsTo(this); VectorFcts.addFunctionsTo(this); diff --git a/test/unit/org/apache/cassandra/cql3/functions/LengthFctsTest.java b/test/unit/org/apache/cassandra/cql3/functions/LengthFctsTest.java new file mode 100644 index 0000000000..b1f1354bc2 --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/functions/LengthFctsTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3.functions; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.quicktheories.WithQuickTheories; + +public class LengthFctsTest extends CQLTester implements WithQuickTheories +{ + @Test + public void testOctetLengthNonStrings() + { + createTable("CREATE TABLE %s (a tinyint primary key," + + " b smallint," + + " c int," + + " d bigint," + + " e float," + + " f double," + + " g decimal," + + " h varint," + + " i int)"); + + execute("INSERT INTO %s (a, b, c, d, e, f, g, h) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (byte) 1, (short) 2, 3, 4L, 5.2F, 6.3, BigDecimal.valueOf(6.3), BigInteger.valueOf(4)); + + assertRows(execute("SELECT OCTET_LENGTH(a), " + + "OCTET_LENGTH(b), " + + "OCTET_LENGTH(c), " + + "OCTET_LENGTH(d), " + + "OCTET_LENGTH(e), " + + "OCTET_LENGTH(f), " + + "OCTET_LENGTH(g), " + + "OCTET_LENGTH(h), " + + "OCTET_LENGTH(i) FROM %s"), + row(1, 2, 4, 8, 4, 8, 5, 1, null)); + } + + @Test + public void testStringLengthUTF8() throws Throwable + { + createTable("CREATE TABLE %s (key text primary key, value blob)"); + // UTF-8 7 codepoint, 21 byte encoded string + String key = "こんにちは世界"; + execute("INSERT INTO %s (key) VALUES (?)", key); + + assertRows(execute("SELECT LENGTH(key), OCTET_LENGTH(key), OCTET_LENGTH(value) FROM %s where key = ?", key), + row(7, 21, null)); + + // Quickly check that multiple arguments leads to an exception as expected + assertInvalidMessage("Invalid number of arguments in call to function system.length", + "SELECT LENGTH(key, value) FROM %s where key = 'こんにちは世界'"); + assertInvalidMessage("Invalid call to function octet_length, none of its type signatures match", + "SELECT OCTET_LENGTH(key, value) FROM %s where key = 'こんにちは世界'"); + } + + @Test + public void testOctetLengthStringFuzz() + { + createTable("CREATE TABLE %s (key text primary key, value blob)"); + + qt().withExamples(1024).forAll(strings().allPossible().ofLengthBetween(32, 100)).checkAssert( + (randString) -> { + int sLen = randString.length(); + byte[] randBytes = randString.getBytes(StandardCharsets.UTF_8); + + // UTF-8 length (code unit count) and byte length are often + // different. Spot checked a few of these, and they are different + // most of the time in this test - but testing that reproducibly + // requires seeding that would decrease the test power... + execute("INSERT INTO %s (key, value) VALUES (?, ?)", randString, ByteBuffer.wrap(randBytes)); + assertRows(execute("SELECT LENGTH(key), OCTET_LENGTH(key), OCTET_LENGTH(value) FROM %s where key = ?", randString), + row(sLen, randBytes.length, randBytes.length)); + }); + } +}