mirror of https://github.com/apache/cassandra
Support octet_length and length functions
Previously users would either have to read the data and check the length themselves or enable UDF and register UDFs to check the length of columns. This patch adds a subset of the SQL99 (binary) string functions: "octet_length" defined on all types and "length" defined on UTF8 strings. patch by Joey Lynch; reviewed by Chris Lohfink and Jordan West for CASSANDRA-20102
This commit is contained in:
Joseph Lynch
Stefan Miklosovic
parent
a8702d6a44
commit
0d39ea4917
|
|
@ -1,4 +1,5 @@
|
|||
5.1
|
||||
* Support octet_length and length functions (CASSANDRA-20102)
|
||||
* Make JsonUtils serialize Instant always with the same format (CASSANDRA-20209)
|
||||
* Port Harry v2 to trunk (CASSANDRA-20200)
|
||||
* Enable filtering of snapshots on keyspace, table and snapshot name in nodetool listsnapshots (CASSANDRA-20151)
|
||||
|
|
|
|||
4
NEWS.txt
4
NEWS.txt
|
|
@ -109,6 +109,10 @@ New features
|
|||
Enabling both ways of configuring JMX will result in a node failing to start.
|
||||
- CEP-43 - it is possible to create a table by "copying" as `CREATE TABLE ks.tb_copy LIKE ks.tb;`.
|
||||
A newly created table will have no data.
|
||||
- New functions `octet_length` and `length` were introduced. Previously, users would either have to read
|
||||
the data and check the length themselves or enable UDF and register UDFs to check the length of columns.
|
||||
CASSANDRA-20102 adds a subset of the SQL99 (binary) string functions: "octet_length" defined on all types
|
||||
and "length" defined on UTF8 strings. See CASSANDRA-20102 for more information.
|
||||
|
||||
Upgrading
|
||||
---------
|
||||
|
|
|
|||
|
|
@ -231,6 +231,54 @@ For every xref:cassandra:developing/cql/types.adoc#native-types[type] supported
|
|||
Conversely, the function `blob_as_type` takes a 64-bit `blob` argument and converts it to a `bigint` value.
|
||||
For example, `bigint_as_blob(3)` returns `0x0000000000000003` and `blob_as_bigint(0x0000000000000003)` returns `3`.
|
||||
|
||||
==== Length Functions
|
||||
CQL supports two functions to retrieve the length of data without returning the data itself. Note that while
|
||||
network bandwidth is reduced and memory is freed earlier, these functions still require data to be read from disk.
|
||||
|
||||
[cols=",,",options="header",]
|
||||
|===
|
||||
|Function name |Input type |Description
|
||||
|
||||
| `octet_length` | All xref:cassandra:developing/cql/types.adoc#native-types[types] | Returns the length of the data in bytes.
|
||||
|
||||
| `length` | `text` | Returns the string's length - the count of UTF-8 code units.
|
||||
|===
|
||||
|
||||
The `octet_length` function is defined for every xref:cassandra:developing/cql/types.adoc#native-types[type]
|
||||
supported by CQL, and represents the number of bytes of the underlying `ByteBuffer` representation, not accounting for
|
||||
metadata overhead. Equivalent to Java's
|
||||
https://docs.oracle.com/en/java/javase/22/docs/api/java.base/java/nio/Buffer.html#remaining()[`ByteBuffer#remaining()`]
|
||||
when data has just been read, which is also the `length` of the type when encoded to `byte[]`. Some examples
|
||||
for common types:
|
||||
|
||||
[cols=",,",options="header",]
|
||||
|===
|
||||
|CQL Type | `octet_length` |Description
|
||||
|
||||
| `tinyint` | `1` | 8-bit signed integer
|
||||
|
||||
| `smallint` | `2` | 16-bit signed integer
|
||||
|
||||
| `int` | `4` | 32-bit signed integer
|
||||
|
||||
| `bigint` | `8` | 64-bit signed integer
|
||||
|
||||
| `float` | `4` | 32-bit IEEE-754 floating point
|
||||
|
||||
| `double` | `8` | 64-bit IEEE-754 floating point
|
||||
|
||||
| `blob` | Number of bytes in blob | Variable length byte string
|
||||
|
||||
| `text` | Number of bytes in `text_as_blob` representation | Variable length character string
|
||||
|===
|
||||
|
||||
The `length` function is only defined for UTF-8 strings (`text`) and returns the length of that string, meaning the
|
||||
number of UTF-8 code units. Equivalent to Java's
|
||||
https://docs.oracle.com/en/java/javase/22/docs/api/java.base/java/lang/String.html#length()[`String#length()`] or
|
||||
Python 3's `len` function when applied to a string.
|
||||
|
||||
These length functions return `null` when the input is `null`.
|
||||
|
||||
==== Math Functions
|
||||
|
||||
Cql provides the following math functions: `abs`, `exp`, `log`, `log10`, and `round`.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.cql3.functions;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.cassandra.cql3.CQL3Type;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.db.marshal.Int32Type;
|
||||
import org.apache.cassandra.db.marshal.UTF8Type;
|
||||
import org.apache.cassandra.exceptions.InvalidRequestException;
|
||||
import org.apache.cassandra.transport.ProtocolVersion;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
|
||||
/**
|
||||
* Lengths of stored data without returning the data.
|
||||
*/
|
||||
public class LengthFcts
|
||||
{
|
||||
public static void addFunctionsTo(NativeFunctions functions)
|
||||
{
|
||||
// As all types ultimately end up as bytebuffers they should all be compatible with
|
||||
// octet_length
|
||||
Set<AbstractType<?>> types = new HashSet<>();
|
||||
for (CQL3Type type : CQL3Type.Native.values())
|
||||
{
|
||||
AbstractType<?> udfType = type.getType().udfType();
|
||||
if (!types.add(udfType))
|
||||
continue;
|
||||
functions.add(makeOctetLengthFunction(type.getType().udfType()));
|
||||
}
|
||||
|
||||
// Special handling for string length which is number of UTF-8 code units
|
||||
functions.add(length);
|
||||
}
|
||||
|
||||
|
||||
public static NativeFunction makeOctetLengthFunction(AbstractType<?> fromType)
|
||||
{
|
||||
// Matches SQL99 OCTET_LENGTH functions defined on bytestring and char strings
|
||||
return new NativeScalarFunction("octet_length", Int32Type.instance, fromType)
|
||||
{
|
||||
// Do not deserialize
|
||||
@Override
|
||||
public Arguments newArguments(ProtocolVersion version)
|
||||
{
|
||||
return FunctionArguments.newNoopInstance(version, 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteBuffer execute(Arguments arguments) throws InvalidRequestException
|
||||
{
|
||||
if (arguments.get(0) == null)
|
||||
return null;
|
||||
|
||||
final ByteBuffer buffer = arguments.get(0);
|
||||
return ByteBufferUtil.bytes(buffer.remaining());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Matches PostgreSQL length function defined as returning the number of UTF-8 code units in the text string
|
||||
public static final NativeFunction length = new NativeScalarFunction("length", Int32Type.instance, UTF8Type.instance)
|
||||
{
|
||||
@Override
|
||||
public ByteBuffer execute(Arguments arguments) throws InvalidRequestException
|
||||
{
|
||||
if (arguments.get(0) == null)
|
||||
return null;
|
||||
|
||||
final String value = arguments.get(0);
|
||||
return ByteBufferUtil.bytes(value.length());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -44,6 +44,7 @@ public class NativeFunctions
|
|||
AggregateFcts.addFunctionsTo(this);
|
||||
CollectionFcts.addFunctionsTo(this);
|
||||
BytesConversionFcts.addFunctionsTo(this);
|
||||
LengthFcts.addFunctionsTo(this);
|
||||
MathFcts.addFunctionsTo(this);
|
||||
MaskingFcts.addFunctionsTo(this);
|
||||
VectorFcts.addFunctionsTo(this);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.cql3.functions;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import org.apache.cassandra.cql3.CQLTester;
|
||||
import org.quicktheories.WithQuickTheories;
|
||||
|
||||
public class LengthFctsTest extends CQLTester implements WithQuickTheories
|
||||
{
|
||||
@Test
|
||||
public void testOctetLengthNonStrings()
|
||||
{
|
||||
createTable("CREATE TABLE %s (a tinyint primary key,"
|
||||
+ " b smallint,"
|
||||
+ " c int,"
|
||||
+ " d bigint,"
|
||||
+ " e float,"
|
||||
+ " f double,"
|
||||
+ " g decimal,"
|
||||
+ " h varint,"
|
||||
+ " i int)");
|
||||
|
||||
execute("INSERT INTO %s (a, b, c, d, e, f, g, h) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(byte) 1, (short) 2, 3, 4L, 5.2F, 6.3, BigDecimal.valueOf(6.3), BigInteger.valueOf(4));
|
||||
|
||||
assertRows(execute("SELECT OCTET_LENGTH(a), " +
|
||||
"OCTET_LENGTH(b), " +
|
||||
"OCTET_LENGTH(c), " +
|
||||
"OCTET_LENGTH(d), " +
|
||||
"OCTET_LENGTH(e), " +
|
||||
"OCTET_LENGTH(f), " +
|
||||
"OCTET_LENGTH(g), " +
|
||||
"OCTET_LENGTH(h), " +
|
||||
"OCTET_LENGTH(i) FROM %s"),
|
||||
row(1, 2, 4, 8, 4, 8, 5, 1, null));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStringLengthUTF8() throws Throwable
|
||||
{
|
||||
createTable("CREATE TABLE %s (key text primary key, value blob)");
|
||||
// UTF-8 7 codepoint, 21 byte encoded string
|
||||
String key = "こんにちは世界";
|
||||
execute("INSERT INTO %s (key) VALUES (?)", key);
|
||||
|
||||
assertRows(execute("SELECT LENGTH(key), OCTET_LENGTH(key), OCTET_LENGTH(value) FROM %s where key = ?", key),
|
||||
row(7, 21, null));
|
||||
|
||||
// Quickly check that multiple arguments leads to an exception as expected
|
||||
assertInvalidMessage("Invalid number of arguments in call to function system.length",
|
||||
"SELECT LENGTH(key, value) FROM %s where key = 'こんにちは世界'");
|
||||
assertInvalidMessage("Invalid call to function octet_length, none of its type signatures match",
|
||||
"SELECT OCTET_LENGTH(key, value) FROM %s where key = 'こんにちは世界'");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOctetLengthStringFuzz()
|
||||
{
|
||||
createTable("CREATE TABLE %s (key text primary key, value blob)");
|
||||
|
||||
qt().withExamples(1024).forAll(strings().allPossible().ofLengthBetween(32, 100)).checkAssert(
|
||||
(randString) -> {
|
||||
int sLen = randString.length();
|
||||
byte[] randBytes = randString.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
// UTF-8 length (code unit count) and byte length are often
|
||||
// different. Spot checked a few of these, and they are different
|
||||
// most of the time in this test - but testing that reproducibly
|
||||
// requires seeding that would decrease the test power...
|
||||
execute("INSERT INTO %s (key, value) VALUES (?, ?)", randString, ByteBuffer.wrap(randBytes));
|
||||
assertRows(execute("SELECT LENGTH(key), OCTET_LENGTH(key), OCTET_LENGTH(value) FROM %s where key = ?", randString),
|
||||
row(sLen, randBytes.length, randBytes.length));
|
||||
});
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue