From f32475a839e01e4eea3989871d293d70e8a360d7 Mon Sep 17 00:00:00 2001 From: Yifan Cai Date: Mon, 22 Feb 2021 13:39:16 -0800 Subject: [PATCH] Fix incorrect encoding for strings can be UTF8 patch by Yifan Cai; reviewed by Ekaterina Dimitrova for CASSANDRA-16429 --- CHANGES.txt | 1 + doc/source/cql/definitions.rst | 6 ++-- .../org/apache/cassandra/cql3/ResultSet.java | 8 ++--- .../apache/cassandra/transport/DataType.java | 8 ++--- .../cassandra/transport/SerDeserTest.java | 30 +++++++++++++++++-- 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6d311eb619..4d3372bbb0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0-beta5 + * Fix incorrect encoding for strings can be UTF8 (CASSANDRA-16429) * Fix node unable to join when RF > N in multi-DC with added warning (CASSANDRA-16296) * Add an option to nodetool tablestats to check sstable location correctness (CASSANDRA-16344) * Unable to ALTER KEYSPACE while decommissioned/assassinated nodes are in gossip (CASSANDRA-16422) diff --git a/doc/source/cql/definitions.rst b/doc/source/cql/definitions.rst index 3df6f20996..8ccb630241 100644 --- a/doc/source/cql/definitions.rst +++ b/doc/source/cql/definitions.rst @@ -46,8 +46,10 @@ To aid in specifying the CQL syntax, we will use the following conventions in th Identifiers and keywords ^^^^^^^^^^^^^^^^^^^^^^^^ -The CQL language uses *identifiers* (or *names*) to identify tables, columns and other objects. An identifier is a token -matching the regular expression ``[a-zA-Z][a-zA-Z0-9_]*``. +The CQL language uses *identifiers* (or *names*) to identify tables, columns and other objects. An identifier can be either +unquoted or quoted. The unquoted identifier is a token matching the regular expression ``[a-zA-Z][a-zA-Z0-9_]*``. +In the case of quoted, the identity can contain non-ASCII characters between the quotation marks, with one exception that +Cassandra does not accept non-ASCII characters, if the quoted identifier is used for keyspace name or table name. A number of such identifiers, like ``SELECT`` or ``WITH``, are *keywords*. They have a fixed meaning for the language and most are reserved. The list of those keywords can be found in :ref:`appendix-A`. diff --git a/src/java/org/apache/cassandra/cql3/ResultSet.java b/src/java/org/apache/cassandra/cql3/ResultSet.java index 4f60445650..de393f7310 100644 --- a/src/java/org/apache/cassandra/cql3/ResultSet.java +++ b/src/java/org/apache/cassandra/cql3/ResultSet.java @@ -452,7 +452,7 @@ public class ResultSet CBUtil.writeAsciiString(name.ksName, dest); CBUtil.writeAsciiString(name.cfName, dest); } - CBUtil.writeAsciiString(name.name.toString(), dest); + CBUtil.writeString(name.name.toString(), dest); DataType.codec.writeOne(DataType.fromType(name.type, version), dest, version); } } @@ -488,7 +488,7 @@ public class ResultSet size += CBUtil.sizeOfAsciiString(name.ksName); size += CBUtil.sizeOfAsciiString(name.cfName); } - size += CBUtil.sizeOfAsciiString(name.name.toString()); + size += CBUtil.sizeOfString(name.name.toString()); size += DataType.codec.oneSerializedSize(DataType.fromType(name.type, version), version); } } @@ -657,7 +657,7 @@ public class ResultSet CBUtil.writeAsciiString(name.ksName, dest); CBUtil.writeAsciiString(name.cfName, dest); } - CBUtil.writeAsciiString(name.name.toString(), dest); + CBUtil.writeString(name.name.toString(), dest); DataType.codec.writeOne(DataType.fromType(name.type, version), dest, version); } } @@ -682,7 +682,7 @@ public class ResultSet size += CBUtil.sizeOfAsciiString(name.ksName); size += CBUtil.sizeOfAsciiString(name.cfName); } - size += CBUtil.sizeOfAsciiString(name.name.toString()); + size += CBUtil.sizeOfString(name.name.toString()); size += DataType.codec.oneSerializedSize(DataType.fromType(name.type, version), version); } return size; diff --git a/src/java/org/apache/cassandra/transport/DataType.java b/src/java/org/apache/cassandra/transport/DataType.java index 9d45e8afe5..1d1a9130b6 100644 --- a/src/java/org/apache/cassandra/transport/DataType.java +++ b/src/java/org/apache/cassandra/transport/DataType.java @@ -161,11 +161,11 @@ public enum DataType case UDT: UserType udt = (UserType)value; CBUtil.writeAsciiString(udt.keyspace, cb); - CBUtil.writeAsciiString(UTF8Type.instance.compose(udt.name), cb); + CBUtil.writeString(UTF8Type.instance.compose(udt.name), cb); cb.writeShort(udt.size()); for (int i = 0; i < udt.size(); i++) { - CBUtil.writeAsciiString(udt.fieldName(i).toString(), cb); + CBUtil.writeString(udt.fieldName(i).toString(), cb); codec.writeOne(DataType.fromType(udt.fieldType(i), version), cb, version); } break; @@ -201,11 +201,11 @@ public enum DataType UserType udt = (UserType)value; int size = 0; size += CBUtil.sizeOfAsciiString(udt.keyspace); - size += CBUtil.sizeOfAsciiString(UTF8Type.instance.compose(udt.name)); + size += CBUtil.sizeOfString(UTF8Type.instance.compose(udt.name)); size += 2; for (int i = 0; i < udt.size(); i++) { - size += CBUtil.sizeOfAsciiString(udt.fieldName(i).toString()); + size += CBUtil.sizeOfString(udt.fieldName(i).toString()); size += codec.oneSerializedSize(DataType.fromType(udt.fieldType(i), version), version); } return size; diff --git a/test/unit/org/apache/cassandra/transport/SerDeserTest.java b/test/unit/org/apache/cassandra/transport/SerDeserTest.java index 42ffa2673a..da76070b6f 100644 --- a/test/unit/org/apache/cassandra/transport/SerDeserTest.java +++ b/test/unit/org/apache/cassandra/transport/SerDeserTest.java @@ -20,6 +20,8 @@ package org.apache.cassandra.transport; import java.nio.ByteBuffer; import java.util.*; +import org.apache.commons.lang3.RandomStringUtils; + import io.netty.buffer.Unpooled; import io.netty.buffer.ByteBuf; @@ -203,20 +205,32 @@ public class SerDeserTest SetType st = SetType.getInstance(UTF8Type.instance, true); MapType mt = MapType.getInstance(UTF8Type.instance, LongType.instance, true); + String typeName = "myType" + randomUTF8(3); + String f1 = 'f' + randomUTF8(3); + UserType udt = new UserType("ks", - bb("myType"), - Arrays.asList(field("f1"), field("f2"), field("f3"), field("f4")), + bb(typeName), + Arrays.asList(field(f1), field("f2"), field("f3"), field("f4")), Arrays.asList(LongType.instance, lt, st, mt), true); Map value = new HashMap<>(); - value.put(field("f1"), lit(42)); + value.put(field(f1), lit(42)); value.put(field("f2"), new Lists.Literal(Arrays.asList(lit(3), lit(1)))); value.put(field("f3"), new Sets.Literal(Arrays.asList(lit("foo"), lit("bar")))); value.put(field("f4"), new Maps.Literal(Arrays.>asList( Pair.create(lit("foo"), lit(24)), Pair.create(lit("bar"), lit(12))))); + ByteBuf buf = Unpooled.buffer(DataType.UDT.serializedValueSize(udt, version)); + DataType.UDT.writeValue(udt, buf, version); + UserType decoded = (UserType) DataType.UDT.readValue(buf, version); + assertNotNull(decoded); + assertEquals("User type name mismatches: " + typeName, + udt.name, decoded.name); + assertEquals("Decoded field name mismatches: " + f1, + udt.fieldNameAsString(0), decoded.fieldNameAsString(0)); + UserTypes.Literal u = new UserTypes.Literal(value); Term t = u.prepare("ks", columnSpec("myValue", udt)); @@ -258,6 +272,9 @@ public class SerDeserTest List columnNames = new ArrayList<>(); for (int i = 0; i < 3; i++) columnNames.add(new ColumnSpecification("ks", "cf", new ColumnIdentifier("col" + i, false), Int32Type.instance)); + // add a column name that contains UTF-8 string (that is valid to cassandra) + String utf8ColName = "col" + randomUTF8(3); + columnNames.add(new ColumnSpecification("ks", "cf", new ColumnIdentifier(utf8ColName, false), Int32Type.instance)); if (version == ProtocolVersion.V3) { @@ -381,4 +398,11 @@ public class SerDeserTest assertEquals(options.getTimestamp(state), decodedOptions.getTimestamp(state)); assertEquals(options.getNowInSeconds(state), decodedOptions.getNowInSeconds(state)); } + + // return utf8 string that contains no ascii chars + public static String randomUTF8(int count) + { + // valid for cassandra + return RandomStringUtils.random(count, 129, 0xD800, false, false); + } }