natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix incorrect encoding for strings can be UTF8 

patch by Yifan Cai; reviewed by Ekaterina Dimitrova for CASSANDRA-16429
This commit is contained in:
Yifan Cai 2021-02-22 13:39:16 -08:00
parent 5ed5e84613
commit f32475a839
5 changed files with 40 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.0-beta5
* Fix incorrect encoding for strings can be UTF8 (CASSANDRA-16429)
* Fix node unable to join when RF > N in multi-DC with added warning (CASSANDRA-16296)
* Add an option to nodetool tablestats to check sstable location correctness (CASSANDRA-16344)
* Unable to ALTER KEYSPACE while decommissioned/assassinated nodes are in gossip (CASSANDRA-16422)

6
doc/source/cql/definitions.rst
View File

@ -46,8 +46,10 @@ To aid in specifying the CQL syntax, we will use the following conventions in th
Identifiers and keywords
^^^^^^^^^^^^^^^^^^^^^^^^
The CQL language uses *identifiers* (or *names*) to identify tables, columns and other objects. An identifier is a token
matching the regular expression ``[a-zA-Z][a-zA-Z0-9_]*``.
The CQL language uses *identifiers* (or *names*) to identify tables, columns and other objects. An identifier can be either
unquoted or quoted. The unquoted identifier is a token matching the regular expression ``[a-zA-Z][a-zA-Z0-9_]*``.
In the case of quoted, the identity can contain non-ASCII characters between the quotation marks, with one exception that
Cassandra does not accept non-ASCII characters, if the quoted identifier is used for keyspace name or table name.
A number of such identifiers, like ``SELECT`` or ``WITH``, are *keywords*. They have a fixed meaning for the language
and most are reserved. The list of those keywords can be found in :ref:`appendix-A`.

8
src/java/org/apache/cassandra/cql3/ResultSet.java
View File

@ -452,7 +452,7 @@ public class ResultSet
CBUtil.writeAsciiString(name.ksName, dest);
CBUtil.writeAsciiString(name.cfName, dest);
}
CBUtil.writeAsciiString(name.name.toString(), dest);
CBUtil.writeString(name.name.toString(), dest);
DataType.codec.writeOne(DataType.fromType(name.type, version), dest, version);
}
}
@ -488,7 +488,7 @@ public class ResultSet
size += CBUtil.sizeOfAsciiString(name.ksName);
size += CBUtil.sizeOfAsciiString(name.cfName);
}
size += CBUtil.sizeOfAsciiString(name.name.toString());
size += CBUtil.sizeOfString(name.name.toString());
size += DataType.codec.oneSerializedSize(DataType.fromType(name.type, version), version);
}
}
@ -657,7 +657,7 @@ public class ResultSet
CBUtil.writeAsciiString(name.ksName, dest);
CBUtil.writeAsciiString(name.cfName, dest);
}
CBUtil.writeAsciiString(name.name.toString(), dest);
CBUtil.writeString(name.name.toString(), dest);
DataType.codec.writeOne(DataType.fromType(name.type, version), dest, version);
}
}
@ -682,7 +682,7 @@ public class ResultSet
size += CBUtil.sizeOfAsciiString(name.ksName);
size += CBUtil.sizeOfAsciiString(name.cfName);
}
size += CBUtil.sizeOfAsciiString(name.name.toString());
size += CBUtil.sizeOfString(name.name.toString());
size += DataType.codec.oneSerializedSize(DataType.fromType(name.type, version), version);
}
return size;

8
src/java/org/apache/cassandra/transport/DataType.java
View File

@ -161,11 +161,11 @@ public enum DataType
case UDT:
UserType udt = (UserType)value;
CBUtil.writeAsciiString(udt.keyspace, cb);
CBUtil.writeAsciiString(UTF8Type.instance.compose(udt.name), cb);
CBUtil.writeString(UTF8Type.instance.compose(udt.name), cb);
cb.writeShort(udt.size());
for (int i = 0; i < udt.size(); i++)
{
CBUtil.writeAsciiString(udt.fieldName(i).toString(), cb);
CBUtil.writeString(udt.fieldName(i).toString(), cb);
codec.writeOne(DataType.fromType(udt.fieldType(i), version), cb, version);
}
break;
@ -201,11 +201,11 @@ public enum DataType
UserType udt = (UserType)value;
int size = 0;
size += CBUtil.sizeOfAsciiString(udt.keyspace);
size += CBUtil.sizeOfAsciiString(UTF8Type.instance.compose(udt.name));
size += CBUtil.sizeOfString(UTF8Type.instance.compose(udt.name));
size += 2;
for (int i = 0; i < udt.size(); i++)
{
size += CBUtil.sizeOfAsciiString(udt.fieldName(i).toString());
size += CBUtil.sizeOfString(udt.fieldName(i).toString());
size += codec.oneSerializedSize(DataType.fromType(udt.fieldType(i), version), version);
}
return size;

30
test/unit/org/apache/cassandra/transport/SerDeserTest.java
View File

@ -20,6 +20,8 @@ package org.apache.cassandra.transport;
import java.nio.ByteBuffer;
import java.util.*;
import org.apache.commons.lang3.RandomStringUtils;
import io.netty.buffer.Unpooled;
import io.netty.buffer.ByteBuf;
@ -203,20 +205,32 @@ public class SerDeserTest
SetType<?> st = SetType.getInstance(UTF8Type.instance, true);
MapType<?, ?> mt = MapType.getInstance(UTF8Type.instance, LongType.instance, true);
String typeName = "myType" + randomUTF8(3);
String f1 = 'f' + randomUTF8(3);
UserType udt = new UserType("ks",
bb("myType"),
Arrays.asList(field("f1"), field("f2"), field("f3"), field("f4")),
bb(typeName),
Arrays.asList(field(f1), field("f2"), field("f3"), field("f4")),
Arrays.asList(LongType.instance, lt, st, mt),
true);
Map<FieldIdentifier, Term.Raw> value = new HashMap<>();
value.put(field("f1"), lit(42));
value.put(field(f1), lit(42));
value.put(field("f2"), new Lists.Literal(Arrays.<Term.Raw>asList(lit(3), lit(1))));
value.put(field("f3"), new Sets.Literal(Arrays.<Term.Raw>asList(lit("foo"), lit("bar"))));
value.put(field("f4"), new Maps.Literal(Arrays.<Pair<Term.Raw, Term.Raw>>asList(
Pair.<Term.Raw, Term.Raw>create(lit("foo"), lit(24)),
Pair.<Term.Raw, Term.Raw>create(lit("bar"), lit(12)))));
ByteBuf buf = Unpooled.buffer(DataType.UDT.serializedValueSize(udt, version));
DataType.UDT.writeValue(udt, buf, version);
UserType decoded = (UserType) DataType.UDT.readValue(buf, version);
assertNotNull(decoded);
assertEquals("User type name mismatches: " + typeName,
udt.name, decoded.name);
assertEquals("Decoded field name mismatches: " + f1,
udt.fieldNameAsString(0), decoded.fieldNameAsString(0));
UserTypes.Literal u = new UserTypes.Literal(value);
Term t = u.prepare("ks", columnSpec("myValue", udt));
@ -258,6 +272,9 @@ public class SerDeserTest
List<ColumnSpecification> columnNames = new ArrayList<>();
for (int i = 0; i < 3; i++)
columnNames.add(new ColumnSpecification("ks", "cf", new ColumnIdentifier("col" + i, false), Int32Type.instance));
// add a column name that contains UTF-8 string (that is valid to cassandra)
String utf8ColName = "col" + randomUTF8(3);
columnNames.add(new ColumnSpecification("ks", "cf", new ColumnIdentifier(utf8ColName, false), Int32Type.instance));
if (version == ProtocolVersion.V3)
{
@ -381,4 +398,11 @@ public class SerDeserTest
assertEquals(options.getTimestamp(state), decodedOptions.getTimestamp(state));
assertEquals(options.getNowInSeconds(state), decodedOptions.getNowInSeconds(state));
}
// return utf8 string that contains no ascii chars
public static String randomUTF8(int count)
{
// valid for cassandra
return RandomStringUtils.random(count, 129, 0xD800, false, false);
}
}