diff --git a/CHANGES.txt b/CHANGES.txt index 4d3372bbb0..c2abba7375 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0-beta5 + * Promote protocol V5 out of beta (CASSANDRA-14973) * Fix incorrect encoding for strings can be UTF8 (CASSANDRA-16429) * Fix node unable to join when RF > N in multi-DC with added warning (CASSANDRA-16296) * Add an option to nodetool tablestats to check sstable location correctness (CASSANDRA-16344) diff --git a/NEWS.txt b/NEWS.txt index 817f286060..3fd26e49c8 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -133,7 +133,7 @@ Upgrading CASSANDRA-16083, backward compatibility layer was added so it can be still exposed under the old 3.11 scope. - Native protocol v5 is promoted from beta in this release. The wire format has changed significantly and users should take care to ensure client drivers are upgraded to a version - with support for the final v5 format, if currently connecting over v5-beta. (CASSANDRA-15299) + with support for the final v5 format, if currently connecting over v5-beta. (CASSANDRA-15299, CASSANDRA-14973) - Cassandra removed support for the OldNetworkTopologyStrategy. Before upgrading you will need to change the replication strategy for the keyspaces using this strategy to the NetworkTopologyStrategy. (CASSANDRA-13990) - Sstables for tables using with a frozen UDT written by C* 3.0 appear as corrupted. diff --git a/bin/cqlsh.py b/bin/cqlsh.py index 5162a00ae4..09183a782d 100644 --- a/bin/cqlsh.py +++ b/bin/cqlsh.py @@ -476,7 +476,7 @@ class Shell(cmd.Cmd): if protocol_version is not None: kwargs['protocol_version'] = protocol_version else: - kwargs['protocol_version'] = 4 + kwargs['protocol_version'] = 5 self.conn = Cluster(contact_points=(self.hostname,), port=self.port, cql_version=cqlver, auth_provider=self.auth_provider, ssl_options=sslhandling.ssl_settings(hostname, CONFIG_FILE) if ssl else None, diff --git a/build.xml b/build.xml index c39701978e..5c93897916 100644 --- a/build.xml +++ b/build.xml @@ -645,6 +645,7 @@ + @@ -748,7 +750,9 @@ + @@ -779,7 +783,9 @@ + @@ -856,14 +862,15 @@ - - - - - - - - + + diff --git a/lib/cassandra-driver-core-3.10.0-shaded.jar b/lib/cassandra-driver-core-3.10.3-1af7382fe-shaded.jar similarity index 77% rename from lib/cassandra-driver-core-3.10.0-shaded.jar rename to lib/cassandra-driver-core-3.10.3-1af7382fe-shaded.jar index d53e577e87..adf439a3a3 100644 Binary files a/lib/cassandra-driver-core-3.10.0-shaded.jar and b/lib/cassandra-driver-core-3.10.3-1af7382fe-shaded.jar differ diff --git a/lib/cassandra-driver-internal-only-3.23.0.post0-1a184b99.zip b/lib/cassandra-driver-internal-only-3.23.0.post0-1a184b99.zip deleted file mode 100644 index 9b25bc17eb..0000000000 Binary files a/lib/cassandra-driver-internal-only-3.23.0.post0-1a184b99.zip and /dev/null differ diff --git a/lib/cassandra-driver-internal-only-3.24.0-1de685b1.zip b/lib/cassandra-driver-internal-only-3.24.0-1de685b1.zip new file mode 100644 index 0000000000..d04607ce84 Binary files /dev/null and b/lib/cassandra-driver-internal-only-3.24.0-1de685b1.zip differ diff --git a/src/java/org/apache/cassandra/cql3/functions/types/CodecUtils.java b/src/java/org/apache/cassandra/cql3/functions/types/CodecUtils.java index 4e97e5c433..22ca4ed2a5 100644 --- a/src/java/org/apache/cassandra/cql3/functions/types/CodecUtils.java +++ b/src/java/org/apache/cassandra/cql3/functions/types/CodecUtils.java @@ -77,6 +77,7 @@ public final class CodecUtils case V3: case V4: case V5: + case V6: return input.getInt(); default: throw new IllegalArgumentException(String.valueOf(version)); @@ -107,6 +108,7 @@ public final class CodecUtils case V3: case V4: case V5: + case V6: output.putInt(size); break; default: @@ -149,6 +151,7 @@ public final class CodecUtils case V3: case V4: case V5: + case V6: if (value == null) { output.putInt(-1); @@ -230,6 +233,7 @@ public final class CodecUtils case V3: case V4: case V5: + case V6: return 4; default: throw new IllegalArgumentException(String.valueOf(version)); @@ -252,6 +256,7 @@ public final class CodecUtils case V3: case V4: case V5: + case V6: return value == null ? 4 : 4 + value.remaining(); default: throw new IllegalArgumentException(String.valueOf(version)); diff --git a/src/java/org/apache/cassandra/transport/CQLMessageHandler.java b/src/java/org/apache/cassandra/transport/CQLMessageHandler.java index 34e619a77e..9c8d9d664f 100644 --- a/src/java/org/apache/cassandra/transport/CQLMessageHandler.java +++ b/src/java/org/apache/cassandra/transport/CQLMessageHandler.java @@ -335,10 +335,23 @@ public class CQLMessageHandler extends AbstractMessageHandler int messageSize = Ints.checkedCast(header.bodySizeInBytes); receivedBytes += buf.remaining(); - if (throwOnOverload) + LargeMessage largeMessage = new LargeMessage(header); + if (!acquireCapacity(header, endpointReserve, globalReserve)) { - LargeMessage largeMessage = new LargeMessage(header); - if (!acquireCapacity(header, endpointReserve, globalReserve)) + // In the case of large messages, never stop processing incoming frames + // as this will halt the client meaning no further frames will be sent, + // leading to starvation. + // If the throwOnOverload option is set, don't process the message once + // read, return an error response to notify the client that resource + // limits have been exceeded. If the option isn't set, the only thing we + // can do is to consume the subsequent frames and process the message. + // Large and small messages are never interleaved for a single client, so + // we know that this client will finish sending the large message before + // anything else. Other clients sending small messages concurrently will + // be backpressured by the global resource limits. The server is still + // vulnerable to overload by multiple clients sending large messages + // concurrently. + if (throwOnOverload) { // discard the request and throw an exception ClientMetrics.instance.markRequestDiscarded(); @@ -350,26 +363,12 @@ public class CQLMessageHandler extends AbstractMessageHandler globalReserve.using(), header); - // mark as overloaded so that we consume - // subsequent frames and then discard the message + // mark as overloaded so that discard the message + // after consuming any subsequent frames largeMessage.markOverloaded(); } - this.largeMessage = largeMessage; - largeMessage.supply(frame); - // Don't stop processing incoming frames, rely on the client to apply - // backpressure when it receives OverloadedException - return true; } - else - { - if (!acquireCapacityAndQueueOnFailure(header, endpointReserve, globalReserve)) - { - receivedBytes += frame.frameSize; - return false; - } - } - - largeMessage = new LargeMessage(header); + this.largeMessage = largeMessage; largeMessage.supply(frame); return true; } diff --git a/src/java/org/apache/cassandra/transport/ClientResourceLimits.java b/src/java/org/apache/cassandra/transport/ClientResourceLimits.java index f9e7dab4ab..17a6e594b3 100644 --- a/src/java/org/apache/cassandra/transport/ClientResourceLimits.java +++ b/src/java/org/apache/cassandra/transport/ClientResourceLimits.java @@ -213,8 +213,11 @@ public class ClientResourceLimits public String toString() { - return String.format("InflightEndpointRequestPayload: %d, InflightOverallRequestPayload: %d", - endpointAndGlobal.endpoint().using(), endpointAndGlobal.global().using()); + return String.format("InflightEndpointRequestPayload: %d/%d, InflightOverallRequestPayload: %d/%d", + endpointAndGlobal.endpoint().using(), + endpointAndGlobal.endpoint().limit(), + endpointAndGlobal.global().using(), + endpointAndGlobal.global().limit()); } } diff --git a/src/java/org/apache/cassandra/transport/ProtocolVersion.java b/src/java/org/apache/cassandra/transport/ProtocolVersion.java index a97597562a..d4c2ac3342 100644 --- a/src/java/org/apache/cassandra/transport/ProtocolVersion.java +++ b/src/java/org/apache/cassandra/transport/ProtocolVersion.java @@ -43,7 +43,8 @@ public enum ProtocolVersion implements Comparable V2(2, "v2", false), // no longer supported V3(3, "v3", false), V4(4, "v4", false), - V5(5, "v5-beta", true); + V5(5, "v5", false), + V6(6, "v6-beta", true); /** The version number */ private final int num; @@ -62,7 +63,7 @@ public enum ProtocolVersion implements Comparable } /** The supported versions stored as an array, these should be private and are required for fast decoding*/ - private final static ProtocolVersion[] SUPPORTED_VERSIONS = new ProtocolVersion[] { V3, V4, V5 }; + private final static ProtocolVersion[] SUPPORTED_VERSIONS = new ProtocolVersion[] { V3, V4, V5, V6 }; final static ProtocolVersion MIN_SUPPORTED_VERSION = SUPPORTED_VERSIONS[0]; final static ProtocolVersion MAX_SUPPORTED_VERSION = SUPPORTED_VERSIONS[SUPPORTED_VERSIONS.length - 1]; @@ -73,8 +74,8 @@ public enum ProtocolVersion implements Comparable public final static EnumSet UNSUPPORTED = EnumSet.complementOf(SUPPORTED); /** The preferred versions */ - public final static ProtocolVersion CURRENT = V4; - public final static Optional BETA = Optional.of(V5); + public final static ProtocolVersion CURRENT = V5; + public final static Optional BETA = Optional.of(V6); public static List supportedVersions() { diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index 0b6d8a3d3b..b8c65fc423 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -801,7 +801,7 @@ public class DescribeStatementTest extends CQLTester // As different ProtocolVersions use different driver instances, we use different ProtocolVersions // for the with and without "USE keyspace" cases. - ProtocolVersion v = useKs != null ? ProtocolVersion.CURRENT : ProtocolVersion.V5; + ProtocolVersion v = useKs != null ? ProtocolVersion.CURRENT : ProtocolVersion.V6; if (useKs != null) executeNet(v, "USE " + useKs); diff --git a/test/unit/org/apache/cassandra/service/ProtocolBetaVersionTest.java b/test/unit/org/apache/cassandra/service/ProtocolBetaVersionTest.java index 4ade4adcfb..a7551f42b3 100644 --- a/test/unit/org/apache/cassandra/service/ProtocolBetaVersionTest.java +++ b/test/unit/org/apache/cassandra/service/ProtocolBetaVersionTest.java @@ -110,7 +110,7 @@ public class ProtocolBetaVersionTest extends CQLTester } catch (Exception e) { - assertEquals("Beta version of server used (5/v5-beta), but USE_BETA flag is not set", + assertEquals("Beta version of server used (6/v6-beta), but USE_BETA flag is not set", e.getMessage()); } } diff --git a/test/unit/org/apache/cassandra/transport/CQLConnectionTest.java b/test/unit/org/apache/cassandra/transport/CQLConnectionTest.java index 46f6ca6f4a..9b1a067be3 100644 --- a/test/unit/org/apache/cassandra/transport/CQLConnectionTest.java +++ b/test/unit/org/apache/cassandra/transport/CQLConnectionTest.java @@ -135,7 +135,7 @@ public class CQLConnectionTest .matches(message -> message.error.getMessage() .equals("Invalid or unsupported protocol version (99); " + - "supported versions are (3/v3, 4/v4, 5/v5-beta)")); + "supported versions are (3/v3, 4/v4, 5/v5, 6/v6-beta)")); server.stop(); // the failure happens before any capacity is allocated diff --git a/test/unit/org/apache/cassandra/transport/ProtocolNegotiationTest.java b/test/unit/org/apache/cassandra/transport/ProtocolNegotiationTest.java index e16959a042..1104a02067 100644 --- a/test/unit/org/apache/cassandra/transport/ProtocolNegotiationTest.java +++ b/test/unit/org/apache/cassandra/transport/ProtocolNegotiationTest.java @@ -43,6 +43,7 @@ import static com.datastax.driver.core.ProtocolVersion.V2; import static com.datastax.driver.core.ProtocolVersion.V3; import static com.datastax.driver.core.ProtocolVersion.V4; import static com.datastax.driver.core.ProtocolVersion.V5; +import static com.datastax.driver.core.ProtocolVersion.V6; import static org.apache.cassandra.transport.messages.StartupMessage.CQL_VERSION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -67,22 +68,23 @@ public class ProtocolNegotiationTest extends CQLTester } @Test - public void serverSupportsV3AndV4ByDefault() + public void serverSupportsV3AndV4AndV5ByDefault() { - // client can explicitly request either V3 or V4 + // client can explicitly request either V3, V4 or V5 testConnection(V3, V3); testConnection(V4, V4); + testConnection(V5, V5); - // if not specified, V4 is the default - testConnection(null, V4); - testConnection(NEWEST_SUPPORTED, V4); + // if not specified, V5 is the default + testConnection(null, V5); + testConnection(NEWEST_SUPPORTED, V5); } @Test - public void supportV5ConnectionWithBetaOption() + public void supportV6ConnectionWithBetaOption() { - testConnection(V5, V5); - testConnection(NEWEST_BETA, V5); + testConnection(V6, V6); + testConnection(NEWEST_BETA, V6); } @Test @@ -176,6 +178,7 @@ public class ProtocolNegotiationTest extends CQLTester { if (expectError) fail("Expected a protocol exception"); + session.execute("SELECT * FROM system.local"); } catch (Exception e) { diff --git a/test/unit/org/apache/cassandra/transport/ProtocolVersionTest.java b/test/unit/org/apache/cassandra/transport/ProtocolVersionTest.java index 74bc407d52..b1ef466973 100644 --- a/test/unit/org/apache/cassandra/transport/ProtocolVersionTest.java +++ b/test/unit/org/apache/cassandra/transport/ProtocolVersionTest.java @@ -80,8 +80,8 @@ public class ProtocolVersionTest Assert.assertTrue(ProtocolVersion.supportedVersions().size() >= 2); // at least one OS and one DSE Assert.assertNotNull(ProtocolVersion.CURRENT); - Assert.assertFalse(ProtocolVersion.V4.isBeta()); - Assert.assertTrue(ProtocolVersion.V5.isBeta()); + Assert.assertFalse(ProtocolVersion.V5.isBeta()); + Assert.assertTrue(ProtocolVersion.V6.isBeta()); } @Test @@ -92,28 +92,34 @@ public class ProtocolVersionTest Assert.assertTrue(ProtocolVersion.V3.isSmallerOrEqualTo(ProtocolVersion.V3)); Assert.assertTrue(ProtocolVersion.V4.isSmallerOrEqualTo(ProtocolVersion.V4)); Assert.assertTrue(ProtocolVersion.V5.isSmallerOrEqualTo(ProtocolVersion.V5)); + Assert.assertTrue(ProtocolVersion.V6.isSmallerOrEqualTo(ProtocolVersion.V6)); Assert.assertTrue(ProtocolVersion.V1.isGreaterOrEqualTo(ProtocolVersion.V1)); Assert.assertTrue(ProtocolVersion.V2.isGreaterOrEqualTo(ProtocolVersion.V2)); Assert.assertTrue(ProtocolVersion.V3.isGreaterOrEqualTo(ProtocolVersion.V3)); Assert.assertTrue(ProtocolVersion.V4.isGreaterOrEqualTo(ProtocolVersion.V4)); Assert.assertTrue(ProtocolVersion.V5.isGreaterOrEqualTo(ProtocolVersion.V5)); + Assert.assertTrue(ProtocolVersion.V6.isGreaterOrEqualTo(ProtocolVersion.V6)); Assert.assertTrue(ProtocolVersion.V1.isSmallerThan(ProtocolVersion.V2)); Assert.assertTrue(ProtocolVersion.V2.isSmallerThan(ProtocolVersion.V3)); Assert.assertTrue(ProtocolVersion.V3.isSmallerThan(ProtocolVersion.V4)); Assert.assertTrue(ProtocolVersion.V4.isSmallerThan(ProtocolVersion.V5)); + Assert.assertTrue(ProtocolVersion.V5.isSmallerThan(ProtocolVersion.V6)); Assert.assertFalse(ProtocolVersion.V1.isGreaterThan(ProtocolVersion.V2)); Assert.assertFalse(ProtocolVersion.V2.isGreaterThan(ProtocolVersion.V3)); Assert.assertFalse(ProtocolVersion.V3.isGreaterThan(ProtocolVersion.V4)); Assert.assertFalse(ProtocolVersion.V4.isGreaterThan(ProtocolVersion.V5)); + Assert.assertFalse(ProtocolVersion.V5.isGreaterThan(ProtocolVersion.V6)); + Assert.assertTrue(ProtocolVersion.V6.isGreaterThan(ProtocolVersion.V5)); Assert.assertTrue(ProtocolVersion.V5.isGreaterThan(ProtocolVersion.V4)); Assert.assertTrue(ProtocolVersion.V4.isGreaterThan(ProtocolVersion.V3)); Assert.assertTrue(ProtocolVersion.V3.isGreaterThan(ProtocolVersion.V2)); Assert.assertTrue(ProtocolVersion.V2.isGreaterThan(ProtocolVersion.V1)); + Assert.assertFalse(ProtocolVersion.V6.isSmallerThan(ProtocolVersion.V5)); Assert.assertFalse(ProtocolVersion.V5.isSmallerThan(ProtocolVersion.V4)); Assert.assertFalse(ProtocolVersion.V4.isSmallerThan(ProtocolVersion.V3)); Assert.assertFalse(ProtocolVersion.V3.isSmallerThan(ProtocolVersion.V2)); diff --git a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java index b7c99c64f3..7e53547e81 100644 --- a/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java +++ b/tools/stress/src/org/apache/cassandra/stress/settings/SettingsMode.java @@ -133,7 +133,7 @@ public class SettingsMode implements Serializable private static abstract class Cql3Options extends GroupedOptions { final OptionSimple api = new OptionSimple("cql3", "", null, "", true); - final OptionSimple protocolVersion = new OptionSimple("protocolVersion=", "[2-4]+", "NEWEST_SUPPORTED", "CQL Protocol Version", false); + final OptionSimple protocolVersion = new OptionSimple("protocolVersion=", "[2-5]+", "NEWEST_SUPPORTED", "CQL Protocol Version", false); final OptionSimple useUnPrepared = new OptionSimple("unprepared", "", null, "force use of unprepared statements", false); final OptionSimple useCompression = new OptionSimple("compression=", "none|lz4|snappy", "none", "", false); final OptionSimple port = new OptionSimple("port=", "[0-9]+", "9046", "", false);