diff --git a/CHANGES.txt b/CHANGES.txt index f7462fa059..f36529a350 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.1 + * Expose auth mode in system_views.clients, nodetool clientstats, metrics (CASSANDRA-19366) * Remove sealed_periods and last_sealed_period tables (CASSANDRA-19189) * Improve setup and initialisation of LocalLog/LogSpec (CASSANDRA-19271) * Refactor structure of caching metrics and expose auth cache metrics via JMX (CASSANDRA-17062) diff --git a/NEWS.txt b/NEWS.txt index dd534dd672..fe8ee12fb8 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -81,6 +81,10 @@ New features - Whether bulk loading of SSTables is allowed. - nodetool tpstats can display core pool size, max pool size and max tasks queued if --verbose / -v flag is specified. system_views.thread_pools adds core_pool_size, max_pool_size and max_tasks_queued columns. + - Authentication mode is exposed in system_views.clients table, nodetool clientstats and ClientMetrics + to help operators identify which authentication modes are being used. nodetool clientstats introduces --verbose flag + behind which this information is visible. + Upgrading --------- diff --git a/doc/modules/cassandra/pages/managing/operating/metrics.adoc b/doc/modules/cassandra/pages/managing/operating/metrics.adoc index 2c709b9d16..c4392fea82 100644 --- a/doc/modules/cassandra/pages/managing/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/managing/operating/metrics.adoc @@ -969,14 +969,87 @@ Reported name format: [cols=",,",options="header",] |=== |Name |Type |Description -|connectedNativeClients |Gauge |Number of clients connected to +|AuthFailure |Meter | Rate of failed authentications + +|AuthSuccess |Meter | Rate of successful authentications + +|ClientsByProtocolVersion |Gauge> | List of +all connections' protocol version and ip address + +|ConnectedNativeClients |Gauge |Number of clients connected to this nodes native protocol server -|connections |Gauge> |List of all connections +|ConnectedNativeClientsByUser |Gauge> |Number of +connnective native clients by username + +|Connections |Gauge> |List of all connections and their state information -|connectedNativeClientsByUser |Gauge |Number of -connnective native clients by username +|PausedConnections|Gauge|Number of connections currently +paused by rate limiter + +|ProtocolException|Meter|Rate of requests resulting in a protocol +exception + +|UnknownException|Meter|Rate of requests resulting in an unknown +exception + +|RequestDiscarded|Meter|Rate of requests discarded by rate limiter + +|RequestDispatched|Meter|Rate of requests dispatched (not discarded) + +|RequestsSizeByIpDistribution|Histogram|Histogram of distribution of +requests coming from unique IPs + +|=== + +== Client Encryption Metrics + +Metrics specific to Client encryption + +*Metric Name*:: +`org.apache.cassandra.metrics.Client..` +*JMX MBean*:: +`org.apache.cassandra.metrics:type=Client scope= name=` + +[cols=",,",options="header",] +|=== +|Name |Type |Description +|ConnectedNativeClients |Gauge |Number of clients connected in +this way to this nodes native protocol server + +|=== + +== Client Authentication Mode-Specific Metrics + +Metrics specific to connectivity for a given Authentication 'mode'. + +*Metric Name*:: +`org.apache.cassandra.metrics.Client..` +*JMX MBean*:: +`org.apache.cassandra.metrics:type=Client scope= name=` + +An authentication mode is a supported method of authentication. The following +authentication modes exist for the given supported `IAuthenticators`: + +* `PasswordAuthenticator` - Password +* `MutualTlsAuthenticator` - MutualTls +* `MutualTlsWithPasswordFallbackAuthenticator` - MutualTls, Password +* `AllowAllAuthenticator` - Unauthenticated + +A custom implementation of `IAuthenticator` may expose metrics on their own +custom mode by implementing `IAuthenticator.getSupportedAuthenticationModes()`. + +[cols=",,",options="header",] +|=== +|Name |Type |Description +|ConnectedNativeClients |Gauge |Number of clients connected and +authenticated to this nodes native protocol server using this mode + +|AuthFailure |Meter | Rate of failed authentications using this mode + +|AuthSuccess |Meter | Rate of successful authentications using this mode + |=== == Batch Metrics diff --git a/src/java/org/apache/cassandra/auth/AllowAllAuthenticator.java b/src/java/org/apache/cassandra/auth/AllowAllAuthenticator.java index 7b21dc60a9..fb0f81a865 100644 --- a/src/java/org/apache/cassandra/auth/AllowAllAuthenticator.java +++ b/src/java/org/apache/cassandra/auth/AllowAllAuthenticator.java @@ -29,6 +29,8 @@ public class AllowAllAuthenticator implements IAuthenticator { private static final SaslNegotiator AUTHENTICATOR_INSTANCE = new Negotiator(); + private static final Set AUTHENTICATION_MODES = Collections.singleton(AuthenticationMode.UNAUTHENTICATED); + public boolean requireAuthentication() { return false; @@ -52,6 +54,12 @@ public class AllowAllAuthenticator implements IAuthenticator return AUTHENTICATOR_INSTANCE; } + @Override + public Set getSupportedAuthenticationModes() + { + return AUTHENTICATION_MODES; + } + public AuthenticatedUser legacyAuthenticate(Map credentialsData) { return AuthenticatedUser.ANONYMOUS_USER; diff --git a/src/java/org/apache/cassandra/auth/AuthenticatedUser.java b/src/java/org/apache/cassandra/auth/AuthenticatedUser.java index 6b31110e95..f8d8a9c790 100644 --- a/src/java/org/apache/cassandra/auth/AuthenticatedUser.java +++ b/src/java/org/apache/cassandra/auth/AuthenticatedUser.java @@ -18,12 +18,17 @@ package org.apache.cassandra.auth; import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.Map; import java.util.Set; import com.google.common.base.Objects; +import org.apache.cassandra.auth.IAuthenticator.AuthenticationMode; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Datacenters; +import static org.apache.cassandra.auth.IAuthenticator.AuthenticationMode.UNAUTHENTICATED; + /** * Returned from IAuthenticator#authenticate(), represents an authenticated user everywhere internally. * @@ -55,13 +60,39 @@ public class AuthenticatedUser private final String name; - // Primary Role of the logged in user + private final AuthenticationMode authenticationMode; + + private final Map metadata; + + // Primary Role of the logged-in user private final RoleResource role; public AuthenticatedUser(String name) + { + this(name, UNAUTHENTICATED); + } + + public AuthenticatedUser(String name, AuthenticationMode authenticationMode) + { + this(name, authenticationMode, Collections.emptyMap()); + } + + /** + * Defines authenticated user context established within a client connection. + * + * @param name The user's role name + * @param authenticationMode How the user was authenticated + * @param metadata contextual metadata about how the user was authenticated. Note that this data is exposed + * through the system_views.clients table, nodetool clientstats and + * {@link org.apache.cassandra.metrics.ClientMetrics}-based JMX Beans. Implementors should + * take care not to store anything sensitive here. + */ + public AuthenticatedUser(String name, AuthenticationMode authenticationMode, Map metadata) { this.name = name; this.role = RoleResource.role(name); + this.authenticationMode = authenticationMode; + this.metadata = metadata; } public String getName() @@ -74,6 +105,26 @@ public class AuthenticatedUser return role; } + /** + * @returns the mode of authentication used to authenticate this user + */ + public AuthenticationMode getAuthenticationMode() + { + return authenticationMode; + } + + /** + * @returns {@link IAuthenticator}-contextual metadata about how the user was authenticated. + *

+ * Note that this data is exposed through the system_views.clients table, + * nodetool clientstats and {@link org.apache.cassandra.metrics.ClientMetrics}-based JMX Beans. + * Implementors should take care not to store anything sensitive here. + */ + public Map getMetadata() + { + return metadata; + } + /** * Checks the user's superuser status. * Only a superuser is allowed to perform CREATE USER and DROP USER queries. @@ -179,6 +230,12 @@ public class AuthenticatedUser @Override public int hashCode() { + // Note: for reasons of maintaining the invariant that an object that equals maintains the same hashCode, + // we do not include mode and metadata in the hashCode calculation. + // This is particularly salient as there are cases where AuthenticatedUser is used as a key in + // Role/Permissions cache. In effect, we would like to treat all connections sharing the same name as the same + // user, where mode and metadata are just additional context about how the user authenticated that + // should not factor into 'equivalence' of users. return Objects.hashCode(name); } } diff --git a/src/java/org/apache/cassandra/auth/IAuthenticator.java b/src/java/org/apache/cassandra/auth/IAuthenticator.java index 89c08243f2..03f48d534d 100644 --- a/src/java/org/apache/cassandra/auth/IAuthenticator.java +++ b/src/java/org/apache/cassandra/auth/IAuthenticator.java @@ -19,9 +19,13 @@ package org.apache.cassandra.auth; import java.net.InetAddress; import java.security.cert.Certificate; +import java.util.Collections; import java.util.Map; +import java.util.Objects; import java.util.Set; +import javax.annotation.Nonnull; + import org.apache.cassandra.exceptions.AuthenticationException; import org.apache.cassandra.exceptions.ConfigurationException; @@ -53,7 +57,7 @@ public interface IAuthenticator return false; } - /** + /** * Set of resources that should be made inaccessible to users and only accessible internally. * * @return Keyspaces, column families that will be unmodifiable by users; other resources. @@ -69,7 +73,7 @@ public interface IAuthenticator /** * Setup is called once upon system startup to initialize the IAuthenticator. - * + *

* For example, use this method to create any required keyspaces/column families. */ void setup(); @@ -78,6 +82,7 @@ public interface IAuthenticator * Provide a SASL handler to perform authentication for an single connection. SASL * is a stateful protocol, so a new instance must be used for each authentication * attempt. + * * @param clientAddress the IP address of the client whom we wish to authenticate, or null * if an internal client (one not connected over the remote transport). * @return org.apache.cassandra.auth.IAuthenticator.SaslNegotiator implementation @@ -90,12 +95,13 @@ public interface IAuthenticator * is a stateful protocol, so a new instance must be used for each authentication * attempt. This method accepts certificates as well. Authentication strategies can * override this method to gain access to client's certificate chain, if present. + * * @param clientAddress the IP address of the client whom we wish to authenticate, or null * if an internal client (one not connected over the remote transport). - * @param certificates the peer's Certificate chain, if present. - * It is expected that these will all be instances of {@link java.security.cert.X509Certificate}, - * but we pass them as the base {@link Certificate} in case future implementations leverage - * other certificate types. + * @param certificates the peer's Certificate chain, if present. + * It is expected that these will all be instances of {@link java.security.cert.X509Certificate}, + * but we pass them as the base {@link Certificate} in case future implementations leverage + * other certificate types. * @return org.apache.cassandra.auth.IAuthenticator.SaslNegotiator implementation * (see {@link org.apache.cassandra.auth.PasswordAuthenticator.PlainTextSaslAuthenticator}) */ @@ -104,12 +110,22 @@ public interface IAuthenticator return newSaslNegotiator(clientAddress); } + /** + * @return The supported authentication 'modes' of this authenticator scheme. + *

+ * This is currently only used for registering metrics tied to authentication by mode. + */ + default Set getSupportedAuthenticationModes() + { + return Collections.emptySet(); + } + /** * A legacy method that is still used by JMX authentication. - * + *

* You should implement this for having JMX authentication through your * authenticator. - * + *

* Should never return null - always throw AuthenticationException instead. * Returning AuthenticatedUser.ANONYMOUS_USER is an option as well if authentication is not required. * @@ -129,7 +145,7 @@ public interface IAuthenticator /** * Evaluates the client response data and generates a byte[] response which may be a further challenge or purely * informational in the case that the negotiation is completed on this round. - * + *

* This method is called each time a {@link org.apache.cassandra.transport.messages.AuthResponse} is received * from a client. After it is called, {@link #isComplete()} is checked to determine whether the negotiation has * finished. If so, an AuthenticatedUser is obtained by calling {@link #getAuthenticatedUser()} and that user @@ -141,8 +157,7 @@ public interface IAuthenticator * * @param clientResponse The non-null (but possibly empty) response sent by the client * @return The possibly null response to send to the client. - * @throws AuthenticationException - * see {@link javax.security.sasl.SaslServer#evaluateResponse(byte[])} + * @throws AuthenticationException see {@link javax.security.sasl.SaslServer#evaluateResponse(byte[])} */ public byte[] evaluateResponse(byte[] clientResponse) throws AuthenticationException; @@ -179,5 +194,68 @@ public interface IAuthenticator { return true; } + + /** + * @return The assumed mode of authentication attempted using this negotiator, this will usually be some value + * of {@link AuthenticationMode#toString()}} unless an implementor provides their own custom authentication + * scheme. + */ + default AuthenticationMode getAuthenticationMode() + { + return AuthenticationMode.UNAUTHENTICATED; + } + } + + /** + * Known modes of authentication supported by Cassandra's provided {@link IAuthenticator} implementations. + */ + abstract class AuthenticationMode + { + private final String displayName; + + /** + * @param displayName How this mode should be displayed in tooling and JMX beans. Note that it is desirable + * for this name to not have spaces as it may not work well with tooling around JMX. + */ + public AuthenticationMode(@Nonnull String displayName) + { + this.displayName = displayName; + } + + /** + * User was not authenticated in any particular way. + */ + public static final AuthenticationMode UNAUTHENTICATED = new AuthenticationMode("Unauthenticated") {}; + + /** + * User authenticated using a password. + */ + public static final AuthenticationMode PASSWORD = new AuthenticationMode("Password") {}; + + /** + * User authenticated using a trusted identity in their client certificate. + */ + public static final AuthenticationMode MTLS = new AuthenticationMode("MutualTls") {}; + + @Override + public String toString() + { + return displayName; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AuthenticationMode that = (AuthenticationMode) o; + return displayName.equals(that.displayName); + } + + @Override + public int hashCode() + { + return Objects.hash(displayName); + } } } diff --git a/src/java/org/apache/cassandra/auth/MutualTlsAuthenticator.java b/src/java/org/apache/cassandra/auth/MutualTlsAuthenticator.java index 8c379a6dbd..a7df4241e9 100644 --- a/src/java/org/apache/cassandra/auth/MutualTlsAuthenticator.java +++ b/src/java/org/apache/cassandra/auth/MutualTlsAuthenticator.java @@ -21,6 +21,7 @@ package org.apache.cassandra.auth; import java.net.InetAddress; import java.security.cert.Certificate; import java.util.Arrays; +import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -40,6 +41,7 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.utils.NoSpamLogger; +import static org.apache.cassandra.auth.IAuthenticator.AuthenticationMode.MTLS; import static org.apache.cassandra.config.EncryptionOptions.ClientAuth.REQUIRED; /* @@ -69,6 +71,10 @@ public class MutualTlsAuthenticator implements IAuthenticator private static final String CACHE_NAME = "IdentitiesCache"; private final IdentityCache identityCache = new IdentityCache(); private final MutualTlsCertificateValidator certificateValidator; + private static final Set AUTHENTICATION_MODES = Collections.singleton(MTLS); + + // key for the 'identity' value in AuthenticatedUser metadata map. + static final String METADATA_IDENTITY_KEY = "identity"; public MutualTlsAuthenticator(Map parameters) { @@ -133,6 +139,12 @@ public class MutualTlsAuthenticator implements IAuthenticator return new CertificateNegotiator(certificates); } + @Override + public Set getSupportedAuthenticationModes() + { + return AUTHENTICATION_MODES; + } + @Override public AuthenticatedUser legacyAuthenticate(Map credentials) throws AuthenticationException { @@ -196,7 +208,13 @@ public class MutualTlsAuthenticator implements IAuthenticator nospamLogger.error(msg, identity); throw new AuthenticationException(MessageFormatter.format(msg, identity).getMessage()); } - return new AuthenticatedUser(role); + return new AuthenticatedUser(role, MTLS, Collections.singletonMap(METADATA_IDENTITY_KEY, identity)); + } + + @Override + public AuthenticationMode getAuthenticationMode() + { + return MTLS; } } diff --git a/src/java/org/apache/cassandra/auth/MutualTlsWithPasswordFallbackAuthenticator.java b/src/java/org/apache/cassandra/auth/MutualTlsWithPasswordFallbackAuthenticator.java index f10b7eb1a0..219353d76a 100644 --- a/src/java/org/apache/cassandra/auth/MutualTlsWithPasswordFallbackAuthenticator.java +++ b/src/java/org/apache/cassandra/auth/MutualTlsWithPasswordFallbackAuthenticator.java @@ -21,6 +21,9 @@ package org.apache.cassandra.auth; import java.net.InetAddress; import java.security.cert.Certificate; import java.util.Map; +import java.util.Set; + +import com.google.common.collect.Sets; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; @@ -34,9 +37,13 @@ import org.apache.cassandra.exceptions.ConfigurationException; public class MutualTlsWithPasswordFallbackAuthenticator extends PasswordAuthenticator { private final MutualTlsAuthenticator mutualTlsAuthenticator; + + private final Set AUTHENTICATION_MODES; + public MutualTlsWithPasswordFallbackAuthenticator(Map parameters) { mutualTlsAuthenticator = new MutualTlsAuthenticator(parameters); + AUTHENTICATION_MODES = Sets.union(super.getSupportedAuthenticationModes(), mutualTlsAuthenticator.getSupportedAuthenticationModes()); } @Override @@ -52,6 +59,12 @@ public class MutualTlsWithPasswordFallbackAuthenticator extends PasswordAuthenti return true; } + @Override + public Set getSupportedAuthenticationModes() + { + return AUTHENTICATION_MODES; + } + @Override public SaslNegotiator newSaslNegotiator(InetAddress clientAddress, Certificate[] certificates) { diff --git a/src/java/org/apache/cassandra/auth/PasswordAuthenticator.java b/src/java/org/apache/cassandra/auth/PasswordAuthenticator.java index b96aabc76a..77bc3e51b3 100644 --- a/src/java/org/apache/cassandra/auth/PasswordAuthenticator.java +++ b/src/java/org/apache/cassandra/auth/PasswordAuthenticator.java @@ -20,6 +20,7 @@ package org.apache.cassandra.auth; import java.net.InetAddress; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -72,6 +73,7 @@ public class PasswordAuthenticator implements IAuthenticator, AuthCache.BulkLoad // really this is a rolename now, but as it only matters for Thrift, we leave it for backwards compatibility public static final String USERNAME_KEY = "username"; public static final String PASSWORD_KEY = "password"; + private static final Set AUTHENTICATION_MODES = Collections.singleton(AuthenticationMode.PASSWORD); static final byte NUL = 0; private SelectStatement authenticateStatement; @@ -164,7 +166,7 @@ public class PasswordAuthenticator implements IAuthenticator, AuthCache.BulkLoad if (!checkpw(password, hash)) throw new AuthenticationException(String.format("Provided username %s and/or password are incorrect", username)); - return new AuthenticatedUser(username); + return new AuthenticatedUser(username, AuthenticationMode.PASSWORD); } private String queryHashedPassword(String username) @@ -238,6 +240,12 @@ public class PasswordAuthenticator implements IAuthenticator, AuthCache.BulkLoad return new PlainTextSaslAuthenticator(); } + @Override + public Set getSupportedAuthenticationModes() + { + return AUTHENTICATION_MODES; + } + private static SelectStatement prepare(String query) { return (SelectStatement) QueryProcessor.getStatement(query, ClientState.forInternalCalls()); @@ -269,6 +277,12 @@ public class PasswordAuthenticator implements IAuthenticator, AuthCache.BulkLoad return authenticate(username, password); } + @Override + public AuthenticationMode getAuthenticationMode() + { + return AuthenticationMode.PASSWORD; + } + /** * SASL PLAIN mechanism specifies that credentials are encoded in a * sequence of UTF-8 bytes, delimited by 0 (US-ASCII NUL). diff --git a/src/java/org/apache/cassandra/db/virtual/ClientsTable.java b/src/java/org/apache/cassandra/db/virtual/ClientsTable.java index 027bb9becd..8f99f3a771 100644 --- a/src/java/org/apache/cassandra/db/virtual/ClientsTable.java +++ b/src/java/org/apache/cassandra/db/virtual/ClientsTable.java @@ -18,8 +18,15 @@ package org.apache.cassandra.db.virtual; import java.net.InetSocketAddress; +import java.util.Map.Entry; +import java.util.stream.Collectors; -import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.marshal.BooleanType; +import org.apache.cassandra.db.marshal.InetAddressType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.metrics.ClientMetrics; import org.apache.cassandra.schema.TableMetadata; @@ -41,6 +48,8 @@ final class ClientsTable extends AbstractVirtualTable private static final String SSL_PROTOCOL = "ssl_protocol"; private static final String SSL_CIPHER_SUITE = "ssl_cipher_suite"; private static final String KEYSPACE_NAME = "keyspace_name"; + private static final String AUTHENTICATION_MODE = "authentication_mode"; + private static final String AUTHENTICATION_METADATA = "authentication_metadata"; ClientsTable(String keyspace) { @@ -62,6 +71,8 @@ final class ClientsTable extends AbstractVirtualTable .addRegularColumn(SSL_PROTOCOL, UTF8Type.instance) .addRegularColumn(SSL_CIPHER_SUITE, UTF8Type.instance) .addRegularColumn(KEYSPACE_NAME, UTF8Type.instance) + .addRegularColumn(AUTHENTICATION_MODE, UTF8Type.instance) + .addRegularColumn(AUTHENTICATION_METADATA, MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false)) .build()); } @@ -86,7 +97,11 @@ final class ClientsTable extends AbstractVirtualTable .column(SSL_ENABLED, client.sslEnabled()) .column(SSL_PROTOCOL, client.sslProtocol().orElse(null)) .column(SSL_CIPHER_SUITE, client.sslCipherSuite().orElse(null)) - .column(KEYSPACE_NAME, client.keyspace().orElse(null)); + .column(KEYSPACE_NAME, client.keyspace().orElse(null)) + .column(AUTHENTICATION_MODE, client.authenticationMode().toString()) + .column(AUTHENTICATION_METADATA, client.authenticationMetadata() + .entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, entry -> String.valueOf(entry.getValue())))); } return result; diff --git a/src/java/org/apache/cassandra/metrics/ClientMetrics.java b/src/java/org/apache/cassandra/metrics/ClientMetrics.java index 4e3eb4f8ec..f0d075bd0c 100644 --- a/src/java/org/apache/cassandra/metrics/ClientMetrics.java +++ b/src/java/org/apache/cassandra/metrics/ClientMetrics.java @@ -18,14 +18,32 @@ */ package org.apache.cassandra.metrics; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; + +import com.google.common.annotations.VisibleForTesting; import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.Reservoir; -import org.apache.cassandra.transport.*; +import org.apache.cassandra.auth.AuthenticatedUser; +import org.apache.cassandra.auth.IAuthenticator; +import org.apache.cassandra.auth.IAuthenticator.AuthenticationMode; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.transport.ClientResourceLimits; +import org.apache.cassandra.transport.ClientStat; +import org.apache.cassandra.transport.ConnectedClient; +import org.apache.cassandra.transport.Server; +import org.apache.cassandra.transport.ServerConnection; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; @@ -38,10 +56,35 @@ public final class ClientMetrics private volatile boolean initialized = false; private Collection servers = Collections.emptyList(); - private Meter authSuccess; - private Meter authFailure; + @VisibleForTesting + Meter authSuccess; + + @VisibleForTesting + final Map authSuccessByMode = new HashMap<>(); + + @VisibleForTesting + Meter authFailure; + + @VisibleForTesting + final Map authFailureByMode = new HashMap<>(); + + @VisibleForTesting + Gauge connectedNativeClients; + + @VisibleForTesting + Gauge encryptedConnectedNativeClients; + + @VisibleForTesting + Gauge unencryptedConnectedNativeClients; + + @VisibleForTesting + Gauge> connectedNativeClientsByUser; + + @VisibleForTesting + final Map> connectedNativeClientsByAuthMode = new HashMap<>(); + private AtomicInteger pausedConnections; - + @SuppressWarnings({ "unused", "FieldCanBeLocal" }) private Gauge pausedConnectionsGauge; @@ -51,18 +94,48 @@ public final class ClientMetrics private Meter protocolException; private Meter unknownException; + private static final String AUTH_SUCCESS = "AuthSuccess"; + + private static final String AUTH_FAILURE = "AuthFailure"; + + private static final String CONNECTED_NATIVE_CLIENTS = "ConnectedNativeClients"; + private ClientMetrics() { } + /** + * @deprecated by {@link #markAuthSuccess(AuthenticationMode)} + */ + @Deprecated(since="5.1", forRemoval = true) public void markAuthSuccess() { - authSuccess.mark(); + markAuthSuccess(null); } + public void markAuthSuccess(AuthenticationMode authenticationMode) + { + authSuccess.mark(); + Meter meterByMode; + if (authenticationMode != null && (meterByMode = authSuccessByMode.get(authenticationMode)) != null) + meterByMode.mark(); + } + + /** + * @deprecated by {@link #markAuthFailure(AuthenticationMode)} + */ + @Deprecated(since="5.1", forRemoval = true) public void markAuthFailure() + { + markAuthFailure(null); + } + + public void markAuthFailure(AuthenticationMode authenticationMode) { authFailure.mark(); + Meter meterByMode; + if (authenticationMode != null && (meterByMode = authFailureByMode.get(authenticationMode)) != null) + meterByMode.mark(); } public void pauseConnection() { pausedConnections.incrementAndGet(); } @@ -99,8 +172,8 @@ public final class ClientMetrics this.servers = servers; // deprecated the lower-cased initial letter metric names in 4.0 - registerGauge("ConnectedNativeClients", "connectedNativeClients", this::countConnectedClients); - registerGauge("ConnectedNativeClientsByUser", "connectedNativeClientsByUser", this::countConnectedClientsByUser); + connectedNativeClients = registerGauge(CONNECTED_NATIVE_CLIENTS, "connectedNativeClients", this::countConnectedClients); + connectedNativeClientsByUser = registerGauge("ConnectedNativeClientsByUser", "connectedNativeClientsByUser", this::countConnectedClientsByUser); registerGauge("Connections", "connections", this::connectedClients); registerGauge("ClientsByProtocolVersion", "clientsByProtocolVersion", this::recentClientStats); registerGauge("RequestsSize", ClientResourceLimits::getCurrentGlobalUsage); @@ -118,6 +191,27 @@ public final class ClientMetrics authSuccess = registerMeter("AuthSuccess"); authFailure = registerMeter("AuthFailure"); + // For each of SSL, non-SSL register a gauge: + encryptedConnectedNativeClients = registerGauge(new DefaultNameFactory("Client", "Encrypted"), CONNECTED_NATIVE_CLIENTS, () -> countConnectedClients((ServerConnection::isSSL))); + unencryptedConnectedNativeClients = registerGauge(new DefaultNameFactory("Client", "Unencrypted"), CONNECTED_NATIVE_CLIENTS, () -> countConnectedClients(((ServerConnection connection) -> !connection.isSSL()))); + + // for each supported authentication mode, register a meter for success and failures. + IAuthenticator authenticator = DatabaseDescriptor.getAuthenticator(); + for (AuthenticationMode mode : authenticator.getSupportedAuthenticationModes()) + { + MetricNameFactory factory = new DefaultNameFactory("Client", mode.toString()); + authSuccessByMode.put(mode, registerMeter(factory, AUTH_SUCCESS)); + authFailureByMode.put(mode, registerMeter(factory, AUTH_FAILURE)); + + Gauge clients = registerGauge(factory, CONNECTED_NATIVE_CLIENTS, () -> countConnectedClients((ServerConnection connection) -> { + AuthenticatedUser user = connection.getClientState().getUser(); + return Optional.ofNullable(user) + .map(u -> mode.equals(u.getAuthenticationMode())) + .orElse(false); + })); + connectedNativeClientsByAuthMode.put(mode, clients); + } + pausedConnections = new AtomicInteger(); pausedConnectionsGauge = registerGauge("PausedConnections", pausedConnections::get); requestDiscarded = registerMeter("RequestDiscarded"); @@ -163,6 +257,16 @@ public final class ClientMetrics return clients; } + private int countConnectedClients(Predicate predicate) + { + int count = 0; + + for (Server server : servers) + count += server.countConnectedClients(predicate); + + return count; + } + private List> recentClientStats() { List> stats = new ArrayList<>(); @@ -178,17 +282,28 @@ public final class ClientMetrics private Gauge registerGauge(String name, Gauge gauge) { - return Metrics.register(factory.createMetricName(name), gauge); + return registerGauge(factory, name, gauge); } - - private void registerGauge(String name, String deprecated, Gauge gauge) + + private Gauge registerGauge(MetricNameFactory metricNameFactory, String name, Gauge gauge) { - Gauge registeredGauge = registerGauge(name, gauge); + return Metrics.register(metricNameFactory.createMetricName(name), gauge); + } + + private Gauge registerGauge(String name, String deprecated, Gauge gauge) + { + Gauge registeredGauge = registerGauge(name, gauge); Metrics.registerMBean(registeredGauge, factory.createMetricName(deprecated).getMBeanName()); + return registeredGauge; } private Meter registerMeter(String name) { - return Metrics.meter(factory.createMetricName(name)); + return registerMeter(factory, name); + } + + private Meter registerMeter(MetricNameFactory metricNameFactory, String name) + { + return Metrics.meter(metricNameFactory.createMetricName(name)); } } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 7b8fd1e096..180fc77766 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2057,7 +2057,7 @@ public class NodeProbe implements AutoCloseable case "connections": // List> - list of all native connections and their properties case "connectedNativeClients": // number of connected native clients case "connectedNativeClientsByUser": // number of native clients by username - case "clientsByProtocolVersion": // number of native clients by username + case "clientsByProtocolVersion": // number of native clients by protocol version return JMX.newMBeanProxy(mbeanServerConn, new ObjectName("org.apache.cassandra.metrics:type=Client,name=" + metricName), CassandraMetricsRegistry.JmxGaugeMBean.class).getValue(); diff --git a/src/java/org/apache/cassandra/tools/nodetool/ClientStats.java b/src/java/org/apache/cassandra/tools/nodetool/ClientStats.java index 7ed9b9d443..fcc9d76629 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/ClientStats.java +++ b/src/java/org/apache/cassandra/tools/nodetool/ClientStats.java @@ -23,7 +23,9 @@ import java.util.Date; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.stream.Collectors; +import com.google.common.collect.ImmutableList; import io.airlift.airline.Command; import io.airlift.airline.Option; import org.apache.cassandra.tools.NodeProbe; @@ -47,6 +49,9 @@ public class ClientStats extends NodeToolCmd @Option(title = "list_connections_with_client_options", name = "--client-options", description = "Lists all connections and the client options") private boolean clientOptions = false; + @Option(title = "verbose", name = "--verbose", description = "Lists all connections with additional details (client options, authenticator-specific metadata and more)") + private boolean verbose = false; + @Override public void execute(NodeProbe probe) { @@ -86,55 +91,36 @@ public class ClientStats extends NodeToolCmd return; } - if (listConnections) + // Note: for compatbility with existing implementation if someone passes --all (listConnections), + // --client-options, and --metadata all three will be printed. + List> clients = (List>) probe.getClientMetric("connections"); + if (!clients.isEmpty() && (listConnections || clientOptions || verbose)) { - List> clients = (List>) probe.getClientMetric("connections"); - if (!clients.isEmpty()) + ImmutableList.Builder tableHeaderBuilder = ImmutableList.builder() + .add("Address", "SSL", "Cipher", "Protocol", "Version", + "User", "Keyspace", "Requests", "Driver-Name", + "Driver-Version"); + ImmutableList.Builder tableFieldsBuilder = ImmutableList.builder() + .add(ConnectedClient.ADDRESS, ConnectedClient.SSL, + ConnectedClient.CIPHER, ConnectedClient.PROTOCOL, + ConnectedClient.VERSION, ConnectedClient.USER, + ConnectedClient.KEYSPACE, ConnectedClient.REQUESTS, + ConnectedClient.DRIVER_NAME, ConnectedClient.DRIVER_VERSION); + if (clientOptions || verbose) { - TableBuilder table = new TableBuilder(); - table.add("Address", "SSL", "Cipher", "Protocol", "Version", "User", "Keyspace", "Requests", "Driver-Name", "Driver-Version"); - for (Map conn : clients) - { - table.add(conn.get(ConnectedClient.ADDRESS), - conn.get(ConnectedClient.SSL), - conn.get(ConnectedClient.CIPHER), - conn.get(ConnectedClient.PROTOCOL), - conn.get(ConnectedClient.VERSION), - conn.get(ConnectedClient.USER), - conn.get(ConnectedClient.KEYSPACE), - conn.get(ConnectedClient.REQUESTS), - conn.get(ConnectedClient.DRIVER_NAME), - conn.get(ConnectedClient.DRIVER_VERSION)); - } - table.printTo(out); - out.println(); + tableHeaderBuilder.add("Client-Options"); + tableFieldsBuilder.add(ConnectedClient.CLIENT_OPTIONS); } - } - if (clientOptions) - { - List> clients = (List>) probe.getClientMetric("connections"); - if (!clients.isEmpty()) + if (verbose) { - TableBuilder table = new TableBuilder(); - table.add("Address", "SSL", "Cipher", "Protocol", "Version", "User", "Keyspace", "Requests", "Driver-Name", "Driver-Version", "Client-Options"); - for (Map conn : clients) - { - table.add(conn.get(ConnectedClient.ADDRESS), - conn.get(ConnectedClient.SSL), - conn.get(ConnectedClient.CIPHER), - conn.get(ConnectedClient.PROTOCOL), - conn.get(ConnectedClient.VERSION), - conn.get(ConnectedClient.USER), - conn.get(ConnectedClient.KEYSPACE), - conn.get(ConnectedClient.REQUESTS), - conn.get(ConnectedClient.DRIVER_NAME), - conn.get(ConnectedClient.DRIVER_VERSION), - conn.get(ConnectedClient.CLIENT_OPTIONS)); - } - table.printTo(out); - out.println(); + tableHeaderBuilder.add("Auth-Mode", "Auth-Metadata"); + tableFieldsBuilder.add(ConnectedClient.AUTHENTICATION_MODE, ConnectedClient.AUTHENTICATION_METADATA); } + + List tableHeader = tableHeaderBuilder.build(); + List tableFields = tableFieldsBuilder.build(); + printTable(out, tableHeader, tableFields, clients); } Map connectionsByUser = (Map) probe.getClientMetric("connectedNativeClientsByUser"); @@ -149,4 +135,27 @@ public class ClientStats extends NodeToolCmd } table.printTo(out); } + + /** + * Convenience function to print a table with the given header and the resolved fields for each connection. + * + * @param out print stream to print to. + * @param headers headers for the table + * @param tableFields the fields from {@link ConnectedClient} to retrieve from each client connection. + * @param clients the clients to print, each client being a row inthe table. + */ + private void printTable(PrintStream out, List headers, List tableFields, List> clients) + { + TableBuilder table = new TableBuilder(); + table.add(headers); + for (Map conn : clients) + { + List connectionFieldValues = tableFields.stream() + .map(conn::get) + .collect(Collectors.toList()); + table.add(connectionFieldValues); + } + table.printTo(out); + out.println(); + } } diff --git a/src/java/org/apache/cassandra/transport/ConnectedClient.java b/src/java/org/apache/cassandra/transport/ConnectedClient.java index a4af32fca1..b8624de983 100644 --- a/src/java/org/apache/cassandra/transport/ConnectedClient.java +++ b/src/java/org/apache/cassandra/transport/ConnectedClient.java @@ -27,8 +27,11 @@ import com.google.common.collect.ImmutableMap; import io.netty.handler.ssl.SslHandler; import org.apache.cassandra.auth.AuthenticatedUser; +import org.apache.cassandra.auth.IAuthenticator; import org.apache.cassandra.service.ClientState; +import static org.apache.cassandra.auth.IAuthenticator.AuthenticationMode.UNAUTHENTICATED; + public final class ConnectedClient { public static final String ADDRESS = "address"; @@ -42,6 +45,8 @@ public final class ConnectedClient public static final String SSL = "ssl"; public static final String CIPHER = "cipher"; public static final String PROTOCOL = "protocol"; + public static final String AUTHENTICATION_MODE = "authenticationMode"; + public static final String AUTHENTICATION_METADATA = "authenticationMetadata"; private static final String UNDEFINED = "undefined"; @@ -124,6 +129,24 @@ public final class ConnectedClient : Optional.empty(); } + public Map authenticationMetadata() + { + AuthenticatedUser user = state().getUser(); + + return null != user + ? user.getMetadata() + : Collections.emptyMap(); + } + + public IAuthenticator.AuthenticationMode authenticationMode() + { + AuthenticatedUser user = state().getUser(); + + return null != user + ? user.getAuthenticationMode() + : UNAUTHENTICATED; + } + private ClientState state() { return connection.getClientState(); @@ -150,6 +173,10 @@ public final class ConnectedClient .put(SSL, Boolean.toString(sslEnabled())) .put(CIPHER, sslCipherSuite().orElse(UNDEFINED)) .put(PROTOCOL, sslProtocol().orElse(UNDEFINED)) + .put(AUTHENTICATION_MODE, authenticationMode().toString()) + .put(AUTHENTICATION_METADATA, Joiner.on(", ") + .withKeyValueSeparator("=") + .join(authenticationMetadata())) .build(); } } diff --git a/src/java/org/apache/cassandra/transport/Server.java b/src/java/org/apache/cassandra/transport/Server.java index 5ec6d20187..02de68596c 100644 --- a/src/java/org/apache/cassandra/transport/Server.java +++ b/src/java/org/apache/cassandra/transport/Server.java @@ -23,6 +23,7 @@ import java.net.UnknownHostException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Predicate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -145,6 +146,14 @@ public class Server implements CassandraDaemon.Server return connectionTracker.countConnectedClientsByUser(); } + /** + * @return A count of the number of clients matching the given predicate. + */ + public int countConnectedClients(Predicate predicate) + { + return connectionTracker.countConnectedClients(predicate); + } + public List getConnectedClients() { List result = new ArrayList<>(); @@ -306,6 +315,24 @@ public class Server implements CassandraDaemon.Server return allChannels.size() != 0 ? allChannels.size() - 1 : 0; } + int countConnectedClients(Predicate predicate) + { + int count = 0; + for (Channel c : allChannels) + { + Connection connection = c.attr(Connection.attributeKey).get(); + if (connection instanceof ServerConnection) + { + ServerConnection conn = (ServerConnection) connection; + if (predicate.test(conn)) + { + count++; + } + } + } + return count; + } + Map countConnectedClientsByUser() { Map result = new HashMap<>(); diff --git a/src/java/org/apache/cassandra/transport/ServerConnection.java b/src/java/org/apache/cassandra/transport/ServerConnection.java index dd6fa313ae..21f2e0b0e6 100644 --- a/src/java/org/apache/cassandra/transport/ServerConnection.java +++ b/src/java/org/apache/cassandra/transport/ServerConnection.java @@ -142,4 +142,15 @@ public class ServerConnection extends Connection } return certificates; } + + /** + * @return Whether this connection is SSL-encrypted. + */ + public boolean isSSL() + { + // If an SslHandler is present on the pipeline, the connection is using ssl. + SslHandler sslHandler = (SslHandler) channel().pipeline() + .get("ssl"); + return sslHandler != null; + } } diff --git a/src/java/org/apache/cassandra/transport/messages/AuthUtil.java b/src/java/org/apache/cassandra/transport/messages/AuthUtil.java index 1009984eac..1c091c7278 100644 --- a/src/java/org/apache/cassandra/transport/messages/AuthUtil.java +++ b/src/java/org/apache/cassandra/transport/messages/AuthUtil.java @@ -58,6 +58,7 @@ public final class AuthUtil static Response handleLogin(Connection connection, QueryState queryState, byte[] token, BiFunction messageToSendBasedOnNegotiation) { + IAuthenticator.SaslNegotiator negotiator = ((ServerConnection) connection).getSaslNegotiator(queryState); try { // client-side timeout can disconnect while sitting in auth executor queue so (client default 12s) @@ -66,13 +67,12 @@ public final class AuthUtil { throw new AuthenticationException("Auth check after connection closed"); } - IAuthenticator.SaslNegotiator negotiator = ((ServerConnection) connection).getSaslNegotiator(queryState); byte[] challenge = negotiator.evaluateResponse(token); if (negotiator.isComplete()) { AuthenticatedUser user = negotiator.getAuthenticatedUser(); queryState.getClientState().login(user); - ClientMetrics.instance.markAuthSuccess(); + ClientMetrics.instance.markAuthSuccess(user.getAuthenticationMode()); AuthEvents.instance.notifyAuthSuccess(queryState); // authentication is complete, complete the authentication flow. return messageToSendBasedOnNegotiation.apply(true, challenge); @@ -85,7 +85,7 @@ public final class AuthUtil } catch (AuthenticationException e) { - ClientMetrics.instance.markAuthFailure(); + ClientMetrics.instance.markAuthFailure(negotiator.getAuthenticationMode()); AuthEvents.instance.notifyAuthFailure(queryState, e); return ErrorMessage.fromException(e); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/AbstractEncryptionOptionsImpl.java b/test/distributed/org/apache/cassandra/distributed/test/AbstractEncryptionOptionsImpl.java index 5f119c6bdf..51af668c29 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/AbstractEncryptionOptionsImpl.java +++ b/test/distributed/org/apache/cassandra/distributed/test/AbstractEncryptionOptionsImpl.java @@ -28,6 +28,8 @@ import javax.net.ssl.SSLHandshakeException; import javax.net.ssl.SSLSession; import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.transport.TlsTestUtils; import org.apache.cassandra.utils.concurrent.Condition; import org.junit.Assert; import org.slf4j.Logger; @@ -62,10 +64,10 @@ import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeConditio public class AbstractEncryptionOptionsImpl extends TestBaseImpl { final Logger logger = LoggerFactory.getLogger(EncryptionOptions.class); - final static String validKeyStorePath = "test/conf/cassandra_ssl_test.keystore"; - final static String validKeyStorePassword = "cassandra"; - final static String validTrustStorePath = "test/conf/cassandra_ssl_test.truststore"; - final static String validTrustStorePassword = "cassandra"; + final static String validKeyStorePath = TlsTestUtils.SERVER_KEYSTORE_PATH; + final static String validKeyStorePassword = TlsTestUtils.SERVER_KEYSTORE_PASSWORD; + final static String validTrustStorePath = TlsTestUtils.SERVER_TRUSTSTORE_PATH; + final static String validTrustStorePassword = TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD; // Base configuration map for a valid keystore that can be opened final static Map validKeystore = ImmutableMap.of("keystore", validKeyStorePath, diff --git a/test/distributed/org/apache/cassandra/distributed/test/AuthTest.java b/test/distributed/org/apache/cassandra/distributed/test/AuthTest.java index 8f67a6ac79..f424691eac 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/AuthTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/AuthTest.java @@ -27,7 +27,6 @@ import org.junit.Test; import com.datastax.driver.core.PlainTextAuthProvider; import com.datastax.driver.core.Session; import com.datastax.driver.core.policies.DCAwareRoundRobinPolicy; -import org.apache.cassandra.auth.CassandraRoleManager; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.ICoordinator; @@ -43,6 +42,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL; import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.apache.cassandra.distributed.util.Auth.waitForExistingRoles; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -161,14 +161,6 @@ public class AuthTest extends TestBaseImpl return cluster.bootstrap(config); } - private void waitForExistingRoles(IInvokableInstance instance) - { - await().pollDelay(1, SECONDS) - .pollInterval(1, SECONDS) - .atMost(30, SECONDS) - .until(() -> instance.callOnInstance(CassandraRoleManager::hasExistingRoles)); - } - private long getPasswordWritetime(ICoordinator coordinator) { return (Long) coordinator.execute("SELECT WRITETIME (salted_hash) from system_auth.roles where role = 'cassandra'", diff --git a/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionEnforcementTest.java b/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionEnforcementTest.java index a37f092a74..142a95b362 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionEnforcementTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/InternodeEncryptionEnforcementTest.java @@ -55,6 +55,7 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.OutboundConnections; import org.apache.cassandra.net.PingRequest; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.transport.TlsTestUtils; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.awaitility.Awaitility; @@ -199,10 +200,10 @@ public final class InternodeEncryptionEnforcementTest extends TestBaseImpl encryption.put("internode_encryption", "none"); if (c.num() == 1) { - encryption.put("keystore", "test/conf/cassandra_ssl_test.keystore"); - encryption.put("keystore_password", "cassandra"); - encryption.put("truststore", "test/conf/cassandra_ssl_test.truststore"); - encryption.put("truststore_password", "cassandra"); + encryption.put("keystore", TlsTestUtils.SERVER_KEYSTORE_PATH); + encryption.put("keystore_password", TlsTestUtils.SERVER_KEYSTORE_PASSWORD); + encryption.put("truststore", TlsTestUtils.SERVER_TRUSTSTORE_PATH); + encryption.put("truststore_password", TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD); encryption.put("internode_encryption", "all"); } c.set("server_encryption_options", encryption); @@ -276,10 +277,10 @@ public final class InternodeEncryptionEnforcementTest extends TestBaseImpl c.with(Feature.NETWORK); c.with(Feature.NATIVE_PROTOCOL); - HashMap encryption = new HashMap<>(); encryption.put("keystore", "test/conf/cassandra_ssl_test.keystore"); - encryption.put("keystore_password", "cassandra"); - encryption.put("truststore", "test/conf/cassandra_ssl_test.truststore"); - encryption.put("truststore_password", "cassandra"); + HashMap encryption = new HashMap<>(); encryption.put("keystore", TlsTestUtils.SERVER_KEYSTORE_PATH); + encryption.put("keystore_password", TlsTestUtils.SERVER_KEYSTORE_PASSWORD); + encryption.put("truststore", TlsTestUtils.SERVER_TRUSTSTORE_PATH); + encryption.put("truststore_password", TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD); encryption.put("internode_encryption", "dc"); c.set("server_encryption_options", encryption); }) @@ -395,10 +396,10 @@ public final class InternodeEncryptionEnforcementTest extends TestBaseImpl c.with(Feature.NATIVE_PROTOCOL); HashMap encryption = new HashMap<>(); - encryption.put("keystore", "test/conf/cassandra_ssl_test.keystore"); - encryption.put("keystore_password", "cassandra"); - encryption.put("truststore", "test/conf/cassandra_ssl_test.truststore"); - encryption.put("truststore_password", "cassandra"); + encryption.put("keystore", TlsTestUtils.SERVER_KEYSTORE_PATH); + encryption.put("keystore_password", TlsTestUtils.SERVER_KEYSTORE_PASSWORD); + encryption.put("truststore", TlsTestUtils.SERVER_TRUSTSTORE_PATH); + encryption.put("truststore_password", TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD); encryption.put("internode_encryption", "all"); encryption.put("require_client_auth", "true"); c.set("server_encryption_options", encryption); @@ -430,8 +431,8 @@ public final class InternodeEncryptionEnforcementTest extends TestBaseImpl // Check if the presented certificates during internode authentication are the ones in the keystores // configured in the cassandra.yaml configuration. KeyStore keyStore = KeyStore.getInstance("JKS"); - char[] keyStorePassword = "cassandra".toCharArray(); - InputStream keyStoreData = new FileInputStream("test/conf/cassandra_ssl_test.keystore"); + char[] keyStorePassword = TlsTestUtils.SERVER_KEYSTORE_PASSWORD.toCharArray(); + InputStream keyStoreData = new FileInputStream(TlsTestUtils.SERVER_KEYSTORE_PATH); keyStore.load(keyStoreData, keyStorePassword); return certificates != null && certificates.length != 0 && keyStore.getCertificate("cassandra_ssl_test").equals(certificates[0]); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java b/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java index 86d4e8fa26..dfb3a6656d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/NativeTransportEncryptionOptionsTest.java @@ -40,6 +40,7 @@ import com.datastax.shaded.netty.handler.ssl.SslContext; import com.datastax.shaded.netty.handler.ssl.SslContextBuilder; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.transport.TlsTestUtils; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -403,10 +404,10 @@ public class NativeTransportEncryptionOptionsTest extends AbstractEncryptionOpti private SSLOptions sslOptions(boolean withKeyStore) throws Exception { SslContextBuilder sslContextBuilder = SslContextBuilder.forClient(); - sslContextBuilder.trustManager(createTrustManagerFactory("test/conf/cassandra_ssl_test.truststore", "cassandra")); + sslContextBuilder.trustManager(createTrustManagerFactory(TlsTestUtils.SERVER_TRUSTSTORE_PATH, TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD)); if (withKeyStore) { - sslContextBuilder.keyManager(createKeyManagerFactory("test/conf/cassandra_ssl_test_outbound.keystore", "cassandra")); + sslContextBuilder.keyManager(createKeyManagerFactory(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PATH, TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PASSWORD)); } SslContext sslContext = sslContextBuilder.build(); @@ -428,11 +429,11 @@ public class NativeTransportEncryptionOptionsTest extends AbstractEncryptionOpti InetAddress address = cluster.get(1).config().broadcastAddress().getAddress(); SslContextBuilder sslContextBuilder = SslContextBuilder.forClient(); if (ipInSAN) - sslContextBuilder.keyManager(createKeyManagerFactory("test/conf/cassandra_ssl_test_endpoint_verify.keystore", "cassandra")); + sslContextBuilder.keyManager(createKeyManagerFactory(TlsTestUtils.SERVER_KEYSTORE_ENDPOINT_VERIFY_PATH, TlsTestUtils.SERVER_KEYSTORE_ENDPOINT_VERIFY_PASSWORD)); else - sslContextBuilder.keyManager(createKeyManagerFactory("test/conf/cassandra_ssl_test_outbound.keystore", "cassandra")); + sslContextBuilder.keyManager(createKeyManagerFactory(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PATH, TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PASSWORD)); - SslContext sslContext = sslContextBuilder.trustManager(createTrustManagerFactory("test/conf/cassandra_ssl_test.truststore", "cassandra")) + SslContext sslContext = sslContextBuilder.trustManager(createTrustManagerFactory(TlsTestUtils.SERVER_TRUSTSTORE_PATH, TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD)) .build(); final SSLOptions sslOptions = socketChannel -> sslContext.newHandler(socketChannel.alloc()); com.datastax.driver.core.Cluster driverCluster = com.datastax.driver.core.Cluster.builder() diff --git a/test/unit/org/apache/cassandra/auth/AuthTestUtils.java b/test/unit/org/apache/cassandra/auth/AuthTestUtils.java index d2e40bd956..01d13bfdd3 100644 --- a/test/unit/org/apache/cassandra/auth/AuthTestUtils.java +++ b/test/unit/org/apache/cassandra/auth/AuthTestUtils.java @@ -59,8 +59,11 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.transport.messages.ResultMessage; import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.auth.AuthKeyspace.CIDR_GROUPS; import static org.apache.cassandra.schema.SchemaConstants.AUTH_KEYSPACE_NAME; +import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertNotNull; @@ -420,4 +423,16 @@ public class AuthTestUtils System.arraycopy(passwordBytes, 0, token, usernameBytes.length + 2, passwordBytes.length); return token; } + + /** + * Block on waiting for existence of the initial superuser roles. This should be used by all tests that + * rely on existence of these users. + */ + public static void waitForExistingRoles() + { + await().pollDelay(0, MILLISECONDS) + .pollInterval(250, MILLISECONDS) + .atMost(10, SECONDS) + .until(CassandraRoleManager::hasExistingRoles); + } } diff --git a/test/unit/org/apache/cassandra/auth/MutualTlsInternodeAuthenticatorTest.java b/test/unit/org/apache/cassandra/auth/MutualTlsInternodeAuthenticatorTest.java index d9fc3d0a3c..8fdd23a444 100644 --- a/test/unit/org/apache/cassandra/auth/MutualTlsInternodeAuthenticatorTest.java +++ b/test/unit/org/apache/cassandra/auth/MutualTlsInternodeAuthenticatorTest.java @@ -41,6 +41,7 @@ import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.transport.TlsTestUtils; import static org.apache.cassandra.auth.AuthTestUtils.loadCertificateChain; import static org.apache.cassandra.auth.IInternodeAuthenticator.InternodeConnectionDirection.INBOUND; @@ -163,10 +164,10 @@ public class MutualTlsInternodeAuthenticatorTest public void testNoIdentitiesInKeystore() { Config config = DatabaseDescriptor.getRawConfig(); - config.server_encryption_options = config.server_encryption_options.withOutboundKeystore("test/conf/cassandra_ssl_test.keystore") - .withOutboundKeystorePassword("cassandra"); + config.server_encryption_options = config.server_encryption_options.withOutboundKeystore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withOutboundKeystorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD); expectedException.expect(ConfigurationException.class); - expectedException.expectMessage("No identity was extracted from the outbound keystore 'test/conf/cassandra_ssl_test.keystore'"); + expectedException.expectMessage(String.format("No identity was extracted from the outbound keystore '%s'", TlsTestUtils.SERVER_KEYSTORE_PATH)); new MutualTlsInternodeAuthenticator(getParams()); } diff --git a/test/unit/org/apache/cassandra/config/EncryptionOptionsEqualityTest.java b/test/unit/org/apache/cassandra/config/EncryptionOptionsEqualityTest.java index 7c86069a80..317c8ec626 100644 --- a/test/unit/org/apache/cassandra/config/EncryptionOptionsEqualityTest.java +++ b/test/unit/org/apache/cassandra/config/EncryptionOptionsEqualityTest.java @@ -25,6 +25,7 @@ import org.junit.Test; import org.apache.cassandra.security.DefaultSslContextFactory; import org.apache.cassandra.security.DummySslContextFactoryImpl; +import org.apache.cassandra.transport.TlsTestUtils; import static org.apache.cassandra.config.EncryptionOptions.ClientAuth.NOT_REQUIRED; import static org.apache.cassandra.config.EncryptionOptions.ClientAuth.REQUIRED; @@ -41,12 +42,12 @@ public class EncryptionOptionsEqualityTest { return new EncryptionOptions.ServerEncryptionOptions() .withStoreType("JKS") - .withKeyStore("test/conf/cassandra.keystore") - .withKeyStorePassword("cassandra") - .withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") - .withOutboundKeystore("test/conf/cassandra_outbound.keystore") - .withOutboundKeystorePassword("cassandra") + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD) + .withOutboundKeystore(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PATH) + .withOutboundKeystorePassword(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PASSWORD) .withProtocol("TLSv1.1") .withRequireClientAuth(REQUIRED) .withRequireEndpointVerification(false); @@ -57,10 +58,10 @@ public class EncryptionOptionsEqualityTest EncryptionOptions encryptionOptions1 = new EncryptionOptions() .withStoreType("JKS") - .withKeyStore("test/conf/cassandra.keystore") - .withKeyStorePassword("cassandra") - .withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD) .withProtocol("TLSv1.1") .withRequireClientAuth(REQUIRED) .withRequireEndpointVerification(false); @@ -68,10 +69,10 @@ public class EncryptionOptionsEqualityTest EncryptionOptions encryptionOptions2 = new EncryptionOptions() .withStoreType("JKS") - .withKeyStore("test/conf/cassandra.keystore") - .withKeyStorePassword("cassandra") - .withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD) .withProtocol("TLSv1.1") .withRequireClientAuth(REQUIRED) .withRequireEndpointVerification(false); diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index de68125f12..5f996e49af 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -58,6 +58,7 @@ import javax.management.remote.JMXConnectorFactory; import javax.management.remote.JMXConnectorServer; import javax.management.remote.JMXServiceURL; import javax.management.remote.rmi.RMIConnectorServer; +import javax.net.ssl.SSLException; import com.google.common.base.Objects; import com.google.common.base.Strings; @@ -165,6 +166,7 @@ import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.transport.Server; import org.apache.cassandra.transport.SimpleClient; +import org.apache.cassandra.transport.TlsTestUtils; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -229,8 +231,8 @@ public abstract class CQLTester protected static int nativePort; protected static final InetAddress nativeAddr; protected static final Set remoteAddrs = new HashSet<>(); - private static final Map, Cluster> clusters = new HashMap<>(); - private static final Map, Session> sessions = new HashMap<>(); + private static final Map clusters = new HashMap<>(); + private static final Map sessions = new HashMap<>(); private static Consumer clusterBuilderConfigurator; @@ -283,6 +285,10 @@ public abstract class CQLTester private User user; + private boolean useEncryption = false; + + private boolean useClientCert = false; + // We don't use USE_PREPARED_VALUES in the code below so some test can foce value preparation (if the result // is not expected to be the same without preparation) private boolean usePrepared = USE_PREPARED_VALUES; @@ -554,6 +560,22 @@ public abstract class CQLTester AuthCacheService.initializeAndRegisterCaches(); } + /** + * @param useEncryption Whether created clients should use encryption. + */ + public void shouldUseEncryption(boolean useEncryption) + { + this.useEncryption = useEncryption; + } + + /** + * @param useClientCert Whether created clients should provide a client cert if encryption is enabled. + */ + public void shouldUseClientCertificate(boolean useClientCert) + { + this.useClientCert = useClientCert; + } + /** * Configures the server to require client encryption for CQL. Useful for tests which exercise TLS specific * behavior. @@ -562,19 +584,18 @@ public abstract class CQLTester * with {@link Server.Builder#withTlsEncryptionPolicy(EncryptionOptions.TlsEncryptionPolicy)} using * {@link org.apache.cassandra.config.EncryptionOptions.TlsEncryptionPolicy#ENCRYPTED}. */ - protected static void requireNativeProtocolClientEncryption() + public static void requireNativeProtocolClientEncryption() { - DatabaseDescriptor.updateNativeProtocolEncryptionOptions((encryptionOptions -> - encryptionOptions.withEnabled(true) - .withKeyStore("test/conf/cassandra_ssl_test.keystore") - .withKeyStorePassword("cassandra") - .withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") - .withRequireEndpointVerification(false) - .withRequireClientAuth(EncryptionOptions.ClientAuth.OPTIONAL))); + DatabaseDescriptor.updateNativeProtocolEncryptionOptions((encryptionOptions) -> + encryptionOptions.withEnabled(true) + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD) + .withRequireEndpointVerification(false) + .withRequireClientAuth(EncryptionOptions.ClientAuth.OPTIONAL)); } - /** * Initialize Native Transport for test that need it. */ @@ -643,7 +664,7 @@ public abstract class CQLTester server.start(); } - private static Cluster initClientCluster(User user, ProtocolVersion version) + private static Cluster initClientCluster(User user, ProtocolVersion version, boolean useEncryption, boolean useClientCert) { SocketOptions socketOptions = new SocketOptions().setConnectTimeoutMillis(TEST_DRIVER_CONNECTION_TIMEOUT_MS.getInt()) // default is 5000 @@ -662,6 +683,19 @@ public abstract class CQLTester if (user != null) builder.withCredentials(user.username, user.password); + if (useEncryption) + { + try + { + builder.withSSL(TlsTestUtils.getSSLOptions(useClientCert)); + } + catch (SSLException e) + { + // generally this should work. + throw new RuntimeException(e); + } + } + if (version.isBeta()) builder = builder.allowBetaProtocolVersion(); else @@ -1540,12 +1574,12 @@ public abstract class CQLTester private Session getSession(ProtocolVersion protocolVersion) { Cluster cluster = getCluster(protocolVersion); - return sessions.computeIfAbsent(Pair.create(user, protocolVersion), userProto -> cluster.connect()); + return sessions.computeIfAbsent(new ClusterSettings(user, protocolVersion, useEncryption, useClientCert), settings -> cluster.connect()); } private Cluster getCluster(ProtocolVersion protocolVersion) { - return clusters.computeIfAbsent(Pair.create(user, protocolVersion), userProto -> initClientCluster(userProto.left, userProto.right)); + return clusters.computeIfAbsent(new ClusterSettings(user, protocolVersion, useEncryption, useClientCert), settings -> initClientCluster(user, protocolVersion, useEncryption, useClientCert)); } protected SimpleClient newSimpleClient(ProtocolVersion version) throws IOException @@ -2868,4 +2902,35 @@ public abstract class CQLTester fs.clearListeners(); } } + + private static class ClusterSettings + { + private final User user; + private final ProtocolVersion protocolVersion; + private final boolean shouldUseEncryption; + private final boolean shouldUseCertificate; + + ClusterSettings(User user, ProtocolVersion protocolVersion, boolean shouldUseEncryption, boolean shouldUseCertificate) + { + this.user = user; + this.protocolVersion = protocolVersion; + this.shouldUseEncryption = shouldUseEncryption; + this.shouldUseCertificate = shouldUseCertificate; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterSettings that = (ClusterSettings) o; + return shouldUseEncryption == that.shouldUseEncryption && shouldUseCertificate == that.shouldUseCertificate && java.util.Objects.equals(user, that.user) && protocolVersion == that.protocolVersion; + } + + @Override + public int hashCode() + { + return java.util.Objects.hash(user, protocolVersion, shouldUseEncryption, shouldUseCertificate); + } + } } diff --git a/test/unit/org/apache/cassandra/db/virtual/ClientsTableTest.java b/test/unit/org/apache/cassandra/db/virtual/ClientsTableTest.java index 862a282a23..7b3c243ffd 100644 --- a/test/unit/org/apache/cassandra/db/virtual/ClientsTableTest.java +++ b/test/unit/org/apache/cassandra/db/virtual/ClientsTableTest.java @@ -19,37 +19,57 @@ package org.apache.cassandra.db.virtual; import java.net.InetAddress; +import java.util.Collections; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; +import org.apache.cassandra.auth.AuthTestUtils; +import org.apache.cassandra.auth.IAuthenticator; +import org.apache.cassandra.auth.MutualTlsAuthenticator; +import org.apache.cassandra.auth.SpiffeCertificateValidator; +import org.apache.cassandra.config.EncryptionOptions; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.transport.TlsTestUtils; import org.assertj.core.api.Assertions; + +import static org.apache.cassandra.auth.AuthTestUtils.waitForExistingRoles; import static org.assertj.core.api.Assertions.assertThat; public class ClientsTableTest extends CQLTester { private static final String KS_NAME = "vts"; - private ClientsTable table; - - @Before - public void config() + @BeforeClass + public static void config() { - table = new ClientsTable(KS_NAME); + // Enable TLS and require native protocol encryption + requireNativeProtocolClientEncryption(); + requireNetwork(server -> server.withTlsEncryptionPolicy(EncryptionOptions.TlsEncryptionPolicy.ENCRYPTED), cluster -> {}); + + IAuthenticator authenticator = new MutualTlsAuthenticator( ImmutableMap.of("validator_class_name", SpiffeCertificateValidator.class.getSimpleName())); + requireAuthentication(authenticator); + + ClientsTable table = new ClientsTable(KS_NAME); VirtualKeyspaceRegistry.instance.register(new VirtualKeyspace(KS_NAME, ImmutableList.of(table))); + + waitForExistingRoles(); + + AuthTestUtils.addIdentityToRole(TlsTestUtils.CLIENT_SPIFFE_IDENTITY, "cassandra"); } @Test - public void testSelectAll() throws Throwable + public void testSelectAll() { + shouldUseEncryption(true); + shouldUseClientCertificate(true); ResultSet result = executeNet("SELECT * FROM vts.clients"); - for (Row r : result) { Assert.assertEquals(InetAddress.getLoopbackAddress(), r.getInet("address")); @@ -62,6 +82,13 @@ public class ClientsTableTest extends CQLTester Assertions.assertThat(r.getMap("client_options", String.class, String.class)) .hasEntrySatisfying("DRIVER_VERSION", value -> assertThat(value.contains(r.getString("driver_name")))) .hasEntrySatisfying("DRIVER_VERSION", value -> assertThat(value.contains(r.getString("driver_version")))); + Assert.assertTrue(r.getBool("ssl_enabled")); + Assert.assertTrue(r.getString("ssl_protocol").startsWith("TLS")); + Assert.assertNotNull(r.getString("ssl_cipher_suite")); + Assert.assertEquals("cassandra", r.getString("username")); + Assert.assertEquals("MutualTls", r.getString("authentication_mode")); + Assert.assertEquals(Collections.singletonMap("identity", TlsTestUtils.CLIENT_SPIFFE_IDENTITY), + r.getMap("authentication_metadata", String.class, String.class)); } } } diff --git a/test/unit/org/apache/cassandra/metrics/ClientMetricsTest.java b/test/unit/org/apache/cassandra/metrics/ClientMetricsTest.java new file mode 100644 index 0000000000..0f1cd26f50 --- /dev/null +++ b/test/unit/org/apache/cassandra/metrics/ClientMetricsTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.metrics; + +import javax.net.ssl.SSLException; + +import org.junit.BeforeClass; +import org.junit.Test; + +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Meter; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.HostDistance; +import com.datastax.driver.core.PoolingOptions; +import com.datastax.driver.core.Session; +import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.auth.IAuthenticator.AuthenticationMode; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.OverrideConfigurationLoader; +import org.apache.cassandra.service.EmbeddedCassandraService; +import org.apache.cassandra.transport.TlsTestUtils; + +import static org.apache.cassandra.auth.AuthTestUtils.waitForExistingRoles; +import static org.apache.cassandra.config.CassandraRelevantProperties.SUPERUSER_SETUP_DELAY_MS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class ClientMetricsTest +{ + private static EmbeddedCassandraService cassandra; + + private static final ClientMetrics clientMetrics = ClientMetrics.instance; + + private static final String USER1 = "user1"; + private static final String USER1PW = "user1pw"; + + @BeforeClass + public static void setUp() throws Throwable + { + OverrideConfigurationLoader.override(TlsTestUtils::configureWithMutualTlsWithPasswordFallbackAuthenticator); + + SUPERUSER_SETUP_DELAY_MS.setLong(0); + + cassandra = ServerTestUtils.startEmbeddedCassandraService(); + + waitForExistingRoles(); + + // Allow client to connect as cassandra using an mTLS identity. + try (Cluster cluster = clusterBuilder() + .withCredentials("cassandra", "cassandra") + .build()) + { + try (Session session = cluster.connect()) + { + session.execute(String.format("CREATE ROLE %s WITH password = '%s' AND LOGIN = true", USER1, USER1PW)); + session.execute(String.format("ADD IDENTITY '%s' TO ROLE '%s'", TlsTestUtils.CLIENT_SPIFFE_IDENTITY, USER1)); + } + } + } + + public static void tearDown() + { + if (cassandra != null) + { + cassandra.stop(); + } + } + + private static Cluster.Builder clusterBuilder() + { + return Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()); + } + + /** + * Validates the behavior of metrics tied to connectivity and authentication by doing the following: + * + *

    + *
  1. Attempt to connect with password authentication and ensure ConnectedNativeClients and AuthSuccess metrics + * are incremented, in addition to the Password specific metrics for these.
    2. + *
  2. Attempt to connect with mtls authentication and ensure ConnectedNativeClients and AuthSuccess metrics + * are incremented, in addition to the MTLS specific metrics.
    3. + *
  3. Attempt to connect with mtls authentication using a non-permitted identity and ensure AuthFailure + * metrics are incremented.
    4. + *
+ * @throws SSLException + */ + @Test + public void testConnectedClientsAndAuthMetrics() throws SSLException + { + // Should be no connections at this time. + assertEquals(0, clientMetrics.connectedNativeClients.getValue().intValue()); + assertEquals(0, clientMetrics.encryptedConnectedNativeClients.getValue().intValue()); + assertEquals(0, clientMetrics.unencryptedConnectedNativeClients.getValue().intValue()); + + long initialAuthSuccess = clientMetrics.authSuccess.getCount(); + long initialAuthFailure = clientMetrics.authFailure.getCount(); + + // For each authentication mode, we should have connectedNativeClientsByAuthMode, authSuccessByMode, and authFailureByMode entries. + Gauge passwordConnections = clientMetrics.connectedNativeClientsByAuthMode.get(AuthenticationMode.PASSWORD); + assertNotNull(passwordConnections); + + Meter authSuccessPassword = clientMetrics.authSuccessByMode.get(AuthenticationMode.PASSWORD); + assertNotNull(authSuccessPassword); + long initialAuthSuccessPassword = authSuccessPassword.getCount(); + + Meter authFailurePassword = clientMetrics.authFailureByMode.get(AuthenticationMode.PASSWORD); + assertNotNull(authFailurePassword); + long initialAuthFailurePassword = authFailurePassword.getCount(); + + Gauge mtlsConnections = clientMetrics.connectedNativeClientsByAuthMode.get(AuthenticationMode.MTLS); + assertNotNull(mtlsConnections); + + Meter authSuccessMtls = clientMetrics.authSuccessByMode.get(AuthenticationMode.MTLS); + assertNotNull(authSuccessMtls); + long initialAuthSuccessMtls = authSuccessMtls.getCount(); + + Meter authFailureMtls = clientMetrics.authFailureByMode.get(AuthenticationMode.MTLS); + assertNotNull(authFailureMtls); + long initialAuthFailureMtls = authFailureMtls.getCount(); + + try (Cluster cluster = clusterBuilder().withCredentials("cassandra", "cassandra").build(); + Session session = cluster.connect()) + { + // Connected native clients should be 2 (1 for core connection, 1 for control connection) + assertEquals(2, clientMetrics.connectedNativeClients.getValue().intValue()); + assertEquals(2, clientMetrics.unencryptedConnectedNativeClients.getValue().intValue()); + assertEquals(0, clientMetrics.encryptedConnectedNativeClients.getValue().intValue()); + + // Should be two connections on the "cassandra" user. + Integer cassandraConnections = clientMetrics.connectedNativeClientsByUser.getValue().get("cassandra"); + assertNotNull(cassandraConnections); + assertEquals(2, cassandraConnections.intValue()); + + // Should be two password-authed connections. + assertEquals(2, passwordConnections.getValue().intValue()); + + // auth success should increment by 2. + assertEquals(initialAuthSuccess+2, clientMetrics.authSuccess.getCount()); + assertEquals(initialAuthFailure, clientMetrics.authFailure.getCount()); + + // should increment by 2 for password mode specific metric as well. + assertEquals(initialAuthSuccessPassword+2L, authSuccessPassword.getCount()); + assertEquals(initialAuthFailurePassword, authFailurePassword.getCount()); + + // Should be no mtls connections. + assertEquals(0, mtlsConnections.getValue().intValue()); + + // Should be no mtls attempts. + assertEquals(initialAuthSuccessMtls, authSuccessMtls.getCount()); + // No auth failure for mtls + assertEquals(initialAuthFailureMtls, authFailureMtls.getCount()); + + Cluster.Builder user1ClusterBuilder = clusterBuilder() + .withSSL(TlsTestUtils.getSSLOptions(true)) + .withPoolingOptions(new PoolingOptions() + .setConnectionsPerHost(HostDistance.LOCAL, 4, 4)); + // Authenticate with 'user1' over mtls with a connection pool of size 4 + try (Cluster user1Cluster = user1ClusterBuilder.build(); + Session user1Session = user1Cluster.connect()) + { + // Connected native clients should increment by 5 for new cluster (4 core + 1 control) + assertEquals(7, clientMetrics.connectedNativeClients.getValue().intValue()); + assertEquals(2, clientMetrics.unencryptedConnectedNativeClients.getValue().intValue()); + assertEquals(5, clientMetrics.encryptedConnectedNativeClients.getValue().intValue()); + + // auth success should increment by 5 (accounting for the 2 initial password connections). + assertEquals(initialAuthSuccess + 2 + 5, clientMetrics.authSuccess.getCount()); + + // Should still be two connections on the "cassandra" user. + cassandraConnections = clientMetrics.connectedNativeClientsByUser.getValue().get("cassandra"); + assertNotNull(cassandraConnections); + assertEquals(2, cassandraConnections.intValue()); + + // User1 should have 5 connections. + Integer user1Connections = clientMetrics.connectedNativeClientsByUser.getValue().get(USER1); + assertNotNull(user1Connections); + assertEquals(5, user1Connections.intValue()); + + // Expect 5 mtls connections, password connections should remain unchanged. + assertEquals(5, mtlsConnections.getValue().intValue()); + assertEquals(2, passwordConnections.getValue().intValue()); + assertEquals(initialAuthSuccessMtls + 5L, authSuccessMtls.getCount()); + assertEquals(initialAuthFailureMtls, authFailureMtls.getCount()); + } + + // after user1's session is closed, metrics should drop to what they were before. + assertEquals(2, clientMetrics.connectedNativeClients.getValue().intValue()); + assertEquals(2, clientMetrics.unencryptedConnectedNativeClients.getValue().intValue()); + assertEquals(0, clientMetrics.encryptedConnectedNativeClients.getValue().intValue()); + + // user1 should not be present in map + Integer user1Connections = clientMetrics.connectedNativeClientsByUser.getValue().get(USER1); + assertNull(user1Connections); + // No mTLS connections should be present. + assertEquals(0, mtlsConnections.getValue().intValue()); + + // Drop the identity used for mTLS to create an auth failure scenario for mTLS. + session.execute(String.format("DROP IDENTITY '%s'", TlsTestUtils.CLIENT_SPIFFE_IDENTITY)); + + // Attempt to connect with mTLS again, which should fail. + try (Cluster user1Cluster = user1ClusterBuilder.build()) + { + user1Cluster.connect(); + } catch (Exception e) + { + // expected. + } + // No mTLS connections should be present. + assertEquals(0, mtlsConnections.getValue().intValue()); + // No new successful mTLS attempts. + assertEquals(initialAuthSuccessMtls + 5L, authSuccessMtls.getCount()); + assertEquals(initialAuthSuccess + 2 + 5, clientMetrics.authSuccess.getCount()); + // 1 failed mTLS attempt (control connection cannot establish) + assertEquals(initialAuthFailureMtls + 1L, authFailureMtls.getCount()); + assertEquals(initialAuthFailure+1, clientMetrics.authFailure.getCount()); + } + + // Should be no connections as all clusters are closed at this point. + assertEquals(0, clientMetrics.connectedNativeClients.getValue().intValue()); + assertEquals(0, clientMetrics.unencryptedConnectedNativeClients.getValue().intValue()); + assertEquals(0, clientMetrics.encryptedConnectedNativeClients.getValue().intValue()); + assertEquals(0, passwordConnections.getValue().intValue()); + } +} diff --git a/test/unit/org/apache/cassandra/net/ConnectionTest.java b/test/unit/org/apache/cassandra/net/ConnectionTest.java index 1f8c6f612d..d0824ac4f8 100644 --- a/test/unit/org/apache/cassandra/net/ConnectionTest.java +++ b/test/unit/org/apache/cassandra/net/ConnectionTest.java @@ -66,6 +66,7 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.transport.TlsTestUtils; import org.apache.cassandra.utils.FBUtilities; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -179,10 +180,10 @@ public class ConnectionTest .withLegacySslStoragePort(true) .withOptional(true) .withInternodeEncryption(EncryptionOptions.ServerEncryptionOptions.InternodeEncryption.all) - .withKeyStore("test/conf/cassandra_ssl_test.keystore") - .withKeyStorePassword("cassandra") - .withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD) .withRequireClientAuth(NOT_REQUIRED) .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA"); diff --git a/test/unit/org/apache/cassandra/net/HandshakeTest.java b/test/unit/org/apache/cassandra/net/HandshakeTest.java index ec7e26b322..99e33eea90 100644 --- a/test/unit/org/apache/cassandra/net/HandshakeTest.java +++ b/test/unit/org/apache/cassandra/net/HandshakeTest.java @@ -46,6 +46,7 @@ import org.apache.cassandra.gms.GossipDigestSyn; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.OutboundConnectionInitiator.Result.MessagingSuccess; import org.apache.cassandra.security.DefaultSslContextFactory; +import org.apache.cassandra.transport.TlsTestUtils; import org.apache.cassandra.utils.concurrent.AsyncPromise; import static org.apache.cassandra.net.ConnectionType.SMALL_MESSAGES; @@ -276,12 +277,12 @@ public class HandshakeTest private ServerEncryptionOptions getServerEncryptionOptions(SslFallbackConnectionType sslConnectionType, boolean optional) { ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions().withOptional(optional) - .withKeyStore("test/conf/cassandra_ssl_test.keystore") - .withKeyStorePassword("cassandra") - .withOutboundKeystore("test/conf/cassandra_ssl_test_outbound.keystore") - .withOutboundKeystorePassword("cassandra") - .withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withOutboundKeystore(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PATH) + .withOutboundKeystorePassword(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PASSWORD) + .withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD) .withSslContextFactory((new ParameterizedClass(DefaultSslContextFactory.class.getName(), new HashMap<>()))); if (sslConnectionType == SslFallbackConnectionType.MTLS) diff --git a/test/unit/org/apache/cassandra/security/DefaultSslContextFactoryTest.java b/test/unit/org/apache/cassandra/security/DefaultSslContextFactoryTest.java index 1b7db326e1..fa3eb7c845 100644 --- a/test/unit/org/apache/cassandra/security/DefaultSslContextFactoryTest.java +++ b/test/unit/org/apache/cassandra/security/DefaultSslContextFactoryTest.java @@ -34,6 +34,7 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslProvider; import org.apache.cassandra.config.EncryptionOptions; import org.apache.cassandra.distributed.shared.WithProperties; +import org.apache.cassandra.transport.TlsTestUtils; import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_TCACTIVE_OPENSSL; @@ -47,33 +48,33 @@ public class DefaultSslContextFactoryTest @Before public void setup() { - commonConfig.put("truststore", "test/conf/cassandra_ssl_test.truststore"); - commonConfig.put("truststore_password", "cassandra"); + commonConfig.put("truststore", TlsTestUtils.SERVER_TRUSTSTORE_PATH); + commonConfig.put("truststore_password", TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD); commonConfig.put("require_client_auth", "false"); commonConfig.put("cipher_suites", Arrays.asList("TLS_RSA_WITH_AES_128_CBC_SHA")); } private void addKeystoreOptions(Map config) { - config.put("keystore", "test/conf/cassandra_ssl_test.keystore"); - config.put("keystore_password", "cassandra"); + config.put("keystore", TlsTestUtils.SERVER_KEYSTORE_PATH); + config.put("keystore_password", TlsTestUtils.SERVER_KEYSTORE_PASSWORD); } private void addOutboundKeystoreOptions(Map config) { - config.put("outbound_keystore", "test/conf/cassandra_ssl_test_outbound.keystore"); - config.put("outbound_keystore_password", "cassandra"); + config.put("outbound_keystore", TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PATH); + config.put("outbound_keystore_password", TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PASSWORD); } @Test public void getSslContextOpenSSL() throws IOException { - EncryptionOptions.ServerEncryptionOptions options = new EncryptionOptions.ServerEncryptionOptions().withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") - .withKeyStore("test/conf/cassandra_ssl_test.keystore") - .withKeyStorePassword("cassandra") - .withOutboundKeystore("test/conf/cassandra_ssl_test_outbound.keystore") - .withOutboundKeystorePassword("cassandra") + EncryptionOptions.ServerEncryptionOptions options = new EncryptionOptions.ServerEncryptionOptions().withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD) + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withOutboundKeystore(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PATH) + .withOutboundKeystorePassword(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PASSWORD) .withRequireClientAuth(NOT_REQUIRED) .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA"); SslContext sslContext = SSLFactory.getOrCreateSslContext(options, REQUIRED, ISslContextFactory.SocketType.CLIENT, "test"); diff --git a/test/unit/org/apache/cassandra/security/FileBasedSslContextFactoryTest.java b/test/unit/org/apache/cassandra/security/FileBasedSslContextFactoryTest.java index 73fe80b304..2916f0d460 100644 --- a/test/unit/org/apache/cassandra/security/FileBasedSslContextFactoryTest.java +++ b/test/unit/org/apache/cassandra/security/FileBasedSslContextFactoryTest.java @@ -27,12 +27,11 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.cassandra.config.EncryptionOptions; import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.distributed.shared.WithProperties; +import org.apache.cassandra.transport.TlsTestUtils; import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG; @@ -40,8 +39,6 @@ import static org.apache.cassandra.config.EncryptionOptions.ClientAuth.NOT_REQUI public class FileBasedSslContextFactoryTest { - private static final Logger logger = LoggerFactory.getLogger(FileBasedSslContextFactoryTest.class); - private EncryptionOptions.ServerEncryptionOptions encryptionOptions; static WithProperties properties; @@ -65,14 +62,14 @@ public class FileBasedSslContextFactoryTest encryptionOptions = new EncryptionOptions.ServerEncryptionOptions() .withSslContextFactory(new ParameterizedClass(TestFileBasedSSLContextFactory.class.getName(), new HashMap<>())) - .withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") + .withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD) .withRequireClientAuth(NOT_REQUIRED) .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA") - .withKeyStore("test/conf/cassandra_ssl_test.keystore") - .withKeyStorePassword("cassandra") - .withOutboundKeystore("test/conf/cassandra_ssl_test_outbound.keystore") - .withOutboundKeystorePassword("cassandra"); + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withOutboundKeystore(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PATH) + .withOutboundKeystorePassword(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PASSWORD); } @Test diff --git a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryTest.java b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryTest.java index ca319de5fd..9781e9d08a 100644 --- a/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryTest.java +++ b/test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryTest.java @@ -36,6 +36,7 @@ import io.netty.handler.ssl.SslProvider; import org.apache.cassandra.config.EncryptionOptions; import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.distributed.shared.WithProperties; +import org.apache.cassandra.transport.TlsTestUtils; import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_TCACTIVE_OPENSSL; import static org.apache.cassandra.config.EncryptionOptions.ClientAuth.NOT_REQUIRED; @@ -195,18 +196,18 @@ public class PEMBasedSslContextFactoryTest private void addFileBaseTrustStoreOptions(Map config) { - config.put("truststore", "test/conf/cassandra_ssl_test.truststore.pem"); + config.put("truststore", TlsTestUtils.SERVER_TRUSTSTORE_PEM_PATH); } private void addFileBaseKeyStoreOptions(Map config) { - config.put("keystore", "test/conf/cassandra_ssl_test.keystore.pem"); - config.put("keystore_password", "cassandra"); + config.put("keystore", TlsTestUtils.SERVER_KEYSTORE_PATH_PEM); + config.put("keystore_password", TlsTestUtils.SERVER_KEYSTORE_PASSWORD); } private void addFileBaseUnencryptedKeyStoreOptions(Map config) { - config.put("keystore", "test/conf/cassandra_ssl_test.unencrypted_keystore.pem"); + config.put("keystore", TlsTestUtils.SERVER_KEYSTORE_PATH_UNENCRYPTED_PEM); } @Test @@ -214,9 +215,9 @@ public class PEMBasedSslContextFactoryTest { ParameterizedClass sslContextFactory = new ParameterizedClass(PEMBasedSslContextFactory.class.getSimpleName() , new HashMap<>()); - EncryptionOptions options = new EncryptionOptions().withTrustStore("test/conf/cassandra_ssl_test.truststore.pem") - .withKeyStore("test/conf/cassandra_ssl_test.keystore.pem") - .withKeyStorePassword("cassandra") + EncryptionOptions options = new EncryptionOptions().withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PEM_PATH) + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH_PEM) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) .withRequireClientAuth(NOT_REQUIRED) .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA") .withSslContextFactory(sslContextFactory); @@ -233,11 +234,11 @@ public class PEMBasedSslContextFactoryTest { ParameterizedClass sslContextFactory = new ParameterizedClass(PEMBasedSslContextFactory.class.getSimpleName() , new HashMap<>()); - EncryptionOptions.ServerEncryptionOptions options = new EncryptionOptions.ServerEncryptionOptions().withTrustStore("test/conf/cassandra_ssl_test.truststore.pem") - .withKeyStore("test/conf/cassandra_ssl_test.keystore.pem") - .withKeyStorePassword("cassandra") - .withOutboundKeystore("test/conf/cassandra_ssl_test.keystore.pem") - .withOutboundKeystorePassword("cassandra") + EncryptionOptions.ServerEncryptionOptions options = new EncryptionOptions.ServerEncryptionOptions().withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PEM_PATH) + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH_PEM) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withOutboundKeystore(TlsTestUtils.SERVER_KEYSTORE_PATH_PEM) + .withOutboundKeystorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) .withRequireClientAuth(NOT_REQUIRED) .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA") .withSslContextFactory(sslContextFactory); diff --git a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java index a7c14dfcfb..ba46588686 100644 --- a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java +++ b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java @@ -37,8 +37,6 @@ import org.apache.commons.io.FileUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import io.netty.handler.ssl.OpenSslClientContext; import io.netty.handler.ssl.OpenSslServerContext; @@ -50,6 +48,7 @@ import org.apache.cassandra.config.EncryptionOptions; import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions; import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.transport.TlsTestUtils; import static org.apache.cassandra.config.EncryptionOptions.ClientAuth.NOT_REQUIRED; import static org.apache.cassandra.config.EncryptionOptions.ClientAuth.REQUIRED; @@ -58,8 +57,6 @@ import static org.junit.Assert.assertNotNull; public class SSLFactoryTest { - private static final Logger logger = LoggerFactory.getLogger(SSLFactoryTest.class); - static final SelfSignedCertificate ssc; static { @@ -81,8 +78,8 @@ public class SSLFactoryTest { SSLFactory.clearSslContextCache(); encryptionOptions = new ServerEncryptionOptions() - .withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") + .withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.SERVER_TRUSTSTORE_PASSWORD) .withRequireClientAuth(NOT_REQUIRED) .withCipherSuites("TLS_RSA_WITH_AES_128_CBC_SHA") .withSslContextFactory(new ParameterizedClass(TestFileBasedSSLContextFactory.class.getName(), @@ -91,10 +88,10 @@ public class SSLFactoryTest private ServerEncryptionOptions addKeystoreOptions(ServerEncryptionOptions options) { - return options.withKeyStore("test/conf/cassandra_ssl_test.keystore") - .withKeyStorePassword("cassandra") - .withOutboundKeystore("test/conf/cassandra_ssl_test_outbound.keystore") - .withOutboundKeystorePassword("cassandra"); + return options.withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withOutboundKeystore(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PATH) + .withOutboundKeystorePassword(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PASSWORD); } private ServerEncryptionOptions addPEMKeystoreOptions(ServerEncryptionOptions options) @@ -102,11 +99,11 @@ public class SSLFactoryTest ParameterizedClass sslContextFactoryClass = new ParameterizedClass("org.apache.cassandra.security.PEMBasedSslContextFactory", new HashMap<>()); return options.withSslContextFactory(sslContextFactoryClass) - .withKeyStore("test/conf/cassandra_ssl_test.keystore.pem") - .withKeyStorePassword("cassandra") - .withOutboundKeystore("test/conf/cassandra_ssl_test.keystore.pem") - .withOutboundKeystorePassword("cassandra") - .withTrustStore("test/conf/cassandra_ssl_test.truststore.pem"); + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH_PEM) + .withKeyStorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withOutboundKeystore(TlsTestUtils.SERVER_KEYSTORE_PATH_PEM) + .withOutboundKeystorePassword(TlsTestUtils.SERVER_KEYSTORE_PASSWORD) + .withTrustStore(TlsTestUtils.SERVER_TRUSTSTORE_PEM_PATH); } @Test @@ -158,7 +155,7 @@ public class SSLFactoryTest // Verify if right certificate is loaded into SslContext final Certificate loadedCertificate = getCertificateLoadedInSslContext(context.sessionContext()); - final Certificate certificate = getCertificates("test/conf/cassandra_ssl_test.keystore", "cassandra"); + final Certificate certificate = getCertificates(TlsTestUtils.SERVER_KEYSTORE_PATH, TlsTestUtils.SERVER_KEYSTORE_PASSWORD); assertEquals(loadedCertificate, certificate); } @@ -175,7 +172,7 @@ public class SSLFactoryTest // Verify if right certificate is loaded into SslContext final Certificate loadedCertificate = getCertificateLoadedInSslContext(context.sessionContext()); - final Certificate certificate = getCertificates("test/conf/cassandra_ssl_test_outbound.keystore", "cassandra"); + final Certificate certificate = getCertificates(TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PATH, TlsTestUtils.SERVER_OUTBOUND_KEYSTORE_PASSWORD); assertEquals(loadedCertificate, certificate); } diff --git a/test/unit/org/apache/cassandra/service/NativeTransportServiceTest.java b/test/unit/org/apache/cassandra/service/NativeTransportServiceTest.java index d5f825fc62..2f45748739 100644 --- a/test/unit/org/apache/cassandra/service/NativeTransportServiceTest.java +++ b/test/unit/org/apache/cassandra/service/NativeTransportServiceTest.java @@ -31,6 +31,7 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.EncryptionOptions; import org.apache.cassandra.transport.Server; +import org.apache.cassandra.transport.TlsTestUtils; import org.apache.cassandra.utils.Pair; import static org.junit.Assert.assertEquals; @@ -168,7 +169,7 @@ public class NativeTransportServiceTest DatabaseDescriptor.updateNativeProtocolEncryptionOptions( options -> options.withEnabled(true) .withOptional(true) - .withKeyStore("test/conf/cassandra_ssl_test.keystore")); + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH)); DatabaseDescriptor.setNativeTransportPortSSL(8432); withService((NativeTransportService service) -> @@ -224,7 +225,7 @@ public class NativeTransportServiceTest DatabaseDescriptor.updateNativeProtocolEncryptionOptions( options -> options.withEnabled(true) .withOptional(false) - .withKeyStore("test/conf/cassandra_ssl_test.keystore")); + .withKeyStore(TlsTestUtils.SERVER_KEYSTORE_PATH)); DatabaseDescriptor.setNativeTransportPortSSL(8432); withService((NativeTransportService service) -> diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ClientStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ClientStatsTest.java index d5b01730ef..2189a0e963 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/ClientStatsTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/ClientStatsTest.java @@ -32,22 +32,34 @@ import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Session; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.OverrideConfigurationLoader; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.service.EmbeddedCassandraService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tools.ToolRunner; +import org.apache.cassandra.transport.TlsTestUtils; import org.assertj.core.groups.Tuple; +import static org.apache.cassandra.auth.AuthTestUtils.waitForExistingRoles; import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_LOCAL_PORT; +import static org.apache.cassandra.config.CassandraRelevantProperties.SUPERUSER_SETUP_DELAY_MS; import static org.assertj.core.api.Assertions.assertThat; public class ClientStatsTest { private static Cluster cluster; + + private static Cluster tlsCluster; + + private static Session tlsSession; + + private static Cluster mtlsCluster; + + private static Session mtlsSession; + private Session session; private static EmbeddedCassandraService cassandra; @@ -55,19 +67,54 @@ public class ClientStatsTest @BeforeClass public static void setup() throws Throwable { + OverrideConfigurationLoader.override(TlsTestUtils::configureWithMutualTlsWithPasswordFallbackAuthenticator); + + SUPERUSER_SETUP_DELAY_MS.setLong(0); // Since we run EmbeddedCassandraServer, we need to manually associate JMX address; otherwise it won't start int jmxPort = CQLTester.getAutomaticallyAllocatedPort(InetAddress.getLoopbackAddress()); CASSANDRA_JMX_LOCAL_PORT.setInt(jmxPort); cassandra = ServerTestUtils.startEmbeddedCassandraService(); - cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()).build(); + + waitForExistingRoles(); + + cluster = clusterBuilder() + .withCredentials("cassandra", "cassandra") + .build(); + + // Allow client to connect as cassandra using an mTLS identity. + try(Session session = cluster.connect()) + { + session.execute(String.format("ADD IDENTITY '%s' TO ROLE 'cassandra'", TlsTestUtils.CLIENT_SPIFFE_IDENTITY)); + } + + // Configure a TLS-based cluster with password authentication. + tlsCluster = clusterBuilder() + .withSSL(TlsTestUtils.getSSLOptions(false)) + .withCredentials("cassandra", "cassandra") + .build(); + + // Configure a TLS-based cluster with certificate (mtls) authentication. + mtlsCluster = clusterBuilder() + .withSSL(TlsTestUtils.getSSLOptions(true)) + .build(); + } + + private static Cluster.Builder clusterBuilder() + { + return Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()); } @Before public void config() throws Throwable { + session = cluster.connect(); - ResultSet result = session.execute("select release_version from system.local"); + session.execute("select release_version from system.local"); + + tlsSession = tlsCluster.connect(); + // connect with system keyspace to assert it's present in output. + mtlsSession = mtlsCluster.connect("system"); } @After @@ -75,6 +122,10 @@ public class ClientStatsTest { if (session != null) session.close(); + if (tlsSession != null) + tlsSession.close(); + if (mtlsSession != null) + mtlsSession.close(); } @AfterClass @@ -82,6 +133,10 @@ public class ClientStatsTest { if (cluster != null) cluster.close(); + if (tlsCluster != null) + tlsCluster.close(); + if (mtlsCluster != null) + mtlsCluster.close(); if (cassandra != null) cassandra.stop(); } @@ -100,7 +155,7 @@ public class ClientStatsTest " [(-pp | --print-port)] [(-pw | --password )]\n" + " [(-pwf | --password-file )]\n" + " [(-u | --username )] clientstats [--all]\n" + - " [--by-protocol] [--clear-history] [--client-options]\n" + + " [--by-protocol] [--clear-history] [--client-options] [--verbose]\n" + "\n" + "OPTIONS\n" + " --all\n" + @@ -111,9 +166,9 @@ public class ClientStatsTest "\n" + " --clear-history\n" + " Clear the history of connected clients\n" + - "\n" + - " --client-options\n" + - " Lists all connections and the client options\n" + + "\n" + + " --client-options\n" + + " Lists all connections and the client options\n" + "\n" + " -h , --host \n" + " Node hostname or ip address\n" + @@ -133,6 +188,10 @@ public class ClientStatsTest " -u , --username \n" + " Remote jmx agent username\n" + "\n" + + " --verbose\n" + + " Lists all connections with additional details (client options,\n" + + " authenticator-specific metadata and more)\n" + + "\n" + "\n"; assertThat(tool.getStdout()).isEqualTo(help); } @@ -143,9 +202,7 @@ public class ClientStatsTest ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("clientstats"); tool.assertOnCleanExit(); String stdout = tool.getStdout(); - assertThat(stdout).contains("Total connected clients: 2"); - assertThat(stdout).contains("User Connections"); - assertThat(stdout).contains("anonymous 2"); + assertClientCount(stdout); } @Test @@ -165,27 +222,85 @@ public class ClientStatsTest ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("clientstats", "--all"); tool.assertOnCleanExit(); String stdout = tool.getStdout(); + /** + * Example expected output: + * Address SSL Cipher Protocol Version User Keyspace Requests Driver-Name Driver-Version + * /127.0.0.1:52549 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra 17 DataStax Java Driver 3.11.5 + * /127.0.0.1:52550 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra 3 DataStax Java Driver 3.11.5 + * /127.0.0.1:52551 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra 16 DataStax Java Driver 3.11.5 + * /127.0.0.1:52552 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra system 3 DataStax Java Driver 3.11.5 + * /127.0.0.1:52546 false undefined undefined 5 cassandra 17 DataStax Java Driver 3.11.5 + * /127.0.0.1:52548 false undefined undefined 5 cassandra 4 DataStax Java Driver 3.11.5 + */ assertThat(stdout).containsPattern("Address +SSL +Cipher +Protocol +Version +User +Keyspace +Requests +Driver-Name +Driver-Version"); - assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ false undefined undefined [0-9]+ +anonymous +[0-9]+ +DataStax Java Driver 3.11.5"); - assertThat(stdout).contains("Total connected clients: 2"); - assertThat(stdout).contains("User Connections"); - assertThat(stdout).contains("anonymous 2"); + // Unencrypted password-based client. + assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ false +undefined +undefined +[0-9]+ +cassandra +[0-9]+ +DataStax Java Driver 3.11.5"); + // TLS-encrypted password-based client. + assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ true +TLS\\S+ +TLS\\S+ +[0-9]+ +cassandra +[0-9]+ +DataStax Java Driver 3.11.5"); + // MTLS-based client. + assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ true +TLS\\S+ +TLS\\S+ +[0-9]+ +cassandra +[0-9]+ +DataStax Java Driver 3.11.5"); + // MTLS-based client with 'system' keyspace set on connection. + assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ true +TLS\\S+ +TLS\\S+ +[0-9]+ +cassandra +system +[0-9]+ +DataStax Java Driver 3.11.5"); + + assertClientCount(stdout); } @Test public void testClientStatsClientOptions() { + // given 'clientstats --metadata' invoked, we expect 'Client-Options' to be present. ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("clientstats", "--client-options"); tool.assertOnCleanExit(); String stdout = tool.getStdout(); + + /* + * Example expected output: + * Address SSL Cipher Protocol Version User Keyspace Requests Driver-Name Driver-Version Client-Options + * /127.0.0.1:51047 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra 17 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 + * /127.0.0.1:51048 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra 3 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 + * /127.0.0.1:51046 false undefined undefined 5 cassandra 4 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 + * /127.0.0.1:51044 false undefined undefined 5 cassandra 17 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 + * /127.0.0.1:51049 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra 16 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 + * /127.0.0.1:51050 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra system 3 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 + */ assertThat(stdout).containsPattern("Address +SSL +Cipher +Protocol +Version +User +Keyspace +Requests +Driver-Name +Driver-Version +Client-Options"); - assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ false undefined undefined [0-9]+ +anonymous +[0-9]+ +DataStax Java Driver 3.11.5"); + assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ false+ undefined +undefined +[0-9]+ +cassandra +[0-9]+ +DataStax Java Driver 3.11.5"); assertThat(stdout).containsPattern("DRIVER_NAME=DataStax Java Driver"); assertThat(stdout).containsPattern("DRIVER_VERSION=3.11.5"); assertThat(stdout).containsPattern("CQL_VERSION=3.0.0"); - assertThat(stdout).contains("Total connected clients: 2"); - assertThat(stdout).contains("User Connections"); - assertThat(stdout).contains("anonymous 2"); + + assertClientCount(stdout); + } + + @Test + public void testClientStatsClientVerbose() + { + // given 'clientstats --verbose' invoked, we expect 'Client-Options', 'Auth-Mode', 'Auth-Metadata', and 'Client-Options' columns to be present. + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("clientstats", "--verbose"); + tool.assertOnCleanExit(); + String stdout = tool.getStdout(); + /* + * Example expected output: + * Address SSL Cipher Protocol Version User Keyspace Requests Driver-Name Driver-Version Client-Options Auth-Mode Auth-Metadata + * /127.0.0.1:57141 false undefined undefined 5 cassandra 17 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 Password + * /127.0.0.1:57165 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra system 3 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 MutualTls identity=spiffe://test.cassandra.apache.org/unitTest/mtls + * /127.0.0.1:57164 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra 3 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 Password + * /127.0.0.1:57144 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra 17 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 Password + * /127.0.0.1:57146 true TLS_AES_256_GCM_SHA384 TLSv1.3 5 cassandra 16 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 MutualTls identity=spiffe://test.cassandra.apache.org/unitTest/mtls + * /127.0.0.1:57163 false undefined undefined 5 cassandra 4 DataStax Java Driver 3.11.5 DRIVER_VERSION=3.11.5, DRIVER_NAME=DataStax Java Driver, CQL_VERSION=3.0.0 Password + */ + // Header + assertThat(stdout).containsPattern("Address +SSL +Cipher +Protocol +Version +User +Keyspace +Requests +Driver-Name +Driver-Version +Client-Options +Auth-Mode +Auth-Metadata"); + // Unencrypted password-based client. Expect 'DRIVER_VERSION' to appear before Password. + assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ false +undefined +undefined +[0-9]+ +cassandra +[0-9]+ +DataStax Java Driver 3.11.5 +.*DRIVER_VERSION.* +Password"); + // TLS-encrypted password-based client. + assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ true +TLS\\S+ +TLS\\S+ +[0-9]+ +cassandra +[0-9]+ +DataStax Java Driver 3.11.5 +.*DRIVER_VERSION.* +Password"); + // MTLS-based client. + assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ true +TLS\\S+ +TLS\\S+ +[0-9]+ +cassandra +[0-9]+ +DataStax Java Driver 3.11.5 +.*DRIVER_VERSION.* +MutualTls +identity=" + TlsTestUtils.CLIENT_SPIFFE_IDENTITY); + // MTLS-based client with 'system' keyspace set on connection. + assertThat(stdout).containsPattern("/127.0.0.1:[0-9]+ true +TLS\\S+ +TLS\\S+ +[0-9]+ +cassandra +system +[0-9]+ +DataStax Java Driver 3.11.5 +.*DRIVER_VERSION.* +MutualTls +identity=" + TlsTestUtils.CLIENT_SPIFFE_IDENTITY); + + assertClientCount(stdout); } @Test @@ -205,4 +320,12 @@ public class ClientStatsTest .extracting(ILoggingEvent::getMessage, ILoggingEvent::getLevel) .contains(Tuple.tuple("Cleared connection history", Level.INFO)); } + + public void assertClientCount(String stdout) + { + // Expect two connections for each client (1 control connection, 1 core pool connection) + assertThat(stdout).contains("Total connected clients: 6"); + assertThat(stdout).contains("User Connections"); + assertThat(stdout).contains("cassandra 6"); + } } diff --git a/test/unit/org/apache/cassandra/transport/ConnectionTrackerTest.java b/test/unit/org/apache/cassandra/transport/ConnectionTrackerTest.java new file mode 100644 index 0000000000..ec3e43efcc --- /dev/null +++ b/test/unit/org/apache/cassandra/transport/ConnectionTrackerTest.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.transport; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Collections; +import java.util.function.Function; +import java.util.function.Predicate; + +import com.google.common.collect.ImmutableMap; +import org.junit.BeforeClass; +import org.junit.Test; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelId; +import io.netty.channel.embedded.EmbeddedChannel; +import org.apache.cassandra.auth.AuthenticatedUser; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.transport.Server.ConnectionTracker; +import org.mockito.Mockito; + +import static org.junit.Assert.assertEquals; + +public class ConnectionTrackerTest +{ + + private static final Predicate PREDICATE_TRUE = (__) -> true; + + @BeforeClass + public static void beforeClass() + { + // perform daemon initialization to intialize core components (like authenticator) that ConnectionTracker + // depends on. + DatabaseDescriptor.daemonInitialization(); + } + + private void registerConnections(ConnectionTracker tracker, int count) + { + registerConnections(tracker, count, (addr) -> null); + } + + private void registerConnections(ConnectionTracker tracker, int count, Function authenticatedUserGenerator) + { + for (int i = 0; i < count; i++) + { + InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 9040 + i); + + Channel c = new EmbeddedChannelWithSocketAddress(socketAddress); + AuthenticatedUser user = authenticatedUserGenerator.apply(socketAddress); + ServerConnection connection = createAuthenticatedConnection(c, user); + c.attr(Connection.attributeKey).set(connection); + + tracker.addConnection(c, connection); + } + } + + @Test + public void testCountConnectedClients() + { + final String ODD_USER = "odd"; + final String EVEN_USER = "even"; + ConnectionTracker connectionTracker = new ConnectionTracker(); + registerConnections(connectionTracker, 6, (addr) -> { + // if port divisible by 10, return 0, this will be treated as an anonymous user. + if (addr.getPort() % 10 == 0) + { + return null; + } + return new AuthenticatedUser(addr.getPort() % 2 == 0 ? EVEN_USER : ODD_USER); + }); + + // Validate that countConnectedClients by predicate returns the proper count. + + // Should be 6 total connections when all matching this predicate + assertEquals(6, connectionTracker.countConnectedClients(PREDICATE_TRUE)); + + // Three connections should have a port > 9042. + assertEquals(3, connectionTracker.countConnectedClients((conn) -> { + int port = conn.getClientState().getRemoteAddress().getPort(); + return port > 9042; + })); + + // Zero connections using ssl. + assertEquals(0, connectionTracker.countConnectedClients(ServerConnection::isSSL)); + + // Verify countConnectedClientsByUser appropriately counts by user. + + // Expect 2 users authenticated as 'even' {2, 4}, 3 as 'odd' {1, 3, 5}, one anonymous {0}. + assertEquals(ImmutableMap.of(EVEN_USER, 2, ODD_USER, 3, "anonymous", 1), connectionTracker.countConnectedClientsByUser()); + } + + @Test + public void testShouldNotCountChannelsMissingConnectionAttribute() + { + ConnectionTracker connectionTracker = new ConnectionTracker(); + registerConnections(connectionTracker, 10); + + // Given a ServerConnection registered that lacks a 'Connection.attributeKey' attribute, it should + // be skipped over when counting client connections. + InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 9042); + Channel c = new EmbeddedChannelWithSocketAddress(socketAddress); + Connection connection = new ServerConnection(c, ProtocolVersion.V5, (c1, c2) -> { + }); + // intentionally omit setting Connection attribute. + connectionTracker.addConnection(c, connection); + + // Expect the 10 existing connections, omitting the one added without a connection attribute. + assertEquals(10, connectionTracker.countConnectedClients(PREDICATE_TRUE)); + assertEquals(Collections.singletonMap("anonymous", 10), connectionTracker.countConnectedClientsByUser()); + } + + @Test + public void testShouldNotCountNonServerConnection() + { + ConnectionTracker connectionTracker = new ConnectionTracker(); + registerConnections(connectionTracker, 10); + + // Given a Connection registered that is not a 'ServerConnection' it should be skipped over when counting client + // connections. + InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 9042); + Channel c = new EmbeddedChannelWithSocketAddress(socketAddress); + Connection connection = new Connection(c, ProtocolVersion.V5, (c1, c2) -> { + }); + c.attr(Connection.attributeKey).set(connection); + connectionTracker.addConnection(c, connection); + + // Expect the 10 existing connections, omitting the one that isn't a ServerConnection. + assertEquals(10, connectionTracker.countConnectedClients(PREDICATE_TRUE)); + assertEquals(Collections.singletonMap("anonymous", 10), connectionTracker.countConnectedClientsByUser()); + } + + @Test(expected = IllegalArgumentException.class) + public void testShouldThrowThrowable() + { + ConnectionTracker connectionTracker = new ConnectionTracker(); + registerConnections(connectionTracker, 10); + + // Mock a ServerConnection such that when you attempt to access its connection attribute, an unhandled throwable + // is raised. + InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 9042); + Channel c = new EmbeddedChannelWithSocketAddress(socketAddress); + Channel spyChannel = Mockito.spy(c); + Connection connection = new ServerConnection(c, ProtocolVersion.V5, (c1, c2) -> { + }); + c.attr(Connection.attributeKey).set(connection); + Mockito.when(spyChannel.attr(Connection.attributeKey)).thenThrow(new IllegalArgumentException("Unexpected behavior")); + connectionTracker.addConnection(spyChannel, connection); + + // When calling countConnectedClients, we expect that expection to be propagated as we can't get a correct calculation. + connectionTracker.countConnectedClients(PREDICATE_TRUE); + } + + /** + * @return a {@link ServerConnection} whose {@link ClientState} has the given {@link AuthenticatedUser} + */ + private ServerConnection createAuthenticatedConnection(Channel channel, AuthenticatedUser user) + { + ServerConnection connection = new ServerConnection(channel, ProtocolVersion.V5, (c1, c2) -> { + }); + + if (user == null) + { + return connection; + } + + ClientState state = connection.getClientState(); + + ClientState spyState = Mockito.spy(state); + Mockito.when(spyState.getUser()).thenReturn(user); + + ServerConnection spyConnection = Mockito.spy(connection); + Mockito.when(spyConnection.getClientState()).thenReturn(spyState); + + return spyConnection; + } + + /** + * An embedded channel that uses the given {@link SocketAddress}'s string as its channel id, and {@link #remoteAddress()} + * returns the provided address. + * + * This is needed because {@link ServerConnection} expects a {@link Channel} with a {@link SocketAddress} + */ + private static class EmbeddedChannelWithSocketAddress extends EmbeddedChannel + { + private final SocketAddress socketAddress; + + EmbeddedChannelWithSocketAddress(SocketAddress socketAddress) + { + super(createChannelId(socketAddress)); + this.socketAddress = socketAddress; + } + + private static ChannelId createChannelId(SocketAddress socketAddress) + { + return new ChannelId() + { + @Override + public String asShortText() + { + return socketAddress.toString(); + } + + @Override + public String asLongText() + { + return asShortText(); + } + + @Override + public int compareTo(ChannelId o) + { + return this.asShortText().compareTo(this.asShortText()); + } + }; + } + + @Override + public SocketAddress remoteAddress() + { + return this.socketAddress; + } + } +} diff --git a/test/unit/org/apache/cassandra/transport/EarlyAuthenticationTest.java b/test/unit/org/apache/cassandra/transport/EarlyAuthenticationTest.java index 113865482a..ed98140b9b 100644 --- a/test/unit/org/apache/cassandra/transport/EarlyAuthenticationTest.java +++ b/test/unit/org/apache/cassandra/transport/EarlyAuthenticationTest.java @@ -60,9 +60,6 @@ public class EarlyAuthenticationTest extends CQLTester static final Map authenticatorParams = ImmutableMap.of("validator_class_name", SpiffeCertificateValidator.class.getSimpleName()); - // identity present in the client cert being used. - static final String spiffeIdentity = "spiffe://test.cassandra.apache.org/unitTest/mtls"; - @BeforeClass public static void setup() { @@ -89,25 +86,18 @@ public class EarlyAuthenticationTest extends CQLTester private EncryptionOptions clientEncryptionOptions(boolean presentClientCertificate) { - // To regenerate: - // 1. generate keystore - // keytool -genkeypair -keystore test/conf/cassandra_ssl_test_spiffe.keystore -validity 100000 -keyalg RSA -dname "CN=Apache Cassandra, OU=ssl_test, O=Unknown, L=Unknown, ST=Unknown, C=Unknown" -keypass cassandra -storepass cassandra -alias spiffecert -ext SAN=URI:spiffe://test.cassandra.apache.org/unitTest/mtls -storetype jks - // 2. export cert - // keytool -export -alias spiffecert -file spiffecert.cer -keystore test/conf/cassandra_ssl_test_spiffe.keystore - // 3. import cert into truststore - // keytool -import -v -trustcacerts -alias spiffecert -file spiffecert.cer -keystore test/conf/cassandra_ssl_test.truststore EncryptionOptions encryptionOptions = new EncryptionOptions() .withEnabled(true) .withRequireClientAuth(EncryptionOptions.ClientAuth.OPTIONAL) - .withTrustStore("test/conf/cassandra_ssl_test.truststore") - .withTrustStorePassword("cassandra") + .withTrustStore(TlsTestUtils.CLIENT_TRUSTSTORE_PATH) + .withTrustStorePassword(TlsTestUtils.CLIENT_TRUSTSTORE_PASSWORD) .withSslContextFactory(new ParameterizedClass(SimpleClientSslContextFactory.class.getName())); if (presentClientCertificate) { - encryptionOptions = encryptionOptions.withKeyStore("test/conf/cassandra_ssl_test_spiffe.keystore") + encryptionOptions = encryptionOptions.withKeyStore(TlsTestUtils.CLIENT_SPIFFE_KEYSTORE_PATH) .withStoreType("JKS") - .withKeyStorePassword("cassandra"); + .withKeyStorePassword(TlsTestUtils.CLIENT_SPIFFE_KEYSTORE_PASSWORD); } return new EncryptionOptions(encryptionOptions); @@ -117,7 +107,7 @@ public class EarlyAuthenticationTest extends CQLTester public void testEarlyAuthSuccess() { // given server is configured with a Mutual TLS Authenticator and the identity in the client's keystore is bound to cassandra. - addIdentityToRole(spiffeIdentity, "cassandra"); + addIdentityToRole(TlsTestUtils.CLIENT_SPIFFE_IDENTITY, "cassandra"); // when connecting, we expect to get a 'READY' message after sending a 'STARTUP' as MutualTlsAuthenticator // supports early authentication and the client presented a cert with an identity that was bound to a role. @@ -135,8 +125,8 @@ public class EarlyAuthenticationTest extends CQLTester // given server is configured with a Mutual TLS Authenticator and the identity in the client's keystore is bound // to a role that is not permitted to log in. // when connecting, we expect an 'ERROR' message. - addIdentityToRole(spiffeIdentity, "readonly_user"); - testStartupResponse(true, expectAuthenticationError(String.format("readonly_user is not permitted to log in", spiffeIdentity))); + addIdentityToRole(TlsTestUtils.CLIENT_SPIFFE_IDENTITY, "readonly_user"); + testStartupResponse(true, expectAuthenticationError("readonly_user is not permitted to log in")); } @Test @@ -144,7 +134,7 @@ public class EarlyAuthenticationTest extends CQLTester { // given server is configured with a Mutual TLS Authenticator, but no identities are bound to roles. // when connecting, we expect an 'ERROR' message. - testStartupResponse(true, expectAuthenticationError(String.format("Certificate identity '%s' not authorized", spiffeIdentity))); + testStartupResponse(true, expectAuthenticationError(String.format("Certificate identity '%s' not authorized", TlsTestUtils.CLIENT_SPIFFE_IDENTITY))); } @Test diff --git a/test/unit/org/apache/cassandra/transport/SimpleClientSslContextFactory.java b/test/unit/org/apache/cassandra/transport/SimpleClientSslContextFactory.java index 709cc50f70..1a6871716a 100644 --- a/test/unit/org/apache/cassandra/transport/SimpleClientSslContextFactory.java +++ b/test/unit/org/apache/cassandra/transport/SimpleClientSslContextFactory.java @@ -19,7 +19,10 @@ package org.apache.cassandra.transport; import java.util.Map; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; import javax.net.ssl.SSLException; +import javax.net.ssl.TrustManager; import io.netty.handler.ssl.CipherSuiteFilter; import io.netty.handler.ssl.SslContext; @@ -27,6 +30,8 @@ import io.netty.handler.ssl.SslContextBuilder; import org.apache.cassandra.config.EncryptionOptions; import org.apache.cassandra.security.FileBasedSslContextFactory; +import static org.apache.cassandra.config.EncryptionOptions.ClientAuth.NOT_REQUIRED; + /** * A custom implementation of {@link FileBasedSslContextFactory} to be used by tests utilizing {@link SimpleClient}. *

@@ -43,6 +48,33 @@ public class SimpleClientSslContextFactory extends FileBasedSslContextFactory super(parameters); } + @Override + public SSLContext createJSSESslContext(EncryptionOptions.ClientAuth clientAuth) throws SSLException + { + TrustManager[] trustManagers = null; + if (clientAuth != NOT_REQUIRED) + trustManagers = buildTrustManagerFactory().getTrustManagers(); + + KeyManagerFactory kmf = null; + + // only provide a client certificate if keystore is present. + if (hasOutboundKeystore()) + { + kmf = buildKeyManagerFactory(); + } + + try + { + SSLContext ctx = SSLContext.getInstance("TLS"); + ctx.init(kmf != null ? kmf.getKeyManagers() : null, trustManagers, null); + return ctx; + } + catch (Exception e) + { + throw new SSLException("Error creating/initializing the SSL Context", e); + } + } + @Override public SslContext createNettySslContext(EncryptionOptions.ClientAuth clientAuth, SocketType socketType, CipherSuiteFilter cipherFilter) throws SSLException diff --git a/test/unit/org/apache/cassandra/transport/TlsTestUtils.java b/test/unit/org/apache/cassandra/transport/TlsTestUtils.java new file mode 100644 index 0000000000..8939937c98 --- /dev/null +++ b/test/unit/org/apache/cassandra/transport/TlsTestUtils.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.transport; + +import java.util.Collections; +import java.util.Map; +import javax.net.ssl.SSLException; + +import com.google.common.collect.ImmutableMap; + +import com.datastax.driver.core.RemoteEndpointAwareJdkSSLOptions; +import com.datastax.driver.core.SSLOptions; +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.EncryptionOptions; +import org.apache.cassandra.config.ParameterizedClass; +import org.apache.cassandra.security.ISslContextFactory; + +public class TlsTestUtils +{ + public static String SERVER_KEYSTORE_PATH = "test/conf/cassandra_ssl_test.keystore"; + public static String SERVER_KEYSTORE_PATH_PEM = "test/conf/cassandra_ssl_test.keystore.pem"; + public static String SERVER_KEYSTORE_PATH_UNENCRYPTED_PEM = "test/conf/cassandra_ssl_test.unencrypted_keystore.pem"; + public static String SERVER_KEYSTORE_PASSWORD = "cassandra"; + + public static String SERVER_KEYSTORE_ENDPOINT_VERIFY_PATH = "test/conf/cassandra_ssl_test_endpoint_verify.keystore"; + public static String SERVER_KEYSTORE_ENDPOINT_VERIFY_PASSWORD = "cassandra"; + + public static String SERVER_OUTBOUND_KEYSTORE_PATH = "test/conf/cassandra_ssl_test_outbound.keystore"; + public static String SERVER_OUTBOUND_KEYSTORE_PASSWORD = "cassandra"; + + public static String SERVER_TRUSTSTORE_PATH = "test/conf/cassandra_ssl_test.truststore"; + public static String SERVER_TRUSTSTORE_PEM_PATH = "test/conf/cassandra_ssl_test.truststore.pem"; + public static String SERVER_TRUSTSTORE_PASSWORD = "cassandra"; + + // To regenerate: + // 1. generate keystore + // keytool -genkeypair -keystore test/conf/cassandra_ssl_test_spiffe.keystore -validity 100000 -keyalg RSA -dname "CN=Apache Cassandra, OU=ssl_test, O=Unknown, L=Unknown, ST=Unknown, C=Unknown" -keypass cassandra -storepass cassandra -alias spiffecert -ext SAN=URI:spiffe://test.cassandra.apache.org/unitTest/mtls -storetype jks + // 2. export cert + // keytool -export -alias spiffecert -file spiffecert.cer -keystore test/conf/cassandra_ssl_test_spiffe.keystore + // 3. import cert into truststore + // keytool -import -v -trustcacerts -alias spiffecert -file spiffecert.cer -keystore test/conf/cassandra_ssl_test.truststore + public static String CLIENT_SPIFFE_KEYSTORE_PATH = "test/conf/cassandra_ssl_test_spiffe.keystore"; + public static String CLIENT_SPIFFE_KEYSTORE_PASSWORD = "cassandra"; + public static String CLIENT_SPIFFE_IDENTITY = "spiffe://test.cassandra.apache.org/unitTest/mtls"; + + public static String CLIENT_TRUSTSTORE_PATH = "test/conf/cassandra_ssl_test.truststore"; + public static String CLIENT_TRUSTSTORE_PASSWORD = "cassandra"; + + public static EncryptionOptions getClientEncryptionOptions() + { + return new EncryptionOptions(new EncryptionOptions() + .withEnabled(true) + .withRequireClientAuth(EncryptionOptions.ClientAuth.OPTIONAL) + .withOptional(true) + .withKeyStore(SERVER_KEYSTORE_PATH) + .withKeyStorePassword(SERVER_KEYSTORE_PASSWORD) + .withTrustStore(SERVER_TRUSTSTORE_PATH) + .withTrustStorePassword(SERVER_TRUSTSTORE_PASSWORD) + .withRequireEndpointVerification(false)); + } + + public static void configureWithMutualTlsWithPasswordFallbackAuthenticator(Config config) + { + // Configure an authenticator that supports multiple authentication mechanisms. + Map parameters = Collections.singletonMap("validator_class_name", "org.apache.cassandra.auth.SpiffeCertificateValidator"); + config.authenticator = new ParameterizedClass("MutualTlsWithPasswordFallbackAuthenticator", parameters); + // Configure client encryption such that we can optionally connect with SSL. + config.client_encryption_options = TlsTestUtils.getClientEncryptionOptions(); + config.role_manager = "CassandraRoleManager"; + config.authorizer = "CassandraAuthorizer"; + } + + public static ISslContextFactory getClientSslContextFactory(boolean provideClientCert) + { + ImmutableMap.Builder params = ImmutableMap.builder() + .put("truststore", CLIENT_TRUSTSTORE_PATH) + .put("truststore_password", CLIENT_TRUSTSTORE_PASSWORD); + + if (provideClientCert) + { + params.put("keystore", CLIENT_SPIFFE_KEYSTORE_PATH) + .put("keystore_password", CLIENT_SPIFFE_KEYSTORE_PASSWORD); + } + + return new SimpleClientSslContextFactory(params.build()); + } + + public static SSLOptions getSSLOptions(boolean provideClientCert) throws SSLException + { + return RemoteEndpointAwareJdkSSLOptions.builder() + .withSSLContext(getClientSslContextFactory(provideClientCert) + .createJSSESslContext(EncryptionOptions.ClientAuth.OPTIONAL)) + .build(); + } + +}