From 473e8dfd7be95815ee10502f021bd7deb8734fba Mon Sep 17 00:00:00 2001 From: Dinesh Joshi Date: Wed, 7 Feb 2018 15:08:01 -0800 Subject: [PATCH] Add hot reloading of SSL Certificates patch by Dinesh Joshi; reviewed by jasobrown for CASSANDRA-14222 --- doc/source/operating/security.rst | 10 ++ .../cassandra/config/DatabaseDescriptor.java | 8 + .../cassandra/net/MessagingService.java | 7 + .../cassandra/net/MessagingServiceMBean.java | 2 + .../apache/cassandra/security/SSLFactory.java | 148 +++++++++++++++++- .../org/apache/cassandra/tools/NodeProbe.java | 5 + .../org/apache/cassandra/tools/NodeTool.java | 3 +- .../tools/ReloadSslCertificates.java | 30 ++++ .../cassandra/security/SSLFactoryTest.java | 38 ++++- 9 files changed, 239 insertions(+), 12 deletions(-) create mode 100644 src/java/org/apache/cassandra/tools/ReloadSslCertificates.java diff --git a/doc/source/operating/security.rst b/doc/source/operating/security.rst index 21245fdcf2..1859dbc59c 100644 --- a/doc/source/operating/security.rst +++ b/doc/source/operating/security.rst @@ -56,6 +56,16 @@ for more details. For information on generating the keystore and truststore files used in SSL communications, see the `java documentation on creating keystores `__ +SSL Certificate Hot Reloading +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Beginning with Cassandra 4, Cassandra supports hot reloading of SSL Certificates. If SSL/TLS support is enabled in Cassandra, +the node periodically polls the Trust and Key Stores specified in cassandra.yaml. When the files are updated, Cassandra will +reload them and use them for subsequent connections. Please note that the Trust & Key Store passwords are part of the yaml so +the updated files should also use the same passwords. The default polling interval is 10 minutes. + +Certificate Hot reloading may also be triggered using the ``nodetool reloadssl`` command. Use this if you want to Cassandra to +immediately notice the changed certificates. + Inter-node Encryption ~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 8e831cf2b7..0714245643 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -60,6 +60,7 @@ import org.apache.cassandra.locator.SeedProvider; import org.apache.cassandra.net.BackPressureStrategy; import org.apache.cassandra.net.RateBasedBackPressure; import org.apache.cassandra.security.EncryptionContext; +import org.apache.cassandra.security.SSLFactory; import org.apache.cassandra.service.CacheService.CacheType; import org.apache.cassandra.utils.FBUtilities; @@ -322,6 +323,8 @@ public class DatabaseDescriptor applySeedProvider(); applyEncryptionContext(); + + applySslContextHotReload(); } private static void applySimpleConfig() @@ -865,6 +868,11 @@ public class DatabaseDescriptor encryptionContext = new EncryptionContext(conf.transparent_data_encryption_options); } + public static void applySslContextHotReload() + { + SSLFactory.initHotReloading(conf.server_encryption_options, conf.client_encryption_options, false); + } + public static void applySeedProvider() { // load the seeds for node contact points diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index 9f00d27c3e..8fdb3954e3 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -101,6 +101,7 @@ import org.apache.cassandra.net.async.NettyFactory.InboundInitializer; import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.schema.MigrationManager; import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.security.SSLFactory; import org.apache.cassandra.service.AbstractWriteResponseHandler; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageService; @@ -1664,4 +1665,10 @@ public final class MessagingService implements MessagingServiceMBean } return true; } + + @Override + public void reloadSslCertificates() + { + SSLFactory.checkCertFilesForHotReloading(); + } } diff --git a/src/java/org/apache/cassandra/net/MessagingServiceMBean.java b/src/java/org/apache/cassandra/net/MessagingServiceMBean.java index f4a0c434cf..6adb891950 100644 --- a/src/java/org/apache/cassandra/net/MessagingServiceMBean.java +++ b/src/java/org/apache/cassandra/net/MessagingServiceMBean.java @@ -129,4 +129,6 @@ public interface MessagingServiceMBean public boolean isBackPressureEnabled(); public int getVersion(String address) throws UnknownHostException; + + void reloadSslCertificates(); } diff --git a/src/java/org/apache/cassandra/security/SSLFactory.java b/src/java/org/apache/cassandra/security/SSLFactory.java index a931f5fbdb..0bf769cdf7 100644 --- a/src/java/org/apache/cassandra/security/SSLFactory.java +++ b/src/java/org/apache/cassandra/security/SSLFactory.java @@ -18,16 +18,20 @@ package org.apache.cassandra.security; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; import java.security.KeyStore; import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.Enumeration; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; @@ -37,6 +41,7 @@ import javax.net.ssl.TrustManagerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; @@ -49,7 +54,7 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.SslProvider; import io.netty.handler.ssl.SupportedCipherSuiteFilter; -import io.netty.util.ReferenceCountUtil; +import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.EncryptionOptions; /** @@ -77,6 +82,67 @@ public final class SSLFactory */ private static final AtomicReference serverSslContext = new AtomicReference<>(); + /** + * List of files that trigger hot reloading of SSL certificates + */ + private static volatile List hotReloadableFiles = ImmutableList.of(); + + /** + * Default initial delay for hot reloading + */ + public static final int DEFAULT_HOT_RELOAD_INITIAL_DELAY_SEC = 600; + + /** + * Default periodic check delay for hot reloading + */ + public static final int DEFAULT_HOT_RELOAD_PERIOD_SEC = 600; + + /** + * State variable to maintain initialization invariant + */ + private static boolean isHotReloadingInitialized = false; + + /** + * Helper class for hot reloading SSL Contexts + */ + private static class HotReloadableFile + { + enum Type + { + SERVER, + CLIENT + } + + private final File file; + private volatile long lastModTime; + private final Type certType; + + HotReloadableFile(String path, Type type) + { + file = new File(path); + lastModTime = file.lastModified(); + certType = type; + } + + boolean shouldReload() + { + long curModTime = file.lastModified(); + boolean result = curModTime != lastModTime; + lastModTime = curModTime; + return result; + } + + public boolean isServer() + { + return certType == Type.SERVER; + } + + public boolean isClient() + { + return certType == Type.CLIENT; + } + } + /** * Create a JSSE {@link SSLContext}. */ @@ -176,10 +242,14 @@ public final class SSLFactory @VisibleForTesting static SslContext getSslContext(EncryptionOptions options, boolean buildTruststore, boolean forServer, boolean useOpenSsl) throws IOException { - if (forServer && serverSslContext.get() != null) - return serverSslContext.get(); - if (!forServer && clientSslContext.get() != null) - return clientSslContext.get(); + + SslContext sslContext; + + if (forServer && (sslContext = serverSslContext.get()) != null) + return sslContext; + + if (!forServer && (sslContext = clientSslContext.get()) != null) + return sslContext; /* There is a case where the netty/openssl combo might not support using KeyManagerFactory. specifically, @@ -219,7 +289,73 @@ public final class SSLFactory if (ref.compareAndSet(null, ctx)) return ctx; - ReferenceCountUtil.release(ctx); return ref.get(); } + + /** + * Performs a lightweight check whether the certificate files have been refreshed. + * + * @throws IllegalStateException if {@link #initHotReloading(EncryptionOptions.ServerEncryptionOptions, EncryptionOptions, boolean)} + * is not called first + */ + public static void checkCertFilesForHotReloading() + { + if (!isHotReloadingInitialized) + throw new IllegalStateException("Hot reloading functionality has not been initialized."); + + logger.trace("Checking whether certificates have been updated"); + + if (hotReloadableFiles.stream().anyMatch(f -> f.isServer() && f.shouldReload())) + { + logger.info("Server ssl certificates have been updated. Reseting the context for new peer connections."); + serverSslContext.set(null); + } + + if (hotReloadableFiles.stream().anyMatch(f -> f.isClient() && f.shouldReload())) + { + logger.info("Client ssl certificates have been updated. Reseting the context for new client connections."); + clientSslContext.set(null); + } + } + + /** + * Determines whether to hot reload certificates and schedules a periodic task for it. + * + * @param serverEncryptionOptions + * @param clientEncryptionOptions + */ + public static synchronized void initHotReloading(EncryptionOptions.ServerEncryptionOptions serverEncryptionOptions, + EncryptionOptions clientEncryptionOptions, + boolean force) + { + if (isHotReloadingInitialized && !force) + return; + + logger.debug("Initializing hot reloading SSLContext"); + + List fileList = new ArrayList<>(); + + if (serverEncryptionOptions.enabled) + { + fileList.add(new HotReloadableFile(serverEncryptionOptions.keystore, HotReloadableFile.Type.SERVER)); + fileList.add(new HotReloadableFile(serverEncryptionOptions.truststore, HotReloadableFile.Type.SERVER)); + } + + if (clientEncryptionOptions.enabled) + { + fileList.add(new HotReloadableFile(clientEncryptionOptions.keystore, HotReloadableFile.Type.CLIENT)); + fileList.add(new HotReloadableFile(clientEncryptionOptions.truststore, HotReloadableFile.Type.CLIENT)); + } + + hotReloadableFiles = ImmutableList.copyOf(fileList); + + if (!isHotReloadingInitialized) + { + ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(SSLFactory::checkCertFilesForHotReloading, + DEFAULT_HOT_RELOAD_INITIAL_DELAY_SEC, + DEFAULT_HOT_RELOAD_PERIOD_SEC, TimeUnit.SECONDS); + } + + isHotReloadingInitialized = true; + } } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 69b64ab9c3..7ce534189f 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -1647,6 +1647,11 @@ public class NodeProbe implements AutoCloseable { return arsProxy; } + + public void reloadSslCerts() + { + msProxy.reloadSslCertificates(); + } } class ColumnFamilyStoreMBeanIterator implements Iterator> diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index 81f2023afe..2b6fabf81e 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -156,7 +156,8 @@ public class NodeTool RefreshSizeEstimates.class, RelocateSSTables.class, ViewBuildStatus.class, - HandoffWindow.class + HandoffWindow.class, + ReloadSslCertificates.class ); Cli.CliBuilder builder = Cli.builder("nodetool"); diff --git a/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java b/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java new file mode 100644 index 0000000000..f38b8c059c --- /dev/null +++ b/src/java/org/apache/cassandra/tools/ReloadSslCertificates.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools; + +import io.airlift.airline.Command; + +@Command(name = "reloadssl", description = "Signals Cassandra to reload SSL certificates") +public class ReloadSslCertificates extends NodeTool.NodeToolCmd +{ + @Override + public void execute(NodeProbe probe) + { + probe.reloadSslCerts(); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java index 61933a507a..5153a1170e 100644 --- a/test/unit/org/apache/cassandra/security/SSLFactoryTest.java +++ b/test/unit/org/apache/cassandra/security/SSLFactoryTest.java @@ -18,16 +18,12 @@ */ package org.apache.cassandra.security; +import java.io.File; import java.io.IOException; -import java.net.InetAddress; import java.security.cert.CertificateException; import java.util.Arrays; -import javax.net.ssl.SSLServerSocket; import javax.net.ssl.TrustManagerFactory; -import com.google.common.base.Predicates; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -162,4 +158,36 @@ public class SSLFactoryTest SSLFactory.buildKeyManagerFactory(options); Assert.assertTrue(SSLFactory.checkedExpiry); } + + @Test + public void testSslContextReload_HappyPath() throws IOException, InterruptedException + { + try + { + EncryptionOptions options = addKeystoreOptions(encryptionOptions); + options.enabled = true; + + SSLFactory.initHotReloading((ServerEncryptionOptions) options, options, true); + + SslContext oldCtx = SSLFactory.getSslContext(options, true, true, OpenSsl.isAvailable()); + File keystoreFile = new File(options.keystore); + + SSLFactory.checkCertFilesForHotReloading(); + Thread.sleep(5000); + keystoreFile.setLastModified(System.currentTimeMillis()); + + SSLFactory.checkCertFilesForHotReloading(); + SslContext newCtx = SSLFactory.getSslContext(options, true, true, OpenSsl.isAvailable()); + + Assert.assertNotSame(oldCtx, newCtx); + } + catch (Exception e) + { + throw e; + } + finally + { + DatabaseDescriptor.loadConfig(); + } + } }