From 75e2fffe5a07faed316fcbf28789864de970f08f Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 13 Mar 2020 02:24:37 -0700 Subject: [PATCH] Add a ProcessMetrics.TLSPolicyFailures metric This reports the number of policy failures over the past 5s interval. It also is step 1 towards getting this information into status json. --- flow/Net2.actor.cpp | 21 ++++++++++++++------- flow/SystemMonitor.cpp | 1 + flow/SystemMonitor.h | 2 ++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index 011f335a4c..c6d999f6bf 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -220,6 +220,7 @@ public: Int64MetricHandle countYieldCallsTrue; Int64MetricHandle countASIOEvents; Int64MetricHandle countSlowTaskSignals; + Int64MetricHandle countTLSPolicyFailures; Int64MetricHandle priorityMetric; DoubleMetricHandle countLaunchTime; DoubleMetricHandle countReactTime; @@ -891,7 +892,7 @@ Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) } #ifndef TLS_DISABLED -void ConfigureSSLContext( const LoadedTLSConfig& loaded, boost::asio::ssl::context* context ) { +void ConfigureSSLContext( const LoadedTLSConfig& loaded, boost::asio::ssl::context* context, std::function onPolicyFailure ) { try { context->set_options(boost::asio::ssl::context::default_workarounds); context->set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert); @@ -900,8 +901,12 @@ void ConfigureSSLContext( const LoadedTLSConfig& loaded, boost::asio::ssl::conte Reference tlsPolicy = Reference(new TLSPolicy(loaded.getEndpointType())); tlsPolicy->set_verify_peers({ loaded.getVerifyPeers() }); - context->set_verify_callback([policy=tlsPolicy](bool preverified, boost::asio::ssl::verify_context& ctx) { - return policy->verify_peer(preverified, ctx.native_handle()); + context->set_verify_callback([policy=tlsPolicy, onPolicyFailure](bool preverified, boost::asio::ssl::verify_context& ctx) { + bool success = policy->verify_peer(preverified, ctx.native_handle()); + if (!success) { + onPolicyFailure(); + } + return success; }); } else { context->set_verify_callback(boost::bind(&insecurely_always_accept, _1, _2)); @@ -959,7 +964,7 @@ ACTOR static Future watchFileForChanges( std::string filename, AsyncTrigge } } -ACTOR static Future reloadCertificatesOnChange( TLSConfig config, AsyncVar>>* contextVar ) { +ACTOR static Future reloadCertificatesOnChange( TLSConfig config, std::function onPolicyFailure, AsyncVar>>* contextVar ) { if (FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS <= 0) { return Void(); } @@ -983,7 +988,7 @@ ACTOR static Future reloadCertificatesOnChange( TLSConfig config, AsyncVar try { LoadedTLSConfig loaded = wait( config.loadAsync() ); boost::asio::ssl::context context(boost::asio::ssl::context::tls); - ConfigureSSLContext(loaded, &context); + ConfigureSSLContext(loaded, &context, onPolicyFailure); TraceEvent(SevInfo, "TLSCertificateRefreshSucceeded"); mismatches = 0; contextVar->set(ReferencedObject::from(std::move(context))); @@ -1006,9 +1011,10 @@ void Net2::initTLS() { #ifndef TLS_DISABLED try { boost::asio::ssl::context newContext(boost::asio::ssl::context::tls); - ConfigureSSLContext( tlsConfig.loadSync(), &newContext ); + auto onPolicyFailure = [this]() { this->countTLSPolicyFailures++; }; + ConfigureSSLContext( tlsConfig.loadSync(), &newContext, onPolicyFailure ); sslContextVar.set(ReferencedObject::from(std::move(newContext))); - backgroundCertRefresh = reloadCertificatesOnChange( tlsConfig, &sslContextVar ); + backgroundCertRefresh = reloadCertificatesOnChange( tlsConfig, onPolicyFailure, &sslContextVar ); } catch (Error& e) { TraceEvent("Net2TLSInitError").error(e); throw tls_error(); @@ -1044,6 +1050,7 @@ void Net2::initMetrics() { countASIOEvents.init(LiteralStringRef("Net2.CountASIOEvents")); countYieldCallsTrue.init(LiteralStringRef("Net2.CountYieldCallsTrue")); countSlowTaskSignals.init(LiteralStringRef("Net2.CountSlowTaskSignals")); + countTLSPolicyFailures.init(LiteralStringRef("Net2.CountTLSPolicyFailures")); priorityMetric.init(LiteralStringRef("Net2.Priority")); awakeMetric.init(LiteralStringRef("Net2.Awake")); slowTaskMetric.init(LiteralStringRef("Net2.SlowTask")); diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp index cc525f098a..47e5c6497c 100644 --- a/flow/SystemMonitor.cpp +++ b/flow/SystemMonitor.cpp @@ -101,6 +101,7 @@ SystemStatistics customSystemMonitor(std::string eventName, StatisticsState *sta .detail("ConnectionsEstablished", (double) (netData.countConnEstablished - statState->networkState.countConnEstablished) / currentStats.elapsed) .detail("ConnectionsClosed", ((netData.countConnClosedWithError - statState->networkState.countConnClosedWithError) + (netData.countConnClosedWithoutError - statState->networkState.countConnClosedWithoutError)) / currentStats.elapsed) .detail("ConnectionErrors", (netData.countConnClosedWithError - statState->networkState.countConnClosedWithError) / currentStats.elapsed) + .detail("TLSPolicyFailures", (netData.countTLSPolicyFailures - statState->networkState.countTLSPolicyFailures)) .trackLatest(eventName); TraceEvent("MemoryMetrics") diff --git a/flow/SystemMonitor.h b/flow/SystemMonitor.h index ac1cb33817..e8c4ca2c59 100644 --- a/flow/SystemMonitor.h +++ b/flow/SystemMonitor.h @@ -80,6 +80,7 @@ struct NetworkData { int64_t countConnEstablished; int64_t countConnClosedWithError; int64_t countConnClosedWithoutError; + int64_t countTLSPolicyFailures; double countLaunchTime; double countReactTime; @@ -107,6 +108,7 @@ struct NetworkData { countConnEstablished = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountConnEstablished")); countConnClosedWithError = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountConnClosedWithError")); countConnClosedWithoutError = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountConnClosedWithoutError")); + countTLSPolicyFailures = Int64Metric::getValueOrDefault(LiteralStringRef("Net2.CountTLSPolicyFailures")); countLaunchTime = DoubleMetric::getValueOrDefault(LiteralStringRef("Net2.CountLaunchTime")); countReactTime = DoubleMetric::getValueOrDefault(LiteralStringRef("Net2.CountReactTime")); countFileLogicalWrites = Int64Metric::getValueOrDefault(LiteralStringRef("AsyncFile.CountLogicalWrites"));