From 0c5faef664aa403998f59cc77c9b39861890cfa1 Mon Sep 17 00:00:00 2001 From: Jeff Jirsa Date: Fri, 17 Feb 2017 14:20:49 -0800 Subject: [PATCH] Add histogram for delay to deliver hints Patch by Jeff Jirsa; Reviewed by Stefan Podkowinski for CASSANDRA-13234 --- CHANGES.txt | 1 + doc/source/operating/metrics.rst | 25 +++++++++++++++ .../cassandra/hints/EncodedHintMessage.java | 5 +++ src/java/org/apache/cassandra/hints/Hint.java | 6 ++++ .../cassandra/hints/HintsDispatcher.java | 3 ++ .../metrics/HintsServiceMetrics.java | 31 +++++++++++++++++++ 6 files changed, 71 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 3acc2b41ee..7247f360aa 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -38,6 +38,7 @@ * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969) * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946) * Add support for arithmetic operators (CASSANDRA-11935) + * Add histogram for delay to deliver hints (CASSANDRA-13234) 3.11.0 diff --git a/doc/source/operating/metrics.rst b/doc/source/operating/metrics.rst index 373d4d2f9a..af2e36e5e0 100644 --- a/doc/source/operating/metrics.rst +++ b/doc/source/operating/metrics.rst @@ -524,6 +524,31 @@ Hints_created- Counter Number of hints on disk for this pee Hints_not_stored- Counter Number of hints not stored for this peer, due to being down past the configured hint window. =========================== ============== =========== +HintsService Metrics +^^^^^^^^^^^^^^^^^^^^^ + +Metrics specific to the Hints delivery service. There are also some metrics related to hints tracked in ``Storage Metrics`` + +These metrics include the peer endpoint **in the metric name** + +Reported name format: + +**Metric Name** + ``org.apache.cassandra.metrics.HintsService.`` + +**JMX MBean** + ``org.apache.cassandra.metrics:type=HintsService name=`` + +=========================== ============== =========== +Name Type Description +=========================== ============== =========== +HintsSucceeded Meter A meter of the hints successfully delivered +HintsFailed Meter A meter of the hints that failed deliver +HintsTimedOut Meter A meter of the hints that timed out +Hints_delays Histogram Histogram of hint delivery delays (in milliseconds) +Hints_delays- Histogram Histogram of hint delivery delays (in milliseconds) per peer +=========================== ============== =========== + SSTable Index Metrics ^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java index 4fe05acaca..50d1302622 100644 --- a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java +++ b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java @@ -58,6 +58,11 @@ final class EncodedHintMessage return new MessageOut<>(MessagingService.Verb.HINT, this, serializer); } + public long getHintCreationTime() + { + return Hint.serializer.getHintCreationTime(hint, version); + } + private static class Serializer implements IVersionedSerializer { public long serializedSize(EncodedHintMessage message, int version) diff --git a/src/java/org/apache/cassandra/hints/Hint.java b/src/java/org/apache/cassandra/hints/Hint.java index 4e8f1395bd..1582a3ccf0 100644 --- a/src/java/org/apache/cassandra/hints/Hint.java +++ b/src/java/org/apache/cassandra/hints/Hint.java @@ -18,6 +18,7 @@ package org.apache.cassandra.hints; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -146,5 +147,10 @@ public final class Hint int gcgs = (int) in.readUnsignedVInt(); return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs); } + + public long getHintCreationTime(ByteBuffer hintBuffer, int version) + { + return hintBuffer.getLong(0); + } } } diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java index 3ac77a3a62..6fc55dfb40 100644 --- a/src/java/org/apache/cassandra/hints/HintsDispatcher.java +++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java @@ -27,6 +27,7 @@ import java.util.function.Function; import com.google.common.util.concurrent.RateLimiter; +import org.apache.cassandra.db.monitoring.ApproximateTime; import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.metrics.HintsServiceMetrics; import org.apache.cassandra.net.IAsyncCallbackWithFailure; @@ -175,6 +176,7 @@ final class HintsDispatcher implements AutoCloseable { Callback callback = new Callback(); HintMessage message = new HintMessage(hostId, hint); + HintsServiceMetrics.updateDelayMetrics(address , ApproximateTime.currentTimeMillis() - hint.creationTime); MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback); return callback; } @@ -187,6 +189,7 @@ final class HintsDispatcher implements AutoCloseable { Callback callback = new Callback(); EncodedHintMessage message = new EncodedHintMessage(hostId, hint, messagingVersion); + HintsServiceMetrics.updateDelayMetrics(address, ApproximateTime.currentTimeMillis() - message.getHintCreationTime()); MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback); return callback; } diff --git a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java index ad85281b90..d6a75f7e20 100644 --- a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java @@ -17,7 +17,16 @@ */ package org.apache.cassandra.metrics; +import java.net.InetAddress; + +import com.google.common.util.concurrent.MoreExecutors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; @@ -26,9 +35,31 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; */ public final class HintsServiceMetrics { + private static final Logger logger = LoggerFactory.getLogger(HintsServiceMetrics.class); + private static final MetricNameFactory factory = new DefaultNameFactory("HintsService"); public static final Meter hintsSucceeded = Metrics.meter(factory.createMetricName("HintsSucceeded")); public static final Meter hintsFailed = Metrics.meter(factory.createMetricName("HintsFailed")); public static final Meter hintsTimedOut = Metrics.meter(factory.createMetricName("HintsTimedOut")); + + /** Histogram of all hint delivery delays */ + private static final Histogram globalDelayHistogram = Metrics.histogram(factory.createMetricName("Hint_delays"), false); + + /** Histograms per-endpoint of hint delivery delays, This is not a cache. */ + private static final LoadingCache delayByEndpoint = Caffeine.newBuilder() + .executor(MoreExecutors.directExecutor()) + .build(address -> Metrics.histogram(factory.createMetricName("Hint_delays-"+address.getHostAddress().replace(':', '.')), false)); + + public static void updateDelayMetrics(InetAddress endpoint, long delay) + { + if (delay <= 0) + { + logger.warn("Invalid negative latency in hint delivery delay: {}", delay); + return; + } + + globalDelayHistogram.update(delay); + delayByEndpoint.get(endpoint).update(delay); + } }