Add histogram for delay to deliver hints

Patch by Jeff Jirsa; Reviewed by Stefan Podkowinski for CASSANDRA-13234

CHANGES.txt

@@ -38,6 +38,7 @@
  * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
  * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
  * Add support for arithmetic operators (CASSANDRA-11935)
+ * Add histogram for delay to deliver hints (CASSANDRA-13234)
 
 
 3.11.0

doc/source/operating/metrics.rst

@@ -524,6 +524,31 @@ Hints_created-<PeerIP>       Counter        Number of hints on disk for this pee
 Hints_not_stored-<PeerIP>    Counter        Number of hints not stored for this peer, due to being down past the configured hint window.
 =========================== ============== ===========
 
+HintsService Metrics
+^^^^^^^^^^^^^^^^^^^^^
+
+Metrics specific to the Hints delivery service.  There are also some metrics related to hints tracked in ``Storage Metrics``
+
+These metrics include the peer endpoint **in the metric name**
+
+Reported name format:
+
+**Metric Name**
+    ``org.apache.cassandra.metrics.HintsService.<MetricName>``
+
+**JMX MBean**
+    ``org.apache.cassandra.metrics:type=HintsService name=<MetricName>``
+
+=========================== ============== ===========
+Name                        Type           Description
+=========================== ============== ===========
+HintsSucceeded               Meter          A meter of the hints successfully delivered
+HintsFailed                  Meter          A meter of the hints that failed deliver
+HintsTimedOut                Meter          A meter of the hints that timed out
+Hints_delays                 Histogram      Histogram of hint delivery delays (in milliseconds)
+Hints_delays-<PeerIP>        Histogram      Histogram of hint delivery delays (in milliseconds) per peer
+=========================== ============== ===========
+
 SSTable Index Metrics
 ^^^^^^^^^^^^^^^^^^^^^
 

src/java/org/apache/cassandra/hints/EncodedHintMessage.java

@@ -58,6 +58,11 @@ final class EncodedHintMessage
         return new MessageOut<>(MessagingService.Verb.HINT, this, serializer);
     }
 
+    public long getHintCreationTime()
+    {
+        return Hint.serializer.getHintCreationTime(hint, version);
+    }
+
     private static class Serializer implements IVersionedSerializer<EncodedHintMessage>
     {
         public long serializedSize(EncodedHintMessage message, int version)

src/java/org/apache/cassandra/hints/Hint.java

@@ -18,6 +18,7 @@
 package org.apache.cassandra.hints;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 
@@ -146,5 +147,10 @@ public final class Hint
             int gcgs = (int) in.readUnsignedVInt();
             return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs);
         }
+
+        public long getHintCreationTime(ByteBuffer hintBuffer, int version)
+        {
+            return hintBuffer.getLong(0);
+        }
     }
 }

src/java/org/apache/cassandra/hints/HintsDispatcher.java

@@ -27,6 +27,7 @@ import java.util.function.Function;
 
 import com.google.common.util.concurrent.RateLimiter;
 
+import org.apache.cassandra.db.monitoring.ApproximateTime;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.metrics.HintsServiceMetrics;
 import org.apache.cassandra.net.IAsyncCallbackWithFailure;
@@ -175,6 +176,7 @@ final class HintsDispatcher implements AutoCloseable
     {
         Callback callback = new Callback();
         HintMessage message = new HintMessage(hostId, hint);
+        HintsServiceMetrics.updateDelayMetrics(address , ApproximateTime.currentTimeMillis() - hint.creationTime);
         MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback);
         return callback;
     }
@@ -187,6 +189,7 @@ final class HintsDispatcher implements AutoCloseable
     {
         Callback callback = new Callback();
         EncodedHintMessage message = new EncodedHintMessage(hostId, hint, messagingVersion);
+        HintsServiceMetrics.updateDelayMetrics(address, ApproximateTime.currentTimeMillis() - message.getHintCreationTime());
         MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback);
         return callback;
     }

src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java

@@ -17,7 +17,16 @@
  */
 package org.apache.cassandra.metrics;
 
+import java.net.InetAddress;
+
+import com.google.common.util.concurrent.MoreExecutors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
@@ -26,9 +35,31 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
  */
 public final class HintsServiceMetrics
 {
+    private static final Logger logger = LoggerFactory.getLogger(HintsServiceMetrics.class);
+
     private static final MetricNameFactory factory = new DefaultNameFactory("HintsService");
 
     public static final Meter hintsSucceeded = Metrics.meter(factory.createMetricName("HintsSucceeded"));
     public static final Meter hintsFailed    = Metrics.meter(factory.createMetricName("HintsFailed"));
     public static final Meter hintsTimedOut  = Metrics.meter(factory.createMetricName("HintsTimedOut"));
+
+    /** Histogram of all hint delivery delays */
+    private static final Histogram globalDelayHistogram = Metrics.histogram(factory.createMetricName("Hint_delays"), false);
+
+    /** Histograms per-endpoint of hint delivery delays, This is not a cache. */
+    private static final LoadingCache<InetAddress, Histogram> delayByEndpoint = Caffeine.newBuilder()
+        .executor(MoreExecutors.directExecutor())
+        .build(address -> Metrics.histogram(factory.createMetricName("Hint_delays-"+address.getHostAddress().replace(':', '.')), false));
+
+    public static void updateDelayMetrics(InetAddress endpoint, long delay)
+    {
+        if (delay <= 0)
+        {
+            logger.warn("Invalid negative latency in hint delivery delay: {}", delay);
+            return;
+        }
+
+        globalDelayHistogram.update(delay);
+        delayByEndpoint.get(endpoint).update(delay);
+    }
 }