From 284271670c389f2b9fd11271c33eca5bb2a228fd Mon Sep 17 00:00:00 2001 From: Simon Zhou Date: Thu, 30 Mar 2017 20:45:03 -0700 Subject: [PATCH] Introduce error metrics for repair patch by Simon Zhou; reviewed by Stefan Podkowinski for CASSANDRA-13387 --- CHANGES.txt | 1 + .../org/apache/cassandra/metrics/StorageMetrics.java | 3 ++- .../org/apache/cassandra/repair/RepairRunnable.java | 10 +++++++++- .../org/apache/cassandra/service/CassandraDaemon.java | 2 +- src/java/org/apache/cassandra/tools/NodeProbe.java | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d747d50e9e..c0abed77e4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Introduce error metrics for repair (CASSANDRA-13387) * Refactoring to primitive functional interfaces in AuthCache (CASSANDRA-13732) * Update metrics to 3.1.5 (CASSANDRA-13648) * batch_size_warn_threshold_in_kb can now be set at runtime (CASSANDRA-13699) diff --git a/src/java/org/apache/cassandra/metrics/StorageMetrics.java b/src/java/org/apache/cassandra/metrics/StorageMetrics.java index 12196f70e8..9399ba6789 100644 --- a/src/java/org/apache/cassandra/metrics/StorageMetrics.java +++ b/src/java/org/apache/cassandra/metrics/StorageMetrics.java @@ -29,7 +29,8 @@ public class StorageMetrics private static final MetricNameFactory factory = new DefaultNameFactory("Storage"); public static final Counter load = Metrics.counter(factory.createMetricName("Load")); - public static final Counter exceptions = Metrics.counter(factory.createMetricName("Exceptions")); + public static final Counter uncaughtExceptions = Metrics.counter(factory.createMetricName("Exceptions")); public static final Counter totalHintsInProgress = Metrics.counter(factory.createMetricName("TotalHintsInProgress")); public static final Counter totalHints = Metrics.counter(factory.createMetricName("TotalHints")); + public static final Counter repairExceptions = Metrics.counter(factory.createMetricName("RepairExceptions")); } diff --git a/src/java/org/apache/cassandra/repair/RepairRunnable.java b/src/java/org/apache/cassandra/repair/RepairRunnable.java index 3f761ee60d..b581ebdbb5 100644 --- a/src/java/org/apache/cassandra/repair/RepairRunnable.java +++ b/src/java/org/apache/cassandra/repair/RepairRunnable.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.repair; +import java.io.IOException; import java.net.InetAddress; import java.nio.ByteBuffer; import java.util.*; @@ -50,6 +51,7 @@ import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.repair.consistent.CoordinatorSession; +import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.QueryState; @@ -121,6 +123,7 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti protected void fireErrorAndComplete(int progressCount, int totalProgress, String message) { + StorageMetrics.repairExceptions.inc(); fireProgressEvent(new ProgressEvent(ProgressEventType.ERROR, progressCount, totalProgress, message)); String completionMessage = String.format("Repair command #%d finished with error", cmd); fireProgressEvent(new ProgressEvent(ProgressEventType.COMPLETE, progressCount, totalProgress, completionMessage)); @@ -144,7 +147,7 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti validColumnFamilies = storageService.getValidColumnFamilies(false, false, keyspace, columnFamilies); progress.incrementAndGet(); } - catch (IllegalArgumentException e) + catch (IllegalArgumentException | IOException e) { logger.error("Repair failed:", e); fireErrorAndComplete(progress.get(), totalProgress, e.getMessage()); @@ -396,6 +399,7 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti public void onFailure(Throwable t) { + StorageMetrics.repairExceptions.inc(); fireProgressEvent(new ProgressEvent(ProgressEventType.ERROR, progress.get(), totalProgress, t.getMessage())); logger.error("Error completing preview repair", t); String completionMessage = complete(); @@ -476,6 +480,8 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti public void onFailure(Throwable t) { + StorageMetrics.repairExceptions.inc(); + String message = String.format("Repair session %s for range %s failed with error %s", session.getId(), session.getRanges().toString(), t.getMessage()); logger.error(message, t); @@ -519,6 +525,7 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti final String message; if (hasFailure.get()) { + StorageMetrics.repairExceptions.inc(); message = "Some repair failed"; fireProgressEvent(new ProgressEvent(ProgressEventType.ERROR, progress.get(), totalProgress, message)); @@ -543,6 +550,7 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti public void onFailure(Throwable t) { + StorageMetrics.repairExceptions.inc(); fireProgressEvent(new ProgressEvent(ProgressEventType.ERROR, progress.get(), totalProgress, t.getMessage())); if (!options.isPreview()) { diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index f412aabbdc..88d812d57d 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -207,7 +207,7 @@ public class CassandraDaemon { public void uncaughtException(Thread t, Throwable e) { - StorageMetrics.exceptions.inc(); + StorageMetrics.uncaughtExceptions.inc(); logger.error("Exception in thread " + t, e); Tracing.trace("Exception in thread {}", t, e); for (Throwable e2 = e; e2 != null; e2 = e2.getCause()) diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 2615412c77..665ea6fb95 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -1100,7 +1100,7 @@ public class NodeProbe implements AutoCloseable public int getExceptionCount() { - return (int)StorageMetrics.exceptions.getCount(); + return (int)StorageMetrics.uncaughtExceptions.getCount(); } public Map getDroppedMessages()