From e301c38d3a592ff943ea0703403542dd2b7f499d Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 5 Jun 2013 15:11:56 -0500 Subject: [PATCH 1/2] More detailed read repair metrics. Patch by Jingsi Zhu, reviewed by brandonwilliams for CASSANDRA-5618 --- CHANGES.txt | 1 + .../cassandra/metrics/ReadRepairMetrics.java | 25 +++++++++++++++++++ .../cassandra/service/ReadCallback.java | 5 +++- .../cassandra/service/StorageProxy.java | 25 ++++++++++++++++++- .../cassandra/service/StorageProxyMBean.java | 6 +++++ .../org/apache/cassandra/tools/NodeCmd.java | 2 ++ .../org/apache/cassandra/tools/NodeProbe.java | 15 +++++++++++ 7 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java diff --git a/CHANGES.txt b/CHANGES.txt index edbb94dbfd..75b3173012 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -19,6 +19,7 @@ * cqlsh: fix COPY FROM with ReversedType (CASSANDRA-5610) * Allow creating CUSTOM indexes on collections (CASSANDRA-5615) * Evaluate now() function at execution time (CASSANDRA-5616) + * Expose detailed read repair metrics (CASSANDRA-5618) Merged from 1.1: * Remove buggy thrift max message length option (CASSANDRA-5529) * Fix NPE in Pig's widerow mode (CASSANDRA-5488) diff --git a/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java new file mode 100644 index 0000000000..3f48fee8ec --- /dev/null +++ b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java @@ -0,0 +1,25 @@ +package org.apache.cassandra.metrics; + +import java.util.concurrent.TimeUnit; + +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Counter; +import com.yammer.metrics.core.Gauge; +import com.yammer.metrics.core.Meter; +import com.yammer.metrics.core.MetricName; +import com.yammer.metrics.util.RatioGauge; + +/** + * Metrics related to Read Repair. + */ +public class ReadRepairMetrics { + public static final String GROUP_NAME = "org.apache.cassandra.metrics"; + public static final String TYPE_NAME = "ReadRepair"; + + public static final Meter repairedBlocking = + Metrics.newMeter(new MetricName(GROUP_NAME, TYPE_NAME, "RepairedBlocking"), "RepairedBlocking", TimeUnit.SECONDS); + public static final Meter repairedBackground = + Metrics.newMeter(new MetricName(GROUP_NAME, TYPE_NAME, "RepairedBackground"), "RepairedBackground", TimeUnit.SECONDS); + public static final Meter attempted = + Metrics.newMeter(new MetricName(GROUP_NAME, TYPE_NAME, "Attempted"), "Attempted", TimeUnit.SECONDS); +} diff --git a/src/java/org/apache/cassandra/service/ReadCallback.java b/src/java/org/apache/cassandra/service/ReadCallback.java index a19df5f9f0..92032f2298 100644 --- a/src/java/org/apache/cassandra/service/ReadCallback.java +++ b/src/java/org/apache/cassandra/service/ReadCallback.java @@ -34,6 +34,7 @@ import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.Table; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.metrics.ReadRepairMetrics; import org.apache.cassandra.net.IAsyncCallback; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.net.MessageOut; @@ -174,7 +175,9 @@ public class ReadCallback implements IAsyncCallback endpoints = getLiveSortedEndpoints(table, command.key); CFMetaData cfm = Schema.instance.getCFMetaData(command.getKeyspace(), command.getColumnFamilyName()); - endpoints = consistency_level.filterForQuery(table, endpoints, cfm.newReadRepairDecision()); + + ReadRepairDecision rrDecision = cfm.newReadRepairDecision(); + endpoints = consistency_level.filterForQuery(table, endpoints, rrDecision); + + if (rrDecision != ReadRepairDecision.NONE) { + ReadRepairMetrics.attempted.mark(); + } RowDigestResolver resolver = new RowDigestResolver(command.table, command.key); ReadCallback handler = new ReadCallback(resolver, consistency_level, command, endpoints); @@ -960,6 +968,9 @@ public class StorageProxy implements StorageProxyMBean catch (DigestMismatchException ex) { logger.debug("Digest mismatch: {}", ex.toString()); + + ReadRepairMetrics.repairedBlocking.mark(); + // Do a full data read to resolve the correct response (and repair node that need be) RowDataResolver resolver = new RowDataResolver(command.table, command.key, command.filter()); ReadCallback repairHandler = handler.withNewResolver(resolver); @@ -1702,4 +1713,16 @@ public class StorageProxy implements StorageProxyMBean public Long getTruncateRpcTimeout() { return DatabaseDescriptor.getTruncateRpcTimeout(); } public void setTruncateRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setTruncateRpcTimeout(timeoutInMillis); } + + public long getReadRepairAttempted() { + return ReadRepairMetrics.attempted.count(); + } + + public long getReadRepairRepairedBlocking() { + return ReadRepairMetrics.repairedBlocking.count(); + } + + public long getReadRepairRepairedBackground() { + return ReadRepairMetrics.repairedBackground.count(); + } } diff --git a/src/java/org/apache/cassandra/service/StorageProxyMBean.java b/src/java/org/apache/cassandra/service/StorageProxyMBean.java index 1bbfd1f71a..e8f5b4aeb1 100644 --- a/src/java/org/apache/cassandra/service/StorageProxyMBean.java +++ b/src/java/org/apache/cassandra/service/StorageProxyMBean.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.service; +import org.apache.cassandra.metrics.ReadRepairMetrics; + public interface StorageProxyMBean { /** @@ -86,4 +88,8 @@ public interface StorageProxyMBean public void setRangeRpcTimeout(Long timeoutInMillis); public Long getTruncateRpcTimeout(); public void setTruncateRpcTimeout(Long timeoutInMillis); + + public long getReadRepairAttempted(); + public long getReadRepairRepairedBlocking(); + public long getReadRepairRepairedBackground(); } diff --git a/src/java/org/apache/cassandra/tools/NodeCmd.java b/src/java/org/apache/cassandra/tools/NodeCmd.java index bca0fcd5a5..1cb30ade9d 100644 --- a/src/java/org/apache/cassandra/tools/NodeCmd.java +++ b/src/java/org/apache/cassandra/tools/NodeCmd.java @@ -669,6 +669,8 @@ public class NodeCmd outs.printf(" Error retrieving file data for %s%n", host); } } + + outs.printf("Read Repair Statistics:%nAttempted: %d%nMismatch (Blocking): %d%nMismatch (Background): %d%n", probe.getReadRepairAttempted(), probe.getReadRepairRepairedBlocking(), probe.getReadRepairRepairedBackground()); MessagingServiceMBean ms = probe.msProxy; outs.printf("%-25s", "Pool Name"); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 5db8f1c90c..dbcb66e59e 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -825,6 +825,21 @@ public class NodeProbe { return failed; } + + public long getReadRepairAttempted() + { + return spProxy.getReadRepairAttempted(); + } + + public long getReadRepairRepairedBlocking() + { + return spProxy.getReadRepairRepairedBlocking(); + } + + public long getReadRepairRepairedBackground() + { + return spProxy.getReadRepairRepairedBackground(); + } } class ColumnFamilyStoreMBeanIterator implements Iterator> From 6850cdc3ed9ea15fecc13beea093012c124f8b8c Mon Sep 17 00:00:00 2001 From: Sylvain Lebresne Date: Thu, 6 Jun 2013 12:16:28 +0200 Subject: [PATCH 2/2] Fix typo in CQL doc --- doc/cql3/CQL.textile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile index 059342eb74..13b6f702a5 100644 --- a/doc/cql3/CQL.textile +++ b/doc/cql3/CQL.textile @@ -976,9 +976,9 @@ CQL distinguishes between _reserved_ and _non-reserved_ keywords. Reserved keywo | @BOOLEAN@ | no | | @BY@ | yes | | @CLUSTERING@ | no | -| @COLUMNFAMLY@ | yes | +| @COLUMNFAMILY@ | yes | | @COMPACT@ | no | -| @CONSITENCY@ | no | +| @CONSISTENCY@ | no | | @COUNT@ | no | | @COUNTER@ | no | | @CREATE@ | yes |