anph
/
hugegraph
forked from hugegraph/hugegraph
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add metrics api test 

Change-Id: Ia94713a411de021cfa1a792e670455e2fa3c6f2b
This commit is contained in:
Zhangmei Li 2018-11-22 21:39:01 +08:00 committed by Linary
parent 3d0bf54191
commit 74a7eb4dcc
13 changed files with 180 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hugegraph-api/src/main/java/com/baidu/hugegraph/api/metrics/MetricsAPI.java
View File

@ -38,7 +38,6 @@ import com.baidu.hugegraph.api.API;
import com.baidu.hugegraph.backend.store.BackendMetrics;
import com.baidu.hugegraph.backend.tx.GraphTransaction;
import com.baidu.hugegraph.core.GraphManager;
import com.baidu.hugegraph.exception.NotSupportException;
import com.baidu.hugegraph.metric.ServerReporter;
import com.baidu.hugegraph.metric.SystemMetrics;
import com.baidu.hugegraph.util.InsertionOrderUtil;
@ -86,7 +85,7 @@ public class MetricsAPI extends API {
metrics.put(BackendMetrics.BACKEND, tx.store().provider().type());
try {
metrics.putAll(tx.metadata(null, "metrics"));
} catch (NotSupportException e) {
} catch (Throwable e) {
metrics.put(BackendMetrics.EXCEPTION, e.toString());
LOG.debug("Failed to get backend metrics", e);
}

7
hugegraph-api/src/main/java/com/baidu/hugegraph/metric/SystemMetrics.java
View File

@ -106,7 +106,7 @@ public class SystemMetrics {
ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean();
metrics.put("peak", threadMxBean.getPeakThreadCount());
metrics.put("daemon", threadMxBean.getDaemonThreadCount());
metrics.put("totalStarted", threadMxBean.getTotalStartedThreadCount());
metrics.put("total_started", threadMxBean.getTotalStartedThreadCount());
metrics.put("count", threadMxBean.getThreadCount());
return metrics;
}
@ -126,14 +126,15 @@ public class SystemMetrics {
List<GarbageCollectorMXBean> gcMxBeans = ManagementFactory
.getGarbageCollectorMXBeans();
for (GarbageCollectorMXBean gcMxBean : gcMxBeans) {
String name = this.formatName(gcMxBean.getName());
String name = formatName(gcMxBean.getName());
metrics.put(name + "_count", gcMxBean.getCollectionCount());
metrics.put(name + "_time", gcMxBean.getCollectionTime());
}
metrics.put("time_unit", "ms");
return metrics;
}
private String formatName(String name) {
private static String formatName(String name) {
return StringUtils.replace(name, " ", "_").toLowerCase();
}
}

12
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraMetrics.java
View File

@ -19,6 +19,7 @@
package com.baidu.hugegraph.backend.store.cassandra;
import java.io.IOException;
import java.lang.management.MemoryUsage;
import java.util.Map;
@ -43,6 +44,7 @@ public class CassandraMetrics implements BackendMetrics {
this.port = conf.get(CassandraOptions.CASSANDRA_JMX_PORT);
this.username = conf.get(CassandraOptions.CASSANDRA_USERNAME);
this.password = conf.get(CassandraOptions.CASSANDRA_PASSWORD);
assert this.username != null && this.password != null;
}
@Override
@ -58,16 +60,22 @@ public class CassandraMetrics implements BackendMetrics {
private Map<String, Object> getMetricsByHost(String host) {
Map<String, Object> metrics = InsertionOrderUtil.newMap();
// JMX client operations for Cassandra.
try (NodeProbe probe = new NodeProbe(host, port, username, password)) {
try (NodeProbe probe = this.newNodeProbe(host)) {
MemoryUsage heapUsage = probe.getHeapMemoryUsage();
metrics.put(MEM_USED, heapUsage.getUsed() / Bytes.MB);
metrics.put(MEM_COMMITED, heapUsage.getCommitted() / Bytes.MB);
metrics.put(MEM_MAX, heapUsage.getMax() / Bytes.MB);
metrics.put(MEM_UNIT, "MB");
metrics.put(DATA_SIZE, probe.getLoadString());
} catch (Exception e) {
} catch (Throwable e) {
metrics.put(EXCEPTION, e.toString());
}
return metrics;
}
private NodeProbe newNodeProbe(String host) throws IOException {
return this.username.isEmpty() ?
new NodeProbe(host, this.port) :
new NodeProbe(host, this.port, this.username, this.password);
}
}

27
hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/IndexLabelBuilder.java
View File

@ -86,6 +86,9 @@ public class IndexLabelBuilder implements IndexLabel.Builder {
return indexLabel;
}
/**
* Create index label with async mode
*/
@Override
public IndexLabel.CreatedIndexLabel createWithTask() {
SchemaTransaction tx = this.transaction;
@ -108,29 +111,33 @@ public class IndexLabelBuilder implements IndexLabel.Builder {
this.checkFields(schemaLabel.properties());
this.checkRepeatIndex(schemaLabel);
// Delete index label which is prefix of the new index label
// Async delete index label which is prefix of the new index label
// TODO: use event to replace direct call
Set<Id> removeTasks = this.removeSubIndex(schemaLabel);
// Create index label
// Create index label (just schema)
indexLabel = this.build();
indexLabel.status(SchemaStatus.CREATING);
tx.addIndexLabel(schemaLabel, indexLabel);
// Async rebuild index
Id rebuildTask = tx.rebuildIndex(indexLabel, removeTasks);
E.checkNotNull(rebuildTask, "rebuild-index task");
return new IndexLabel.CreatedIndexLabel(indexLabel, rebuildTask);
}
/**
* Create index label with sync mode
*/
@Override
public IndexLabel create() {
// Create index label async
IndexLabel.CreatedIndexLabel createdIndexLabel = this.createWithTask();
Id task = createdIndexLabel.task();
IndexLabel indexLabel = createdIndexLabel.indexLabel();
if (task == null) {
E.checkNotNull(indexLabel, "index label");
return indexLabel;
}
// Wait task completed (change to sync mode)
HugeGraph graph = this.transaction.graph();
Id task = createdIndexLabel.task();
long timeout = graph.configuration().get(CoreOptions.TASK_WAIT_TIMEOUT);
try {
graph.taskScheduler().waitUntilTaskCompleted(task, timeout);
@ -138,7 +145,9 @@ public class IndexLabelBuilder implements IndexLabel.Builder {
throw new HugeException(
"Failed to wait index-creating task completed", e);
}
return indexLabel;
// Return index label without task-info
return createdIndexLabel.indexLabel();
}
@Override

12
hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskScheduler.java
View File

@ -72,6 +72,7 @@ public class TaskScheduler {
private volatile TaskTransaction taskTx;
private static final long NO_LIMIT = -1L;
private static final long QUERY_INTERVAL = 100L;
public TaskScheduler(HugeGraph graph,
ExecutorService taskExecutor,
@ -319,16 +320,17 @@ public class TaskScheduler {
public <V> HugeTask<V> waitUntilTaskCompleted(Id id, long seconds)
throws TimeoutException {
long passes = seconds * 1000 / QUERY_INTERVAL;
for (long pass = 0;; pass++) {
HugeTask<V> task = this.task(id);
if (task.completed()) {
return task;
}
if (pass >= seconds) {
if (pass >= passes) {
break;
}
try {
Thread.sleep(1000L);
Thread.sleep(QUERY_INTERVAL);
} catch (InterruptedException ignored) {
// Ignore InterruptedException
}
@ -339,18 +341,18 @@ public class TaskScheduler {
public void waitUntilAllTasksCompleted(long seconds)
throws TimeoutException {
long t100ms = seconds * 10L;
long passes = seconds * 1000 / QUERY_INTERVAL;
int taskSize = 0;
for (long pass = 0;; pass++) {
taskSize = this.pendingTasks();
if (taskSize == 0) {
return;
}
if (pass >= t100ms) {
if (pass >= passes) {
break;
}
try {
Thread.sleep(100L);
Thread.sleep(QUERY_INTERVAL);
} catch (InterruptedException ignored) {
// Ignore InterruptedException
}

5
hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBMetrics.java
View File

@ -21,6 +21,8 @@ package com.baidu.hugegraph.backend.store.rocksdb;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import com.baidu.hugegraph.backend.store.BackendMetrics;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBSessions.Session;
import com.baidu.hugegraph.util.Bytes;
@ -45,7 +47,8 @@ public class RocksDBMetrics implements BackendMetrics {
// NOTE: the unit of rocksdb mem property is kb
metrics.put(MEM_USED, this.getMemUsed() / Bytes.BASE);
metrics.put(MEM_UNIT, "MB");
metrics.put(DATA_SIZE, this.getDataSize());
String size = FileUtils.byteCountToDisplaySize(this.getDataSize());
metrics.put(DATA_SIZE, size);
return metrics;
}

3
hugegraph-test/src/main/java/com/baidu/hugegraph/api/ApiTestSuite.java
View File

@ -34,7 +34,8 @@ import com.baidu.hugegraph.dist.RegisterUtil;
IndexLabelApiTest.class,
VertexApiTest.class,
EdgeApiTest.class,
GremlinApiTest.class
GremlinApiTest.class,
MetricsApiTest.class
})
public class ApiTestSuite {

27
hugegraph-test/src/main/java/com/baidu/hugegraph/api/BaseApiTest.java
View File

@ -38,6 +38,7 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import com.baidu.hugegraph.HugeException;
import com.baidu.hugegraph.util.JsonUtil;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -307,8 +308,8 @@ public class BaseApiTest {
+ "}");
}
protected String getVertexId(String label, String key, String value)
throws IOException {
protected static String getVertexId(String label, String key, String value)
throws IOException {
String props = mapper.writeValueAsString(ImmutableMap.of(key, value));
Map<String, Object> params = ImmutableMap.of(
"label", label,
@ -328,6 +329,11 @@ public class BaseApiTest {
return (String) list.get(0).get("id");
}
protected static String parseId(String content) throws IOException {
Map<?, ?> map = mapper.readValue(content, Map.class);
return (String) map.get("id");
}
protected static <T> List<T> readList(String content,
String key,
Class<T> clazz) {
@ -359,12 +365,21 @@ public class BaseApiTest {
protected static String assertResponseStatus(int status,
Response response) {
String content = response.readEntity(String.class);
Assert.assertEquals(content, status, response.getStatus());
String message = String.format("Response with status %s and content %s",
response.getStatus(), content);
Assert.assertEquals(message, status, response.getStatus());
return content;
}
protected static String parseId(String content) throws IOException {
Map<?, ?> map = mapper.readValue(content, Map.class);
return (String) map.get("id");
public static Object assertJsonContains(String response, String key) {
Map<?, ?> json = JsonUtil.fromJson(response, Map.class);
return assertMapContains(json, key);
}
public static Object assertMapContains(Map<?, ?> map, String key) {
String message = String.format("Expect contains key '%s' in %s",
key, map);
Assert.assertTrue(message, map.containsKey(key));
return map.get(key);
}
}

2
hugegraph-test/src/main/java/com/baidu/hugegraph/api/EdgeLabelApiTest.java
View File

@ -134,6 +134,6 @@ public class EdgeLabelApiTest extends BaseApiTest {
String name = "created";
r = client().delete(path, name);
assertResponseStatus(204, r);
assertResponseStatus(202, r);
}
}

10
hugegraph-test/src/main/java/com/baidu/hugegraph/api/IndexLabelApiTest.java
View File

@ -45,7 +45,7 @@ public class IndexLabelApiTest extends BaseApiTest {
+ "\"fields\":[\"age\"]"
+ "}";
Response r = client().post(path, indexLabel);
assertResponseStatus(201, r);
assertResponseStatus(202, r);
}
@Test
@ -58,7 +58,7 @@ public class IndexLabelApiTest extends BaseApiTest {
+ "\"fields\":[\"age\"]"
+ "}";
Response r = client().post(path, indexLabel);
assertResponseStatus(201, r);
assertResponseStatus(202, r);
String name = "personByAge";
r = client().get(path, name);
@ -75,7 +75,7 @@ public class IndexLabelApiTest extends BaseApiTest {
+ "\"fields\":[\"age\"]"
+ "}";
Response r = client().post(path, indexLabel);
assertResponseStatus(201, r);
assertResponseStatus(202, r);
r = client().get(path);
assertResponseStatus(200, r);
@ -91,10 +91,10 @@ public class IndexLabelApiTest extends BaseApiTest {
+ "\"fields\":[\"age\"]"
+ "}";
Response r = client().post(path, indexLabel);
assertResponseStatus(201, r);
assertResponseStatus(202, r);
String name = "personByAge";
r = client().delete(path, name);
assertResponseStatus(204, r);
assertResponseStatus(202, r);
}
}

104
hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java Normal file
View File

@ -0,0 +1,104 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.api;
import java.util.Map;
import javax.ws.rs.core.Response;
import org.junit.Test;
import com.baidu.hugegraph.testutil.Assert;
public class MetricsApiTest extends BaseApiTest {
private static String path = "/metrics";
@Test
public void testMetricsAll() {
Response r = client().get(path);
String result = assertResponseStatus(200, r);
assertJsonContains(result, "gauges");
assertJsonContains(result, "counters");
assertJsonContains(result, "histograms");
assertJsonContains(result, "meters");
assertJsonContains(result, "timers");
}
@Test
public void testMetricsSystem() {
Response r = client().get(path, "system");
String result = assertResponseStatus(200, r);
assertJsonContains(result, "basic");
assertJsonContains(result, "heap");
assertJsonContains(result, "nonheap");
assertJsonContains(result, "thread");
assertJsonContains(result, "class_loading");
assertJsonContains(result, "garbage_collector");
}
@Test
public void testMetricsBackend() {
Response r = client().get(path, "backend");
String result = assertResponseStatus(200, r);
Object value = assertJsonContains(result, "hugegraph");
Assert.assertTrue(value instanceof Map);
Map<?, ?> graph = (Map<?, ?>) value;
String backend = (String) graph.get("backend");
String notSupport = "Not support metadata 'metrics'";
switch (backend) {
case "memory":
case "mysql":
case "hbase":
String except = (String) assertMapContains(graph, "exception");
Assert.assertTrue(except, except.contains(notSupport));
break;
case "rocksdb":
assertMapContains(graph, "mem_used");
assertMapContains(graph, "mem_unit");
assertMapContains(graph, "data_size");
break;
case "cassandra":
case "scylladb":
for (Map.Entry<?, ?> e : graph.entrySet()) {
String key = (String) e.getKey();
value = e.getValue();
if (key.equals("backend")) {
continue;
}
Assert.assertTrue(String.format(
"Expect map value for key %s but got %s",
key, value),
value instanceof Map);
Map<?, ?> host = (Map<?, ?>) value;
assertMapContains(host, "mem_used");
assertMapContains(host, "mem_commited");
assertMapContains(host, "mem_max");
assertMapContains(host, "mem_unit");
assertMapContains(host, "data_size");
}
break;
default:
Assert.assertTrue("Unexpected backend " + backend, false);
break;
}
}
}

2
hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexApiTest.java
View File

@ -62,6 +62,7 @@ public class VertexApiTest extends BaseApiTest {
String content = assertResponseStatus(201, r);
String id = parseId(content);
id = String.format("\"%s\"", id);
r = client().get(path, id);
assertResponseStatus(200, r);
}
@ -95,6 +96,7 @@ public class VertexApiTest extends BaseApiTest {
String content = assertResponseStatus(201, r);
String id = parseId(content);
id = String.format("\"%s\"", id);
r = client().delete(path, id);
assertResponseStatus(204, r);
}

2
hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexLabelApiTest.java
View File

@ -121,6 +121,6 @@ public class VertexLabelApiTest extends BaseApiTest {
String name = "person";
r = client().delete(path, name);
assertResponseStatus(204, r);
assertResponseStatus(202, r);
}
}