diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/metrics/MetricsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/metrics/MetricsAPI.java index 7b8e052a9..91e290ac6 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/metrics/MetricsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/metrics/MetricsAPI.java @@ -38,7 +38,6 @@ import com.baidu.hugegraph.api.API; import com.baidu.hugegraph.backend.store.BackendMetrics; import com.baidu.hugegraph.backend.tx.GraphTransaction; import com.baidu.hugegraph.core.GraphManager; -import com.baidu.hugegraph.exception.NotSupportException; import com.baidu.hugegraph.metric.ServerReporter; import com.baidu.hugegraph.metric.SystemMetrics; import com.baidu.hugegraph.util.InsertionOrderUtil; @@ -86,7 +85,7 @@ public class MetricsAPI extends API { metrics.put(BackendMetrics.BACKEND, tx.store().provider().type()); try { metrics.putAll(tx.metadata(null, "metrics")); - } catch (NotSupportException e) { + } catch (Throwable e) { metrics.put(BackendMetrics.EXCEPTION, e.toString()); LOG.debug("Failed to get backend metrics", e); } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/metric/SystemMetrics.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/metric/SystemMetrics.java index 161a04137..7de6c50f7 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/metric/SystemMetrics.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/metric/SystemMetrics.java @@ -106,7 +106,7 @@ public class SystemMetrics { ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean(); metrics.put("peak", threadMxBean.getPeakThreadCount()); metrics.put("daemon", threadMxBean.getDaemonThreadCount()); - metrics.put("totalStarted", threadMxBean.getTotalStartedThreadCount()); + metrics.put("total_started", threadMxBean.getTotalStartedThreadCount()); metrics.put("count", threadMxBean.getThreadCount()); return metrics; } @@ -126,14 +126,15 @@ public class SystemMetrics { List gcMxBeans = ManagementFactory .getGarbageCollectorMXBeans(); for (GarbageCollectorMXBean gcMxBean : gcMxBeans) { - String name = this.formatName(gcMxBean.getName()); + String name = formatName(gcMxBean.getName()); metrics.put(name + "_count", gcMxBean.getCollectionCount()); metrics.put(name + "_time", gcMxBean.getCollectionTime()); } + metrics.put("time_unit", "ms"); return metrics; } - private String formatName(String name) { + private static String formatName(String name) { return StringUtils.replace(name, " ", "_").toLowerCase(); } } diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraMetrics.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraMetrics.java index 1781d0213..2f6370bb1 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraMetrics.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraMetrics.java @@ -19,6 +19,7 @@ package com.baidu.hugegraph.backend.store.cassandra; +import java.io.IOException; import java.lang.management.MemoryUsage; import java.util.Map; @@ -43,6 +44,7 @@ public class CassandraMetrics implements BackendMetrics { this.port = conf.get(CassandraOptions.CASSANDRA_JMX_PORT); this.username = conf.get(CassandraOptions.CASSANDRA_USERNAME); this.password = conf.get(CassandraOptions.CASSANDRA_PASSWORD); + assert this.username != null && this.password != null; } @Override @@ -58,16 +60,22 @@ public class CassandraMetrics implements BackendMetrics { private Map getMetricsByHost(String host) { Map metrics = InsertionOrderUtil.newMap(); // JMX client operations for Cassandra. - try (NodeProbe probe = new NodeProbe(host, port, username, password)) { + try (NodeProbe probe = this.newNodeProbe(host)) { MemoryUsage heapUsage = probe.getHeapMemoryUsage(); metrics.put(MEM_USED, heapUsage.getUsed() / Bytes.MB); metrics.put(MEM_COMMITED, heapUsage.getCommitted() / Bytes.MB); metrics.put(MEM_MAX, heapUsage.getMax() / Bytes.MB); metrics.put(MEM_UNIT, "MB"); metrics.put(DATA_SIZE, probe.getLoadString()); - } catch (Exception e) { + } catch (Throwable e) { metrics.put(EXCEPTION, e.toString()); } return metrics; } + + private NodeProbe newNodeProbe(String host) throws IOException { + return this.username.isEmpty() ? + new NodeProbe(host, this.port) : + new NodeProbe(host, this.port, this.username, this.password); + } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/IndexLabelBuilder.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/IndexLabelBuilder.java index a874dbedf..d25c09d87 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/IndexLabelBuilder.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/IndexLabelBuilder.java @@ -86,6 +86,9 @@ public class IndexLabelBuilder implements IndexLabel.Builder { return indexLabel; } + /** + * Create index label with async mode + */ @Override public IndexLabel.CreatedIndexLabel createWithTask() { SchemaTransaction tx = this.transaction; @@ -108,29 +111,33 @@ public class IndexLabelBuilder implements IndexLabel.Builder { this.checkFields(schemaLabel.properties()); this.checkRepeatIndex(schemaLabel); - // Delete index label which is prefix of the new index label + // Async delete index label which is prefix of the new index label // TODO: use event to replace direct call Set removeTasks = this.removeSubIndex(schemaLabel); - // Create index label + // Create index label (just schema) indexLabel = this.build(); indexLabel.status(SchemaStatus.CREATING); tx.addIndexLabel(schemaLabel, indexLabel); + // Async rebuild index Id rebuildTask = tx.rebuildIndex(indexLabel, removeTasks); + E.checkNotNull(rebuildTask, "rebuild-index task"); + return new IndexLabel.CreatedIndexLabel(indexLabel, rebuildTask); } + /** + * Create index label with sync mode + */ @Override public IndexLabel create() { + // Create index label async IndexLabel.CreatedIndexLabel createdIndexLabel = this.createWithTask(); - Id task = createdIndexLabel.task(); - IndexLabel indexLabel = createdIndexLabel.indexLabel(); - if (task == null) { - E.checkNotNull(indexLabel, "index label"); - return indexLabel; - } + + // Wait task completed (change to sync mode) HugeGraph graph = this.transaction.graph(); + Id task = createdIndexLabel.task(); long timeout = graph.configuration().get(CoreOptions.TASK_WAIT_TIMEOUT); try { graph.taskScheduler().waitUntilTaskCompleted(task, timeout); @@ -138,7 +145,9 @@ public class IndexLabelBuilder implements IndexLabel.Builder { throw new HugeException( "Failed to wait index-creating task completed", e); } - return indexLabel; + + // Return index label without task-info + return createdIndexLabel.indexLabel(); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskScheduler.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskScheduler.java index 60c1edd8c..a6fcfc75f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskScheduler.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskScheduler.java @@ -72,6 +72,7 @@ public class TaskScheduler { private volatile TaskTransaction taskTx; private static final long NO_LIMIT = -1L; + private static final long QUERY_INTERVAL = 100L; public TaskScheduler(HugeGraph graph, ExecutorService taskExecutor, @@ -319,16 +320,17 @@ public class TaskScheduler { public HugeTask waitUntilTaskCompleted(Id id, long seconds) throws TimeoutException { + long passes = seconds * 1000 / QUERY_INTERVAL; for (long pass = 0;; pass++) { HugeTask task = this.task(id); if (task.completed()) { return task; } - if (pass >= seconds) { + if (pass >= passes) { break; } try { - Thread.sleep(1000L); + Thread.sleep(QUERY_INTERVAL); } catch (InterruptedException ignored) { // Ignore InterruptedException } @@ -339,18 +341,18 @@ public class TaskScheduler { public void waitUntilAllTasksCompleted(long seconds) throws TimeoutException { - long t100ms = seconds * 10L; + long passes = seconds * 1000 / QUERY_INTERVAL; int taskSize = 0; for (long pass = 0;; pass++) { taskSize = this.pendingTasks(); if (taskSize == 0) { return; } - if (pass >= t100ms) { + if (pass >= passes) { break; } try { - Thread.sleep(100L); + Thread.sleep(QUERY_INTERVAL); } catch (InterruptedException ignored) { // Ignore InterruptedException } diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBMetrics.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBMetrics.java index ef5862614..c835f7e98 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBMetrics.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBMetrics.java @@ -21,6 +21,8 @@ package com.baidu.hugegraph.backend.store.rocksdb; import java.util.Map; +import org.apache.commons.io.FileUtils; + import com.baidu.hugegraph.backend.store.BackendMetrics; import com.baidu.hugegraph.backend.store.rocksdb.RocksDBSessions.Session; import com.baidu.hugegraph.util.Bytes; @@ -45,7 +47,8 @@ public class RocksDBMetrics implements BackendMetrics { // NOTE: the unit of rocksdb mem property is kb metrics.put(MEM_USED, this.getMemUsed() / Bytes.BASE); metrics.put(MEM_UNIT, "MB"); - metrics.put(DATA_SIZE, this.getDataSize()); + String size = FileUtils.byteCountToDisplaySize(this.getDataSize()); + metrics.put(DATA_SIZE, size); return metrics; } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/ApiTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/ApiTestSuite.java index 5d15a2d9b..6a2cd2e12 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/ApiTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/ApiTestSuite.java @@ -34,7 +34,8 @@ import com.baidu.hugegraph.dist.RegisterUtil; IndexLabelApiTest.class, VertexApiTest.class, EdgeApiTest.class, - GremlinApiTest.class + GremlinApiTest.class, + MetricsApiTest.class }) public class ApiTestSuite { diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/BaseApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/BaseApiTest.java index 380690c29..c14b1e4ce 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/BaseApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/BaseApiTest.java @@ -38,6 +38,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import com.baidu.hugegraph.HugeException; +import com.baidu.hugegraph.util.JsonUtil; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -307,8 +308,8 @@ public class BaseApiTest { + "}"); } - protected String getVertexId(String label, String key, String value) - throws IOException { + protected static String getVertexId(String label, String key, String value) + throws IOException { String props = mapper.writeValueAsString(ImmutableMap.of(key, value)); Map params = ImmutableMap.of( "label", label, @@ -328,6 +329,11 @@ public class BaseApiTest { return (String) list.get(0).get("id"); } + protected static String parseId(String content) throws IOException { + Map map = mapper.readValue(content, Map.class); + return (String) map.get("id"); + } + protected static List readList(String content, String key, Class clazz) { @@ -359,12 +365,21 @@ public class BaseApiTest { protected static String assertResponseStatus(int status, Response response) { String content = response.readEntity(String.class); - Assert.assertEquals(content, status, response.getStatus()); + String message = String.format("Response with status %s and content %s", + response.getStatus(), content); + Assert.assertEquals(message, status, response.getStatus()); return content; } - protected static String parseId(String content) throws IOException { - Map map = mapper.readValue(content, Map.class); - return (String) map.get("id"); + public static Object assertJsonContains(String response, String key) { + Map json = JsonUtil.fromJson(response, Map.class); + return assertMapContains(json, key); + } + + public static Object assertMapContains(Map map, String key) { + String message = String.format("Expect contains key '%s' in %s", + key, map); + Assert.assertTrue(message, map.containsKey(key)); + return map.get(key); } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/EdgeLabelApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/EdgeLabelApiTest.java index 95df3b096..a1513aba6 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/EdgeLabelApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/EdgeLabelApiTest.java @@ -134,6 +134,6 @@ public class EdgeLabelApiTest extends BaseApiTest { String name = "created"; r = client().delete(path, name); - assertResponseStatus(204, r); + assertResponseStatus(202, r); } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/IndexLabelApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/IndexLabelApiTest.java index 5c8d04fb7..0fba250dc 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/IndexLabelApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/IndexLabelApiTest.java @@ -45,7 +45,7 @@ public class IndexLabelApiTest extends BaseApiTest { + "\"fields\":[\"age\"]" + "}"; Response r = client().post(path, indexLabel); - assertResponseStatus(201, r); + assertResponseStatus(202, r); } @Test @@ -58,7 +58,7 @@ public class IndexLabelApiTest extends BaseApiTest { + "\"fields\":[\"age\"]" + "}"; Response r = client().post(path, indexLabel); - assertResponseStatus(201, r); + assertResponseStatus(202, r); String name = "personByAge"; r = client().get(path, name); @@ -75,7 +75,7 @@ public class IndexLabelApiTest extends BaseApiTest { + "\"fields\":[\"age\"]" + "}"; Response r = client().post(path, indexLabel); - assertResponseStatus(201, r); + assertResponseStatus(202, r); r = client().get(path); assertResponseStatus(200, r); @@ -91,10 +91,10 @@ public class IndexLabelApiTest extends BaseApiTest { + "\"fields\":[\"age\"]" + "}"; Response r = client().post(path, indexLabel); - assertResponseStatus(201, r); + assertResponseStatus(202, r); String name = "personByAge"; r = client().delete(path, name); - assertResponseStatus(204, r); + assertResponseStatus(202, r); } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java new file mode 100644 index 000000000..bff59662d --- /dev/null +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java @@ -0,0 +1,104 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.api; + +import java.util.Map; + +import javax.ws.rs.core.Response; + +import org.junit.Test; + +import com.baidu.hugegraph.testutil.Assert; + +public class MetricsApiTest extends BaseApiTest { + + private static String path = "/metrics"; + + @Test + public void testMetricsAll() { + Response r = client().get(path); + String result = assertResponseStatus(200, r); + assertJsonContains(result, "gauges"); + assertJsonContains(result, "counters"); + assertJsonContains(result, "histograms"); + assertJsonContains(result, "meters"); + assertJsonContains(result, "timers"); + } + + @Test + public void testMetricsSystem() { + Response r = client().get(path, "system"); + String result = assertResponseStatus(200, r); + assertJsonContains(result, "basic"); + assertJsonContains(result, "heap"); + assertJsonContains(result, "nonheap"); + assertJsonContains(result, "thread"); + assertJsonContains(result, "class_loading"); + assertJsonContains(result, "garbage_collector"); + } + + @Test + public void testMetricsBackend() { + Response r = client().get(path, "backend"); + String result = assertResponseStatus(200, r); + Object value = assertJsonContains(result, "hugegraph"); + + Assert.assertTrue(value instanceof Map); + Map graph = (Map) value; + String backend = (String) graph.get("backend"); + String notSupport = "Not support metadata 'metrics'"; + switch (backend) { + case "memory": + case "mysql": + case "hbase": + String except = (String) assertMapContains(graph, "exception"); + Assert.assertTrue(except, except.contains(notSupport)); + break; + case "rocksdb": + assertMapContains(graph, "mem_used"); + assertMapContains(graph, "mem_unit"); + assertMapContains(graph, "data_size"); + break; + case "cassandra": + case "scylladb": + for (Map.Entry e : graph.entrySet()) { + String key = (String) e.getKey(); + value = e.getValue(); + if (key.equals("backend")) { + continue; + } + Assert.assertTrue(String.format( + "Expect map value for key %s but got %s", + key, value), + value instanceof Map); + Map host = (Map) value; + assertMapContains(host, "mem_used"); + assertMapContains(host, "mem_commited"); + assertMapContains(host, "mem_max"); + assertMapContains(host, "mem_unit"); + assertMapContains(host, "data_size"); + } + break; + default: + Assert.assertTrue("Unexpected backend " + backend, false); + break; + } + } +} diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexApiTest.java index c694107fa..25b878b55 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexApiTest.java @@ -62,6 +62,7 @@ public class VertexApiTest extends BaseApiTest { String content = assertResponseStatus(201, r); String id = parseId(content); + id = String.format("\"%s\"", id); r = client().get(path, id); assertResponseStatus(200, r); } @@ -95,6 +96,7 @@ public class VertexApiTest extends BaseApiTest { String content = assertResponseStatus(201, r); String id = parseId(content); + id = String.format("\"%s\"", id); r = client().delete(path, id); assertResponseStatus(204, r); } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexLabelApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexLabelApiTest.java index 09d554505..90820de42 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexLabelApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/VertexLabelApiTest.java @@ -121,6 +121,6 @@ public class VertexLabelApiTest extends BaseApiTest { String name = "person"; r = client().delete(path, name); - assertResponseStatus(204, r); + assertResponseStatus(202, r); } }