From 3fcfce2bb786c5ae852a0164be790a1e8221c2ea Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 23 Jul 2019 18:36:49 +0800 Subject: [PATCH] Support batch updating elements' property by multiple strategy (#493) support strategies: SUM, BIGGER, SMALLER on Number or Date UNION, INTERSECTION, APPEND, ELIMINATE on Collection --- .../java/com/baidu/hugegraph/api/API.java | 12 +- .../baidu/hugegraph/api/graph/BatchAPI.java | 85 ++++++++ .../baidu/hugegraph/api/graph/EdgeAPI.java | 157 +++++++++++--- .../hugegraph/api/graph/UpdateStrategy.java | 193 ++++++++++++++++++ .../baidu/hugegraph/api/graph/VertexAPI.java | 153 +++++++++++--- .../api/traversers/NeighborRankAPI.java | 2 +- .../baidu/hugegraph/version/ApiVersion.java | 1 + .../backend/serializer/BinarySerializer.java | 2 +- .../store/rocksdbsst/RocksDBSstSessions.java | 2 +- 9 files changed, 552 insertions(+), 55 deletions(-) create mode 100644 hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/UpdateStrategy.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java index eb46c97bf..c549fbcad 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java @@ -148,10 +148,20 @@ public class API { protected static void checkCreatingBody( Collection bodys) { E.checkArgumentNotNull(bodys, "The request body can't be empty"); + for (Checkable body : bodys) { + E.checkArgument(body != null, + "The batch body can't contain null record"); + body.checkCreate(true); + } + } + + protected static void checkUpdatingBody( + Collection bodys) { + E.checkArgumentNotNull(bodys, "The request body can't be empty"); for (Checkable body : bodys) { E.checkArgumentNotNull(body, "The batch body can't contain null record"); - body.checkCreate(true); + body.checkUpdate(); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/BatchAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/BatchAPI.java index 517581da5..3f3135a57 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/BatchAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/BatchAPI.java @@ -19,20 +19,27 @@ package com.baidu.hugegraph.api.graph; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.tinkerpop.gremlin.structure.Element; import org.slf4j.Logger; import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.api.API; +import com.baidu.hugegraph.api.schema.Checkable; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.ServerOptions; import com.baidu.hugegraph.metrics.MetricsUtil; import com.baidu.hugegraph.server.RestServer; +import com.baidu.hugegraph.structure.HugeElement; +import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.Meter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; public class BatchAPI extends API { @@ -71,4 +78,82 @@ public class BatchAPI extends API { batchWriteThreads.decrementAndGet(); } } + + @JsonIgnoreProperties(value = {"type"}) + protected static abstract class JsonElement implements Checkable { + + @JsonProperty("id") + public Object id; + @JsonProperty("label") + public String label; + @JsonProperty("properties") + public Map properties; + @JsonProperty("type") + public String type; + + @Override + public abstract void checkCreate(boolean isBatch); + + @Override + public abstract void checkUpdate(); + + protected abstract Object[] properties(); + } + + protected void updateExistElement(JsonElement oldElement, + JsonElement newElement, + Map strategies) { + if (oldElement == null) { + return; + } + E.checkArgument(newElement != null, "The json element can't be null"); + + for (Map.Entry kv : strategies.entrySet()) { + String key = kv.getKey(); + UpdateStrategy updateStrategy = kv.getValue(); + if (oldElement.properties.get(key) != null) { + Object value = updateStrategy.checkAndUpdateProperty( + oldElement.properties.get(key), + newElement.properties.get(key)); + newElement.properties.put(key, value); + } + } + } + + protected void updateExistElement(HugeGraph g, + Element oldElement, + JsonElement newElement, + Map strategies) { + if (oldElement == null) { + return; + } + E.checkArgument(newElement != null, "The json element can't be null"); + + for (Map.Entry kv : strategies.entrySet()) { + String key = kv.getKey(); + UpdateStrategy updateStrategy = kv.getValue(); + if (oldElement.property(key).isPresent() && + newElement.properties.containsKey(key)) { + Object value = updateStrategy.checkAndUpdateProperty( + oldElement.property(key).value(), + newElement.properties.get(key)); + value = g.propertyKey(key).convValue(value, false); + newElement.properties.put(key, value); + } + } + } + + protected static void updateProperties(HugeElement element, + JsonElement jsonElement, + boolean append) { + for (Map.Entry e : jsonElement.properties.entrySet()) { + String key = e.getKey(); + Object value = e.getValue(); + if (append) { + element.property(key, value); + } else { + element.property(key).remove(); + } + } + } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/EdgeAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/EdgeAPI.java index ba54eba04..128d362cf 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/EdgeAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/EdgeAPI.java @@ -20,6 +20,7 @@ package com.baidu.hugegraph.api.graph; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -50,8 +51,9 @@ import com.baidu.hugegraph.api.API; import com.baidu.hugegraph.api.filter.CompressInterceptor.Compress; import com.baidu.hugegraph.api.filter.DecompressInterceptor.Decompress; import com.baidu.hugegraph.api.filter.StatusFilter.Status; -import com.baidu.hugegraph.api.schema.Checkable; +import com.baidu.hugegraph.backend.id.EdgeId; import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.id.SplicingIdGenerator; import com.baidu.hugegraph.backend.tx.SchemaTransaction; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.ServerOptions; @@ -64,10 +66,10 @@ import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.structure.HugeEdge; import com.baidu.hugegraph.structure.HugeVertex; import com.baidu.hugegraph.type.HugeType; +import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.annotation.Timed; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @Path("graphs/{graph}/graph/edges") @@ -130,9 +132,9 @@ public class EdgeAPI extends BatchAPI { List jsonEdges) { LOG.debug("Graph [{}] create edges: {}", graph, jsonEdges); checkCreatingBody(jsonEdges); + checkBatchSize(config, jsonEdges); HugeGraph g = graph(manager, graph); - checkBatchSize(config, jsonEdges); TriFunction getVertex = checkVertex ? EdgeAPI::getVertex : EdgeAPI::newVertex; @@ -157,6 +159,64 @@ public class EdgeAPI extends BatchAPI { }); } + /** + * Batch update steps are same like vertices + */ + @PUT + @Timed + @Decompress + @Path("batch") + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON_WITH_CHARSET) + public String update(@Context HugeConfig config, + @Context GraphManager manager, + @PathParam("graph") String graph, + BatchEdgeRequest req) { + BatchEdgeRequest.checkUpdate(req); + LOG.debug("Graph [{}] update edges: {}", graph, req); + checkUpdatingBody(req.jsonEdges); + checkBatchSize(config, req.jsonEdges); + + HugeGraph g = graph(manager, graph); + Map map = new HashMap<>(req.jsonEdges.size()); + TriFunction getVertex = + req.checkVertex ? EdgeAPI::getVertex : EdgeAPI::newVertex; + + return this.commit(config, g, map.size(), () -> { + // 1.Put all newEdges' properties into map (combine first) + req.jsonEdges.forEach(newEdge -> { + Id newEdgeId = getEdgeId(g, newEdge); + JsonEdge oldEdge = map.get(newEdgeId); + this.updateExistElement(oldEdge, newEdge, + req.updateStrategies); + map.put(newEdgeId, newEdge); + }); + + // 2.Get all oldEdges and update with new ones + Object[] ids = map.keySet().toArray(); + Iterator oldEdges = g.edges(ids); + oldEdges.forEachRemaining(oldEdge -> { + JsonEdge newEdge = map.get(oldEdge.id()); + this.updateExistElement(g, oldEdge, newEdge, + req.updateStrategies); + }); + + // 3.Add all finalEdges + List edges = new ArrayList<>(map.size()); + map.values().forEach(finalEdge -> { + Vertex srcVertex = getVertex.apply(g, finalEdge.source, + finalEdge.sourceLabel); + Vertex tgtVertex = getVertex.apply(g, finalEdge.target, + finalEdge.targetLabel); + edges.add(srcVertex.addEdge(finalEdge.label, tgtVertex, + finalEdge.properties())); + }); + + // If return ids, the ids.size() maybe different with the origins' + return manager.serializer(g).writeEdges(edges.iterator(), false); + }); + } + @PUT @Timed @Path("{id}") @@ -191,17 +251,7 @@ public class EdgeAPI extends BatchAPI { id, key); } - commit(g, () -> { - for (Map.Entry e : jsonEdge.properties.entrySet()) { - String key = e.getKey(); - Object value = e.getValue(); - if (append) { - edge.property(key, value); - } else { - edge.property(key).remove(); - } - } - }); + commit(g, () -> updateProperties(edge, jsonEdge, append)); return manager.serializer(g).writeEdge(edge); } @@ -312,9 +362,13 @@ public class EdgeAPI extends BatchAPI { int max = config.get(ServerOptions.MAX_EDGES_PER_BATCH); if (edges.size() > max) { throw new IllegalArgumentException(String.format( - "Too many counts of edges for one time post, " + + "Too many edges for one time post, " + "the maximum number is '%s'", max)); } + if (edges.size() == 0) { + throw new IllegalArgumentException(String.format( + "The number of edges can't be 0")); + } } private static Vertex getVertex(HugeGraph graph, Object id, String label) { @@ -350,25 +404,77 @@ public class EdgeAPI extends BatchAPI { } } - @JsonIgnoreProperties(value = {"type"}) - private static class JsonEdge implements Checkable { + private Id getEdgeId(HugeGraph g, JsonEdge newEdge) { + if (newEdge.id != null) { + return EdgeId.parse(newEdge.id.toString()); + } + + String sortKeys = ""; + Id labelId = g.edgeLabel(newEdge.label).id(); + List sortKeyIds = g.edgeLabel(labelId).sortKeys(); + if (!sortKeyIds.isEmpty()) { + List sortKeyValues = new ArrayList<>(sortKeyIds.size()); + sortKeyIds.forEach(skId -> { + String sortKey = g.propertyKey(skId).name(); + Object sortKeyValue = newEdge.properties.get(sortKey); + E.checkArgument(sortKeyValue != null, + "The value of sort key '%s' can't be null", + sortKey); + sortKeyValues.add(sortKeyValue); + }); + sortKeys = SplicingIdGenerator.concatValues(sortKeyValues); + } + + // TODO: How to get Direction from JsonEdge easily? or any better way? + EdgeId edgeId = new EdgeId(HugeVertex.getIdValue(newEdge.source), + Directions.OUT, labelId, sortKeys, + HugeVertex.getIdValue(newEdge.target)); + return edgeId; + } + + protected static class BatchEdgeRequest { + + @JsonProperty("edges") + public List jsonEdges; + @JsonProperty("update_strategies") + public Map updateStrategies; + @JsonProperty("check_vertex") + public boolean checkVertex = false; + @JsonProperty("create_if_not_exist") + public boolean createIfNotExist = true; + + private static void checkUpdate(BatchEdgeRequest req) { + E.checkArgumentNotNull(req, "BatchEdgeRequest can't be null"); + E.checkArgumentNotNull(req.jsonEdges, + "Parameter 'edges' can't be null"); + E.checkArgument(req.updateStrategies != null && + !req.updateStrategies.isEmpty(), + "Parameter 'update_strategies' can't be empty"); + E.checkArgument(req.createIfNotExist == true, + "Parameter 'create_if_not_exist' " + + "dose not support false now"); + } + + @Override + public String toString() { + return String.format("BatchEdgeRequest{jsonEdges=%s," + + "updateStrategies=%s," + + "checkVertex=%s,createIfNotExist=%s}", + this.jsonEdges, this.updateStrategies, + this.checkVertex, this.createIfNotExist); + } + } + + private static class JsonEdge extends JsonElement { - @JsonProperty("id") - public String id; @JsonProperty("outV") public Object source; @JsonProperty("outVLabel") public String sourceLabel; - @JsonProperty("label") - public String label; @JsonProperty("inV") public Object target; @JsonProperty("inVLabel") public String targetLabel; - @JsonProperty("properties") - public Map properties; - @JsonProperty("type") - public String type; @Override public void checkCreate(boolean isBatch) { @@ -405,6 +511,7 @@ public class EdgeAPI extends BatchAPI { } } + @Override public Object[] properties() { return API.properties(this.properties); } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/UpdateStrategy.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/UpdateStrategy.java new file mode 100644 index 000000000..22da3f468 --- /dev/null +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/UpdateStrategy.java @@ -0,0 +1,193 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.api.graph; + +import java.math.BigDecimal; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.NumericUtil; +import com.google.common.collect.Sets; + +public enum UpdateStrategy { + + // Only number support sum + SUM { + @Override + Object updatePropertyValue(Object oldProperty, Object newProperty) { + // TODO: Improve preformance? (like write a method in common module) + BigDecimal oldNumber = new BigDecimal(oldProperty.toString()); + BigDecimal newNumber = new BigDecimal(newProperty.toString()); + return oldNumber.add(newNumber); + } + + @Override + void checkPropertyType(Object oldProperty, Object newProperty) { + E.checkArgument(oldProperty instanceof Number && + newProperty instanceof Number, + this.formatError(oldProperty, newProperty, + "Number")); + } + }, + + // Only Date & Number support compare + BIGGER { + @Override + Object updatePropertyValue(Object oldProperty, Object newProperty) { + return compareNumber(oldProperty, newProperty, BIGGER); + } + + @Override + void checkPropertyType(Object oldProperty, Object newProperty) { + E.checkArgument((oldProperty instanceof Date || + oldProperty instanceof Number) && + (newProperty instanceof Date || + newProperty instanceof Number), + this.formatError(oldProperty, newProperty, + "Date or Number")); + } + }, + + SMALLER { + @Override + Object updatePropertyValue(Object oldProperty, Object newProperty) { + return compareNumber(oldProperty, newProperty, SMALLER); + } + + @Override + void checkPropertyType(Object oldProperty, Object newProperty) { + E.checkArgument((oldProperty instanceof Date || + oldProperty instanceof Number) && + (newProperty instanceof Date || + newProperty instanceof Number), + this.formatError(oldProperty, newProperty, + "Date or Number")); + } + }, + + // Only Set support union & intersection + UNION { + @Override + Object updatePropertyValue(Object oldProperty, Object newProperty) { + return combineSet(oldProperty, newProperty, UNION); + } + + @Override + void checkPropertyType(Object oldProperty, Object newProperty) { + // JsonElements are always List-type, so allows two type now. + this.checkCollectionType(oldProperty, newProperty); + } + }, + + INTERSECTION { + @Override + Object updatePropertyValue(Object oldProperty, Object newProperty) { + return combineSet(oldProperty, newProperty, INTERSECTION); + } + + @Override + void checkPropertyType(Object oldProperty, Object newProperty) { + this.checkCollectionType(oldProperty, newProperty); + } + }, + + // Batch update Set should use union because of higher efficiency + APPEND { + @Override + @SuppressWarnings({ "rawtypes", "unchecked" }) + Object updatePropertyValue(Object oldProperty, Object newProperty) { + ((Collection) oldProperty).addAll((Collection) newProperty); + return oldProperty; + } + + @Override + void checkPropertyType(Object oldProperty, Object newProperty) { + this.checkCollectionType(oldProperty, newProperty); + } + }, + + ELIMINATE { + @Override + @SuppressWarnings({ "rawtypes", "unchecked" }) + Object updatePropertyValue(Object oldProperty, Object newProperty) { + ((Collection) oldProperty).removeAll((Collection) newProperty); + return oldProperty; + } + + @Override + void checkPropertyType(Object oldProperty, Object newProperty) { + this.checkCollectionType(oldProperty, newProperty); + } + }; + + abstract Object updatePropertyValue(Object oldProperty, Object newProperty); + + abstract void checkPropertyType(Object oldProperty, Object newProperty); + + public Object checkAndUpdateProperty(Object oldProperty, + Object newProperty) { + this.checkPropertyType(oldProperty, newProperty); + return this.updatePropertyValue(oldProperty, newProperty); + } + + protected String formatError(Object oldProperty, Object newProperty, + String className) { + return String.format("Property type must be %s for strategy %s, " + + "but got type %s, %s", className, this, + oldProperty.getClass().getSimpleName(), + newProperty.getClass().getSimpleName()); + } + + protected void checkCollectionType(Object oldProperty, + Object newProperty) { + E.checkArgument((oldProperty instanceof Set || + oldProperty instanceof List) && + (newProperty instanceof Set || + newProperty instanceof List), + this.formatError(oldProperty, newProperty, + "Set or List")); + } + + protected static Object compareNumber(Object oldProperty, + Object newProperty, + UpdateStrategy strategy) { + Number oldNum = NumericUtil.convertToNumber(oldProperty); + Number newNum = NumericUtil.convertToNumber(newProperty); + int result = NumericUtil.compareNumber(oldNum, newNum); + return strategy == BIGGER ? (result > 0 ? oldProperty : newProperty) : + (result < 0 ? oldProperty : newProperty); + } + + protected static Set combineSet(Object oldProperty, Object newProperty, + UpdateStrategy strategy) { + Set oldSet = oldProperty instanceof Set ? + (Set) oldProperty : + new HashSet<>((List) oldProperty); + Set newSet = newProperty instanceof Set ? + (Set) newProperty : + new HashSet<>((List) newProperty); + return strategy == UNION ? Sets.union(oldSet, newSet) : + Sets.intersection(oldSet, newSet); + } +} diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/VertexAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/VertexAPI.java index faac4545a..c1d5c670b 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/VertexAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/VertexAPI.java @@ -21,6 +21,7 @@ package com.baidu.hugegraph.api.graph; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -48,8 +49,8 @@ import com.baidu.hugegraph.api.API; import com.baidu.hugegraph.api.filter.CompressInterceptor.Compress; import com.baidu.hugegraph.api.filter.DecompressInterceptor.Decompress; import com.baidu.hugegraph.api.filter.StatusFilter.Status; -import com.baidu.hugegraph.api.schema.Checkable; import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.id.SplicingIdGenerator; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.ServerOptions; import com.baidu.hugegraph.core.GraphManager; @@ -58,11 +59,11 @@ import com.baidu.hugegraph.schema.VertexLabel; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.structure.HugeVertex; import com.baidu.hugegraph.type.HugeType; +import com.baidu.hugegraph.type.define.IdStrategy; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.JsonUtil; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.annotation.Timed; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @Path("graphs/{graph}/graph/vertices") @@ -101,9 +102,9 @@ public class VertexAPI extends BatchAPI { List jsonVertices) { LOG.debug("Graph [{}] create vertices: {}", graph, jsonVertices); checkCreatingBody(jsonVertices); + checkBatchSize(config, jsonVertices); HugeGraph g = graph(manager, graph); - checkBatchSize(config, jsonVertices); return this.commit(config, g, jsonVertices.size(), () -> { List ids = new ArrayList<>(jsonVertices.size()); @@ -114,6 +115,64 @@ public class VertexAPI extends BatchAPI { }); } + /** + * Batch update steps like: + * 1. Get all newVertices' ID & combine first + * 2. Get all oldVertices & update + * 3. Add the final vertex together + */ + @PUT + @Timed + @Decompress + @Path("batch") + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON_WITH_CHARSET) + public String update(@Context HugeConfig config, + @Context GraphManager manager, + @PathParam("graph") String graph, + BatchVertexRequest req) { + BatchVertexRequest.checkUpdate(req); + LOG.debug("Graph [{}] update vertices: {}", graph, req); + checkUpdatingBody(req.jsonVertices); + checkBatchSize(config, req.jsonVertices); + + HugeGraph g = graph(manager, graph); + Map map = new HashMap<>(req.jsonVertices.size()); + + return this.commit(config, g, map.size(), () -> { + /* + * 1.Put all newVertices' properties into map (combine first) + * - Consider primary-key & user-define ID mode first + */ + req.jsonVertices.forEach(newVertex -> { + Id newVertexId = getVertexId(g, newVertex); + JsonVertex oldVertex = map.get(newVertexId); + this.updateExistElement(oldVertex, newVertex, + req.updateStrategies); + map.put(newVertexId, newVertex); + }); + + // 2.Get all oldVertices and update with new vertices + Object[] ids = map.keySet().toArray(); + Iterator oldVertices = g.vertices(ids); + oldVertices.forEachRemaining(oldVertex -> { + JsonVertex newVertex = map.get(oldVertex.id()); + this.updateExistElement(g, oldVertex, newVertex, + req.updateStrategies); + }); + + // 3.Add finalVertices and return them + List vertices = new ArrayList<>(map.size()); + map.values().forEach(finalVertex -> { + vertices.add(g.addVertex(finalVertex.properties())); + }); + + // If return ids, the ids.size() maybe different with the origins' + return manager.serializer(g) + .writeVertices(vertices.iterator(), false); + }); + } + @PUT @Timed @Path("{id}") @@ -143,18 +202,7 @@ public class VertexAPI extends BatchAPI { id, key); } - commit(g, () -> { - for (Map.Entry e : - jsonVertex.properties.entrySet()) { - String key = e.getKey(); - Object value = e.getValue(); - if (append) { - vertex.property(key, value); - } else { - vertex.property(key).remove(); - } - } - }); + commit(g, () -> updateProperties(vertex, jsonVertex, append)); return manager.serializer(g).writeVertex(vertex); } @@ -259,22 +307,74 @@ public class VertexAPI extends BatchAPI { int max = config.get(ServerOptions.MAX_VERTICES_PER_BATCH); if (vertices.size() > max) { throw new IllegalArgumentException(String.format( - "Too many counts of vertices for one time post, " + + "Too many vertices for one time post, " + "the maximum number is '%s'", max)); } + if (vertices.size() == 0) { + throw new IllegalArgumentException(String.format( + "The number of vertices can't be 0")); + } } - @JsonIgnoreProperties(value = {"type"}) - private static class JsonVertex implements Checkable { + private Id getVertexId(HugeGraph g, JsonVertex vertex) { + VertexLabel vertexLabel = g.vertexLabel(vertex.label); + String labelId = vertexLabel.id().asString(); + IdStrategy idStrategy = vertexLabel.idStrategy(); + E.checkArgument(idStrategy != IdStrategy.AUTOMATIC, + "Automatic Id strategy is not supported now"); - @JsonProperty("id") - public Object id; - @JsonProperty("label") - public String label; - @JsonProperty("properties") - public Map properties; - @JsonProperty("type") - public String type; + if (idStrategy == IdStrategy.PRIMARY_KEY) { + List pkIds = vertexLabel.primaryKeys(); + List pkValues = new ArrayList<>(pkIds.size()); + for (Id pkId : pkIds) { + String propertyKey = g.propertyKey(pkId).name(); + Object propertyValue = vertex.properties.get(propertyKey); + E.checkArgument(propertyValue != null, + "The value of primary key '%s' can't be null", + propertyKey); + pkValues.add(propertyValue); + } + + String value = SplicingIdGenerator.concatValues(pkValues); + return SplicingIdGenerator.splicing(labelId, value); + } else { + assert idStrategy == IdStrategy.CUSTOMIZE_NUMBER || + idStrategy == IdStrategy.CUSTOMIZE_STRING; + return HugeVertex.getIdValue(vertex.id); + } + } + + private static class BatchVertexRequest { + + @JsonProperty("vertices") + public List jsonVertices; + @JsonProperty("update_strategies") + public Map updateStrategies; + @JsonProperty("create_if_not_exist") + public boolean createIfNotExist = true; + + private static void checkUpdate(BatchVertexRequest req) { + E.checkArgumentNotNull(req, "BatchVertexRequest can't be null"); + E.checkArgumentNotNull(req.jsonVertices, + "Parameter 'vertices' can't be null"); + E.checkArgument(req.updateStrategies != null && + !req.updateStrategies.isEmpty(), + "Parameter 'update_strategies' can't be empty"); + E.checkArgument(req.createIfNotExist == true, + "Parameter 'create_if_not_exist' " + + "dose not support false now"); + } + + @Override + public String toString() { + return String.format("BatchVertexRequest{jsonVertices=%s," + + "updateStrategies=%s,createIfNotExist=%s}", + this.jsonVertices, this.updateStrategies, + this.createIfNotExist); + } + } + + private static class JsonVertex extends JsonElement { @Override public void checkCreate(boolean isBatch) { @@ -297,6 +397,7 @@ public class VertexAPI extends BatchAPI { } } + @Override public Object[] properties() { Object[] props = API.properties(this.properties); List list = new ArrayList<>(Arrays.asList(props)); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java index 4b7b38e51..349a869a6 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/NeighborRankAPI.java @@ -144,7 +144,7 @@ public class NeighborRankAPI extends API { "The degree must be > 0, but got: %s", this.degree); E.checkArgument(this.top > 0 && this.top <= MAX_TOP, - "The top of each layer cannot exceed %s", MAX_TOP); + "The top of each layer can't exceed %s", MAX_TOP); Map labelIds = new HashMap<>(); if (this.labels != null) { for (String label : this.labels) { diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java index 427048464..fed98937d 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java @@ -88,6 +88,7 @@ public final class ApiVersion { * version 0.10: * [0.39] Issue-522: Add profile RESTful API * [0.40] Issue-523: Add source_in_ring args for rings RESTful API + * [0.41] Issue-493: Support batch updating properties by multiple strategy */ // The second parameter of Version.of() is for IDE running without JAR diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java index 89ee06b72..bf8082d0a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java @@ -1150,7 +1150,7 @@ public class BinarySerializer extends AbstractSerializer { private byte[] writeIds(Collection ids) { E.checkState(ids.size() <= BytesBuffer.UINT16_MAX, "The number of properties of vertex/edge label " + - "cannot exceed '%s'", BytesBuffer.UINT16_MAX); + "can't exceed '%s'", BytesBuffer.UINT16_MAX); int size = 2; for (Id id : ids) { size += (1 + id.length()); diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java index 5dbb3f2ec..7ad46cedd 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java @@ -145,7 +145,7 @@ public class RocksDBSstSessions extends RocksDBSessions { @Override protected synchronized void doClose() { - final String NO_ENTRIES = "Cannot create sst file with no entries"; + final String NO_ENTRIES = "Can't create sst file with no entries"; for (SstFileWriter sst : this.tables.values()) { E.checkState(sst.isOwningHandle(), "SstFileWriter closed");