From 1208522699dd45075b457bd8b5d8a805cfbd0ef7 Mon Sep 17 00:00:00 2001 From: Jermy Li Date: Thu, 12 Sep 2019 21:12:25 +0800 Subject: [PATCH] Reduce the storage of vertex/edge id (#661) Change-Id: I374deb904ae8ca20b5768ebc7976f831ec558e3e --- .../store/cassandra/CassandraTables.java | 4 +- .../baidu/hugegraph/backend/id/EdgeId.java | 40 ++++++-- .../com/baidu/hugegraph/backend/id/Id.java | 28 ++++++ .../hugegraph/backend/id/IdGenerator.java | 8 +- .../baidu/hugegraph/backend/id/IdUtil.java | 67 ++++++++++--- .../backend/serializer/BinarySerializer.java | 18 ++-- .../backend/serializer/TableSerializer.java | 39 ++++---- .../exception/NotFoundException.java | 4 + .../baidu/hugegraph/schema/VertexLabel.java | 2 +- .../schema/builder/VertexLabelBuilder.java | 2 +- .../hugegraph/structure/HugeElement.java | 8 +- .../backend/store/mysql/MysqlTables.java | 4 +- .../backend/store/palo/PaloTables.java | 4 +- .../postgresql/PostgresqlSerializer.java | 2 +- .../store/scylladb/ScyllaDBTables.java | 6 +- .../baidu/hugegraph/core/EdgeCoreTest.java | 96 +++++++++++++++++++ .../baidu/hugegraph/core/VertexCoreTest.java | 10 +- .../hugegraph/core/VertexLabelCoreTest.java | 2 +- .../com/baidu/hugegraph/unit/core/IdTest.java | 19 +++- .../baidu/hugegraph/unit/util/IdUtilTest.java | 65 +++++++++++++ 20 files changed, 353 insertions(+), 75 deletions(-) diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTables.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTables.java index 860798b58..fd1e4210f 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTables.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTables.java @@ -362,11 +362,11 @@ public class CassandraTables { protected final List idColumnValue(EdgeId edgeId) { List list = new ArrayList<>(5); - list.add(IdUtil.writeString(edgeId.ownerVertexId())); + list.add(IdUtil.writeStoredString(edgeId.ownerVertexId())); list.add(edgeId.direction().code()); list.add(edgeId.edgeLabelId().asLong()); list.add(edgeId.sortValues()); - list.add(IdUtil.writeString(edgeId.otherVertexId())); + list.add(IdUtil.writeStoredString(edgeId.otherVertexId())); return list; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java index ecfd41656..1e8cfe243 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/EdgeId.java @@ -24,6 +24,7 @@ import com.baidu.hugegraph.structure.HugeVertex; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.type.define.Directions; import com.baidu.hugegraph.type.define.HugeKeys; +import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.StringEncoding; /** @@ -132,13 +133,13 @@ public class EdgeId implements Id { this.cache = SplicingIdGenerator.concat( IdUtil.writeString(this.ownerVertexId), this.direction.type().string(), - this.edgeLabelId.asString(), + IdUtil.writeLong(this.edgeLabelId), this.sortValues, IdUtil.writeString(this.otherVertexId)); } else { this.cache = SplicingIdGenerator.concat( IdUtil.writeString(this.sourceVertexId()), - this.edgeLabelId.asString(), + IdUtil.writeLong(this.edgeLabelId), this.sortValues, IdUtil.writeString(this.targetVertexId())); } @@ -162,7 +163,7 @@ public class EdgeId implements Id { @Override public IdType type() { - return IdType.STRING; + return IdType.EDGE; } @Override @@ -190,15 +191,15 @@ public class EdgeId implements Id { } public static EdgeId parse(String id) throws NotFoundException { - String[] idParts = SplicingIdGenerator.split(id); + String[] idParts = split(id); if (!(idParts.length == 4 || idParts.length == 5)) { - throw new NotFoundException("Edge id must be formatted as 4~5 parts" - + ", but got '%s'", id); + throw new NotFoundException("Edge id must be formatted as 4~5 " + + "parts, but got '%s'", id); } try { if (idParts.length == 4) { Id ownerVertexId = IdUtil.readString(idParts[0]); - Id edgeLabelId = IdGenerator.of(Long.parseLong(idParts[1])); + Id edgeLabelId = IdUtil.readLong(idParts[1]); String sortValues = idParts[2]; Id otherVertexId = IdUtil.readString(idParts[3]); return new EdgeId(ownerVertexId, Directions.OUT, edgeLabelId, @@ -207,17 +208,38 @@ public class EdgeId implements Id { assert idParts.length == 5; Id ownerVertexId = IdUtil.readString(idParts[0]); HugeType direction = HugeType.fromString(idParts[1]); - Id edgeLabelId = IdGenerator.of(Long.parseLong(idParts[2])); + Id edgeLabelId = IdUtil.readLong(idParts[2]); String sortValues = idParts[3]; Id otherVertexId = IdUtil.readString(idParts[4]); return new EdgeId(ownerVertexId, Directions.convert(direction), edgeLabelId, sortValues, otherVertexId); } } catch (Exception e) { - throw new NotFoundException("Invalid format of edge id '%s'", id); + throw new NotFoundException("Invalid format of edge id '%s'", + e, id); } } + public static Id parseStoredString(String id) { + String[] idParts = split(id); + E.checkArgument(idParts.length == 4, "Invalid id format: %s", id); + Id ownerVertexId = IdUtil.readStoredString(idParts[0]); + Id edgeLabelId = IdGenerator.ofStoredString(idParts[1], IdType.LONG); + String sortValues = idParts[2]; + Id otherVertexId = IdUtil.readStoredString(idParts[3]); + return new EdgeId(ownerVertexId, Directions.OUT, edgeLabelId, + sortValues, otherVertexId); + } + + public static String asStoredString(Id id) { + EdgeId eid = (EdgeId) id; + return SplicingIdGenerator.concat( + IdUtil.writeStoredString(eid.sourceVertexId()), + IdGenerator.asStoredString(eid.edgeLabelId()), + eid.sortValues(), + IdUtil.writeStoredString(eid.targetVertexId())); + } + public static String concat(String... ids) { return SplicingIdGenerator.concat(ids); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java index e00f3c322..f7627bae5 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java @@ -19,6 +19,8 @@ package com.baidu.hugegraph.backend.id; +import com.baidu.hugegraph.util.E; + public interface Id extends Comparable { public static final int UUID_LENGTH = 16; @@ -48,9 +50,35 @@ public interface Id extends Comparable { } public enum IdType { + UNKNOWN, LONG, UUID, STRING, + EDGE; + + public char prefix() { + if (this == UNKNOWN) { + return 'N'; + } + return this.name().charAt(0); + } + + public static IdType valueOfPrefix(String id) { + E.checkArgument(id != null && id.length() > 0, + "Invalid id '%s'", id); + switch (id.charAt(0)) { + case 'L': + return IdType.LONG; + case 'U': + return IdType.UUID; + case 'S': + return IdType.STRING; + case 'E': + return IdType.EDGE; + default: + return IdType.UNKNOWN; + } + } } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java index 34fc8d88e..850086372 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java @@ -39,6 +39,10 @@ public abstract class IdGenerator { return new StringId(id); } + public final static Id of(UUID id) { + return new UuidId(id); + } + public final static Id of(String id, boolean uuid) { return uuid ? new UuidId(id) : new StringId(id); } @@ -63,7 +67,7 @@ public abstract class IdGenerator { public final static Id ofStoredString(String id, IdType type) { switch (type) { case LONG: - return of(LongEncoding.decode(id)); + return of(LongEncoding.decodeSortable(id)); case UUID: byte[] bytes = Base64.getDecoder().decode(id); return of(bytes, IdType.UUID); @@ -77,7 +81,7 @@ public abstract class IdGenerator { public final static String asStoredString(Id id) { switch (id.type()) { case LONG: - return LongEncoding.encode(id.asLong()); + return LongEncoding.encodeSortable(id.asLong()); case UUID: return Base64.getEncoder().encodeToString(id.asBytes()); case STRING: diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdUtil.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdUtil.java index 5891f07ab..788d759d4 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdUtil.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdUtil.java @@ -21,33 +21,70 @@ package com.baidu.hugegraph.backend.id; import org.apache.commons.lang3.StringUtils; -import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.backend.id.Id.IdType; public final class IdUtil { - private static final String NUMBER_PREFIX = "L"; - private static final String STRING_PREFIX = "S"; + public static String writeStoredString(Id id) { + String idString; + switch (id.type()) { + case LONG: + case STRING: + case UUID: + idString = IdGenerator.asStoredString(id); + break; + case EDGE: + idString = EdgeId.asStoredString(id); + break; + default: + throw new AssertionError("Invalid id type " + id.type()); + } + return id.type().prefix() + idString; + } + + public static Id readStoredString(String id) { + IdType type = IdType.valueOfPrefix(id); + id = id.substring(1); + switch (type) { + case LONG: + case STRING: + case UUID: + return IdGenerator.ofStoredString(id, type); + case EDGE: + return EdgeId.parseStoredString(id); + default: + throw new AssertionError("Invalid id type " + type); + } + } public static String writeString(Id id) { - return (id.number() ? NUMBER_PREFIX : STRING_PREFIX) + id.asObject(); + return "" + id.type().prefix() + id.asObject(); } public static Id readString(String id) { - E.checkNotNull(id, "id"); - String signal = id.substring(0, 1); - E.checkState(signal.equals(NUMBER_PREFIX) || - signal.equals(STRING_PREFIX), - "The serialized id value must start with '%s' or '%s', " + - "but got '%s'", NUMBER_PREFIX, STRING_PREFIX, id); + IdType type = IdType.valueOfPrefix(id); id = id.substring(1); - boolean number = signal.equals(NUMBER_PREFIX); - if (number) { - return IdGenerator.of(Long.parseLong(id)); - } else { - return IdGenerator.of(id); + switch (type) { + case LONG: + return IdGenerator.of(Long.parseLong(id)); + case STRING: + case UUID: + return IdGenerator.of(id, type == IdType.UUID); + case EDGE: + return EdgeId.parse(id); + default: + throw new AssertionError("Invalid id type " + type); } } + public static String writeLong(Id id) { + return String.valueOf(id.asLong()); + } + + public static Id readLong(String id) { + return IdGenerator.of(Long.parseLong(id)); + } + public static String escape(char splitor, char escape, String... values) { StringBuilder escaped = new StringBuilder((values.length + 1) << 4); // Do escape for every item in values diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java index 8edf697e2..e10ef05cb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java @@ -30,6 +30,7 @@ import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.id.EdgeId; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.id.IdGenerator; +import com.baidu.hugegraph.backend.id.IdUtil; import com.baidu.hugegraph.backend.page.PageState; import com.baidu.hugegraph.backend.query.Condition; import com.baidu.hugegraph.backend.query.Condition.RangeConditions; @@ -326,11 +327,10 @@ public class BinarySerializer extends AbstractSerializer { } protected byte[] formatIndexName(HugeIndex index) { - Id elemId = index.elementId(); - int idLen = 1 + elemId.length(); - BytesBuffer buffer; if (!this.indexWithIdPrefix) { + Id elemId = index.elementId(); + int idLen = 1 + elemId.length(); buffer = BytesBuffer.allocate(idLen); // Write element-id buffer.writeId(elemId, true); @@ -340,12 +340,13 @@ public class BinarySerializer extends AbstractSerializer { if (!type.isNumericIndex() && indexIdLengthExceedLimit(indexId)) { indexId = index.hashId(); } - idLen += 1 + indexId.length(); + String elemId = IdUtil.writeStoredString(index.elementId()); + int idLen = 1 + elemId.length() + 1 + indexId.length(); buffer = BytesBuffer.allocate(idLen); // Write index-id buffer.writeIndexId(indexId, type); // Write element-id - buffer.writeId(elemId, true); + buffer.writeString(elemId); } return buffer.bytes(); @@ -362,11 +363,8 @@ public class BinarySerializer extends AbstractSerializer { if (this.indexWithIdPrefix) { buffer.readIndexId(index.type()); } - Id elemId = buffer.readId(true); - if (index.indexLabel().queryType().isEdge()) { - elemId = EdgeId.parse(elemId.asString()); - } - index.elementIds(elemId); + String elemId = buffer.readString(); + index.elementIds(IdUtil.readStoredString(elemId)); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java index d86fbe8da..ee15a087e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java @@ -86,9 +86,8 @@ public abstract class TableSerializer extends AbstractSerializer { protected void formatProperty(HugeProperty prop, TableBackendEntry.Row row) { - row.column(HugeKeys.PROPERTIES, - prop.propertyKey().id().asLong(), - JsonUtil.toJson(prop.value())); + long pkid = prop.propertyKey().id().asLong(); + row.column(HugeKeys.PROPERTIES, pkid, JsonUtil.toJson(prop.value())); } protected void parseProperty(Id key, String colValue, HugeElement owner) { @@ -118,11 +117,13 @@ public abstract class TableSerializer extends AbstractSerializer { EdgeId id = edge.idWithDirection(); TableBackendEntry.Row row = new TableBackendEntry.Row(edge.type(), id); // Id: ownerVertex + direction + edge-label + sortValues + otherVertex - row.column(HugeKeys.OWNER_VERTEX, IdUtil.writeString(id.ownerVertexId())); + row.column(HugeKeys.OWNER_VERTEX, + IdUtil.writeStoredString(id.ownerVertexId())); row.column(HugeKeys.DIRECTION, id.direction().code()); row.column(HugeKeys.LABEL, id.edgeLabelId().asLong()); row.column(HugeKeys.SORT_VALUES, id.sortValues()); - row.column(HugeKeys.OTHER_VERTEX, IdUtil.writeString(id.otherVertexId())); + row.column(HugeKeys.OTHER_VERTEX, + IdUtil.writeStoredString(id.otherVertexId())); this.formatProperties(edge, row); return row; @@ -146,7 +147,7 @@ public abstract class TableSerializer extends AbstractSerializer { String otherVertexId = row.column(HugeKeys.OTHER_VERTEX); if (vertex == null) { - Id ownerId = IdUtil.readString(ownerVertexId); + Id ownerId = IdUtil.readStoredString(ownerVertexId); vertex = new HugeVertex(graph, ownerId, null); } @@ -154,7 +155,7 @@ public abstract class TableSerializer extends AbstractSerializer { VertexLabel srcLabel = graph.vertexLabel(edgeLabel.sourceLabel()); VertexLabel tgtLabel = graph.vertexLabel(edgeLabel.targetLabel()); - Id otherId = IdUtil.readString(otherVertexId); + Id otherId = IdUtil.readStoredString(otherVertexId); boolean isOutEdge = direction == Directions.OUT; HugeVertex otherVertex; if (isOutEdge) { @@ -189,7 +190,7 @@ public abstract class TableSerializer extends AbstractSerializer { @Override public BackendEntry writeVertex(HugeVertex vertex) { TableBackendEntry entry = newBackendEntry(vertex); - entry.column(HugeKeys.ID, IdUtil.writeString(vertex.id())); + entry.column(HugeKeys.ID, IdUtil.writeStoredString(vertex.id())); entry.column(HugeKeys.LABEL, vertex.schemaLabel().id().asLong()); // Add all properties of a Vertex this.formatProperties(vertex, entry.row()); @@ -201,7 +202,7 @@ public abstract class TableSerializer extends AbstractSerializer { HugeVertex vertex = prop.element(); TableBackendEntry entry = newBackendEntry(vertex); entry.subId(IdGenerator.of(prop.key())); - entry.column(HugeKeys.ID, IdUtil.writeString(vertex.id())); + entry.column(HugeKeys.ID, IdUtil.writeStoredString(vertex.id())); entry.column(HugeKeys.LABEL, vertex.schemaLabel().id().asLong()); this.formatProperty(prop, entry.row()); @@ -218,7 +219,7 @@ public abstract class TableSerializer extends AbstractSerializer { TableBackendEntry entry = this.convertEntry(backendEntry); assert entry.type().isVertex(); - Id id = IdUtil.readString(entry.column(HugeKeys.ID)); + Id id = IdUtil.readStoredString(entry.column(HugeKeys.ID)); Number label = entry.column(HugeKeys.LABEL); VertexLabel vertexLabel = VertexLabel.NONE; @@ -247,11 +248,13 @@ public abstract class TableSerializer extends AbstractSerializer { EdgeId id = edge.idWithDirection(); TableBackendEntry.Row row = new TableBackendEntry.Row(edge.type(), id); // Id: ownerVertex + direction + edge-label + sortValues + otherVertex - row.column(HugeKeys.OWNER_VERTEX, IdUtil.writeString(id.ownerVertexId())); + row.column(HugeKeys.OWNER_VERTEX, + IdUtil.writeStoredString(id.ownerVertexId())); row.column(HugeKeys.DIRECTION, id.direction().code()); row.column(HugeKeys.LABEL, id.edgeLabelId().asLong()); row.column(HugeKeys.SORT_VALUES, id.sortValues()); - row.column(HugeKeys.OTHER_VERTEX, IdUtil.writeString(id.otherVertexId())); + row.column(HugeKeys.OTHER_VERTEX, + IdUtil.writeStoredString(id.otherVertexId())); // Format edge property this.formatProperty(prop, row); @@ -284,7 +287,7 @@ public abstract class TableSerializer extends AbstractSerializer { entry.column(HugeKeys.FIELD_VALUES, index.fieldValues()); entry.column(HugeKeys.INDEX_LABEL_ID, index.indexLabel().longId()); entry.column(HugeKeys.ELEMENT_IDS, - IdUtil.writeString(index.elementId())); + IdUtil.writeStoredString(index.elementId())); entry.subId(index.elementId()); } return entry; @@ -308,11 +311,7 @@ public abstract class TableSerializer extends AbstractSerializer { HugeIndex index = new HugeIndex(indexLabel); index.fieldValues(indexValues); for (String elemId : elemIds) { - Id id = IdUtil.readString(elemId); - if (indexLabel.queryType().isEdge()) { - id = EdgeId.parse(id.asString()); - } - index.elementIds(id); + index.elementIds(IdUtil.readStoredString(elemId)); } return index; } @@ -329,7 +328,7 @@ public abstract class TableSerializer extends AbstractSerializer { id = EdgeId.parse(id.asString()); } } else if (type.isGraph()) { - id = IdGenerator.of(IdUtil.writeString(id)); + id = IdGenerator.of(IdUtil.writeStoredString(id)); } return id; } @@ -343,7 +342,7 @@ public abstract class TableSerializer extends AbstractSerializer { if (r.key() == HugeKeys.OWNER_VERTEX || r.key() == HugeKeys.OTHER_VERTEX) { // Serialize vertex id - String id = IdUtil.writeString((Id) value); + String id = IdUtil.writeStoredString((Id) value); r.serialValue(this.escapeString(id)); } else { // Serialize label id diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/exception/NotFoundException.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/exception/NotFoundException.java index ca4073b59..1f7dc0852 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/exception/NotFoundException.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/exception/NotFoundException.java @@ -32,4 +32,8 @@ public class NotFoundException extends HugeException { public NotFoundException(String message, Object... args) { super(message, args); } + + public NotFoundException(String message, Throwable cause, Object... args) { + super(message, cause, args); + } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/VertexLabel.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/VertexLabel.java index d5551973e..8d1a2588d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/VertexLabel.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/VertexLabel.java @@ -85,7 +85,7 @@ public class VertexLabel extends SchemaLabel { Builder useCustomizeNumberId(); - Builder useCustomizeUUid(); + Builder useCustomizeUUID(); Builder properties(String... properties); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/VertexLabelBuilder.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/VertexLabelBuilder.java index dcc13ac92..a6e8793d1 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/VertexLabelBuilder.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/schema/builder/VertexLabelBuilder.java @@ -256,7 +256,7 @@ public class VertexLabelBuilder implements VertexLabel.Builder { } @Override - public Builder useCustomizeUUid() { + public Builder useCustomizeUUID() { E.checkArgument(this.idStrategy == IdStrategy.DEFAULT || this.idStrategy == IdStrategy.CUSTOMIZE_UUID, "Not allowed to change id strategy for " + diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java index 4e6a89538..1fb346da6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeElement.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.function.Supplier; import org.apache.tinkerpop.gremlin.structure.Element; @@ -336,6 +337,9 @@ public abstract class HugeElement implements Element, GraphType, Idfiable { } else if (idValue instanceof Number) { // Long id return IdGenerator.of(((Number) idValue).longValue()); + } else if (idValue instanceof UUID) { + // UUID id + return IdGenerator.of((UUID) idValue); } else if (idValue instanceof Id) { // Id itself return (Id) idValue; @@ -346,8 +350,8 @@ public abstract class HugeElement implements Element, GraphType, Idfiable { // Throw if error type throw new UnsupportedOperationException(String.format( - "Invalid element id type: %s, must be a string", - idValue.getClass().getSimpleName())); + "Invalid element id: %s(%s)", + idValue, idValue.getClass().getSimpleName())); } @Watched(prefix = "element") diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java index d742a413c..93d418224 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java @@ -304,11 +304,11 @@ public class MysqlTables { edgeId.direction(), this.direction); List list = new ArrayList<>(5); - list.add(IdUtil.writeString(edgeId.ownerVertexId())); + list.add(IdUtil.writeStoredString(edgeId.ownerVertexId())); list.add(edgeId.direction().code()); list.add(edgeId.edgeLabelId().asLong()); list.add(edgeId.sortValues()); - list.add(IdUtil.writeString(edgeId.otherVertexId())); + list.add(IdUtil.writeStoredString(edgeId.otherVertexId())); return list; } diff --git a/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloTables.java b/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloTables.java index a222c398f..540a0de8d 100644 --- a/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloTables.java +++ b/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloTables.java @@ -221,11 +221,11 @@ public class PaloTables { edgeId.direction(), this.direction); List list = new ArrayList<>(5); - list.add(IdUtil.writeString(edgeId.ownerVertexId())); + list.add(IdUtil.writeStoredString(edgeId.ownerVertexId())); list.add(edgeId.direction().code()); list.add(edgeId.edgeLabelId().asLong()); list.add(edgeId.sortValues()); - list.add(IdUtil.writeString(edgeId.otherVertexId())); + list.add(IdUtil.writeStoredString(edgeId.otherVertexId())); return list; } diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java index 6780ed32c..3d1590264 100644 --- a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java @@ -47,7 +47,7 @@ public class PostgresqlSerializer extends MysqlSerializer { entry.column(HugeKeys.FIELD_VALUES, value); entry.column(HugeKeys.INDEX_LABEL_ID, index.indexLabel().longId()); entry.column(HugeKeys.ELEMENT_IDS, - IdUtil.writeString(index.elementId())); + IdUtil.writeStoredString(index.elementId())); entry.subId(index.elementId()); } return entry; diff --git a/hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBTables.java b/hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBTables.java index a4094369c..3a2ceab2a 100644 --- a/hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBTables.java +++ b/hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBTables.java @@ -85,7 +85,7 @@ public class ScyllaDBTables { Update update = QueryBuilder.update(table); update.with(QueryBuilder.append(ELEMENT_IDS, - IdUtil.writeString(entry.id()))); + IdUtil.writeStoredString(entry.id()))); update.where(CassandraTable.formatEQ(HugeKeys.LABEL, entry.column(HugeKeys.LABEL))); session.add(update); @@ -106,7 +106,7 @@ public class ScyllaDBTables { return; } update.with(QueryBuilder.remove(ELEMENT_IDS, - IdUtil.writeString(entry.id()))); + IdUtil.writeStoredString(entry.id()))); update.where(CassandraTable.formatEQ(HugeKeys.LABEL, label)); session.add(update); } @@ -307,7 +307,7 @@ public class ScyllaDBTables { }; for (String idValue : ids) { - Id rawId = IdUtil.readString(idValue); + Id rawId = IdUtil.readStoredString(idValue); EdgeId id = EdgeId.parse(rawId.asString()).directed(true); assert id.direction() == Directions.OUT; deleteEdge.accept(id); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java index f69da5bef..073513f70 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java @@ -28,6 +28,7 @@ import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import java.util.UUID; import java.util.function.Function; import org.apache.tinkerpop.gremlin.process.traversal.Order; @@ -1119,6 +1120,101 @@ public class EdgeCoreTest extends BaseCoreTest { Assert.assertEquals(edges2, edges); } + @Test + public void testQueryEdgesOfVertexWithCustomizeId() { + HugeGraph graph = graph(); + SchemaManager schema = graph.schema(); + + schema.vertexLabel("author2") + .properties("name", "age", "lived") + .useCustomizeNumberId() + .enableLabelIndex(false) + .create(); + schema.vertexLabel("language2") + .properties("name", "dynamic") + .useCustomizeUUID() + .nullableKeys("dynamic") + .enableLabelIndex(false) + .create(); + schema.vertexLabel("book2") + .properties("name") + .useAutomaticId() + .enableLabelIndex(false) + .create(); + + schema.edgeLabel("created2").singleTime() + .link("author2", "language2") + .enableLabelIndex(true) + .create(); + schema.edgeLabel("know2").singleTime() + .link("author2", "author2") + .enableLabelIndex(true) + .create(); + schema.edgeLabel("authored2").singleTime() + .properties("contribution", "comment", "score") + .nullableKeys("score", "contribution", "comment") + .link("author2", "book2") + .enableLabelIndex(true) + .create(); + + Vertex james = graph.addVertex(T.label, "author2", T.id, 13579, + "name", "James Gosling", "age", 62, + "lived", "Canadian"); + Vertex guido = graph.addVertex(T.label, "author2", T.id, 24680, + "name", "Guido van Rossum", "age", 61, + "lived", "California"); + + Vertex java = graph.addVertex(T.label, "language2", "name", "java", + T.id, UUID.randomUUID()); + Vertex python = graph.addVertex(T.label, "language2", + T.id, UUID.randomUUID(), + "name", "python", "dynamic", true); + + Vertex java1 = graph.addVertex(T.label, "book2", "name", "java-1"); + Vertex java2 = graph.addVertex(T.label, "book2", "name", "java-2"); + Vertex java3 = graph.addVertex(T.label, "book2", "name", "java-3"); + + Edge e1 = james.addEdge("created2", java); + Edge e2 = guido.addEdge("created2", python); + + Edge e3 = guido.addEdge("know2", james); + + Edge e4 = james.addEdge("authored2", java1); + Edge e5 = james.addEdge("authored2", java2); + Edge e6 = james.addEdge("authored2", java3, "score", 3); + + graph.tx().commit(); + + // Query OUT edges of a vertex + List edges = graph.traversal().V(james.id()).outE().toList(); + Assert.assertEquals(4, edges.size()); + + edges = ImmutableList.copyOf(james.edges(Direction.OUT)); + Assert.assertEquals(4, edges.size()); + + // Query IN edges of a vertex + edges = graph.traversal().V(james.id()).inE().toList(); + Assert.assertEquals(1, edges.size()); + + edges = ImmutableList.copyOf(james.edges(Direction.IN)); + Assert.assertEquals(1, edges.size()); + + // Query BOTH edges of a vertex + edges = graph.traversal().V(james.id()).bothE().toList(); + Assert.assertEquals(5, edges.size()); + + edges = ImmutableList.copyOf(james.edges(Direction.BOTH)); + Assert.assertEquals(5, edges.size()); + + // Query by edge id + Assert.assertEquals(1L, graph.traversal().E(e1.id()).count().next()); + Assert.assertEquals(1L, graph.traversal().E(e2.id()).count().next()); + Assert.assertEquals(1L, graph.traversal().E(e3.id()).count().next()); + Assert.assertEquals(1L, graph.traversal().E(e4.id()).count().next()); + Assert.assertEquals(1L, graph.traversal().E(e5.id()).count().next()); + Assert.assertEquals(1L, graph.traversal().E(e6.id()).count().next()); + } + @Test public void testQueryVerticesOfVertex() { HugeGraph graph = graph(); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java index eb7a037bd..fc6e01a0f 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java @@ -28,6 +28,7 @@ import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import org.apache.tinkerpop.gremlin.process.traversal.P; @@ -753,16 +754,21 @@ public class VertexCoreTest extends BaseCoreTest { SchemaManager schema = graph.schema(); schema.vertexLabel("programmer") - .useCustomizeUUid() + .useCustomizeUUID() .properties("name", "age", "city") .create(); graph.addVertex(T.label, "programmer", T.id, "835e1153-9281-4957-8691-cf79258e90eb", "name", "marko", "age", 18, "city", "Beijing"); + graph.addVertex(T.label, "programmer", + T.id, UUID.fromString( + "835e1153-9281-4957-8691-cf79258e90eb"), + "name", "marko", "age", 18, "city", "Beijing"); graph.addVertex(T.label, "programmer", T.id, "835e1153928149578691cf79258e90ee", "name", "marko", "age", 19, "city", "Beijing"); graph.tx().commit(); + Assert.assertEquals(2L, graph.traversal().V().count().next()); Object uuid = Text.uuid("835e1153928149578691cf79258e90eb"); List vertices = graph.traversal().V(uuid).toList(); @@ -793,7 +799,7 @@ public class VertexCoreTest extends BaseCoreTest { SchemaManager schema = graph.schema(); schema.vertexLabel("programmer") - .useCustomizeUUid() + .useCustomizeUUID() .properties("name", "age", "city") .create(); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java index f1bcb9f3b..70e19d43a 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java @@ -118,7 +118,7 @@ public class VertexLabelCoreTest extends SchemaCoreTest { Assert.assertEquals(IdStrategy.CUSTOMIZE_NUMBER, person3.idStrategy()); VertexLabel person4 = schema.vertexLabel("person4") - .useCustomizeUUid() + .useCustomizeUUID() .properties("name", "age", "city") .create(); Assert.assertEquals(IdStrategy.CUSTOMIZE_UUID, person4.idStrategy()); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdTest.java index a57752711..2b8588dae 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/IdTest.java @@ -98,8 +98,8 @@ public class IdTest extends BaseUnitTest { Assert.assertNotEquals(IdGenerator.of(1233), id); Assert.assertNotEquals(IdGenerator.of("123"), id); - Assert.assertEquals("1w", IdGenerator.asStoredString(id)); - Assert.assertEquals(id, IdGenerator.ofStoredString("1w", IdType.LONG)); + Assert.assertEquals("21w", IdGenerator.asStoredString(id)); + Assert.assertEquals(id, IdGenerator.ofStoredString("21w", IdType.LONG)); } @Test @@ -143,4 +143,19 @@ public class IdTest extends BaseUnitTest { Assert.assertEquals(id, IdGenerator.ofStoredString( "g14RU5KBSVeGkc95JY6Q6w==", IdType.UUID)); } + + @Test + public void testIdType() { + Assert.assertEquals(IdType.LONG, IdType.valueOfPrefix("L")); + Assert.assertEquals(IdType.UUID, IdType.valueOfPrefix("U")); + Assert.assertEquals(IdType.STRING, IdType.valueOfPrefix("S")); + Assert.assertEquals(IdType.EDGE, IdType.valueOfPrefix("E")); + Assert.assertEquals(IdType.UNKNOWN, IdType.valueOfPrefix("N")); + + Assert.assertEquals('L', IdType.LONG.prefix()); + Assert.assertEquals('U', IdType.UUID.prefix()); + Assert.assertEquals('S', IdType.STRING.prefix()); + Assert.assertEquals('E', IdType.EDGE.prefix()); + Assert.assertEquals('N', IdType.UNKNOWN.prefix()); + } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/IdUtilTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/IdUtilTest.java index 2fe0b3b79..1ca37919a 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/IdUtilTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/IdUtilTest.java @@ -22,10 +22,75 @@ package com.baidu.hugegraph.unit.util; import org.junit.Assert; import org.junit.Test; +import com.baidu.hugegraph.backend.id.EdgeId; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.id.IdGenerator; import com.baidu.hugegraph.backend.id.IdUtil; public class IdUtilTest { + @Test + public void testWriteReadString() { + Id id = IdGenerator.of(123); + Assert.assertEquals("L123", IdUtil.writeString(id)); + Assert.assertEquals(id, IdUtil.readString("L123")); + + id = IdGenerator.of("123"); + Assert.assertEquals("S123", IdUtil.writeString(id)); + Assert.assertEquals(id, IdUtil.readString("S123")); + + String uuid = "835e1153-9281-4957-8691-cf79258e90eb"; + id = IdGenerator.of(uuid, true); + Assert.assertEquals("U" + uuid, IdUtil.writeString(id)); + Assert.assertEquals(id, IdUtil.readString("U" + uuid)); + + id = EdgeId.parse("S1>2>3>L4"); + Assert.assertEquals("ES1>2>3>L4", IdUtil.writeString(id)); + Assert.assertEquals(id, IdUtil.readString("ES1>2>3>L4")); + + id = EdgeId.parse("S1111>2222>3>L4444"); + Assert.assertEquals("ES1111>2222>3>L4444", IdUtil.writeString(id)); + Assert.assertEquals(id, IdUtil.readString("ES1111>2222>3>L4444")); + } + + @Test + public void testWriteReadStoredString() { + Id id = IdGenerator.of(123); + Assert.assertEquals("L21w", IdUtil.writeStoredString(id)); + Assert.assertEquals(id, IdUtil.readStoredString("L21w")); + + id = IdGenerator.of("123"); + Assert.assertEquals("S123", IdUtil.writeStoredString(id)); + Assert.assertEquals(id, IdUtil.readStoredString("S123")); + + id = IdGenerator.of("835e1153-9281-4957-8691-cf79258e90eb", true); + String uuid = "Ug14RU5KBSVeGkc95JY6Q6w=="; + Assert.assertEquals(uuid, IdUtil.writeStoredString(id)); + Assert.assertEquals(id, IdUtil.readStoredString(uuid)); + + id = EdgeId.parse("S1>2>3>L4"); + Assert.assertEquals("ES1>12>3>L14", IdUtil.writeStoredString(id)); + Assert.assertEquals(id, IdUtil.readStoredString("ES1>12>3>L14")); + + id = EdgeId.parse("S1111>2222>3>L4444"); + Assert.assertEquals("ES1111>2Yj>3>L315S", IdUtil.writeStoredString(id)); + Assert.assertEquals(id, IdUtil.readStoredString("ES1111>2Yj>3>L315S")); + + id = EdgeId.parse("L1111>2222>3>L4444"); + Assert.assertEquals("EL2HN>2Yj>3>L315S", IdUtil.writeStoredString(id)); + Assert.assertEquals(id, IdUtil.readStoredString("EL2HN>2Yj>3>L315S")); + + id = EdgeId.parse("L11111111>2222>3>L44444444"); + String eid = "EL4fOg7>2Yj>3>L52eYhS"; + Assert.assertEquals(eid, IdUtil.writeStoredString(id)); + Assert.assertEquals(id, IdUtil.readStoredString(eid)); + + id = EdgeId.parse("L-1111>2222>33>L4444"); + eid = "EL0B7~~~~~~~~je>2Yj>33>L315S"; + Assert.assertEquals(eid, IdUtil.writeStoredString(id)); + Assert.assertEquals(id, IdUtil.readStoredString(eid)); + } + @Test public void testEscape() { Assert.assertEquals("a2b2c",