anph
/
hugegraph
forked from hugegraph/hugegraph
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Skip expired_time when not needed (#1044) 

* improve perf: skip expired_time for TextSerializer and TableSerializer
* fix mysql ttl insert template
* fix pgsql ttl insert template
* fix HugePermission.match()

Change-Id: Ic607b372ba61ac5768b81497d4ae7b2d4f4342f4
This commit is contained in:
Jermy Li 2020-06-18 14:31:20 +08:00 committed by GitHub
parent 85535cf758
commit 07cc67e8a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 112 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java
View File

@ -386,8 +386,13 @@ public abstract class CassandraTable
List<Definition> cols = row.getColumnDefinitions().asList();
for (Definition col : cols) {
String name = col.getName();
HugeKeys key = CassandraTable.parseKey(name);
Object value = row.getObject(name);
entry.column(CassandraTable.parseKey(name), value);
if (value == null) {
assert key == HugeKeys.EXPIRED_TIME;
continue;
}
entry.column(key, value);
}
return entry;

8
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTables.java
View File

@ -624,7 +624,7 @@ public class CassandraTables {
@Override
public void append(CassandraSessionPool.Session session,
CassandraBackendEntry.Row entry) {
assert entry.columns().size() == 4;
assert entry.columns().size() == 3 || entry.columns().size() == 4;
Insert insert = this.buildInsert(entry);
session.add(setTtl(insert, entry));
}
@ -632,7 +632,7 @@ public class CassandraTables {
@Override
public void eliminate(CassandraSessionPool.Session session,
CassandraBackendEntry.Row entry) {
assert entry.columns().size() == 4;
assert entry.columns().size() == 3 || entry.columns().size() == 4;
this.delete(session, entry);
}
}
@ -740,7 +740,7 @@ public class CassandraTables {
@Override
public void append(CassandraSessionPool.Session session,
CassandraBackendEntry.Row entry) {
assert entry.columns().size() == 4;
assert entry.columns().size() == 3 || entry.columns().size() == 4;
Insert insert = this.buildInsert(entry);
session.add(setTtl(insert, entry));
}
@ -748,7 +748,7 @@ public class CassandraTables {
@Override
public void eliminate(CassandraSessionPool.Session session,
CassandraBackendEntry.Row entry) {
assert entry.columns().size() == 4;
assert entry.columns().size() == 3 || entry.columns().size() == 4;
this.delete(session, entry);
}
}

9
hugegraph-core/src/main/java/com/baidu/hugegraph/auth/HugePermission.java
View File

@ -49,15 +49,14 @@ public enum HugePermission implements SerialEnum {
}
public String string() {
String string = this.name().toLowerCase();
return string;
return this.name().toLowerCase();
}
public boolean match(HugePermission other) {
if ((this.code & other.code) != 0) {
return true;
if (other == ANY) {
return this == ANY;
}
return false;
return (this.code & other.code) != 0;
}
public static HugePermission fromCode(byte code) {

5
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java
View File

@ -412,6 +412,7 @@ public class BinarySerializer extends AbstractSerializer {
// Write vertex expired time if needed
if (vertex.hasTtl()) {
entry.ttl(vertex.ttl());
this.formatExpiredTime(vertex.expiredTime(), buffer);
}
@ -419,10 +420,6 @@ public class BinarySerializer extends AbstractSerializer {
byte[] name = this.keyWithIdPrefix ? entry.id().asBytes() : EMPTY_BYTES;
entry.column(name, buffer.bytes());
if (vertex.hasTtl()) {
entry.ttl(vertex.ttl());
}
return entry;
}

21
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java
View File

@ -138,6 +138,7 @@ public abstract class TableSerializer extends AbstractSerializer {
TableBackendEntry.Row row = new TableBackendEntry.Row(edge.type(), id);
if (edge.hasTtl()) {
row.ttl(edge.ttl());
row.column(HugeKeys.EXPIRED_TIME, edge.expiredTime());
}
// Id: ownerVertex + direction + edge-label + sortValues + otherVertex
row.column(HugeKeys.OWNER_VERTEX, this.writeId(id.ownerVertexId()));
@ -145,7 +146,6 @@ public abstract class TableSerializer extends AbstractSerializer {
row.column(HugeKeys.LABEL, id.edgeLabelId().asLong());
row.column(HugeKeys.SORT_VALUES, id.sortValues());
row.column(HugeKeys.OTHER_VERTEX, this.writeId(id.otherVertexId()));
row.column(HugeKeys.EXPIRED_TIME, edge.expiredTime());
this.formatProperties(edge, row);
return row;
@ -206,7 +206,11 @@ public abstract class TableSerializer extends AbstractSerializer {
// Parse edge properties
this.parseProperties(edge, row);
edge.expiredTime(expiredTime.longValue());
// The expired time is null when the edge is non-ttl
long expired = edge.hasTtl() ? expiredTime.longValue() : 0L;
edge.expiredTime(expired);
return edge;
}
@ -215,10 +219,10 @@ public abstract class TableSerializer extends AbstractSerializer {
TableBackendEntry entry = newBackendEntry(vertex);
if (vertex.hasTtl()) {
entry.ttl(vertex.ttl());
entry.column(HugeKeys.EXPIRED_TIME, vertex.expiredTime());
}
entry.column(HugeKeys.ID, this.writeId(vertex.id()));
entry.column(HugeKeys.LABEL, vertex.schemaLabel().id().asLong());
entry.column(HugeKeys.EXPIRED_TIME, vertex.expiredTime());
// Add all properties of a Vertex
this.formatProperties(vertex, entry.row());
return entry;
@ -230,11 +234,11 @@ public abstract class TableSerializer extends AbstractSerializer {
TableBackendEntry entry = newBackendEntry(vertex);
if (vertex.hasTtl()) {
entry.ttl(vertex.ttl());
entry.column(HugeKeys.EXPIRED_TIME, vertex.expiredTime());
}
entry.subId(IdGenerator.of(prop.key()));
entry.column(HugeKeys.ID, this.writeId(vertex.id()));
entry.column(HugeKeys.LABEL, vertex.schemaLabel().id().asLong());
entry.column(HugeKeys.EXPIRED_TIME, vertex.expiredTime());
this.formatProperty(prop, entry.row());
return entry;
@ -266,7 +270,7 @@ public abstract class TableSerializer extends AbstractSerializer {
for (TableBackendEntry.Row edge : entry.subRows()) {
this.parseEdge(edge, vertex, graph);
}
// The expired time is null when this is fake vertex of edge
// The expired time is null when this is fake vertex of edge or non-ttl
if (expiredTime != null) {
vertex.expiredTime(expiredTime.longValue());
}
@ -285,6 +289,7 @@ public abstract class TableSerializer extends AbstractSerializer {
TableBackendEntry.Row row = new TableBackendEntry.Row(edge.type(), id);
if (edge.hasTtl()) {
row.ttl(edge.ttl());
row.column(HugeKeys.EXPIRED_TIME, edge.expiredTime());
}
// Id: ownerVertex + direction + edge-label + sortValues + otherVertex
row.column(HugeKeys.OWNER_VERTEX, this.writeId(id.ownerVertexId()));
@ -292,7 +297,6 @@ public abstract class TableSerializer extends AbstractSerializer {
row.column(HugeKeys.LABEL, id.edgeLabelId().asLong());
row.column(HugeKeys.SORT_VALUES, id.sortValues());
row.column(HugeKeys.OTHER_VERTEX, this.writeId(id.otherVertexId()));
row.column(HugeKeys.EXPIRED_TIME, edge.expiredTime());
// Format edge property
this.formatProperty(prop, row);
@ -326,10 +330,10 @@ public abstract class TableSerializer extends AbstractSerializer {
entry.column(HugeKeys.FIELD_VALUES, index.fieldValues());
entry.column(HugeKeys.INDEX_LABEL_ID, index.indexLabel().longId());
entry.column(HugeKeys.ELEMENT_IDS, this.writeId(index.elementId()));
entry.column(HugeKeys.EXPIRED_TIME, index.expiredTime());
entry.subId(index.elementId());
if (index.hasTtl()) {
entry.ttl(index.ttl());
entry.column(HugeKeys.EXPIRED_TIME, index.expiredTime());
}
}
return entry;
@ -353,8 +357,9 @@ public abstract class TableSerializer extends AbstractSerializer {
IndexLabel indexLabel = graph.indexLabel(this.toId(indexLabelId));
HugeIndex index = new HugeIndex(graph, indexLabel);
index.fieldValues(indexValues);
long expired = index.hasTtl() ? expiredTime.longValue() : 0L;
for (Object elemId : elemIds) {
index.elementIds(this.readId(elemId), expiredTime.longValue());
index.elementIds(this.readId(elemId), expired);
}
return index;
}

25
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextBackendEntry.java
View File

@ -20,7 +20,6 @@
package com.baidu.hugegraph.backend.serializer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@ -183,11 +182,17 @@ public class TextBackendEntry implements BackendEntry, Cloneable {
}
// TODO: ensure the old value is a list and json format (for index)
if (oldValue.equals("[]")) {
this.column(col.getKey(), newValue);
continue;
}
List<Object> values = new ArrayList<>();
Object[] oldValues = JsonUtil.fromJson(oldValue, Object[].class);
Object[] newValues = JsonUtil.fromJson(newValue, Object[].class);
values.addAll(Arrays.asList(oldValues));
values.addAll(Arrays.asList(newValues));
@SuppressWarnings("unchecked")
List<Object> oldValues = JsonUtil.fromJson(oldValue, List.class);
@SuppressWarnings("unchecked")
List<Object> newValues = JsonUtil.fromJson(newValue, List.class);
values.addAll(oldValues);
values.addAll(newValues);
// Update the old value
this.column(col.getKey(), JsonUtil.toJson(values));
}
@ -213,10 +218,12 @@ public class TextBackendEntry implements BackendEntry, Cloneable {
// TODO: ensure the old value is a list and json format (for index)
List<Object> values = new ArrayList<>();
Object[] oldValues = JsonUtil.fromJson(oldValue, Object[].class);
Object[] newValues = JsonUtil.fromJson(newValue, Object[].class);
values.addAll(Arrays.asList(oldValues));
values.removeAll(Arrays.asList(newValues));
@SuppressWarnings("unchecked")
List<Object> oldValues = JsonUtil.fromJson(oldValue, List.class);
@SuppressWarnings("unchecked")
List<Object> newValues = JsonUtil.fromJson(newValue, List.class);
values.addAll(oldValues);
values.removeAll(newValues);
// Update the old value
this.column(col.getKey(), JsonUtil.toJson(values));
}

38
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java
View File

@ -824,14 +824,6 @@ public class TextSerializer extends AbstractSerializer {
return JsonUtil.toJson(array);
}
private static String writeElementId(Id id, long expiredTime) {
Object[] array = new Object[1];
Object idValue = id.number() ? id.asLong() : id.asString();
array[0] = ImmutableMap.of(HugeKeys.ID.string(), idValue,
HugeKeys.EXPIRED_TIME.string(), expiredTime);
return JsonUtil.toJson(array);
}
private static Id[] readIds(String str) {
Object[] values = JsonUtil.fromJson(str, Object[].class);
Id[] ids = new Id[values.length];
@ -847,15 +839,35 @@ public class TextSerializer extends AbstractSerializer {
return ids;
}
private static String writeElementId(Id id, long expiredTime) {
Object[] array = new Object[1];
Object idValue = id.number() ? id.asLong() : id.asString();
if (expiredTime <= 0L) {
array[0] = id;
} else {
array[0] = ImmutableMap.of(HugeKeys.ID.string(), idValue,
HugeKeys.EXPIRED_TIME.string(),
expiredTime);
}
return JsonUtil.toJson(array);
}
private static IdWithExpiredTime[] readElementIds(String str) {
Object[] values = JsonUtil.fromJson(str, Object[].class);
IdWithExpiredTime[] ids = new IdWithExpiredTime[values.length];
for (int i = 0; i < values.length; i++) {
@SuppressWarnings("unchecked")
Map<String, Object> map = (Map<String, Object>) values[i];
Object idValue = map.get(HugeKeys.ID.string());
long expiredTime = ((Number) map.get(
HugeKeys.EXPIRED_TIME.string())).longValue();
Object idValue;
long expiredTime;
if (values[i] instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> map = (Map<String, Object>) values[i];
idValue = map.get(HugeKeys.ID.string());
expiredTime = ((Number) map.get(
HugeKeys.EXPIRED_TIME.string())).longValue();
} else {
idValue = values[i];
expiredTime = 0L;
}
Id id;
if (idValue instanceof Number) {
id = IdGenerator.of(((Number) idValue).longValue());

2
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java
View File

@ -407,7 +407,7 @@ public class SchemaTransaction extends IndexableTransaction {
E.checkArgument(name.length() < 256,
"The length of name must less than 256 bytes.");
E.checkArgument(!name.matches(illegalReg),
String.format("Illegal schema name '%s'", name));
"Illegal schema name '%s'", name);
final char[] filters = {'#', '>', ':', '!'};
for (char c : filters) {

2
hugegraph-core/src/main/java/com/baidu/hugegraph/plugin/HugeGraphGremlinPlugin.java
View File

@ -49,7 +49,7 @@ public class HugeGraphGremlinPlugin extends AbstractGremlinPlugin {
classInfos = ReflectionUtil.classes(PACKAGE);
} catch (IOException e) {
throw new HugeException("Failed to scan classes under package %s",
PACKAGE);
e, PACKAGE);
}
@SuppressWarnings("rawtypes")

3
hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/optimize/TraversalUtil.java
View File

@ -778,8 +778,7 @@ public final class TraversalUtil {
}
}
public static <T> Condition parsePredicate(PropertyKey pk,
String predicate) {
public static Condition parsePredicate(PropertyKey pk, String predicate) {
Pattern pattern = Pattern.compile("^P\\.([a-z]+)\\(([\\S ]*)\\)$");
Matcher matcher = pattern.matcher(predicate);
if (!matcher.find()) {

9
hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlEntryIterator.java
View File

@ -101,7 +101,7 @@ public class MysqlEntryIterator extends BackendEntryIterator {
protected PageState pageState() {
byte[] position;
// There is no latest or no next page
if (this.lastest == null || !exceedLimit &&
if (this.lastest == null || !this.exceedLimit &&
this.fetched() <= this.query.limit() && this.next == null) {
position = PageState.EMPTY_BYTES;
} else {
@ -144,8 +144,13 @@ public class MysqlEntryIterator extends BackendEntryIterator {
ResultSetMetaData metaData = result.getMetaData();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String name = metaData.getColumnLabel(i);
HugeKeys key = MysqlTable.parseKey(name);
Object value = result.getObject(i);
entry.column(MysqlTable.parseKey(name), value);
if (value == null) {
assert key == HugeKeys.EXPIRED_TIME;
continue;
}
entry.column(key, value);
}
return entry;
}

36
hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java
View File

@ -63,6 +63,7 @@ public abstract class MysqlTable
// The template for insert and delete statements
private String insertTemplate;
private String insertTemplateTtl;
private String deleteTemplate;
private final MysqlShardSpliter shardSpliter;
@ -70,6 +71,7 @@ public abstract class MysqlTable
public MysqlTable(String table) {
super(table);
this.insertTemplate = null;
this.insertTemplateTtl = null;
this.deleteTemplate = null;
this.shardSpliter = new MysqlShardSpliter(this.table());
}
@ -175,10 +177,28 @@ public abstract class MysqlTable
}
protected String buildInsertTemplate(MysqlBackendEntry.Row entry) {
if (entry.ttl() != 0L) {
return this.buildInsertTemplateWithTtl(entry);
}
if (this.insertTemplate != null) {
return this.insertTemplate;
}
this.insertTemplate = this.buildInsertTemplateForce(entry);
return this.insertTemplate;
}
protected String buildInsertTemplateWithTtl(MysqlBackendEntry.Row entry) {
assert entry.ttl() != 0L;
if (this.insertTemplateTtl != null) {
return this.insertTemplateTtl;
}
this.insertTemplateTtl = this.buildInsertTemplateForce(entry);
return this.insertTemplateTtl;
}
protected String buildInsertTemplateForce(MysqlBackendEntry.Row entry) {
StringBuilder insert = new StringBuilder();
insert.append("REPLACE INTO ").append(this.table()).append(" (");
@ -200,8 +220,7 @@ public abstract class MysqlTable
}
insert.append(")");
this.insertTemplate = insert.toString();
return this.insertTemplate;
return insert.toString();
}
protected String buildDeleteTemplate(List<HugeKeys> idNames) {
@ -221,18 +240,6 @@ public abstract class MysqlTable
return this.deleteTemplate;
}
protected String buildDeleteTemplateWithoutCache(List<HugeKeys> idNames) {
StringBuilder delete = new StringBuilder();
delete.append("DELETE FROM ").append(this.table());
this.appendPartition(delete);
WhereBuilder where = this.newWhereBuilder();
where.and(formatKeys(idNames), "=");
delete.append(where.build());
return delete.toString();
}
protected String buildDropTemplate() {
return String.format("DROP TABLE IF EXISTS %s;", this.table());
}
@ -701,6 +708,7 @@ public abstract class MysqlTable
super(table);
}
@Override
public List<Shard> getSplits(Session session, long splitSize) {
E.checkArgument(splitSize >= MIN_SHARD_SIZE,
"The split-size must be >= %s bytes, but got %s",

22
hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java
View File

@ -32,8 +32,7 @@ import com.baidu.hugegraph.type.define.HugeKeys;
public abstract class PostgresqlTable extends MysqlTable {
private String insertTemplate = null;
private String orderByKeys = null;
private String orderByKeysTemplate = null;
public PostgresqlTable(String table) {
super(table);
@ -63,11 +62,7 @@ public abstract class PostgresqlTable extends MysqlTable {
}
@Override
protected String buildInsertTemplate(MysqlBackendEntry.Row entry) {
if (this.insertTemplate != null) {
return this.insertTemplate;
}
protected String buildInsertTemplateForce(MysqlBackendEntry.Row entry) {
StringBuilder insert = new StringBuilder();
insert.append("INSERT INTO ").append(this.table()).append(" (");
@ -110,15 +105,14 @@ public abstract class PostgresqlTable extends MysqlTable {
}
}
this.insertTemplate = insert.toString();
return this.insertTemplate;
return insert.toString();
}
// Set order-by to keep results order consistence for PostgreSQL result
@Override
protected String orderByKeys() {
if (this.orderByKeys != null) {
return this.orderByKeys;
// Set order-by to keep results order consistence for PostgreSQL result
if (this.orderByKeysTemplate != null) {
return this.orderByKeysTemplate;
}
int i = 0;
int size = this.tableDefine().keys().size();
@ -131,8 +125,8 @@ public abstract class PostgresqlTable extends MysqlTable {
select.append(", ");
}
}
this.orderByKeys = select.toString();
return this.orderByKeys;
this.orderByKeysTemplate = select.toString();
return this.orderByKeysTemplate;
}
@Override

6
hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java
View File

@ -620,7 +620,7 @@ public class EdgeCoreTest extends BaseCoreTest {
Vertex java = graph().addVertex(T.label, "book",
"name", "Java in action");
Edge edge = baby.addEdge("borrow", java, "place", "library of school",
"date", DateUtil.now().getTime() - 1000L);
"date", DateUtil.now().getTime() - 2000L);
graph().tx().commit();
Iterator<Edge> edges = graph().edges(edge);
@ -629,7 +629,7 @@ public class EdgeCoreTest extends BaseCoreTest {
graph().tx().commit();
try {
Thread.sleep(1100L);
Thread.sleep(500L);
} catch (InterruptedException e) {
// Ignore
}
@ -640,7 +640,7 @@ public class EdgeCoreTest extends BaseCoreTest {
graph().tx().commit();
try {
Thread.sleep(1100L);
Thread.sleep(1000L);
} catch (InterruptedException e) {
// Ignore
}