[Fix][Common] rewrite code generate,fix bit shift (#6914)
* rewrite code generate,fix bit shift * fix ut * add algorithm from licenses file * add algorithm from licenses file * add algorithm from licenses file * add algorithm from licenses file * add algorithm from licenses file * fix ut
This commit is contained in:
parent
595e4843d0
commit
4bec792160
|
|
@ -26,6 +26,7 @@ header:
|
||||||
- LICENSE
|
- LICENSE
|
||||||
- DISCLAIMER
|
- DISCLAIMER
|
||||||
- dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
|
- dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
|
||||||
|
- dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java
|
||||||
- mvnw.cmd
|
- mvnw.cmd
|
||||||
- dolphinscheduler-dao/src/main/resources/sql/soft_version
|
- dolphinscheduler-dao/src/main/resources/sql/soft_version
|
||||||
- .mvn
|
- .mvn
|
||||||
|
|
|
||||||
2
LICENSE
2
LICENSE
|
|
@ -219,4 +219,4 @@ The text of each license is the standard Apache 2.0 license.
|
||||||
DolphinPluginClassLoader from https://github.com/prestosql/presto Apache 2.0
|
DolphinPluginClassLoader from https://github.com/prestosql/presto Apache 2.0
|
||||||
DolphinPluginDiscovery from https://github.com/prestosql/presto Apache 2.0
|
DolphinPluginDiscovery from https://github.com/prestosql/presto Apache 2.0
|
||||||
DolphinPluginLoader from https://github.com/prestosql/presto Apache 2.0
|
DolphinPluginLoader from https://github.com/prestosql/presto Apache 2.0
|
||||||
|
CodeGenerateUtils from https://github.com/twitter-archive/snowflake/tree/snowflake-2010 Apache 2.0
|
||||||
|
|
|
||||||
|
|
@ -23,9 +23,9 @@ import org.apache.dolphinscheduler.api.service.EnvironmentService;
|
||||||
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
||||||
import org.apache.dolphinscheduler.api.utils.Result;
|
import org.apache.dolphinscheduler.api.utils.Result;
|
||||||
import org.apache.dolphinscheduler.common.Constants;
|
import org.apache.dolphinscheduler.common.Constants;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
|
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
|
|
||||||
import org.apache.dolphinscheduler.dao.entity.Environment;
|
import org.apache.dolphinscheduler.dao.entity.Environment;
|
||||||
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
|
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
|
||||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||||
|
|
@ -115,9 +115,9 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
|
||||||
env.setUpdateTime(new Date());
|
env.setUpdateTime(new Date());
|
||||||
long code = 0L;
|
long code = 0L;
|
||||||
try {
|
try {
|
||||||
code = SnowFlakeUtils.getInstance().nextId();
|
code = CodeGenerateUtils.getInstance().genCode();
|
||||||
env.setCode(code);
|
env.setCode(code);
|
||||||
} catch (SnowFlakeException e) {
|
} catch (CodeGenerateException e) {
|
||||||
logger.error("Environment code get error, ", e);
|
logger.error("Environment code get error, ", e);
|
||||||
}
|
}
|
||||||
if (code == 0L) {
|
if (code == 0L) {
|
||||||
|
|
|
||||||
|
|
@ -40,10 +40,10 @@ import org.apache.dolphinscheduler.common.graph.DAG;
|
||||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||||
import org.apache.dolphinscheduler.common.thread.Stopper;
|
import org.apache.dolphinscheduler.common.thread.Stopper;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
|
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
|
|
||||||
import org.apache.dolphinscheduler.dao.entity.DagData;
|
import org.apache.dolphinscheduler.dao.entity.DagData;
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
|
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
|
||||||
|
|
@ -222,8 +222,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
||||||
}
|
}
|
||||||
long processDefinitionCode;
|
long processDefinitionCode;
|
||||||
try {
|
try {
|
||||||
processDefinitionCode = SnowFlakeUtils.getInstance().nextId();
|
processDefinitionCode = CodeGenerateUtils.getInstance().genCode();
|
||||||
} catch (SnowFlakeException e) {
|
} catch (CodeGenerateException e) {
|
||||||
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
|
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
@ -874,8 +874,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
||||||
processDefinition.setProjectCode(projectCode);
|
processDefinition.setProjectCode(projectCode);
|
||||||
processDefinition.setUserId(loginUser.getId());
|
processDefinition.setUserId(loginUser.getId());
|
||||||
try {
|
try {
|
||||||
processDefinition.setCode(SnowFlakeUtils.getInstance().nextId());
|
processDefinition.setCode(CodeGenerateUtils.getInstance().genCode());
|
||||||
} catch (SnowFlakeException e) {
|
} catch (CodeGenerateException e) {
|
||||||
putMsg(result, Status.CREATE_PROCESS_DEFINITION_ERROR);
|
putMsg(result, Status.CREATE_PROCESS_DEFINITION_ERROR);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
@ -894,10 +894,10 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
||||||
taskDefinitionLog.setOperator(loginUser.getId());
|
taskDefinitionLog.setOperator(loginUser.getId());
|
||||||
taskDefinitionLog.setOperateTime(now);
|
taskDefinitionLog.setOperateTime(now);
|
||||||
try {
|
try {
|
||||||
long code = SnowFlakeUtils.getInstance().nextId();
|
long code = CodeGenerateUtils.getInstance().genCode();
|
||||||
taskCodeMap.put(taskDefinitionLog.getCode(), code);
|
taskCodeMap.put(taskDefinitionLog.getCode(), code);
|
||||||
taskDefinitionLog.setCode(code);
|
taskDefinitionLog.setCode(code);
|
||||||
} catch (SnowFlakeException e) {
|
} catch (CodeGenerateException e) {
|
||||||
logger.error("Task code get error, ", e);
|
logger.error("Task code get error, ", e);
|
||||||
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
|
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
|
||||||
return false;
|
return false;
|
||||||
|
|
@ -1363,8 +1363,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
||||||
processDefinition.setProjectCode(targetProjectCode);
|
processDefinition.setProjectCode(targetProjectCode);
|
||||||
if (isCopy) {
|
if (isCopy) {
|
||||||
try {
|
try {
|
||||||
processDefinition.setCode(SnowFlakeUtils.getInstance().nextId());
|
processDefinition.setCode(CodeGenerateUtils.getInstance().genCode());
|
||||||
} catch (SnowFlakeException e) {
|
} catch (CodeGenerateException e) {
|
||||||
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
|
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
|
||||||
throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS);
|
throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,8 +25,8 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
|
||||||
import org.apache.dolphinscheduler.api.utils.Result;
|
import org.apache.dolphinscheduler.api.utils.Result;
|
||||||
import org.apache.dolphinscheduler.common.Constants;
|
import org.apache.dolphinscheduler.common.Constants;
|
||||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||||
import org.apache.dolphinscheduler.dao.entity.Project;
|
import org.apache.dolphinscheduler.dao.entity.Project;
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
|
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
|
||||||
|
|
@ -97,14 +97,14 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
|
||||||
project = Project
|
project = Project
|
||||||
.newBuilder()
|
.newBuilder()
|
||||||
.name(name)
|
.name(name)
|
||||||
.code(SnowFlakeUtils.getInstance().nextId())
|
.code(CodeGenerateUtils.getInstance().genCode())
|
||||||
.description(desc)
|
.description(desc)
|
||||||
.userId(loginUser.getId())
|
.userId(loginUser.getId())
|
||||||
.userName(loginUser.getUserName())
|
.userName(loginUser.getUserName())
|
||||||
.createTime(now)
|
.createTime(now)
|
||||||
.updateTime(now)
|
.updateTime(now)
|
||||||
.build();
|
.build();
|
||||||
} catch (SnowFlakeException e) {
|
} catch (CodeGenerateException e) {
|
||||||
putMsg(result, Status.CREATE_PROJECT_ERROR);
|
putMsg(result, Status.CREATE_PROJECT_ERROR);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,9 +27,9 @@ import org.apache.dolphinscheduler.api.utils.Result;
|
||||||
import org.apache.dolphinscheduler.common.Constants;
|
import org.apache.dolphinscheduler.common.Constants;
|
||||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
|
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
|
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
|
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
|
||||||
import org.apache.dolphinscheduler.dao.entity.Project;
|
import org.apache.dolphinscheduler.dao.entity.Project;
|
||||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||||
|
|
@ -431,9 +431,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
|
||||||
List<Long> taskCodes = new ArrayList<>();
|
List<Long> taskCodes = new ArrayList<>();
|
||||||
try {
|
try {
|
||||||
for (int i = 0; i < genNum; i++) {
|
for (int i = 0; i < genNum; i++) {
|
||||||
taskCodes.add(SnowFlakeUtils.getInstance().nextId());
|
taskCodes.add(CodeGenerateUtils.getInstance().genCode());
|
||||||
}
|
}
|
||||||
} catch (SnowFlakeException e) {
|
} catch (CodeGenerateException e) {
|
||||||
logger.error("Task code get error, ", e);
|
logger.error("Task code get error, ", e);
|
||||||
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
|
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,74 @@
|
||||||
|
/** Copyright 2010-2012 Twitter, Inc.*/
|
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.common.utils;
|
||||||
|
|
||||||
|
import java.net.InetAddress;
|
||||||
|
import java.net.UnknownHostException;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rewriting based on Twitter snowflake algorithm
|
||||||
|
*/
|
||||||
|
public class CodeGenerateUtils {
|
||||||
|
// start timestamp
|
||||||
|
private static final long START_TIMESTAMP = 1609430400000L; //2021-01-01 00:00:00
|
||||||
|
// Each machine generates 32 in the same millisecond
|
||||||
|
private static final long LOW_DIGIT_BIT = 5L;
|
||||||
|
private static final long MIDDLE_BIT = 2L;
|
||||||
|
private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT);
|
||||||
|
// The displacement to the left
|
||||||
|
private static final long MIDDLE_LEFT = LOW_DIGIT_BIT;
|
||||||
|
private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MIDDLE_BIT;
|
||||||
|
private final long machineHash;
|
||||||
|
private long lowDigit = 0L;
|
||||||
|
private long recordMillisecond = -1L;
|
||||||
|
|
||||||
|
private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis();
|
||||||
|
private static final long SYSTEM_NANOTIME = System.nanoTime();
|
||||||
|
|
||||||
|
private CodeGenerateUtils() throws CodeGenerateException {
|
||||||
|
try {
|
||||||
|
this.machineHash = Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % (2 << (MIDDLE_BIT - 1));
|
||||||
|
} catch (UnknownHostException e) {
|
||||||
|
throw new CodeGenerateException(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CodeGenerateUtils instance = null;
|
||||||
|
|
||||||
|
public static synchronized CodeGenerateUtils getInstance() throws CodeGenerateException {
|
||||||
|
if (instance == null) {
|
||||||
|
instance = new CodeGenerateUtils();
|
||||||
|
}
|
||||||
|
return instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized long genCode() throws CodeGenerateException {
|
||||||
|
long nowtMillisecond = systemMillisecond();
|
||||||
|
if (nowtMillisecond < recordMillisecond) {
|
||||||
|
throw new CodeGenerateException("New code exception because time is set back.");
|
||||||
|
}
|
||||||
|
if (nowtMillisecond == recordMillisecond) {
|
||||||
|
lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT;
|
||||||
|
if (lowDigit == 0L) {
|
||||||
|
while (nowtMillisecond <= recordMillisecond) {
|
||||||
|
nowtMillisecond = systemMillisecond();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
lowDigit = 0L;
|
||||||
|
}
|
||||||
|
recordMillisecond = nowtMillisecond;
|
||||||
|
return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << MIDDLE_LEFT | lowDigit;
|
||||||
|
}
|
||||||
|
|
||||||
|
private long systemMillisecond() {
|
||||||
|
return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class CodeGenerateException extends Exception {
|
||||||
|
public CodeGenerateException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,94 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
* contributor license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright ownership.
|
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
* (the "License"); you may not use this file except in compliance with
|
|
||||||
* the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.common.utils;
|
|
||||||
|
|
||||||
import java.net.InetAddress;
|
|
||||||
import java.net.UnknownHostException;
|
|
||||||
import java.util.Objects;
|
|
||||||
|
|
||||||
public class SnowFlakeUtils {
|
|
||||||
// start timestamp
|
|
||||||
private static final long START_TIMESTAMP = 1609430400000L; //2021-01-01 00:00:00
|
|
||||||
// Each machine generates 32 in the same millisecond
|
|
||||||
private static final long SEQUENCE_BIT = 5;
|
|
||||||
private static final long MACHINE_BIT = 2;
|
|
||||||
private static final long MAX_SEQUENCE = ~(-1L << SEQUENCE_BIT);
|
|
||||||
// The displacement to the left
|
|
||||||
private static final long MACHINE_LEFT = SEQUENCE_BIT + MACHINE_BIT;
|
|
||||||
private static final long TIMESTAMP_LEFT = SEQUENCE_BIT + MACHINE_BIT + MACHINE_LEFT;
|
|
||||||
private final int machineId;
|
|
||||||
private long sequence = 0L;
|
|
||||||
private long lastTimestamp = -1L;
|
|
||||||
|
|
||||||
private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis();
|
|
||||||
private static final long SYSTEM_NANOTIME = System.nanoTime();
|
|
||||||
|
|
||||||
private SnowFlakeUtils() throws SnowFlakeException {
|
|
||||||
try {
|
|
||||||
this.machineId = Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % 4;
|
|
||||||
} catch (UnknownHostException e) {
|
|
||||||
throw new SnowFlakeException(e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static SnowFlakeUtils instance = null;
|
|
||||||
|
|
||||||
public static synchronized SnowFlakeUtils getInstance() throws SnowFlakeException {
|
|
||||||
if (instance == null) {
|
|
||||||
instance = new SnowFlakeUtils();
|
|
||||||
}
|
|
||||||
return instance;
|
|
||||||
}
|
|
||||||
|
|
||||||
public synchronized long nextId() throws SnowFlakeException {
|
|
||||||
long currStmp = nowTimestamp();
|
|
||||||
if (currStmp < lastTimestamp) {
|
|
||||||
throw new SnowFlakeException("Clock moved backwards. Refusing to generate id");
|
|
||||||
}
|
|
||||||
if (currStmp == lastTimestamp) {
|
|
||||||
sequence = (sequence + 1) & MAX_SEQUENCE;
|
|
||||||
if (sequence == 0L) {
|
|
||||||
currStmp = getNextMill();
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
sequence = 0L;
|
|
||||||
}
|
|
||||||
lastTimestamp = currStmp;
|
|
||||||
return (currStmp - START_TIMESTAMP) << TIMESTAMP_LEFT
|
|
||||||
| machineId << MACHINE_LEFT
|
|
||||||
| sequence;
|
|
||||||
}
|
|
||||||
|
|
||||||
private long getNextMill() {
|
|
||||||
long mill = nowTimestamp();
|
|
||||||
while (mill <= lastTimestamp) {
|
|
||||||
mill = nowTimestamp();
|
|
||||||
}
|
|
||||||
return mill;
|
|
||||||
}
|
|
||||||
|
|
||||||
private long nowTimestamp() {
|
|
||||||
return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class SnowFlakeException extends Exception {
|
|
||||||
public SnowFlakeException(String message) {
|
|
||||||
super(message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -22,14 +22,14 @@ import java.util.HashSet;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
public class SnowFlakeUtilsTest {
|
public class CodeGenerateUtilsTest {
|
||||||
@Test
|
@Test
|
||||||
public void testNoGenerateDuplicateId() throws SnowFlakeUtils.SnowFlakeException {
|
public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException {
|
||||||
HashSet<Long> existsSnowFlakeId = new HashSet<>();
|
HashSet<Long> existsCode = new HashSet<>();
|
||||||
for (int i = 0; i < 100; i++) {
|
for (int i = 0; i < 100; i++) {
|
||||||
Long currentId = SnowFlakeUtils.getInstance().nextId();
|
Long currentCode = CodeGenerateUtils.getInstance().genCode();
|
||||||
Assert.assertFalse(existsSnowFlakeId.contains(currentId));
|
Assert.assertFalse(existsCode.contains(currentCode));
|
||||||
existsSnowFlakeId.add(currentId);
|
existsCode.add(currentCode);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.dao.upgrade;
|
||||||
import org.apache.dolphinscheduler.common.Constants;
|
import org.apache.dolphinscheduler.common.Constants;
|
||||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
|
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
|
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||||
|
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
|
|
@ -110,7 +110,7 @@ public class ProcessDefinitionDao {
|
||||||
processDefinition.setId(rs.getInt(1));
|
processDefinition.setId(rs.getInt(1));
|
||||||
long code = rs.getLong(2);
|
long code = rs.getLong(2);
|
||||||
if (code == 0L) {
|
if (code == 0L) {
|
||||||
code = SnowFlakeUtils.getInstance().nextId();
|
code = CodeGenerateUtils.getInstance().genCode();
|
||||||
}
|
}
|
||||||
processDefinition.setCode(code);
|
processDefinition.setCode(code);
|
||||||
processDefinition.setVersion(Constants.VERSION_FIRST);
|
processDefinition.setVersion(Constants.VERSION_FIRST);
|
||||||
|
|
|
||||||
|
|
@ -17,8 +17,8 @@
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.dao.upgrade;
|
package org.apache.dolphinscheduler.dao.upgrade;
|
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
|
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
|
|
||||||
|
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
import java.sql.PreparedStatement;
|
import java.sql.PreparedStatement;
|
||||||
|
|
@ -51,7 +51,7 @@ public class ProjectDao {
|
||||||
Integer id = rs.getInt(1);
|
Integer id = rs.getInt(1);
|
||||||
long code = rs.getLong(2);
|
long code = rs.getLong(2);
|
||||||
if (code == 0L) {
|
if (code == 0L) {
|
||||||
code = SnowFlakeUtils.getInstance().nextId();
|
code = CodeGenerateUtils.getInstance().genCode();
|
||||||
}
|
}
|
||||||
projectMap.put(id, code);
|
projectMap.put(id, code);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,11 +25,11 @@ import org.apache.dolphinscheduler.common.enums.TaskType;
|
||||||
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
|
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
|
||||||
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||||
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
|
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
|
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.SchemaUtils;
|
import org.apache.dolphinscheduler.common.utils.SchemaUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.ScriptRunner;
|
import org.apache.dolphinscheduler.common.utils.ScriptRunner;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
|
|
||||||
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
|
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
|
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
|
||||||
|
|
@ -673,7 +673,7 @@ public abstract class UpgradeDao {
|
||||||
String name = task.get("name").asText();
|
String name = task.get("name").asText();
|
||||||
taskDefinitionLog.setName(name);
|
taskDefinitionLog.setName(name);
|
||||||
taskDefinitionLog.setWorkerGroup(task.get("workerGroup").asText());
|
taskDefinitionLog.setWorkerGroup(task.get("workerGroup").asText());
|
||||||
long taskCode = SnowFlakeUtils.getInstance().nextId();
|
long taskCode = CodeGenerateUtils.getInstance().genCode();
|
||||||
taskDefinitionLog.setCode(taskCode);
|
taskDefinitionLog.setCode(taskCode);
|
||||||
taskDefinitionLog.setVersion(Constants.VERSION_FIRST);
|
taskDefinitionLog.setVersion(Constants.VERSION_FIRST);
|
||||||
taskDefinitionLog.setProjectCode(processDefinition.getProjectCode());
|
taskDefinitionLog.setProjectCode(processDefinition.getProjectCode());
|
||||||
|
|
|
||||||
|
|
@ -402,6 +402,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
|
||||||
protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0
|
protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0
|
||||||
protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0
|
protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0
|
||||||
prometheus client_java(simpleclient) 0.12.0: https://github.com/prometheus/client_java, Apache 2.0
|
prometheus client_java(simpleclient) 0.12.0: https://github.com/prometheus/client_java, Apache 2.0
|
||||||
|
snowflake snowflake-2010: https://github.com/twitter-archive/snowflake/tree/snowflake-2010, Apache 2.0
|
||||||
|
|
||||||
========================================================================
|
========================================================================
|
||||||
BSD licenses
|
BSD licenses
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
Copyright 2010-2012 Twitter, Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
|
||||||
|
file except in compliance with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software distributed
|
||||||
|
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||||
|
CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||||
|
specific language governing permissions and limitations under the License.
|
||||||
|
|
@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.common.enums.RunMode;
|
||||||
import org.apache.dolphinscheduler.common.enums.TaskDependType;
|
import org.apache.dolphinscheduler.common.enums.TaskDependType;
|
||||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||||
import org.apache.dolphinscheduler.common.enums.WarningType;
|
import org.apache.dolphinscheduler.common.enums.WarningType;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||||
import org.apache.dolphinscheduler.dao.entity.Project;
|
import org.apache.dolphinscheduler.dao.entity.Project;
|
||||||
import org.apache.dolphinscheduler.dao.entity.Queue;
|
import org.apache.dolphinscheduler.dao.entity.Queue;
|
||||||
|
|
@ -150,18 +150,18 @@ public class PythonGatewayServer extends SpringBootServletInitializer {
|
||||||
return taskDefinitionService.genTaskCodeList(genNum);
|
return taskDefinitionService.genTaskCodeList(genNum);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<String, Long> getCodeAndVersion(String projectName, String taskName) throws SnowFlakeUtils.SnowFlakeException {
|
public Map<String, Long> getCodeAndVersion(String projectName, String taskName) throws CodeGenerateUtils.CodeGenerateException {
|
||||||
Project project = projectMapper.queryByName(projectName);
|
Project project = projectMapper.queryByName(projectName);
|
||||||
Map<String, Long> result = new HashMap<>();
|
Map<String, Long> result = new HashMap<>();
|
||||||
// project do not exists, mean task not exists too, so we should directly return init value
|
// project do not exists, mean task not exists too, so we should directly return init value
|
||||||
if (project == null) {
|
if (project == null) {
|
||||||
result.put("code", SnowFlakeUtils.getInstance().nextId());
|
result.put("code", CodeGenerateUtils.getInstance().genCode());
|
||||||
result.put("version", 0L);
|
result.put("version", 0L);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), taskName);
|
TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), taskName);
|
||||||
if (taskDefinition == null) {
|
if (taskDefinition == null) {
|
||||||
result.put("code", SnowFlakeUtils.getInstance().nextId());
|
result.put("code", CodeGenerateUtils.getInstance().genCode());
|
||||||
result.put("version", 0L);
|
result.put("version", 0L);
|
||||||
} else {
|
} else {
|
||||||
result.put("code", taskDefinition.getCode());
|
result.put("code", taskDefinition.getCode());
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.service.process;
|
package org.apache.dolphinscheduler.service.process;
|
||||||
|
|
||||||
|
import static java.util.stream.Collectors.toSet;
|
||||||
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
|
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
|
||||||
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
|
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
|
||||||
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS;
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS;
|
||||||
|
|
@ -27,8 +28,6 @@ import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS
|
||||||
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID;
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID;
|
||||||
import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS;
|
import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS;
|
||||||
|
|
||||||
import static java.util.stream.Collectors.toSet;
|
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.Constants;
|
import org.apache.dolphinscheduler.common.Constants;
|
||||||
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
|
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
|
||||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||||
|
|
@ -50,11 +49,11 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo;
|
||||||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||||
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
|
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
|
||||||
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
|
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
|
|
||||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
|
|
||||||
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
|
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
|
||||||
import org.apache.dolphinscheduler.dao.entity.Command;
|
import org.apache.dolphinscheduler.dao.entity.Command;
|
||||||
import org.apache.dolphinscheduler.dao.entity.DagData;
|
import org.apache.dolphinscheduler.dao.entity.DagData;
|
||||||
|
|
@ -2290,8 +2289,8 @@ public class ProcessService {
|
||||||
taskDefinitionLog.setCreateTime(now);
|
taskDefinitionLog.setCreateTime(now);
|
||||||
if (taskDefinitionLog.getCode() == 0) {
|
if (taskDefinitionLog.getCode() == 0) {
|
||||||
try {
|
try {
|
||||||
taskDefinitionLog.setCode(SnowFlakeUtils.getInstance().nextId());
|
taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode());
|
||||||
} catch (SnowFlakeException e) {
|
} catch (CodeGenerateException e) {
|
||||||
logger.error("Task code get error, ", e);
|
logger.error("Task code get error, ", e);
|
||||||
return Constants.DEFINITION_FAILURE;
|
return Constants.DEFINITION_FAILURE;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
Copyright 2010-2012 Twitter, Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
|
||||||
|
file except in compliance with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software distributed
|
||||||
|
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||||
|
CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||||
|
specific language governing permissions and limitations under the License.
|
||||||
Loading…
Reference in New Issue