natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix

This commit is contained in:
gongzijian 2019-05-21 14:57:29 +08:00
parent 7b1f95d067 0cc61f5d8d
commit 0cfa35da4e
10 changed files with 73 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java
View File

@ -701,17 +701,19 @@ public class ResourcesService extends BaseService {
if (checkAdmin(loginUser, result)) {
return result;
}
List<Resource> resourceList = resourcesMapper.queryResourceExceptUserId(userId);
Set<Resource> resourceSet = null;
List<Object> list ;
if (resourceList != null && resourceList.size() > 0) {
resourceSet = new HashSet<>(resourceList);
Set<Resource> resourceSet = new HashSet<>(resourceList);
List<Resource> authedResourceList = resourcesMapper.queryAuthorizedResourceList(userId);
getAuthorizedResourceList(resourceSet, authedResourceList);
list = new ArrayList<>(resourceSet);
}else {
list = new ArrayList<>(0);
}
result.put(Constants.DATA_LIST, new ArrayList<>(resourceSet));
result.put(Constants.DATA_LIST, list);
putMsg(result,Status.SUCCESS);
return result;
}

61
escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java
View File

@ -20,6 +20,7 @@ import cn.escheduler.common.Constants;
import cn.escheduler.common.enums.*;
import cn.escheduler.common.model.DateInterval;
import cn.escheduler.common.model.TaskNode;
import cn.escheduler.common.process.Property;
import cn.escheduler.common.queue.ITaskQueue;
import cn.escheduler.common.queue.TaskQueueFactory;
import cn.escheduler.common.task.subprocess.SubProcessParameters;
@ -41,6 +42,7 @@ import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import java.util.*;
import java.util.stream.Collectors;
import static cn.escheduler.common.Constants.*;
import static cn.escheduler.dao.datasource.ConnectionFactory.getMapper;
@ -689,41 +691,62 @@ public class ProcessDao extends AbstractBaseDao {
* handle sub work process instance, update relation table and command parameters
* set sub work process flag, extends parent work process command parameters.
*/
public ProcessInstance setSubProcessParam(ProcessInstance processInstance){
String cmdParam = processInstance.getCommandParam();
public ProcessInstance setSubProcessParam(ProcessInstance subProcessInstance){
String cmdParam = subProcessInstance.getCommandParam();
if(StringUtils.isEmpty(cmdParam)){
return processInstance;
return subProcessInstance;
}
Map<String, String> paramMap = JSONUtils.toMap(cmdParam);
// write sub process id into cmd param.
if(paramMap.containsKey(CMDPARAM_SUB_PROCESS)
&& CMDPARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMDPARAM_SUB_PROCESS))){
paramMap.remove(CMDPARAM_SUB_PROCESS);
paramMap.put(CMDPARAM_SUB_PROCESS, String.valueOf(processInstance.getId()));
processInstance.setCommandParam(JSONUtils.toJson(paramMap));
processInstance.setIsSubProcess(Flag.YES);
this.saveProcessInstance(processInstance);
paramMap.put(CMDPARAM_SUB_PROCESS, String.valueOf(subProcessInstance.getId()));
subProcessInstance.setCommandParam(JSONUtils.toJson(paramMap));
subProcessInstance.setIsSubProcess(Flag.YES);
this.saveProcessInstance(subProcessInstance);
}
// copy parent instance user def params to sub process..
String parentInstanceId = paramMap.get(CMDPARAM_SUB_PROCESS_PARENT_INSTANCE_ID);
if(StringUtils.isNotEmpty(parentInstanceId)){
ProcessInstance parentInstance = findProcessInstanceDetailById(Integer.parseInt(parentInstanceId));
if(parentInstance != null){
processInstance.setGlobalParams(parentInstance.getGlobalParams());
this.saveProcessInstance(processInstance);
subProcessInstance.setGlobalParams(
joinGlobalParams(parentInstance.getGlobalParams(), subProcessInstance.getGlobalParams()));
this.saveProcessInstance(subProcessInstance);
}else{
logger.error("sub process command params error, cannot find parent instance: {} ", cmdParam);
}
}
ProcessInstanceMap processInstanceMap = JSONUtils.parseObject(cmdParam, ProcessInstanceMap.class);
if(processInstanceMap == null || processInstanceMap.getParentProcessInstanceId() == 0){
return processInstance;
return subProcessInstance;
}
// update sub process id to process map table
processInstanceMap.setProcessInstanceId(processInstance.getId());
processInstanceMap.setProcessInstanceId(subProcessInstance.getId());
this.updateWorkProcessInstanceMap(processInstanceMap);
return processInstance;
return subProcessInstance;
}
/**
* join parent global params into sub process.
* only the keys doesn't in sub process global would be joined.
* @param parentGlobalParams
* @param subGlobalParams
* @return
*/
private String joinGlobalParams(String parentGlobalParams, String subGlobalParams){
List<Property> parentPropertyList = JSONUtils.toList(parentGlobalParams, Property.class);
List<Property> subPropertyList = JSONUtils.toList(subGlobalParams, Property.class);
Map<String,String> subMap = subPropertyList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue));
for(Property parent : parentPropertyList){
if(!subMap.containsKey(parent.getProp())){
subPropertyList.add(parent);
}
}
return JSONUtils.toJson(subPropertyList);
}
/**
@ -898,7 +921,11 @@ public class ProcessDao extends AbstractBaseDao {
taskInstance.setFlag(Flag.NO);
updateTaskInstance(taskInstance);
// crate new task instance
taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1 );
if(taskInstance.getState() != ExecutionStatus.NEED_FAULT_TOLERANCE){
taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1 );
}
taskInstance.setEndTime(null);
taskInstance.setStartTime(new Date());
taskInstance.setFlag(Flag.YES);
taskInstance.setHost(null);
taskInstance.setId(0);
@ -1526,6 +1553,14 @@ public class ProcessDao extends AbstractBaseDao {
}
public void selfFaultTolerant(int ... states){
List<ProcessInstance> processInstanceList = processInstanceMapper.listByStatus(states);
for (ProcessInstance processInstance:processInstanceList){
selfFaultTolerant(processInstance);
}
}
@Transactional(value = "TransactionManager",rollbackFor = Exception.class)
public void selfFaultTolerant(ProcessInstance processInstance){

6
escheduler-dao/src/main/java/cn/escheduler/dao/model/TaskInstance.java
View File

@ -422,8 +422,12 @@ public class TaskInstance {
if(this.isSubProcess()){
return false;
}
return (this.getState().typeIsFailure()
if(this.getState() == ExecutionStatus.NEED_FAULT_TOLERANCE){
return true;
}else {
return (this.getState().typeIsFailure()
&& this.getRetryTimes() < this.getMaxRetryTimes());
}
}
public void setDependency(String dependency) {

2
escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java
View File

@ -869,7 +869,7 @@ public class MasterExecThread implements Runnable {
}
Date now = new Date();
long runningTime = DateUtils.differMs(now, processInstance.getStartTime());
long runningTime = DateUtils.diffMin(now, processInstance.getStartTime());
if(runningTime > processInstance.getTimeout()){
return true;

2
escheduler-server/src/main/java/cn/escheduler/server/utils/AlertManager.java
View File

@ -175,7 +175,7 @@ public class AlertManager {
alert.setContent(content);
alert.setAlertType(AlertType.EMAIL);
alert.setCreateTime(new Date());
alert.setAlertGroupId(processInstance.getWarningGroupId());
alert.setAlertGroupId(processInstance.getWarningGroupId() == null ? 1:processInstance.getWarningGroupId());
alert.setReceivers(processInstance.getProcessDefinition().getReceivers());
alert.setReceiversCc(processInstance.getProcessDefinition().getReceiversCc());

4
escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java
View File

@ -123,9 +123,9 @@ public class ZKMasterClient extends AbstractZKClient {
// register master
this.registMaster();
// check if fault tolerance is required
// check if fault tolerance is required，failure and tolerance
if (getActiveMasterNum() == 1) {
processDao.selfFaultTolerant(ExecutionStatus.RUNNING_EXEUTION.ordinal());
processDao.selfFaultTolerant(ExecutionStatus.RUNNING_EXEUTION.ordinal(),ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal());
}
}

5
escheduler-ui/.env
View File

@ -1,7 +1,6 @@
# 后端接口地址
#API_BASE = http://192.168.220.154:12345
API_BASE = http://192.168.221.188:12345
# 后端接口地址11
API_BASE = http://192.168.xx.xx:12345
# 本地开发如需ip访问项目把"#"号去掉
#DEV_HOST = 192.168.xx.xx

8
escheduler-ui/install-escheduler-ui.sh
View File

@ -88,9 +88,9 @@ eschedulerConf(){
proxy_set_header remote_addr $E_remote_addr;
proxy_set_header X-Forwarded-For $E_proxy_add_x_forwarded_for;
proxy_http_version 1.1;
proxy_connect_timeout 4s;
proxy_read_timeout 30s;
proxy_send_timeout 12s;
proxy_connect_timeout 300s;
proxy_read_timeout 300s;
proxy_send_timeout 300s;
proxy_set_header Upgrade $E_http_upgrade;
proxy_set_header Connection "upgrade";
}
@ -184,7 +184,7 @@ if [[ $version -eq 6 ]]; then
fi
# centos 7
if [[ $version -eq 7 ]]; then
if [[ $version -eq 7 ]]; then
centos7
fi

4
escheduler-ui/src/js/module/i18n/locale/zh_CN.js
View File

@ -130,7 +130,7 @@ export default {
'Please enter database name': '请输入数据库名',
'jdbc connect parameters': 'jdbc连接参数',
'Test Connect': '测试连接',
'Please enter resource name': '请输入资源名称',
'Please enter resource name': '请输入数据源名称',
'Please enter IP/hostname': '请输入IP/主机名',
'jdbc connection parameters is not a correct JSON format': 'jdbc连接参数不是一个正确的JSON格式',
'#': '编号',
@ -419,7 +419,7 @@ export default {
'Create token': '创建令牌',
'Edit token': '编辑令牌',
'Please enter the IP address separated by commas': '请输入IP地址多个用逗号隔开',
'Note: Multiple IP addresses have been comma separated': '注意：多个IP地址已逗号分割',
'Note: Multiple IP addresses have been comma separated': '注意：多个IP地址以逗号分割',
'Failure time': '失效时间',
'User': '用户',
'Please enter token': '请输入令牌',

2
script/env/.escheduler_env.sh vendored
View File

@ -6,4 +6,4 @@ export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/opt/soft/java
export HIVE_HOME=/opt/soft/hive
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH