natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Feature][JsonSplit] Fix api bug (#5102) 

* modify checkDAGRing and ProcessService method

* merge

* modify dagRing

* modify process instance for project home page

* fix save process bug

* codeStyle

* Fix logical bug in saving process definition

* codeSytle

* Fix bug in interface of  queryProcessDefinitionList

* codeSytle

* Fix api bug"

Co-authored-by: JinyLeeChina <297062848@qq.com>
This commit is contained in:
JinyLeeChina 2021-03-23 10:44:25 +08:00 committed by GitHub
parent 010b49c933
commit d99934d0da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 80 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java
View File

@ -43,13 +43,14 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
@ -148,8 +149,8 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
}
}
Project project = projectMapper.selectById(projectId);
Long[] projectCodeArray = getProjectCodesArrays(loginUser, project.getCode());
Long[] projectCodeArray = projectId == 0 ? getProjectCodesArrays(loginUser)
: new Long[] { projectMapper.selectById(projectId).getCode() };
List<ExecuteStatusCount> processInstanceStateCounts =
instanceStateCounter.apply(start, end, projectCodeArray);
@ -172,9 +173,12 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
@Override
public Map<String, Object> countDefinitionByUser(User loginUser, int projectId) {
Map<String, Object> result = new HashMap<>();
Project project = projectMapper.selectById(projectId);
Long[] projectCodeArray = getProjectCodesArrays(loginUser, project.getCode());
boolean checkProject = checkProject(loginUser, projectId, result);
if (!checkProject) {
return result;
}
Long[] projectCodeArray = projectId == 0 ? getProjectCodesArrays(loginUser)
: new Long[] { projectMapper.selectById(projectId).getCode() };
List<DefinitionGroupByUser> defineGroupByUsers = processDefinitionMapper.countDefinitionGroupByUser(
loginUser.getId(), projectCodeArray, isAdmin(loginUser));
@ -203,8 +207,6 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
return result;
}
Project project = projectMapper.selectById(projectId);
/**
* find all the task lists in the project under the user
* statistics based on task status execution, failure, completion, wait, total
@ -226,7 +228,8 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
}
}
Long[] projectCodeArray = getProjectCodesArrays(loginUser, project.getCode());
Long[] projectCodeArray = projectId == 0 ? getProjectCodesArrays(loginUser)
: new Long[] { projectMapper.selectById(projectId).getCode() };
// count normal command state
Map<CommandType, Integer> normalCountCommandCounts = commandMapper.countCommandState(loginUser.getId(), start, end, projectCodeArray)
.stream()
@ -249,15 +252,14 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
return result;
}
private Long[] getProjectCodesArrays(User loginUser, Long projectCode) {
List<Long> projectCodes = new ArrayList<>();
if (projectCode != 0) {
projectCodes.add(projectCode);
} else if (loginUser.getUserType() == UserType.GENERAL_USER) {
projectCodes = processService.getProjectIdListHavePerm(loginUser.getId());
if (projectCodes.isEmpty()) {
projectCodes.add(0L);
}
private Long[] getProjectCodesArrays(User loginUser) {
List<Project> projectList = projectMapper.queryRelationProjectListByUserId(
loginUser.getUserType() == UserType.ADMIN_USER ? 0 : loginUser.getId());
Set<Long> projectCodes = new HashSet<>();
projectList.forEach(project -> projectCodes.add(project.getCode()));
if (loginUser.getUserType() == UserType.GENERAL_USER) {
List<Project> createProjects = projectMapper.queryProjectCreatedByUser(loginUser.getId());
createProjects.forEach(project -> projectCodes.add(project.getCode()));
}
return projectCodes.toArray(new Long[0]);
}

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
View File

@ -259,9 +259,9 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefineId);
IPage<ProcessInstance> processInstanceList =
processInstanceMapper.queryProcessInstanceListPaging(page,
project.getCode(), processDefinition.getCode(), searchVal, executorId, statusArray, host, start, end);
IPage<ProcessInstance> processInstanceList = processInstanceMapper.queryProcessInstanceListPaging(page,
project.getCode(), processDefinition == null ? 0L : processDefinition.getCode(), searchVal,
executorId, statusArray, host, start, end);
List<ProcessInstance> processInstances = processInstanceList.getRecords();

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java
View File

@ -444,7 +444,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
return Constants.ALL_PERMISSIONS;
}
ProjectUser projectUser = projectUserMapper.queryProjectRelation(project.getCode(), user.getId());
ProjectUser projectUser = projectUserMapper.queryProjectRelation(project.getId(), user.getId());
if (projectUser == null) {
return 0;

22
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
View File

@ -271,25 +271,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
return result;
}
TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(taskCode, version);
taskDefinition.setVersion(version);
taskDefinition.setCode(taskCode);
taskDefinition.setName(taskDefinitionLog.getName());
taskDefinition.setDescription(taskDefinitionLog.getDescription());
taskDefinition.setProjectCode(taskDefinitionLog.getProjectCode());
taskDefinition.setUserId(loginUser.getId());
taskDefinition.setTaskType(taskDefinitionLog.getTaskType());
taskDefinition.setTaskParams(taskDefinitionLog.getTaskParams());
taskDefinition.setFlag(taskDefinitionLog.getFlag());
taskDefinition.setTaskPriority(taskDefinitionLog.getTaskPriority());
taskDefinition.setWorkerGroup(taskDefinitionLog.getWorkerGroup());
taskDefinition.setFailRetryTimes(taskDefinitionLog.getFailRetryTimes());
taskDefinition.setFailRetryInterval(taskDefinitionLog.getFailRetryInterval());
taskDefinition.setTimeoutFlag(taskDefinitionLog.getTimeoutFlag());
taskDefinition.setTimeoutNotifyStrategy(taskDefinitionLog.getTimeoutNotifyStrategy());
taskDefinition.setTimeout(taskDefinitionLog.getTimeout());
taskDefinition.setUpdateTime(new Date());
taskDefinition.setResourceIds(taskDefinitionLog.getResourceIds());
taskDefinitionMapper.updateById(taskDefinition);
taskDefinitionLog.setUserId(loginUser.getId());
taskDefinitionLog.setUpdateTime(new Date());
taskDefinitionMapper.updateById(taskDefinitionLog);
result.put(Constants.DATA_LIST, taskCode);
putMsg(result, Status.SUCCESS);
return result;

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
View File

@ -525,10 +525,10 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
//if the selected projectIds are empty, delete all items associated with the user
projectUserMapper.deleteProjectRelation(0L, userId);
//if the selected projectIds are empty, delete all items associated with the user
if (check(result, StringUtils.isEmpty(projectIds), Status.SUCCESS)) {
projectUserMapper.deleteProjectRelation(0, userId);
return result;
}

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java
View File

@ -116,7 +116,7 @@ public class ProjectServiceTest {
@Test
public void testCheckProjectAndAuth() {
Mockito.when(projectUserMapper.queryProjectRelation(1L, 1)).thenReturn(getProjectUser());
Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser());
User loginUser = getLoginUser();
Map<String, Object> result = projectService.checkProjectAndAuth(loginUser, null, projectName);

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
View File

@ -305,7 +305,7 @@ public class UsersServiceTest {
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
when(projectUserMapper.deleteProjectRelation(Mockito.anyLong(), Mockito.anyInt())).thenReturn(1);
when(projectUserMapper.deleteProjectRelation(Mockito.anyInt(), Mockito.anyInt())).thenReturn(1);
result = usersService.grantProject(loginUser, 1, projectIds);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));

15
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
View File

@ -137,12 +137,6 @@ public class TaskNode {
*/
private String workerGroup;
/**
* worker group id
*/
private Integer workerGroupId;
/**
* task time out
*/
@ -325,14 +319,6 @@ public class TaskNode {
this.conditionResult = conditionResult;
}
public Integer getWorkerGroupId() {
return workerGroupId;
}
public void setWorkerGroupId(Integer workerGroupId) {
this.workerGroupId = workerGroupId;
}
public int getDelayTime() {
return delayTime;
}
@ -405,7 +391,6 @@ public class TaskNode {
+ ", conditionResult='" + conditionResult + '\''
+ ", taskInstancePriority=" + taskInstancePriority
+ ", workerGroup='" + workerGroup + '\''
+ ", workerGroupId=" + workerGroupId
+ ", timeout='" + timeout + '\''
+ ", delayTime=" + delayTime
+ '}';

14
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectUser.java
View File

@ -34,12 +34,9 @@ public class ProjectUser {
@TableField("user_id")
private int userId;
@TableField(exist = false)
@TableField("project_id")
private int projectId;
@TableField("project_code")
private long projectCode;
/**
* project name
*/
@ -63,14 +60,6 @@ public class ProjectUser {
@TableField("update_time")
private Date updateTime;
public long getProjectCode() {
return projectCode;
}
public void setProjectCode(long projectCode) {
this.projectCode = projectCode;
}
public int getId() {
return id;
}
@ -141,7 +130,6 @@ public class ProjectUser {
+ "id=" + id
+ ", userId=" + userId
+ ", projectId=" + projectId
+ ", projectCode=" + projectCode
+ ", projectName='" + projectName + '\''
+ ", userName='" + userName + '\''
+ ", perm=" + perm

7
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java
View File

@ -85,6 +85,13 @@ public interface ProjectMapper extends BaseMapper<Project> {
*/
List<Project> queryAuthedProjectListByUserId(@Param("userId") int userId);
/**
* query relation project list by userId
* @param userId userId
* @return project list
*/
List<Project> queryRelationProjectListByUserId(@Param("userId") int userId);
/**
* query project except userId
* @param userId userId

14
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapper.java
View File

@ -17,9 +17,11 @@
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Param;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* project user mapper interface
*/
@ -27,19 +29,21 @@ public interface ProjectUserMapper extends BaseMapper<ProjectUser> {
/**
* delte prject user relation
* @param projectCode projectCode
*
* @param projectId projectId
* @param userId userId
* @return delete result
*/
int deleteProjectRelation(@Param("projectCode") Long projectCode,
int deleteProjectRelation(@Param("projectId") int projectId,
@Param("userId") int userId);
/**
* query project relation
* @param projectCode projectCode
*
* @param projectId projectId
* @param userId userId
* @return project user relation
*/
ProjectUser queryProjectRelation(@Param("projectCode") Long projectCode,
ProjectUser queryProjectRelation(@Param("projectId") int projectId,
@Param("userId") int userId);
}

2
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.xml
View File

@ -29,7 +29,7 @@
</foreach>
</if>
<if test="startTime != null and endTime != null">
and cmd.startTime <![CDATA[ >= ]]> #{startTime} and cmd.update_time <![CDATA[ <= ]]> #{endTime}
and cmd.start_time <![CDATA[ >= ]]> #{startTime} and cmd.update_time <![CDATA[ <= ]]> #{endTime}
</if>
group by cmd.command_type
</select>

29
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
View File

@ -19,7 +19,6 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ProjectMapper">
<sql id="baseSql">
id, name, code, description, user_id, flag, create_time, update_time
</sql>
<sql id="baseSqlV2">
@ -83,9 +82,9 @@
join t_ds_user u on u.id=p.user_id
where 1=1
<if test="userId != 0">
and p.code in
(select project_code from t_ds_relation_project_user where user_id=#{userId}
union select id as project_code from t_ds_project where user_id=#{userId}
and p.id in
(select project_id from t_ds_relation_project_user where user_id=#{userId}
union select id as project_id from t_ds_project where user_id=#{userId}
)
</if>
<if test="searchName!=null and searchName != ''">
@ -101,6 +100,18 @@
from t_ds_project p,t_ds_relation_project_user rel
where p.id = rel.project_id and rel.user_id= #{userId}
</select>
<select id="queryRelationProjectListByUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project">
select
<include refid="baseSqlV2">
<property name="alias" value="p"/>
</include>
from t_ds_project p left join t_ds_relation_project_user rel
on p.id = rel.project_id
where 1=1
<if test="userId != 0 ">
and rel.user_id= #{userId}
</if>
</select>
<select id="queryProjectExceptUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project">
select
<include refid="baseSql"/>
@ -114,16 +125,18 @@
where user_id = #{userId}
</select>
<select id="queryProjectCreatedAndAuthorizedByUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project">
select * from t_ds_project where id in
select
<include refid="baseSql"/>
from t_ds_project where id in
(select project_id from t_ds_relation_project_user where user_id=#{userId}
union select id as project_id from t_ds_project where user_id=#{userId})
</select>
<select id="queryProjectWithUserByProcessInstanceId" resultType="org.apache.dolphinscheduler.dao.entity.ProjectUser">
select
dp.code projectCode,
dp.name projectName,
u.user_name userName
dp.id project_id,
dp.name project_name,
u.user_name user_name
from t_ds_process_instance di
join t_ds_process_definition dpd on di.process_definition_code = dpd.code
join t_ds_project dp on dpd.project_code = dp.code

8
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapper.xml
View File

@ -19,21 +19,21 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper">
<sql id="baseSql">
id, user_id, project_code, perm, create_time, update_time
id, user_id, project_id, perm, create_time, update_time
</sql>
<delete id="deleteProjectRelation">
delete from t_ds_relation_project_user
where 1=1
and user_id = #{userId}
<if test="projectCode != 0 ">
and project_code = #{projectCode}
<if test="projectId != 0 ">
and project_id = #{projectId}
</if>
</delete>
<select id="queryProjectRelation" resultType="org.apache.dolphinscheduler.dao.entity.ProjectUser">
select
<include refid="baseSql"/>
from t_ds_relation_project_user
where project_code = #{projectCode}
where project_id = #{projectId}
and user_id = #{userId}
limit 1
</select>

5
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java
View File

@ -52,7 +52,6 @@ public class ProjectUserMapperTest {
private ProjectUser insertOne(){
//insertOne
ProjectUser projectUser = new ProjectUser();
projectUser.setProjectCode(1010L);
projectUser.setProjectId(1010);
projectUser.setUserId(111);
projectUserMapper.insert(projectUser);
@ -101,7 +100,7 @@ public class ProjectUserMapperTest {
ProjectUser projectUser = insertOne();
int delete = projectUserMapper.deleteProjectRelation(projectUser.getProjectCode(), projectUser.getUserId());
int delete = projectUserMapper.deleteProjectRelation(projectUser.getProjectId(), projectUser.getUserId());
assertThat(delete,greaterThanOrEqualTo(1));
}
@ -112,7 +111,7 @@ public class ProjectUserMapperTest {
@Test
public void testQueryProjectRelation() {
ProjectUser projectUser = insertOne();
ProjectUser projectUser1 = projectUserMapper.queryProjectRelation(projectUser.getProjectCode(), projectUser.getUserId());
ProjectUser projectUser1 = projectUserMapper.queryProjectRelation(projectUser.getProjectId(), projectUser.getUserId());
Assert.assertNotEquals(projectUser1, null);
}

6
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
View File

@ -2222,7 +2222,6 @@ public class ProcessService {
* update task definition
*/
public int updateTaskDefinition(User operator, Long projectCode, TaskNode taskNode, TaskDefinition taskDefinition) {
Integer version = taskDefinitionLogMapper.queryMaxVersionForDefinition(taskDefinition.getCode());
Date now = new Date();
taskDefinition.setProjectCode(projectCode);
@ -2243,7 +2242,7 @@ public class ProcessService {
taskDefinition.setName(taskNode.getName());
taskDefinition.setDescription(taskNode.getDesc());
taskDefinition.setTaskType(TaskType.of(taskNode.getType()));
taskDefinition.setTaskParams(taskNode.getParams());
taskDefinition.setTaskParams(TaskType.of(taskNode.getType()) == TaskType.DEPENDENT ? taskNode.getDependence() : taskNode.getParams());
taskDefinition.setFlag(taskNode.isForbidden() ? Flag.NO : Flag.YES);
taskDefinition.setTaskPriority(taskNode.getTaskInstancePriority());
taskDefinition.setWorkerGroup(taskNode.getWorkerGroup());
@ -2508,7 +2507,8 @@ public class ProcessService {
v.setRunFlag(taskDefinitionLog.getFlag() == Flag.YES ? Constants.FLOWNODE_RUN_FLAG_NORMAL : Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
v.setMaxRetryTimes(taskDefinitionLog.getFailRetryTimes());
v.setRetryInterval(taskDefinitionLog.getFailRetryInterval());
v.setParams(taskDefinitionLog.getTaskParams());
v.setParams(taskDefinitionLog.getTaskType() == TaskType.DEPENDENT ? "" : taskDefinitionLog.getTaskParams());
v.setDependence(taskDefinitionLog.getTaskType() == TaskType.DEPENDENT ? taskDefinitionLog.getTaskParams() : "");
v.setTaskInstancePriority(taskDefinitionLog.getTaskPriority());
v.setWorkerGroup(taskDefinitionLog.getWorkerGroup());
v.setTimeout(JSONUtils.toJsonString(new TaskTimeoutParameter(taskDefinitionLog.getTimeoutFlag() == TimeoutFlag.OPEN,

2
sql/dolphinscheduler_mysql.sql
View File

@ -696,7 +696,7 @@ DROP TABLE IF EXISTS `t_ds_relation_project_user`;
CREATE TABLE `t_ds_relation_project_user` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
`user_id` int(11) NOT NULL COMMENT 'user id',
`project_code` bigint(20) DEFAULT NULL COMMENT 'project code',
`project_id` int(11) DEFAULT NULL COMMENT 'project id',
`perm` int(11) DEFAULT '1' COMMENT 'limits of authority',
`create_time` datetime DEFAULT NULL COMMENT 'create time',
`update_time` datetime DEFAULT NULL COMMENT 'update time',