createDirectory(User loginUser,
- String name,
- ResourceType type,
- int pid,
- String currentDir);
+ void createDirectory(CreateDirectoryRequest createDirectoryRequest);
/**
- * create resource
- *
- * @param loginUser login user
- * @param name alias
- * @param type type
- * @param file file
- * @param currentDir current directory
- * @return create result code
+ * Rename the directory in the resource storage, if the origin directory not exists or the new directory already exists will throw exception.
+ * If the origin directory is empty will only update the directory name.
+ *
If the origin directory is not empty will move all the files and directories to the new directory.
+ *
After update the origin directory will be deleted.
*/
- Result uploadResource(User loginUser,
- String name,
- ResourceType type,
- MultipartFile file,
- String currentDir);
+ void renameDirectory(RenameDirectoryRequest renameDirectoryRequest);
/**
- * update resource
- * @param loginUser login user
- * @param name name
- * @param type resource type
- * @param file resource file
- * @return update result code
+ * Upload a new file to the resource storage, if the file already exists will throw exception
*/
- Result updateResource(User loginUser,
- String fullName,
- String tenantCode,
- String name,
- ResourceType type,
- MultipartFile file);
+ void createFile(CreateFileRequest createFileRequest);
/**
- * query resources list paging
- *
- * @param loginUser login user
- * @param type resource type
- * @param searchVal search value
- * @param pageNo page number
- * @param pageSize page size
- * @return resource list page
+ * Update the file in the resource storage, if the origin file not exists or the new file already exists will throw exception.
+ * If the new file is empty will only update the file name.
+ *
If the new file is not empty will update the file content and name.
+ *
After update the origin file will be deleted.
*/
- Result> queryResourceListPaging(User loginUser, String fullName, String resTenantCode,
- ResourceType type, String searchVal, Integer pageNo,
- Integer pageSize);
+ void updateFile(UpdateFileRequest updateFileRequest);
/**
- * query resource list
- *
- * @param loginUser login user
- * @param type resource type
- * @return resource list
+ * Rename the file in the resource storage, if the origin file not exists or the new file already exists will throw exception.
*/
- Map queryResourceList(User loginUser, ResourceType type, String fullName);
+ void renameFile(RenameFileRequest renameFileRequest);
/**
- * query resource list by program type
- *
- * @param loginUser login user
- * @param type resource type
- * @return resource list
+ * Create a new file in the resource storage, if the file already exists will throw exception.
+ * Different with {@link ResourcesService#createFile(CreateFileRequest)} this method will create a new file with the given content.
*/
- Result queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType);
+ void createFileFromContent(CreateFileFromContentRequest createFileFromContentRequest);
/**
- * delete resource
- *
- * @param loginUser login user
- * @return delete result code
- * @throws IOException exception
+ * Update the file content.
*/
- Result delete(User loginUser, String fullName, String tenantCode) throws IOException;
+ void updateFileFromContent(UpdateFileFromContentRequest updateFileContentRequest);
/**
- * verify resource by name and type
- * @param loginUser login user
- * @param fullName resource full name
- * @param type resource type
- * @return true if the resource name not exists, otherwise return false
+ * Paging query resource items.
+ * If the login user is not admin will only query the resource items that under the user's tenant.
+ *
If the login user is admin and {@link PagingResourceItemRequest##resourceAbsolutePath} is null will return all the resource items.
*/
- Result verifyResourceName(String fullName, ResourceType type, User loginUser);
+ PageInfo pagingResourceItem(PagingResourceItemRequest pagingResourceItemRequest);
/**
- * verify resource by file name
- * @param fileName resource file name
- * @param type resource type
- * @return true if the resource file name, otherwise return false
+ * Query the resource file items by the given resource type and program type.
*/
- Result queryResourceByFileName(User loginUser, String fileName, ResourceType type, String resTenantCode);
+ List queryResourceFiles(User loginUser, ResourceType type);
/**
- * view resource file online
- *
- * @param skipLineNum skip line number
- * @param limit limit
- * @param fullName fullName
- * @return resource content
+ * Delete the resource item.
+ * If the resource item is a directory will delete all the files and directories under the directory.
+ *
If the resource item is a file will delete the file.
+ *
If the resource item not exists will throw exception.
*/
- Result readResource(User loginUser, String fullName, String tenantCode, int skipLineNum, int limit);
+ void delete(DeleteResourceRequest deleteResourceRequest);
/**
- * create resource file online
- *
- * @param loginUser login user
- * @param type resource type
- * @param fileName file name
- * @param fileSuffix file suffix
- * @param content content
- * @return create result code
+ * Fetch the file content.
*/
- Result createResourceFile(User loginUser, ResourceType type, String fileName, String fileSuffix,
- String content, String currentDirectory);
+ FetchFileContentResponse fetchResourceFileContent(FetchFileContentRequest fetchFileContentRequest);
- /**
- * create or update resource.
- * If the folder is not already created, it will be ignored and directly create the new file
- *
- * @param userName user who create or update resource
- * @param fullName The fullname of resource.Includes path and suffix.
- * @param resourceContent content of resource
- */
- StorageEntity createOrUpdateResource(String userName, String fullName, String resourceContent) throws Exception;
-
- /**
- * updateProcessInstance resource
- *
- * @param loginUser login user
- * @param fullName full name
- * @param tenantCode tenantCode
- * @param content content
- * @return update result cod
- */
- Result updateResourceContent(User loginUser, String fullName, String tenantCode,
- String content);
-
- /**
- * download file
- *
- * @return resource content
- * @throws IOException exception
- */
- org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException;
+ void downloadResource(HttpServletResponse response, DownloadFileRequest downloadFileRequest);
/**
* Get resource by given resource type and file name.
@@ -202,21 +120,6 @@ public interface ResourcesService {
*/
StorageEntity queryFileStatus(String userName, String fileName) throws Exception;
- /**
- * delete DATA_TRANSFER data in resource center
- *
- * @param loginUser user who query resource
- * @param days number of days
- */
- DeleteDataTransferResponse deleteDataTransferData(User loginUser, Integer days);
-
- /**
- * get resource base dir
- *
- * @param loginUser login user
- * @param type resource type
- * @return
- */
- Result queryResourceBaseDir(User loginUser, ResourceType type);
+ String queryResourceBaseDir(User loginUser, ResourceType type);
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
index 2c87e4be2e..b85d3912cb 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
@@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.utils.Result;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import java.util.List;
@@ -77,13 +76,6 @@ public interface WorkerGroupService {
*/
Map getWorkerAddressList();
- /**
- * Get task instance's worker group
- * @param taskInstance task instance
- * @return worker group
- */
- String getTaskWorkerGroup(TaskInstance taskInstance);
-
/**
* Query worker group by process definition codes
* @param processDefinitionCodeList processDefinitionCodeList
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
index 3e613b11fa..6bb7d228ff 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
@@ -89,8 +89,8 @@ import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse;
import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
-import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
+import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.service.command.CommandService;
import org.apache.dolphinscheduler.service.cron.CronUtils;
@@ -361,7 +361,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
// find out the process definition code
Set processDefinitionCodeSet = new HashSet<>();
taskDefinitions.stream()
- .filter(task -> TaskConstants.TASK_TYPE_SUB_PROCESS.equalsIgnoreCase(task.getTaskType())).forEach(
+ .filter(task -> TaskTypeUtils.isSubWorkflowTask(task.getTaskType())).forEach(
taskDefinition -> processDefinitionCodeSet.add(Long.valueOf(
JSONUtils.getNodeString(taskDefinition.getTaskParams(),
CMD_PARAM_SUB_PROCESS_DEFINE_CODE))));
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
index e6893b04fb..820d35b3b1 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
@@ -34,13 +34,12 @@ import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_DEFINE_NOT_EX
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE;
import static org.apache.dolphinscheduler.common.constants.Constants.COPY_SUFFIX;
import static org.apache.dolphinscheduler.common.constants.Constants.DATA_LIST;
-import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.common.constants.Constants.GLOBAL_PARAMS;
import static org.apache.dolphinscheduler.common.constants.Constants.IMPORT_SUFFIX;
import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOCAL_PARAMS_LIST;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SQL;
+import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters;
import org.apache.dolphinscheduler.api.dto.DagDataSchedule;
import org.apache.dolphinscheduler.api.dto.treeview.Instance;
@@ -109,12 +108,13 @@ import org.apache.dolphinscheduler.dao.model.PageListingResult;
import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionDao;
import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionLogDao;
import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao;
-import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters;
+import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
+import org.apache.dolphinscheduler.plugin.task.sql.SqlTaskChannelFactory;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.dolphinscheduler.service.model.TaskNode;
import org.apache.dolphinscheduler.service.process.ProcessService;
@@ -421,11 +421,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJson);
}
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
- if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
- .taskType(taskDefinitionLog.getTaskType())
- .taskParams(taskDefinitionLog.getTaskParams())
- .dependence(taskDefinitionLog.getDependence())
- .build())) {
+ if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) {
log.error(
"Generate task definition list failed, the given task definition parameter is invalided, taskName: {}, taskDefinition: {}",
taskDefinitionLog.getName(), taskDefinitionLog);
@@ -1386,11 +1382,11 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
sqlParameters.setLocalParams(Collections.emptyList());
taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters));
taskDefinition.setCode(CodeGenerateUtils.genCode());
- taskDefinition.setTaskType(TASK_TYPE_SQL);
+ taskDefinition.setTaskType(SqlTaskChannelFactory.NAME);
taskDefinition.setFailRetryTimes(0);
taskDefinition.setFailRetryInterval(0);
taskDefinition.setTimeoutFlag(TimeoutFlag.CLOSE);
- taskDefinition.setWorkerGroup(DEFAULT_WORKER_GROUP);
+ taskDefinition.setWorkerGroup(WorkerGroupUtils.getDefaultWorkerGroup());
taskDefinition.setTaskPriority(Priority.MEDIUM);
taskDefinition.setEnvironmentCode(-1);
taskDefinition.setTimeout(0);
@@ -1615,13 +1611,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
// check whether the process definition json is normal
for (TaskNode taskNode : taskNodes) {
- if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
- .taskType(taskNode.getType())
- .taskParams(taskNode.getTaskParams())
- .dependence(taskNode.getDependence())
- .switchResult(taskNode.getSwitchResult())
- .build())) {
- log.error("Task node {} parameter invalid.", taskNode.getName());
+ if (!checkTaskParameters(taskNode.getType(), taskNode.getParams())) {
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName());
return result;
}
@@ -1891,7 +1881,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
long subProcessCode = 0L;
// if process is sub process, the return sub id, or sub id=0
- if (taskInstance.isSubProcess()) {
+ if (TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType())) {
TaskDefinition taskDefinition = taskDefinitionMap.get(taskInstance.getTaskCode());
subProcessCode = Long.parseLong(JSONUtils.parseObject(
taskDefinition.getTaskParams()).path(CMD_PARAM_SUB_PROCESS_DEFINE_CODE).asText());
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
index 06e9fd95db..5e5d703017 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
@@ -26,8 +26,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.GLOBAL_PARA
import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS;
import static org.apache.dolphinscheduler.common.constants.Constants.PROCESS_INSTANCE_STATE;
import static org.apache.dolphinscheduler.common.constants.Constants.TASK_LIST;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
+import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters;
import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
import org.apache.dolphinscheduler.api.dto.DynamicSubWorkflowDto;
@@ -77,11 +76,10 @@ import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceMapDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.dao.utils.WorkflowUtils;
-import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
+import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
import org.apache.dolphinscheduler.service.model.TaskNode;
import org.apache.dolphinscheduler.service.process.ProcessService;
@@ -498,7 +496,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId);
}
- if (!taskInstance.isDynamic()) {
+ if (!TaskTypeUtils.isDynamicTask(taskInstance.getTaskType())) {
putMsg(result, Status.TASK_INSTANCE_NOT_DYNAMIC_TASK, taskInstance.getName());
throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId);
}
@@ -548,7 +546,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
*/
private void addDependResultForTaskList(User loginUser, List taskInstanceList) throws IOException {
for (TaskInstance taskInstance : taskInstanceList) {
- if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) {
+ if (TaskTypeUtils.isDependentTask(taskInstance.getTaskType())) {
log.info("DEPENDENT type task instance need to set dependent result, taskCode:{}, taskInstanceId:{}",
taskInstance.getTaskCode(), taskInstance.getId());
// TODO The result of dependent item should not be obtained from the log, waiting for optimization.
@@ -628,9 +626,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
return result;
}
- if (!taskInstance.isSubProcess()) {
- log.warn("Task instance is not {} type instance, projectCode:{}, taskInstanceId:{}.",
- TASK_TYPE_SUB_PROCESS, projectCode, taskId);
+ if (!TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType())) {
putMsg(result, Status.TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE, taskInstance.getName());
return result;
}
@@ -714,11 +710,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
return result;
}
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
- if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
- .taskType(taskDefinitionLog.getTaskType())
- .taskParams(taskDefinitionLog.getTaskParams())
- .dependence(taskDefinitionLog.getDependence())
- .build())) {
+ if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) {
log.error("Task parameters are invalid, taskDefinitionName:{}.", taskDefinitionLog.getName());
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName());
return result;
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java
index d8c568e50a..1f9fe6ff53 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java
@@ -18,9 +18,6 @@
package org.apache.dolphinscheduler.api.service.impl;
import static java.util.stream.Collectors.toSet;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationCreateRequest;
import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationFilterRequest;
@@ -46,6 +43,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
+import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.collections4.CollectionUtils;
@@ -354,9 +352,9 @@ public class ProcessTaskRelationServiceImpl extends BaseServiceImpl implements P
}
updateProcessDefiniteVersion(loginUser, result, processDefinition);
updateRelation(loginUser, result, processDefinition, processTaskRelationList);
- if (TASK_TYPE_CONDITIONS.equals(taskDefinition.getTaskType())
- || TASK_TYPE_DEPENDENT.equals(taskDefinition.getTaskType())
- || TASK_TYPE_SUB_PROCESS.equals(taskDefinition.getTaskType())) {
+ if (TaskTypeUtils.isConditionTask(taskDefinition.getTaskType())
+ || TaskTypeUtils.isSubWorkflowTask(taskDefinition.getTaskType())
+ || TaskTypeUtils.isDependentTask(taskDefinition.getTaskType())) {
int deleteTaskDefinition = taskDefinitionMapper.deleteByCode(taskCode);
if (0 == deleteTaskDefinition) {
log.error("Delete task definition error, taskDefinitionCode:{}.", taskCode);
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java
index 31dc113dbb..c11915667f 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java
@@ -32,6 +32,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectWorkerGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections4.SetUtils;
@@ -118,7 +119,7 @@ public class ProjectWorkerGroupRelationServiceImpl extends BaseServiceImpl
workerGroupMapper.queryAllWorkerGroup().stream().map(WorkerGroup::getName).collect(
Collectors.toSet());
- workerGroupNames.add(Constants.DEFAULT_WORKER_GROUP);
+ workerGroupNames.add(WorkerGroupUtils.getDefaultWorkerGroup());
Set assignedWorkerGroupNames = new HashSet<>(workerGroups);
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
index 6a15da17a8..0757acd816 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
@@ -17,16 +17,29 @@
package org.apache.dolphinscheduler.api.service.impl;
-import static org.apache.dolphinscheduler.common.constants.Constants.ALIAS;
-import static org.apache.dolphinscheduler.common.constants.Constants.CONTENT;
-import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING;
-import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
-import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_SS;
-import static org.apache.dolphinscheduler.common.constants.Constants.JAR;
-import static org.apache.dolphinscheduler.common.constants.Constants.PERIOD;
-
-import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
-import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter;
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
+import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceDto;
+import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest;
+import org.apache.dolphinscheduler.api.dto.resources.DownloadFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest;
+import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto;
+import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest;
+import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
+import org.apache.dolphinscheduler.api.dto.resources.QueryResourceDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
+import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor;
import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor;
import org.apache.dolphinscheduler.api.enums.Status;
@@ -34,1269 +47,374 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
-import org.apache.dolphinscheduler.api.utils.RegexUtils;
-import org.apache.dolphinscheduler.api.utils.Result;
-import org.apache.dolphinscheduler.common.constants.Constants;
-import org.apache.dolphinscheduler.common.enums.ProgramType;
-import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.api.validator.resource.CreateDirectoryDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.CreateDirectoryRequestTransformer;
+import org.apache.dolphinscheduler.api.validator.resource.CreateFileDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.CreateFileFromContentDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.DeleteResourceDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.DownloadFileDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.FetchFileContentDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.FileFromContentRequestTransformer;
+import org.apache.dolphinscheduler.api.validator.resource.FileRequestTransformer;
+import org.apache.dolphinscheduler.api.validator.resource.PagingResourceItemRequestTransformer;
+import org.apache.dolphinscheduler.api.validator.resource.RenameDirectoryDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.RenameDirectoryRequestTransformer;
+import org.apache.dolphinscheduler.api.validator.resource.RenameFileDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.RenameFileRequestTransformer;
+import org.apache.dolphinscheduler.api.validator.resource.UpdateFileDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.UpdateFileFromContentDtoValidator;
+import org.apache.dolphinscheduler.api.validator.resource.UpdateFileFromContentRequestTransformer;
+import org.apache.dolphinscheduler.api.validator.resource.UpdateFileRequestTransformer;
+import org.apache.dolphinscheduler.api.vo.ResourceItemVO;
+import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse;
import org.apache.dolphinscheduler.common.utils.FileUtils;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
-import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
-import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
import java.io.File;
-import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Paths;
-import java.text.MessageFormat;
-import java.time.LocalDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.UUID;
import java.util.stream.Collectors;
+import javax.servlet.http.HttpServletResponse;
+
+import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
-import com.google.common.io.Files;
-
@Service
@Slf4j
public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesService {
@Autowired
- private UdfFuncMapper udfFunctionMapper;
-
- @Autowired
- private TenantMapper tenantMapper;
+ private TenantDao tenantDao;
@Autowired
private UserMapper userMapper;
- @Autowired(required = false)
- private StorageOperate storageOperate;
+ @Autowired
+ private StorageOperator storageOperator;
+
+ @Autowired
+ private CreateDirectoryRequestTransformer createDirectoryRequestTransformer;
+
+ @Autowired
+ private CreateDirectoryDtoValidator createDirectoryDtoValidator;
+
+ @Autowired
+ private RenameDirectoryRequestTransformer renameDirectoryRequestTransformer;
+
+ @Autowired
+ private RenameDirectoryDtoValidator renameDirectoryDtoValidator;
+
+ @Autowired
+ private RenameFileRequestTransformer renameFileRequestTransformer;
+
+ @Autowired
+ private RenameFileDtoValidator renameFileDtoValidator;
+
+ @Autowired
+ private FileFromContentRequestTransformer createFileFromContentRequestTransformer;
+
+ @Autowired
+ private CreateFileFromContentDtoValidator createFileFromContentDtoValidator;
+
+ @Autowired
+ private FetchFileContentDtoValidator fetchFileContentDtoValidator;
+
+ @Autowired
+ private UpdateFileFromContentRequestTransformer updateFileFromContentRequestTransformer;
+
+ @Autowired
+ private UpdateFileFromContentDtoValidator updateFileFromContentDtoValidator;
+
+ @Autowired
+ private FileRequestTransformer createFileRequestTransformer;
+
+ @Autowired
+ private CreateFileDtoValidator createFileDtoValidator;
+
+ @Autowired
+ private UpdateFileRequestTransformer updateFileRequestTransformer;
+
+ @Autowired
+ private UpdateFileDtoValidator updateFileDtoValidator;
+
+ @Autowired
+ private DeleteResourceDtoValidator deleteResourceDtoValidator;
+
+ @Autowired
+ private DownloadFileDtoValidator downloadFileDtoValidator;
+
+ @Autowired
+ private PagingResourceItemRequestTransformer pagingResourceItemRequestTransformer;
- /**
- * create directory
- *
- * @param loginUser login user
- * @param name alias
- * @param type type
- * @param pid parent id
- * @param currentDir current directory
- * @return create directory result
- */
@Override
- @Transactional
- public Result createDirectory(User loginUser, String name, ResourceType type, int pid, String currentDir) {
- Result result = new Result<>();
- if (FileUtils.directoryTraversal(name)) {
- log.warn("Parameter name is invalid, name:{}.", RegexUtils.escapeNRT(name));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
+ public void createDirectory(CreateDirectoryRequest createDirectoryRequest) {
+ CreateDirectoryDto createDirectoryDto = createDirectoryRequestTransformer.transform(createDirectoryRequest);
+ createDirectoryDtoValidator.validate(createDirectoryDto);
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, currentDir);
-
- String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode)
- : storageOperate.getResDir(tenantCode);
- String fullName = !currentDir.contains(userResRootPath) ? userResRootPath + name : currentDir + name;
-
- try {
- if (checkResourceExists(fullName)) {
- log.error("resource directory {} has exist, can't recreate", fullName);
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- } catch (Exception e) {
- log.warn("Resource exists, can not create again, fullName:{}.", fullName, e);
- throw new ServiceException("resource already exists, can't recreate");
- }
-
- // create directory in storage
- createDirectory(loginUser, fullName, type, result);
- return result;
+ storageOperator.createStorageDir(createDirectoryDto.getDirectoryAbsolutePath());
+ log.info("Success create directory: {}", createDirectoryRequest.getParentAbsoluteDirectory());
}
- /**
- * create resource
- *
- * @param loginUser login user
- * @param name alias
- * @param type type
- * @param file file
- * @param currentDir current directory
- * @return create result code
- */
@Override
- @Transactional
- public Result uploadResource(User loginUser, String name, ResourceType type, MultipartFile file,
- String currentDir) {
- Result result = new Result<>();
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, currentDir);
-
- result = verifyFile(name, type, file);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- return result;
- }
-
- // check resource name exists
- String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode)
- : storageOperate.getResDir(tenantCode);
- String currDirNFileName = !currentDir.contains(userResRootPath) ? userResRootPath + name : currentDir + name;
+ public void createFile(CreateFileRequest createFileRequest) {
+ CreateFileDto createFileDto = createFileRequestTransformer.transform(createFileRequest);
+ createFileDtoValidator.validate(createFileDto);
+ // todo: use storage proxy
+ MultipartFile file = createFileDto.getFile();
+ String fileAbsolutePath = createFileDto.getFileAbsolutePath();
+ String srcLocalTmpFileAbsolutePath = copyFileToLocal(file);
try {
- if (checkResourceExists(currDirNFileName)) {
- log.error("resource {} has exist, can't recreate", RegexUtils.escapeNRT(name));
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- } catch (Exception e) {
- throw new ServiceException("resource already exists, can't recreate");
- }
- if (currDirNFileName.length() > Constants.RESOURCE_FULL_NAME_MAX_LENGTH) {
- log.error(
- "Resource file's name is longer than max full name length, fullName:{}, "
- + "fullNameSize:{}, maxFullNameSize:{}",
- RegexUtils.escapeNRT(name), currDirNFileName.length(), Constants.RESOURCE_FULL_NAME_MAX_LENGTH);
- putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR);
- return result;
- }
-
- // fail upload
- if (!upload(loginUser, currDirNFileName, file, type)) {
- log.error("upload resource: {} file: {} failed.", RegexUtils.escapeNRT(name),
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.STORE_OPERATE_CREATE_ERROR);
- throw new ServiceException(
- String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename()));
- } else
+ storageOperator.upload(srcLocalTmpFileAbsolutePath, fileAbsolutePath, true, false);
ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
- log.info("Upload resource file complete, resourceName:{}, fileName:{}.", RegexUtils.escapeNRT(name),
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.SUCCESS);
- return result;
- }
-
- /**
- * check resource is exists
- *
- * @param fullName fullName
- * @return true if resource exists
- */
- private boolean checkResourceExists(String fullName) {
- Boolean existResource = false;
- try {
- existResource = storageOperate.exists(fullName);
- } catch (IOException e) {
- log.error("error occurred when checking resource: " + fullName, e);
+ log.info("Success upload resource file: {} complete.", fileAbsolutePath);
+ } catch (Exception ex) {
+ // If exception, clear the tmp path
+ FileUtils.deleteFile(srcLocalTmpFileAbsolutePath);
+ throw ex;
}
- return Boolean.TRUE.equals(existResource);
}
- /**
- * update resource
- *
- * @param loginUser login user
- * @param resourceFullName resource full name
- * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
- * can be different from the login user in the case of logging in as admin users.
- * @param name name
- * @param type resource type
- * @param file resource file
- * @return update result code
- */
@Override
- @Transactional
- public Result updateResource(User loginUser, String resourceFullName, String resTenantCode, String name,
- ResourceType type, MultipartFile file) {
- Result result = new Result<>();
+ public void createFileFromContent(CreateFileFromContentRequest createFileFromContentRequest) {
+ CreateFileFromContentDto createFileFromContentDto =
+ createFileFromContentRequestTransformer.transform(createFileFromContentRequest);
+ createFileFromContentDtoValidator.validate(createFileFromContentDto);
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, resourceFullName);
-
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
-
- String defaultPath = storageOperate.getDir(type, tenantCode);
-
- StorageEntity resource;
+ // todo: use storage proxy
+ String fileContent = createFileFromContentDto.getFileContent();
+ String fileAbsolutePath = createFileFromContentDto.getFileAbsolutePath();
+ String srcLocalTmpFileAbsolutePath = copyFileToLocal(fileContent);
try {
- resource = storageOperate.getFileStatus(resourceFullName, defaultPath, resTenantCode, type);
- } catch (Exception e) {
- log.error("Get file status fail, resource path: {}", resourceFullName, e);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- throw new ServiceException((String.format("Get file status fail, resource path: %s", resourceFullName)));
+ storageOperator.upload(srcLocalTmpFileAbsolutePath, fileAbsolutePath, true, false);
+ ApiServerMetrics.recordApiResourceUploadSize(fileContent.length());
+ log.info("Success upload resource file: {} complete.", fileAbsolutePath);
+ } catch (Exception ex) {
+ // If exception, clear the tmp path
+ FileUtils.deleteFile(srcLocalTmpFileAbsolutePath);
+ throw ex;
}
+ }
- // TODO: deal with OSS
- if (resource.isDirectory() && storageOperate.returnStorageType().equals(ResUploadType.S3)
- && !resource.getFileName().equals(name)) {
- log.warn("Directory in S3 storage can not be renamed.");
- putMsg(result, Status.S3_CANNOT_RENAME);
- return result;
- }
+ @Override
+ public void renameDirectory(RenameDirectoryRequest renameDirectoryRequest) {
+ RenameDirectoryDto renameDirectoryDto = renameDirectoryRequestTransformer.transform(renameDirectoryRequest);
+ renameDirectoryDtoValidator.validate(renameDirectoryDto);
- // check if updated name of the resource already exists
- String originFullName = resource.getFullName();
- String originResourceName = resource.getAlias();
+ String originDirectoryAbsolutePath = renameDirectoryDto.getOriginDirectoryAbsolutePath();
+ String targetDirectoryAbsolutePath = renameDirectoryDto.getTargetDirectoryAbsolutePath();
+ storageOperator.copy(originDirectoryAbsolutePath, targetDirectoryAbsolutePath, true, true);
+ log.info("Success rename directory: {} -> {} ", originDirectoryAbsolutePath, targetDirectoryAbsolutePath);
+ }
- // the format of hdfs folders in the implementation has a "/" at the very end, we need to remove it.
- originFullName = originFullName.endsWith("/") ? StringUtils.chop(originFullName) : originFullName;
- name = name.endsWith("/") ? StringUtils.chop(name) : name;
- // updated fullName
- String fullName = String.format(FORMAT_SS,
- originFullName.substring(0, originFullName.lastIndexOf(FOLDER_SEPARATOR) + 1), name);
- if (!originResourceName.equals(name)) {
- try {
- if (checkResourceExists(fullName)) {
- log.error("resource {} already exists, can't recreate", fullName);
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- } catch (Exception e) {
- throw new ServiceException(String.format("error occurs while querying resource: %s", fullName));
- }
+ @Override
+ public void renameFile(RenameFileRequest renameFileRequest) {
+ RenameFileDto renameFileDto = renameFileRequestTransformer.transform(renameFileRequest);
+ renameFileDtoValidator.validate(renameFileDto);
- }
+ String originFileAbsolutePath = renameFileDto.getOriginFileAbsolutePath();
+ String targetFileAbsolutePath = renameFileDto.getTargetFileAbsolutePath();
+ storageOperator.copy(originFileAbsolutePath, targetFileAbsolutePath, true, true);
+ log.info("Success rename file: {} -> {} ", originFileAbsolutePath, targetFileAbsolutePath);
+ }
- result = verifyFile(name, type, file);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- return result;
- }
+ @Override
+ public void updateFile(UpdateFileRequest updateFileRequest) {
+ UpdateFileDto updateFileDto = updateFileRequestTransformer.transform(updateFileRequest);
+ updateFileDtoValidator.validate(updateFileDto);
- Date now = new Date();
-
- resource.setAlias(name);
- resource.setFileName(name);
- resource.setFullName(fullName);
- resource.setUpdateTime(now);
- if (file != null) {
- resource.setSize(file.getSize());
- }
-
- // if name unchanged, return directly without moving on HDFS
- if (originResourceName.equals(name) && file == null) {
- return result;
- }
-
- if (file != null) {
- // fail upload
- if (!upload(loginUser, fullName, file, type)) {
- log.error("Storage operation error, resourceName:{}, originFileName:{}.", name,
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.HDFS_OPERATION_ERROR);
- throw new ServiceException(
- String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename()));
- }
- if (!fullName.equals(originFullName)) {
- try {
- storageOperate.delete(originFullName, false);
- } catch (IOException e) {
- log.error("Resource delete error, resourceFullName:{}.", originFullName, e);
- throw new ServiceException(String.format("delete resource: %s failed.", originFullName));
- }
- }
-
- ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
- return result;
- }
-
- // get the path of dest file in hdfs
- String destHdfsFileName = fullName;
+ String srcLocalTmpFileAbsolutePath = copyFileToLocal(updateFileDto.getFile());
try {
- log.info("start copy {} -> {}", originFullName, destHdfsFileName);
- storageOperate.copy(originFullName, destHdfsFileName, true, true);
- putMsg(result, Status.SUCCESS);
+ storageOperator.upload(srcLocalTmpFileAbsolutePath, updateFileDto.getFileAbsolutePath(), true, true);
+ ApiServerMetrics.recordApiResourceUploadSize(updateFileDto.getFile().getSize());
+ log.info("Success upload resource file: {} complete.", updateFileDto.getFileAbsolutePath());
+ } catch (Exception ex) {
+ // If exception, clear the tmp path
+ FileUtils.deleteFile(srcLocalTmpFileAbsolutePath);
+ throw ex;
+ }
+ }
+
+ @Override
+ public PageInfo pagingResourceItem(PagingResourceItemRequest pagingResourceItemRequest) {
+
+ QueryResourceDto queryResourceDto = pagingResourceItemRequestTransformer.transform(pagingResourceItemRequest);
+ List resourceAbsolutePaths = queryResourceDto.getResourceAbsolutePaths();
+ if (CollectionUtils.isEmpty(resourceAbsolutePaths)) {
+ return new PageInfo<>(pagingResourceItemRequest.getPageNo(), pagingResourceItemRequest.getPageSize());
+ }
+
+ for (String resourceAbsolutePath : resourceAbsolutePaths) {
+ createDirectoryDtoValidator.exceptionResourceAbsolutePathInvalidated(resourceAbsolutePath);
+ createDirectoryDtoValidator.exceptionUserNoResourcePermission(pagingResourceItemRequest.getLoginUser(),
+ resourceAbsolutePath);
+ }
+
+ Integer pageNo = pagingResourceItemRequest.getPageNo();
+ Integer pageSize = pagingResourceItemRequest.getPageSize();
+
+ List storageEntities = resourceAbsolutePaths.stream()
+ .flatMap(resourceAbsolutePath -> storageOperator.listStorageEntity(resourceAbsolutePath).stream())
+ .collect(Collectors.toList());
+
+ List result = storageEntities
+ .stream()
+ .filter(storageEntity -> storageEntity.getFileName()
+ .contains(pagingResourceItemRequest.getResourceNameKeyWord()))
+ .skip((long) (pageNo - 1) * pageSize)
+ .limit(pageSize)
+ .map(ResourceItemVO::new)
+ .collect(Collectors.toList());
+
+ return PageInfo.builder()
+ .pageNo(pagingResourceItemRequest.getPageNo())
+ .pageSize(pagingResourceItemRequest.getPageSize())
+ .total(storageEntities.size())
+ .totalList(result)
+ .build();
+ }
+
+ @Override
+ public List queryResourceFiles(User loginUser, ResourceType resourceType) {
+ Tenant tenant = tenantDao.queryOptionalById(loginUser.getTenantId())
+ .orElseThrow(() -> new ServiceException(Status.TENANT_NOT_EXIST, loginUser.getTenantId()));
+ String storageBaseDirectory = storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType);
+ List allResourceFiles = storageOperator.listFileStorageEntityRecursively(storageBaseDirectory);
+
+ Visitor visitor = new ResourceTreeVisitor(allResourceFiles);
+ return visitor.visit("").getChildren();
+ }
+
+ @Override
+ public void delete(DeleteResourceRequest deleteResourceRequest) {
+ DeleteResourceDto deleteResourceDto = DeleteResourceDto.builder()
+ .loginUser(deleteResourceRequest.getLoginUser())
+ .resourceAbsolutePath(deleteResourceRequest.getResourceAbsolutePath())
+ .build();
+ deleteResourceDtoValidator.validate(deleteResourceDto);
+ storageOperator.delete(deleteResourceDto.getResourceAbsolutePath(), true);
+ }
+
+ @Override
+ public FetchFileContentResponse fetchResourceFileContent(FetchFileContentRequest fetchFileContentRequest) {
+ FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
+ .loginUser(fetchFileContentRequest.getLoginUser())
+ .resourceFileAbsolutePath(fetchFileContentRequest.getResourceFileAbsolutePath())
+ .skipLineNum(fetchFileContentRequest.getSkipLineNum())
+ .limit(fetchFileContentRequest.getLimit())
+ .build();
+ fetchFileContentDtoValidator.validate(fetchFileContentDto);
+
+ String content = storageOperator
+ .fetchFileContent(
+ fetchFileContentRequest.getResourceFileAbsolutePath(),
+ fetchFileContentRequest.getSkipLineNum(),
+ fetchFileContentRequest.getLimit())
+ .stream()
+ .collect(Collectors.joining("\n"));
+
+ ApiServerMetrics.recordApiResourceDownloadSize(content.length());
+
+ return FetchFileContentResponse.builder()
+ .content(content)
+ .build();
+ }
+
+ @Override
+ public void updateFileFromContent(UpdateFileFromContentRequest updateFileContentRequest) {
+ UpdateFileFromContentDto updateFileFromContentDto =
+ updateFileFromContentRequestTransformer.transform(updateFileContentRequest);
+ updateFileFromContentDtoValidator.validate(updateFileFromContentDto);
+
+ String srcLocalTmpFileAbsolutePath = copyFileToLocal(updateFileFromContentDto.getFileContent());
+ try {
+ storageOperator.upload(srcLocalTmpFileAbsolutePath, updateFileFromContentDto.getFileAbsolutePath(), true,
+ true);
+ ApiServerMetrics.recordApiResourceUploadSize(updateFileFromContentDto.getFileContent().length());
+ log.info("Success upload resource file: {} complete.", updateFileFromContentDto.getFileAbsolutePath());
+ } catch (Exception ex) {
+ // If exception, clear the tmp path
+ FileUtils.deleteFile(srcLocalTmpFileAbsolutePath);
+ throw new ServiceException("Update the resource file from content: "
+ + updateFileFromContentDto.getFileAbsolutePath() + " failed", ex);
+ }
+ }
+
+ @Override
+ public void downloadResource(HttpServletResponse response, DownloadFileRequest downloadFileRequest) {
+ DownloadFileDto downloadFileDto = DownloadFileDto.builder()
+ .loginUser(downloadFileRequest.getLoginUser())
+ .fileAbsolutePath(downloadFileRequest.getFileAbsolutePath())
+ .build();
+ downloadFileDtoValidator.validate(downloadFileDto);
+
+ String fileName = new File(downloadFileDto.getFileAbsolutePath()).getName();
+ String localTmpFileAbsolutePath = FileUtils.getDownloadFilename(fileName);
+
+ try {
+ storageOperator.download(downloadFileRequest.getFileAbsolutePath(), localTmpFileAbsolutePath, true);
+ int length = (int) new File(localTmpFileAbsolutePath).length();
+ ApiServerMetrics.recordApiResourceDownloadSize(length);
+
+ response.reset();
+ response.setContentType("application/octet-stream");
+ response.setCharacterEncoding("utf-8");
+ response.setContentLength(length);
+ response.setHeader("Content-Disposition", "attachment;filename=" + fileName);
+ Files.copy(Paths.get(localTmpFileAbsolutePath), response.getOutputStream());
} catch (Exception e) {
- log.error(MessageFormat.format(" copy {0} -> {1} fail", originFullName, destHdfsFileName), e);
- putMsg(result, Status.HDFS_COPY_FAIL);
throw new ServiceException(
- MessageFormat.format(Status.HDFS_COPY_FAIL.getMsg(), originFullName, destHdfsFileName));
- }
-
- return result;
- }
-
- private Result verifyFile(String name, ResourceType type, MultipartFile file) {
- Result result = new Result<>();
- putMsg(result, Status.SUCCESS);
-
- if (FileUtils.directoryTraversal(name)) {
- log.warn("Parameter file alias name verify failed, fileAliasName:{}.", RegexUtils.escapeNRT(name));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
-
- if (file != null && FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename()))) {
- log.warn("File original name verify failed, fileOriginalName:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
-
- if (file != null) {
- // file is empty
- if (file.isEmpty()) {
- log.warn("Parameter file is empty, fileOriginalName:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.RESOURCE_FILE_IS_EMPTY);
- return result;
- }
-
- // file suffix
- String fileSuffix = Files.getFileExtension(file.getOriginalFilename());
- String nameSuffix = Files.getFileExtension(name);
-
- // determine file suffix
- if (!fileSuffix.equalsIgnoreCase(nameSuffix)) {
- // rename file suffix and original suffix must be consistent
- log.warn("Rename file suffix and original suffix must be consistent, fileOriginalName:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.RESOURCE_SUFFIX_FORBID_CHANGE);
- return result;
- }
-
- // If resource type is UDF, only jar packages are allowed to be uploaded, and the suffix must be .jar
- if (Constants.UDF.equals(type.name()) && !JAR.equalsIgnoreCase(fileSuffix)) {
- log.warn(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg());
- putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR);
- return result;
- }
- if (file.getSize() > Constants.MAX_FILE_SIZE) {
- log.warn(
- "Resource file size is larger than max file size, fileOriginalName:{}, fileSize:{}, maxFileSize:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()), file.getSize(), Constants.MAX_FILE_SIZE);
- putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT);
- return result;
- }
- }
- return result;
- }
-
- /**
- * query resources list paging
- *
- * @param loginUser login user
- * @param fullName resource full name
- * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
- * can be different from the login user in the case of logging in as admin users.
- * @param type resource type
- * @param searchVal search value
- * @param pageNo page number
- * @param pageSize page size
- * @return resource list page
- */
- @Override
- public Result> queryResourceListPaging(User loginUser, String fullName,
- String resTenantCode, ResourceType type,
- String searchVal, Integer pageNo, Integer pageSize) {
- Result> result = new Result<>();
- PageInfo pageInfo = new PageInfo<>(pageNo, pageSize);
- if (storageOperate == null) {
- log.warn("The resource storage is not opened.");
- return Result.success(pageInfo);
- }
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, fullName);
-
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
-
- List resourcesList;
- try {
- resourcesList = queryStorageEntityList(loginUser, fullName, type, tenantCode, false);
- } catch (ServiceException e) {
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
-
- // remove leading and trailing spaces in searchVal
- String trimmedSearchVal = searchVal != null ? searchVal.trim() : "";
- // filter based on trimmed searchVal
- List filteredResourceList = resourcesList.stream()
- .filter(x -> x.getFileName().contains(trimmedSearchVal)).collect(Collectors.toList());
- // inefficient pagination
- List slicedResourcesList = filteredResourceList.stream().skip((long) (pageNo - 1) * pageSize)
- .limit(pageSize).collect(Collectors.toList());
-
- pageInfo.setTotal(filteredResourceList.size());
- pageInfo.setTotalList(slicedResourcesList);
- result.setData(pageInfo);
- putMsg(result, Status.SUCCESS);
- return result;
- }
-
- private List queryStorageEntityList(User loginUser, String fullName, ResourceType type,
- String tenantCode, boolean recursive) {
- String defaultPath = "";
- List resourcesList = new ArrayList<>();
- String resourceStorageType =
- PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE, ResUploadType.LOCAL.name());
- if (isAdmin(loginUser) && StringUtils.isBlank(fullName)) {
- // list all tenants' resources to admin users in the root directory
- List userList = userMapper.selectList(null);
- Set visitedTenantEntityCode = new HashSet<>();
- for (User userEntity : userList) {
- String tenantEntityCode = getTenantCode(userEntity);
- if (!visitedTenantEntityCode.contains(tenantEntityCode)) {
- defaultPath = storageOperate.getResDir(tenantEntityCode);
- if (type.equals(ResourceType.UDF)) {
- defaultPath = storageOperate.getUdfDir(tenantEntityCode);
- }
- try {
- resourcesList.addAll(recursive
- ? storageOperate.listFilesStatusRecursively(defaultPath, defaultPath, tenantEntityCode,
- type)
- : storageOperate.listFilesStatus(defaultPath, defaultPath, tenantEntityCode, type));
-
- visitedTenantEntityCode.add(tenantEntityCode);
- } catch (Exception e) {
- log.error(e.getMessage() + " Resource path: {}", defaultPath, e);
- throw new ServiceException(
- String.format(e.getMessage() + " make sure resource path: %s exists in %s", defaultPath,
- resourceStorageType));
- }
- }
- }
- } else {
- defaultPath = storageOperate.getResDir(tenantCode);
- if (type.equals(ResourceType.UDF)) {
- defaultPath = storageOperate.getUdfDir(tenantCode);
- }
-
- try {
- if (StringUtils.isBlank(fullName)) {
- fullName = defaultPath;
- }
- resourcesList =
- recursive ? storageOperate.listFilesStatusRecursively(fullName, defaultPath, tenantCode, type)
- : storageOperate.listFilesStatus(fullName, defaultPath, tenantCode, type);
- } catch (Exception e) {
- log.error(e.getMessage() + " Resource path: {}", fullName, e);
- throw new ServiceException(String.format(e.getMessage() + " make sure resource path: %s exists in %s",
- defaultPath, resourceStorageType));
- }
- }
-
- return resourcesList;
- }
-
- /**
- * create directory
- * xxx The steps to verify resources are cumbersome and can be optimized
- *
- * @param loginUser login user
- * @param fullName full name
- * @param type resource type
- * @param result Result
- */
- private void createDirectory(User loginUser, String fullName, ResourceType type, Result result) {
- String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode();
- // String directoryName = storageOperate.getFileName(type, tenantCode, fullName);
- String resourceRootPath = storageOperate.getDir(type, tenantCode);
- try {
- if (!storageOperate.exists(resourceRootPath)) {
- storageOperate.createTenantDirIfNotExists(tenantCode);
- }
-
- if (!storageOperate.mkdir(tenantCode, fullName)) {
- throw new ServiceException(String.format("Create resource directory: %s failed.", fullName));
- }
- putMsg(result, Status.SUCCESS);
- } catch (Exception e) {
- throw new ServiceException(String.format("create resource directory: %s failed.", fullName));
- }
- }
-
- /**
- * upload file to hdfs
- *
- * @param loginUser login user
- * @param fullName full name
- * @param file file
- * @param type resource type
- * @return upload success return true, otherwise false
- */
- private boolean upload(User loginUser, String fullName, MultipartFile file, ResourceType type) {
- // save to local
- String fileSuffix = Files.getFileExtension(file.getOriginalFilename());
- String nameSuffix = Files.getFileExtension(fullName);
-
- // determine file suffix
- if (!fileSuffix.equalsIgnoreCase(nameSuffix)) {
- return false;
- }
- // query tenant
- String tenantCode = getTenantCode(loginUser);
- // random file name
- String localFilename = FileUtils.getUploadFilename(tenantCode, UUID.randomUUID().toString());
-
- // save file to hdfs, and delete original file
- String resourcePath = storageOperate.getDir(type, tenantCode);
- try {
- // if tenant dir not exists
- if (!storageOperate.exists(resourcePath)) {
- storageOperate.createTenantDirIfNotExists(tenantCode);
- }
- org.apache.dolphinscheduler.api.utils.FileUtils.copyInputStreamToFile(file, localFilename);
- storageOperate.upload(tenantCode, localFilename, fullName, true, true);
- FileUtils.deleteFile(localFilename);
- } catch (Exception e) {
- FileUtils.deleteFile(localFilename);
- log.error(e.getMessage(), e);
- return false;
- }
- return true;
- }
-
- /**
- * query resource list
- *
- * @param loginUser login user
- * @param type resource type
- * @param fullName resource full name
- * @return resource list
- */
- @Override
- public Map queryResourceList(User loginUser, ResourceType type, String fullName) {
- Map result = new HashMap<>();
- if (storageOperate == null) {
- result.put(Constants.DATA_LIST, Collections.emptyList());
- result.put(Constants.STATUS, Status.SUCCESS);
- return result;
- }
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return null;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, fullName);
-
- String baseDir = storageOperate.getDir(type, tenantCode);
-
- List resourcesList = new ArrayList<>();
- if (StringUtils.isBlank(fullName)) {
- if (isAdmin(loginUser)) {
- List userList = userMapper.selectList(null);
- Set visitedTenantEntityCode = new HashSet<>();
- for (User userEntity : userList) {
- String tenantEntityCode = getTenantCode(userEntity);
- if (!visitedTenantEntityCode.contains(tenantEntityCode)) {
- baseDir = storageOperate.getDir(type, tenantEntityCode);
- resourcesList.addAll(storageOperate.listFilesStatusRecursively(baseDir, baseDir,
- tenantEntityCode, type));
- visitedTenantEntityCode.add(tenantEntityCode);
- }
- }
- } else {
- resourcesList = storageOperate.listFilesStatusRecursively(baseDir, baseDir, tenantCode, type);
- }
- } else {
- resourcesList = storageOperate.listFilesStatusRecursively(fullName, baseDir, tenantCode, type);
- }
-
- Visitor resourceTreeVisitor = new ResourceTreeVisitor(resourcesList);
- result.put(Constants.DATA_LIST, resourceTreeVisitor.visit(baseDir).getChildren());
- putMsg(result, Status.SUCCESS);
-
- return result;
- }
-
- /**
- * query resource list by program type
- *
- * @param loginUser login user
- * @param type resource type
- * @return resource list
- */
- @Override
- public Result queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) {
- Result result = new Result<>();
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- Tenant tenant = tenantMapper.queryById(user.getTenantId());
- if (tenant == null) {
- log.error("tenant not exists");
- putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
- return result;
- }
-
- String tenantCode = tenant.getTenantCode();
-
- List allResourceList = queryStorageEntityList(loginUser, "", type, tenantCode, true);
-
- String suffix = ".jar";
- if (programType != null) {
- switch (programType) {
- case JAVA:
- case SCALA:
- break;
- case PYTHON:
- suffix = ".py";
- break;
- default:
- }
- }
- List resources = new ResourceFilter(suffix, new ArrayList<>(allResourceList)).filter();
- Visitor visitor = new ResourceTreeVisitor(resources);
- result.setData(visitor.visit("").getChildren());
- putMsg(result, Status.SUCCESS);
- return result;
- }
-
- /**
- * delete resource
- *
- * @param loginUser login user
- * @param fullName resource full name
- * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
- * can be different from the login user in the case of logging in as admin users.
- * @return delete result code
- * @throws IOException exception
- */
- @Override
- @Transactional(rollbackFor = Exception.class)
- public Result delete(User loginUser, String fullName, String resTenantCode) throws IOException {
- Result result = new Result<>();
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, fullName);
-
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
-
- String baseDir = storageOperate.getResDir(tenantCode);
-
- StorageEntity resource;
- try {
- resource = storageOperate.getFileStatus(fullName, baseDir, resTenantCode, null);
- } catch (Exception e) {
- log.error(e.getMessage() + " Resource path: {}", fullName, e);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- throw new ServiceException(String.format(e.getMessage() + " Resource path: %s", fullName));
- }
-
- if (resource == null) {
- log.error("Resource does not exist, resource full name:{}.", fullName);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
-
- // recursively delete a folder
- List allChildren =
- storageOperate.listFilesStatusRecursively(fullName, baseDir, resTenantCode, resource.getType())
- .stream().map(storageEntity -> storageEntity.getFullName()).collect(Collectors.toList());
-
- String[] allChildrenFullNameArray = allChildren.stream().toArray(String[]::new);
-
- // if resource type is UDF,need check whether it is bound by UDF function
- if (resource.getType() == (ResourceType.UDF)) {
- List udfFuncs = udfFunctionMapper.listUdfByResourceFullName(allChildrenFullNameArray);
- if (CollectionUtils.isNotEmpty(udfFuncs)) {
- log.warn("Resource can not be deleted because it is bound by UDF functions, udfFuncIds:{}", udfFuncs);
- putMsg(result, Status.UDF_RESOURCE_IS_BOUND, udfFuncs.get(0).getFuncName());
- return result;
- }
- }
-
- // delete file on hdfs,S3
- storageOperate.delete(fullName, allChildren, true);
-
- putMsg(result, Status.SUCCESS);
-
- return result;
- }
-
- /**
- * verify resource by name and type
- *
- * @param loginUser login user
- * @param fullName resource full name
- * @param type resource type
- * @return true if the resource name not exists, otherwise return false
- */
- @Override
- public Result verifyResourceName(String fullName, ResourceType type, User loginUser) {
- Result result = new Result<>();
- putMsg(result, Status.SUCCESS);
- if (checkResourceExists(fullName)) {
- log.error("Resource with same name exists so can not create again, resourceType:{}, resourceName:{}.", type,
- RegexUtils.escapeNRT(fullName));
- putMsg(result, Status.RESOURCE_EXIST);
- }
-
- return result;
- }
-
- /**
- * verify resource by full name or pid and type
- *
- * @param fileName resource file name
- * @param type resource type
- * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
- * can be different from the login user in the case of logging in as admin users.
- * @return true if the resource full name or pid not exists, otherwise return false
- */
- @Override
- public Result queryResourceByFileName(User loginUser, String fileName, ResourceType type,
- String resTenantCode) {
- Result result = new Result<>();
- if (StringUtils.isBlank(fileName)) {
- putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR);
- return result;
- }
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
-
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
-
- String defaultPath = storageOperate.getDir(type, resTenantCode);
- StorageEntity file;
- try {
- file = storageOperate.getFileStatus(defaultPath + fileName, defaultPath, resTenantCode, type);
- } catch (Exception e) {
- log.error(e.getMessage() + " Resource path: {}", defaultPath + fileName, e);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
-
- putMsg(result, Status.SUCCESS);
- result.setData(file);
- return result;
- }
-
- /**
- * view resource file online
- *
- * @param fullName resource fullName
- * @param resTenantCode owner's tenant code of the resource
- * @param skipLineNum skip line number
- * @param limit limit
- * @return resource content
- */
- @Override
- public Result readResource(User loginUser, String fullName, String resTenantCode, int skipLineNum,
- int limit) {
- Result result = new Result<>();
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, fullName);
-
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
-
- // check preview or not by file suffix
- String nameSuffix = Files.getFileExtension(fullName);
- String resourceViewSuffixes = FileUtils.getResourceViewSuffixes();
- if (StringUtils.isNotEmpty(resourceViewSuffixes)) {
- List strList = Arrays.asList(resourceViewSuffixes.split(","));
- if (!strList.contains(nameSuffix)) {
- log.error("Resource suffix does not support view,resourceFullName:{}, suffix:{}.", fullName,
- nameSuffix);
- putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW);
- return result;
- }
- }
-
- List content;
- try {
- if (storageOperate.exists(fullName)) {
- content = storageOperate.vimFile(tenantCode, fullName, skipLineNum, limit);
- long size = content.stream().mapToLong(String::length).sum();
- ApiServerMetrics.recordApiResourceDownloadSize(size);
- } else {
- log.error("read file {} not exist in storage", fullName);
- putMsg(result, Status.RESOURCE_FILE_NOT_EXIST, fullName);
- return result;
- }
-
- } catch (Exception e) {
- log.error("Resource {} read failed", fullName, e);
- putMsg(result, Status.HDFS_OPERATION_ERROR);
- return result;
- }
-
- putMsg(result, Status.SUCCESS);
- Map map = new HashMap<>();
- map.put(ALIAS, fullName);
- map.put(CONTENT, String.join("\n", content));
- result.setData(map);
-
- return result;
- }
-
- /**
- * create resource file online
- *
- * @param loginUser login user
- * @param type resource type
- * @param fileName file name
- * @param fileSuffix file suffix
- * @param content content
- * @param currentDir current directory
- * @return create result code
- */
- @Override
- @Transactional
- public Result createResourceFile(User loginUser, ResourceType type, String fileName, String fileSuffix,
- String content, String currentDir) {
- Result result = new Result<>();
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, currentDir);
-
- if (FileUtils.directoryTraversal(fileName)) {
- log.warn("File name verify failed, fileName:{}.", RegexUtils.escapeNRT(fileName));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
-
- // check file suffix
- String nameSuffix = fileSuffix.trim();
- String resourceViewSuffixes = FileUtils.getResourceViewSuffixes();
- if (StringUtils.isNotEmpty(resourceViewSuffixes)) {
- List strList = Arrays.asList(resourceViewSuffixes.split(","));
- if (!strList.contains(nameSuffix)) {
- log.warn("Resource suffix does not support view, suffix:{}.", nameSuffix);
- putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW);
- return result;
- }
- }
-
- String name = fileName.trim() + "." + nameSuffix;
-
- String userResRootPath = storageOperate.getResDir(tenantCode);
- String fullName = currentDir.contains(userResRootPath) ? currentDir + name : userResRootPath + name;
-
- result = verifyResourceName(fullName, type, loginUser);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- return result;
- }
-
- result = uploadContentToStorage(fullName, tenantCode, content);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- throw new ServiceException(result.getMsg());
- }
- return result;
- }
-
- @Override
- @Transactional
- public StorageEntity createOrUpdateResource(String userName, String filepath,
- String resourceContent) throws Exception {
- User user = userMapper.queryByUserNameAccurately(userName);
- int suffixLabelIndex = filepath.indexOf(PERIOD);
- if (suffixLabelIndex == -1) {
- throw new IllegalArgumentException(String
- .format("Not allow create or update resources without extension name, filepath: %s", filepath));
- }
-
- String defaultPath = storageOperate.getResDir(user.getTenantCode());
- String fullName = defaultPath + filepath;
-
- Result result = uploadContentToStorage(fullName, user.getTenantCode(), resourceContent);
- if (result.getCode() != Status.SUCCESS.getCode()) {
- throw new ServiceException(result.getMsg());
- }
- return storageOperate.getFileStatus(fullName, defaultPath, user.getTenantCode(), ResourceType.FILE);
- }
-
- /**
- * updateProcessInstance resource
- *
- * @param fullName resource full name
- * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
- * can be different from the login user in the case of logging in as admin users.
- * @param content content
- * @return update result cod
- */
- @Override
- @Transactional
- public Result updateResourceContent(User loginUser, String fullName, String resTenantCode, String content) {
- Result result = new Result<>();
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, fullName);
-
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
-
- StorageEntity resource;
- try {
- resource = storageOperate.getFileStatus(fullName, "", resTenantCode, ResourceType.FILE);
- } catch (Exception e) {
- log.error("error occurred when fetching resource information , resource full name {}", fullName);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
-
- if (resource == null) {
- log.error("Resource does not exist, resource full name:{}.", fullName);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
-
- // check can edit by file suffix
- String nameSuffix = Files.getFileExtension(resource.getAlias());
- String resourceViewSuffixes = FileUtils.getResourceViewSuffixes();
- if (StringUtils.isNotEmpty(resourceViewSuffixes)) {
- List strList = Arrays.asList(resourceViewSuffixes.split(","));
- if (!strList.contains(nameSuffix)) {
- log.warn("Resource suffix does not support view, resource full name:{}, suffix:{}.", fullName,
- nameSuffix);
- putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW);
- return result;
- }
- }
-
- result = uploadContentToStorage(resource.getFullName(), resTenantCode, content);
-
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- throw new ServiceException(result.getMsg());
- } else
- log.info("Update resource content complete, resource full name:{}.", fullName);
- return result;
- }
-
- /**
- * @param fullName resource full name
- * @param tenantCode tenant code
- * @param content content
- * @return result
- */
- private Result uploadContentToStorage(String fullName, String tenantCode, String content) {
- Result result = new Result<>();
- String localFilename = "";
- try {
- localFilename = FileUtils.getUploadFilename(tenantCode, UUID.randomUUID().toString());
-
- if (!FileUtils.writeContent2File(content, localFilename)) {
- // write file fail
- log.error("Write file error, fileName:{}, content:{}.", localFilename, RegexUtils.escapeNRT(content));
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
-
- // get resource file path
- String resourcePath = storageOperate.getResDir(tenantCode);
- log.info("resource path is {}, resource dir is {}", fullName, resourcePath);
-
- if (!storageOperate.exists(resourcePath)) {
- // create if tenant dir not exists
- storageOperate.createTenantDirIfNotExists(tenantCode);
- log.info("Create tenant dir because path {} does not exist, tenantCode:{}.", resourcePath, tenantCode);
- }
- if (storageOperate.exists(fullName)) {
- storageOperate.delete(fullName, false);
- }
-
- storageOperate.upload(tenantCode, localFilename, fullName, true, true);
- } catch (Exception e) {
- log.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode, localFilename, e);
- result.setCode(Status.HDFS_OPERATION_ERROR.getCode());
- result.setMsg(String.format("copy %s to hdfs %s fail", localFilename, fullName));
- return result;
+ "Download the resource file: " + downloadFileRequest.getFileAbsolutePath() + " failed", e);
} finally {
- FileUtils.deleteFile(localFilename);
- }
- log.info("Upload content to storage complete, tenantCode:{}, destFileName:{}.", tenantCode, localFilename);
- putMsg(result, Status.SUCCESS);
- return result;
- }
-
- /**
- * download file
- *
- * @return resource content
- */
- @Override
- public org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) {
- if (fullName.endsWith("/")) {
- log.error("resource id {} is directory,can't download it", fullName);
- throw new ServiceException("can't download directory");
- }
-
- int userId = loginUser.getId();
- User user = userMapper.selectById(userId);
- if (user == null) {
- log.error("User does not exits, userId:{}.", userId);
- throw new ServiceException(String.format("Resource owner id %d does not exist", userId));
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, fullName);
-
- String[] aliasArr = fullName.split("/");
- String alias = aliasArr[aliasArr.length - 1];
- String localFileName = FileUtils.getDownloadFilename(alias);
- log.info("Resource path is {}, download local filename is {}", alias, localFileName);
-
- try {
- storageOperate.download(fullName, localFileName, true);
- ApiServerMetrics.recordApiResourceDownloadSize(java.nio.file.Files.size(Paths.get(localFileName)));
- return org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName);
- } catch (IOException e) {
- log.error("Download resource error, the path is {}, and local filename is {}, the error message is {}",
- fullName, localFileName, e.getMessage());
- throw new ServiceException("Download the resource file failed ,it may be related to your storage");
+ FileUtils.deleteFile(localTmpFileAbsolutePath);
}
}
@Override
- public StorageEntity queryFileStatus(String userName, String fileName) throws Exception {
- // TODO: It is used in PythonGateway, should be revised
- User user = userMapper.queryByUserNameAccurately(userName);
-
- String defaultPath = storageOperate.getResDir(user.getTenantCode());
- return storageOperate.getFileStatus(defaultPath + fileName, defaultPath, user.getTenantCode(),
- ResourceType.FILE);
+ public StorageEntity queryFileStatus(String userName, String fileAbsolutePath) {
+ return storageOperator.getStorageEntity(fileAbsolutePath);
}
@Override
- public DeleteDataTransferResponse deleteDataTransferData(User loginUser, Integer days) {
- DeleteDataTransferResponse result = new DeleteDataTransferResponse();
+ public String queryResourceBaseDir(User loginUser, ResourceType type) {
User user = userMapper.selectById(loginUser.getId());
if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
+ throw new ServiceException(Status.USER_NOT_EXIST);
}
- String tenantCode = getTenantCode(user);
-
- String baseFolder = storageOperate.getResourceFullName(tenantCode, "DATA_TRANSFER");
-
- LocalDateTime now = LocalDateTime.now();
- now = now.minus(days, ChronoUnit.DAYS);
- String deleteDate = now.toLocalDate().toString().replace("-", "");
- List storageEntities;
- try {
- storageEntities = new ArrayList<>(
- storageOperate.listFilesStatus(baseFolder, baseFolder, tenantCode, ResourceType.FILE));
- } catch (Exception e) {
- log.error("delete data transfer data error", e);
- putMsg(result, Status.DELETE_RESOURCE_ERROR);
- return result;
- }
-
- List successList = new ArrayList<>();
- List failList = new ArrayList<>();
-
- for (StorageEntity storageEntity : storageEntities) {
- File path = new File(storageEntity.getFullName());
- String date = path.getName();
- if (date.compareTo(deleteDate) <= 0) {
- try {
- storageOperate.delete(storageEntity.getFullName(), true);
- successList.add(storageEntity.getFullName());
- } catch (Exception ex) {
- log.error("delete data transfer data {} error, please delete it manually", date, ex);
- failList.add(storageEntity.getFullName());
- }
- }
- }
-
- result.setSuccessList(successList);
- result.setFailedList(failList);
- putMsg(result, Status.SUCCESS);
- return result;
+ Tenant tenant = tenantDao.queryOptionalById(user.getTenantId())
+ .orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST));
+ return storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), type);
}
- /**
- * get resource base dir
- *
- * @param loginUser login user
- * @param type resource type
- * @return
- */
- @Override
- public Result queryResourceBaseDir(User loginUser, ResourceType type) {
- Result result = new Result<>();
- if (storageOperate == null) {
- putMsg(result, Status.SUCCESS);
- result.setData(EMPTY_STRING);
- return result;
- }
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
-
- String baseDir = isAdmin(loginUser) ? storageOperate.getDir(ResourceType.ALL, tenantCode)
- : storageOperate.getDir(type, tenantCode);
-
- putMsg(result, Status.SUCCESS);
- result.setData(baseDir);
-
- return result;
+ // Copy the file to the local file system and return the local file absolute path
+ @SneakyThrows
+ private String copyFileToLocal(MultipartFile multipartFile) {
+ String localTmpFileAbsolutePath = FileUtils.getUploadFileLocalTmpAbsolutePath();
+ FileUtils.copyInputStreamToFile(multipartFile.getInputStream(), localTmpFileAbsolutePath);
+ return localTmpFileAbsolutePath;
}
- /**
- * check permission by comparing login user's tenantCode with tenantCode in the request
- *
- * @param isAdmin is the login user admin
- * @param userTenantCode loginUser's tenantCode
- * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
- * can be different from the login user in the case of logging in as admin users.
- * @return isValid
- */
- private boolean isUserTenantValid(boolean isAdmin, String userTenantCode,
- String resTenantCode) throws ServiceException {
- if (isAdmin) {
- return true;
- }
- if (StringUtils.isEmpty(resTenantCode)) {
- // TODO: resource tenant code will be empty when query resources list, need to be optimized
- return true;
- }
- return resTenantCode.equals(userTenantCode);
+ // Copy the file to the local file system and return the local file absolute path
+ private String copyFileToLocal(String fileContent) {
+ String localTmpFileAbsolutePath = FileUtils.getUploadFileLocalTmpAbsolutePath();
+ FileUtils.writeContent2File(fileContent, localTmpFileAbsolutePath);
+ return localTmpFileAbsolutePath;
}
- private String getTenantCode(User user) {
- Tenant tenant = tenantMapper.queryById(user.getTenantId());
- if (tenant == null) {
- throw new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
- }
- return tenant.getTenantCode();
- }
-
- private void checkFullName(String userTenantCode, String fullName) {
- if (StringUtils.isEmpty(fullName)) {
- return;
- }
- if (FOLDER_SEPARATOR.equalsIgnoreCase(fullName)) {
- return;
- }
- // Avoid returning to the parent directory
- if (fullName.contains("../")) {
- throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName);
- }
- String baseDir = storageOperate.getDir(ResourceType.ALL, userTenantCode);
- if (!StringUtils.startsWith(fullName, baseDir)) {
- throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName);
- }
- }
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
index 2887606a1d..ae3410cb77 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
@@ -24,6 +24,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationCon
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_VERSION_VIEW;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_DEFINITION;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_SWITCH_TO_THIS_VERSION;
+import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters;
import org.apache.dolphinscheduler.api.dto.task.TaskCreateRequest;
import org.apache.dolphinscheduler.api.dto.task.TaskFilterRequest;
@@ -67,8 +68,6 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao;
import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao;
-import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.collections4.CollectionUtils;
@@ -167,11 +166,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
return result;
}
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
- if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
- .taskType(taskDefinitionLog.getTaskType())
- .taskParams(taskDefinitionLog.getTaskParams())
- .dependence(taskDefinitionLog.getDependence())
- .build())) {
+ if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) {
log.warn("Task definition {} parameters are invalid.", taskDefinitionLog.getName());
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName());
return result;
@@ -208,11 +203,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
Project project = projectMapper.queryByCode(taskDefinition.getProjectCode());
projectService.checkProjectAndAuthThrowException(user, project, permissions);
- if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
- .taskType(taskDefinition.getTaskType())
- .taskParams(taskDefinition.getTaskParams())
- .dependence(taskDefinition.getDependence())
- .build())) {
+ if (!checkTaskParameters(taskDefinition.getTaskType(), taskDefinition.getTaskParams())) {
throw new ServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName());
}
}
@@ -321,12 +312,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj);
return result;
}
- if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
- .taskType(taskDefinition.getTaskType())
- .taskParams(taskDefinition.getTaskParams())
- .dependence(taskDefinition.getDependence())
- .build())) {
- log.error("Task definition {} parameters are invalid", taskDefinition.getName());
+ if (!checkTaskParameters(taskDefinition.getTaskType(), taskDefinition.getTaskParams())) {
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName());
return result;
}
@@ -732,13 +718,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj);
return null;
}
- if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
- .taskType(taskDefinitionToUpdate.getTaskType())
- .taskParams(taskDefinitionToUpdate.getTaskParams())
- .dependence(taskDefinitionToUpdate.getDependence())
- .build())) {
- log.warn("Task definition parameters are invalid, taskDefinitionName:{}.",
- taskDefinitionToUpdate.getName());
+ if (!checkTaskParameters(taskDefinitionToUpdate.getTaskType(), taskDefinitionToUpdate.getTaskParams())) {
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionToUpdate.getName());
return null;
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
index e77baed1d3..7bec4d6780 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
@@ -39,7 +39,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -84,7 +84,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
private QueueService queueService;
@Autowired(required = false)
- private StorageOperate storageOperate;
+ private StorageOperator storageOperator;
/**
* Check the tenant new object valid or not
@@ -136,14 +136,13 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
* @param queueId queue id
* @param desc description
* @return create result code
- * @throws Exception exception
*/
@Override
@Transactional(rollbackFor = Exception.class)
public Tenant createTenant(User loginUser,
String tenantCode,
int queueId,
- String desc) throws Exception {
+ String desc) {
if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_CREATE)) {
throw new ServiceException(Status.USER_NO_OPERATION_PERM);
}
@@ -154,7 +153,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
createTenantValid(tenant);
tenantMapper.insert(tenant);
- storageOperate.createTenantDirIfNotExists(tenantCode);
return tenant;
}
@@ -209,11 +207,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
updateTenantValid(existsTenant, updateTenant);
updateTenant.setCreateTime(existsTenant.getCreateTime());
- // updateProcessInstance tenant
- // if the tenant code is modified, the original resource needs to be copied to the new tenant.
- if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode())) {
- storageOperate.createTenantDirIfNotExists(tenantCode);
- }
int update = tenantMapper.updateById(updateTenant);
if (update <= 0) {
throw new ServiceException(Status.UPDATE_TENANT_ERROR);
@@ -262,7 +255,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
}
processInstanceMapper.updateProcessInstanceByTenantCode(tenant.getTenantCode(), Constants.DEFAULT);
- storageOperate.deleteTenant(tenant.getTenantCode());
}
private List getProcessInstancesByTenant(Tenant tenant) {
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
index 1bf7d23a6b..7cf16b3567 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
@@ -28,11 +28,10 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -62,7 +61,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
private UDFUserMapper udfUserMapper;
@Autowired(required = false)
- private StorageOperate storageOperate;
+ private StorageOperator storageOperator;
/**
* create udf function
@@ -107,12 +106,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
return result;
}
- Boolean existResource = false;
- try {
- existResource = storageOperate.exists(fullName);
- } catch (IOException e) {
- log.error("Check resource error: {}", fullName, e);
- }
+ boolean existResource = storageOperator.exists(fullName);
if (!existResource) {
log.error("resource full name {} is not exist", fullName);
@@ -241,7 +235,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
Boolean doesResExist = false;
try {
- doesResExist = storageOperate.exists(fullName);
+ doesResExist = storageOperator.exists(fullName);
} catch (Exception e) {
log.error("udf resource :{} checking error", fullName, e);
result.setCode(Status.RESOURCE_NOT_EXIST.getCode());
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
index 7b9746921c..0e91dc582d 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service.impl;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.USER_MANAGER;
-import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.service.MetricsCleanUpService;
@@ -50,7 +49,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -110,7 +109,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
private ProjectMapper projectMapper;
@Autowired(required = false)
- private StorageOperate storageOperate;
+ private StorageOperator storageOperator;
@Autowired
private K8sNamespaceUserMapper k8sNamespaceUserMapper;
@@ -171,9 +170,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
User user = createUser(userName, userPassword, email, tenantId, phone, queue, state);
- Tenant tenant = tenantMapper.queryById(tenantId);
- storageOperate.createTenantDirIfNotExists(tenant.getTenantCode());
-
log.info("User is created and id is {}.", user.getId());
result.put(Constants.DATA_LIST, user);
putMsg(result, Status.SUCCESS);
@@ -1128,54 +1124,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
return msg;
}
- /**
- * copy resource files
- * xxx unchecked
- *
- * @param resourceComponent resource component
- * @param srcBasePath src base path
- * @param dstBasePath dst base path
- * @throws IOException io exception
- */
- private void copyResourceFiles(String oldTenantCode, String newTenantCode, ResourceComponent resourceComponent,
- String srcBasePath, String dstBasePath) {
- List components = resourceComponent.getChildren();
-
- try {
- if (CollectionUtils.isNotEmpty(components)) {
- for (ResourceComponent component : components) {
- // verify whether exist
- if (!storageOperate.exists(
- String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()))) {
- log.error("Resource file: {} does not exist, copy error.", component.getFullName());
- throw new ServiceException(Status.RESOURCE_NOT_EXIST);
- }
-
- if (!component.isDirctory()) {
- // copy it to dst
- storageOperate.copy(String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()),
- String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()), false, true);
- continue;
- }
-
- if (CollectionUtils.isEmpty(component.getChildren())) {
- // if not exist,need create it
- if (!storageOperate
- .exists(String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) {
- storageOperate.mkdir(newTenantCode,
- String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()));
- }
- } else {
- copyResourceFiles(oldTenantCode, newTenantCode, component, srcBasePath, dstBasePath);
- }
- }
-
- }
- } catch (IOException e) {
- log.error("copy the resources failed,the error message is {}", e.getMessage());
- }
- }
-
/**
* registry user, default state is 0, default tenant_id is 1, no phone, no queue
*
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
index 014d22af57..9763f21b0f 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
@@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.api.service.impl;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
-
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
@@ -39,6 +37,7 @@ import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
+import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.commons.lang3.StringUtils;
@@ -147,7 +146,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
List processDefinitionCodes) {
for (DependentProcessDefinition dependentProcessDefinition : dependentDefinitionList) {
for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters()
- .getDependTaskList()) {
+ .getDependence().getDependTaskList()) {
for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) {
if (!processDefinitionCodes.contains(dependentItem.getDefinitionCode())) {
processDefinitionCodes.add(dependentItem.getDefinitionCode());
@@ -220,12 +219,12 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
List taskDefinitionLogs = taskDefinitionLogMapper.queryByTaskDefinitions(taskDefinitionList);
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
if (taskDefinitionLog.getProjectCode() == projectCode) {
- if (taskDefinitionLog.getTaskType().equals(TASK_TYPE_DEPENDENT)) {
+ if (TaskTypeUtils.isDependentTask(taskDefinitionLog.getTaskType())) {
DependentParameters dependentParameters =
JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class);
if (dependentParameters != null) {
List dependTaskList =
- dependentParameters.getDependTaskList();
+ dependentParameters.getDependence().getDependTaskList();
if (!CollectionUtils.isEmpty(dependTaskList)) {
for (DependentTaskModel taskModel : dependTaskList) {
List dependItemList = taskModel.getDependItemList();
@@ -247,9 +246,9 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
/**
* Query and return tasks dependence with string format, is a wrapper of queryTaskDepOnTask and task query method.
*
- * @param projectCode Project code want to query tasks dependence
+ * @param projectCode Project code want to query tasks dependence
* @param processDefinitionCode Process definition code want to query tasks dependence
- * @param taskCode Task code want to query tasks dependence
+ * @param taskCode Task code want to query tasks dependence
* @return Optional of formatter message
*/
@Override
@@ -271,7 +270,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
/**
* Query tasks depend on process definition, include upstream or downstream
*
- * @param projectCode Project code want to query tasks dependence
+ * @param projectCode Project code want to query tasks dependence
* @param processDefinitionCode Process definition code want to query tasks dependence
* @return Set of TaskMainInfo
*/
@@ -291,7 +290,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
* Query downstream tasks depend on a process definition or a task
*
* @param processDefinitionCode Process definition code want to query tasks dependence
- * @param taskCode Task code want to query tasks dependence
+ * @param taskCode Task code want to query tasks dependence
* @return downstream dependent tasks
*/
@Override
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
index 9c98880740..4652347987 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
@@ -32,7 +32,6 @@ import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper;
@@ -41,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.registry.api.RegistryClient;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.service.process.ProcessService;
@@ -357,12 +357,12 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
workerGroups = workerGroupMapper.queryAllWorkerGroup();
}
boolean containDefaultWorkerGroups = workerGroups.stream()
- .anyMatch(workerGroup -> Constants.DEFAULT_WORKER_GROUP.equals(workerGroup.getName()));
+ .anyMatch(workerGroup -> WorkerGroupUtils.isWorkerGroupEmpty(workerGroup.getName()));
if (!containDefaultWorkerGroups) {
// there doesn't exist a default WorkerGroup, we will add all worker to the default worker group.
Set activeWorkerNodes = registryClient.getServerNodeSet(RegistryNodeType.WORKER);
WorkerGroup defaultWorkerGroup = new WorkerGroup();
- defaultWorkerGroup.setName(Constants.DEFAULT_WORKER_GROUP);
+ defaultWorkerGroup.setName(WorkerGroupUtils.getDefaultWorkerGroup());
defaultWorkerGroup.setAddrList(String.join(Constants.COMMA, activeWorkerNodes));
defaultWorkerGroup.setCreateTime(new Date());
defaultWorkerGroup.setUpdateTime(new Date());
@@ -431,27 +431,6 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
return result;
}
- @Override
- public String getTaskWorkerGroup(TaskInstance taskInstance) {
- if (taskInstance == null) {
- return null;
- }
-
- String workerGroup = taskInstance.getWorkerGroup();
-
- if (StringUtils.isNotEmpty(workerGroup)) {
- return workerGroup;
- }
- int processInstanceId = taskInstance.getProcessInstanceId();
- ProcessInstance processInstance = processService.findProcessInstanceById(processInstanceId);
-
- if (processInstance != null) {
- return processInstance.getWorkerGroup();
- }
- log.info("task : {} will use default worker group", taskInstance.getId());
- return Constants.DEFAULT_WORKER_GROUP;
- }
-
@Override
public Map queryWorkerGroupByProcessDefinitionCodes(List processDefinitionCodeList) {
List processDefinitionScheduleList =
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java
index 2cfdd8f840..85e47c96b9 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java
@@ -20,12 +20,16 @@ package org.apache.dolphinscheduler.api.utils;
import java.util.Collections;
import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
import lombok.Data;
import lombok.Setter;
import com.baomidou.mybatisplus.core.metadata.IPage;
@Data
+@Builder
+@AllArgsConstructor
public class PageInfo {
/**
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/ITransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/ITransformer.java
new file mode 100644
index 0000000000..8aedad3a90
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/ITransformer.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator;
+
+public interface ITransformer {
+
+ R transform(T t);
+
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/IValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/IValidator.java
new file mode 100644
index 0000000000..7570fa67d5
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/IValidator.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator;
+
+public interface IValidator {
+
+ void validate(T t);
+
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceTransformer.java
new file mode 100644
index 0000000000..4f721188dc
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceTransformer.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.exceptions.ServiceException;
+import org.apache.dolphinscheduler.api.validator.ITransformer;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
+
+import org.apache.commons.lang3.StringUtils;
+
+import lombok.AllArgsConstructor;
+
+@AllArgsConstructor
+public abstract class AbstractResourceTransformer implements ITransformer {
+
+ protected TenantDao tenantDao;
+
+ protected StorageOperator storageOperator;
+
+ protected String getParentDirectoryAbsolutePath(User loginUser, String parentAbsoluteDirectory, ResourceType type) {
+ String tenantCode = tenantDao.queryOptionalById(loginUser.getTenantId())
+ .orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST))
+ .getTenantCode();
+ String userResRootPath = storageOperator.getStorageBaseDirectory(tenantCode, type);
+ // If the parent directory is / then will transform to userResRootPath
+ // This only happens when the front-end go into the resource page first
+ // todo: we need to change the front-end logic to avoid this
+ if (parentAbsoluteDirectory.equals("/")) {
+ return userResRootPath;
+ }
+
+ if (!StringUtils.startsWith(parentAbsoluteDirectory, userResRootPath)) {
+ throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, parentAbsoluteDirectory);
+ }
+ return parentAbsoluteDirectory;
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceValidator.java
new file mode 100644
index 0000000000..35656b4d82
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceValidator.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.AbstractResourceDto;
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.exceptions.ServiceException;
+import org.apache.dolphinscheduler.api.validator.IValidator;
+import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.springframework.web.multipart.MultipartFile;
+
+import com.google.common.io.Files;
+
+public abstract class AbstractResourceValidator implements IValidator {
+
+ private static final Set FILE_SUFFIXES_WHICH_CAN_FETCH_CONTENT = new HashSet<>(Arrays.asList(
+ StringUtils.defaultIfBlank(FileUtils.getResourceViewSuffixes(), "").split(",")));
+
+ protected final StorageOperator storageOperator;
+
+ private final TenantDao tenantDao;
+
+ public AbstractResourceValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ this.storageOperator = storageOperator;
+ this.tenantDao = tenantDao;
+ }
+
+ public void exceptionResourceAbsolutePathInvalidated(String resourceAbsolutePath) {
+ if (StringUtils.isBlank(resourceAbsolutePath)) {
+ throw new ServiceException("The resource path is null");
+ }
+ if (!resourceAbsolutePath.startsWith(storageOperator.getStorageBaseDirectory())) {
+ throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath);
+ }
+ if (resourceAbsolutePath.contains("..")) {
+ throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath);
+ }
+ }
+
+ public void exceptionFileInvalidated(MultipartFile file) {
+ if (file == null) {
+ throw new ServiceException("The file is null");
+ }
+ }
+
+ public void exceptionFileContentInvalidated(String fileContent) {
+ if (StringUtils.isEmpty(fileContent)) {
+ throw new ServiceException("The file content is null");
+ }
+ }
+
+ public void exceptionFileContentCannotFetch(String fileAbsolutePath) {
+ String fileExtension = Files.getFileExtension(fileAbsolutePath);
+ if (!FILE_SUFFIXES_WHICH_CAN_FETCH_CONTENT.contains(fileExtension)) {
+ throw new ServiceException("The file type: " + fileExtension + " cannot be fetched");
+ }
+ }
+
+ public void exceptionResourceNotExists(String resourceAbsolutePath) {
+ if (!storageOperator.exists(resourceAbsolutePath)) {
+ throw new ServiceException("Thr resource is not exists: " + resourceAbsolutePath);
+ }
+ }
+
+ public void exceptionResourceExists(String resourceAbsolutePath) {
+ if (storageOperator.exists(resourceAbsolutePath)) {
+ throw new ServiceException("The resource is already exist: " + resourceAbsolutePath);
+ }
+ }
+
+ public void exceptionResourceIsNotDirectory(String resourceAbsolutePath) {
+ if (StringUtils.isNotEmpty(Files.getFileExtension(resourceAbsolutePath))) {
+ throw new ServiceException("The path is not a directory: " + resourceAbsolutePath);
+ }
+ }
+
+ public void exceptionResourceIsNotFile(String fileAbsolutePath) {
+ if (StringUtils.isEmpty(Files.getFileExtension(fileAbsolutePath))) {
+ throw new ServiceException("The path is not a file: " + fileAbsolutePath);
+ }
+ }
+
+ public void exceptionUserNoResourcePermission(User user, AbstractResourceDto resourceDto) {
+ exceptionUserNoResourcePermission(user, resourceDto.getResourceAbsolutePath());
+ }
+
+ public void exceptionUserNoResourcePermission(User user, String resourceAbsolutePath) {
+ if (user.getUserType() == UserType.ADMIN_USER) {
+ return;
+ }
+ // check if the user have resource tenant permission
+ // Parse the resource path to get the tenant code
+ ResourceMetadata resourceMetaData = storageOperator.getResourceMetaData(resourceAbsolutePath);
+
+ if (!resourceAbsolutePath.startsWith(resourceMetaData.getResourceBaseDirectory())) {
+ throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath);
+ }
+
+ // todo: inject the tenant when login
+ Tenant tenant = tenantDao.queryOptionalById(user.getTenantId())
+ .orElseThrow(() -> new ServiceException(Status.TENANT_NOT_EXIST, user.getTenantId()));
+ String userTenant = tenant.getTenantCode();
+ if (!userTenant.equals(resourceMetaData.getTenant())) {
+ throw new ServiceException(
+ "The user's tenant is " + userTenant + " have no permission to access the resource: "
+ + resourceAbsolutePath);
+ }
+ }
+
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidator.java
new file mode 100644
index 0000000000..248f7fbff3
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
+import org.apache.dolphinscheduler.api.exceptions.ServiceException;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.springframework.stereotype.Component;
+
+import com.google.common.io.Files;
+
+@Component
+public class CreateDirectoryDtoValidator extends AbstractResourceValidator {
+
+ public CreateDirectoryDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(CreateDirectoryDto createDirectoryDto) {
+ String directoryAbsolutePath = createDirectoryDto.getDirectoryAbsolutePath();
+
+ exceptionResourceAbsolutePathInvalidated(directoryAbsolutePath);
+ exceptionResourceExists(directoryAbsolutePath);
+ exceptionUserNoResourcePermission(createDirectoryDto.getLoginUser(), directoryAbsolutePath);
+ exceptionResourceIsNotDirectory(directoryAbsolutePath);
+ if (StringUtils.isNotEmpty(Files.getFileExtension(directoryAbsolutePath))) {
+ throw new ServiceException("The path is not a directory: " + directoryAbsolutePath);
+ }
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryRequestTransformer.java
new file mode 100644
index 0000000000..75985477f3
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryRequestTransformer.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.exceptions.ServiceException;
+import org.apache.dolphinscheduler.api.validator.ITransformer;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class CreateDirectoryRequestTransformer implements ITransformer {
+
+ @Autowired
+ private TenantDao tenantDao;
+
+ @Autowired
+ private StorageOperator storageOperator;
+
+ @Override
+ public CreateDirectoryDto transform(CreateDirectoryRequest createDirectoryRequest) {
+ validateCreateDirectoryRequest(createDirectoryRequest);
+ return doTransform(createDirectoryRequest);
+ }
+
+ private CreateDirectoryDto doTransform(CreateDirectoryRequest createDirectoryRequest) {
+ String directoryAbsolutePath = getDirectoryAbsolutePath(createDirectoryRequest);
+ return CreateDirectoryDto.builder()
+ .loginUser(createDirectoryRequest.getLoginUser())
+ .directoryAbsolutePath(directoryAbsolutePath)
+ .build();
+ }
+
+ private void validateCreateDirectoryRequest(CreateDirectoryRequest createDirectoryRequest) {
+ checkNotNull(createDirectoryRequest.getLoginUser(), "loginUser is null");
+ checkNotNull(createDirectoryRequest.getType(), "resource type is null");
+ checkNotNull(createDirectoryRequest.getDirectoryName(), "directory name is null");
+ checkNotNull(createDirectoryRequest.getParentAbsoluteDirectory(), "parent directory is null");
+
+ }
+
+ private String getDirectoryAbsolutePath(CreateDirectoryRequest createDirectoryRequest) {
+ String tenantCode = tenantDao.queryOptionalById(createDirectoryRequest.getLoginUser().getTenantId())
+ .orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST))
+ .getTenantCode();
+ String userResRootPath = storageOperator.getStorageBaseDirectory(tenantCode, createDirectoryRequest.getType());
+ String parentDirectoryName = createDirectoryRequest.getParentAbsoluteDirectory();
+ String directoryName = createDirectoryRequest.getDirectoryName();
+
+ // If the parent directory is / then will transform to userResRootPath
+ // This only happens when the front-end go into the resource page first
+ // todo: we need to change the front-end logic to avoid this
+ if (parentDirectoryName.equals("/")) {
+ return FileUtils.concatFilePath(userResRootPath, directoryName);
+ }
+
+ if (!StringUtils.startsWith(parentDirectoryName, userResRootPath)) {
+ throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, parentDirectoryName);
+ }
+ return FileUtils.concatFilePath(parentDirectoryName, directoryName);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileDtoValidator.java
new file mode 100644
index 0000000000..d2c91387a6
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileDtoValidator.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+import org.springframework.web.multipart.MultipartFile;
+
+@Component
+public class CreateFileDtoValidator extends AbstractResourceValidator {
+
+ public CreateFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(CreateFileDto createFileDto) {
+ String fileAbsolutePath = createFileDto.getFileAbsolutePath();
+ User loginUser = createFileDto.getLoginUser();
+ MultipartFile file = createFileDto.getFile();
+
+ exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
+ exceptionResourceExists(fileAbsolutePath);
+ exceptionFileInvalidated(file);
+ exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
+ exceptionResourceIsNotFile(fileAbsolutePath);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidator.java
new file mode 100644
index 0000000000..ae61f47041
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class CreateFileFromContentDtoValidator extends AbstractResourceValidator {
+
+ public CreateFileFromContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(CreateFileFromContentDto createFileFromContentDto) {
+ String fileAbsolutePath = createFileFromContentDto.getFileAbsolutePath();
+ User loginUser = createFileFromContentDto.getLoginUser();
+ String fileContent = createFileFromContentDto.getFileContent();
+
+ exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
+ exceptionResourceIsNotFile(fileAbsolutePath);
+ exceptionResourceExists(fileAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
+ exceptionFileContentInvalidated(fileContent);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DeleteResourceDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DeleteResourceDtoValidator.java
new file mode 100644
index 0000000000..e337d9e07c
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DeleteResourceDtoValidator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceDto;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class DeleteResourceDtoValidator extends AbstractResourceValidator {
+
+ public DeleteResourceDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(DeleteResourceDto deleteResourceDto) {
+ String resourceAbsolutePath = deleteResourceDto.getResourceAbsolutePath();
+ User loginUser = deleteResourceDto.getLoginUser();
+
+ exceptionResourceAbsolutePathInvalidated(resourceAbsolutePath);
+ exceptionResourceNotExists(resourceAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, resourceAbsolutePath);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DownloadFileDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DownloadFileDtoValidator.java
new file mode 100644
index 0000000000..ac15d279ac
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DownloadFileDtoValidator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.DownloadFileDto;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class DownloadFileDtoValidator extends AbstractResourceValidator {
+
+ public DownloadFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(DownloadFileDto downloadFileDto) {
+ String fileAbsolutePath = downloadFileDto.getFileAbsolutePath();
+ User loginUser = downloadFileDto.getLoginUser();
+
+ exceptionResourceNotExists(fileAbsolutePath);
+ exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
+ exceptionResourceIsNotFile(fileAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidator.java
new file mode 100644
index 0000000000..b69ec68a7c
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidator.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto;
+import org.apache.dolphinscheduler.api.exceptions.ServiceException;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class FetchFileContentDtoValidator extends AbstractResourceValidator {
+
+ public FetchFileContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(FetchFileContentDto fetchFileContentDto) {
+ if (fetchFileContentDto.getSkipLineNum() < 0) {
+ throw new ServiceException("skipLineNum must be greater than or equal to 0");
+ }
+ String resourceFileAbsolutePath = fetchFileContentDto.getResourceFileAbsolutePath();
+ User loginUser = fetchFileContentDto.getLoginUser();
+
+ exceptionResourceAbsolutePathInvalidated(resourceFileAbsolutePath);
+ exceptionResourceIsNotFile(resourceFileAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, resourceFileAbsolutePath);
+ exceptionFileContentCannotFetch(resourceFileAbsolutePath);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileFromContentRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileFromContentRequestTransformer.java
new file mode 100644
index 0000000000..7f29646e36
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileFromContentRequestTransformer.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class FileFromContentRequestTransformer
+ extends
+ AbstractResourceTransformer {
+
+ public FileFromContentRequestTransformer(TenantDao tenantDao, StorageOperator storageOperator) {
+ super(tenantDao, storageOperator);
+ }
+
+ @Override
+ public CreateFileFromContentDto transform(CreateFileFromContentRequest createFileFromContentRequest) {
+ validateCreateFileRequest(createFileFromContentRequest);
+ return doTransform(createFileFromContentRequest);
+ }
+
+ private void validateCreateFileRequest(CreateFileFromContentRequest createFileFromContentRequest) {
+ checkNotNull(createFileFromContentRequest.getLoginUser(), "loginUser is null");
+ checkNotNull(createFileFromContentRequest.getType(), "resource type is null");
+ checkNotNull(createFileFromContentRequest.getFileName(), "file name is null");
+ checkNotNull(createFileFromContentRequest.getParentAbsoluteDirectory(), "parent directory is null");
+ checkNotNull(createFileFromContentRequest.getFileContent(), "file content is null");
+ }
+
+ private CreateFileFromContentDto doTransform(CreateFileFromContentRequest createFileFromContentRequest) {
+ String fileAbsolutePath = getFileAbsolutePath(createFileFromContentRequest);
+ return CreateFileFromContentDto.builder()
+ .loginUser(createFileFromContentRequest.getLoginUser())
+ .fileAbsolutePath(fileAbsolutePath)
+ .fileContent(createFileFromContentRequest.getFileContent())
+ .build();
+
+ }
+
+ private String getFileAbsolutePath(CreateFileFromContentRequest createFileFromContentRequest) {
+ String parentDirectoryAbsolutePath = getParentDirectoryAbsolutePath(
+ createFileFromContentRequest.getLoginUser(),
+ createFileFromContentRequest.getParentAbsoluteDirectory(),
+ createFileFromContentRequest.getType());
+ return FileUtils.concatFilePath(parentDirectoryAbsolutePath, createFileFromContentRequest.getFileName());
+ }
+
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileRequestTransformer.java
new file mode 100644
index 0000000000..c2f007dbf7
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileRequestTransformer.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class FileRequestTransformer extends AbstractResourceTransformer {
+
+ public FileRequestTransformer(TenantDao tenantDao, StorageOperator storageOperator) {
+ super(tenantDao, storageOperator);
+ }
+
+ @Override
+ public CreateFileDto transform(CreateFileRequest createFileRequest) {
+ validateCreateFileRequest(createFileRequest);
+ return doTransform(createFileRequest);
+ }
+
+ private void validateCreateFileRequest(CreateFileRequest createFileRequest) {
+ checkNotNull(createFileRequest.getLoginUser(), "loginUser is null");
+ checkNotNull(createFileRequest.getType(), "resource type is null");
+ checkNotNull(createFileRequest.getFileName(), "file name is null");
+ checkNotNull(createFileRequest.getParentAbsoluteDirectory(), "parent directory is null");
+ checkNotNull(createFileRequest.getFile(), "file is null");
+ }
+
+ private CreateFileDto doTransform(CreateFileRequest createFileRequest) {
+ String fileAbsolutePath = getFileAbsolutePath(createFileRequest);
+ return CreateFileDto.builder()
+ .loginUser(createFileRequest.getLoginUser())
+ .file(createFileRequest.getFile())
+ .fileAbsolutePath(fileAbsolutePath)
+ .build();
+
+ }
+
+ private String getFileAbsolutePath(CreateFileRequest createFileRequest) {
+ String parentDirectoryAbsolutePath = getParentDirectoryAbsolutePath(
+ createFileRequest.getLoginUser(),
+ createFileRequest.getParentAbsoluteDirectory(),
+ createFileRequest.getType());
+ return FileUtils.concatFilePath(parentDirectoryAbsolutePath, createFileRequest.getFileName());
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/PagingResourceItemRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/PagingResourceItemRequestTransformer.java
new file mode 100644
index 0000000000..9be51e413e
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/PagingResourceItemRequestTransformer.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
+import org.apache.dolphinscheduler.api.dto.resources.QueryResourceDto;
+import org.apache.dolphinscheduler.api.validator.ITransformer;
+import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.AllArgsConstructor;
+
+import org.springframework.stereotype.Component;
+
+import com.google.common.collect.Lists;
+
+@Component
+@AllArgsConstructor
+public class PagingResourceItemRequestTransformer implements ITransformer {
+
+ private final StorageOperator storageOperator;
+
+ private final TenantDao tenantDao;
+
+ @Override
+ public QueryResourceDto transform(PagingResourceItemRequest pagingResourceItemRequest) {
+ validatePagingResourceItemRequest(pagingResourceItemRequest);
+
+ if (StringUtils.isNotEmpty(pagingResourceItemRequest.getResourceAbsolutePath())) {
+ // query from the given path
+ return QueryResourceDto.builder()
+ .resourceAbsolutePaths(Lists.newArrayList(pagingResourceItemRequest.getResourceAbsolutePath()))
+ .build();
+ }
+
+ ResourceType resourceType = pagingResourceItemRequest.getResourceType();
+ User loginUser = pagingResourceItemRequest.getLoginUser();
+ if (loginUser.getUserType() == UserType.ADMIN_USER) {
+ // If the current user is admin
+ // then will query all tenant resources
+ List resourceAbsolutePaths = tenantDao.queryAll()
+ .stream()
+ .map(tenant -> storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType))
+ .collect(Collectors.toList());
+ return QueryResourceDto.builder()
+ .resourceAbsolutePaths(resourceAbsolutePaths)
+ .build();
+ } else {
+ // todo: inject the tenantCode when login
+ Tenant tenant = tenantDao.queryById(loginUser.getTenantId());
+ String storageBaseDirectory = storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType);
+ return QueryResourceDto.builder()
+ .resourceAbsolutePaths(Lists.newArrayList(storageBaseDirectory))
+ .build();
+ }
+
+ }
+
+ private void validatePagingResourceItemRequest(PagingResourceItemRequest pagingResourceItemRequest) {
+ checkNotNull(pagingResourceItemRequest.getLoginUser(), "loginUser is null");
+ checkNotNull(pagingResourceItemRequest.getResourceType(), "resourceType is null");
+ }
+
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidator.java
new file mode 100644
index 0000000000..b76ea5b6fc
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class RenameDirectoryDtoValidator extends AbstractResourceValidator {
+
+ public RenameDirectoryDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(RenameDirectoryDto renameDirectoryDto) {
+ String originDirectoryAbsolutePath = renameDirectoryDto.getOriginDirectoryAbsolutePath();
+ User loginUser = renameDirectoryDto.getLoginUser();
+ String targetDirectoryAbsolutePath = renameDirectoryDto.getTargetDirectoryAbsolutePath();
+
+ exceptionResourceAbsolutePathInvalidated(originDirectoryAbsolutePath);
+ exceptionResourceIsNotDirectory(originDirectoryAbsolutePath);
+ exceptionResourceNotExists(originDirectoryAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, originDirectoryAbsolutePath);
+
+ exceptionResourceAbsolutePathInvalidated(targetDirectoryAbsolutePath);
+ exceptionResourceIsNotDirectory(targetDirectoryAbsolutePath);
+ exceptionResourceExists(targetDirectoryAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, targetDirectoryAbsolutePath);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryRequestTransformer.java
new file mode 100644
index 0000000000..4d6745df3d
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryRequestTransformer.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
+import org.apache.dolphinscheduler.api.validator.ITransformer;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.File;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Component
+public class RenameDirectoryRequestTransformer implements ITransformer {
+
+ @Override
+ public RenameDirectoryDto transform(RenameDirectoryRequest renameDirectoryRequest) {
+ String originDirectoryAbsolutePath = renameDirectoryRequest.getDirectoryAbsolutePath();
+ String targetDirectoryName = renameDirectoryRequest.getNewDirectoryName();
+
+ String targetDirectoryAbsolutePath =
+ getTargetDirectoryAbsolutePath(originDirectoryAbsolutePath, targetDirectoryName);
+
+ return RenameDirectoryDto.builder()
+ .loginUser(renameDirectoryRequest.getLoginUser())
+ .originDirectoryAbsolutePath(originDirectoryAbsolutePath)
+ .targetDirectoryAbsolutePath(targetDirectoryAbsolutePath)
+ .build();
+ }
+
+ private String getTargetDirectoryAbsolutePath(String originDirectoryAbsolutePath, String targetDirectoryName) {
+ String originDirectoryParentAbsolutePath = StringUtils.substringBeforeLast(
+ originDirectoryAbsolutePath, File.separator);
+ return originDirectoryParentAbsolutePath + File.separator + targetDirectoryName;
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidator.java
new file mode 100644
index 0000000000..8c9e333dbb
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class RenameFileDtoValidator extends AbstractResourceValidator {
+
+ public RenameFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(RenameFileDto renameFileDto) {
+ String originFileAbsolutePath = renameFileDto.getOriginFileAbsolutePath();
+ User loginUser = renameFileDto.getLoginUser();
+ String targetFileAbsolutePath = renameFileDto.getTargetFileAbsolutePath();
+
+ exceptionResourceAbsolutePathInvalidated(originFileAbsolutePath);
+ exceptionResourceNotExists(originFileAbsolutePath);
+ exceptionResourceIsNotFile(originFileAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, originFileAbsolutePath);
+
+ exceptionResourceAbsolutePathInvalidated(targetFileAbsolutePath);
+ exceptionResourceExists(targetFileAbsolutePath);
+ exceptionResourceIsNotFile(targetFileAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, targetFileAbsolutePath);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileRequestTransformer.java
new file mode 100644
index 0000000000..43cb55c461
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileRequestTransformer.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
+import org.apache.dolphinscheduler.api.validator.ITransformer;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class RenameFileRequestTransformer implements ITransformer {
+
+ @Autowired
+ private StorageOperator storageOperator;
+
+ @Override
+ public RenameFileDto transform(RenameFileRequest renameFileRequest) {
+ ResourceMetadata resourceMetaData =
+ storageOperator.getResourceMetaData(renameFileRequest.getFileAbsolutePath());
+ return RenameFileDto.builder()
+ .loginUser(renameFileRequest.getLoginUser())
+ .originFileAbsolutePath(renameFileRequest.getFileAbsolutePath())
+ .targetFileAbsolutePath(FileUtils.concatFilePath(resourceMetaData.getResourceParentAbsolutePath(),
+ renameFileRequest.getNewFileName()))
+ .build();
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileDtoValidator.java
new file mode 100644
index 0000000000..468c651087
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileDtoValidator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto;
+import org.apache.dolphinscheduler.api.exceptions.ServiceException;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import java.util.Objects;
+
+import org.springframework.stereotype.Component;
+import org.springframework.web.multipart.MultipartFile;
+
+import com.google.common.io.Files;
+
+@Component
+public class UpdateFileDtoValidator extends AbstractResourceValidator {
+
+ public UpdateFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(UpdateFileDto updateFileDto) {
+ String fileAbsolutePath = updateFileDto.getFileAbsolutePath();
+ User loginUser = updateFileDto.getLoginUser();
+ MultipartFile file = updateFileDto.getFile();
+
+ if (!Objects.equals(Files.getFileExtension(file.getName()),
+ Files.getFileExtension(updateFileDto.getFileAbsolutePath()))) {
+ throw new ServiceException("file extension cannot not change");
+ }
+
+ exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
+ exceptionResourceNotExists(fileAbsolutePath);
+ exceptionResourceIsNotFile(fileAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
+ exceptionFileInvalidated(file);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentDtoValidator.java
new file mode 100644
index 0000000000..5245759639
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentDtoValidator.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class UpdateFileFromContentDtoValidator extends AbstractResourceValidator {
+
+ public UpdateFileFromContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
+ super(storageOperator, tenantDao);
+ }
+
+ @Override
+ public void validate(UpdateFileFromContentDto updateFileFromContentDto) {
+ String fileAbsolutePath = updateFileFromContentDto.getFileAbsolutePath();
+ User loginUser = updateFileFromContentDto.getLoginUser();
+ String fileContent = updateFileFromContentDto.getFileContent();
+
+ exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
+ exceptionResourceNotExists(fileAbsolutePath);
+ exceptionResourceIsNotFile(fileAbsolutePath);
+ exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
+ exceptionFileContentCannotFetch(fileAbsolutePath);
+ exceptionFileContentInvalidated(fileContent);
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentRequestTransformer.java
new file mode 100644
index 0000000000..961d0e6049
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentRequestTransformer.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
+import org.apache.dolphinscheduler.api.validator.ITransformer;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class UpdateFileFromContentRequestTransformer
+ implements
+ ITransformer {
+
+ @Override
+ public UpdateFileFromContentDto transform(UpdateFileFromContentRequest updateFileContentRequest) {
+ return UpdateFileFromContentDto.builder()
+ .loginUser(updateFileContentRequest.getLoginUser())
+ .fileAbsolutePath(updateFileContentRequest.getFileAbsolutePath())
+ .fileContent(updateFileContentRequest.getFileContent())
+ .build();
+ }
+}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/common/UiChannelFactory.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileRequestTransformer.java
similarity index 51%
rename from dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/common/UiChannelFactory.java
rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileRequestTransformer.java
index 8b89215049..5c7646fbf9 100644
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/common/UiChannelFactory.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileRequestTransformer.java
@@ -15,29 +15,24 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.spi.common;
+package org.apache.dolphinscheduler.api.validator.resource;
-import org.apache.dolphinscheduler.spi.params.base.PluginParams;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
+import org.apache.dolphinscheduler.api.validator.ITransformer;
-import java.util.List;
+import org.springframework.stereotype.Component;
-public interface UiChannelFactory {
+@Component
+public class UpdateFileRequestTransformer implements ITransformer {
- /**
- * plugin name
- * Must be UNIQUE .
- * This alert plugin name eg: email , message ...
- * Name can often be displayed on the page ui eg : email , message , MR , spark , hive ...
- *
- * @return this alert plugin name
- */
- String getName();
-
- /**
- * Returns the configurable parameters that this plugin needs to display on the web ui
- *
- * @return this alert plugin params
- */
- List getParams();
+ @Override
+ public UpdateFileDto transform(UpdateFileRequest updateFileRequest) {
+ return UpdateFileDto.builder()
+ .loginUser(updateFileRequest.getLoginUser())
+ .fileAbsolutePath(updateFileRequest.getFileAbsolutePath())
+ .file(updateFileRequest.getFile())
+ .build();
+ }
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/ResourceItemVO.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/ResourceItemVO.java
new file mode 100644
index 0000000000..9470ded4ba
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/ResourceItemVO.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.vo;
+
+import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.File;
+import java.util.Date;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class ResourceItemVO {
+
+ // todo: remove this field, directly use fileName
+ private String alias;
+
+ // todo: use tenantName instead of userName
+ private String userName;
+
+ private String fileName;
+
+ private String fullName;
+
+ private boolean isDirectory;
+
+ private ResourceType type;
+
+ private long size;
+
+ private Date createTime;
+
+ private Date updateTime;
+
+ public ResourceItemVO(StorageEntity storageEntity) {
+ this.isDirectory = storageEntity.isDirectory();
+ this.alias = storageEntity.getFileName();
+ this.fileName = storageEntity.getFileName();
+ this.fullName = storageEntity.getFullName();
+ this.type = storageEntity.getType();
+ this.size = storageEntity.getSize();
+ this.createTime = storageEntity.getCreateTime();
+ this.updateTime = storageEntity.getUpdateTime();
+
+ if (isDirectory) {
+ alias = StringUtils.removeEndIgnoreCase(alias, File.separator);
+ fileName = StringUtils.removeEndIgnoreCase(fileName, File.separator);
+ fullName = StringUtils.removeEndIgnoreCase(fullName, File.separator);
+ }
+ }
+
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/resources/FetchFileContentResponse.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/resources/FetchFileContentResponse.java
new file mode 100644
index 0000000000..1f228f42fb
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/resources/FetchFileContentResponse.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.vo.resources;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class FetchFileContentResponse {
+
+ private String content;
+
+}
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml
index e38d0c5a8e..9b0e94d644 100644
--- a/dolphinscheduler-api/src/main/resources/application.yaml
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -75,6 +75,8 @@ spring:
pathmatch:
matching-strategy: ANT_PATH_MATCHER
static-path-pattern: /static/**
+ cloud.discovery.client.composite-indicator.enabled: false
+
springdoc:
swagger-ui:
path: /swagger-ui.html
@@ -118,8 +120,8 @@ registry:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
- base-sleep-time: 60ms
- max-sleep: 300ms
+ base-sleep-time: 1s
+ max-sleep: 3s
max-retries: 5
session-timeout: 60s
connection-timeout: 15s
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java
index eae064bb24..21977bd50d 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java
@@ -27,6 +27,11 @@ import org.junit.jupiter.api.function.Executable;
public class AssertionsHelper extends Assertions {
+ public static void assertThrowServiceException(String message, Executable executable) {
+ ServiceException exception = Assertions.assertThrows(ServiceException.class, executable);
+ Assertions.assertEquals(message, exception.getMessage());
+ }
+
public static void assertThrowsServiceException(Status status, Executable executable) {
ServiceException exception = Assertions.assertThrows(ServiceException.class, executable);
Assertions.assertEquals(status.getCode(), exception.getCode());
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
index bfed64f9f6..01a2d47998 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.api.controller;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
@@ -28,29 +29,24 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.service.UdfFuncService;
import org.apache.dolphinscheduler.api.utils.Result;
-import org.apache.dolphinscheduler.common.constants.Constants;
+import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
-import java.util.HashMap;
-import java.util.Map;
-
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
-/**
- * resources controller test
- */
+import com.fasterxml.jackson.core.type.TypeReference;
+
public class ResourcesControllerTest extends AbstractControllerTest {
private static final Logger logger = LoggerFactory.getLogger(ResourcesControllerTest.class);
@@ -61,37 +57,12 @@ public class ResourcesControllerTest extends AbstractControllerTest {
@MockBean(name = "udfFuncServiceImpl")
private UdfFuncService udfFuncService;
- @Test
- public void testQuerytResourceList() throws Exception {
- Map mockResult = new HashMap<>();
- mockResult.put(Constants.STATUS, Status.SUCCESS);
- Mockito.when(resourcesService.queryResourceList(Mockito.any(), Mockito.any(), Mockito.anyString()))
- .thenReturn(mockResult);
-
- MultiValueMap paramsMap = new LinkedMultiValueMap<>();
- paramsMap.add("fullName", "dolphinscheduler/resourcePath");
- paramsMap.add("type", ResourceType.FILE.name());
- MvcResult mvcResult = mockMvc.perform(get("/resources/list")
- .header(SESSION_ID, sessionId)
- .params(paramsMap))
- .andExpect(status().isOk())
- .andExpect(content().contentType(MediaType.APPLICATION_JSON))
- .andReturn();
-
- Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
-
- Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
- logger.info(mvcResult.getResponse().getContentAsString());
- }
-
@Test
public void testQueryResourceListPaging() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.SUCCESS.getCode());
- Mockito.when(resourcesService.queryResourceListPaging(
- Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
- Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()))
- .thenReturn(mockResult);
+ // Mockito.when(resourcesService.pagingResourceItem()
+ // .thenReturn(mockResult);
MultiValueMap paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("type", String.valueOf(ResourceType.FILE));
@@ -111,41 +82,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
- Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
- logger.info(mvcResult.getResponse().getContentAsString());
- }
-
- @Test
- public void testVerifyResourceName() throws Exception {
- Result mockResult = new Result<>();
- mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
- Mockito.when(resourcesService.verifyResourceName(Mockito.anyString(), Mockito.any(), Mockito.any()))
- .thenReturn(mockResult);
-
- MultiValueMap paramsMap = new LinkedMultiValueMap<>();
- paramsMap.add("fullName", "list_resources_1.sh");
- paramsMap.add("type", "FILE");
-
- MvcResult mvcResult = mockMvc.perform(get("/resources/verify-name")
- .header(SESSION_ID, sessionId)
- .params(paramsMap))
- .andExpect(status().isOk())
- .andExpect(content().contentType(MediaType.APPLICATION_JSON))
- .andReturn();
-
- Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
-
- Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
+ assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void testViewResource() throws Exception {
- Result mockResult = new Result<>();
- mockResult.setCode(Status.HDFS_NOT_STARTUP.getCode());
- Mockito.when(resourcesService.readResource(Mockito.any(),
- Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()))
- .thenReturn(mockResult);
+ FetchFileContentResponse fetchFileContentResponse = FetchFileContentResponse.builder()
+ .content("echo hello")
+ .build();
+ Mockito.when(resourcesService.fetchResourceFileContent(Mockito.any()))
+ .thenReturn(fetchFileContentResponse);
MultiValueMap paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("skipLineNum", "2");
@@ -160,19 +107,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
- Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(),
+ new TypeReference>() {
+ });
- Assertions.assertEquals(Status.HDFS_NOT_STARTUP.getCode(), result.getCode().intValue());
- logger.info(mvcResult.getResponse().getContentAsString());
+ assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
+ assertEquals(fetchFileContentResponse, result.getData());
}
@Test
public void testCreateResourceFile() throws Exception {
- Result mockResult = new Result<>();
- mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
- Mockito.when(resourcesService.createResourceFile(Mockito.any(), Mockito.any(), Mockito.anyString(),
- Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
- .thenReturn(mockResult);
+ Mockito.doNothing().when(resourcesService).createFileFromContent(Mockito.any());
MultiValueMap paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("type", String.valueOf(ResourceType.FILE));
@@ -190,19 +135,16 @@ public class ResourcesControllerTest extends AbstractControllerTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
- Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+ Result result =
+ JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference>() {
+ });
- Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
- logger.info(mvcResult.getResponse().getContentAsString());
+ assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}
@Test
public void testUpdateResourceContent() throws Exception {
- Result mockResult = new Result<>();
- mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
- Mockito.when(resourcesService.updateResourceContent(Mockito.any(), Mockito.anyString(),
- Mockito.anyString(), Mockito.anyString()))
- .thenReturn(mockResult);
+ Mockito.doNothing().when(resourcesService).updateFileFromContent(Mockito.any());
MultiValueMap paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("id", "1");
@@ -217,17 +159,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
- Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+ Result result =
+ JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference>() {
+ });
- Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
- logger.info(mvcResult.getResponse().getContentAsString());
+ assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}
@Test
public void testDownloadResource() throws Exception {
- Mockito.when(resourcesService.downloadResource(Mockito.any(), Mockito.anyString()))
- .thenReturn(null);
+ Mockito.doNothing().when(resourcesService).downloadResource(Mockito.any(), Mockito.any());
MultiValueMap paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
@@ -235,7 +177,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
MvcResult mvcResult = mockMvc.perform(get("/resources/download")
.params(paramsMap)
.header(SESSION_ID, sessionId))
- .andExpect(status().is(HttpStatus.BAD_REQUEST.value()))
+ .andExpect(status().isOk())
.andReturn();
Assertions.assertNotNull(mvcResult);
@@ -269,7 +211,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
- Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
+ assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@@ -289,7 +231,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
- Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
+ assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@@ -323,7 +265,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
- Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
+ assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@@ -348,7 +290,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
- Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
+ assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@@ -370,7 +312,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
- Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
+ assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@@ -392,7 +334,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
- Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
+ assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@@ -410,17 +352,13 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
- Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
+ assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void testDeleteResource() throws Exception {
- Result mockResult = new Result<>();
- mockResult.setCode(Status.SUCCESS.getCode());
- Mockito.when(resourcesService.delete(Mockito.any(), Mockito.anyString(),
- Mockito.anyString()))
- .thenReturn(mockResult);
+ Mockito.doNothing().when(resourcesService).delete(Mockito.any());
MultiValueMap paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
paramsMap.add("tenantCode", "123");
@@ -431,9 +369,10 @@ public class ResourcesControllerTest extends AbstractControllerTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
- Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+ Result result =
+ JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference>() {
+ });
- Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
- logger.info(mvcResult.getResponse().getContentAsString());
+ assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}
}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java
index 173ae98854..5ae26af826 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java
@@ -97,19 +97,6 @@ public class PythonGatewayTest {
Assertions.assertEquals((long) result.get("taskDefinitionCode"), taskDefinition.getCode());
}
- @Test
- public void testCreateResource() {
- User user = getTestUser();
- String resourceDir = "/dir1/dir2/";
- String resourceName = "test";
- String resourceSuffix = "py";
- String content = "content";
- String resourceFullName = resourceDir + resourceName + "." + resourceSuffix;
-
- Assertions.assertDoesNotThrow(
- () -> pythonGateway.createOrUpdateResource(user.getUserName(), resourceFullName, content));
- }
-
@Test
public void testQueryResourcesFileInfo() throws Exception {
User user = getTestUser();
@@ -118,12 +105,11 @@ public class PythonGatewayTest {
Mockito.when(resourcesService.queryFileStatus(user.getUserName(), storageEntity.getFullName()))
.thenReturn(storageEntity);
StorageEntity result = pythonGateway.queryResourcesFileInfo(user.getUserName(), storageEntity.getFullName());
- Assertions.assertEquals(result.getId(), storageEntity.getId());
+ Assertions.assertEquals(result.getFullName(), storageEntity.getFullName());
}
private StorageEntity getTestResource() {
StorageEntity storageEntity = new StorageEntity();
- storageEntity.setId(1);
storageEntity.setType(ResourceType.FILE);
storageEntity.setFullName("/dev/test.py");
return storageEntity;
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java
index 8f1869c1fb..04dabf5ead 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java
@@ -67,6 +67,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.service.command.CommandService;
import org.apache.dolphinscheduler.service.process.ProcessService;
@@ -271,7 +272,7 @@ public class ExecuteFunctionServiceTest {
null, null,
null, null, null,
RunMode.RUN_MODE_SERIAL,
- Priority.LOW, Constants.DEFAULT_WORKER_GROUP, tenantCode, 100L, 10, null, null,
+ Priority.LOW, WorkerGroupUtils.getDefaultWorkerGroup(), tenantCode, 100L, 10, null, null,
Constants.DRY_RUN_FLAG_NO,
Constants.TEST_FLAG_NO,
ComplementDependentMode.OFF_MODE, null,
@@ -298,7 +299,7 @@ public class ExecuteFunctionServiceTest {
null, "123456789,987654321",
null, null, null,
RunMode.RUN_MODE_SERIAL,
- Priority.LOW, Constants.DEFAULT_WORKER_GROUP, tenantCode, 100L, 110, null, null,
+ Priority.LOW, WorkerGroupUtils.getDefaultWorkerGroup(), tenantCode, 100L, 110, null, null,
Constants.DRY_RUN_FLAG_NO,
Constants.TEST_FLAG_NO,
ComplementDependentMode.OFF_MODE, null,
@@ -323,7 +324,7 @@ public class ExecuteFunctionServiceTest {
null, "1123456789,987654321",
null, null, null,
RunMode.RUN_MODE_SERIAL,
- Priority.LOW, Constants.DEFAULT_WORKER_GROUP, tenantCode, 100L, 110, null, 0,
+ Priority.LOW, WorkerGroupUtils.getDefaultWorkerGroup(), tenantCode, 100L, 110, null, 0,
Constants.DRY_RUN_FLAG_NO,
Constants.TEST_FLAG_NO,
ComplementDependentMode.OFF_MODE, null,
@@ -354,14 +355,14 @@ public class ExecuteFunctionServiceTest {
dependentProcessDefinition.setProcessDefinitionCode(2);
dependentProcessDefinition.setProcessDefinitionVersion(1);
dependentProcessDefinition.setTaskDefinitionCode(1);
- dependentProcessDefinition.setWorkerGroup(Constants.DEFAULT_WORKER_GROUP);
+ dependentProcessDefinition.setWorkerGroup(WorkerGroupUtils.getDefaultWorkerGroup());
dependentProcessDefinition.setTaskParams(
"{\"localParams\":[],\"resourceList\":[],\"dependence\":{\"relation\":\"AND\",\"dependTaskList\":[{\"relation\":\"AND\",\"dependItemList\":[{\"depTaskCode\":2,\"status\":\"SUCCESS\"}]}]},\"conditionResult\":{\"successNode\":[1],\"failedNode\":[1]}}");
Mockito.when(processService.queryDependentProcessDefinitionByProcessDefinitionCode(processDefinitionCode))
.thenReturn(Lists.newArrayList(dependentProcessDefinition));
Map processDefinitionWorkerGroupMap = new HashMap<>();
- processDefinitionWorkerGroupMap.put(1L, Constants.DEFAULT_WORKER_GROUP);
+ processDefinitionWorkerGroupMap.put(1L, WorkerGroupUtils.getDefaultWorkerGroup());
Mockito.when(workerGroupService.queryWorkerGroupByProcessDefinitionCodes(Lists.newArrayList(1L)))
.thenReturn(processDefinitionWorkerGroupMap);
@@ -370,7 +371,7 @@ public class ExecuteFunctionServiceTest {
command.setCommandType(CommandType.COMPLEMENT_DATA);
command.setCommandParam(
"{\"StartNodeList\":\"1\",\"complementStartDate\":\"2020-01-01 00:00:00\",\"complementEndDate\":\"2020-01-31 23:00:00\"}");
- command.setWorkerGroup(Constants.DEFAULT_WORKER_GROUP);
+ command.setWorkerGroup(WorkerGroupUtils.getDefaultWorkerGroup());
command.setProcessDefinitionCode(processDefinitionCode);
command.setExecutorId(1);
@@ -383,7 +384,7 @@ public class ExecuteFunctionServiceTest {
childDependent.setProcessDefinitionCode(3);
childDependent.setProcessDefinitionVersion(1);
childDependent.setTaskDefinitionCode(4);
- childDependent.setWorkerGroup(Constants.DEFAULT_WORKER_GROUP);
+ childDependent.setWorkerGroup(WorkerGroupUtils.getDefaultWorkerGroup());
childDependent.setTaskParams(
"{\"localParams\":[],\"resourceList\":[],\"dependence\":{\"relation\":\"AND\",\"dependTaskList\":[{\"relation\":\"AND\",\"dependItemList\":[{\"depTaskCode\":3,\"status\":\"SUCCESS\"}]}]},\"conditionResult\":{\"successNode\":[1],\"failedNode\":[1]}}");
Mockito.when(processService.queryDependentProcessDefinitionByProcessDefinitionCode(
@@ -409,7 +410,8 @@ public class ExecuteFunctionServiceTest {
null, null,
null, null, null,
RunMode.RUN_MODE_SERIAL,
- Priority.LOW, Constants.DEFAULT_WORKER_GROUP, tenantCode, 100L, 110, null, 2, Constants.DRY_RUN_FLAG_NO,
+ Priority.LOW, WorkerGroupUtils.getDefaultWorkerGroup(), tenantCode, 100L, 110, null, 2,
+ Constants.DRY_RUN_FLAG_NO,
Constants.TEST_FLAG_NO,
ComplementDependentMode.OFF_MODE, null,
false,
@@ -434,7 +436,7 @@ public class ExecuteFunctionServiceTest {
null, null,
null, null, null,
RunMode.RUN_MODE_SERIAL,
- Priority.LOW, Constants.DEFAULT_WORKER_GROUP, tenantCode, 100L, 110, null, null,
+ Priority.LOW, WorkerGroupUtils.getDefaultWorkerGroup(), tenantCode, 100L, 110, null, null,
Constants.DRY_RUN_FLAG_NO,
Constants.TEST_FLAG_NO,
ComplementDependentMode.OFF_MODE, null,
@@ -460,7 +462,8 @@ public class ExecuteFunctionServiceTest {
null, null,
null, null, null,
RunMode.RUN_MODE_PARALLEL,
- Priority.LOW, Constants.DEFAULT_WORKER_GROUP, tenantCode, 100L, 110, null, 2, Constants.DRY_RUN_FLAG_NO,
+ Priority.LOW, WorkerGroupUtils.getDefaultWorkerGroup(), tenantCode, 100L, 110, null, 2,
+ Constants.DRY_RUN_FLAG_NO,
Constants.TEST_FLAG_NO,
ComplementDependentMode.OFF_MODE, null,
false,
@@ -486,7 +489,7 @@ public class ExecuteFunctionServiceTest {
null, null,
null, null, null,
RunMode.RUN_MODE_PARALLEL,
- Priority.LOW, Constants.DEFAULT_WORKER_GROUP, tenantCode, 100L, 110, null, 15,
+ Priority.LOW, WorkerGroupUtils.getDefaultWorkerGroup(), tenantCode, 100L, 110, null, 15,
Constants.DRY_RUN_FLAG_NO,
Constants.TEST_FLAG_NO,
ComplementDependentMode.OFF_MODE, null,
@@ -514,7 +517,7 @@ public class ExecuteFunctionServiceTest {
null,
RunMode.RUN_MODE_PARALLEL,
Priority.LOW,
- Constants.DEFAULT_WORKER_GROUP,
+ WorkerGroupUtils.getDefaultWorkerGroup(),
tenantCode,
100L,
110,
@@ -553,7 +556,7 @@ public class ExecuteFunctionServiceTest {
null, null,
null, null, 0,
RunMode.RUN_MODE_PARALLEL,
- Priority.LOW, Constants.DEFAULT_WORKER_GROUP, tenantCode, 100L, 110, null, 15,
+ Priority.LOW, WorkerGroupUtils.getDefaultWorkerGroup(), tenantCode, 100L, 110, null, 15,
Constants.DRY_RUN_FLAG_NO,
Constants.TEST_FLAG_YES,
ComplementDependentMode.OFF_MODE, null,
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
index ed3a5f639b..37af53dbbd 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
@@ -72,6 +72,7 @@ import org.apache.dolphinscheduler.dao.model.PageListingResult;
import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionDao;
import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionLogDao;
import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.spi.enums.DbType;
@@ -1143,7 +1144,7 @@ public class ProcessDefinitionServiceTest extends BaseServiceTestTool {
schedule.setProcessInstancePriority(Priority.MEDIUM);
schedule.setWarningType(WarningType.NONE);
schedule.setWarningGroupId(1);
- schedule.setWorkerGroup(Constants.DEFAULT_WORKER_GROUP);
+ schedule.setWorkerGroup(WorkerGroupUtils.getDefaultWorkerGroup());
return schedule;
}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java
index 7d9a4f9338..208d880fc1 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java
@@ -627,7 +627,8 @@ public class ProcessInstanceServiceTest {
try (
MockedStatic taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) {
- taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any()))
+ taskPluginManagerMockedStatic
+ .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true);
Map processInstanceFinishRes =
processInstanceService.updateProcessInstance(loginUser, projectCode, 1,
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
deleted file mode 100644
index 6e94a25861..0000000000
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
+++ /dev/null
@@ -1,657 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.api.service;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.when;
-
-import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
-import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
-import org.apache.dolphinscheduler.api.enums.Status;
-import org.apache.dolphinscheduler.api.exceptions.ServiceException;
-import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService;
-import org.apache.dolphinscheduler.api.service.impl.ResourcesServiceImpl;
-import org.apache.dolphinscheduler.api.utils.PageInfo;
-import org.apache.dolphinscheduler.api.utils.Result;
-import org.apache.dolphinscheduler.common.constants.Constants;
-import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.FileUtils;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
-import org.apache.dolphinscheduler.dao.entity.Tenant;
-import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
-import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
-import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
-import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
-import org.apache.dolphinscheduler.spi.enums.ResourceType;
-
-import org.apache.commons.collections4.CollectionUtils;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.time.LocalDateTime;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
-import org.mockito.junit.jupiter.MockitoExtension;
-import org.mockito.junit.jupiter.MockitoSettings;
-import org.mockito.quality.Strictness;
-import org.springframework.mock.web.MockMultipartFile;
-
-import com.google.common.io.Files;
-
-/**
- * resources service test
- */
-@ExtendWith(MockitoExtension.class)
-@MockitoSettings(strictness = Strictness.LENIENT)
-public class ResourcesServiceTest {
-
- private static final String basePath = "/dolphinscheduler";
- private static final String tenantCode = "123";
- private static final String tenantFileResourceDir = "/dolphinscheduler/123/resources/";
- private static final String tenantUdfResourceDir = "/dolphinscheduler/123/udfs/";
-
- @InjectMocks
- private ResourcesServiceImpl resourcesService;
-
- @Mock
- private TenantMapper tenantMapper;
-
- @Mock
- private StorageOperate storageOperate;
-
- @Mock
- private UserMapper userMapper;
-
- @Mock
- private UdfFuncMapper udfFunctionMapper;
-
- @Mock
- private ProcessDefinitionMapper processDefinitionMapper;
-
- @Mock
- private ResourcePermissionCheckService resourcePermissionCheckService;
-
- private MockedStatic mockedStaticFileUtils;
-
- private MockedStatic mockedStaticFiles;
-
- private MockedStatic mockedStaticDolphinschedulerFileUtils;
-
- private MockedStatic mockedStaticPropertyUtils;
-
- private MockedStatic mockedStaticPaths;
-
- private MockedStatic filesMockedStatic;
-
- private Exception exception;
-
- @BeforeEach
- public void setUp() {
- mockedStaticFileUtils = Mockito.mockStatic(FileUtils.class);
- mockedStaticFiles = Mockito.mockStatic(Files.class);
- mockedStaticDolphinschedulerFileUtils =
- Mockito.mockStatic(org.apache.dolphinscheduler.api.utils.FileUtils.class);
-
- mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
- mockedStaticPaths = Mockito.mockStatic(Paths.class);
- filesMockedStatic = Mockito.mockStatic(java.nio.file.Files.class);
- }
-
- @AfterEach
- public void after() {
- mockedStaticFileUtils.close();
- mockedStaticFiles.close();
- mockedStaticDolphinschedulerFileUtils.close();
- mockedStaticPropertyUtils.close();
- mockedStaticPaths.close();
- filesMockedStatic.close();
- }
-
- @Test
- public void testCreateResource() {
- User user = new User();
- user.setId(1);
- user.setUserType(UserType.GENERAL_USER);
-
- // CURRENT_LOGIN_USER_TENANT_NOT_EXIST
- when(userMapper.selectById(user.getId())).thenReturn(getUser());
- when(tenantMapper.queryById(1)).thenReturn(null);
- ServiceException serviceException = Assertions.assertThrows(ServiceException.class,
- () -> resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE,
- new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes()), "/"));
- assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), serviceException.getMessage());
-
- // set tenant for user
- user.setTenantId(1);
- when(tenantMapper.queryById(1)).thenReturn(getTenant());
- when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
-
- // ILLEGAL_RESOURCE_FILE
- String illegal_path = "/dolphinscheduler/123/../";
- serviceException = Assertions.assertThrows(ServiceException.class,
- () -> {
- MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes());
- resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE,
- mockMultipartFile, illegal_path);
- });
- assertEquals(new ServiceException(Status.ILLEGAL_RESOURCE_PATH, illegal_path), serviceException);
-
- // RESOURCE_FILE_IS_EMPTY
- MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes());
- Result result = resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE,
- mockMultipartFile, tenantFileResourceDir);
- assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg());
-
- // RESOURCE_SUFFIX_FORBID_CHANGE
- mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes());
- when(Files.getFileExtension("test.pdf")).thenReturn("pdf");
- when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
- result = resourcesService.uploadResource(user, "ResourcesServiceTest.jar", ResourceType.FILE, mockMultipartFile,
- tenantFileResourceDir);
- assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg());
-
- // UDF_RESOURCE_SUFFIX_NOT_JAR
- mockMultipartFile =
- new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes());
- when(Files.getFileExtension("ResourcesServiceTest.pdf")).thenReturn("pdf");
- result = resourcesService.uploadResource(user, "ResourcesServiceTest.pdf", ResourceType.UDF, mockMultipartFile,
- tenantUdfResourceDir);
- assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg());
-
- // FULL_FILE_NAME_TOO_LONG
- String tooLongFileName = getRandomStringWithLength(Constants.RESOURCE_FULL_NAME_MAX_LENGTH) + ".pdf";
- mockMultipartFile = new MockMultipartFile(tooLongFileName, tooLongFileName, "pdf", "test".getBytes());
- when(Files.getFileExtension(tooLongFileName)).thenReturn("pdf");
-
- // '/databasePath/tenantCode/RESOURCE/'
- when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
- result = resourcesService.uploadResource(user, tooLongFileName, ResourceType.FILE, mockMultipartFile,
- tenantFileResourceDir);
- assertEquals(Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR.getMsg(), result.getMsg());
- }
-
- @Test
- public void testCreateDirecotry() throws IOException {
- User user = new User();
- user.setId(1);
- user.setUserType(UserType.GENERAL_USER);
-
- String fileName = "directoryTest";
- // RESOURCE_EXIST
- user.setId(1);
- user.setTenantId(1);
- when(tenantMapper.queryById(1)).thenReturn(getTenant());
- when(userMapper.selectById(user.getId())).thenReturn(getUser());
- when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
- when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
- when(storageOperate.exists(tenantFileResourceDir + fileName)).thenReturn(true);
- Result result = resourcesService.createDirectory(user, fileName, ResourceType.FILE, -1, tenantFileResourceDir);
- assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
- }
-
- @Test
- public void testUpdateResource() throws Exception {
- User user = new User();
- user.setId(1);
- user.setUserType(UserType.GENERAL_USER);
- user.setTenantId(1);
-
- when(userMapper.selectById(user.getId())).thenReturn(getUser());
- when(tenantMapper.queryById(1)).thenReturn(getTenant());
- when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
- when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
-
- // TENANT_NOT_EXIST
- when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null);
- Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResource(user,
- "ResourcesServiceTest1.jar", "", "ResourcesServiceTest", ResourceType.UDF, null));
-
- // USER_NO_OPERATION_PERM
- user.setUserType(UserType.GENERAL_USER);
- // tenant who have access to resource is 123,
- Tenant tenantWNoPermission = new Tenant();
- tenantWNoPermission.setTenantCode("321");
- when(tenantMapper.queryById(1)).thenReturn(tenantWNoPermission);
- when(storageOperate.getDir(ResourceType.ALL, "321")).thenReturn(basePath);
-
- String fileName = "ResourcesServiceTest";
- Result result = resourcesService.updateResource(user, tenantFileResourceDir + fileName,
- tenantCode, fileName, ResourceType.FILE, null);
- assertEquals(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), result.getMsg());
-
- // SUCCESS
- when(tenantMapper.queryById(1)).thenReturn(getTenant());
- when(storageOperate.exists(Mockito.any())).thenReturn(false);
-
- when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir);
- when(storageOperate.getFileStatus(tenantFileResourceDir + fileName,
- tenantFileResourceDir, tenantCode, ResourceType.FILE))
- .thenReturn(getStorageEntityResource(fileName));
- result = resourcesService.updateResource(user, tenantFileResourceDir + fileName,
- tenantCode, fileName, ResourceType.FILE, null);
- assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
-
- // Tests for udf resources.
- fileName = "ResourcesServiceTest.jar";
- when(storageOperate.getDir(ResourceType.UDF, tenantCode)).thenReturn(tenantUdfResourceDir);
- when(storageOperate.exists(tenantUdfResourceDir + fileName)).thenReturn(true);
- when(storageOperate.getFileStatus(tenantUdfResourceDir + fileName, tenantUdfResourceDir, tenantCode,
- ResourceType.UDF))
- .thenReturn(getStorageEntityUdfResource(fileName));
- result = resourcesService.updateResource(user, tenantUdfResourceDir + fileName,
- tenantCode, fileName, ResourceType.UDF, null);
- assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
- }
-
- @Test
- public void testQueryResourceListPaging() throws Exception {
- User loginUser = new User();
- loginUser.setId(1);
- loginUser.setTenantId(1);
- loginUser.setTenantCode("tenant1");
- loginUser.setUserType(UserType.ADMIN_USER);
-
- String fileName = "ResourcesServiceTest";
- List mockResList = new ArrayList<>();
- mockResList.add(getStorageEntityResource(fileName));
- List mockUserList = new ArrayList<>();
- mockUserList.add(getUser());
- when(userMapper.selectList(null)).thenReturn(mockUserList);
- when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
- when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
- when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
- when(storageOperate.listFilesStatus(tenantFileResourceDir, tenantFileResourceDir,
- tenantCode, ResourceType.FILE)).thenReturn(mockResList);
-
- Result result = resourcesService.queryResourceListPaging(loginUser, "", "", ResourceType.FILE, "Test", 1, 10);
- assertEquals(Status.SUCCESS.getCode(), (int) result.getCode());
- PageInfo pageInfo = (PageInfo) result.getData();
- Assertions.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList()));
-
- }
-
- @Test
- public void testQueryResourceList() {
- User loginUser = getUser();
- String fileName = "ResourcesServiceTest";
-
- when(userMapper.selectList(null)).thenReturn(Collections.singletonList(loginUser));
- when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
- when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
- when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
- when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir);
- when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
- when(storageOperate.listFilesStatusRecursively(tenantFileResourceDir,
- tenantFileResourceDir, tenantCode, ResourceType.FILE))
- .thenReturn(Collections.singletonList(getStorageEntityResource(fileName)));
- Map result =
- resourcesService.queryResourceList(loginUser, ResourceType.FILE, tenantFileResourceDir);
- assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
- List resourceList = (List) result.get(Constants.DATA_LIST);
- Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
-
- // test udf
- when(storageOperate.getDir(ResourceType.UDF, tenantCode)).thenReturn(tenantUdfResourceDir);
- when(storageOperate.getUdfDir(tenantCode)).thenReturn(tenantUdfResourceDir);
- when(storageOperate.listFilesStatusRecursively(tenantUdfResourceDir, tenantUdfResourceDir,
- tenantCode, ResourceType.UDF)).thenReturn(Arrays.asList(getStorageEntityUdfResource("test.jar")));
- loginUser.setUserType(UserType.GENERAL_USER);
- result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, tenantUdfResourceDir);
- assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
- resourceList = (List) result.get(Constants.DATA_LIST);
- Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
- }
-
- @Test
- public void testDelete() throws Exception {
- User loginUser = new User();
- loginUser.setId(0);
- loginUser.setUserType(UserType.GENERAL_USER);
-
- // TENANT_NOT_EXIST
- loginUser.setUserType(UserType.ADMIN_USER);
- loginUser.setTenantId(2);
- when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
- Assertions.assertThrows(ServiceException.class, () -> resourcesService.delete(loginUser, "", ""));
-
- // RESOURCE_NOT_EXIST
- String fileName = "ResourcesServiceTest";
- when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
- when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
- when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn(tenantFileResourceDir);
- when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, tenantFileResourceDir, tenantCode, null))
- .thenReturn(getStorageEntityResource(fileName));
- Result result = resourcesService.delete(loginUser, tenantFileResourceDir + "ResNotExist", tenantCode);
- assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
-
- // SUCCESS
- loginUser.setTenantId(1);
- result = resourcesService.delete(loginUser, tenantFileResourceDir + fileName, tenantCode);
- assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
- }
-
- @Test
- public void testVerifyResourceName() throws IOException {
- User user = new User();
- user.setId(1);
- user.setUserType(UserType.GENERAL_USER);
-
- String fileName = "ResourcesServiceTest";
- when(storageOperate.exists(tenantFileResourceDir + fileName)).thenReturn(true);
-
- Result result = resourcesService.verifyResourceName(tenantFileResourceDir + fileName, ResourceType.FILE, user);
- assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
-
- // RESOURCE_FILE_EXIST
- result = resourcesService.verifyResourceName(tenantFileResourceDir + fileName, ResourceType.FILE, user);
- Assertions.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode());
-
- // SUCCESS
- result = resourcesService.verifyResourceName("test2", ResourceType.FILE, user);
- assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
- }
-
- @Test
- public void testReadResource() throws IOException {
- // RESOURCE_NOT_EXIST
- when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
- when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
- Result result = resourcesService.readResource(getUser(), "", "", 1, 10);
- assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode());
-
- // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
- when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
- result = resourcesService.readResource(getUser(), "", "", 1, 10);
- assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
-
- // USER_NOT_EXIST
- when(userMapper.selectById(getUser().getId())).thenReturn(null);
- when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
- when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
- result = resourcesService.readResource(getUser(), "", "", 1, 10);
- assertEquals(Status.USER_NOT_EXIST.getCode(), (int) result.getCode());
-
- // TENANT_NOT_EXIST
- when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
- when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(null);
- Assertions.assertThrows(ServiceException.class, () -> resourcesService.readResource(getUser(), "", "", 1, 10));
-
- // SUCCESS
- when(FileUtils.getResourceViewSuffixes()).thenReturn("jar,sh");
- when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
- when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn(tenantFileResourceDir);
- when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
- when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
- when(storageOperate.exists(Mockito.any())).thenReturn(true);
- when(storageOperate.vimFile(Mockito.any(), Mockito.any(), eq(1), eq(10))).thenReturn(getContent());
- when(Files.getFileExtension("/dolphinscheduler/123/resources/test.jar")).thenReturn("jar");
- result = resourcesService.readResource(getUser(), "/dolphinscheduler/123/resources/test.jar", tenantCode, 1,
- 10);
- assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
- }
-
- @Test
- public void testCreateOrUpdateResource() throws Exception {
- User user = getUser();
- when(userMapper.queryByUserNameAccurately(user.getUserName())).thenReturn(getUser());
-
- // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
- exception = Assertions.assertThrows(IllegalArgumentException.class,
- () -> resourcesService.createOrUpdateResource(user.getUserName(), "filename", "my-content"));
- Assertions.assertTrue(
- exception.getMessage().contains("Not allow create or update resources without extension name"));
-
- // SUCCESS
- String fileName = "ResourcesServiceTest";
- when(storageOperate.getResDir(user.getTenantCode())).thenReturn(tenantFileResourceDir);
- when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
- when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
- when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any()))
- .thenReturn(getStorageEntityResource(fileName));
- StorageEntity storageEntity =
- resourcesService.createOrUpdateResource(user.getUserName(), "filename.txt", "my-content");
- Assertions.assertNotNull(storageEntity);
- assertEquals(tenantFileResourceDir + fileName, storageEntity.getFullName());
- }
-
- @Test
- public void testUpdateResourceContent() throws Exception {
- // RESOURCE_PATH_ILLEGAL
- when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
- when(tenantMapper.queryById(1)).thenReturn(getTenant());
- when(storageOperate.getResDir(Mockito.anyString())).thenReturn("/tmp");
-
- String fileName = "ResourcesServiceTest.jar";
- ServiceException serviceException =
- Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(),
- tenantFileResourceDir + fileName, tenantCode, "content"));
- assertEquals(new ServiceException(Status.ILLEGAL_RESOURCE_PATH, tenantFileResourceDir + fileName),
- serviceException);
-
- // RESOURCE_NOT_EXIST
- when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
- when(storageOperate.getResDir(Mockito.anyString())).thenReturn(tenantFileResourceDir);
- when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, "", tenantCode, ResourceType.FILE))
- .thenReturn(null);
- Result result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir + fileName, tenantCode,
- "content");
- assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
-
- // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
- when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
- when(storageOperate.getFileStatus(tenantFileResourceDir, "", tenantCode, ResourceType.FILE))
- .thenReturn(getStorageEntityResource(fileName));
-
- result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir, tenantCode,
- "content");
- assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
-
- // USER_NOT_EXIST
- when(userMapper.selectById(getUser().getId())).thenReturn(null);
- result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir + "123.class",
- tenantCode,
- "content");
- Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
-
- // TENANT_NOT_EXIST
- when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
- when(tenantMapper.queryById(1)).thenReturn(null);
- Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(),
- tenantFileResourceDir + fileName, tenantCode, "content"));
-
- // SUCCESS
- when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, "", tenantCode,
- ResourceType.FILE)).thenReturn(getStorageEntityResource(fileName));
-
- when(Files.getFileExtension(Mockito.anyString())).thenReturn("jar");
- when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
- when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
- when(tenantMapper.queryById(1)).thenReturn(getTenant());
- when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
- when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
- result = resourcesService.updateResourceContent(getUser(),
- tenantFileResourceDir + fileName, tenantCode, "content");
- assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
- }
-
- @Test
- public void testDownloadResource() throws IOException {
- when(tenantMapper.queryById(1)).thenReturn(getTenant());
- when(userMapper.selectById(1)).thenReturn(getUser());
- org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class);
- Path path = Mockito.mock(Path.class);
- when(Paths.get(Mockito.any())).thenReturn(path);
- when(java.nio.file.Files.size(Mockito.any())).thenReturn(1L);
- // resource null
- org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), "");
- Assertions.assertNull(resource);
-
- when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any())).thenReturn(resourceMock);
- resource = resourcesService.downloadResource(getUser(), "");
- Assertions.assertNotNull(resource);
- }
-
- @Test
- public void testDeleteDataTransferData() throws Exception {
- User user = getUser();
- when(userMapper.selectById(user.getId())).thenReturn(getUser());
- when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant());
-
- StorageEntity storageEntity1 = Mockito.mock(StorageEntity.class);
- StorageEntity storageEntity2 = Mockito.mock(StorageEntity.class);
- StorageEntity storageEntity3 = Mockito.mock(StorageEntity.class);
- StorageEntity storageEntity4 = Mockito.mock(StorageEntity.class);
- StorageEntity storageEntity5 = Mockito.mock(StorageEntity.class);
-
- when(storageEntity1.getFullName()).thenReturn("DATA_TRANSFER/20220101");
- when(storageEntity2.getFullName()).thenReturn("DATA_TRANSFER/20220102");
- when(storageEntity3.getFullName()).thenReturn("DATA_TRANSFER/20220103");
- when(storageEntity4.getFullName()).thenReturn("DATA_TRANSFER/20220104");
- when(storageEntity5.getFullName()).thenReturn("DATA_TRANSFER/20220105");
-
- List storageEntityList = new ArrayList<>();
- storageEntityList.add(storageEntity1);
- storageEntityList.add(storageEntity2);
- storageEntityList.add(storageEntity3);
- storageEntityList.add(storageEntity4);
- storageEntityList.add(storageEntity5);
-
- when(storageOperate.listFilesStatus(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
- .thenReturn(storageEntityList);
-
- LocalDateTime localDateTime = LocalDateTime.of(2022, 1, 5, 0, 0, 0);
- try (MockedStatic mockHook = Mockito.mockStatic(LocalDateTime.class)) {
- mockHook.when(LocalDateTime::now).thenReturn(localDateTime);
- DeleteDataTransferResponse response = resourcesService.deleteDataTransferData(user, 3);
-
- assertEquals(response.getSuccessList().size(), 2);
- assertEquals(response.getSuccessList().get(0), "DATA_TRANSFER/20220101");
- assertEquals(response.getSuccessList().get(1), "DATA_TRANSFER/20220102");
- }
-
- try (MockedStatic mockHook = Mockito.mockStatic(LocalDateTime.class)) {
- mockHook.when(LocalDateTime::now).thenReturn(localDateTime);
- DeleteDataTransferResponse response = resourcesService.deleteDataTransferData(user, 0);
- assertEquals(response.getSuccessList().size(), 5);
- }
-
- }
-
- @Test
- public void testCatFile() throws IOException {
- // SUCCESS
- List list = storageOperate.vimFile(Mockito.any(), Mockito.anyString(), eq(1), eq(10));
- Assertions.assertNotNull(list);
- }
-
- @Test
- void testQueryBaseDir() throws Exception {
- User user = getUser();
- String fileName = "ResourcesServiceTest.jar";
- when(userMapper.selectById(user.getId())).thenReturn(getUser());
- when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant());
- when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir);
- when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
- Mockito.any())).thenReturn(getStorageEntityResource(fileName));
- Result result = resourcesService.queryResourceBaseDir(user, ResourceType.FILE);
- assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
- }
-
- private Tenant getTenant() {
- Tenant tenant = new Tenant();
- tenant.setTenantCode(tenantCode);
- return tenant;
- }
-
- private User getUser() {
- User user = new User();
- user.setId(1);
- user.setUserType(UserType.GENERAL_USER);
- user.setTenantId(1);
- user.setTenantCode(tenantCode);
- return user;
- }
-
- private StorageEntity getStorageEntityResource(String fileName) {
- StorageEntity entity = new StorageEntity();
- entity.setAlias(fileName);
- entity.setFileName(fileName);
- entity.setDirectory(false);
- entity.setUserName(tenantCode);
- entity.setType(ResourceType.FILE);
- entity.setFullName(tenantFileResourceDir + fileName);
- return entity;
- }
-
- private StorageEntity getStorageEntityUdfResource(String fileName) {
- StorageEntity entity = new StorageEntity();
- entity.setAlias(fileName);
- entity.setFileName(fileName);
- entity.setDirectory(false);
- entity.setUserName(tenantCode);
- entity.setType(ResourceType.UDF);
- entity.setFullName(tenantUdfResourceDir + fileName);
-
- return entity;
- }
-
- private List getContent() {
- List contentList = new ArrayList<>();
- contentList.add("test");
- return contentList;
- }
-
- private List> getResources() {
- List> resources = new ArrayList<>();
- Map resource = new HashMap<>();
- resource.put("id", 1);
- resource.put("resource_ids", "1");
- resources.add(resource);
- return resources;
- }
-
- private static String getRandomStringWithLength(int length) {
- Random r = new Random();
- StringBuilder sb = new StringBuilder();
- while (sb.length() < length) {
- char c = (char) (r.nextInt(26) + 'a');
- sb.append(c);
- }
- return sb.toString();
- }
-}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java
index ff8ca6ba2c..8460c59924 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java
@@ -64,6 +64,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
+import org.apache.dolphinscheduler.plugin.task.shell.ShellTaskChannelFactory;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.process.ProcessServiceImpl;
@@ -165,7 +166,8 @@ public class TaskDefinitionServiceImplTest {
try (
MockedStatic taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) {
- taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any()))
+ taskPluginManagerMockedStatic
+ .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true);
Project project = getProject();
when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project);
@@ -194,7 +196,8 @@ public class TaskDefinitionServiceImplTest {
try (
MockedStatic taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) {
- taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any()))
+ taskPluginManagerMockedStatic
+ .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true);
String taskDefinitionJson = getTaskDefinitionJson();
@@ -441,7 +444,8 @@ public class TaskDefinitionServiceImplTest {
() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest));
// error task definition
- taskCreateRequest.setTaskParams(TASK_PARAMETER);
+ taskCreateRequest.setTaskType(ShellTaskChannelFactory.NAME);
+ taskCreateRequest.setTaskParams(JSONUtils.toJsonString(new HashMap<>()));
doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE);
assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID,
() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest));
@@ -449,7 +453,8 @@ public class TaskDefinitionServiceImplTest {
try (
MockedStatic taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) {
- taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any()))
+ taskPluginManagerMockedStatic
+ .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true);
// error create task definition object
@@ -502,7 +507,8 @@ public class TaskDefinitionServiceImplTest {
try (
MockedStatic taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) {
- taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any()))
+ taskPluginManagerMockedStatic
+ .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(false);
assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID,
() -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest));
@@ -511,7 +517,8 @@ public class TaskDefinitionServiceImplTest {
try (
MockedStatic taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) {
- taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any()))
+ taskPluginManagerMockedStatic
+ .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true);
// error task definition nothing update
when(processService.isTaskOnline(TASK_CODE)).thenReturn(false);
@@ -616,7 +623,8 @@ public class TaskDefinitionServiceImplTest {
try (
MockedStatic taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) {
- taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any()))
+ taskPluginManagerMockedStatic
+ .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true);
String taskDefinitionJson = getTaskDefinitionJson();
TaskDefinition taskDefinition = getTaskDefinition();
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
index 5746840e90..6925441b10 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
@@ -41,7 +41,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
@@ -95,7 +95,7 @@ public class TenantServiceTest {
private ResourcePermissionCheckService resourcePermissionCheckService;
@Mock
- private StorageOperate storageOperate;
+ private StorageOperator storageOperator;
private static final String tenantCode = "hayden";
private static final String tenantDesc = "This is the tenant desc";
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
index f1721cbb46..5c7bba5821 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
@@ -32,11 +32,10 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -83,7 +82,7 @@ public class UdfFuncServiceTest {
private UDFUserMapper udfUserMapper;
@Mock
- private StorageOperate storageOperate;
+ private StorageOperator storageOperator;
@BeforeEach
public void setUp() {
@@ -111,11 +110,7 @@ public class UdfFuncServiceTest {
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// success
- try {
- Mockito.when(storageOperate.exists("String")).thenReturn(true);
- } catch (IOException e) {
- logger.error("AmazonServiceException when checking resource: String");
- }
+ Mockito.when(storageOperator.exists("String")).thenReturn(true);
result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
@@ -176,11 +171,7 @@ public class UdfFuncServiceTest {
// success
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF, 1,
ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE, serviceLogger)).thenReturn(true);
- try {
- Mockito.when(storageOperate.exists("")).thenReturn(true);
- } catch (IOException e) {
- logger.error("AmazonServiceException when checking resource: ");
- }
+ Mockito.when(storageOperator.exists("")).thenReturn(true);
result = udfFuncService.updateUdfFunc(getLoginUser(), 11, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
index 3cb71d97a0..9e2a359667 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
@@ -48,7 +48,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
@@ -117,7 +117,7 @@ public class UsersServiceTest {
private ProjectMapper projectMapper;
@Mock
- private StorageOperate storageOperate;
+ private StorageOperator storageOperator;
@Mock
private ResourcePermissionCheckService resourcePermissionCheckService;
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java
index 08a541c5bb..fce9aa3f1c 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java
@@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper;
@@ -65,8 +64,6 @@ import org.slf4j.LoggerFactory;
@MockitoSettings(strictness = Strictness.LENIENT)
public class WorkerGroupServiceTest {
- private static final Logger logger = LoggerFactory.getLogger(WorkerGroupServiceTest.class);
-
private static final Logger baseServiceLogger = LoggerFactory.getLogger(BaseServiceImpl.class);
private static final Logger serviceLogger = LoggerFactory.getLogger(WorkerGroupService.class);
@@ -288,47 +285,6 @@ public class WorkerGroupServiceTest {
Assertions.assertEquals("default", workerGroups.toArray()[0]);
}
- @Test
- public void giveNull_whenGetTaskWorkerGroup_expectNull() {
- String nullWorkerGroup = workerGroupService.getTaskWorkerGroup(null);
- Assertions.assertNull(nullWorkerGroup);
- }
-
- @Test
- public void giveCorrectTaskInstance_whenGetTaskWorkerGroup_expectTaskWorkerGroup() {
- TaskInstance taskInstance = new TaskInstance();
- taskInstance.setId(1);
- taskInstance.setWorkerGroup("cluster1");
-
- String workerGroup = workerGroupService.getTaskWorkerGroup(taskInstance);
- Assertions.assertEquals("cluster1", workerGroup);
- }
-
- @Test
- public void giveNullWorkerGroup_whenGetTaskWorkerGroup_expectProcessWorkerGroup() {
- TaskInstance taskInstance = new TaskInstance();
- taskInstance.setId(1);
- taskInstance.setProcessInstanceId(1);
- ProcessInstance processInstance = new ProcessInstance();
- processInstance.setId(1);
- processInstance.setWorkerGroup("cluster1");
- Mockito.when(processService.findProcessInstanceById(1)).thenReturn(processInstance);
-
- String workerGroup = workerGroupService.getTaskWorkerGroup(taskInstance);
- Assertions.assertEquals("cluster1", workerGroup);
- }
-
- @Test
- public void giveNullTaskAndProcessWorkerGroup_whenGetTaskWorkerGroup_expectDefault() {
- TaskInstance taskInstance = new TaskInstance();
- taskInstance.setId(1);
- taskInstance.setProcessInstanceId(1);
- Mockito.when(processService.findProcessInstanceById(1)).thenReturn(null);
-
- String defaultWorkerGroup = workerGroupService.getTaskWorkerGroup(taskInstance);
- Assertions.assertEquals(Constants.DEFAULT_WORKER_GROUP, defaultWorkerGroup);
- }
-
/**
* get Group
*/
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidatorTest.java
new file mode 100644
index 0000000000..11803cba92
--- /dev/null
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidatorTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
+import static org.mockito.Mockito.when;
+
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
+import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import java.util.Locale;
+import java.util.Optional;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.context.i18n.LocaleContextHolder;
+
+@ExtendWith(MockitoExtension.class)
+class CreateDirectoryDtoValidatorTest {
+
+ @Mock
+ private StorageOperator storageOperator;
+
+ @Mock
+ private TenantDao tenantDao;
+
+ @InjectMocks
+ private CreateDirectoryDtoValidator createDirectoryDtoValidator;
+
+ private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
+
+ private User loginUser;
+
+ @BeforeEach
+ public void setup() {
+ when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
+ loginUser = new User();
+ loginUser.setTenantId(1);
+ LocaleContextHolder.setLocale(Locale.ENGLISH);
+ }
+
+ @Test
+ void testValidate_notUnderBaseDirectory() {
+ CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
+ .loginUser(loginUser)
+ .directoryAbsolutePath("/tmp")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp",
+ () -> createDirectoryDtoValidator.validate(createDirectoryDto));
+ }
+
+ @Test
+ public void testValidate_directoryPathContainsIllegalSymbolic() {
+ CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
+ .loginUser(loginUser)
+ .directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/..")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..",
+ () -> createDirectoryDtoValidator.validate(createDirectoryDto));
+ }
+
+ @Test
+ public void testValidate_directoryExist() {
+ CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
+ .loginUser(loginUser)
+ .directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/demo")
+ .build();
+ when(storageOperator.exists(createDirectoryDto.getDirectoryAbsolutePath())).thenReturn(true);
+ assertThrowServiceException(
+ "Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/demo",
+ () -> createDirectoryDtoValidator.validate(createDirectoryDto));
+ }
+
+ @Test
+ public void testValidate_NoPermission() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("test");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
+ .loginUser(loginUser)
+ .directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/demo")
+ .build();
+ when(storageOperator.getResourceMetaData(createDirectoryDto.getDirectoryAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(createDirectoryDto.getDirectoryAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("demo")
+ .isDirectory(true)
+ .tenant("default")
+ .build());
+ when(storageOperator.exists(createDirectoryDto.getDirectoryAbsolutePath())).thenReturn(false);
+ assertThrowServiceException(
+ "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/demo",
+ () -> createDirectoryDtoValidator.validate(createDirectoryDto));
+ }
+
+ @Test
+ public void testValidate_pathNotDirectory() {
+ CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
+ .loginUser(loginUser)
+ .directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/demo.sql")
+ .build();
+ loginUser.setUserType(UserType.ADMIN_USER);
+ assertThrowServiceException(
+ "Internal Server Error: The path is not a directory: /tmp/dolphinscheduler/default/resources/demo.sql",
+ () -> createDirectoryDtoValidator.validate(createDirectoryDto));
+ }
+
+}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidatorTest.java
new file mode 100644
index 0000000000..6312346e40
--- /dev/null
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidatorTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow;
+import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
+import static org.mockito.Mockito.when;
+
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import java.util.Locale;
+import java.util.Optional;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.context.i18n.LocaleContextHolder;
+
+@ExtendWith(MockitoExtension.class)
+class CreateFileFromContentDtoValidatorTest {
+
+ @Mock
+ private StorageOperator storageOperator;
+
+ @Mock
+ private TenantDao tenantDao;
+
+ @InjectMocks
+ private CreateFileFromContentDtoValidator createFileFromContentDtoValidator;
+
+ private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
+
+ private User loginUser;
+
+ @BeforeEach
+ public void setup() {
+ when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
+ loginUser = new User();
+ loginUser.setTenantId(1);
+ LocaleContextHolder.setLocale(Locale.ENGLISH);
+ }
+
+ @Test
+ void testValidate_notUnderBaseDirectory() {
+ CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
+ .loginUser(loginUser)
+ .fileAbsolutePath("/tmp")
+ .fileContent("select * from t")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp",
+ () -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
+
+ }
+
+ @Test
+ public void testValidate_filePathContainsIllegalSymbolic() {
+ CreateFileFromContentDto renameDirectoryDto = CreateFileFromContentDto.builder()
+ .loginUser(loginUser)
+ .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/..")
+ .fileContent("select * from t")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..",
+ () -> createFileFromContentDtoValidator.validate(renameDirectoryDto));
+ }
+
+ @Test
+ public void testValidate_IsNotFile() {
+ CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
+ .loginUser(loginUser)
+ .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
+ .fileContent("select * from t")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: The path is not a file: /tmp/dolphinscheduler/default/resources/a",
+ () -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
+ }
+
+ @Test
+ public void testValidate_fileAlreadyExist() {
+ CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
+ .loginUser(loginUser)
+ .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
+ .fileContent("select * from t")
+ .build();
+ when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(true);
+ assertThrowServiceException(
+ "Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/a.sql",
+ () -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
+ }
+
+ @Test
+ public void testValidate_fileNoPermission() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("test");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
+ .loginUser(loginUser)
+ .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
+ .fileContent("select * from t")
+ .build();
+ when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(false);
+ when(storageOperator.getResourceMetaData(createFileFromContentDto.getFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(createFileFromContentDto.getFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a.sql")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+ assertThrowServiceException(
+ "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a.sql",
+ () -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
+ }
+
+ @Test
+ public void testValidate_contentIsInvalidated() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("default");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
+ .loginUser(loginUser)
+ .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
+ .fileContent("")
+ .build();
+ when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(false);
+ when(storageOperator.getResourceMetaData(createFileFromContentDto.getFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(createFileFromContentDto.getFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a.sql")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+ assertThrowServiceException(
+ "Internal Server Error: The file content is null",
+ () -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
+ }
+
+ @Test
+ public void testValidate() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("default");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
+ .loginUser(loginUser)
+ .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
+ .fileContent("select * from t")
+ .build();
+ when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(false);
+ when(storageOperator.getResourceMetaData(createFileFromContentDto.getFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(createFileFromContentDto.getFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a.sql")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+ assertDoesNotThrow(() -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
+ }
+}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidatorTest.java
new file mode 100644
index 0000000000..69c31f2a0e
--- /dev/null
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidatorTest.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
+import static org.mockito.Mockito.when;
+
+import org.apache.dolphinscheduler.api.AssertionsHelper;
+import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto;
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import java.util.Locale;
+import java.util.Optional;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
+import org.springframework.context.i18n.LocaleContextHolder;
+
+@MockitoSettings(strictness = Strictness.LENIENT)
+@ExtendWith(MockitoExtension.class)
+class FetchFileContentDtoValidatorTest {
+
+ @Mock
+ private StorageOperator storageOperator;
+
+ @Mock
+ private TenantDao tenantDao;
+
+ @InjectMocks
+ private FetchFileContentDtoValidator fetchFileContentDtoValidator;
+
+ private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
+
+ private User loginUser;
+
+ @BeforeEach
+ public void setup() {
+ when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
+ loginUser = new User();
+ loginUser.setTenantId(1);
+ LocaleContextHolder.setLocale(Locale.ENGLISH);
+ }
+
+ @Test
+ void testValidate_skipLineNumInvalid() {
+ FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
+ .loginUser(loginUser)
+ .resourceFileAbsolutePath("/tmp")
+ .skipLineNum(-1)
+ .limit(-1)
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: skipLineNum must be greater than or equal to 0",
+ () -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
+
+ }
+
+ @Test
+ void testValidate_notUnderBaseDirectory() {
+ FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
+ .loginUser(loginUser)
+ .resourceFileAbsolutePath("/tmp")
+ .skipLineNum(0)
+ .limit(-1)
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp",
+ () -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
+
+ }
+
+ @Test
+ public void testValidate_filePathContainsIllegalSymbolic() {
+ FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
+ .loginUser(loginUser)
+ .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/..")
+ .skipLineNum(0)
+ .limit(-1)
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..",
+ () -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
+ }
+
+ @Test
+ public void testValidate_IsNotFile() {
+ FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
+ .loginUser(loginUser)
+ .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
+ .skipLineNum(0)
+ .limit(-1)
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: The path is not a file: /tmp/dolphinscheduler/default/resources/a",
+ () -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
+ }
+
+ @Test
+ public void testValidate_fileNoPermission() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("test");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
+ .loginUser(loginUser)
+ .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
+ .skipLineNum(0)
+ .limit(-1)
+ .build();
+ when(storageOperator.exists(fetchFileContentDto.getResourceFileAbsolutePath())).thenReturn(false);
+ when(storageOperator.getResourceMetaData(fetchFileContentDto.getResourceFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(fetchFileContentDto.getResourceFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a.sql")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+ assertThrowServiceException(
+ "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a.sql",
+ () -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
+ }
+
+ @Test
+ void validate_fileExtensionInvalid() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("default");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
+ .loginUser(loginUser)
+ .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.jar")
+ .skipLineNum(0)
+ .limit(-1)
+ .build();
+ when(storageOperator.exists(fetchFileContentDto.getResourceFileAbsolutePath())).thenReturn(false);
+ when(storageOperator.getResourceMetaData(fetchFileContentDto.getResourceFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(fetchFileContentDto.getResourceFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a.jar")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+ assertThrowServiceException(
+ "Internal Server Error: The file type: jar cannot be fetched",
+ () -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
+ }
+
+ @Test
+ void validate() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("default");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
+ .loginUser(loginUser)
+ .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
+ .skipLineNum(0)
+ .limit(-1)
+ .build();
+ when(storageOperator.exists(fetchFileContentDto.getResourceFileAbsolutePath())).thenReturn(false);
+ when(storageOperator.getResourceMetaData(fetchFileContentDto.getResourceFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(fetchFileContentDto.getResourceFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a.sql")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+ AssertionsHelper.assertDoesNotThrow(() -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
+ }
+}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidatorTest.java
new file mode 100644
index 0000000000..43e73f3b60
--- /dev/null
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidatorTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow;
+import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
+import static org.mockito.Mockito.when;
+
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import java.util.Locale;
+import java.util.Optional;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.context.i18n.LocaleContextHolder;
+
+@ExtendWith(MockitoExtension.class)
+class RenameDirectoryDtoValidatorTest {
+
+ @Mock
+ private StorageOperator storageOperator;
+
+ @Mock
+ private TenantDao tenantDao;
+
+ @InjectMocks
+ private RenameDirectoryDtoValidator renameDirectoryDtoValidator;
+
+ private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
+
+ private User loginUser;
+
+ @BeforeEach
+ public void setup() {
+ when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
+ loginUser = new User();
+ loginUser.setTenantId(1);
+ LocaleContextHolder.setLocale(Locale.ENGLISH);
+ }
+
+ @Test
+ void testValidate_notUnderBaseDirectory() {
+ RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
+ .loginUser(loginUser)
+ .originDirectoryAbsolutePath("/tmp")
+ .targetDirectoryAbsolutePath("/tmp1")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp",
+ () -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
+
+ }
+
+ @Test
+ public void testValidate_directoryPathContainsIllegalSymbolic() {
+ RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
+ .loginUser(loginUser)
+ .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/..")
+ .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..",
+ () -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
+ }
+
+ @Test
+ public void testValidate_originDirectoryNotExist() {
+ RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
+ .loginUser(loginUser)
+ .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
+ .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Thr resource is not exists: /tmp/dolphinscheduler/default/resources/a",
+ () -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
+ }
+
+ @Test
+ public void testValidate_originDirectoryNoPermission() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("test");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
+ .loginUser(loginUser)
+ .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
+ .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b")
+ .build();
+ when(storageOperator.exists(renameDirectoryDto.getOriginDirectoryAbsolutePath())).thenReturn(true);
+ when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginDirectoryAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a")
+ .isDirectory(true)
+ .tenant("default")
+ .build());
+ assertThrowServiceException(
+ "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a",
+ () -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
+ }
+
+ @Test
+ public void testValidate_targetDirectoryAlreadyExist() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("default");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
+ .loginUser(loginUser)
+ .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
+ .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b")
+ .build();
+ when(storageOperator.exists(renameDirectoryDto.getOriginDirectoryAbsolutePath())).thenReturn(true);
+ when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginDirectoryAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a")
+ .isDirectory(true)
+ .tenant("default")
+ .build());
+
+ when(storageOperator.exists(renameDirectoryDto.getTargetDirectoryAbsolutePath())).thenReturn(true);
+ assertThrowServiceException(
+ "Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/b",
+ () -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
+ }
+
+ @Test
+ public void testValidate() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("default");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
+ .loginUser(loginUser)
+ .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
+ .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b")
+ .build();
+ when(storageOperator.exists(renameDirectoryDto.getOriginDirectoryAbsolutePath())).thenReturn(true);
+ when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginDirectoryAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a")
+ .isDirectory(true)
+ .tenant("default")
+ .build());
+
+ when(storageOperator.exists(renameDirectoryDto.getTargetDirectoryAbsolutePath())).thenReturn(false);
+ when(storageOperator.getResourceMetaData(renameDirectoryDto.getTargetDirectoryAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("b")
+ .isDirectory(true)
+ .tenant("default")
+ .build());
+
+ assertDoesNotThrow(() -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
+ }
+}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidatorTest.java
new file mode 100644
index 0000000000..79fcd8c124
--- /dev/null
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidatorTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.validator.resource;
+
+import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.mockito.Mockito.when;
+
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import java.util.Locale;
+import java.util.Optional;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.context.i18n.LocaleContextHolder;
+
+@ExtendWith(MockitoExtension.class)
+public class RenameFileDtoValidatorTest {
+
+ @Mock
+ private StorageOperator storageOperator;
+
+ @Mock
+ private TenantDao tenantDao;
+
+ @InjectMocks
+ private RenameFileDtoValidator renameFileDtoValidator;
+
+ private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
+
+ private User loginUser;
+
+ @BeforeEach
+ public void setup() {
+ when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
+ loginUser = new User();
+ loginUser.setTenantId(1);
+ LocaleContextHolder.setLocale(Locale.ENGLISH);
+ }
+
+ @Test
+ void testValidate_notUnderBaseDirectory() {
+ RenameFileDto renameFileDto = RenameFileDto.builder()
+ .loginUser(loginUser)
+ .originFileAbsolutePath("/tmp")
+ .targetFileAbsolutePath("/tmp1")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp",
+ () -> renameFileDtoValidator.validate(renameFileDto));
+
+ }
+
+ @Test
+ public void testValidate_fileAbsolutePathContainsIllegalSymbolic() {
+ RenameFileDto renameFileDto = RenameFileDto.builder()
+ .loginUser(loginUser)
+ .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/../a.txt")
+ .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/../a.txt",
+ () -> renameFileDtoValidator.validate(renameFileDto));
+ }
+
+ @Test
+ public void testValidate_originFileNotExist() {
+ RenameFileDto renameFileDto = RenameFileDto.builder()
+ .loginUser(loginUser)
+ .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt")
+ .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
+ .build();
+ assertThrowServiceException(
+ "Internal Server Error: Thr resource is not exists: /tmp/dolphinscheduler/default/resources/a.txt",
+ () -> renameFileDtoValidator.validate(renameFileDto));
+ }
+
+ @Test
+ public void testValidate_originFileIsNotFile() {
+ RenameFileDto renameFileDto = RenameFileDto.builder()
+ .loginUser(loginUser)
+ .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
+ .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
+ .build();
+ when(storageOperator.exists(renameFileDto.getOriginFileAbsolutePath())).thenReturn(true);
+ assertThrowServiceException(
+ "Internal Server Error: The path is not a file: /tmp/dolphinscheduler/default/resources/a",
+ () -> renameFileDtoValidator.validate(renameFileDto));
+ }
+
+ @Test
+ public void testValidate_originFileNoPermission() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("test");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ RenameFileDto renameFileDto = RenameFileDto.builder()
+ .loginUser(loginUser)
+ .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt")
+ .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
+ .build();
+ when(storageOperator.exists(renameFileDto.getOriginFileAbsolutePath())).thenReturn(true);
+ when(storageOperator.getResourceMetaData(renameFileDto.getOriginFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(renameFileDto.getOriginFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a.txt")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+ assertThrowServiceException(
+ "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a.txt",
+ () -> renameFileDtoValidator.validate(renameFileDto));
+ }
+
+ @Test
+ public void testValidate_targetFileAlreadyExist() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("default");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ RenameFileDto renameDirectoryDto = RenameFileDto.builder()
+ .loginUser(loginUser)
+ .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt")
+ .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
+ .build();
+ when(storageOperator.exists(renameDirectoryDto.getOriginFileAbsolutePath())).thenReturn(true);
+ when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(renameDirectoryDto.getOriginFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a.txt")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+
+ when(storageOperator.exists(renameDirectoryDto.getTargetFileAbsolutePath())).thenReturn(true);
+ assertThrowServiceException(
+ "Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/b.txt",
+ () -> renameFileDtoValidator.validate(renameDirectoryDto));
+ }
+
+ @Test
+ public void testValidate() {
+ Tenant tenant = new Tenant();
+ tenant.setTenantCode("default");
+ when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
+
+ RenameFileDto renameDirectoryDto = RenameFileDto.builder()
+ .loginUser(loginUser)
+ .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt")
+ .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
+ .build();
+ when(storageOperator.exists(renameDirectoryDto.getOriginFileAbsolutePath())).thenReturn(true);
+ when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(renameDirectoryDto.getOriginFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("a.txt")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+
+ when(storageOperator.exists(renameDirectoryDto.getTargetFileAbsolutePath())).thenReturn(false);
+ when(storageOperator.getResourceMetaData(renameDirectoryDto.getTargetFileAbsolutePath()))
+ .thenReturn(ResourceMetadata.builder()
+ .resourceAbsolutePath(renameDirectoryDto.getTargetFileAbsolutePath())
+ .resourceBaseDirectory(BASE_DIRECTORY)
+ .resourceRelativePath("b.txt")
+ .isDirectory(false)
+ .tenant("default")
+ .build());
+
+ assertDoesNotThrow(() -> renameFileDtoValidator.validate(renameDirectoryDto));
+ }
+
+}
diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml
index 10c4f0e4a1..8c59d1c4ca 100644
--- a/dolphinscheduler-bom/pom.xml
+++ b/dolphinscheduler-bom/pom.xml
@@ -965,6 +965,13 @@
test
+
+ org.testcontainers
+ minio
+ ${testcontainer.version}
+ test
+
+
org.checkerframework
checker-qual
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java
index 5806a20fd7..6f4bb34eab 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java
@@ -43,6 +43,10 @@ public class ImmutableYamlDelegate implements IPropertyDelegate {
// read from classpath
for (String fileName : yamlAbsolutePath) {
try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) {
+ if (fis == null) {
+ log.warn("Cannot find the file: {} under classpath", fileName);
+ continue;
+ }
YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean();
factory.setResources(new InputStreamResource(fis));
factory.afterPropertiesSet();
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
index 5ec2a2b181..4800bff7f9 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
@@ -59,28 +59,6 @@ public final class Constants {
*/
public static final String HDFS_DEFAULT_FS = "fs.defaultFS";
- /**
- * hadoop configuration
- */
- public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE";
-
- public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port";
-
- /**
- * yarn.resourcemanager.ha.rm.ids
- */
- public static final String YARN_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm.ids";
-
- /**
- * yarn.application.status.address
- */
- public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address";
-
- /**
- * yarn.job.history.status.address
- */
- public static final String YARN_JOB_HISTORY_STATUS_ADDRESS = "yarn.job.history.status.address";
-
/**
* hdfs configuration
* resource.hdfs.root.user
@@ -294,8 +272,6 @@ public final class Constants {
*/
public static final String FLOWNODE_RUN_FLAG_NORMAL = "NORMAL";
- public static final String COMMON_TASK_TYPE = "common";
-
public static final String DEFAULT = "default";
public static final String PASSWORD = "password";
public static final String XXXXXX = "******";
@@ -392,8 +368,6 @@ public final class Constants {
public static final String QUEUE_NAME = "queueName";
public static final int LOG_QUERY_SKIP_LINE_NUMBER = 0;
public static final int LOG_QUERY_LIMIT = 4096;
- public static final String ALIAS = "alias";
- public static final String CONTENT = "content";
public static final String DEPENDENT_SPLIT = ":||";
public static final long DEPENDENT_ALL_TASK_CODE = -1;
public static final long DEPENDENT_WORKFLOW_CODE = 0;
@@ -532,14 +506,9 @@ public final class Constants {
* session timeout
*/
public static final int SESSION_TIME_OUT = 7200;
- public static final int MAX_FILE_SIZE = 1024 * 1024 * 1024;
public static final String UDF = "UDF";
public static final String CLASS = "class";
- /**
- * default worker group
- */
- public static final String DEFAULT_WORKER_GROUP = "default";
/**
* authorize writable perm
*/
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java
index 6e25d2ed4b..220b56a939 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java
@@ -27,7 +27,6 @@ public enum StateEventType {
TASK_TIMEOUT(3, "task timeout"),
WAKE_UP_TASK_GROUP(4, "wait task group"),
TASK_RETRY(5, "task retry"),
- PROCESS_BLOCKED(6, "process blocked"),
PROCESS_SUBMIT_FAILED(7, "process submit failed");
StateEventType(int code, String descp) {
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResUploadType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StorageType.java
similarity index 97%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResUploadType.java
rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StorageType.java
index d60a657002..ba3cad45cd 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResUploadType.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StorageType.java
@@ -20,6 +20,6 @@ package org.apache.dolphinscheduler.common.enums;
/**
* data base types
*/
-public enum ResUploadType {
+public enum StorageType {
LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java
index 880db60257..923192ea7b 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java
@@ -38,8 +38,6 @@ public enum WorkflowExecutionStatus {
SUCCESS(7, "success"),
DELAY_EXECUTION(12, "delay execution"),
SERIAL_WAIT(14, "serial wait"),
- READY_BLOCK(15, "ready block"),
- BLOCK(16, "block"),
WAIT_TO_RUN(17, "wait to run"),
;
@@ -59,7 +57,6 @@ public enum WorkflowExecutionStatus {
READY_PAUSE.getCode(),
READY_STOP.getCode(),
SERIAL_WAIT.getCode(),
- READY_BLOCK.getCode(),
WAIT_TO_RUN.getCode()
};
@@ -91,7 +88,7 @@ public enum WorkflowExecutionStatus {
public boolean isFinished() {
// todo: do we need to remove pause/block in finished judge?
- return isSuccess() || isFailure() || isStop() || isPause() || isBlock();
+ return isSuccess() || isFailure() || isStop() || isPause();
}
/**
@@ -119,10 +116,6 @@ public enum WorkflowExecutionStatus {
return this == STOP;
}
- public boolean isBlock() {
- return this == BLOCK;
- }
-
public static int[] getNeedFailoverWorkflowInstanceState() {
return NEED_FAILOVER_STATES;
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index 60629576d9..31765b19bd 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -17,14 +17,15 @@
package org.apache.dolphinscheduler.common.utils;
+import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.dolphinscheduler.common.constants.Constants.DATA_BASEDIR_PATH;
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE;
-import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import java.io.ByteArrayOutputStream;
@@ -33,17 +34,21 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Optional;
import java.util.Set;
import java.util.zip.CRC32;
import java.util.zip.CheckedInputStream;
import lombok.NonNull;
+import lombok.SneakyThrows;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@@ -66,32 +71,14 @@ public class FileUtils {
* @return download file name
*/
public static String getDownloadFilename(String filename) {
- String fileName =
- String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename);
-
- File file = new File(fileName);
- if (!file.getParentFile().exists()) {
- file.getParentFile().mkdirs();
- }
-
- return fileName;
+ return Paths.get(DATA_BASEDIR, "tmp", CodeGenerateUtils.genCode() + "-" + filename).toString();
}
/**
- * get upload file absolute path and name
- *
- * @param tenantCode tenant code
- * @param filename file name
- * @return local file path
+ * Generate a local tmp absolute path of the uploaded file
*/
- public static String getUploadFilename(String tenantCode, String filename) {
- String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename);
- File file = new File(fileName);
- if (!file.getParentFile().exists()) {
- file.getParentFile().mkdirs();
- }
-
- return fileName;
+ public static String getUploadFileLocalTmpAbsolutePath() {
+ return Paths.get(DATA_BASEDIR, "tmp", String.valueOf(CodeGenerateUtils.genCode())).toString();
}
/**
@@ -135,7 +122,7 @@ public class FileUtils {
/**
* absolute path of appInfo file
*
- * @param execPath directory of process execution
+ * @param execPath directory of process execution
* @return
*/
public static String getAppInfoPath(String execPath) {
@@ -152,7 +139,7 @@ public class FileUtils {
/**
* write content to file ,if parent path not exists, it will do one's utmost to mkdir
*
- * @param content content
+ * @param content content
* @param filePath target file path
* @return true if write success
*/
@@ -236,6 +223,7 @@ public class FileUtils {
/**
* Calculate file checksum with CRC32 algorithm
+ *
* @param pathName
* @return checksum of file/dir
*/
@@ -315,4 +303,45 @@ public class FileUtils {
}
}
+ public static String concatFilePath(String... paths) {
+ if (paths.length == 0) {
+ throw new IllegalArgumentException("At least one path should be provided");
+ }
+ StringBuilder finalPath = new StringBuilder(paths[0]);
+ if (StringUtils.isEmpty(finalPath)) {
+ throw new IllegalArgumentException("The path should not be empty");
+ }
+ String separator = File.separator;
+ for (int i = 1; i < paths.length; i++) {
+ String path = paths[i];
+ if (StringUtils.isEmpty(path)) {
+ throw new IllegalArgumentException("The path should not be empty");
+ }
+ if (finalPath.toString().endsWith(separator) && path.startsWith(separator)) {
+ finalPath.append(path.substring(separator.length()));
+ continue;
+ }
+ if (!finalPath.toString().endsWith(separator) && !path.startsWith(separator)) {
+ finalPath.append(separator).append(path);
+ continue;
+ }
+ finalPath.append(path);
+ }
+ return finalPath.toString();
+ }
+
+ public static String getClassPathAbsolutePath(Class clazz) {
+ checkNotNull(clazz, "class is null");
+ return Optional.ofNullable(clazz.getResource("/"))
+ .map(URL::getPath)
+ .orElseThrow(() -> new IllegalArgumentException("class path: " + clazz + " is null"));
+ }
+
+ /**
+ * copy input stream to file, if the file already exists, will append the content to the beginning of the file, otherwise will create a new file.
+ */
+ @SneakyThrows
+ public static void copyInputStreamToFile(InputStream inputStream, String destFilename) {
+ org.apache.commons.io.FileUtils.copyInputStreamToFile(inputStream, new File(destFilename));
+ }
}
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
index f06ece2a56..72ceed630f 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
@@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.common.utils;
-import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
-
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@@ -31,26 +29,21 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
+import com.google.common.truth.Truth;
+
@ExtendWith(MockitoExtension.class)
public class FileUtilsTest {
@Test
public void testGetDownloadFilename() {
- try (MockedStatic mockedDateUtils = Mockito.mockStatic(DateUtils.class)) {
- mockedDateUtils.when(() -> DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059");
- Assertions.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test",
- FileUtils.getDownloadFilename("test"));
- }
+ Truth.assertThat(FileUtils.getDownloadFilename("test")).startsWith("/tmp/dolphinscheduler/tmp/");
}
@Test
public void testGetUploadFilename() {
- Assertions.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb",
- FileUtils.getUploadFilename("aaa", "bbb"));
+ Truth.assertThat(FileUtils.getUploadFileLocalTmpAbsolutePath()).startsWith("/tmp/dolphinscheduler/tmp/");
}
@Test
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java
index 9fd77c944f..fcce2a33d3 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java
@@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.entity;
-import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.CycleEnum;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
@@ -67,7 +66,7 @@ public class DependentProcessDefinition {
*/
public CycleEnum getDependentCycle(long upstreamProcessDefinitionCode) {
DependentParameters dependentParameters = this.getDependentParameters();
- List dependentTaskModelList = dependentParameters.getDependTaskList();
+ List dependentTaskModelList = dependentParameters.getDependence().getDependTaskList();
for (DependentTaskModel dependentTaskModel : dependentTaskModelList) {
List dependentItemList = dependentTaskModel.getDependItemList();
@@ -104,11 +103,7 @@ public class DependentProcessDefinition {
}
public DependentParameters getDependentParameters() {
- return JSONUtils.parseObject(getDependence(), DependentParameters.class);
- }
-
- public String getDependence() {
- return JSONUtils.getNodeString(this.taskParams, Constants.DEPENDENCE);
+ return JSONUtils.parseObject(taskParams, DependentParameters.class);
}
public String getProcessDefinitionName() {
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
index d8f01f473f..9ed8bca0ef 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
@@ -201,12 +201,6 @@ public class ProcessInstance {
*/
private Date restartTime;
- /**
- * workflow block flag
- */
- @TableField(exist = false)
- private boolean isBlocked;
-
/**
* test flag
*/
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
index 4897449753..629cfe9aea 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
@@ -17,26 +17,15 @@
package org.apache.dolphinscheduler.dao.entity;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DYNAMIC;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH;
-
-import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.TaskExecuteType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
+import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import java.io.Serializable;
import java.util.Date;
-import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.Data;
@@ -46,7 +35,6 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
-import com.fasterxml.jackson.core.type.TypeReference;
/**
* task instance
@@ -186,18 +174,6 @@ public class TaskInstance implements Serializable {
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String cacheKey;
- /**
- * dependency
- */
- @TableField(exist = false)
- private DependentParameters dependency;
-
- /**
- * switch dependency
- */
- @TableField(exist = false)
- private SwitchParameters switchDependency;
-
/**
* duration
*/
@@ -303,43 +279,6 @@ public class TaskInstance implements Serializable {
this.executePath = executePath;
}
- public DependentParameters getDependency() {
- if (this.dependency == null) {
- Map taskParamsMap =
- JSONUtils.parseObject(this.getTaskParams(), new TypeReference>() {
- });
- this.dependency =
- JSONUtils.parseObject((String) taskParamsMap.get(Constants.DEPENDENCE), DependentParameters.class);
- }
- return this.dependency;
- }
-
- public void setDependency(DependentParameters dependency) {
- this.dependency = dependency;
- }
-
- public SwitchParameters getSwitchDependency() {
- // todo: We need to directly use Jackson to deserialize the taskParam, rather than parse the map and get from
- // field.
- if (this.switchDependency == null) {
- Map taskParamsMap =
- JSONUtils.parseObject(this.getTaskParams(), new TypeReference>() {
- });
- this.switchDependency =
- JSONUtils.parseObject((String) taskParamsMap.get(Constants.SWITCH_RESULT), SwitchParameters.class);
- }
- return this.switchDependency;
- }
-
- public void setSwitchDependency(SwitchParameters switchDependency) {
- Map taskParamsMap =
- JSONUtils.parseObject(this.getTaskParams(), new TypeReference>() {
- });
- taskParamsMap.put(Constants.SWITCH_RESULT, JSONUtils.toJsonString(switchDependency));
- this.switchDependency = switchDependency;
- this.setTaskParams(JSONUtils.toJsonString(taskParamsMap));
- }
-
public boolean isTaskComplete() {
return this.getState().isSuccess()
@@ -348,30 +287,6 @@ public class TaskInstance implements Serializable {
|| this.getState().isForceSuccess();
}
- public boolean isSubProcess() {
- return TASK_TYPE_SUB_PROCESS.equalsIgnoreCase(this.taskType);
- }
-
- public boolean isDependTask() {
- return TASK_TYPE_DEPENDENT.equalsIgnoreCase(this.taskType);
- }
-
- public boolean isDynamic() {
- return TASK_TYPE_DYNAMIC.equalsIgnoreCase(this.taskType);
- }
-
- public boolean isConditionsTask() {
- return TASK_TYPE_CONDITIONS.equalsIgnoreCase(this.taskType);
- }
-
- public boolean isSwitchTask() {
- return TASK_TYPE_SWITCH.equalsIgnoreCase(this.taskType);
- }
-
- public boolean isBlockingTask() {
- return TASK_TYPE_BLOCKING.equalsIgnoreCase(this.taskType);
- }
-
public boolean isFirstRun() {
return endTime == null;
}
@@ -383,7 +298,7 @@ public class TaskInstance implements Serializable {
* @return can try result
*/
public boolean taskCanRetry() {
- if (this.isSubProcess()) {
+ if (TaskTypeUtils.isSubWorkflowTask(getTaskType())) {
return false;
}
if (this.getState() == TaskExecutionStatus.NEED_FAULT_TOLERANCE) {
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TenantDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TenantDao.java
new file mode 100644
index 0000000000..9f48f372da
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TenantDao.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.repository;
+
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+
+public interface TenantDao extends IDao {
+
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TenantDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TenantDaoImpl.java
new file mode 100644
index 0000000000..45f9a5fcba
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TenantDaoImpl.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.repository.impl;
+
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
+import org.apache.dolphinscheduler.dao.repository.BaseDao;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
+
+import lombok.NonNull;
+
+import org.springframework.stereotype.Repository;
+
+@Repository
+public class TenantDaoImpl extends BaseDao implements TenantDao {
+
+ public TenantDaoImpl(@NonNull TenantMapper tenantMapper) {
+ super(tenantMapper);
+ }
+
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/EnvironmentUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/EnvironmentUtils.java
new file mode 100644
index 0000000000..89ea647f75
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/EnvironmentUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.utils;
+
+public class EnvironmentUtils {
+
+ private static final long EMPTY_ENVIRONMENT_CODE = -1L;
+
+ /**
+ * Check if the environment code is empty (we should use null instead of -1, this is used to comply with the original code)
+ *
+ * @return true if the environment code is empty, false otherwise
+ */
+ public static boolean isEnvironmentCodeEmpty(Long environmentCode) {
+ return environmentCode == null || environmentCode <= 0;
+ }
+
+ /**
+ * Get the empty environment code
+ */
+ public static Long getDefaultEnvironmentCode() {
+ return EMPTY_ENVIRONMENT_CODE;
+ }
+
+ /**
+ * Get the environment code or the default environment code if the environment code is empty
+ */
+ public static Long getEnvironmentCodeOrDefault(Long environmentCode) {
+ return getEnvironmentCodeOrDefault(environmentCode, getDefaultEnvironmentCode());
+ }
+
+ /**
+ * Get the environment code or the default environment code if the environment code is empty
+ */
+ public static Long getEnvironmentCodeOrDefault(Long environmentCode, Long defaultEnvironmentCode) {
+ return isEnvironmentCodeEmpty(environmentCode) ? defaultEnvironmentCode : environmentCode;
+ }
+
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java
index 36cf6c2356..90b648386b 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java
@@ -22,7 +22,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.CRC_SUFFIX;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
@@ -65,17 +65,17 @@ public class TaskCacheUtils {
* 4. input VarPool, from upstream task and workflow global parameters
* @param taskInstance task instance
* @param taskExecutionContext taskExecutionContext
- * @param storageOperate storageOperate
+ * @param storageOperator storageOperate
* @return cache key
*/
public static String generateCacheKey(TaskInstance taskInstance, TaskExecutionContext taskExecutionContext,
- StorageOperate storageOperate) {
+ StorageOperator storageOperator) {
List keyElements = new ArrayList<>();
keyElements.add(String.valueOf(taskInstance.getTaskCode()));
keyElements.add(String.valueOf(taskInstance.getTaskDefinitionVersion()));
keyElements.add(String.valueOf(taskInstance.getIsCache().getCode()));
keyElements.add(String.valueOf(taskInstance.getEnvironmentConfig()));
- keyElements.add(getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperate));
+ keyElements.add(getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperator));
String data = StringUtils.join(keyElements, "_");
return DigestUtils.sha256Hex(data);
}
@@ -123,7 +123,7 @@ public class TaskCacheUtils {
* taskExecutionContext taskExecutionContext
*/
public static String getTaskInputVarPoolData(TaskInstance taskInstance, TaskExecutionContext context,
- StorageOperate storageOperate) {
+ StorageOperator storageOperator) {
JsonNode taskParams = JSONUtils.parseObject(taskInstance.getTaskParams());
// The set of input values considered from localParams in the taskParams
@@ -141,7 +141,8 @@ public class TaskCacheUtils {
List fileInput = varPool.stream().filter(property -> property.getType().equals(DataType.FILE))
.collect(Collectors.toList());
fileInput.forEach(
- property -> fileCheckSumMap.put(property.getProp(), getValCheckSum(property, context, storageOperate)));
+ property -> fileCheckSumMap.put(property.getProp(),
+ getValCheckSum(property, context, storageOperator)));
// var pool value from workflow global parameters
if (context.getPrepareParamsMap() != null) {
@@ -173,17 +174,18 @@ public class TaskCacheUtils {
* cache can be used if content of upstream output files are the same
* @param fileProperty
* @param context
- * @param storageOperate
+ * @param storageOperator
*/
public static String getValCheckSum(Property fileProperty, TaskExecutionContext context,
- StorageOperate storageOperate) {
+ StorageOperator storageOperator) {
String resourceCRCPath = fileProperty.getValue() + CRC_SUFFIX;
- String resourceCRCWholePath = storageOperate.getResourceFullName(context.getTenantCode(), resourceCRCPath);
+ String resourceCRCWholePath =
+ storageOperator.getStorageFileAbsolutePath(context.getTenantCode(), resourceCRCPath);
String targetPath = String.format("%s/%s", context.getExecutePath(), resourceCRCPath);
log.info("{} --- Remote:{} to Local:{}", "CRC file", resourceCRCWholePath, targetPath);
String crcString = "";
try {
- storageOperate.download(resourceCRCWholePath, targetPath, true);
+ storageOperator.download(resourceCRCWholePath, targetPath, true);
crcString = FileUtils.readFile2Str(new FileInputStream(targetPath));
fileProperty.setValue(crcString);
} catch (IOException e) {
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java
index f75a2f5e11..e23fcb97b5 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java
@@ -55,11 +55,9 @@ public class TaskInstanceUtils {
target.setPid(source.getPid());
target.setAppLink(source.getAppLink());
target.setFlag(source.getFlag());
- target.setDependency(source.getDependency());
// todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on
// task params, this is really a very bad practice.
target.setTaskParams(source.getTaskParams());
- target.setSwitchDependency(source.getSwitchDependency());
target.setDuration(source.getDuration());
target.setMaxRetryTimes(source.getMaxRetryTimes());
target.setRetryInterval(source.getRetryInterval());
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/BlockingParameters.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkerGroupUtils.java
similarity index 51%
rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/BlockingParameters.java
rename to dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkerGroupUtils.java
index 2942a08c30..2436d45a02 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/BlockingParameters.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkerGroupUtils.java
@@ -15,37 +15,31 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.plugin.task.api.parameters;
+package org.apache.dolphinscheduler.dao.utils;
import org.apache.commons.lang3.StringUtils;
-public class BlockingParameters extends AbstractParameters {
+public class WorkerGroupUtils {
- // condition of blocking: BlockingOnFailed or BlockingOnSuccess
- private String blockingOpportunity;
+ private static final String DEFAULT_WORKER_GROUP = "default";
- // if true, alert when blocking, otherwise do nothing
-
- private boolean isAlertWhenBlocking;
-
- @Override
- public boolean checkParameters() {
- return !StringUtils.isEmpty(blockingOpportunity);
+ /**
+ * Check if the worker group is empty, if the worker group is default, it is considered empty
+ */
+ public static boolean isWorkerGroupEmpty(String workerGroup) {
+ return StringUtils.isEmpty(workerGroup) || getDefaultWorkerGroup().equals(workerGroup);
}
- public String getBlockingOpportunity() {
- return blockingOpportunity;
+ public static String getWorkerGroupOrDefault(String workerGroup) {
+ return getWorkerGroupOrDefault(workerGroup, getDefaultWorkerGroup());
}
- public void setBlockingCondition(String blockingOpportunity) {
- this.blockingOpportunity = blockingOpportunity;
+ public static String getWorkerGroupOrDefault(String workerGroup, String defaultWorkerGroup) {
+ return isWorkerGroupEmpty(workerGroup) ? defaultWorkerGroup : workerGroup;
}
- public boolean isAlertWhenBlocking() {
- return isAlertWhenBlocking;
+ public static String getDefaultWorkerGroup() {
+ return DEFAULT_WORKER_GROUP;
}
- public void setAlertWhenBlocking(boolean alertWhenBlocking) {
- isAlertWhenBlocking = alertWhenBlocking;
- }
}
diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_dml.sql
index e5d97fab94..21189c77a3 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_dml.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_dml.sql
@@ -30,6 +30,9 @@ END IF;
END;
d//
+-- If the admin account is not associated with a tenant, the admin's tenant will be set to the default tenant.
+UPDATE `t_ds_user` SET `tenant_id` = '-1' WHERE (`user_name` = 'admin') AND (`tenant_id` = '0');
+
delimiter ;
CALL dolphin_t_ds_tenant_insert_default();
DROP PROCEDURE dolphin_t_ds_tenant_insert_default;
diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_dml.sql
index 482b317a60..f47d16f7c7 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_dml.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_dml.sql
@@ -21,6 +21,9 @@ INSERT INTO t_ds_tenant(id, tenant_code, description, queue_id, create_time, upd
UPDATE t_ds_schedules as t1 SET tenant_code = COALESCE(t3.tenant_code, 'default') FROM t_ds_process_definition as t2 LEFT JOIN t_ds_tenant t3 ON t2.tenant_id = t3.id WHERE t1.process_definition_code = t2.code;
UPDATE t_ds_process_instance SET tenant_code = 'default' WHERE tenant_code IS NULL;
+-- If the admin account is not associated with a tenant, the admin's tenant will be set to the default tenant.
+UPDATE t_ds_user SET tenant_id = '-1' WHERE (user_name = 'admin') AND (tenant_id = '0');
+
-- data quality support choose database
INSERT INTO t_ds_dq_rule_input_entry
(id, field, "type", title, value, "options", placeholder, option_source_type, value_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time)
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java
index cb68cf7494..1bcc784cbc 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java
@@ -17,12 +17,7 @@
package org.apache.dolphinscheduler.dao.entity;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
-
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
@@ -30,35 +25,10 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameter
import java.util.ArrayList;
import java.util.List;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class TaskInstanceTest {
- /**
- * task instance sub process
- */
- @Test
- public void testTaskInstanceIsSubProcess() {
- TaskInstance taskInstance = new TaskInstance();
-
- // sub process
- taskInstance.setTaskType(TASK_TYPE_SUB_PROCESS);
- Assertions.assertTrue(taskInstance.isSubProcess());
-
- // not sub process
- taskInstance.setTaskType("HTTP");
- Assertions.assertFalse(taskInstance.isSubProcess());
-
- // sub process
- taskInstance.setTaskType(TASK_TYPE_CONDITIONS);
- Assertions.assertTrue(taskInstance.isConditionsTask());
-
- // sub process
- taskInstance.setTaskType(TASK_TYPE_DEPENDENT);
- Assertions.assertTrue(taskInstance.isDependTask());
- }
-
/**
* test for TaskInstance.getDependence
*/
@@ -66,7 +36,6 @@ public class TaskInstanceTest {
public void testTaskInstanceGetDependence() {
TaskInstance taskInstance = new TaskInstance();
taskInstance.setTaskParams(JSONUtils.toJsonString(getDependentParameters()));
- taskInstance.getDependency();
}
/**
@@ -82,8 +51,6 @@ public class TaskInstanceTest {
dependentItem.setDefinitionCode(222L);
dependentItem.setCycle("today");
dependentItems.add(dependentItem);
- dependentParameters.setDependTaskList(dependTaskList);
- dependentParameters.setRelation(DependentRelation.AND);
return dependentParameters;
}
}
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java
index 2105885fa3..3f4b0768aa 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java
@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.dao.mapper;
import static com.google.common.truth.Truth.assertThat;
-import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Flag;
@@ -32,6 +31,7 @@ import org.apache.dolphinscheduler.dao.BaseDaoTest;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.CommandCount;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import java.util.Date;
import java.util.HashMap;
@@ -303,7 +303,7 @@ public class CommandMapperTest extends BaseDaoTest {
command.setProcessInstancePriority(Priority.MEDIUM);
command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
- command.setWorkerGroup(Constants.DEFAULT_WORKER_GROUP);
+ command.setWorkerGroup(WorkerGroupUtils.getDefaultWorkerGroup());
command.setProcessInstanceId(0);
command.setProcessDefinitionVersion(0);
commandMapper.insert(command);
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java
index 85867ef3b5..df5a2562aa 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java
@@ -18,8 +18,8 @@
package org.apache.dolphinscheduler.dao.repository.impl;
import static com.google.common.truth.Truth.assertThat;
+import static org.junit.jupiter.api.Assertions.assertFalse;
-import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
@@ -29,44 +29,58 @@ import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.BaseDaoTest;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.repository.CommandDao;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.commons.lang3.RandomUtils;
import java.util.List;
+import java.util.stream.Collectors;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.RepeatedTest;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.annotation.DirtiesContext;
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
class CommandDaoImplTest extends BaseDaoTest {
@Autowired
private CommandDao commandDao;
- @Test
+ @RepeatedTest(value = 10)
void fetchCommandByIdSlot() {
- int commandSize = RandomUtils.nextInt(1, 1000);
- for (int i = 0; i < commandSize; i++) {
- createCommand(CommandType.START_PROCESS, 0);
- }
int totalSlot = RandomUtils.nextInt(1, 10);
int currentSlotIndex = RandomUtils.nextInt(0, totalSlot);
int fetchSize = RandomUtils.nextInt(10, 100);
- for (int i = 1; i < 5; i++) {
- int idStep = i;
- List commands = commandDao.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchSize);
- assertThat(commands.size()).isGreaterThan(0);
- assertThat(commands.size())
- .isEqualTo(commandDao.queryAll()
- .stream()
- .filter(command -> (command.getId() / idStep) % totalSlot == currentSlotIndex)
- .limit(fetchSize)
- .count());
-
+ int idStep = RandomUtils.nextInt(1, 5);
+ int commandSize = RandomUtils.nextInt(currentSlotIndex, 1000);
+ // Generate commandSize commands
+ int id = 1;
+ for (int j = 0; j < commandSize; j++) {
+ Command command = generateCommand(CommandType.START_PROCESS, 0);
+ command.setId(id);
+ commandDao.insert(command);
+ id += idStep;
}
+ List commands = commandDao.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchSize);
+ assertFalse(commands.isEmpty(),
+ "Commands should not be empty, currentSlotIndex: " + currentSlotIndex +
+ ", totalSlot: " + totalSlot +
+ ", idStep: " + idStep +
+ ", fetchSize: " + fetchSize +
+ ", total command size: " + commandSize +
+ ", total commands: "
+ + commandDao.queryAll().stream().map(Command::getId).collect(Collectors.toList()));
+ assertThat(commands.size())
+ .isEqualTo(commandDao.queryAll()
+ .stream()
+ .filter(command -> (command.getId() / idStep) % totalSlot == currentSlotIndex)
+ .limit(fetchSize)
+ .count());
+
}
- private void createCommand(CommandType commandType, int processDefinitionCode) {
+ private Command generateCommand(CommandType commandType, int processDefinitionCode) {
Command command = new Command();
command.setCommandType(commandType);
command.setProcessDefinitionCode(processDefinitionCode);
@@ -80,9 +94,9 @@ class CommandDaoImplTest extends BaseDaoTest {
command.setProcessInstancePriority(Priority.MEDIUM);
command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
- command.setWorkerGroup(Constants.DEFAULT_WORKER_GROUP);
+ command.setWorkerGroup(WorkerGroupUtils.getDefaultWorkerGroup());
command.setProcessInstanceId(0);
command.setProcessDefinitionVersion(0);
- commandDao.insert(command);
+ return command;
}
}
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java
index 421309130f..9f69b7d90f 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java
@@ -65,11 +65,9 @@ class ProcessInstanceDaoImplTest extends BaseDaoTest {
WorkflowExecutionStatus.READY_STOP));
processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion,
WorkflowExecutionStatus.SERIAL_WAIT));
- processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion,
- WorkflowExecutionStatus.READY_BLOCK));
processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion,
WorkflowExecutionStatus.WAIT_TO_RUN));
- assertEquals(8, processInstanceDao
+ assertEquals(7, processInstanceDao
.queryByWorkflowCodeVersionStatus(workflowDefinitionCode, workflowDefinitionVersion, status).size());
}
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/EnvironmentUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/EnvironmentUtilsTest.java
new file mode 100644
index 0000000000..dd5356169d
--- /dev/null
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/EnvironmentUtilsTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.utils;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class EnvironmentUtilsTest {
+
+ @ParameterizedTest
+ @ValueSource(longs = {0, -1})
+ void testIsEnvironmentCodeEmpty_emptyEnvironmentCode(Long environmentCode) {
+ assertThat(EnvironmentUtils.isEnvironmentCodeEmpty(environmentCode)).isTrue();
+ }
+
+ @ParameterizedTest
+ @ValueSource(longs = {123})
+ void testIsEnvironmentCodeEmpty_nonEmptyEnvironmentCode(Long environmentCode) {
+ assertThat(EnvironmentUtils.isEnvironmentCodeEmpty(environmentCode)).isFalse();
+ }
+
+ @Test
+ void testGetDefaultEnvironmentCode() {
+ assertThat(EnvironmentUtils.getDefaultEnvironmentCode()).isEqualTo(-1L);
+ }
+
+ @ParameterizedTest
+ @ValueSource(longs = {0, -1})
+ void testGetEnvironmentCodeOrDefault_emptyEnvironmentCode(Long environmentCode) {
+ assertThat(EnvironmentUtils.getEnvironmentCodeOrDefault(environmentCode)).isEqualTo(-1L);
+ }
+
+ @ParameterizedTest
+ @ValueSource(longs = {123})
+ void testGetEnvironmentCodeOrDefault_nonEmptyEnvironmentCode(Long environmentCode) {
+ assertThat(EnvironmentUtils.getEnvironmentCodeOrDefault(environmentCode)).isEqualTo(environmentCode);
+ }
+
+ @ParameterizedTest
+ @CsvSource(value = {",123", "-1,123"})
+ void testGetEnvironmentCodeOrDefault_withDefaultValue_emptyEnvironmentCode(Long environmentCode,
+ Long defaultValue) {
+ assertThat(EnvironmentUtils.getEnvironmentCodeOrDefault(environmentCode, defaultValue)).isEqualTo(defaultValue);
+ }
+
+ @ParameterizedTest
+ @CsvSource(value = {"1,123"})
+ void testGetEnvironmentCodeOrDefault_withDefaultValue_nonEmptyEnvironmentCode(Long environmentCode,
+ Long defaultValue) {
+ assertThat(EnvironmentUtils.getEnvironmentCodeOrDefault(environmentCode, defaultValue))
+ .isEqualTo(environmentCode);
+ }
+}
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtilsTest.java
index 49d5a87bb6..ee88d1cd1a 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtilsTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtilsTest.java
@@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.dao.utils;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
@@ -46,7 +46,7 @@ class TaskCacheUtilsTest {
private TaskExecutionContext taskExecutionContext;
- private StorageOperate storageOperate;
+ private StorageOperator storageOperator;
@BeforeEach
void setUp() {
@@ -101,7 +101,7 @@ class TaskCacheUtilsTest {
prepareParamsMap.put("a", property);
taskExecutionContext.setPrepareParamsMap(prepareParamsMap);
- storageOperate = Mockito.mock(StorageOperate.class);
+ storageOperator = Mockito.mock(StorageOperator.class);
}
@Test
@@ -128,26 +128,26 @@ class TaskCacheUtilsTest {
@Test
void TestGetTaskInputVarPoolData() {
- TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperate);
+ TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperator);
// only a=aa and c=cc will influence the result,
// b=bb is a fixed value, will be considered in task version
// k=kk is not in task params, will be ignored
String except =
"[{\"prop\":\"a\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"aa\"},{\"prop\":\"c\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"cc\"}]";
Assertions.assertEquals(except,
- TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperate));
+ TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperator));
}
@Test
void TestGenerateCacheKey() {
- String cacheKeyBase = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
+ String cacheKeyBase = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
Property propertyI = new Property();
propertyI.setProp("i");
propertyI.setDirect(Direct.IN);
propertyI.setType(DataType.VARCHAR);
propertyI.setValue("ii");
taskExecutionContext.getPrepareParamsMap().put("i", propertyI);
- String cacheKeyNew = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
+ String cacheKeyNew = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
// i will not influence the result, because task instance not use it
Assertions.assertEquals(cacheKeyBase, cacheKeyNew);
@@ -157,17 +157,17 @@ class TaskCacheUtilsTest {
propertyD.setType(DataType.VARCHAR);
propertyD.setValue("dd");
taskExecutionContext.getPrepareParamsMap().put("i", propertyD);
- String cacheKeyD = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
+ String cacheKeyD = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
// d will influence the result, because task instance use it
Assertions.assertNotEquals(cacheKeyBase, cacheKeyD);
taskInstance.setTaskDefinitionVersion(100);
- String cacheKeyE = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
+ String cacheKeyE = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
// task definition version is changed, so cache key changed
Assertions.assertNotEquals(cacheKeyD, cacheKeyE);
taskInstance.setEnvironmentConfig("export PYTHON_LAUNCHER=/bin/python3");
- String cacheKeyF = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
+ String cacheKeyF = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
// EnvironmentConfig is changed, so cache key changed
Assertions.assertNotEquals(cacheKeyE, cacheKeyF);
}
@@ -193,7 +193,7 @@ class TaskCacheUtilsTest {
taskExecutionContext.setExecutePath("test");
taskExecutionContext.setTenantCode("aaa");
- String crc = TaskCacheUtils.getValCheckSum(property, taskExecutionContext, storageOperate);
+ String crc = TaskCacheUtils.getValCheckSum(property, taskExecutionContext, storageOperator);
Assertions.assertEquals(crc, content);
}
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkerGroupUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkerGroupUtilsTest.java
new file mode 100644
index 0000000000..60bf74695d
--- /dev/null
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkerGroupUtilsTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.utils;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class WorkerGroupUtilsTest {
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", "default"})
+ void testIsWorkerGroupEmpty_emptyWorkerGroup(String workerGroup) {
+ assertThat(WorkerGroupUtils.isWorkerGroupEmpty(workerGroup)).isTrue();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"123", "default1"})
+ void testIsWorkerGroupEmpty_nonEmptyWorkerGroup(String workerGroup) {
+ assertThat(WorkerGroupUtils.isWorkerGroupEmpty(workerGroup)).isFalse();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", "default"})
+ void testGetWorkerGroupOrDefault_emptyWorkerGroup(String workerGroup) {
+ assertThat(WorkerGroupUtils.getWorkerGroupOrDefault(workerGroup))
+ .isEqualTo(WorkerGroupUtils.getDefaultWorkerGroup());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"test"})
+ void testGetWorkerGroupOrDefault_nonEmptyWorkerGroup(String workerGroup) {
+ assertThat(WorkerGroupUtils.getWorkerGroupOrDefault(workerGroup)).isEqualTo(workerGroup);
+ }
+
+ @ParameterizedTest
+ @CsvSource(value = {",test", "default,test"})
+ void testGetWorkerGroupOrDefault_withDefaultValue_emptyWorkerGroup(String workerGroup, String defaultValue) {
+ assertThat(WorkerGroupUtils.getWorkerGroupOrDefault(workerGroup, defaultValue)).isEqualTo(defaultValue);
+ }
+
+ @ParameterizedTest
+ @CsvSource(value = {"test1,test"})
+ void testGetWorkerGroupOrDefault_withDefaultValue_nonEmptyWorkerGroup(String workerGroup, String defaultValue) {
+ assertThat(WorkerGroupUtils.getWorkerGroupOrDefault(workerGroup)).isEqualTo(workerGroup);
+ }
+
+ @Test
+ void getDefaultWorkerGroup() {
+ assertThat(WorkerGroupUtils.getDefaultWorkerGroup()).isEqualTo("default");
+ }
+}
diff --git a/dolphinscheduler-dao/src/test/resources/logback.xml b/dolphinscheduler-dao/src/test/resources/logback.xml
new file mode 100644
index 0000000000..9a182a18ef
--- /dev/null
+++ b/dolphinscheduler-dao/src/test/resources/logback.xml
@@ -0,0 +1,21 @@
+
+
+
+
+
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
index 1c24785c2f..eb8f2ddef0 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
@@ -29,7 +29,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOGIN_US
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RESOURCE_UPLOAD_PATH;
import org.apache.dolphinscheduler.common.constants.Constants;
-import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.enums.StorageType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.commons.lang3.StringUtils;
@@ -72,9 +72,9 @@ public class CommonUtils {
*/
public static boolean getKerberosStartupState() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(RESOURCE_STORAGE_TYPE);
- ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
+ StorageType storageType = StorageType.valueOf(resUploadStartupType);
Boolean kerberosStartupState = PropertyUtils.getBoolean(HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
- return resUploadType == ResUploadType.HDFS && kerberosStartupState;
+ return storageType == StorageType.HDFS && kerberosStartupState;
}
/**
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java
index 168ff3bdca..a39cdaf614 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java
@@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.hive.security;
import static org.apache.dolphinscheduler.common.constants.Constants.JAVA_SECURITY_KRB5_CONF;
import org.apache.dolphinscheduler.common.constants.Constants;
-import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.enums.StorageType;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
@@ -120,10 +120,10 @@ public class UserGroupInformationFactory {
public static boolean openKerberos() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
- ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
+ StorageType storageType = StorageType.valueOf(resUploadStartupType);
Boolean kerberosStartupState =
PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
- return resUploadType == ResUploadType.HDFS && kerberosStartupState;
+ return storageType == StorageType.HDFS && kerberosStartupState;
}
}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java
index c3b73580dd..687d853fff 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java
@@ -35,6 +35,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import com.alibaba.druid.sql.parser.SQLParserUtils;
import com.google.auto.service.AutoService;
@@ -128,8 +129,10 @@ public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public List splitAndRemoveComment(String sql) {
- String cleanSQL = SQLParserUtils.removeComment(sql, com.alibaba.druid.DbType.sqlserver);
- return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.sqlserver);
+ return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.sqlserver)
+ .stream()
+ .map(subSql -> subSql.concat(";"))
+ .collect(Collectors.toList());
}
private String transformOther(Map otherMap) {
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessorTest.java
index de30e27e0b..7d34755d65 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessorTest.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessorTest.java
@@ -17,12 +17,15 @@
package org.apache.dolphinscheduler.plugin.datasource.sqlserver.param;
+import static com.google.common.truth.Truth.assertThat;
+
import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
@@ -95,4 +98,32 @@ public class SQLServerDataSourceProcessorTest {
Assertions.assertEquals(DataSourceConstants.SQLSERVER_VALIDATION_QUERY,
sqlServerDatasourceProcessor.getValidationQuery());
}
+
+ @Test
+ void splitAndRemoveComment_singleSelect() {
+ String sql = "select * from table;";
+ List subSqls = sqlServerDatasourceProcessor.splitAndRemoveComment(sql);
+ assertThat(subSqls).hasSize(1);
+ assertThat(subSqls.get(0)).isEqualTo("select * from table;");
+ }
+
+ @Test
+ void splitAndRemoveComment_singleMerge() {
+ String sql = "MERGE\n" +
+ " [ TOP ( expression ) [ PERCENT ] ]\n" +
+ " [ INTO ] [ WITH ( ) ] [ [ AS ] table_alias ]\n" +
+ " USING [ [ AS ] table_alias ]\n" +
+ " ON \n" +
+ " [ WHEN MATCHED [ AND ]\n" +
+ " THEN ] [ ...n ]\n" +
+ " [ WHEN NOT MATCHED [ BY TARGET ] [ AND ]\n" +
+ " THEN ]\n" +
+ " [ WHEN NOT MATCHED BY SOURCE [ AND ]\n" +
+ " THEN ] [ ...n ]\n" +
+ " [ ]\n" +
+ " [ OPTION ( [ ,...n ] ) ];";
+ List subSqls = sqlServerDatasourceProcessor.splitAndRemoveComment(sql);
+ assertThat(subSqls).hasSize(1);
+ assertThat(subSqls.get(0)).isEqualTo(sql);
+ }
}
diff --git a/dolphinscheduler-e2e/README.md b/dolphinscheduler-e2e/README.md
index f2fbcead77..eb332de90e 100644
--- a/dolphinscheduler-e2e/README.md
+++ b/dolphinscheduler-e2e/README.md
@@ -97,3 +97,27 @@ class TenantE2ETest {
- For UI tests, it's common that the pages might need some time to load, or the operations might need some time to
complete, we can use `await().untilAsserted(() -> {})` to wait for the assertions.
+## Local development
+
+### Mac M1
+Add VM options to the test configuration in IntelliJ IDEA:
+```
+# In this mode you need to install docker desktop for mac and run it with locally
+-Dm1_chip=true
+```
+
+### Running locally(without Docker)
+```
+# In this mode you need to start frontend and backend services locally
+-Dlocal=true
+```
+
+### Running locally(with Docker)
+```
+# In this mode you only need to install docker locally
+```
+
+- To run the tests locally, you need to have the DolphinScheduler running locally. You should add `dolphinscheduler-e2e/pom.xml` to the maven project
+ Since it does not participate in project compilation, it is not in the main project.
+- Running run test class `org.apache.dolphinscheduler.e2e.cases.UserE2ETest` in the IDE. After execution, the test video will be saved as mp4 in a local temporary directory. Such as
+ `/var/folders/hf/123/T/record-3123/PASSED-[engine:junit-jupiter]/[class:org.apache.dolphinscheduler.e2e.cases.UserE2ETest]-20240606-152333.mp4`
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowJavaTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowJavaTaskE2ETest.java
index d61a9ddd2a..77c4e554bc 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowJavaTaskE2ETest.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowJavaTaskE2ETest.java
@@ -92,8 +92,8 @@ public class WorkflowJavaTaskE2ETest {
.goToNav(SecurityPage.class)
.goToTab(UserPage.class);
- new WebDriverWait(userPage.driver(), Duration.ofSeconds(20)).until(ExpectedConditions.visibilityOfElementLocated(
- new By.ByClassName("name")));
+ new WebDriverWait(userPage.driver(), Duration.ofSeconds(20))
+ .until(ExpectedConditions.visibilityOfElementLocated(new By.ByClassName("name")));
userPage.update(user, user, email, phone, tenant)
.goToNav(ProjectPage.class)
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java
index 21c740952c..dcf22a2d6d 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java
@@ -134,8 +134,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
private void setBrowserContainerByOsName() {
DockerImageName imageName;
- if (LOCAL_MODE && M1_CHIP_FLAG) {
- imageName = DockerImageName.parse("seleniarm/standalone-chromium:4.1.2-20220227")
+ if (M1_CHIP_FLAG) {
+ imageName = DockerImageName.parse("seleniarm/standalone-chromium:124.0-chromedriver-124.0")
.asCompatibleSubstituteFor("selenium/standalone-chrome");
browser = new BrowserWebDriverContainer<>(imageName)
@@ -143,6 +143,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
.withCreateContainerCmdModifier(cmd -> cmd.withUser("root"))
.withFileSystemBind(Constants.HOST_CHROME_DOWNLOAD_PATH.toFile().getAbsolutePath(),
Constants.SELENIUM_CONTAINER_CHROME_DOWNLOAD_PATH)
+ .withRecordingMode(RECORD_ALL, record.toFile(), MP4)
.withStartupTimeout(Duration.ofSeconds(300));
} else {
browser = new BrowserWebDriverContainer<>()
@@ -203,7 +204,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
.map(URL::getPath)
.map(File::new)
.collect(Collectors.toList());
-
+
ComposeContainer compose = new ComposeContainer(files)
.withPull(true)
.withTailChildContainers(true)
@@ -213,7 +214,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
DOCKER_PORT, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300)))
.withLogConsumer(serviceName, outputFrame -> LOGGER.info(outputFrame.getUtf8String()))
.waitingFor(serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
-
+
return compose;
}
diff --git a/dolphinscheduler-e2e/pom.xml b/dolphinscheduler-e2e/pom.xml
index c7ce90b1a6..6731240486 100644
--- a/dolphinscheduler-e2e/pom.xml
+++ b/dolphinscheduler-e2e/pom.xml
@@ -36,7 +36,7 @@
UTF-8
5.8.1
- 4.6.0
+ 4.13.0
1.18.20
3.20.2
1.5.30
@@ -119,7 +119,7 @@
org.testcontainers
testcontainers-bom
- 1.19.3
+ 1.19.8
import
pom
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml b/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml
index 9501d55706..0554ea3c3c 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml
@@ -47,6 +47,13 @@
dolphinscheduler-common
${project.version}
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-meter
+ ${project.version}
+
+
io.netty
netty-all
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcMethod.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcMethod.java
index cd1b778c9a..2dcf689ad8 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcMethod.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcMethod.java
@@ -28,6 +28,6 @@ import java.lang.annotation.Target;
@Documented
public @interface RpcMethod {
- long timeout() default 3000L;
+ long timeout() default -1;
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/SyncRequestDto.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/SyncRequestDto.java
new file mode 100644
index 0000000000..7df649cd57
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/SyncRequestDto.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.base;
+
+import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
+import org.apache.dolphinscheduler.extract.base.utils.Host;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class SyncRequestDto {
+
+ private Host serverHost;
+ private Transporter transporter;
+ private long timeoutMillis;
+
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java
index be570eb577..5f50e2441c 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyClientHandler.java
@@ -77,7 +77,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
.writeAndFlush(HeartBeatTransporter.getHeartBeatTransporter())
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
if (log.isDebugEnabled()) {
- log.debug("Client send heart beat to: {}", ChannelUtils.getRemoteAddress(ctx.channel()));
+ log.info("Client send heartbeat to: {}", ctx.channel().remoteAddress());
}
} else {
super.userEventTriggered(ctx, evt);
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java
index 8ded68669d..4aea4d6dfe 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java
@@ -19,14 +19,17 @@ package org.apache.dolphinscheduler.extract.base.client;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.extract.base.IRpcResponse;
+import org.apache.dolphinscheduler.extract.base.SyncRequestDto;
import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig;
import org.apache.dolphinscheduler.extract.base.exception.RemotingException;
import org.apache.dolphinscheduler.extract.base.exception.RemotingTimeoutException;
import org.apache.dolphinscheduler.extract.base.future.ResponseFuture;
+import org.apache.dolphinscheduler.extract.base.metrics.ClientSyncDurationMetrics;
+import org.apache.dolphinscheduler.extract.base.metrics.ClientSyncExceptionMetrics;
+import org.apache.dolphinscheduler.extract.base.metrics.RpcMetrics;
import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder;
-import org.apache.dolphinscheduler.extract.base.utils.Constants;
import org.apache.dolphinscheduler.extract.base.utils.Host;
import org.apache.dolphinscheduler.extract.base.utils.NettyUtils;
@@ -97,8 +100,8 @@ public class NettyRemotingClient implements AutoCloseable {
ch.pipeline()
.addLast("client-idle-handler",
new IdleStateHandler(
- Constants.NETTY_CLIENT_HEART_BEAT_TIME,
0,
+ clientConfig.getHeartBeatIntervalMillis(),
0,
TimeUnit.MILLISECONDS))
.addLast(new TransporterDecoder(), clientHandler, new TransporterEncoder());
@@ -107,38 +110,60 @@ public class NettyRemotingClient implements AutoCloseable {
isStarted.compareAndSet(false, true);
}
- public IRpcResponse sendSync(final Host host,
- final Transporter transporter,
- final long timeoutMillis) throws InterruptedException, RemotingException {
- final Channel channel = getOrCreateChannel(host);
- if (channel == null) {
- throw new RemotingException(String.format("connect to : %s fail", host));
- }
+ public IRpcResponse sendSync(SyncRequestDto syncRequestDto) throws RemotingException {
+ long start = System.currentTimeMillis();
+
+ final Host host = syncRequestDto.getServerHost();
+ final Transporter transporter = syncRequestDto.getTransporter();
+ final long timeoutMillis = syncRequestDto.getTimeoutMillis() < 0 ? clientConfig.getConnectTimeoutMillis()
+ : syncRequestDto.getTimeoutMillis();
final long opaque = transporter.getHeader().getOpaque();
- final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis);
- channel.writeAndFlush(transporter).addListener(future -> {
- if (future.isSuccess()) {
- responseFuture.setSendOk(true);
- return;
- } else {
- responseFuture.setSendOk(false);
+
+ try {
+ final Channel channel = getOrCreateChannel(host);
+ if (channel == null) {
+ throw new RemotingException(String.format("connect to : %s fail", host));
}
- responseFuture.setCause(future.cause());
- responseFuture.putResponse(null);
- log.error("Send Sync request {} to host {} failed", transporter, host, responseFuture.getCause());
- });
- /*
- * sync wait for result
- */
- IRpcResponse iRpcResponse = responseFuture.waitResponse();
- if (iRpcResponse == null) {
- if (responseFuture.isSendOK()) {
- throw new RemotingTimeoutException(host.toString(), timeoutMillis, responseFuture.getCause());
- } else {
- throw new RemotingException(host.toString(), responseFuture.getCause());
+ final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis);
+ channel.writeAndFlush(transporter).addListener(future -> {
+ if (future.isSuccess()) {
+ responseFuture.setSendOk(true);
+ return;
+ } else {
+ responseFuture.setSendOk(false);
+ }
+ responseFuture.setCause(future.cause());
+ responseFuture.putResponse(null);
+ log.error("Send Sync request {} to host {} failed", transporter, host, responseFuture.getCause());
+ });
+ /*
+ * sync wait for result
+ */
+ IRpcResponse iRpcResponse = responseFuture.waitResponse();
+ if (iRpcResponse == null) {
+ if (responseFuture.isSendOK()) {
+ throw new RemotingTimeoutException(host.toString(), timeoutMillis, responseFuture.getCause());
+ } else {
+ throw new RemotingException(host.toString(), responseFuture.getCause());
+ }
}
+ return iRpcResponse;
+ } catch (Exception ex) {
+ ClientSyncExceptionMetrics clientSyncExceptionMetrics = ClientSyncExceptionMetrics
+ .of(syncRequestDto)
+ .withThrowable(ex);
+ RpcMetrics.recordClientSyncRequestException(clientSyncExceptionMetrics);
+ if (ex instanceof RemotingException) {
+ throw (RemotingException) ex;
+ } else {
+ throw new RemotingException(ex);
+ }
+ } finally {
+ ClientSyncDurationMetrics clientSyncDurationMetrics = ClientSyncDurationMetrics
+ .of(syncRequestDto)
+ .withMilliseconds(System.currentTimeMillis() - start);
+ RpcMetrics.recordClientSyncRequestDuration(clientSyncDurationMetrics);
}
- return iRpcResponse;
}
Channel getOrCreateChannel(Host host) {
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java
index 4731a22d0a..ccbdad945b 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.extract.base.client;
import org.apache.dolphinscheduler.extract.base.IRpcResponse;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.StandardRpcRequest;
+import org.apache.dolphinscheduler.extract.base.SyncRequestDto;
import org.apache.dolphinscheduler.extract.base.exception.MethodInvocationException;
import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterHeader;
@@ -41,8 +42,12 @@ class SyncClientMethodInvoker extends AbstractClientMethodInvoker {
transporter.setBody(JsonSerializer.serialize(StandardRpcRequest.of(args)));
transporter.setHeader(TransporterHeader.of(methodIdentifier));
- IRpcResponse iRpcResponse =
- nettyRemotingClient.sendSync(serverHost, transporter, sync.timeout());
+ SyncRequestDto syncRequestDto = SyncRequestDto.builder()
+ .timeoutMillis(sync.timeout())
+ .transporter(transporter)
+ .serverHost(serverHost)
+ .build();
+ IRpcResponse iRpcResponse = nettyRemotingClient.sendSync(syncRequestDto);
if (!iRpcResponse.isSuccess()) {
throw MethodInvocationException.of(iRpcResponse.getMessage());
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java
index a41a439b5c..a00ff540f4 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java
@@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.extract.base.config;
+import java.time.Duration;
+
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@@ -64,4 +66,14 @@ public class NettyClientConfig {
@Builder.Default
private int connectTimeoutMillis = 3000;
+ /**
+ * Will send {@link org.apache.dolphinscheduler.extract.base.protocal.HeartBeatTransporter} to netty server every
+ * heartBeatIntervalMillis, used to keep the {@link io.netty.channel.Channel} active.
+ */
+ @Builder.Default
+ private long heartBeatIntervalMillis = Duration.ofSeconds(10).toMillis();
+
+ @Builder.Default
+ private int defaultRpcTimeoutMillis = 10_000;
+
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java
index 9d4a2ee3d2..cc0aa04f68 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java
@@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.extract.base.config;
+import java.time.Duration;
+
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@@ -37,7 +39,7 @@ public class NettyServerConfig {
private int soBacklog = 1024;
/**
- * whether tpc delay
+ * whether tcp delay
*/
@Builder.Default
private boolean tcpNoDelay = true;
@@ -66,6 +68,12 @@ public class NettyServerConfig {
@Builder.Default
private int workerThread = Runtime.getRuntime().availableProcessors() * 2;
+ /**
+ * If done's receive any data from a {@link io.netty.channel.Channel} during 180s then will close it.
+ */
+ @Builder.Default
+ private long connectionIdleTime = Duration.ofSeconds(60).toMillis();
+
/**
* listen port
*/
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/metrics/ClientSyncDurationMetrics.java
similarity index 51%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java
rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/metrics/ClientSyncDurationMetrics.java
index 8ad4013868..60124363b7 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/metrics/ClientSyncDurationMetrics.java
@@ -15,13 +15,11 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.dispatch.context;
+package org.apache.dolphinscheduler.extract.base.metrics;
-import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_WORKER_GROUP;
-
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.extract.base.utils.Host;
-import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
+import org.apache.dolphinscheduler.common.utils.NetUtils;
+import org.apache.dolphinscheduler.extract.base.SyncRequestDto;
+import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
import lombok.AllArgsConstructor;
import lombok.Builder;
@@ -32,26 +30,27 @@ import lombok.NoArgsConstructor;
@Builder
@NoArgsConstructor
@AllArgsConstructor
-public class ExecutionContext {
+public class ClientSyncDurationMetrics {
- private Host host;
+ private Transporter transporter;
- private TaskInstance taskInstance;
+ private long milliseconds;
- private ExecutorType executorType;
+ @Builder.Default
+ private String clientHost = NetUtils.getHost();
- /**
- * worker group
- */
- private String workerGroup;
+ private String serverHost;
- public ExecutionContext(ExecutorType executorType, TaskInstance taskInstance) {
- this(executorType, DEFAULT_WORKER_GROUP, taskInstance);
+ public static ClientSyncDurationMetrics of(SyncRequestDto syncRequestDto) {
+ return ClientSyncDurationMetrics.builder()
+ .transporter(syncRequestDto.getTransporter())
+ .serverHost(syncRequestDto.getServerHost().getIp())
+ .build();
}
- public ExecutionContext(ExecutorType executorType, String workerGroup, TaskInstance taskInstance) {
- this.executorType = executorType;
- this.workerGroup = workerGroup;
- this.taskInstance = taskInstance;
+ public ClientSyncDurationMetrics withMilliseconds(long milliseconds) {
+ this.milliseconds = milliseconds;
+ return this;
}
+
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/metrics/ClientSyncExceptionMetrics.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/metrics/ClientSyncExceptionMetrics.java
new file mode 100644
index 0000000000..6f91132547
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/metrics/ClientSyncExceptionMetrics.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.base.metrics;
+
+import org.apache.dolphinscheduler.common.utils.NetUtils;
+import org.apache.dolphinscheduler.extract.base.SyncRequestDto;
+import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ClientSyncExceptionMetrics {
+
+ private Transporter transporter;
+
+ private String clientHost;
+
+ @Builder.Default
+ private String serverHost = NetUtils.getHost();
+
+ private Throwable throwable;
+
+ public static ClientSyncExceptionMetrics of(SyncRequestDto syncRequestDto) {
+ return ClientSyncExceptionMetrics.builder()
+ .transporter(syncRequestDto.getTransporter())
+ .build();
+
+ }
+
+ public ClientSyncExceptionMetrics withThrowable(Throwable throwable) {
+ this.throwable = throwable;
+ return this;
+ }
+
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/metrics/RpcMetrics.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/metrics/RpcMetrics.java
new file mode 100644
index 0000000000..c2dbbfefa6
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/metrics/RpcMetrics.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.base.metrics;
+
+import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
+import org.apache.dolphinscheduler.extract.base.protocal.TransporterHeader;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Timer;
+
+public class RpcMetrics {
+
+ private static final Map rpcRequestDurationTimer = new ConcurrentHashMap<>();
+
+ private static final Map rpcRequestExceptionCounter = new ConcurrentHashMap<>();
+
+ public static void recordClientSyncRequestException(ClientSyncExceptionMetrics clientSyncExceptionMetrics) {
+ recordClientSyncRequestException(
+ clientSyncExceptionMetrics.getThrowable(),
+ Optional.of(clientSyncExceptionMetrics)
+ .map(ClientSyncExceptionMetrics::getTransporter)
+ .map(Transporter::getHeader)
+ .map(TransporterHeader::getMethodIdentifier)
+ .orElseGet(() -> "unknown"),
+ clientSyncExceptionMetrics.getClientHost(),
+ clientSyncExceptionMetrics.getServerHost());
+ }
+
+ public static void recordClientSyncRequestException(final Throwable throwable,
+ final String methodName,
+ final String clientHost,
+ final String serverHost) {
+ final String exceptionType = throwable == null ? "unknown" : throwable.getClass().getSimpleName();
+ final Counter counter = rpcRequestExceptionCounter.computeIfAbsent(exceptionType,
+ (et) -> Counter.builder("ds.rpc.client.sync.request.exception.count")
+ .tag("method_name", methodName)
+ .tag("client_host", clientHost)
+ .tag("server_host", serverHost)
+ .tag("exception_name", et)
+ .description("rpc sync request exception counter for exception type: " + et)
+ .register(Metrics.globalRegistry));
+ counter.increment();
+ }
+
+ public static void recordClientSyncRequestDuration(ClientSyncDurationMetrics clientSyncDurationMetrics) {
+ recordClientSyncRequestDuration(
+ Optional.of(clientSyncDurationMetrics)
+ .map(ClientSyncDurationMetrics::getTransporter)
+ .map(Transporter::getHeader)
+ .map(TransporterHeader::getMethodIdentifier)
+ .orElseGet(() -> "unknown"),
+ clientSyncDurationMetrics.getMilliseconds(),
+ clientSyncDurationMetrics.getClientHost(),
+ clientSyncDurationMetrics.getServerHost());
+ }
+
+ public static void recordClientSyncRequestDuration(final String methodName,
+ final long milliseconds,
+ final String clientHost,
+ final String serverHost) {
+ rpcRequestDurationTimer.computeIfAbsent(methodName,
+ (method) -> Timer.builder("ds.rpc.client.sync.request.duration.time")
+ .tag("method_name", method)
+ .tag("client_host", clientHost)
+ .tag("server_host", serverHost)
+ .publishPercentiles(0.5, 0.75, 0.95, 0.99)
+ .publishPercentileHistogram()
+ .description("time cost of sync rpc request, unit ms")
+ .register(Metrics.globalRegistry))
+ .record(milliseconds, TimeUnit.MILLISECONDS);
+ }
+
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java
index f57ff0b609..4f9a7034c8 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java
@@ -38,6 +38,7 @@ import io.netty.channel.ChannelConfig;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.handler.timeout.IdleState;
import io.netty.handler.timeout.IdleStateEvent;
@Slf4j
@@ -160,7 +161,11 @@ class JdkDynamicServerHandler extends ChannelInboundHandlerAdapter {
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
if (evt instanceof IdleStateEvent) {
- ctx.channel().close();
+ IdleStateEvent event = (IdleStateEvent) evt;
+ if (event.state() == IdleState.READER_IDLE) {
+ log.warn("Not receive heart beat from: {}, will close the channel", ctx.channel().remoteAddress());
+ ctx.close();
+ }
} else {
super.userEventTriggered(ctx, evt);
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java
index 9beeaced3d..9ebf802b1e 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java
@@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig;
import org.apache.dolphinscheduler.extract.base.exception.RemoteException;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder;
-import org.apache.dolphinscheduler.extract.base.utils.Constants;
import org.apache.dolphinscheduler.extract.base.utils.NettyUtils;
import java.util.concurrent.ExecutorService;
@@ -135,7 +134,7 @@ class NettyRemotingServer {
.addLast("encoder", new TransporterEncoder())
.addLast("decoder", new TransporterDecoder())
.addLast("server-idle-handle",
- new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS))
+ new IdleStateHandler(serverConfig.getConnectionIdleTime(), 0, 0, TimeUnit.MILLISECONDS))
.addLast("handler", channelHandler);
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Constants.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Constants.java
index 76e3872d31..94b92b2539 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Constants.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Constants.java
@@ -35,10 +35,6 @@ public class Constants {
public static final String SLASH = "/";
- public static final int NETTY_SERVER_HEART_BEAT_TIME = 1000 * 60 * 3 + 1000;
-
- public static final int NETTY_CLIENT_HEART_BEAT_TIME = 1000 * 6;
-
/**
* charset
*/
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/metrics/RpcMetricsTest.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/metrics/RpcMetricsTest.java
new file mode 100644
index 0000000000..352e218206
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/metrics/RpcMetricsTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.base.metrics;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import org.apache.dolphinscheduler.common.utils.NetUtils;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+
+class RpcMetricsTest {
+
+ @BeforeEach
+ public void setup() {
+ Metrics.globalRegistry.clear();
+ Metrics.addRegistry(new SimpleMeterRegistry());
+ }
+
+ @Test
+ void testRecordClientSyncRequestException() {
+ assertThat(Metrics.globalRegistry.find("ds.rpc.client.sync.request.exception.count").counter()).isNull();
+
+ String clientHost = NetUtils.getHost();
+ String serverHost = NetUtils.getHost();
+
+ RpcMetrics.recordClientSyncRequestException(
+ new IllegalArgumentException("id is null"), "getById", clientHost, serverHost);
+ RpcMetrics.recordClientSyncRequestException(
+ new IllegalArgumentException("name is null"), "getByName", clientHost, serverHost);
+ RpcMetrics.recordClientSyncRequestException(
+ new IllegalArgumentException("age is null"), "getByAge", clientHost, serverHost);
+ RpcMetrics.recordClientSyncRequestException(new UnsupportedOperationException("update id is not supported"),
+ "updateById", clientHost, serverHost);
+ assertThat(Metrics.globalRegistry.find("ds.rpc.client.sync.request.exception.count").counter()).isNotNull();
+ }
+
+ @Test
+ void testRecordRpcRequestDuration() {
+ assertThat(Metrics.globalRegistry.find("ds.rpc.client.sync.request.duration.time").timer()).isNull();
+
+ String clientHost = NetUtils.getHost();
+ String serverHost = NetUtils.getHost();
+
+ RpcMetrics.recordClientSyncRequestDuration("getById", 100, clientHost, serverHost);
+ RpcMetrics.recordClientSyncRequestDuration("getByName", 200, clientHost, serverHost);
+ RpcMetrics.recordClientSyncRequestDuration("getByAge", 300, clientHost, serverHost);
+ RpcMetrics.recordClientSyncRequestDuration("updateById", 400, clientHost, serverHost);
+ assertThat(Metrics.globalRegistry.find("ds.rpc.client.sync.request.duration.time").timer()).isNotNull();
+ }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 9b507500b9..92d7dcfaec 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -108,7 +108,7 @@ public class MasterServer implements IStoppable {
this.masterRPCServer.start();
// install task plugin
- TaskPluginManager.loadPlugin();
+ TaskPluginManager.loadTaskPlugin();
DataSourceProcessorProvider.initialize();
this.masterSlotManager.start();
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
index 20d3cccef3..d9e6ab3b7d 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
@@ -96,10 +96,14 @@ public class MasterConfig implements Validator {
private CommandFetchStrategy commandFetchStrategy = new CommandFetchStrategy();
- // ip:listenPort
+ /**
+ * The IP address and listening port of the master server in the format 'ip:listenPort'.
+ */
private String masterAddress;
- // /nodes/master/ip:listenPort
+ /**
+ * The registry path for the master server in the format '/nodes/master/ip:listenPort'.
+ */
private String masterRegistryPath;
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java
index d03fd59ada..9c3c4fd696 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java
@@ -25,10 +25,12 @@ import java.util.Collection;
public class LowerWeightRoundRobin extends AbstractSelector {
/**
- * select
+ * Selects a HostWeight from a collection of HostWeight objects.
+ * The selection is based on the current weight of each HostWeight.
+ * The HostWeight with the smallest current weight is selected.
*
- * @param sources sources
- * @return HostWeight
+ * @param sources A collection of HostWeight objects to select from.
+ * @return The selected HostWeight with the smallest current weight.
*/
@Override
public HostWeight doSelect(Collection sources) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java
index 2b7488a370..a0f83232a2 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java
@@ -27,6 +27,14 @@ import java.util.concurrent.ThreadLocalRandom;
*/
public class RandomSelector extends AbstractSelector {
+ /**
+ * This method selects a HostWorker from a collection of HostWorker objects using a weighted random algorithm.
+ * The selection is based on the weight of each HostWorker.
+ * A random number is generated and the HostWorker whose weight spans this random number is selected.
+ *
+ * @param source A collection of HostWorker objects to select from.
+ * @return The selected HostWorker based on the weighted random algorithm.
+ */
@Override
public HostWorker doSelect(final Collection source) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java
index 8f21acef6d..b47eff87b9 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java
@@ -73,6 +73,14 @@ public class RoundRobinSelector extends AbstractSelector {
}
+ /**
+ * This method selects a HostWorker from a collection of HostWorker objects using a weighted round-robin algorithm.
+ * The selection is based on the current weight of each HostWorker.
+ * The HostWorker with the highest current weight is selected.
+ *
+ * @param source A collection of HostWorker objects to select from.
+ * @return The selected HostWorker with the highest current weight.
+ */
@Override
public HostWorker doSelect(Collection source) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java
deleted file mode 100644
index 07f89fc544..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.event;
-
-import org.apache.dolphinscheduler.common.enums.StateEventType;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
-
-import java.util.Optional;
-
-import lombok.extern.slf4j.Slf4j;
-
-import com.google.auto.service.AutoService;
-
-@AutoService(StateEventHandler.class)
-@Slf4j
-public class WorkflowBlockStateEventHandler implements StateEventHandler {
-
- @Override
- public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable,
- StateEvent stateEvent) throws StateEventHandleError {
- log.info("Handle workflow instance state block event");
- Optional taskInstanceOptional =
- workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId());
- if (!taskInstanceOptional.isPresent()) {
- throw new StateEventHandleError("Cannot find taskInstance from taskMap by taskInstanceId: "
- + stateEvent.getTaskInstanceId());
- }
- TaskInstance task = taskInstanceOptional.get();
-
- BlockingParameters parameters = JSONUtils.parseObject(task.getTaskParams(), BlockingParameters.class);
- if (parameters != null && parameters.isAlertWhenBlocking()) {
- workflowExecuteRunnable.processBlock();
- }
- return true;
- }
-
- @Override
- public StateEventType getEventType() {
- return StateEventType.PROCESS_BLOCKED;
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
index f066f0403d..d532168b2a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
@@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.registry.api.Event;
import org.apache.dolphinscheduler.registry.api.Event.Type;
import org.apache.dolphinscheduler.registry.api.RegistryClient;
@@ -36,7 +37,6 @@ import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
@@ -273,8 +273,8 @@ public class ServerNodeManager implements InitializingBean {
.filter(workerNodeInfo::containsKey).collect(Collectors.toSet());
tmpWorkerGroupMappings.put(workerGroupName, activeWorkerNodes);
}
- if (!tmpWorkerGroupMappings.containsKey(Constants.DEFAULT_WORKER_GROUP)) {
- tmpWorkerGroupMappings.put(Constants.DEFAULT_WORKER_GROUP, workerNodeInfo.keySet());
+ if (!tmpWorkerGroupMappings.containsKey(WorkerGroupUtils.getDefaultWorkerGroup())) {
+ tmpWorkerGroupMappings.put(WorkerGroupUtils.getDefaultWorkerGroup(), workerNodeInfo.keySet());
}
} finally {
workerNodeInfoReadLock.unlock();
@@ -307,9 +307,7 @@ public class ServerNodeManager implements InitializingBean {
public Set getWorkerGroupNodes(String workerGroup) throws WorkerGroupNotFoundException {
workerGroupReadLock.lock();
try {
- if (StringUtils.isEmpty(workerGroup)) {
- workerGroup = Constants.DEFAULT_WORKER_GROUP;
- }
+ workerGroup = WorkerGroupUtils.getWorkerGroupOrDefault(workerGroup);
Set nodes = workerGroupNodes.get(workerGroup);
if (nodes == null) {
throw new WorkerGroupNotFoundException(String.format("WorkerGroup: %s is invalidated", workerGroup));
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java
index 2f61507e72..90fb25fcf0 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java
@@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.server.master.runner;
-import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_WORKER_GROUP;
-
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.Priority;
@@ -31,6 +29,8 @@ import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
+import org.apache.dolphinscheduler.dao.utils.EnvironmentUtils;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest;
import org.apache.dolphinscheduler.extract.worker.ITaskInstanceExecutionEventAckListener;
@@ -43,7 +43,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
@@ -89,8 +88,6 @@ public class StreamTaskExecuteRunnable implements Runnable {
protected ProcessTaskRelationMapper processTaskRelationMapper;
- protected TaskPluginManager taskPluginManager;
-
private StreamTaskInstanceExecCacheManager streamTaskInstanceExecCacheManager;
protected TaskDefinition taskDefinition;
@@ -115,7 +112,6 @@ public class StreamTaskExecuteRunnable implements Runnable {
this.processService = SpringApplicationContext.getBean(ProcessService.class);
this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class);
this.workerTaskDispatcher = SpringApplicationContext.getBean(WorkerTaskDispatcher.class);
- this.taskPluginManager = SpringApplicationContext.getBean(TaskPluginManager.class);
this.processTaskRelationMapper = SpringApplicationContext.getBean(ProcessTaskRelationMapper.class);
this.taskInstanceDao = SpringApplicationContext.getBean(TaskInstanceDao.class);
this.streamTaskInstanceExecCacheManager =
@@ -270,12 +266,11 @@ public class StreamTaskExecuteRunnable implements Runnable {
// task dry run flag
taskInstance.setDryRun(taskExecuteStartMessage.getDryRun());
- taskInstance.setWorkerGroup(StringUtils.isBlank(taskDefinition.getWorkerGroup()) ? DEFAULT_WORKER_GROUP
- : taskDefinition.getWorkerGroup());
- taskInstance.setEnvironmentCode(
- taskDefinition.getEnvironmentCode() == 0 ? -1 : taskDefinition.getEnvironmentCode());
+ taskInstance.setWorkerGroup(WorkerGroupUtils.getWorkerGroupOrDefault(taskDefinition.getWorkerGroup()));
+ taskInstance
+ .setEnvironmentCode(EnvironmentUtils.getEnvironmentCodeOrDefault(taskDefinition.getEnvironmentCode()));
- if (!taskInstance.getEnvironmentCode().equals(-1L)) {
+ if (!EnvironmentUtils.isEnvironmentCodeEmpty(taskInstance.getEnvironmentCode())) {
Environment environment = processService.findEnvironmentByCode(taskInstance.getEnvironmentCode());
if (Objects.nonNull(environment) && StringUtils.isNotEmpty(environment.getConfig())) {
taskInstance.setEnvironmentConfig(environment.getConfig());
@@ -312,14 +307,11 @@ public class StreamTaskExecuteRunnable implements Runnable {
return null;
}
- TaskChannel taskChannel = taskPluginManager.getTaskChannel(taskInstance.getTaskType());
- ResourceParametersHelper resources = taskChannel.getResources(taskInstance.getTaskParams());
+ TaskChannel taskChannel = TaskPluginManager.getTaskChannel(taskInstance.getTaskType());
+ ResourceParametersHelper resources = taskChannel.parseParameters(taskInstance.getTaskParams()).getResources();
- AbstractParameters baseParam = taskPluginManager.getParameters(
- ParametersNode.builder()
- .taskType(taskInstance.getTaskType())
- .taskParams(taskInstance.getTaskParams())
- .build());
+ AbstractParameters baseParam =
+ TaskPluginManager.parseTaskParameters(taskInstance.getTaskType(), taskInstance.getTaskParams());
Map propertyMap = paramParsingPreparation(taskInstance, baseParam);
TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
.buildWorkflowInstanceHost(masterConfig.getMasterAddress())
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java
index b3b0cf67ea..da4f5308d0 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java
@@ -49,7 +49,6 @@ import org.apache.dolphinscheduler.plugin.task.api.model.JdbcInfo;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.AbstractResourceParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
@@ -74,7 +73,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
@@ -102,16 +100,15 @@ public class TaskExecutionContextFactory {
public TaskExecutionContext createTaskExecutionContext(TaskInstance taskInstance) throws TaskExecutionContextCreateException {
ProcessInstance workflowInstance = taskInstance.getProcessInstance();
- ResourceParametersHelper resources =
- Optional.ofNullable(TaskPluginManager.getTaskChannel(taskInstance.getTaskType()))
- .map(taskChannel -> taskChannel.getResources(taskInstance.getTaskParams()))
- .orElse(null);
+ ResourceParametersHelper resources = TaskPluginManager.getTaskChannel(taskInstance.getTaskType())
+ .parseParameters(taskInstance.getTaskParams())
+ .getResources();
setTaskResourceInfo(resources);
Map businessParamsMap = curingParamsService.preBuildBusinessParams(workflowInstance);
- AbstractParameters baseParam = TaskPluginManager.getParameters(ParametersNode.builder()
- .taskType(taskInstance.getTaskType()).taskParams(taskInstance.getTaskParams()).build());
+ AbstractParameters baseParam =
+ TaskPluginManager.parseTaskParameters(taskInstance.getTaskType(), taskInstance.getTaskParams());
Map propertyMap =
curingParamsService.paramParsingPreparation(taskInstance, baseParam, workflowInstance);
TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index c35fd9991f..c58d741d41 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -25,9 +25,9 @@ import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.C
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_START_NODES;
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_START_PARAMS;
import static org.apache.dolphinscheduler.common.constants.Constants.COMMA;
-import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYY_MM_DD_HH_MM_SS;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
+import static org.apache.dolphinscheduler.dao.utils.EnvironmentUtils.getEnvironmentCodeOrDefault;
+import static org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils.getWorkerGroupOrDefault;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -51,15 +51,17 @@ import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
+import org.apache.dolphinscheduler.dao.utils.EnvironmentUtils;
import org.apache.dolphinscheduler.dao.utils.TaskCacheUtils;
+import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator;
import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostRequest;
import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
+import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.VarPoolUtils;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.event.StateEvent;
@@ -73,7 +75,6 @@ import org.apache.dolphinscheduler.server.master.graph.IWorkflowGraph;
import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory;
import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator;
-import org.apache.dolphinscheduler.server.master.utils.TaskUtils;
import org.apache.dolphinscheduler.server.master.utils.WorkflowInstanceUtils;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
@@ -297,7 +298,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
stateEvent,
stateEventHandleException);
ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
- } catch (Exception e) {
+ } catch (Throwable e) {
// we catch the exception here, since if the state event handle failed, the state event will still
// keep
// in the stateEvents queue.
@@ -378,25 +379,24 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
if (taskInstance.getIsCache().equals(Flag.YES)) {
saveCacheTaskInstance(taskInstance);
}
- if (!workflowInstance.isBlocked()) {
- submitPostNode(taskInstance.getTaskCode());
- }
+ submitPostNode(taskInstance.getTaskCode());
} else if (taskInstance.taskCanRetry() && !workflowInstance.getState().isReadyStop()) {
// retry task
log.info("Retry taskInstance taskInstance state: {}", taskInstance.getState());
retryTaskInstance(taskInstance);
- } else if (taskInstance.getState().isFailure()) {
+ } else if (taskInstance.getState().isFailure() || taskInstance.getState().isKill()
+ || taskInstance.getState().isStop()) {
completeTaskSet.add(taskInstance.getTaskCode());
- ProjectUser projectUser =
- processService.queryProjectWithUserByProcessInstanceId(workflowInstance.getId());
- listenerEventAlertManager.publishTaskFailListenerEvent(workflowInstance, taskInstance, projectUser);
+ listenerEventAlertManager.publishTaskFailListenerEvent(workflowInstance, taskInstance);
+ if (isTaskNeedPutIntoErrorMap(taskInstance)) {
+ errorTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
+ }
// There are child nodes and the failure policy is: CONTINUE
if (workflowInstance.getFailureStrategy() == FailureStrategy.CONTINUE && DagHelper.haveAllNodeAfterNode(
taskInstance.getTaskCode(),
workflowExecuteContext.getWorkflowGraph().getDag())) {
submitPostNode(taskInstance.getTaskCode());
} else {
- errorTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
if (workflowInstance.getFailureStrategy() == FailureStrategy.END) {
killAllTasks();
}
@@ -805,10 +805,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
completeTaskSet.add(task.getTaskCode());
continue;
}
- if (task.isConditionsTask() || DagHelper.haveConditionsAfterNode(task.getTaskCode(),
- workflowExecuteContext.getWorkflowGraph().getDag())) {
- continue;
- }
+
if (task.taskCanRetry()) {
if (task.getState().isNeedFaultTolerance()) {
log.info("TaskInstance needs fault tolerance, will be added to standby list.");
@@ -824,7 +821,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
}
continue;
}
- if (task.getState().isFailure()) {
+ if (isTaskNeedPutIntoErrorMap(task)) {
errorTaskMap.put(task.getTaskCode(), task.getId());
}
} finally {
@@ -949,16 +946,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
if (!taskInstance.getState().isFinished()) {
taskExecuteRunnable.dispatch();
} else {
- if (workflowExecuteContext.getWorkflowInstance().isBlocked()) {
- TaskStateEvent processBlockEvent = TaskStateEvent.builder()
- .processInstanceId(workflowExecuteContext.getWorkflowInstance().getId())
- .taskInstanceId(taskInstance.getId())
- .status(taskInstance.getState())
- .type(StateEventType.PROCESS_BLOCKED)
- .build();
- this.stateEvents.add(processBlockEvent);
- }
-
TaskStateEvent taskStateChangeEvent = TaskStateEvent.builder()
.processInstanceId(workflowExecuteContext.getWorkflowInstance().getId())
.taskInstanceId(taskInstance.getId())
@@ -1095,7 +1082,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
taskInstance.setRetryInterval(taskNode.getRetryInterval());
// set task param
- taskInstance.setTaskParams(taskNode.getTaskParams());
+ taskInstance.setTaskParams(taskNode.getParams());
// set task group and priority
taskInstance.setTaskGroupId(taskNode.getTaskGroupId());
@@ -1114,25 +1101,22 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
taskInstance.setTaskInstancePriority(taskNode.getTaskInstancePriority());
}
- String processWorkerGroup = processInstance.getWorkerGroup();
- processWorkerGroup = StringUtils.isBlank(processWorkerGroup) ? DEFAULT_WORKER_GROUP : processWorkerGroup;
- String taskWorkerGroup =
- StringUtils.isBlank(taskNode.getWorkerGroup()) ? processWorkerGroup : taskNode.getWorkerGroup();
+ String processWorkerGroup = getWorkerGroupOrDefault(processInstance.getWorkerGroup());
+ String taskWorkerGroup = getWorkerGroupOrDefault(taskNode.getWorkerGroup());
- Long processEnvironmentCode =
- Objects.isNull(processInstance.getEnvironmentCode()) ? -1 : processInstance.getEnvironmentCode();
- Long taskEnvironmentCode =
- Objects.isNull(taskNode.getEnvironmentCode()) ? processEnvironmentCode : taskNode.getEnvironmentCode();
+ Long processEnvironmentCode = getEnvironmentCodeOrDefault(processInstance.getEnvironmentCode());
+ Long taskEnvironmentCode = getEnvironmentCodeOrDefault(taskNode.getEnvironmentCode());
- if (!processWorkerGroup.equals(DEFAULT_WORKER_GROUP) && taskWorkerGroup.equals(DEFAULT_WORKER_GROUP)) {
+ if (WorkerGroupUtils.isWorkerGroupEmpty(taskWorkerGroup)) {
+ // If the task workerGroup is empty, then use the workflow workerGroup/environment
taskInstance.setWorkerGroup(processWorkerGroup);
- taskInstance.setEnvironmentCode(processEnvironmentCode);
+ taskInstance.setEnvironmentCode(getEnvironmentCodeOrDefault(taskEnvironmentCode, processEnvironmentCode));
} else {
taskInstance.setWorkerGroup(taskWorkerGroup);
- taskInstance.setEnvironmentCode(taskEnvironmentCode);
+ taskInstance.setEnvironmentCode(getEnvironmentCodeOrDefault(taskEnvironmentCode, processEnvironmentCode));
}
- if (!taskInstance.getEnvironmentCode().equals(-1L)) {
+ if (!EnvironmentUtils.isEnvironmentCodeEmpty(taskInstance.getEnvironmentCode())) {
Environment environment = processService.findEnvironmentByCode(taskInstance.getEnvironmentCode());
if (Objects.nonNull(environment) && StringUtils.isNotEmpty(environment.getConfig())) {
taskInstance.setEnvironmentConfig(environment.getConfig());
@@ -1219,12 +1203,12 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
|| state == TaskExecutionStatus.DISPATCH
|| state == TaskExecutionStatus.SUBMITTED_SUCCESS
|| state == TaskExecutionStatus.DELAY_EXECUTION) {
- // try to take over task instance
- if (state == TaskExecutionStatus.SUBMITTED_SUCCESS || state == TaskExecutionStatus.DELAY_EXECUTION
- || state == TaskExecutionStatus.DISPATCH) {
+ if (state == TaskExecutionStatus.SUBMITTED_SUCCESS
+ || state == TaskExecutionStatus.DELAY_EXECUTION) {
// The taskInstance is not in running, directly takeover it
} else if (tryToTakeOverTaskInstance(existTaskInstance)) {
- log.info("Success take over task {}", existTaskInstance.getName());
+ // If the taskInstance has already dispatched to worker then will try to take-over it
+ log.info("Success take over task {} -> status: {}", existTaskInstance.getName(), state);
continue;
} else {
// set the task instance state to fault tolerance
@@ -1277,7 +1261,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
private boolean tryToTakeOverTaskInstance(TaskInstance taskInstance) {
ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance();
- if (TaskUtils.isMasterTask(taskInstance.getTaskType())) {
+ if (TaskTypeUtils.isLogicTask(taskInstance.getTaskType())) {
return false;
}
try {
@@ -1343,13 +1327,14 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
if (depTaskState.isKill()) {
return DependResult.NON_EXEC;
}
- // ignore task state if current task is block
- if (taskNode.isBlockingTask()) {
+
+ // always return success if current task is condition
+ if (TaskTypeUtils.isConditionTask(taskNode.getType())) {
continue;
}
- // always return success if current task is condition
- if (taskNode.isConditionsTask()) {
+ // always return success if current task is switch
+ if (TaskTypeUtils.isSwitchTask(taskNode.getType())) {
continue;
}
@@ -1394,7 +1379,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
private boolean dependTaskSuccess(Long dependNodeCode, Long nextNodeCode) {
DAG dag = workflowExecuteContext.getWorkflowGraph().getDag();
TaskNode dependentNode = dag.getNode(dependNodeCode);
- if (dependentNode.isConditionsTask()) {
+ if (TaskTypeUtils.isConditionTask(dependentNode.getType())) {
// condition task need check the branch to run
List nextTaskList =
DagHelper.parseConditionTask(dependNodeCode, skipTaskNodeMap, dag, getCompleteTaskInstanceMap());
@@ -1407,12 +1392,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
}
return true;
}
- if (dependentNode.isSwitchTask()) {
- TaskInstance dependentTaskInstance = taskInstanceMap.get(validTaskMap.get(dependentNode.getCode()));
- SwitchParameters switchParameters = dependentTaskInstance.getSwitchDependency();
- return switchParameters.getDependTaskList().get(switchParameters.getResultConditionLocation()).getNextNode()
- .contains(nextNodeCode);
- }
Optional existTaskInstanceOptional = getTaskInstance(dependNodeCode);
if (!existTaskInstanceOptional.isPresent()) {
return false;
@@ -1452,8 +1431,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
*/
private WorkflowExecutionStatus runningState(WorkflowExecutionStatus state) {
if (state == WorkflowExecutionStatus.READY_STOP || state == WorkflowExecutionStatus.READY_PAUSE
- || state == WorkflowExecutionStatus.READY_BLOCK ||
- state == WorkflowExecutionStatus.DELAY_EXECUTION) {
+ || state == WorkflowExecutionStatus.DELAY_EXECUTION) {
// if the running task is not completed, the state remains unchanged
return state;
} else {
@@ -1512,7 +1490,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
}
List pauseList = getCompleteTaskByState(TaskExecutionStatus.PAUSE);
- if (CollectionUtils.isNotEmpty(pauseList) || workflowInstance.isBlocked() || !isComplementEnd()
+ if (CollectionUtils.isNotEmpty(pauseList) || !isComplementEnd()
|| standByTaskInstancePriorityQueue.size() > 0) {
return WorkflowExecutionStatus.PAUSE;
} else {
@@ -1520,30 +1498,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
}
}
- /**
- * prepare for block
- * if process has tasks still running, pause them
- * if readyToSubmitTaskQueue is not empty, kill them
- * else return block status directly
- *
- * @return ExecutionStatus
- */
- private WorkflowExecutionStatus processReadyBlock() {
- if (taskExecuteRunnableMap.size() > 0) {
- for (DefaultTaskExecuteRunnable taskExecuteRunnable : taskExecuteRunnableMap.values()) {
- if (!TASK_TYPE_BLOCKING.equals(taskExecuteRunnable.getTaskInstance().getTaskType())) {
- taskExecuteRunnable.pause();
- }
- }
- }
- if (standByTaskInstancePriorityQueue.size() > 0) {
- for (Iterator iter = standByTaskInstancePriorityQueue.iterator(); iter.hasNext();) {
- iter.next().setState(TaskExecutionStatus.PAUSE);
- }
- }
- return WorkflowExecutionStatus.BLOCK;
- }
-
/**
* generate the latest process instance status by the tasks state
*
@@ -1559,13 +1513,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
return executionStatus;
}
- // block
- if (state == WorkflowExecutionStatus.READY_BLOCK) {
- WorkflowExecutionStatus executionStatus = processReadyBlock();
- log.info("The workflowInstance is ready to block, the workflowInstance status is {}", executionStatus);
- return executionStatus;
- }
-
// pause
if (state == WorkflowExecutionStatus.READY_PAUSE) {
WorkflowExecutionStatus executionStatus = processReadyPause();
@@ -2015,6 +1962,24 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
}
}
+ /**
+ * Whether the task instance need to put into {@link #errorTaskMap}.
+ * Only the task instance is failed or killed, and it is parent of condition task.
+ * Then it should be put into {@link #errorTaskMap}.
+ * Once a task instance is put into {@link #errorTaskMap}, it will be thought as failed and make the workflow be failed.
+ */
+ private boolean isTaskNeedPutIntoErrorMap(TaskInstance taskInstance) {
+ if (!taskInstance.getState().isFailure() && !taskInstance.getState().isStop()
+ && !taskInstance.getState().isKill()) {
+ return false;
+ }
+ TaskNode taskNode = workflowExecuteContext.getWorkflowGraph().getTaskNodeByCode(taskInstance.getTaskCode());
+ if (DagHelper.haveConditionsAfterNode(taskNode.getCode(), workflowExecuteContext.getWorkflowGraph().getDag())) {
+ return false;
+ }
+ return true;
+ }
+
private enum WorkflowRunnableStatus {
CREATED, INITIALIZE_QUEUE, STARTED,
;
@@ -2022,7 +1987,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
}
private void sendTaskLogOnMasterToRemoteIfNeeded(TaskInstance taskInstance) {
- if (RemoteLogUtils.isRemoteLoggingEnable() && TaskUtils.isMasterTask(taskInstance.getTaskType())) {
+ if (RemoteLogUtils.isRemoteLoggingEnable() && TaskTypeUtils.isLogicTask(taskInstance.getTaskType())) {
RemoteLogUtils.sendRemoteLog(taskInstance.getLogPath());
log.info("Master sends task log {} to remote storage asynchronously.", taskInstance.getLogPath());
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java
index 52469fb54c..9943ee1b5b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java
@@ -18,7 +18,7 @@
package org.apache.dolphinscheduler.server.master.runner.dispatcher;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.server.master.utils.TaskUtils;
+import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import lombok.extern.slf4j.Slf4j;
@@ -36,7 +36,10 @@ public class TaskDispatchFactory {
private WorkerTaskDispatcher workerTaskDispatcher;
public TaskDispatcher getTaskDispatcher(String taskType) {
- return TaskUtils.isMasterTask(taskType) ? masterTaskDispatcher : workerTaskDispatcher;
+ if (TaskTypeUtils.isLogicTask(taskType)) {
+ return masterTaskDispatcher;
+ }
+ return workerTaskDispatcher;
}
public TaskDispatcher getTaskDispatcher(TaskInstance taskInstance) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java
index 1b92f5e75c..0a67e801dc 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java
@@ -17,8 +17,8 @@
package org.apache.dolphinscheduler.server.master.runner.operator;
+import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.server.master.runner.DefaultTaskExecuteRunnable;
-import org.apache.dolphinscheduler.server.master.utils.TaskUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -51,28 +51,29 @@ public class TaskExecuteRunnableOperatorManager {
private LogicTaskExecuteRunnableTimeoutOperator logicTaskTimeoutOperator;
public TaskExecuteRunnableOperator getTaskKillOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) {
- if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
+ if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
return logicTaskKillOperator;
}
return taskKillOperator;
}
public TaskExecuteRunnableOperator getTaskPauseOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) {
- if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
+ if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
+
return logicTaskPauseOperator;
}
return taskPauseOperator;
}
public TaskExecuteRunnableOperator getTaskDispatchOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) {
- if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
+ if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
return logicTaskDispatchOperator;
}
return taskDispatchOperator;
}
public TaskExecuteRunnableOperator getTaskTimeoutOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) {
- if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
+ if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
return logicTaskTimeoutOperator;
}
return taskTimeoutOperator;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java
index 10a4ec1e7c..064e054ed3 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java
@@ -18,9 +18,12 @@
package org.apache.dolphinscheduler.server.master.runner.task;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
+import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import lombok.extern.slf4j.Slf4j;
@@ -28,10 +31,20 @@ import lombok.extern.slf4j.Slf4j;
public abstract class BaseSyncLogicTask implements ISyncLogicTask {
protected final TaskExecutionContext taskExecutionContext;
+
+ protected final WorkflowExecuteRunnable workflowExecuteRunnable;
+ protected final TaskInstance taskInstance;
protected final T taskParameters;
- protected BaseSyncLogicTask(TaskExecutionContext taskExecutionContext, T taskParameters) {
+ protected BaseSyncLogicTask(WorkflowExecuteRunnable workflowExecuteRunnable,
+ TaskExecutionContext taskExecutionContext,
+ T taskParameters) throws LogicTaskInitializeException {
this.taskExecutionContext = taskExecutionContext;
+ this.workflowExecuteRunnable = workflowExecuteRunnable;
+ this.taskInstance =
+ workflowExecuteRunnable.getTaskInstance(taskExecutionContext.getTaskInstanceId()).orElseThrow(
+ () -> new LogicTaskInitializeException(
+ "Cannot find the task instance in workflow execute runnable"));
this.taskParameters = taskParameters;
log.info("Success initialize task parameters: \n{}", JSONUtils.toPrettyJsonString(taskParameters));
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTask.java
deleted file mode 100644
index acc05aaf2d..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTask.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.task.blocking;
-
-import org.apache.dolphinscheduler.common.constants.Constants;
-import org.apache.dolphinscheduler.common.enums.BlockingOpportunity;
-import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
-import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
-import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
-import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
-import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
-import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils;
-import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
-import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import lombok.extern.slf4j.Slf4j;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-
-@Slf4j
-public class BlockingLogicTask extends BaseSyncLogicTask {
-
- public static final String TASK_TYPE = "BLOCKING";
-
- private final ProcessInstanceExecCacheManager processInstanceExecCacheManager;
-
- private final ProcessInstanceDao processInstanceDao;
-
- private final TaskInstanceDao taskInstanceDao;
-
- public BlockingLogicTask(TaskExecutionContext taskExecutionContext,
- ProcessInstanceExecCacheManager processInstanceExecCacheManager,
- ProcessInstanceDao processInstanceDao,
- TaskInstanceDao taskInstanceDao) {
- super(taskExecutionContext,
- JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference() {
- }));
- this.processInstanceExecCacheManager = processInstanceExecCacheManager;
- this.processInstanceDao = processInstanceDao;
- this.taskInstanceDao = taskInstanceDao;
- }
-
- @Override
- public void handle() throws MasterTaskExecuteException {
- DependResult conditionResult = calculateConditionResult();
- DependResult expected = taskParameters.getBlockingOpportunity()
- .equals(BlockingOpportunity.BLOCKING_ON_SUCCESS.getDesc())
- ? DependResult.SUCCESS
- : DependResult.FAILED;
- boolean isBlocked = (expected == conditionResult);
- log.info("blocking opportunity: expected-->{}, actual-->{}", expected, conditionResult);
- ProcessInstance workflowInstance = processInstanceExecCacheManager
- .getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()).getWorkflowExecuteContext()
- .getWorkflowInstance();
- workflowInstance.setBlocked(isBlocked);
- if (isBlocked) {
- workflowInstance.setStateWithDesc(WorkflowExecutionStatus.READY_BLOCK, "ready block");
- }
- taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
- }
-
- private DependResult calculateConditionResult() throws MasterTaskExecuteException {
- // todo: Directly get the task instance from the cache
- Map completeTaskList = taskInstanceDao
- .queryValidTaskListByWorkflowInstanceId(taskExecutionContext.getProcessInstanceId(),
- taskExecutionContext.getTestFlag())
- .stream()
- .collect(Collectors.toMap(TaskInstance::getTaskCode, Function.identity()));
-
- // todo: we need to parse the task parameter from TaskExecutionContext
- TaskInstance taskInstance =
- processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId())
- .getTaskInstance(taskExecutionContext.getTaskInstanceId())
- .orElseThrow(() -> new MasterTaskExecuteException("Task instance not found"));
- DependentParameters dependentParameters = taskInstance.getDependency();
-
- List tempResultList = new ArrayList<>();
- for (DependentTaskModel dependentTaskModel : dependentParameters.getDependTaskList()) {
- List itemDependResult = new ArrayList<>();
- for (DependentItem item : dependentTaskModel.getDependItemList()) {
- itemDependResult.add(getDependResultForItem(item, completeTaskList));
- }
- DependResult tempResult =
- DependentUtils.getDependResultForRelation(dependentTaskModel.getRelation(), itemDependResult);
- tempResultList.add(tempResult);
- }
- return DependentUtils.getDependResultForRelation(dependentParameters.getRelation(), tempResultList);
- }
-
- private DependResult getDependResultForItem(DependentItem item, Map completeTaskList) {
-
- DependResult dependResult = DependResult.SUCCESS;
- if (!completeTaskList.containsKey(item.getDepTaskCode())) {
- log.info("depend item: {} have not completed yet.", item.getDepTaskCode());
- dependResult = DependResult.FAILED;
- return dependResult;
- }
- TaskInstance taskInstance = completeTaskList.get(item.getDepTaskCode());
- if (taskInstance.getState() != item.getStatus()) {
- log.info("depend item : {} expect status: {}, actual status: {}", item.getDepTaskCode(), item.getStatus(),
- taskInstance.getState().name());
- dependResult = DependResult.FAILED;
- }
- log.info("Dependent item complete {} {},{}",
- Constants.DEPENDENT_SPLIT, item.getDepTaskCode(), dependResult);
- return dependResult;
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTaskPluginFactory.java
deleted file mode 100644
index b4fdd56c10..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTaskPluginFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.task.blocking;
-
-import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
-import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
-import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class BlockingLogicTaskPluginFactory implements ILogicTaskPluginFactory {
-
- @Autowired
- private ProcessInstanceDao processInstanceDao;
-
- @Autowired
- private TaskInstanceDao taskInstanceDao;
-
- @Autowired
- private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
-
- @Override
- public BlockingLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) {
- return new BlockingLogicTask(taskExecutionContext, processInstanceExecCacheManager, processInstanceDao,
- taskInstanceDao);
- }
-
- @Override
- public String getTaskType() {
- return BlockingLogicTask.TASK_TYPE;
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java
index 803a8043ff..10f7c52cf5 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java
@@ -17,18 +17,17 @@
package org.apache.dolphinscheduler.server.master.runner.task.condition;
-import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
+import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils;
-import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask;
import java.util.List;
@@ -39,50 +38,42 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
+import com.fasterxml.jackson.core.type.TypeReference;
+
@Slf4j
-public class ConditionLogicTask extends BaseSyncLogicTask {
+public class ConditionLogicTask extends BaseSyncLogicTask {
public static final String TASK_TYPE = "CONDITIONS";
private final TaskInstanceDao taskInstanceDao;
- private final ProcessInstanceDao workflowInstanceDao;
- public ConditionLogicTask(TaskExecutionContext taskExecutionContext,
- ProcessInstanceExecCacheManager processInstanceExecCacheManager,
- TaskInstanceDao taskInstanceDao,
- ProcessInstanceDao workflowInstanceDao) throws LogicTaskInitializeException {
- // todo: we need to change the parameter in front-end, so that we can directly use json to parse
- super(taskExecutionContext,
- processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId())
- .getTaskInstance(taskExecutionContext.getTaskInstanceId())
- .orElseThrow(() -> new LogicTaskInitializeException(
- "Cannot find the task instance in workflow execute runnable"))
- .getDependency());
- // todo：check the parameters, why we don't use conditionTask? taskInstance.getDependency();
+ public ConditionLogicTask(WorkflowExecuteRunnable workflowExecuteRunnable,
+ TaskExecutionContext taskExecutionContext,
+ TaskInstanceDao taskInstanceDao) throws LogicTaskInitializeException {
+ super(workflowExecuteRunnable, taskExecutionContext,
+ JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference() {
+ }));
this.taskInstanceDao = taskInstanceDao;
- this.workflowInstanceDao = workflowInstanceDao;
}
@Override
public void handle() {
- // calculate the conditionResult
DependResult conditionResult = calculateConditionResult();
- TaskExecutionStatus taskExecutionStatus =
- (conditionResult == DependResult.SUCCESS) ? TaskExecutionStatus.SUCCESS : TaskExecutionStatus.FAILURE;
- log.info("The condition result is {}, task instance statue will be: {}", conditionResult, taskExecutionStatus);
- taskExecutionContext.setCurrentExecutionStatus(taskExecutionStatus);
+ log.info("The condition result is {}", conditionResult);
+ taskParameters.getConditionResult().setConditionSuccess(conditionResult == DependResult.SUCCESS);
+ taskInstance.setTaskParams(JSONUtils.toJsonString(taskParameters));
+ taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
}
private DependResult calculateConditionResult() {
- final ProcessInstance processInstance =
- workflowInstanceDao.queryById(taskExecutionContext.getProcessInstanceId());
- final List taskInstances =
- taskInstanceDao.queryValidTaskListByWorkflowInstanceId(processInstance.getId(),
- processInstance.getTestFlag());
- final Map taskInstanceMap =
- taskInstances.stream().collect(Collectors.toMap(TaskInstance::getTaskCode, Function.identity()));
+ final List taskInstances = taskInstanceDao.queryValidTaskListByWorkflowInstanceId(
+ taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTestFlag());
+ final Map taskInstanceMap = taskInstances.stream()
+ .collect(Collectors.toMap(TaskInstance::getTaskCode, Function.identity()));
- List dependResults = taskParameters.getDependTaskList().stream()
+ ConditionsParameters.ConditionDependency dependence = taskParameters.getDependence();
+ List dependResults = dependence.getDependTaskList()
+ .stream()
.map(dependentTaskModel -> DependentUtils.getDependResultForRelation(
dependentTaskModel.getRelation(),
dependentTaskModel.getDependItemList()
@@ -90,7 +81,7 @@ public class ConditionLogicTask extends BaseSyncLogicTask {
.map(dependentItem -> getDependResultForItem(dependentItem, taskInstanceMap))
.collect(Collectors.toList())))
.collect(Collectors.toList());
- return DependentUtils.getDependResultForRelation(taskParameters.getRelation(), dependResults);
+ return DependentUtils.getDependResultForRelation(dependence.getRelation(), dependResults);
}
private DependResult getDependResultForItem(DependentItem item, Map taskInstanceMap) {
@@ -101,8 +92,9 @@ public class ConditionLogicTask extends BaseSyncLogicTask